| import json | |
| import xml.etree.ElementTree as ET | |
| from bs4 import BeautifulSoup | |
| import requests | |
| import tools | |
| import threading | |
| def fetch_links(category): | |
| links = [] | |
| xml_data = tools.fetch_page(f"https://www.sciencedaily.com/rss/top/{category.lower()}.xml") | |
| items = ET.fromstring(xml_data).findall('channel/item') | |
| for item in items: | |
| link = item.find('link').text | |
| links.append(link) | |
| return links | |
| def fetch_all_links(): | |
| categories = ["Science", "Health", "Environment", "Technology", "Society"] | |
| sd_links_data = {} | |
| for category in categories: | |
| links = fetch_links(category) | |
| sd_links_data[category] = links | |
| data = json.dumps(sd_links_data, indent=4, ensure_ascii=False) | |
| return data | |
| def fetch_dois(): | |
| doi_data = {} | |
| data = json.loads(fetch_all_links()) | |
| for topic, links in data.items(): | |
| doi_list = [] | |
| for link in links: | |
| page_content = tools.fetch_page(link) | |
| page_datas = BeautifulSoup(page_content, 'html.parser').find_all("div", id="journal_references") | |
| for page_data in page_datas: | |
| if not page_data.find("a", href=True): | |
| continue | |
| else: | |
| doi = page_data.find("a", href=True).text | |
| if doi.startswith('10.'): | |
| doi_list.append(doi) | |
| else: | |
| continue | |
| doi_data[topic] = doi_list | |
| data = json.dumps(doi_data, indent=4, ensure_ascii=False) | |
| return data | |
| def fetch_doi_data(): | |
| result = [] | |
| def fetch_and_store(): | |
| result.append(fetch_dois()) | |
| thread = threading.Thread(target=fetch_and_store) | |
| thread.start() | |
| thread.join() | |
| if len(result) == 0 or not result or result[0] == None: | |
| return [] | |
| return result[0] | |
| def doi_to_pmc(): | |
| data = json.loads(fetch_doi_data()) | |
| pmc_data = {} | |
| for topic, dois in data.items(): | |
| if len(dois) > 0: | |
| doi_list = "" | |
| for doi in dois: | |
| doi_list += doi + "," | |
| doi_list = doi_list.rstrip(',') | |
| try: | |
| url = f"https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/[email protected]&ids={doi_list}&format=json" | |
| doi_pmc_data = requests.get(url).json() | |
| except Exception as e: | |
| print(f"Error: {str(e)}") | |
| if doi_pmc_data['status'] == 'ok': | |
| pmc_list = [] | |
| for record in doi_pmc_data['records']: | |
| if 'pmcid' in record: | |
| if 'live' in record and record['live'] == False: | |
| continue | |
| pmc_list.append(record['pmcid']) | |
| else: | |
| continue | |
| pmc_data[topic] = pmc_list | |
| else: | |
| continue | |
| else: | |
| continue | |
| data = json.dumps(pmc_data, indent=4, ensure_ascii=False) | |
| return data | |
| def extract_pmc_data(): | |
| if not tools.download_datafile('pmc.txt'): | |
| raise Exception("Failed to download datafile") | |
| pmc_data ={} | |
| pmcid_data = json.loads(doi_to_pmc()) | |
| for topic, pmcids in pmcid_data.items(): | |
| pmc_ids = [] | |
| for id in pmcids: | |
| if tools.check_data_in_file(id, 'pmc.txt'): | |
| continue | |
| else: | |
| tools.write_data_to_file(id, 'pmc.txt') | |
| pmc_ids.append(id) | |
| pmc_data[topic] = {} | |
| pmc_data[topic]['count'] = len(pmc_ids) | |
| pmc_data[topic]['ids'] = pmc_ids | |
| data = json.dumps(pmc_data, indent=4, ensure_ascii=False) | |
| if not tools.upload_datafile('pmc.txt'): | |
| raise Exception("Failed to upload datafile") | |
| return data | |
| if __name__ == "__main__": | |
| data = extract_pmc_data() | |
| with open('pmc_data.json', 'w') as f: | |
| f.write(data) |