{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import string\n", "import pandas as pd\n", "import time\n", "import urllib\n", "import urllib.request\n", "import json" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'computer science': ['machine learning', 'artificial intelligence', 'hardware architecture', 'computational complexity', 'data structures', 'algorithms', 'graphics', 'databases', 'discrete mathematics', 'human-computer interaction', 'information retrieval', 'multiagent systems', 'neural network'], 'economics': ['general economics', 'theoretical economics', 'econometrics'], 'electrical engineering and system science': ['audio processing', 'speech processing', 'signal processing', 'image and video processing', 'system and controls'], 'mathematics': ['general mathematics', 'general topology', 'group theory', 'numerical analysis', 'probability', 'number theory', 'statistic theory']}\n" ] } ], "source": [ "baseurl = 'http://export.arxiv.org/api/query?search_query='\n", "\n", "# still ambigious, what are keywords?\n", "\n", "timestamp = \"2020-01-01\" \n", "max_results = 10000\n", "date = pd.Timestamp(str(timestamp), tz='US/Pacific')\n", "\n", "topics = json.load(open(\"topics.txt\",\"r\"))\n", "print(topics)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for key in topics:\n", " # print(key)\n", " # prepare url for each topic\n", " keyword_list = topics[key]\n", " i = 0\n", " for keyword in keyword_list:\n", " if i ==0:\n", " url = baseurl + 'all:' + keyword\n", " i = i + 1 \n", " else:\n", " url = url + '+OR+' + 'all:' + keyword\n", " url = url+ '&max_results=' + str(max_results)\n", " url = url.replace(' ', '%20')\n", "\n", " arxiv_page = urllib.request.urlopen(url,timeout=100).read()\n", " with open(key+\".xml\",\"wb\") as outfile:\n", " outfile.write(arxiv_page)\n", " print(url)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def crawl_from_url(url):\n", " try: \n", " arxiv_page = urllib.request.urlopen(url,timeout=100).read()\n", " with open(\"save.xml\",\"wb\") as outfile:\n", " outfile.write(arxiv_page)\n", " arxiv_page = str(arxiv_page) \n", " # Mỗi record nằm trong một thẻ \n", " # chứa đường dẫn tới paper trên arxiv\n", " # , là thời gian gần nhất cập nhật/xuất bản\n", " # là tiêu đề paper\n", " # <summary> là abstract paper\n", " # có thể có nhiều thẻ <author> chứa tên các tác giả\n", " # <link title=\"pdf\" href=\" ... chứa link tải paper\n", "\n", " # trích 1 record dựa vào thẻ <entry>\n", " start = arxiv_page.find(\"<entry>\")\n", " end = arxiv_page.find(\"</entry>\")\n", " extract = arxiv_page[start+7:end]\n", " # print(extract)\n", "\n", " except Exception as e:\n", " print(\"Error occured: \",e)\n", "\n", "crawl_from_url(url)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def extract_tag(txt,tagname):\n", " return txt[txt.find(\"<\"+tagname+\">\")+len(tagname)+2:txt.find(\"</\"+tagname+\">\")]\n", "\n", "def get_record(extract):\n", " # id = extract[extract.find(\"<id>\")+4:extract.find(\"</id>\")]\n", " # updated = extract[extract.find(\"<updated>\")+9:extract.find(\"</updated>\")]\n", " # published = extract[extract.find(\"<published>\")+11:extract.find(\"</published>\")]\n", " # title = extract[extract.find(\"<title>\")+7:extract.find(\"\")]\n", " # summary = extract[extract.find(\"\")+9:extract.find(\"\")]\n", " id = extract_tag(extract,\"id\")\n", " updated = extract_tag(extract,\"updated\")\n", " published = extract_tag(extract,\"published\")\n", " title = extract_tag(extract,\"title\").replace(\"\\n \",\"\").strip()\n", " summary = extract_tag(extract,\"summary\").replace(\"\\n\",\"\").strip()\n", " authors = []\n", " while extract.find(\"\")!=-1:\n", " # author = extract[extract.find(\"\")+6:extract.find(\"\")]\n", " author = extract_tag(extract,\"name\")\n", " extract = extract[extract.find(\"\")+9:]\n", " authors.append(author)\n", " pattern = '\") != -1:\n", " extract = xml[xml.find(\"\")+7:xml.find(\"\")]\n", " xml = xml[xml.find(\"\")+8:]\n", " records.append([key,*get_record(extract)])" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3000\n", "\n" ] } ], "source": [ "print(len(records))\n", "print(type(records[32][5]))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "df = pd.DataFrame(records,columns=[\"topic\",\"id\",\"updated\",\"published\",\"title\",\"author\",\"link\",\"summary\",])\n", "df.to_csv(\"arxiv_crawl.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "topics_descriptions = json.load(open(\"topic_descriptions.txt\",\"r\"))\n", "print(topics_descriptions)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "embed = model.encode(\"\"\"Recommendation systems for different Document Networks (DN) such as the World\n", "Wide Web (WWW) and Digital Libraries, often use distance functions extracted\n", "from relationships among documents and keywords. For instance, documents in the\n", "WWW are related via a hyperlink network, while documents in bibliographic\n", "databases are related by citation and collaboration networks. Furthermore,\n", "documents are related to keyterms. The distance functions computed from these\n", "relations establish associative networks among items of the DN, referred to as\n", "Distance Graphs, which allow recommendation systems to identify relevant\n", "associations for individual users. However, modern recommendation systems need\n", "to integrate associative data from multiple sources such as different\n", "databases, web sites, and even other users. Thus, we are presented with a\n", "problem of combining evidence (about associations between items) from different\n", "sources characterized by distance functions. In this paper we describe our work\n", "on (1) inferring relevant associations from, as well as characterizing,\n", "semi-metric distance graphs and (2) combining evidence from different distance\n", "graphs in a recommendation system. Regarding (1), we present the idea of\n", "semi-metric distance graphs, and introduce ratios to measure semi-metric\n", "behavior. We compute these ratios for several DN such as digital libraries and\n", "web sites and show that they are useful to identify implicit associations.\n", "Regarding (2), we describe an algorithm to combine evidence from distance\n", "graphs that uses Evidence Sets, a set structure based on Interval Valued Fuzzy\n", "Sets and Dempster-Shafer Theory of Evidence. This algorithm has been developed\n", "for a recommendation system named TalkMine.\"\"\")\n", "for topic in topics_descriptions:\n", " description = topics_descriptions[topic]\n", " embed_desc = model.encode(description)\n", " print(topic+\": \"+str(cos_sim(embed,embed_desc)))" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import chromadb\n", "from chromadb import Documents, EmbeddingFunction, Embeddings\n", "\n", "from transformers import AutoModel\n", "from numpy.linalg import norm\n", "\n", "cos_sim = lambda a,b: (a @ b.T) / (norm(a)*norm(b))\n", "model = AutoModel.from_pretrained('jinaai/jina-embeddings-v2-base-en',\n", " trust_remote_code=True,\n", " cache_dir='models') # trust_remote_code is needed to use the encode method\n", "class JinaAIEmbeddingFunction(EmbeddingFunction):\n", " def __init__(self, model):\n", " super().__init__()\n", " self.model = model\n", "\n", " def __call__(self, input: Documents) -> Embeddings:\n", " embeddings = self.model.encode(input)\n", " return embeddings.tolist()\n", "\n", "ef = JinaAIEmbeddingFunction(model)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "client = chromadb.PersistentClient(path=\"arxivdb/\")\n", "# first creation, embedding function = default\n", "# collection = client.create_collection(name=\"arxiv_records\",metadata={\"hnsw:space\": \"cosine\"})\n", "# later call\n", "collection = client.get_or_create_collection(name=\"arxiv_records\", embedding_function=ef, metadata={\"hnsw:space\": \"cosine\"})\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "client.delete_collection(name=\"arxiv_records\")" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "import sqlite3\n", "con = sqlite3.connect(\"arxiv_records_sql\")\n", "cur = con.cursor()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "ename": "OperationalError", "evalue": "table arxivsql already exists", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mOperationalError\u001b[0m Traceback (most recent call last)", "Cell \u001b[1;32mIn[14], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mcur\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\"\"\u001b[39;49m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;124;43m create table arxivsql(\u001b[39;49m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;124;43m id,\u001b[39;49m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;124;43m topic,\u001b[39;49m\n\u001b[0;32m 5\u001b[0m \u001b[38;5;124;43m title,\u001b[39;49m\n\u001b[0;32m 6\u001b[0m \u001b[38;5;124;43m authors,\u001b[39;49m\n\u001b[0;32m 7\u001b[0m \u001b[38;5;124;43m year_updated,\u001b[39;49m\n\u001b[0;32m 8\u001b[0m \u001b[38;5;124;43m year_published,\u001b[39;49m\n\u001b[0;32m 9\u001b[0m \u001b[38;5;124;43m link\u001b[39;49m\n\u001b[0;32m 10\u001b[0m \u001b[38;5;124;43m )\u001b[39;49m\n\u001b[0;32m 11\u001b[0m \u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m 12\u001b[0m con\u001b[38;5;241m.\u001b[39mcommit()\n", "\u001b[1;31mOperationalError\u001b[0m: table arxivsql already exists" ] } ], "source": [ "cur.execute(\"\"\"\n", " create table arxivsql(\n", " id,\n", " topic,\n", " title,\n", " authors,\n", " year_updated,\n", " year_published,\n", " link\n", " )\n", "\"\"\")\n", "con.commit()" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "cur.execute(\"drop table arxivsql\")\n", "con.commit()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(3000, 8)\n", "\n" ] } ], "source": [ "import pandas as pd\n", "df = pd.read_csv(\"arxiv_crawl.csv\",index_col=0,header=0)\n", "print(df.shape)\n", "records = df.values\n", "print(type(records))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Domenico Amato, Giosue' Lo Bosco, Raffaele Giancarl\n" ] } ], "source": [ "def chunk_text(text, max_char=400):\n", " \"\"\"\n", " Chunk a long text into several chunks, with each chunk about 300-400 characters long,\n", " but make sure no word is cut in half.\n", " Args:\n", " text: The long text to be chunked.\n", " max_char: The maximum number of characters per chunk (default: 400).\n", " Returns:\n", " A list of chunks.\n", " \"\"\"\n", " chunks = []\n", " current_chunk = \"\"\n", " words = text.split()\n", " for word in words:\n", " # Check if adding the word would exceed the chunk limit (including overlap)\n", " if len(current_chunk) + len(word) + 1 >= max_char:\n", " chunks.append(current_chunk)\n", " current_chunk = word\n", " else:\n", " current_chunk += \" \" + word\n", " chunks.append(current_chunk.strip())\n", " return chunks\n", "\n", "def process_authors(authors):\n", " text = \"\"\n", " for author in authors:\n", " text+=author+\", \"\n", " return text[:-3]\n", "\n", "print(process_authors(records[32][5]))" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "200\n", "400\n", "600\n", "800\n", "1000\n", "1200\n", "1400\n", "1600\n", "1800\n", "2000\n", "2200\n", "2400\n", "2600\n", "2800\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Insert of existing embedding ID: 2111.13171v1_0\n", "Insert of existing embedding ID: 2111.13171v1_1\n", "Insert of existing embedding ID: 2111.13171v1_2\n", "Insert of existing embedding ID: 2111.13171v1_3\n", "Insert of existing embedding ID: 2111.13171v1_4\n", "Add of existing embedding ID: 2111.13171v1_0\n", "Add of existing embedding ID: 2111.13171v1_1\n", "Add of existing embedding ID: 2111.13171v1_2\n", "Add of existing embedding ID: 2111.13171v1_3\n", "Add of existing embedding ID: 2111.13171v1_4\n", "Insert of existing embedding ID: 2211.03756v1_0\n", "Insert of existing embedding ID: 2211.03756v1_1\n", "Insert of existing embedding ID: 2211.03756v1_2\n", "Insert of existing embedding ID: 2211.03756v1_3\n", "Insert of existing embedding ID: 2211.03756v1_4\n", "Insert of existing embedding ID: 2211.03756v1_5\n", "Add of existing embedding ID: 2211.03756v1_0\n", "Add of existing embedding ID: 2211.03756v1_1\n", "Add of existing embedding ID: 2211.03756v1_2\n", "Add of existing embedding ID: 2211.03756v1_3\n", "Add of existing embedding ID: 2211.03756v1_4\n", "Add of existing embedding ID: 2211.03756v1_5\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "3000\n" ] } ], "source": [ "count = 0\n", "for record in records:\n", " # add to vector db\n", " embed_text = \"\"\"\n", " Topic: {},\n", " Title: {},\n", " Summary: {}\n", "\"\"\".format(\n", " record[0], record[4], record[7]\n", " )\n", " chunks = chunk_text(embed_text)\n", " ids = [record[1][21:] + \"_\" + str(j) for j in range(len(chunks))]\n", " paper_ids = [{\"paper_id\": record[1][21:]} for _ in range(len(chunks))]\n", " collection.add(documents=chunks, metadatas=paper_ids, ids=ids)\n", " # try:\n", " # query = \"\"\"insert into arxivsql values(\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\")\"\"\".format(\n", " # record[1][21:],\n", " # record[0],\n", " # record[4].replace('\"', \"'\"),\n", " # process_authors(record[5]),\n", " # record[2][:10],\n", " # record[3][:10],\n", " # record[6],\n", " # )\n", " # cur.execute(query)\n", " # con.commit()\n", " # except Exception as e:\n", " # print(e)\n", " # print(query)\n", " count += 1\n", " if count % 200 == 0:\n", " print(count)" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "cur.execute(\"\"\"insert into arxivsql values(\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\")\"\"\".format(\n", " \"1906.04027v2\", #editing\n", " \"electrical engineering and system science\",\n", " \"'Did You Hear That?'' Learning to Play Video Games from Audio Cues\",\"Raluca D. Gaina, Matthew Stephenso\",\n", " \"Hadi Abdullah, Muhammad Sajidur Rahman, Washington Garcia, Logan Blue, Kevin Warren, Anurag Swarnim Yadav, Tom Shrimpton, Patrick Trayno\",\n", " \"2019-06-11\",\n", " \"2019-06-10\",\n", " \"http://arxiv.org/pdf/1910.05262v1\"\n", " ))\n", "con.commit()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'cur' is not defined", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[1;32mIn[11], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m res \u001b[38;5;241m=\u001b[39m \u001b[43mcur\u001b[49m\u001b[38;5;241m.\u001b[39mexecute(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mselect * from arxivsql where True and True\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(res\u001b[38;5;241m.\u001b[39mfetchall())\n", "\u001b[1;31mNameError\u001b[0m: name 'cur' is not defined" ] } ], "source": [ "res = cur.execute(\"select * from arxivsql where True and True\")\n", "print(res.fetchall())" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10740\n" ] } ], "source": [ "print(collection.count())" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['2211.03756v1_0', '2211.03756v1_1', '2211.03756v1_2', '2211.03756v1_3', '2211.03756v1_4', '2211.03756v1_5', '2211.03756v1_6']\n" ] } ], "source": [ "id = \"2211.03756v1\"\n", "ids = [\"{}_{}\".format(id,j) for j in range(0,10)]\n", "results = collection.get(ids=ids)\n", "print(results[\"ids\"])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "results = collection.query(\n", " query_texts = \"recommend academic articles or books related to the field of artificial intelligence, machine learning and technology for the AI intern to explore further\",\n", " where_document = {\n", " \"$or\":[\n", " {\"$contains\":\"AI\"},\n", " {\"$contains\":\"machine learning\"},\n", " {\"$contains\":\"technology\"}\n", " ]\n", " },\n", " n_results=3\n", ")" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['title', 'author']\n" ] } ], "source": [ "args = {\"title\":\"Attention is all you need\",\n", " \"author\": \"Vaswani, Ashish and Shazeer\"}\n", "keys = list(dict.keys(args))\n", "print(keys)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def printline(txt, maxline = 100):\n", " for i in range(len(txt)):\n", " if i%maxline == maxline-1:\n", " print(txt[i],end=\"\\n\")\n", " else: print(txt[i],end=\"\")\n", "\n", "print(dict.keys(results))\n", "# get metadatas\n", "target = results['metadatas'][0]\n", "for rec in target:\n", " print(rec['author'])\n", " print(rec['link'])\n", " printline(rec['summary'])\n", " print(\"\\n------------------------------------------\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "t = target[0]\n", "print(t['link'])\n", "print(t['title'])\n", "print(t['summary'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "args = '[\"AI technologies\",\"Find academic papers\"]'\n", "print(list(args))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.2" } }, "nbformat": 4, "nbformat_minor": 2 }