test

.gitattributes

@@ -17,7 +17,7 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text 
 *.tar.* filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
@@ -25,3 +25,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+AbstractGenerator/ filter=lfs diff=lfs merge=lfs -text
+AbstractGenerator/** filter=lfs diff=lfs merge=lfs -text
+models/** filter=lfs diff=lfs merge=lfs -text
+models/ filter=lfs diff=lfs merge=lfs -text

AbstractGenerator.ipynb

@@ -0,0 +1,247 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'gpt_2_simple'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "\u001b[1;32mc:\\Users\\franz\\AbstractGenerator\\AbstractGenerator.ipynb Cell 1'\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/franz/AbstractGenerator/AbstractGenerator.ipynb#ch0000000?line=0'>1</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mgpt_2_simple\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mgpt2\u001b[39;00m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/franz/AbstractGenerator/AbstractGenerator.ipynb#ch0000000?line=1'>2</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mos\u001b[39;00m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/franz/AbstractGenerator/AbstractGenerator.ipynb#ch0000000?line=2'>3</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mtensorflow\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mtf\u001b[39;00m\n",
+      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'gpt_2_simple'"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import gpt_2_simple as gpt2\n",
+    "import os\n",
+    "import tensorflow as tf\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "print(\"GPU is\", \"available\" if tf.test.is_gpu_available() else \"NOT AVAILABLE\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_name = \"124M\"\n",
+    "if not os.path.isdir(os.path.join(\"models\", model_name)):\n",
+    "\tprint(f\"Downloading {model_name} model...\")\n",
+    "\tgpt2.download_gpt2(model_name=model_name)  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path = 'AbstractGenerator/'\n",
+    "checkpoint_dir =path+'weights/'\n",
+    "data_path = path+'Tokenized_data/'\n",
+    "\n",
+    "\n",
+    "file_name = 'resumen'\n",
+    "file_path = data_path+file_name\n",
+    "\n",
+    "prefix= '<|startoftext|>'\n",
+    "sufix ='<|endoftext|>'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# pretrained"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sess = gpt2.start_tf_sess()\n",
+    "gpt2.load_gpt2(sess,checkpoint_dir=checkpoint_dir,run_name='run1')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# train "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tf.compat.v1.reset_default_graph()\n",
+    "sess = gpt2.start_tf_sess()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gpt2.finetune(sess,\n",
+    "              file_path+'.txt',\n",
+    "              model_name=model_name,\n",
+    "              checkpoint_dir=checkpoint_dir,   \n",
+    "              steps=1000\n",
+    "              )   "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"\"\"Introduction and preliminaries\n",
+    "The focus of this paper is decompositions of (k, `)-sparse graphs into edge-disjoint subgraphs\n",
+    "that certify sparsity. We use graph to mean a multigraph, possibly with loops. We say that a\n",
+    "graph is (k, `)-sparse if no subset of n′ vertices spans more than kn′− ` edges in the graph; a\n",
+    "(k, `)-sparse graph with kn′− ` edges is (k, `)-tight. We call the range k ≤ `≤ 2k−1 the upper\n",
+    "range of sparse graphs and 0≤ `≤ k the lower range.\n",
+    "In this paper, we present efficient algorithms for finding decompositions that certify sparsity\n",
+    "in the upper range of `. Our algorithms also apply in the lower range, which was already ad-\n",
+    "dressed by [3, 4, 5, 6, 19]. A decomposition certifies the sparsity of a graph if the sparse graphs\n",
+    "and graphs admitting the decomposition coincide.\n",
+    "Our algorithms are based on a new characterization of sparse graphs, which we call the\n",
+    "pebble game with colors. The pebble game with colors is a simple graph construction rule that\n",
+    "produces a sparse graph along with a sparsity-certifying decomposition.\n",
+    "We define and study a canonical class of pebble game constructions, which correspond to\n",
+    "previously studied decompositions of sparse graphs into edge disjoint trees. Our results provide\n",
+    "a unifying framework for all the previously known special cases, including Nash-Williams-\n",
+    "Tutte and [7, 24]. Indeed, in the lower range, canonical pebble game constructions capture the\n",
+    "properties of the augmenting paths used in matroid union and intersection algorithms[5, 6].\n",
+    "Since the sparse graphs in the upper range are not known to be unions or intersections of the\n",
+    "matroids for which there are efficient augmenting path algorithms, these do not easily apply in\n",
+    "∗ Research of both authors funded by the NSF under grants NSF CCF-0430990 and NSF-DARPA CARGO\n",
+    "CCR-0310661 to the first author.\n",
+    "2 Ileana Streinu, Louis Theran\n",
+    "Term Meaning\n",
+    "Sparse graph G Every non-empty subgraph on n′ vertices has ≤ kn′− ` edges\n",
+    "Tight graph G G = (V,E) is sparse and |V |= n, |E|= kn− `\n",
+    "Block H in G G is sparse, and H is a tight subgraph\n",
+    "Component H of G G is sparse and H is a maximal block\n",
+    "Map-graph Graph that admits an out-degree-exactly-one orientation\n",
+    "(k, `)-maps-and-trees Edge-disjoint union of ` trees and (k− `) map-grpahs\n",
+    "`Tk Union of ` trees, each vertex is in exactly k of them\n",
+    "Set of tree-pieces of an `Tk induced on V ′ ⊂V Pieces of trees in the `Tk spanned by E(V ′)\n",
+    "Proper `Tk Every V ′ ⊂V contains ≥ ` pieces of trees from the `Tk\n",
+    "Table 1. Sparse graph and decomposition terminology used in this paper.\n",
+    "the upper range. Pebble game with colors constructions may thus be considered a strengthening\n",
+    "of augmenting paths to the upper range of matroidal sparse graphs.\n",
+    "1.1. Sparse graphs\n",
+    "\n",
+    "ABSTRACT\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gpt2.generate(sess,prefix=text,truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Data Tokeniser"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = pd.read_csv('Recipe-Creator\\data\\scientific_paper_full_text_translated.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import codecs\n",
+    "with codecs.open(\"Recipe-Creator/Tokenized_data/resumen.txt\",'a','utf-8') as f:\n",
+    "    for i in ds.index:\n",
+    "        f.write(prefix+\"\\n\")\n",
+    "        f.write(ds.iloc[i]['text_no_abstract'])\n",
+    "        f.write(\"ABSTRACT\\n\")\n",
+    "        f.write(ds.iloc[i]['abstract']+\"\\n\")\n",
+    "        f.write(sufix)\n",
+    "       "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gradio as gr\n",
+    "\n",
+    "def greet(text):\n",
+    "    return gpt2.generate(sess,prefix=str(text),truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)\n",
+    "\n",
+    "iface = gr.Interface(fn=greet, inputs=\"text\", outputs=\"text\")\n",
+    "iface.launch(share=True,debug=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "53fbdc69e3e12c371950068c144423682c30d04ec68c2bd46937202e33e0058d"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.7.11 ('receta')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

AbstractGenerator.yml

@@ -0,0 +1,14 @@
+name: Recipe-Creator
+
+dependencies:
+    - python>=3.7
+    - pip>=19.0
+    - jupyter
+    - pandas   
+    - pip:
+        - gpt-2-simple
+        - tensorflow-estimator==1.15.1
+        - tensorflow-gpu==1.15
+        
+
+        

AbstractGenerator/Tokenized_data/reduced_dataset_47MB.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ddebcb25fcc12a0029f83374ac9ea5e010e5dcbc58162ad6cac42360391f4f5
+size 48623670

AbstractGenerator/Tokenized_data/resumen.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ee5de5835a959b24bb1fba6bed5b14a6c5e7682f04483850109123d448c2853
+size 62458342

AbstractGenerator/data/scientific_paper_full_text_translated.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58700a294dbdc6a78ef056f207cbee864e47b81c59e0209d0f5b622276e373f0
+size 227783472

AbstractGenerator/weights/run1/checkpoint

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cdce1a7de49e734203b3af24a79c6e7f92b12c50462e0a6ee9e5ce254e8c5a7
+size 77

AbstractGenerator/weights/run1/counter

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b7b041d37d1d693c3afe25d2af7d56ad2ee7b98eddb2cc1a055c1117e55542e
+size 6

AbstractGenerator/weights/run1/encoder.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783
+size 1042301

AbstractGenerator/weights/run1/events.out.tfevents.1648099088.FRANZ96521-W11

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ce17a8b02dce07c39e62333105e012aad5f62825b207ccb692becbe31bdc2b2
+size 42913

AbstractGenerator/weights/run1/hparams.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d56e4121c427164e0c55c6f03c08e1daf9002b9b672825112d19097b680318
+size 90

AbstractGenerator/weights/run1/model-1000.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:904922cd8b9620aec00f8072edc3717b03dfb4a700b585fe88012da8af0ce67a
+size 497759232

AbstractGenerator/weights/run1/model-1000.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75db8ca12b433be77ffdf420d041d20837f2698ee5ab7132773f6d25cf841637
+size 5215

AbstractGenerator/weights/run1/model-1000.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e2968f20985aa601ae18b0ddf2b29bb5625822a4e11e3a45c0c5406f032e7b7
+size 3884257

AbstractGenerator/weights/run1/vocab.bpe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5
+size 456318

App.py

@@ -0,0 +1,40 @@
+import os
+os.system('pip install gpt-2-simple')
+os.system('pip install tensorflow-estimator==1.15.1')
+
+import gpt_2_simple as gpt2
+import tensorflow as tf
+import pandas as pd
+import re
+import gradio as gr
+
+model_name = "124M"
+
+path = 'AbstractGenerator/'
+checkpoint_dir =path+'weights/'
+data_path = path+'Tokenized_data/'
+
+
+file_name = 'resumen'
+file_path = data_path+file_name
+
+prefix= '<|startoftext|>'
+sufix ='<|endoftext|>'
+
+print("GPU is", "available" if tf.test.is_gpu_available() else "NOT AVAILABLE")
+
+sess = gpt2.start_tf_sess()
+gpt2.load_gpt2(sess,checkpoint_dir=checkpoint_dir,run_name='run1')
+
+
+
+
+
+def greet(text):
+    return gpt2.generate(sess,prefix=text,truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)
+
+
+
+
+iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+iface.launch(share=True)

models/124M/checkpoint

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd1b025d2e155283f5e300ce95bf6d5b6bc0f7fe010db73daa6975eb896ab9cb
+size 77

models/124M/encoder.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783
+size 1042301

models/124M/hparams.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d56e4121c427164e0c55c6f03c08e1daf9002b9b672825112d19097b680318
+size 90

models/124M/model.ckpt.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2060c885360cc0cf41d7a6dbc4d24b5127aae20260c8b5ae521b5a6578407118
+size 497759232

models/124M/model.ckpt.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71916f763f9746f9b2a06b12d91996cf1084ae008d0424543d39391c5f2dc687
+size 5215

models/124M/model.ckpt.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4668c448fa11531fd6700460487f73e82d3272960cea942252f8744bf225c77b
+size 471155

models/124M/vocab.bpe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5
+size 456318