working basic app + dockerfile

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

Dockerfile

@@ -1,14 +1,19 @@
 # read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
 # you will also find guides on how best to write your Dockerfile
 
+# Use an official Python runtime as a parent image
 FROM python:3.9
 
-WORKDIR /code
+# Set the working directory in the container to /app
+WORKDIR /app
 
-COPY ./requirements.txt /code/requirements.txt
+# Add the current directory contents into the container at /app
+ADD . /app
 
-RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+# Install any needed packages specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
-COPY . .
+# Make port 7860 available to the world outside this container
+EXPOSE 7860
 
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -1,36 +0,0 @@
-from fastapi import FastAPI, Request, WebSocket
-from fastapi.responses import HTMLResponse
-from fastapi.templating import Jinja2Templates
-from typing import Optional
-
-
-
-from pydantic import BaseModel
-
-
-app = FastAPI()
-templates = Jinja2Templates(directory="templates")
-
-
-# Create a Pydantic model for the form data
-class DatasetForm(BaseModel):
-    dataset_name: str
-
-
-
-def long_running_function():
-    pass
-
-@app.get("/", response_class=HTMLResponse)
-async def read_form(request: Request):
-    # Render the form.html template
-    return templates.TemplateResponse("form.html", {"request": request})
-
-@app.post("/submit_form")
-async def form_post(form_data: DatasetForm):
-    # Do something with form_data
-    print(form_data.dict())
-    return {"message": "Form data received!", "received_data": form_data.dict()}
-
-
-

build_map.py

@@ -109,12 +109,13 @@ def load_dataset_and_metadata(dataset_name,
     return dataset_dict
 
 
-def upload_project_to_atlas(dataset_dict, 
+def upload_dataset_to_atlas(dataset_dict, 
                             project_name = None,
                             unique_id_field_name=None, 
                             indexed_field = None, 
                             modality=None,
-                            organization_name=None):
+                            organization_name=None,
+                            wait_for_map=True):
 
     if modality is None:
         modality = "text"
@@ -231,6 +232,10 @@ def upload_project_to_atlas(dataset_dict,
             topic_label_field = topic_label_field,
             build_topic_model=True)
     
+    if wait_for_map:
+        with proj.wait_for_project_lock():
+            time.sleep(1)
+    
     return projection.map_link
 
 # Run test
@@ -240,4 +245,4 @@ if __name__ == "__main__":
     project_name = "huggingface_auto_upload_test-dolly-15k"
 
     dataset_dict = load_dataset_and_metadata(dataset_name)
-    print(upload_project_to_atlas(dataset_dict, project_name=project_name))
+    print(upload_dataset_to_atlas(dataset_dict, project_name=project_name))

form.html

@@ -1,11 +0,0 @@
-<!DOCTYPE html>
-<html>
-    <body>
-        <h2>HTML Form</h2>
-        <form action="/submit_form" method="post">
-            <label for="name">Name:</label><br>
-            <input type="text" id="dataset_name" name="dataset_name" value=""><br>
-            <input type="submit" value="Submit">
-        </form> 
-    </body>
-</html>

main.py

@@ -0,0 +1,57 @@
+from fastapi import FastAPI, Form, Request, BackgroundTasks
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+#from pydantic import BaseModel
+
+from uuid import uuid4
+import asyncio
+
+from build_map import load_dataset_and_metadata, upload_dataset_to_atlas
+
+
+app = FastAPI()
+# TODO: use task management queue
+tasks = {}
+templates = Jinja2Templates(directory="templates")
+
+def upload_atlas_task(task_id, dataset_name):
+    dataset_dict = load_dataset_and_metadata(dataset_name)
+    map_url = upload_dataset_to_atlas(dataset_dict, project_name="atlas-space-test")
+    tasks[task_id]['status'] = 'done'
+    tasks[task_id]['url'] = map_url
+
+@app.on_event("startup")
+async def startup_event():
+    asyncio.create_task(cleanup_tasks())
+
+async def cleanup_tasks():
+    while True:
+        current_time = time.time()
+        tasks_to_delete = []
+        for task_id, task in tasks.items():
+            if task['status'] == 'done' and current_time - task.get('finish_time', current_time) > 1800:  # 30 minutes
+                tasks_to_delete.append(task_id)
+        for task_id in tasks_to_delete:
+            del tasks[task_id]
+        await asyncio.sleep(1800)  # Wait for 30 minutes
+
+@app.get("/", response_class=HTMLResponse)
+async def read_form(request: Request):
+    # Render the form.html template
+    return templates.TemplateResponse("form.html", {"request": request})
+
+@app.post("/submit_form")
+async def form_post(background_tasks: BackgroundTasks, dataset_name: str = Form(...)):
+    task_id = str(uuid4())
+    tasks[task_id] = {'status': 'running'}
+    #form_data = DatasetForm(dataset_name=dataset_name)
+    background_tasks.add_task(upload_atlas_task, task_id, dataset_name)
+    
+    return {'task_id': task_id}
+
+@app.get("/status/{task_id}")
+async def read_task(task_id: str):
+    if task_id not in tasks:
+        return {'status': 'not found'}
+    else:
+        return tasks[task_id]

templates/form.html

@@ -0,0 +1,98 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>Atlas Map</title>
+
+    <!-- CSS only -->
+    <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" rel="stylesheet">
+
+    <style>
+        body {
+            display: flex;
+            justify-content: center;
+            align-items: center;
+            height: 100vh;
+            background-color: #f5f5f5;
+        }
+
+        .form-container {
+            background-color: white;
+            padding: 30px;
+            border-radius: 10px;
+            box-shadow: 0px 0px 10px 0px rgba(0,0,0,0.1);
+        }
+
+        .form-container h2 {
+            margin-bottom: 30px;
+        }
+        
+        #loading {
+            position: absolute;
+            top: 0;
+            left: 0;
+            right: 0;
+            bottom: 0;
+            background-color: rgba(0, 0, 0, 0.5);
+            display: flex;
+            justify-content: center;
+            align-items: center;
+        }
+
+        .loading-content {
+            padding: 20px;
+            background-color: white;
+            border-radius: 5px;
+            font-size: 20px;
+            color: #333;
+            box-shadow: 0px 0px 10px 0px rgba(0,0,0,0.1);
+        }
+    </style>
+</head>
+<body>
+
+<div class="container form-container">
+    <h2>HuggingFace Dataset to Atlas Map</h2>
+    <form id="myForm" action="/submit_form" method="post">
+        <div class="form-group">
+            <label for="dataset_name">Dataset Name:</label>
+            <input type="text" class="form-control" id="dataset_name" name="dataset_name">
+        </div>
+        <button type="submit" class="btn btn-primary">Submit</button>
+    </form> 
+
+    <div id="loading" style="display: none;">
+        <div class="loading-content">
+            Building map...
+        </div>
+    </div>
+</div>
+
+<script>
+    window.onload = function() {
+        document.getElementById('myForm').onsubmit = async function(e) {
+            e.preventDefault();  // Prevent the form from submitting normally
+    
+            document.getElementById('loading').style.display = 'block';  // Show the loading message
+    
+            let formData = new FormData(this);
+            let response = await fetch('/submit_form', {method: 'POST', body: formData});  // Send the form data to the server
+            let data = await response.json();  // Parse the server's response
+            checkStatus(data.task_id);
+        };
+    }
+    
+    async function checkStatus(taskId) {
+        let response = await fetch(`/status/${taskId}`);
+        let data = await response.json();
+        if (data.status === 'running') {
+            setTimeout(() => checkStatus(taskId), 5000);  // Check again in 5 seconds
+        } else {
+            window.location.href = data.url;  // Redirect to the finished URL
+        }
+    }
+</script>
+
+</body>
+</html>