Upload 5 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,109 @@
+import streamlit as st
+import sparknlp
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    documentAssembler = DocumentAssembler() \
+        .setInputCol("text") \
+        .setOutputCol("documents")
+        
+    t5 = T5Transformer.pretrained(model) \
+        .setTask("cola:") \
+        .setInputCols(["documents"])\
+        .setMaxOutputLength(200)\
+        .setOutputCol("corrections")
+
+    pipeline = Pipeline().setStages([documentAssembler, t5])
+    return pipeline
+
+def fit_data(pipeline, data):
+    df = spark.createDataFrame([[data]]).toDF("text")
+    result = pipeline.fit(df).transform(df)
+    return result.select('corrections.result').collect()
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ['t5_base',  't5_small', 't5_large'],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Set up the page layout
+title = "Evaluate Sentence Grammar"
+sub_title = "This demo uses a text-to-text model fine-tuned to evaluate grammatical errors when the task is set to 'cola:'"
+
+st.markdown(f'<div class="main-title">{title}</div>', unsafe_allow_html=True)
+st.markdown(f'<div style="text-align: center; color: #666666;">{sub_title}</div>', unsafe_allow_html=True)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5_LINGUISTIC.ipynb#scrollTo=QAZ3vOX_SW7B">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Define the examples
+examples = [
+    "She don't knows nothing about what's happening in the office.",
+    "They was playing soccer yesterday when it start raining heavily.",
+    "This car are more faster than that one, but it costed less money.",
+    "I seen him go to the store, but he don't buy nothing from there.",
+    "We was going to the park but it start raining before we could leave."
+]
+
+# Text selection and analysis
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.write('Text to be evaluated:')
+HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, text_to_analyze)
+
+# Display transformed sentence
+st.write("Prediction:")
+output_text = "".join(output[0][0])
+st.markdown(f'<div class="scroll">{output_text}</div>', unsafe_allow_html=True)

Dockerfile

@@ -0,0 +1,72 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean
+
+# Set up JAVA_HOME
+RUN echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> /etc/profile \
+    && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set up PySpark to use Python 3.8 for both driver and workers
+ENV PYSPARK_PYTHON=/usr/bin/python3.8
+ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3.8
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,167 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Title
+st.markdown('<div class="main-title">Evaluate Sentence Grammar</div>', unsafe_allow_html=True)
+
+# Introduction Section
+st.markdown("""
+<div class="section">
+    <p>Evaluating sentence grammar is crucial for maintaining the clarity and accuracy of written communication. Whether you're reviewing content for publication, editing academic work, or checking everyday writing, ensuring grammatical correctness is key.</p>
+    <p>This page showcases the implementation of a grammar evaluation pipeline using advanced NLP models. We leverage the T5 Transformer model, fine-tuned for assessing sentence grammar, to evaluate and identify potential errors in sentences.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# T5 Transformer Overview
+st.markdown('<div class="sub-title">Understanding the T5 Transformer for Grammar Evaluation</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>The T5 (Text-To-Text Transfer Transformer) model, developed by Google, is a powerful tool for various NLP tasks, including grammar evaluation. When configured with the appropriate task, T5 can assess sentences for grammatical correctness, helping users identify and correct errors.</p>
+    <p>This capability is particularly useful in proofreading tools, automated editing software, and educational applications, where precise grammar is essential.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Performance Section
+st.markdown('<div class="sub-title">Performance and Use Cases</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>The T5 model exhibits strong performance in grammar evaluation tasks, providing accurate and contextually relevant assessments. This makes it a valuable resource for anyone looking to improve the quality of written content.</p>
+    <p>Use cases include academic proofreading, professional editing, and everyday writing checks, where maintaining grammatical integrity is of utmost importance.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Implementation Section
+st.markdown('<div class="sub-title">Implementing Grammar Evaluation</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>The following example demonstrates how to implement a grammar evaluation pipeline using Spark NLP. The pipeline includes a document assembler and the T5 model configured for evaluating sentence grammar.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code('''
+import sparknlp 
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Initialize Spark NLP
+spark = sparknlp.start()
+
+# Define the pipeline stages
+documentAssembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("documents")
+    
+t5 = T5Transformer.pretrained('t5_base') \\
+    .setTask("cola:") \\
+    .setInputCols(["documents"])\\
+    .setMaxOutputLength(200)\\
+    .setOutputCol("prediction")
+
+pipeline = Pipeline().setStages([documentAssembler, t5])
+
+# Input data example
+data = spark.createDataFrame([["She don't knows nothing about what's happening in the office."]]).toDF("text")
+
+# Apply the pipeline for grammar evaluation
+result = pipeline.fit(data).transform(data)
+result.select("prediction.result").show(truncate=False)
+''', language='python')
+
+# Example Output
+st.text("""
++--------------------+
+|corrections.result  |
++--------------------+
+|unacceptable        |
++--------------------+
+""")
+
+# Model Info Section
+st.markdown('<div class="sub-title">Choosing the Right T5 Model for Grammar Evaluation</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>For evaluating sentence grammar, we use the model: "t5_grammar_error_corrector" with the task set to "cola:". This model is specifically tuned to assess grammatical correctness in English sentences.</p>
+    <p>Explore other T5 models tailored for different NLP tasks on the <a class="link" href="https://sparknlp.org/models?annotator=T5Transformer" target="_blank">Spark NLP Models Hub</a> to find the best fit for your specific needs.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References Section
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html" target="_blank">Google AI Blog</a>: Exploring Transfer Learning with T5</li>
+        <li><a class="link" href="https://sparknlp.org/models?annotator=T5Transformer" target="_blank">Spark NLP Model Hub</a>: Explore T5 models</li>
+        <li><a class="link" href="https://github.com/google-research/text-to-text-transfer-transformer" target="_blank">GitHub</a>: T5 Transformer repository</li>
+        <li><a class="link" href="https://arxiv.org/abs/1910.10683" target="_blank">T5 Paper</a>: Detailed insights from the developers</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Community & Support Section
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Quick Links Section
+st.markdown('<div class="sub-title">Quick Links</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/docs/en/quickstart" target="_blank">Getting Started</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/models" target="_blank">Pretrained Models</a></li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp/tree/master/examples/python/annotation/text/english" target="_blank">Example Notebooks</a></li>
+        <li><a class="link" href="https://sparknlp.org/docs/en/install" target="_blank">Installation Guide</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,7 @@
+streamlit
+st-annotated-text
+streamlit-tags
+pandas
+numpy
+spark-nlp
+pyspark