add first commit

README.md

@@ -1,12 +1,45 @@
 ---
-title: Newspaper
-emoji: 🏃
-colorFrom: yellow
-colorTo: indigo
+title: News Generator
+emoji: 🦀
+colorFrom: pink
+colorTo: red
 sdk: streamlit
-sdk_version: 1.10.0
-app_file: app.py
+app_file: app/app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Indonesian GPT-2 Applications
+This is Application that generates sentences using Indonesian GPT-2 models finetuned on 6GB online news dataset!
+
+
+## How did we create it
+
+## Development
+
+### Dependencies Installation
+
+### Inference Pipeline
+
+## Authors
+
+Following are the authors of this work (listed alphabetically):
+- [Cahya Wirawan](https://github.com/cahya-wirawan)
+
+## Acknowledgements
+
+- 🤗 Hugging Face for organizing [the FLAX/JAX community week](https://github.com/huggingface/transformers/tree/master/examples/research_projects/jax-projects)
+- Google [TPU Research Cloud (TRC) program](https://sites.research.google/trc/) for providing computing resources
+- [Weights & Biases](https://wandb.com/) for providing the infrastructure for experiment tracking and model management
+
+## Citing Indonesian GPT-2 Applications
+
+If you find this is useful in your research or wish to refer, please use the following BibTeX entry.
+
+```
+@misc{Indonesian_GPT2_App_2021,
+author = {Cahya Wirawan},
+title = {Abstract Generator using Indonesian GPT-2},
+url = {https://github.com/cahya-wirawan/abstract-generator},
+year = {2021}
+}
+```

app/SessionState.py

@@ -0,0 +1,95 @@
+"""Hack to add per-session state to Streamlit.
+Usage
+-----
+>>> import SessionState
+>>>
+>>> session_state = SessionState.get(user_name='', favorite_color='black')
+>>> session_state.user_name
+''
+>>> session_state.user_name = 'Mary'
+>>> session_state.favorite_color
+'black'
+Since you set user_name above, next time your script runs this will be the
+result:
+>>> session_state = get(user_name='', favorite_color='black')
+>>> session_state.user_name
+'Mary'
+"""
+from streamlit.scriptrunner import get_script_run_ctx
+from streamlit.server.server import Server
+
+
+class SessionState(object):
+    def __init__(self, **kwargs):
+        """A new SessionState object.
+        Parameters
+        ----------
+        **kwargs : any
+            Default values for the session state.
+        Example
+        -------
+        >>> session_state = SessionState(user_name='', favorite_color='black')
+        >>> session_state.user_name = 'Mary'
+        ''
+        >>> session_state.favorite_color
+        'black'
+        """
+        for key, val in kwargs.items():
+            setattr(self, key, val)
+
+
+def get(**kwargs):
+    """Gets a SessionState object for the current session.
+    Creates a new object if necessary.
+    Parameters
+    ----------
+    **kwargs : any
+        Default values you want to add to the session state, if we're creating a
+        new one.
+    Example
+    -------
+    >>> session_state = get(user_name='', favorite_color='black')
+    >>> session_state.user_name
+    ''
+    >>> session_state.user_name = 'Mary'
+    >>> session_state.favorite_color
+    'black'
+    Since you set user_name above, next time your script runs this will be the
+    result:
+    >>> session_state = get(user_name='', favorite_color='black')
+    >>> session_state.user_name
+    'Mary'
+    """
+    # Hack to get the session object from Streamlit.
+
+    ctx = get_script_run_ctx()
+
+    this_session = None
+
+    current_server = Server.get_current()
+    if hasattr(current_server, '_session_infos'):
+        # Streamlit < 0.56
+        session_infos = Server.get_current()._session_infos.values()
+    else:
+        session_infos = Server.get_current()._session_info_by_id.values()
+
+    for session_info in session_infos:
+        s = session_info.session
+        if (
+            (not hasattr(s, '_main_dg') and s._uploaded_file_mgr == ctx.uploaded_file_mgr)
+        ):
+            this_session = s
+
+    if this_session is None:
+        raise RuntimeError(
+            "Oh noes. Couldn't get your Streamlit Session object. "
+            'Are you doing something fancy with threads?')
+
+    # Got the session object! Now let's attach some state into it.
+
+    if not hasattr(this_session, '_custom_session_state'):
+        this_session._custom_session_state = SessionState(**kwargs)
+
+    return this_session._custom_session_state
+
+__all__ = ['get']

app/abstract_dataset.py

@@ -0,0 +1,62 @@
+import torch
+import random
+from torch.utils.data import Dataset
+
+
+class AbstractDataset(Dataset):
+    special_tokens = {"bos_token": "<|BOS|>",
+                      "eos_token": "<|EOS|>",
+                      "unk_token": "<|UNK|>",
+                      "pad_token": "<|PAD|>",
+                      "sep_token": "<|SEP|>"}
+    max_length = 1024
+
+    def __init__(self, data, tokenizer, randomize=True):
+        title, text, keywords = [], [], []
+        for k, v in data.items():
+            title.append(v[0])
+            text.append(v[1])
+            keywords.append(v[2])
+
+        self.randomize = randomize
+        self.tokenizer = tokenizer
+        self.title = title
+        self.text = text
+        self.keywords = keywords
+
+    @staticmethod
+    def join_keywords(keywords, randomize=True):
+        N = len(keywords)
+
+        # random sampling and shuffle
+        if randomize:
+            # M = random.choice(range(N + 1))
+            # keywords = keywords[:M]
+            random.shuffle(keywords)
+
+        return ','.join(keywords)
+
+
+    def __len__(self):
+        return len(self.text)
+
+
+    def __getitem__(self, i):
+        keywords = self.keywords[i].copy()
+        kw = self.join_keywords(keywords, self.randomize)
+
+        input = self.special_tokens['bos_token'] + self.title[i] + \
+                self.special_tokens['sep_token'] + kw + self.special_tokens['sep_token'] + \
+                self.text[i] + self.special_tokens['eos_token']
+
+        encodings_dict = self.tokenizer(input,
+                                   truncation=True,
+                                   max_length=self.max_length,
+                                   padding="max_length")
+
+        input_ids = encodings_dict['input_ids']
+        attention_mask = encodings_dict['attention_mask']
+
+        return {'label': torch.tensor(input_ids),
+                'input_ids': torch.tensor(input_ids),
+                'attention_mask': torch.tensor(attention_mask)}

app/app.py

@@ -0,0 +1,244 @@
+import streamlit as st
+import SessionState
+from mtranslate import translate
+from prompts import PROMPT_LIST
+import random
+import time
+from transformers import pipeline, set_seed, AutoConfig, AutoTokenizer, GPT2LMHeadModel, GPT2Tokenizer
+import psutil
+import torch
+import os
+from abstract_dataset import AbstractDataset
+
+
+# st.set_page_config(page_title="Indonesian GPT-2")
+
+mirror_url = "https://abstract-generator.ai-research.id/"
+if "MIRROR_URL" in os.environ:
+    mirror_url = os.environ["MIRROR_URL"]
+
+MODELS = {
+    "Indonesian Academic Journal - Indonesian GPT-2 Medium": {
+        "group": "Indonesian Journal",
+        "name": "cahya/abstract-generator",
+        "description": "Abstract Generator using Indonesian GPT-2 Medium.",
+        "text_generator": None,
+        "tokenizer": None
+    },
+}
+
+st.sidebar.markdown("""
+<style>
+.centeralign {
+    text-align: center;
+}
+</style>
+<p class="centeralign">
+    <img src="https://huggingface.co/spaces/flax-community/gpt2-indonesian/resolve/main/huggingwayang.png"/>
+</p>
+""", unsafe_allow_html=True)
+st.sidebar.markdown(f"""
+___
+<p class="centeralign">
+This is a collection of applications that generates sentences using Indonesian GPT-2 models!
+</p>
+<p class="centeralign">
+Created by <a href="https://huggingface.co/indonesian-nlp">Indonesian NLP</a> team @2021
+<br/>
+<a href="https://github.com/indonesian-nlp/gpt2-app" target="_blank">GitHub</a> | <a href="https://github.com/indonesian-nlp/gpt2-app" target="_blank">Project Report</a>
+<br/>
+A mirror of the application is available <a href="{mirror_url}" target="_blank">here</a>
+</p>
+""", unsafe_allow_html=True)
+
+st.sidebar.markdown("""
+___
+        """, unsafe_allow_html=True)
+
+model_type = st.sidebar.selectbox('Model', (MODELS.keys()))
+
+
+@st.cache(suppress_st_warning=True, allow_output_mutation=True)
+def get_generator(model_name: str):
+    st.write(f"Loading the GPT2 model {model_name}, please wait...")
+    special_tokens = AbstractDataset.special_tokens
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    tokenizer.add_special_tokens(special_tokens)
+    config = AutoConfig.from_pretrained(model_name,
+                                        bos_token_id=tokenizer.bos_token_id,
+                                        eos_token_id=tokenizer.eos_token_id,
+                                        sep_token_id=tokenizer.sep_token_id,
+                                        pad_token_id=tokenizer.pad_token_id,
+                                        output_hidden_states=False)
+    model = GPT2LMHeadModel.from_pretrained(model_name, config=config)
+    model.resize_token_embeddings(len(tokenizer))
+    return model, tokenizer
+
+
+# Disable the st.cache for this function due to issue on newer version of streamlit
+# @st.cache(suppress_st_warning=True, hash_funcs={tokenizers.Tokenizer: id})
+def process(text_generator, tokenizer, title: str, keywords: str, text: str,
+            max_length: int = 200, do_sample: bool = True, top_k: int = 50, top_p: float = 0.95,
+            temperature: float = 1.0, max_time: float = 120.0, seed=42, repetition_penalty=1.0):
+    # st.write("Cache miss: process")
+    set_seed(seed)
+    if repetition_penalty == 0.0:
+        min_penalty = 1.05
+        max_penalty = 1.5
+        repetition_penalty = max(min_penalty + (1.0-temperature) * (max_penalty-min_penalty), 0.8)
+
+    keywords = [keyword.strip() for keyword in keywords.split(",")]
+    keywords = AbstractDataset.join_keywords(keywords, randomize=False)
+
+    special_tokens = AbstractDataset.special_tokens
+    prompt = special_tokens['bos_token'] + title + \
+             special_tokens['sep_token'] + keywords + special_tokens['sep_token'] + text
+
+    print(f"title: {title}, keywords: {keywords}, text: {text}")
+
+    generated = torch.tensor(tokenizer.encode(prompt)).unsqueeze(0)
+    # device = torch.device("cuda")
+    # generated = generated.to(device)
+
+    text_generator.eval()
+    sample_outputs = text_generator.generate(generated,
+                                    do_sample=do_sample,
+                                    min_length=200,
+                                    max_length=max_length,
+                                    top_k=top_k,
+                                    top_p=top_p,
+                                    temperature=temperature,
+                                    repetition_penalty=repetition_penalty,
+                                    num_return_sequences=1
+                                    )
+    result = tokenizer.decode(sample_outputs[0], skip_special_tokens=True)
+    print(f"result: {result}")
+    prefix_length = len(title) + len(keywords)
+    result = result[prefix_length:]
+    return result
+
+
+st.title("Indonesian GPT-2 Applications")
+prompt_group_name = MODELS[model_type]["group"]
+st.header(prompt_group_name)
+description = f"This is a bilingual (Indonesian and English) abstract generator using Indonesian GPT-2 Medium. We finetuned it with the Indonesian paper abstract dataset."
+st.markdown(description)
+model_name = f"Model name: [{MODELS[model_type]['name']}](https://huggingface.co/{MODELS[model_type]['name']})"
+st.markdown(model_name)
+if prompt_group_name in ["Indonesian GPT-2", "Indonesian Literature", "Indonesian Journal"]:
+    session_state = SessionState.get(prompt=None, prompt_box=None, text=None)
+    ALL_PROMPTS = list(PROMPT_LIST[prompt_group_name].keys())+["Custom"]
+
+    prompt = st.selectbox('Prompt', ALL_PROMPTS, index=len(ALL_PROMPTS)-1)
+
+    # Update prompt
+    if session_state.prompt is None:
+        session_state.prompt = prompt
+    elif session_state.prompt is not None and (prompt != session_state.prompt):
+        session_state.prompt = prompt
+        session_state.prompt_box = None
+    else:
+        session_state.prompt = prompt
+
+    # Update prompt box
+    if session_state.prompt == "Custom":
+        session_state.prompt_box = ""
+        session_state.title = ""
+        session_state.keywords = ""
+    else:
+        if session_state.prompt is not None and session_state.prompt_box is None:
+            session_state.prompt_box = random.choice(PROMPT_LIST[prompt_group_name][session_state.prompt])
+
+    session_state.title = st.text_input("Title", session_state.title)
+    session_state.keywords = st.text_input("Keywords", session_state.keywords)
+    session_state.text = st.text_area("Prompt", session_state.prompt_box)
+
+    max_length = st.sidebar.number_input(
+        "Maximum length",
+        value=200,
+        max_value=512,
+        help="The maximum length of the sequence to be generated."
+    )
+
+    temperature = st.sidebar.slider(
+        "Temperature",
+        value=0.4,
+        min_value=0.0,
+        max_value=2.0
+    )
+
+    do_sample = st.sidebar.checkbox(
+        "Use sampling",
+        value=True
+    )
+
+    top_k = 30
+    top_p = 0.95
+
+    if do_sample:
+        top_k = st.sidebar.number_input(
+            "Top k",
+            value=top_k,
+            help="The number of highest probability vocabulary tokens to keep for top-k-filtering."
+        )
+        top_p = st.sidebar.number_input(
+            "Top p",
+            value=top_p,
+            help="If set to float < 1, only the most probable tokens with probabilities that add up to top_p or higher "
+                 "are kept for generation."
+        )
+
+    seed = st.sidebar.number_input(
+        "Random Seed",
+        value=25,
+        help="The number used to initialize a pseudorandom number generator"
+    )
+
+    repetition_penalty = 0.0
+    automatic_repetition_penalty = st.sidebar.checkbox(
+        "Automatic Repetition Penalty",
+        value=True
+    )
+
+    if not automatic_repetition_penalty:
+        repetition_penalty = st.sidebar.slider(
+            "Repetition Penalty",
+            value=1.0,
+            min_value=1.0,
+            max_value=2.0
+        )
+
+    for group_name in MODELS:
+        if MODELS[group_name]["group"] in ["Indonesian GPT-2", "Indonesian Literature", "Indonesian Journal"]:
+            MODELS[group_name]["text_generator"], MODELS[group_name]["tokenizer"] = \
+                get_generator(MODELS[group_name]["name"])
+
+    if st.button("Run"):
+        with st.spinner(text="Getting results..."):
+            memory = psutil.virtual_memory()
+            st.subheader("Result")
+            time_start = time.time()
+            # text_generator = MODELS[model_type]["text_generator"]
+            result = process(MODELS[model_type]["text_generator"], MODELS[model_type]["tokenizer"],
+                             title=session_state.title,
+                             keywords=session_state.keywords,
+                             text=session_state.text, max_length=int(max_length),
+                             temperature=temperature, do_sample=do_sample,
+                             top_k=int(top_k), top_p=float(top_p), seed=seed, repetition_penalty=repetition_penalty)
+            time_end = time.time()
+            time_diff = time_end-time_start
+            #result = result[0]["generated_text"]
+            st.write(result.replace("\n", "  \n"))
+            st.text("Translation")
+            translation = translate(result, "en", "id")
+            st.write(translation.replace("\n", "  \n"))
+            # st.write(f"*do_sample: {do_sample}, top_k: {top_k}, top_p: {top_p}, seed: {seed}*")
+            info = f"""
+            *Memory: {memory.total/(1024*1024*1024):.2f}GB, used: {memory.percent}%, available: {memory.available/(1024*1024*1024):.2f}GB*        
+            *Text generated in {time_diff:.5} seconds*
+            """
+            st.write(info)
+
+            # Reset state
+            session_state.prompt = None
+            session_state.prompt_box = None

app/prompts.py

@@ -0,0 +1,57 @@
+PROMPT_LIST = {
+    "Indonesian GPT-2": {
+        "Resep masakan (recipe)": [
+            "Berikut adalah cara memasak sate ayam:\n",
+            "Langkah-langkah membuat nasi goreng:\n",
+            "Berikut adalah bahan-bahan membuat nastar:\n"
+        ],
+        "Puisi (poetry)": [
+            "Aku ingin jadi merpati\nTerbang di langit yang damai\nBernyanyi-nyanyi tentang masa depan\n",
+            "Terdiam aku satu persatu dengan tatapan binar\nSenyawa merasuk dalam sukma membuat lara\nKefanaan membentuk kelemahan"
+        ],
+        "Cerpen (short story)": [
+            "Putri memakai sepatunya dengan malas. Kalau bisa, selama seminggu ini ia bolos sekolah saja. Namun, Mama pasti akan marah. Ulangan tengah semester telah selesai. Minggu ini, di sekolah sedang berlangsung pekan olahraga.",
+            "\"Wah, hari ini cerah sekali ya,\" ucap Budi ketika ia keluar rumah.",
+            "Sewindu sudah kita tak berjumpa, rinduku padamu sudah tak terkira."
+        ],
+        "Sejarah (history)": [
+            "Mohammad Natsir adalah seorang ulama, politisi, dan pejuang kemerdekaan Indonesia.",
+            "Ir. H. Soekarno adalah Presiden pertama Republik Indonesia. Ia adalah seorang tokoh perjuangan yang memainkan peranan penting dalam memerdekakan bangsa Indonesia",
+            "Borobudur adalah sebuah candi Buddha yang terletak di sebelah barat laut Yogyakarta. Monumen ini merupakan model alam semesta dan dibangun sebagai tempat suci untuk memuliakan Buddha"
+        ],
+    },
+    "Indonesian Literature": {
+        "Adult Romance": [
+            "Ini adalah kisah tentang seorang laki-laki yang berusaha memperjuangkan cintanya",
+            "Alunan musik terdengar memenuhi ruangan kantor, cowok itu duduk di balik meja kerjanya sambil memejamkan mata. Berusaha meresapi nada per nada",
+            "Aku mencari dan terus mencari\nDimana bahagia akan kutemui\nKumencari terus mencari\nHingga ku tak mengerti arti hari-hari",
+            "Gadis itu mengharuskan dirinya tegar, dan kuat dalam menghadapi masalah. Menahan air matanya jatuh setiap kali ingin menangis"
+        ],
+        "Horror": [
+            "Ditengah-tengah perbincangan mereka berdua, datanglah sesosok mahluk tinggi hitam dan besar",
+            "Sesosok hantu perempuan seperti kuntilanak yang melayang keluar dan bergerak perlahan dari pintu kamar kecil tadi yang tertutup.",
+            "Sejak pertemuannya dengan leak, yang ternyata tinggal satu atap dengannya, hidupnya terus dihantui oleh berbagai sosok seram."
+        ],
+        "Poetry": [
+            "Aku ingin menulis sajak\nyang melesat dalam kejap\nmenembus hati yang pejam\nmemaksa mimpimu terjaga\ndari semu",
+            "Malam ini langitku lengang\ntiada hujan yang membasuh rindu\npun awan yang biasanya temani seruput kopimu",
+            "Di sisimu waktu menjelma\nsetangkai kembang api\ngelora membakar tanpa jeda\nmemercik pijar binar kita."
+        ]
+    },
+    "Indonesian Journal": {
+        "Biologi (biology)": [
+            "Tujuan  penelitian ini untuk menentukan keanekaragaman Arthropoda pada lahan pertanian kacang",
+            "Identifikasi spesies secara molekuler sangat diperlukan dalam mempelajari taksonomi",
+            "Penelitian ini bertujuan untuk menentukan identitas invertebrata laut dari Perairan Papua dengan teknik DNA barcoding"],
+        "Psikologi (psychology)": [
+            "Penelitian ini bertujuan untuk mengetahui perilaku wirausaha remaja yang diprediksi dari motivasi intrinsik",
+            "Tujuan dari penelitian ini adalah untuk mendapatkan data empiris mengenai gambaran peta bakat mahasiswa Fakultas Psikologi Unjani"],
+        "Ekonomi (economics)": [
+            "Faktor kepuasan dan kepercayaan konsumen merupakan dua faktor kunci dalam meningkatkan penetrasi e-commerce. Penelitian yang dilakukan",
+            "Penelitian ini bertujuan untuk menganalisis pola konsumsi pangan di Indonesia",
+            "Model GTAP diimplementasikan untuk melihat dampak yang ditimbulkan pada PDB"],
+        "Teknologi Informasi (IT)": [
+            "pembuatan aplikasi ini menggunakan pengembangan metode Waterfall dan dirancang mengguynakan Unified Modeling Language (UML) dengan bahasa pemrograman",
+            "Berdasarkan masalah tersebut, maka penulis termotivasi untuk membangun Pengembangan Sistem Informasi Manajemen"]
+    },
+}

requirements.txt

@@ -0,0 +1,10 @@
+numpy
+torch
+tokenizers
+transformers
+datasets
+mtranslate
+# streamlit version 0.67.1 is needed due to issue with caching
+# streamlit==0.67.1
+streamlit
+psutil