add event-identifier

app.py

@@ -0,0 +1,177 @@
+import streamlit as st
+from annotated_text import annotated_text
+
+import torch
+from transformers import pipeline
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+
+import json
+
+
+st.set_page_config(layout="wide")
+
+
+
+model = AutoModelForTokenClassification.from_pretrained("models/lusa")
+tokenizer = AutoTokenizer.from_pretrained("models/lusa", model_max_length=512)
+tagger = pipeline("ner", model=model, tokenizer=tokenizer, aggregation_strategy='first') #aggregation_strategy='max'
+  
+
+
+def aggregate_subwords(input_tokens, labels):
+    new_inputs = []
+    new_labels = []
+    current_word = ""
+    current_label = ""
+    for i, token in enumerate(input_tokens):     
+        label = labels[i]
+        # Handle subwords
+        if token.startswith('##'):
+            current_word += token[2:]
+        else:
+            # Finish previous word
+            if current_word:
+                new_inputs.append(current_word)
+                new_labels.append(current_label)
+            # Start new word
+            current_word = token
+            current_label = label
+    new_inputs.append(current_word)
+    new_labels.append(current_label)
+    return new_inputs, new_labels
+
+def annotateTriggers(line):
+    line = line.strip()
+    inputs = tokenizer(line, return_tensors="pt")
+    input_tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])
+
+    with torch.no_grad():
+        logits = model(**inputs).logits
+
+    predictions = torch.argmax(logits, dim=2)
+    predicted_token_class = [model.config.id2label[t.item()] for t in predictions[0]]
+    input_tokens, predicted_token_class =  aggregate_subwords(input_tokens,predicted_token_class)
+    token_labels = []
+    current_entity = ''
+    for i, label in enumerate(predicted_token_class):
+        token = input_tokens[i]
+        if label == 'O':
+            token_labels.append((token, 'O', ''))
+            current_entity = ''
+        elif label.startswith('B-'):
+            current_entity = label[2:]
+            token_labels.append((token, 'B', current_entity))
+        elif label.startswith('I-'):
+            if current_entity == '':
+                raise ValueError(f"Invalid label sequence: {predicted_token_class}")
+            token_labels[-1] = (token_labels[-1][0] + f" {token}", 'I', current_entity)
+        else:
+            raise ValueError(f"Invalid label: {label}")
+    return token_labels[1:-1]
+
+
+
+
+
+def joinEntities(entities):
+
+    joined_entities = []
+    i = 0
+    while i < len(entities):
+        curr_entity = entities[i]
+        if curr_entity['entity'][0] == 'B':
+            label = curr_entity['entity'][2:]
+            j = i + 1
+            while j < len(entities) and entities[j]['entity'][0] == 'I':
+                j += 1
+            joined_entity = {
+                 'entity': label,
+                'score': max(e['score'] for e in entities[i:j]),
+                'index': min(e['index'] for e in entities[i:j]),
+                'word': ' '.join(e['word'] for e in entities[i:j]),
+                'start': entities[i]['start'],
+                'end': entities[j-1]['end']
+            }
+            joined_entities.append(joined_entity)
+            i = j - 1
+        i += 1
+    return joined_entities
+
+
+
+import pysbd
+seg = pysbd.Segmenter(language="es", clean=False)
+
+def sent_tokenize(text):
+    return seg.segment(text)
+
+def getSentenceIndex(lines,span):
+    i = 1
+    sum = len(lines[0])
+    while sum < span:
+        sum += len(lines[i])
+        i = i + 1
+    return i - 1
+
+def generateContext(text, window,span):
+    lines = sent_tokenize(text)
+    index = getSentenceIndex(lines,span)
+    text = " ".join(lines[max(0,index-window):index+window +1])
+    return text
+
+
+def annotateEvents(text,squad,window):
+    text = text.strip()
+    ner_results = tagger(text)
+    #print(ner_results)
+    #ner_results = joinEntities(ner_results)
+    i = 0
+    #exit()
+    while i < len(ner_results):
+        ner_results[i]["entity"] = ner_results[i]["entity_group"].lstrip("B-")
+        ner_results[i]["entity"] = ner_results[i]["entity_group"].lstrip("I-")
+        i = i + 1
+
+    events = [] 
+    for trigger in ner_results:
+        tipo = trigger["entity_group"]
+        context = generateContext(text,window,trigger["start"])
+        event = {
+            "trigger":trigger["word"],
+            "type": tipo,
+            "score": trigger["score"],
+            "context": context,
+        }
+        events.append(event)
+    return events
+
+
+#"A Joana foi atacada pelo João nas ruas do Porto, com uma faca."
+
+st.title('Extract Events')
+
+options = ["O presidente da Federação Haitiana de Futebol, Yves Jean-Bart, foi banido para sempre de toda a atividade ligada ao futebol, por ter sido considerado culpado de abuso sexual sistemático de jogadoras, anunciou hoje a FIFA."]
+
+option = st.selectbox(
+        'Select examples',
+        options)
+#option = options [index]
+line = st.text_area("Insert Text",option)
+
+st.button('Run')
+
+
+st.sidebar.write("## Hyperparameters :gear:")
+window = 1
+if line != "":
+    st.header("Triggers:")
+    triggerss = annotateTriggers(line)   
+    annotated_text(*[word[0]+" " if word[1] == 'O' else (word[0]+" ",word[2]) for word in triggerss ])
+
+    eventos_1 = annotateEvents(line,1,window)
+    eventos_2 = annotateEvents(line,2,window)
+
+    for mention1, mention2 in zip(eventos_1,eventos_2):
+        st.text(f"| Trigger: {mention1['trigger']:20} | Type: {mention1['type']:10} | Score: {str(round(mention1['score'],3)):5} |") 
+        st.markdown("""---""")
+    

models/lusa/config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "neuralmind/bert-base-portuguese-cased",
+  "architectures": [
+    "BertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "I-Event",
+    "1": "B-Event",
+    "2": "O"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "B-Event": 1,
+    "I-Event": 0,
+    "O": 2
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.18.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 29794
+}

models/lusa/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc02b2488c39a686b32d518c2dbf42b5ccc349371d7b238c6e9a1d67053a6fd6
+size 433412010

models/lusa/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

models/lusa/tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "special_tokens_map_file": "/home/dock/.cache/huggingface/transformers/eecc45187d085a1169eed91017d358cc0e9cbdd5dc236bcd710059dbf0a2f816.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d", "name_or_path": "neuralmind/bert-base-portuguese-cased", "do_basic_tokenize": true, "never_split": null, "tokenizer_class": "BertTokenizer"}

models/lusa/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72ae981987da18cf3cbbd058377b693d1e5b1d57242a27551b56f605e5bb3d11
+size 3512

requirements.txt

@@ -0,0 +1,8 @@
+st-annotated-text==3.0.0
+torch
+transformers==4.21.0
+tqdm==4.64.0
+numpy==1.22.3
+pysbd==0.3.4
+altair==4.2.2
+streamlit==1.19.0