96abhishekarora
commited on
Commit
•
6aa6cec
1
Parent(s):
f403d17
Updated model with better training and evaluation. Test and val data included as pickle files. Older Legacy files were removed to avoid confusion.
Browse files- .gitattributes +4 -35
- 1_Pooling/config.json +7 -0
- LT_training_config.json +29 -0
- README.md +146 -0
- added_tokens.json +4 -0
- binary_classification_evaluation_eval_results.csv +3 -0
- binary_classification_evaluation_test_results.csv +3 -0
- config.json +32 -0
- config_sentence_transformers.json +7 -0
- entity_vocab.json +6 -0
- eval/binary_classification_evaluation_eval_results.csv +0 -0
- model.safetensors +3 -0
- modules.json +14 -0
- sentence_bert_config.json +4 -0
- sentencepiece.bpe.model +3 -0
- special_tokens_map.json +73 -0
- test_data.pickle +3 -0
- test_results_allmethods.csv +2 -0
- tokenizer_config.json +106 -0
- val_data.pickle +3 -0
.gitattributes
CHANGED
@@ -1,35 +1,4 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
|
4 |
-
|
5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
1 |
+
model.safetensors filter=lfs diff=lfs merge=lfs -text
|
2 |
+
sentencepiece.bpe.model filter=lfs diff=lfs merge=lfs -text
|
3 |
+
test_data.pickle filter=lfs diff=lfs merge=lfs -text
|
4 |
+
val_data.pickle filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1_Pooling/config.json
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"word_embedding_dimension": 768,
|
3 |
+
"pooling_mode_cls_token": false,
|
4 |
+
"pooling_mode_mean_tokens": true,
|
5 |
+
"pooling_mode_max_tokens": false,
|
6 |
+
"pooling_mode_mean_sqrt_len_tokens": false
|
7 |
+
}
|
LT_training_config.json
ADDED
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"model_save_dir": "models",
|
3 |
+
"model_save_name": "lt-historicjapanesecompanies-comp-prod-ind_supcon_full",
|
4 |
+
"opt_model_description": "This model was trained on a dataset of historic Japanese companies, products, industry, addresses, and shareholders. Take a look at our paper for more details. The task is to link indices of japanese companies",
|
5 |
+
"opt_model_lang": "ja",
|
6 |
+
"train_batch_size": 64,
|
7 |
+
"num_epochs": 50,
|
8 |
+
"warm_up_perc": 1,
|
9 |
+
"learning_rate": 2e-05,
|
10 |
+
"loss_type": "supcon",
|
11 |
+
"val_perc": 0.2,
|
12 |
+
"wandb_names": {
|
13 |
+
"id": "econabhishek",
|
14 |
+
"run": "lt-historicjapanesecompanies-comp-prod-ind_supcon_full",
|
15 |
+
"project": "linkage",
|
16 |
+
"entity": "econabhishek"
|
17 |
+
},
|
18 |
+
"add_pooling_layer": false,
|
19 |
+
"large_val": true,
|
20 |
+
"eval_steps_perc": 0.5,
|
21 |
+
"test_at_end": true,
|
22 |
+
"save_val_test_pickles": true,
|
23 |
+
"val_query_prop": 0.5,
|
24 |
+
"loss_params": {},
|
25 |
+
"eval_type": "classification",
|
26 |
+
"training_dataset": "dataframe",
|
27 |
+
"base_model_path": "oshizo/sbert-jsnli-luke-japanese-base-lite",
|
28 |
+
"best_model_path": "models/lt-historicjapanesecompanies-comp-prod-ind_supcon_full"
|
29 |
+
}
|
README.md
ADDED
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
pipeline_tag: sentence-similarity
|
3 |
+
language:
|
4 |
+
- ja
|
5 |
+
tags:
|
6 |
+
- linktransformer
|
7 |
+
- sentence-transformers
|
8 |
+
- sentence-similarity
|
9 |
+
- tabular-classification
|
10 |
+
|
11 |
+
---
|
12 |
+
|
13 |
+
# {MODEL_NAME}
|
14 |
+
|
15 |
+
This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class.
|
16 |
+
It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.
|
17 |
+
Notwithstanding that, it can be used for any sentence similarity task within the sentence-transformers framework as well.
|
18 |
+
It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
|
19 |
+
Take a look at the documentation of [sentence-transformers](https://www.sbert.net/index.html) if you want to use this model for more than what we support in our applications.
|
20 |
+
|
21 |
+
|
22 |
+
This model has been fine-tuned on the model : oshizo/sbert-jsnli-luke-japanese-base-lite. It is pretrained for the language : - ja.
|
23 |
+
|
24 |
+
|
25 |
+
This model was trained on a dataset of historic Japanese companies, products, industry, addresses, and shareholders. Take a look at our paper for more details. The task is to link indices of japanese companies
|
26 |
+
|
27 |
+
## Usage (LinkTransformer)
|
28 |
+
|
29 |
+
Using this model becomes easy when you have [LinkTransformer](https://github.com/dell-research-harvard/linktransformer) installed:
|
30 |
+
|
31 |
+
```
|
32 |
+
pip install -U linktransformer
|
33 |
+
```
|
34 |
+
|
35 |
+
Then you can use the model like this:
|
36 |
+
|
37 |
+
```python
|
38 |
+
import linktransformer as lt
|
39 |
+
import pandas as pd
|
40 |
+
|
41 |
+
##Load the two dataframes that you want to link. For example, 2 dataframes with company names that are written differently
|
42 |
+
df1=pd.read_csv("data/df1.csv") ###This is the left dataframe with key CompanyName for instance
|
43 |
+
df2=pd.read_csv("data/df2.csv") ###This is the right dataframe with key CompanyName for instance
|
44 |
+
|
45 |
+
###Merge the two dataframes on the key column!
|
46 |
+
df_merged = lt.merge(df1, df2, on="CompanyName", how="inner")
|
47 |
+
|
48 |
+
##Done! The merged dataframe has a column called "score" that contains the similarity score between the two company names
|
49 |
+
|
50 |
+
```
|
51 |
+
|
52 |
+
|
53 |
+
## Training your own LinkTransformer model
|
54 |
+
Any Sentence Transformers can be used as a backbone by simply adding a pooling layer. Any other transformer on HuggingFace can also be used by specifying the option add_pooling_layer==True
|
55 |
+
The model was trained using SupCon loss.
|
56 |
+
Usage can be found in the package docs.
|
57 |
+
The training config can be found in the repo with the name LT_training_config.json
|
58 |
+
To replicate the training, you can download the file and specify the path in the config_path argument of the training function. You can also override the config by specifying the training_args argument.
|
59 |
+
Here is an example.
|
60 |
+
|
61 |
+
|
62 |
+
```python
|
63 |
+
|
64 |
+
##Consider the example in the paper that has a dataset of Mexican products and their tariff codes from 1947 and 1948 and we want train a model to link the two tariff codes.
|
65 |
+
saved_model_path = train_model(
|
66 |
+
model_path="hiiamsid/sentence_similarity_spanish_es",
|
67 |
+
dataset_path=dataset_path,
|
68 |
+
left_col_names=["description47"],
|
69 |
+
right_col_names=['description48'],
|
70 |
+
left_id_name=['tariffcode47'],
|
71 |
+
right_id_name=['tariffcode48'],
|
72 |
+
log_wandb=False,
|
73 |
+
config_path=LINKAGE_CONFIG_PATH,
|
74 |
+
training_args={"num_epochs": 1}
|
75 |
+
)
|
76 |
+
|
77 |
+
```
|
78 |
+
|
79 |
+
|
80 |
+
You can also use this package for deduplication (clusters a df on the supplied key column). Merging a fine class (like product) to a coarse class (like HS code) is also possible.
|
81 |
+
Read our paper and the documentation for more!
|
82 |
+
|
83 |
+
|
84 |
+
|
85 |
+
## Evaluation Results
|
86 |
+
|
87 |
+
<!--- Describe how your model was evaluated -->
|
88 |
+
|
89 |
+
You can evaluate the model using the [LinkTransformer](https://github.com/dell-research-harvard/linktransformer) package's inference functions.
|
90 |
+
We have provided a few datasets in the package for you to try out. We plan to host more datasets on Huggingface and our website (Coming soon) that you can take a look at.
|
91 |
+
|
92 |
+
|
93 |
+
## Training
|
94 |
+
The model was trained with the parameters:
|
95 |
+
|
96 |
+
**DataLoader**:
|
97 |
+
|
98 |
+
`torch.utils.data.dataloader.DataLoader` of length 16 with parameters:
|
99 |
+
```
|
100 |
+
{'batch_size': 64, 'sampler': 'torch.utils.data.dataloader._InfiniteConstantSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
|
101 |
+
```
|
102 |
+
|
103 |
+
**Loss**:
|
104 |
+
|
105 |
+
`linktransformer.modified_sbert.losses.SupConLoss_wandb`
|
106 |
+
|
107 |
+
Parameters of the fit()-Method:
|
108 |
+
```
|
109 |
+
{
|
110 |
+
"epochs": 50,
|
111 |
+
"evaluation_steps": 8,
|
112 |
+
"evaluator": "sentence_transformers.evaluation.SequentialEvaluator.SequentialEvaluator",
|
113 |
+
"max_grad_norm": 1,
|
114 |
+
"optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
|
115 |
+
"optimizer_params": {
|
116 |
+
"lr": 2e-05
|
117 |
+
},
|
118 |
+
"scheduler": "WarmupLinear",
|
119 |
+
"steps_per_epoch": null,
|
120 |
+
"warmup_steps": 800,
|
121 |
+
"weight_decay": 0.01
|
122 |
+
}
|
123 |
+
```
|
124 |
+
|
125 |
+
|
126 |
+
|
127 |
+
|
128 |
+
LinkTransformer(
|
129 |
+
(0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: LukeModel
|
130 |
+
(1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
|
131 |
+
)
|
132 |
+
```
|
133 |
+
|
134 |
+
## Citing & Authors
|
135 |
+
|
136 |
+
```
|
137 |
+
@misc{arora2023linktransformer,
|
138 |
+
title={LinkTransformer: A Unified Package for Record Linkage with Transformer Language Models},
|
139 |
+
author={Abhishek Arora and Melissa Dell},
|
140 |
+
year={2023},
|
141 |
+
eprint={2309.00789},
|
142 |
+
archivePrefix={arXiv},
|
143 |
+
primaryClass={cs.CL}
|
144 |
+
}
|
145 |
+
|
146 |
+
```
|
added_tokens.json
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<ent2>": 32771,
|
3 |
+
"<ent>": 32770
|
4 |
+
}
|
binary_classification_evaluation_eval_results.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
|
2 |
+
0,0,0.9382022471910112,0.8532869815826416,0.7636363636363636,0.7,0.84,0.8262333869934082,0.8184099257358792,0.9382022471910112,84.1578369140625,0.7572815533980584,0.7358490566037735,0.78,89.29296112060547,0.8270661393011416,0.9438202247191011,4.08632755279541,0.7872340425531915,0.8409090909090909,0.74,4.08632755279541,0.8277502672701681,0.8848314606741573,44.88762664794922,0.5925925925925927,0.5517241379310345,0.64,43.54036331176758,0.5417359362559628
|
3 |
+
0,0,0.9382022471910112,0.8532869815826416,0.7636363636363636,0.7,0.84,0.8262333869934082,0.8184099257358792,0.9382022471910112,84.1578369140625,0.7572815533980584,0.7358490566037735,0.78,89.29296112060547,0.8270661393011416,0.9438202247191011,4.08632755279541,0.7872340425531915,0.8409090909090909,0.74,4.08632755279541,0.8277502672701681,0.8848314606741573,44.88762283325195,0.5925925925925927,0.5517241379310345,0.64,43.54036331176758,0.5417359362559628
|
binary_classification_evaluation_test_results.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
|
2 |
+
0,0,0.9578651685393258,0.7885568141937256,0.854368932038835,0.9166666666666666,0.8,0.7569339871406555,0.9027201028709122,0.9606741573033708,121.89360046386719,0.86,0.9555555555555556,0.7818181818181819,121.89360046386719,0.9092811528662349,0.9606741573033708,5.6210103034973145,0.8653846153846154,0.9183673469387755,0.8181818181818182,5.979313373565674,0.9083606695270187,0.9410112359550562,58.51844024658203,0.7789473684210527,0.925,0.6727272727272727,58.51844024658203,0.8613143757051128
|
3 |
+
0,0,0.9578651685393258,0.7794706225395203,0.845360824742268,0.9761904761904762,0.7454545454545455,0.7794706225395203,0.9070055457080543,0.9578651685393258,119.23606872558594,0.8545454545454545,0.8545454545454545,0.8545454545454545,140.44253540039062,0.9139460658879931,0.9578651685393258,5.471532344818115,0.8545454545454545,0.8545454545454545,0.8545454545454545,6.480866432189941,0.9147327683242549,0.9241573033707865,56.068626403808594,0.7518796992481204,0.6410256410256411,0.9090909090909091,46.432830810546875,0.8419716262659404
|
config.json
ADDED
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "/home/abhishekarora/.cache/torch/sentence_transformers/oshizo_sbert-jsnli-luke-japanese-base-lite/",
|
3 |
+
"architectures": [
|
4 |
+
"LukeModel"
|
5 |
+
],
|
6 |
+
"attention_probs_dropout_prob": 0.1,
|
7 |
+
"bert_model_name": "models/luke-japanese/hf_xlm_roberta",
|
8 |
+
"bos_token_id": 0,
|
9 |
+
"classifier_dropout": null,
|
10 |
+
"cls_entity_prediction": false,
|
11 |
+
"entity_emb_size": 256,
|
12 |
+
"entity_vocab_size": 4,
|
13 |
+
"eos_token_id": 2,
|
14 |
+
"hidden_act": "gelu",
|
15 |
+
"hidden_dropout_prob": 0.1,
|
16 |
+
"hidden_size": 768,
|
17 |
+
"initializer_range": 0.02,
|
18 |
+
"intermediate_size": 3072,
|
19 |
+
"layer_norm_eps": 1e-05,
|
20 |
+
"max_position_embeddings": 514,
|
21 |
+
"model_type": "luke",
|
22 |
+
"num_attention_heads": 12,
|
23 |
+
"num_hidden_layers": 12,
|
24 |
+
"pad_token_id": 1,
|
25 |
+
"position_embedding_type": "absolute",
|
26 |
+
"torch_dtype": "float32",
|
27 |
+
"transformers_version": "4.35.1",
|
28 |
+
"type_vocab_size": 1,
|
29 |
+
"use_cache": true,
|
30 |
+
"use_entity_aware_attention": true,
|
31 |
+
"vocab_size": 32772
|
32 |
+
}
|
config_sentence_transformers.json
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"__version__": {
|
3 |
+
"sentence_transformers": "2.2.2",
|
4 |
+
"transformers": "4.25.1",
|
5 |
+
"pytorch": "1.13.0+cu116"
|
6 |
+
}
|
7 |
+
}
|
entity_vocab.json
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"[MASK2]": 3,
|
3 |
+
"[MASK]": 0,
|
4 |
+
"[PAD]": 2,
|
5 |
+
"[UNK]": 1
|
6 |
+
}
|
eval/binary_classification_evaluation_eval_results.csv
ADDED
The diff for this file is too large to render.
See raw diff
|
|
model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0c43b11600beb431e99a5285efa15ec24aba97c430c0108c0b0a8075c5beb129
|
3 |
+
size 532299592
|
modules.json
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[
|
2 |
+
{
|
3 |
+
"idx": 0,
|
4 |
+
"name": "0",
|
5 |
+
"path": "",
|
6 |
+
"type": "sentence_transformers.models.Transformer"
|
7 |
+
},
|
8 |
+
{
|
9 |
+
"idx": 1,
|
10 |
+
"name": "1",
|
11 |
+
"path": "1_Pooling",
|
12 |
+
"type": "sentence_transformers.models.Pooling"
|
13 |
+
}
|
14 |
+
]
|
sentence_bert_config.json
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"max_seq_length": 128,
|
3 |
+
"do_lower_case": false
|
4 |
+
}
|
sentencepiece.bpe.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d8b73a5e054936c920cf5b7d1ec21ce9c281977078269963beb821c6c86fbff7
|
3 |
+
size 841889
|
special_tokens_map.json
ADDED
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
{
|
4 |
+
"content": "<ent>",
|
5 |
+
"lstrip": false,
|
6 |
+
"normalized": true,
|
7 |
+
"rstrip": false,
|
8 |
+
"single_word": false
|
9 |
+
},
|
10 |
+
{
|
11 |
+
"content": "<ent2>",
|
12 |
+
"lstrip": false,
|
13 |
+
"normalized": true,
|
14 |
+
"rstrip": false,
|
15 |
+
"single_word": false
|
16 |
+
},
|
17 |
+
{
|
18 |
+
"content": "<ent>",
|
19 |
+
"lstrip": false,
|
20 |
+
"normalized": true,
|
21 |
+
"rstrip": false,
|
22 |
+
"single_word": false
|
23 |
+
},
|
24 |
+
{
|
25 |
+
"content": "<ent2>",
|
26 |
+
"lstrip": false,
|
27 |
+
"normalized": true,
|
28 |
+
"rstrip": false,
|
29 |
+
"single_word": false
|
30 |
+
},
|
31 |
+
{
|
32 |
+
"content": "<ent>",
|
33 |
+
"lstrip": false,
|
34 |
+
"normalized": false,
|
35 |
+
"rstrip": false,
|
36 |
+
"single_word": false
|
37 |
+
},
|
38 |
+
{
|
39 |
+
"content": "<ent2>",
|
40 |
+
"lstrip": false,
|
41 |
+
"normalized": false,
|
42 |
+
"rstrip": false,
|
43 |
+
"single_word": false
|
44 |
+
},
|
45 |
+
{
|
46 |
+
"content": "<ent>",
|
47 |
+
"lstrip": false,
|
48 |
+
"normalized": true,
|
49 |
+
"rstrip": false,
|
50 |
+
"single_word": false
|
51 |
+
},
|
52 |
+
{
|
53 |
+
"content": "<ent2>",
|
54 |
+
"lstrip": false,
|
55 |
+
"normalized": true,
|
56 |
+
"rstrip": false,
|
57 |
+
"single_word": false
|
58 |
+
}
|
59 |
+
],
|
60 |
+
"bos_token": "<s>",
|
61 |
+
"cls_token": "<s>",
|
62 |
+
"eos_token": "</s>",
|
63 |
+
"mask_token": {
|
64 |
+
"content": "<mask>",
|
65 |
+
"lstrip": true,
|
66 |
+
"normalized": true,
|
67 |
+
"rstrip": false,
|
68 |
+
"single_word": false
|
69 |
+
},
|
70 |
+
"pad_token": "<pad>",
|
71 |
+
"sep_token": "</s>",
|
72 |
+
"unk_token": "<unk>"
|
73 |
+
}
|
test_data.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:71d2b470c96d3e8108f0a18ffe2707122bb0f30bf07bc11666de18b384714b0c
|
3 |
+
size 206402
|
test_results_allmethods.csv
ADDED
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
1 |
+
edit_distance,edit_distance_threshold,SBERT,SBERT_threshold,LT ZS Wiki,LT ZS Wiki_threshold,LT,LT_threshold,OpenAI,OpenAI_threshold
|
2 |
+
0.267639902676399,0.7790753674033062,0.6902654867256638,0.8263489127309215,0.723404255319149,0.6465095054004382,0.8367346938775511,0.7741220046283592,0.8627450980392157,0.9252961885425512
|
tokenizer_config.json
ADDED
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"added_tokens_decoder": {
|
3 |
+
"0": {
|
4 |
+
"content": "<s>",
|
5 |
+
"lstrip": false,
|
6 |
+
"normalized": false,
|
7 |
+
"rstrip": false,
|
8 |
+
"single_word": false,
|
9 |
+
"special": true
|
10 |
+
},
|
11 |
+
"1": {
|
12 |
+
"content": "<pad>",
|
13 |
+
"lstrip": false,
|
14 |
+
"normalized": false,
|
15 |
+
"rstrip": false,
|
16 |
+
"single_word": false,
|
17 |
+
"special": true
|
18 |
+
},
|
19 |
+
"2": {
|
20 |
+
"content": "</s>",
|
21 |
+
"lstrip": false,
|
22 |
+
"normalized": false,
|
23 |
+
"rstrip": false,
|
24 |
+
"single_word": false,
|
25 |
+
"special": true
|
26 |
+
},
|
27 |
+
"3": {
|
28 |
+
"content": "<unk>",
|
29 |
+
"lstrip": false,
|
30 |
+
"normalized": false,
|
31 |
+
"rstrip": false,
|
32 |
+
"single_word": false,
|
33 |
+
"special": true
|
34 |
+
},
|
35 |
+
"32769": {
|
36 |
+
"content": "<mask>",
|
37 |
+
"lstrip": true,
|
38 |
+
"normalized": true,
|
39 |
+
"rstrip": false,
|
40 |
+
"single_word": false,
|
41 |
+
"special": true
|
42 |
+
},
|
43 |
+
"32770": {
|
44 |
+
"content": "<ent>",
|
45 |
+
"lstrip": false,
|
46 |
+
"normalized": true,
|
47 |
+
"rstrip": false,
|
48 |
+
"single_word": false,
|
49 |
+
"special": true
|
50 |
+
},
|
51 |
+
"32771": {
|
52 |
+
"content": "<ent2>",
|
53 |
+
"lstrip": false,
|
54 |
+
"normalized": true,
|
55 |
+
"rstrip": false,
|
56 |
+
"single_word": false,
|
57 |
+
"special": true
|
58 |
+
}
|
59 |
+
},
|
60 |
+
"additional_special_tokens": [
|
61 |
+
"<ent>",
|
62 |
+
"<ent2>",
|
63 |
+
"<ent>",
|
64 |
+
"<ent2>",
|
65 |
+
"<ent>",
|
66 |
+
"<ent2>",
|
67 |
+
"<ent>",
|
68 |
+
"<ent2>"
|
69 |
+
],
|
70 |
+
"bos_token": "<s>",
|
71 |
+
"clean_up_tokenization_spaces": true,
|
72 |
+
"cls_token": "<s>",
|
73 |
+
"entity_mask2_token": "[MASK2]",
|
74 |
+
"entity_mask_token": "[MASK]",
|
75 |
+
"entity_pad_token": "[PAD]",
|
76 |
+
"entity_token_1": {
|
77 |
+
"__type": "AddedToken",
|
78 |
+
"content": "<ent>",
|
79 |
+
"lstrip": false,
|
80 |
+
"normalized": true,
|
81 |
+
"rstrip": false,
|
82 |
+
"single_word": false,
|
83 |
+
"special": false
|
84 |
+
},
|
85 |
+
"entity_token_2": {
|
86 |
+
"__type": "AddedToken",
|
87 |
+
"content": "<ent2>",
|
88 |
+
"lstrip": false,
|
89 |
+
"normalized": true,
|
90 |
+
"rstrip": false,
|
91 |
+
"single_word": false,
|
92 |
+
"special": false
|
93 |
+
},
|
94 |
+
"entity_unk_token": "[UNK]",
|
95 |
+
"eos_token": "</s>",
|
96 |
+
"mask_token": "<mask>",
|
97 |
+
"max_entity_length": 32,
|
98 |
+
"max_mention_length": 30,
|
99 |
+
"model_max_length": 512,
|
100 |
+
"pad_token": "<pad>",
|
101 |
+
"sep_token": "</s>",
|
102 |
+
"sp_model_kwargs": {},
|
103 |
+
"task": null,
|
104 |
+
"tokenizer_class": "MLukeTokenizer",
|
105 |
+
"unk_token": "<unk>"
|
106 |
+
}
|
val_data.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2e9a840c70377691526c516f49b5712585452cf7e7f6e0a30ce621fadb84b6f6
|
3 |
+
size 211003
|