96abhishekarora commited on
Commit
6aa6cec
1 Parent(s): f403d17

Updated model with better training and evaluation. Test and val data included as pickle files. Older Legacy files were removed to avoid confusion.

Browse files
.gitattributes CHANGED
@@ -1,35 +1,4 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ model.safetensors filter=lfs diff=lfs merge=lfs -text
2
+ sentencepiece.bpe.model filter=lfs diff=lfs merge=lfs -text
3
+ test_data.pickle filter=lfs diff=lfs merge=lfs -text
4
+ val_data.pickle filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1_Pooling/config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "word_embedding_dimension": 768,
3
+ "pooling_mode_cls_token": false,
4
+ "pooling_mode_mean_tokens": true,
5
+ "pooling_mode_max_tokens": false,
6
+ "pooling_mode_mean_sqrt_len_tokens": false
7
+ }
LT_training_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_save_dir": "models",
3
+ "model_save_name": "lt-historicjapanesecompanies-comp-prod-ind_supcon_full",
4
+ "opt_model_description": "This model was trained on a dataset of historic Japanese companies, products, industry, addresses, and shareholders. Take a look at our paper for more details. The task is to link indices of japanese companies",
5
+ "opt_model_lang": "ja",
6
+ "train_batch_size": 64,
7
+ "num_epochs": 50,
8
+ "warm_up_perc": 1,
9
+ "learning_rate": 2e-05,
10
+ "loss_type": "supcon",
11
+ "val_perc": 0.2,
12
+ "wandb_names": {
13
+ "id": "econabhishek",
14
+ "run": "lt-historicjapanesecompanies-comp-prod-ind_supcon_full",
15
+ "project": "linkage",
16
+ "entity": "econabhishek"
17
+ },
18
+ "add_pooling_layer": false,
19
+ "large_val": true,
20
+ "eval_steps_perc": 0.5,
21
+ "test_at_end": true,
22
+ "save_val_test_pickles": true,
23
+ "val_query_prop": 0.5,
24
+ "loss_params": {},
25
+ "eval_type": "classification",
26
+ "training_dataset": "dataframe",
27
+ "base_model_path": "oshizo/sbert-jsnli-luke-japanese-base-lite",
28
+ "best_model_path": "models/lt-historicjapanesecompanies-comp-prod-ind_supcon_full"
29
+ }
README.md ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ pipeline_tag: sentence-similarity
3
+ language:
4
+ - ja
5
+ tags:
6
+ - linktransformer
7
+ - sentence-transformers
8
+ - sentence-similarity
9
+ - tabular-classification
10
+
11
+ ---
12
+
13
+ # {MODEL_NAME}
14
+
15
+ This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class.
16
+ It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.
17
+ Notwithstanding that, it can be used for any sentence similarity task within the sentence-transformers framework as well.
18
+ It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
19
+ Take a look at the documentation of [sentence-transformers](https://www.sbert.net/index.html) if you want to use this model for more than what we support in our applications.
20
+
21
+
22
+ This model has been fine-tuned on the model : oshizo/sbert-jsnli-luke-japanese-base-lite. It is pretrained for the language : - ja.
23
+
24
+
25
+ This model was trained on a dataset of historic Japanese companies, products, industry, addresses, and shareholders. Take a look at our paper for more details. The task is to link indices of japanese companies
26
+
27
+ ## Usage (LinkTransformer)
28
+
29
+ Using this model becomes easy when you have [LinkTransformer](https://github.com/dell-research-harvard/linktransformer) installed:
30
+
31
+ ```
32
+ pip install -U linktransformer
33
+ ```
34
+
35
+ Then you can use the model like this:
36
+
37
+ ```python
38
+ import linktransformer as lt
39
+ import pandas as pd
40
+
41
+ ##Load the two dataframes that you want to link. For example, 2 dataframes with company names that are written differently
42
+ df1=pd.read_csv("data/df1.csv") ###This is the left dataframe with key CompanyName for instance
43
+ df2=pd.read_csv("data/df2.csv") ###This is the right dataframe with key CompanyName for instance
44
+
45
+ ###Merge the two dataframes on the key column!
46
+ df_merged = lt.merge(df1, df2, on="CompanyName", how="inner")
47
+
48
+ ##Done! The merged dataframe has a column called "score" that contains the similarity score between the two company names
49
+
50
+ ```
51
+
52
+
53
+ ## Training your own LinkTransformer model
54
+ Any Sentence Transformers can be used as a backbone by simply adding a pooling layer. Any other transformer on HuggingFace can also be used by specifying the option add_pooling_layer==True
55
+ The model was trained using SupCon loss.
56
+ Usage can be found in the package docs.
57
+ The training config can be found in the repo with the name LT_training_config.json
58
+ To replicate the training, you can download the file and specify the path in the config_path argument of the training function. You can also override the config by specifying the training_args argument.
59
+ Here is an example.
60
+
61
+
62
+ ```python
63
+
64
+ ##Consider the example in the paper that has a dataset of Mexican products and their tariff codes from 1947 and 1948 and we want train a model to link the two tariff codes.
65
+ saved_model_path = train_model(
66
+ model_path="hiiamsid/sentence_similarity_spanish_es",
67
+ dataset_path=dataset_path,
68
+ left_col_names=["description47"],
69
+ right_col_names=['description48'],
70
+ left_id_name=['tariffcode47'],
71
+ right_id_name=['tariffcode48'],
72
+ log_wandb=False,
73
+ config_path=LINKAGE_CONFIG_PATH,
74
+ training_args={"num_epochs": 1}
75
+ )
76
+
77
+ ```
78
+
79
+
80
+ You can also use this package for deduplication (clusters a df on the supplied key column). Merging a fine class (like product) to a coarse class (like HS code) is also possible.
81
+ Read our paper and the documentation for more!
82
+
83
+
84
+
85
+ ## Evaluation Results
86
+
87
+ <!--- Describe how your model was evaluated -->
88
+
89
+ You can evaluate the model using the [LinkTransformer](https://github.com/dell-research-harvard/linktransformer) package's inference functions.
90
+ We have provided a few datasets in the package for you to try out. We plan to host more datasets on Huggingface and our website (Coming soon) that you can take a look at.
91
+
92
+
93
+ ## Training
94
+ The model was trained with the parameters:
95
+
96
+ **DataLoader**:
97
+
98
+ `torch.utils.data.dataloader.DataLoader` of length 16 with parameters:
99
+ ```
100
+ {'batch_size': 64, 'sampler': 'torch.utils.data.dataloader._InfiniteConstantSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
101
+ ```
102
+
103
+ **Loss**:
104
+
105
+ `linktransformer.modified_sbert.losses.SupConLoss_wandb`
106
+
107
+ Parameters of the fit()-Method:
108
+ ```
109
+ {
110
+ "epochs": 50,
111
+ "evaluation_steps": 8,
112
+ "evaluator": "sentence_transformers.evaluation.SequentialEvaluator.SequentialEvaluator",
113
+ "max_grad_norm": 1,
114
+ "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
115
+ "optimizer_params": {
116
+ "lr": 2e-05
117
+ },
118
+ "scheduler": "WarmupLinear",
119
+ "steps_per_epoch": null,
120
+ "warmup_steps": 800,
121
+ "weight_decay": 0.01
122
+ }
123
+ ```
124
+
125
+
126
+
127
+
128
+ LinkTransformer(
129
+ (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: LukeModel
130
+ (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
131
+ )
132
+ ```
133
+
134
+ ## Citing & Authors
135
+
136
+ ```
137
+ @misc{arora2023linktransformer,
138
+ title={LinkTransformer: A Unified Package for Record Linkage with Transformer Language Models},
139
+ author={Abhishek Arora and Melissa Dell},
140
+ year={2023},
141
+ eprint={2309.00789},
142
+ archivePrefix={arXiv},
143
+ primaryClass={cs.CL}
144
+ }
145
+
146
+ ```
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<ent2>": 32771,
3
+ "<ent>": 32770
4
+ }
binary_classification_evaluation_eval_results.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
2
+ 0,0,0.9382022471910112,0.8532869815826416,0.7636363636363636,0.7,0.84,0.8262333869934082,0.8184099257358792,0.9382022471910112,84.1578369140625,0.7572815533980584,0.7358490566037735,0.78,89.29296112060547,0.8270661393011416,0.9438202247191011,4.08632755279541,0.7872340425531915,0.8409090909090909,0.74,4.08632755279541,0.8277502672701681,0.8848314606741573,44.88762664794922,0.5925925925925927,0.5517241379310345,0.64,43.54036331176758,0.5417359362559628
3
+ 0,0,0.9382022471910112,0.8532869815826416,0.7636363636363636,0.7,0.84,0.8262333869934082,0.8184099257358792,0.9382022471910112,84.1578369140625,0.7572815533980584,0.7358490566037735,0.78,89.29296112060547,0.8270661393011416,0.9438202247191011,4.08632755279541,0.7872340425531915,0.8409090909090909,0.74,4.08632755279541,0.8277502672701681,0.8848314606741573,44.88762283325195,0.5925925925925927,0.5517241379310345,0.64,43.54036331176758,0.5417359362559628
binary_classification_evaluation_test_results.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
2
+ 0,0,0.9578651685393258,0.7885568141937256,0.854368932038835,0.9166666666666666,0.8,0.7569339871406555,0.9027201028709122,0.9606741573033708,121.89360046386719,0.86,0.9555555555555556,0.7818181818181819,121.89360046386719,0.9092811528662349,0.9606741573033708,5.6210103034973145,0.8653846153846154,0.9183673469387755,0.8181818181818182,5.979313373565674,0.9083606695270187,0.9410112359550562,58.51844024658203,0.7789473684210527,0.925,0.6727272727272727,58.51844024658203,0.8613143757051128
3
+ 0,0,0.9578651685393258,0.7794706225395203,0.845360824742268,0.9761904761904762,0.7454545454545455,0.7794706225395203,0.9070055457080543,0.9578651685393258,119.23606872558594,0.8545454545454545,0.8545454545454545,0.8545454545454545,140.44253540039062,0.9139460658879931,0.9578651685393258,5.471532344818115,0.8545454545454545,0.8545454545454545,0.8545454545454545,6.480866432189941,0.9147327683242549,0.9241573033707865,56.068626403808594,0.7518796992481204,0.6410256410256411,0.9090909090909091,46.432830810546875,0.8419716262659404
config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/home/abhishekarora/.cache/torch/sentence_transformers/oshizo_sbert-jsnli-luke-japanese-base-lite/",
3
+ "architectures": [
4
+ "LukeModel"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bert_model_name": "models/luke-japanese/hf_xlm_roberta",
8
+ "bos_token_id": 0,
9
+ "classifier_dropout": null,
10
+ "cls_entity_prediction": false,
11
+ "entity_emb_size": 256,
12
+ "entity_vocab_size": 4,
13
+ "eos_token_id": 2,
14
+ "hidden_act": "gelu",
15
+ "hidden_dropout_prob": 0.1,
16
+ "hidden_size": 768,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 3072,
19
+ "layer_norm_eps": 1e-05,
20
+ "max_position_embeddings": 514,
21
+ "model_type": "luke",
22
+ "num_attention_heads": 12,
23
+ "num_hidden_layers": 12,
24
+ "pad_token_id": 1,
25
+ "position_embedding_type": "absolute",
26
+ "torch_dtype": "float32",
27
+ "transformers_version": "4.35.1",
28
+ "type_vocab_size": 1,
29
+ "use_cache": true,
30
+ "use_entity_aware_attention": true,
31
+ "vocab_size": 32772
32
+ }
config_sentence_transformers.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "sentence_transformers": "2.2.2",
4
+ "transformers": "4.25.1",
5
+ "pytorch": "1.13.0+cu116"
6
+ }
7
+ }
entity_vocab.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "[MASK2]": 3,
3
+ "[MASK]": 0,
4
+ "[PAD]": 2,
5
+ "[UNK]": 1
6
+ }
eval/binary_classification_evaluation_eval_results.csv ADDED
The diff for this file is too large to render. See raw diff
 
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c43b11600beb431e99a5285efa15ec24aba97c430c0108c0b0a8075c5beb129
3
+ size 532299592
modules.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "sentence_transformers.models.Transformer"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Pooling",
12
+ "type": "sentence_transformers.models.Pooling"
13
+ }
14
+ ]
sentence_bert_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "max_seq_length": 128,
3
+ "do_lower_case": false
4
+ }
sentencepiece.bpe.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8b73a5e054936c920cf5b7d1ec21ce9c281977078269963beb821c6c86fbff7
3
+ size 841889
special_tokens_map.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ {
4
+ "content": "<ent>",
5
+ "lstrip": false,
6
+ "normalized": true,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "<ent2>",
12
+ "lstrip": false,
13
+ "normalized": true,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ },
17
+ {
18
+ "content": "<ent>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ {
25
+ "content": "<ent2>",
26
+ "lstrip": false,
27
+ "normalized": true,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ },
31
+ {
32
+ "content": "<ent>",
33
+ "lstrip": false,
34
+ "normalized": false,
35
+ "rstrip": false,
36
+ "single_word": false
37
+ },
38
+ {
39
+ "content": "<ent2>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false
44
+ },
45
+ {
46
+ "content": "<ent>",
47
+ "lstrip": false,
48
+ "normalized": true,
49
+ "rstrip": false,
50
+ "single_word": false
51
+ },
52
+ {
53
+ "content": "<ent2>",
54
+ "lstrip": false,
55
+ "normalized": true,
56
+ "rstrip": false,
57
+ "single_word": false
58
+ }
59
+ ],
60
+ "bos_token": "<s>",
61
+ "cls_token": "<s>",
62
+ "eos_token": "</s>",
63
+ "mask_token": {
64
+ "content": "<mask>",
65
+ "lstrip": true,
66
+ "normalized": true,
67
+ "rstrip": false,
68
+ "single_word": false
69
+ },
70
+ "pad_token": "<pad>",
71
+ "sep_token": "</s>",
72
+ "unk_token": "<unk>"
73
+ }
test_data.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71d2b470c96d3e8108f0a18ffe2707122bb0f30bf07bc11666de18b384714b0c
3
+ size 206402
test_results_allmethods.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ edit_distance,edit_distance_threshold,SBERT,SBERT_threshold,LT ZS Wiki,LT ZS Wiki_threshold,LT,LT_threshold,OpenAI,OpenAI_threshold
2
+ 0.267639902676399,0.7790753674033062,0.6902654867256638,0.8263489127309215,0.723404255319149,0.6465095054004382,0.8367346938775511,0.7741220046283592,0.8627450980392157,0.9252961885425512
tokenizer_config.json ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<s>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32769": {
36
+ "content": "<mask>",
37
+ "lstrip": true,
38
+ "normalized": true,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "32770": {
44
+ "content": "<ent>",
45
+ "lstrip": false,
46
+ "normalized": true,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "32771": {
52
+ "content": "<ent2>",
53
+ "lstrip": false,
54
+ "normalized": true,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ }
59
+ },
60
+ "additional_special_tokens": [
61
+ "<ent>",
62
+ "<ent2>",
63
+ "<ent>",
64
+ "<ent2>",
65
+ "<ent>",
66
+ "<ent2>",
67
+ "<ent>",
68
+ "<ent2>"
69
+ ],
70
+ "bos_token": "<s>",
71
+ "clean_up_tokenization_spaces": true,
72
+ "cls_token": "<s>",
73
+ "entity_mask2_token": "[MASK2]",
74
+ "entity_mask_token": "[MASK]",
75
+ "entity_pad_token": "[PAD]",
76
+ "entity_token_1": {
77
+ "__type": "AddedToken",
78
+ "content": "<ent>",
79
+ "lstrip": false,
80
+ "normalized": true,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": false
84
+ },
85
+ "entity_token_2": {
86
+ "__type": "AddedToken",
87
+ "content": "<ent2>",
88
+ "lstrip": false,
89
+ "normalized": true,
90
+ "rstrip": false,
91
+ "single_word": false,
92
+ "special": false
93
+ },
94
+ "entity_unk_token": "[UNK]",
95
+ "eos_token": "</s>",
96
+ "mask_token": "<mask>",
97
+ "max_entity_length": 32,
98
+ "max_mention_length": 30,
99
+ "model_max_length": 512,
100
+ "pad_token": "<pad>",
101
+ "sep_token": "</s>",
102
+ "sp_model_kwargs": {},
103
+ "task": null,
104
+ "tokenizer_class": "MLukeTokenizer",
105
+ "unk_token": "<unk>"
106
+ }
val_data.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e9a840c70377691526c516f49b5712585452cf7e7f6e0a30ce621fadb84b6f6
3
+ size 211003