update model

.gitattributes

@@ -32,3 +32,16 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+optimizer.pt filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
+scaler.pt filter=lfs diff=lfs merge=lfs -text
+scheduler.pt filter=lfs diff=lfs merge=lfs -text
+special_tokens_map.json filter=lfs diff=lfs merge=lfs -text
+training_args.bin filter=lfs diff=lfs merge=lfs -text
+config.json filter=lfs diff=lfs merge=lfs -text
+rng_state.pth filter=lfs diff=lfs merge=lfs -text
+tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+trainer_state.json filter=lfs diff=lfs merge=lfs -text
+vocab.json filter=lfs diff=lfs merge=lfs -text
+merges.txt filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,35 +0,0 @@
-This is a deepfake text detector (Longformer) trained using the testbeds in Github project 🏃 [Deepfake Text Detection in the Wild](https://github.com/yafuly/DeepfakeTextDetect).
-Here is a simple example of how to use the detector.
-However, we recommend utilizing the full detection pipeline available on our Github, which includes text preprocessing.
-
-
-```python
-import torch
-import os
-from transformers import AutoModelForSequenceClassification,AutoTokenizer
-
-
-device = 'cuda:0'
-model_dir = "nealcly/detection-longformer"
-tokenizer = AutoTokenizer.from_pretrained(model_dir)
-model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device)
-
-label2decisions = {
-    0: "machine-generated",
-    1: "human-written",
-}
-def detect(input_text,th=-3.08583984375):
-    tokenize_input = tokenizer(input_text)
-    tensor_input = torch.tensor([tokenize_input["input_ids"]]).to(device)
-    outputs = model(tensor_input)
-    is_machine = -outputs.logits[0][0].item()
-    if is_machine < th:
-        decision = 0
-    else:
-        decision = 1
-    print(f"The text is {label2decisions[decision]}.")
-
-input_text = "Researchers at Stanford University and the SLAC National Accelerator Laboratory have discovered a way to transform a substance found in fossil fuels into diamonds with pressure and low heat. Diamond synthesis usually requires a large amount of energy, time, or the addition of a catalyst, which adds impurities. Diamondoids are tiny, odorless, and slightly sticky powders that resemble rock salt. They are made up of atoms arranged in the same pattern as diamonds, but they contain hydrogen. Diamondoids can reorganize into diamonds with surprisingly little energy, without passing through other forms of carbon, such as graphite. The method is currently only able to make specks of diamonds, and it is impractical until larger crystals can be formed." # human-written
-input_text = "Reddit Talk is a new social audio product that allows subreddit moderators to start Clubhouse-like Talks. While moderators will have control over who can speak in the sessions, anybody on Reddit or Discord can join and listen in. It's like an open mic with your own personal mods in charge of taking care of everything else (like banning trolls). The idea is to create more friendly and interactive conversations among users rather than just endless battles between assholes. There are even 'subreddits for each type of topic moderated by their in context moderation team members."" The current moderation was created very quickly as popularity spiked within days after Reddit acquired it back in February 2019. We think this could be a great way to keep discussions active without having someone run them off into the abyss." # machine-generated
-detect(input_text)
-```

config.json

@@ -1,53 +1,3 @@
-{
-  "_name_or_path": "/apdcephfs/share_916081/effidit_shared_data/yafuli/Codes/LLM-results/classfication/models/longformer_base_4096",
-  "architectures": [
-    "LongformerForSequenceClassification"
-  ],
-  "attention_mode": "longformer",
-  "attention_probs_dropout_prob": 0.1,
-  "attention_window": [
-    512,
-    512,
-    512,
-    512,
-    512,
-    512,
-    512,
-    512,
-    512,
-    512,
-    512,
-    512
-  ],
-  "bos_token_id": 0,
-  "eos_token_id": 2,
-  "gradient_checkpointing": false,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "id2label": {
-    "0": 0,
-    "1": 1
-  },
-  "ignore_attention_mask": false,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "label2id": {
-    "0": 0,
-    "1": 1
-  },
-  "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 4098,
-  "model_type": "longformer",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "onnx_export": false,
-  "pad_token_id": 1,
-  "position_embedding_type": "absolute",
-  "problem_type": "single_label_classification",
-  "sep_token_id": 2,
-  "torch_dtype": "float32",
-  "transformers_version": "4.27.4",
-  "type_vocab_size": 1,
-  "vocab_size": 50265
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce51c2bbfd353433e384ea042883ef20a67c458aa3f8d88b95fc582728382328
+size 1161

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6b9b31fc7deb68db3738f6a96fe2b0adc075b9e0592418ce49056340664b658
+oid sha256:82e8a624dc1273ba5eb689856ab591869eadd100edbd884ec5d4b7026fdebdc1
 size 1189446589

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a8853ee3563ba5d6f251dba1421a028dd2251ff80c0ba110650dfed5a53766e
+oid sha256:99b8df5eff67a96c9cd47c53bd914d31dd80bbe9a3c3b8b5fb101c60458a03bf
 size 594737055

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bae425a9a22152b4243f1294e3c4f6f1ac182f90d2c74645f2626619079787a7
+oid sha256:525961baca9138129946e3341b7059e0fb39e609f2aa53c27e1291a8e85cb774
 size 21579

scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4eb96d3c159ec49bfc76551332a7f52392c99a69b649688e4dc3f1979f6d527b
+oid sha256:9a7b2e65afa3c51b1ad587b8bc79d39d4d0b86949cde1725008fb133af677ff8
 size 559

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d95c1e1c6ec2c56daac9bd96a1fee70862322dcd845123edfab43941f796769
+oid sha256:ecdb69babb24440944c1b22dc3e00801f9b10a77365526be86a60fd6738e17d4
 size 623

special_tokens_map.json

@@ -1,15 +1,3 @@
-{
-  "bos_token": "<s>",
-  "cls_token": "<s>",
-  "eos_token": "</s>",
-  "mask_token": {
-    "content": "<mask>",
-    "lstrip": true,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "unk_token": "<unk>"
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:06e405a36dfe4b9604f484f6a1e619af1a7f7d09e34a8555eb0b77b66318067f
+size 280

tokenizer_config.json

@@ -1,15 +1,3 @@
-{
-  "add_prefix_space": false,
-  "bos_token": "<s>",
-  "cls_token": "<s>",
-  "eos_token": "</s>",
-  "errors": "replace",
-  "mask_token": "<mask>",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "special_tokens_map_file": null,
-  "tokenizer_class": "LongformerTokenizer",
-  "trim_offsets": true,
-  "unk_token": "<unk>"
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:e667b8a2c557b7012bfdec7dd4cbb6066cf6ec9069f0515db0c4fc8cdb162c0f
+size 377

trainer_state.json

@@ -1,478 +1,3 @@
-{
-  "best_metric": 0.262811541557312,
-  "best_model_checkpoint": "./output_samples_202302241350_lfbase/checkpoint-6000",
-  "epoch": 4.245465071401004,
-  "global_step": 22000,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.1,
-      "learning_rate": 2.94245465071401e-05,
-      "loss": 0.2869,
-      "step": 500
-    },
-    {
-      "epoch": 0.19,
-      "learning_rate": 2.8845619451949058e-05,
-      "loss": 0.1797,
-      "step": 1000
-    },
-    {
-      "epoch": 0.19,
-      "eval_accuracy": 0.8138957619667053,
-      "eval_loss": 0.49187058210372925,
-      "eval_runtime": 490.7956,
-      "eval_samples_per_second": 119.795,
-      "eval_steps_per_second": 1.872,
-      "step": 1000
-    },
-    {
-      "epoch": 0.29,
-      "learning_rate": 2.826669239675801e-05,
-      "loss": 0.1589,
-      "step": 1500
-    },
-    {
-      "epoch": 0.39,
-      "learning_rate": 2.7687765341566965e-05,
-      "loss": 0.1369,
-      "step": 2000
-    },
-    {
-      "epoch": 0.39,
-      "eval_accuracy": 0.8265668749809265,
-      "eval_loss": 0.4808556139469147,
-      "eval_runtime": 472.7272,
-      "eval_samples_per_second": 124.374,
-      "eval_steps_per_second": 1.944,
-      "step": 2000
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 2.71099961404863e-05,
-      "loss": 0.128,
-      "step": 2500
-    },
-    {
-      "epoch": 0.58,
-      "learning_rate": 2.6531069085295255e-05,
-      "loss": 0.1274,
-      "step": 3000
-    },
-    {
-      "epoch": 0.58,
-      "eval_accuracy": 0.8054426312446594,
-      "eval_loss": 0.6082525253295898,
-      "eval_runtime": 485.5844,
-      "eval_samples_per_second": 121.081,
-      "eval_steps_per_second": 1.893,
-      "step": 3000
-    },
-    {
-      "epoch": 0.68,
-      "learning_rate": 2.5953299884214592e-05,
-      "loss": 0.1161,
-      "step": 3500
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 2.5374372829023545e-05,
-      "loss": 0.1109,
-      "step": 4000
-    },
-    {
-      "epoch": 0.77,
-      "eval_accuracy": 0.8235734105110168,
-      "eval_loss": 0.510552704334259,
-      "eval_runtime": 488.158,
-      "eval_samples_per_second": 120.443,
-      "eval_steps_per_second": 1.883,
-      "step": 4000
-    },
-    {
-      "epoch": 0.87,
-      "learning_rate": 2.47954457738325e-05,
-      "loss": 0.1078,
-      "step": 4500
-    },
-    {
-      "epoch": 0.96,
-      "learning_rate": 2.4216518718641452e-05,
-      "loss": 0.1036,
-      "step": 5000
-    },
-    {
-      "epoch": 0.96,
-      "eval_accuracy": 0.8834084272384644,
-      "eval_loss": 0.312045156955719,
-      "eval_runtime": 486.8674,
-      "eval_samples_per_second": 120.762,
-      "eval_steps_per_second": 1.888,
-      "step": 5000
-    },
-    {
-      "epoch": 1.06,
-      "learning_rate": 2.3637591663450405e-05,
-      "loss": 0.0823,
-      "step": 5500
-    },
-    {
-      "epoch": 1.16,
-      "learning_rate": 2.305866460825936e-05,
-      "loss": 0.0718,
-      "step": 6000
-    },
-    {
-      "epoch": 1.16,
-      "eval_accuracy": 0.9076281785964966,
-      "eval_loss": 0.262811541557312,
-      "eval_runtime": 488.7776,
-      "eval_samples_per_second": 120.29,
-      "eval_steps_per_second": 1.88,
-      "step": 6000
-    },
-    {
-      "epoch": 1.25,
-      "learning_rate": 2.2479737553068312e-05,
-      "loss": 0.0714,
-      "step": 6500
-    },
-    {
-      "epoch": 1.35,
-      "learning_rate": 2.190081049787727e-05,
-      "loss": 0.0729,
-      "step": 7000
-    },
-    {
-      "epoch": 1.35,
-      "eval_accuracy": 0.8230121731758118,
-      "eval_loss": 0.6299644112586975,
-      "eval_runtime": 480.4556,
-      "eval_samples_per_second": 122.373,
-      "eval_steps_per_second": 1.913,
-      "step": 7000
-    },
-    {
-      "epoch": 1.45,
-      "learning_rate": 2.1321883442686222e-05,
-      "loss": 0.0687,
-      "step": 7500
-    },
-    {
-      "epoch": 1.54,
-      "learning_rate": 2.0742956387495175e-05,
-      "loss": 0.0664,
-      "step": 8000
-    },
-    {
-      "epoch": 1.54,
-      "eval_accuracy": 0.8701080083847046,
-      "eval_loss": 0.5767059922218323,
-      "eval_runtime": 472.3986,
-      "eval_samples_per_second": 124.461,
-      "eval_steps_per_second": 1.945,
-      "step": 8000
-    },
-    {
-      "epoch": 1.64,
-      "learning_rate": 2.0164029332304132e-05,
-      "loss": 0.0665,
-      "step": 8500
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 1.9585102277113085e-05,
-      "loss": 0.0622,
-      "step": 9000
-    },
-    {
-      "epoch": 1.74,
-      "eval_accuracy": 0.8938174843788147,
-      "eval_loss": 0.46610942482948303,
-      "eval_runtime": 472.4306,
-      "eval_samples_per_second": 124.452,
-      "eval_steps_per_second": 1.945,
-      "step": 9000
-    },
-    {
-      "epoch": 1.83,
-      "learning_rate": 1.9007333076032422e-05,
-      "loss": 0.0639,
-      "step": 9500
-    },
-    {
-      "epoch": 1.93,
-      "learning_rate": 1.8428406020841375e-05,
-      "loss": 0.0638,
-      "step": 10000
-    },
-    {
-      "epoch": 1.93,
-      "eval_accuracy": 0.913428008556366,
-      "eval_loss": 0.27155911922454834,
-      "eval_runtime": 484.5228,
-      "eval_samples_per_second": 121.346,
-      "eval_steps_per_second": 1.897,
-      "step": 10000
-    },
-    {
-      "epoch": 2.03,
-      "learning_rate": 1.784947896565033e-05,
-      "loss": 0.0564,
-      "step": 10500
-    },
-    {
-      "epoch": 2.12,
-      "learning_rate": 1.7270551910459282e-05,
-      "loss": 0.0385,
-      "step": 11000
-    },
-    {
-      "epoch": 2.12,
-      "eval_accuracy": 0.9116081595420837,
-      "eval_loss": 0.38816747069358826,
-      "eval_runtime": 488.2826,
-      "eval_samples_per_second": 120.412,
-      "eval_steps_per_second": 1.882,
-      "step": 11000
-    },
-    {
-      "epoch": 2.22,
-      "learning_rate": 1.669162485526824e-05,
-      "loss": 0.0412,
-      "step": 11500
-    },
-    {
-      "epoch": 2.32,
-      "learning_rate": 1.6113855654187572e-05,
-      "loss": 0.0406,
-      "step": 12000
-    },
-    {
-      "epoch": 2.32,
-      "eval_accuracy": 0.863440752029419,
-      "eval_loss": 0.6258434653282166,
-      "eval_runtime": 473.3305,
-      "eval_samples_per_second": 124.216,
-      "eval_steps_per_second": 1.942,
-      "step": 12000
-    },
-    {
-      "epoch": 2.41,
-      "learning_rate": 1.553492859899653e-05,
-      "loss": 0.0391,
-      "step": 12500
-    },
-    {
-      "epoch": 2.51,
-      "learning_rate": 1.495600154380548e-05,
-      "loss": 0.037,
-      "step": 13000
-    },
-    {
-      "epoch": 2.51,
-      "eval_accuracy": 0.891011118888855,
-      "eval_loss": 0.48729509115219116,
-      "eval_runtime": 473.8644,
-      "eval_samples_per_second": 124.076,
-      "eval_steps_per_second": 1.939,
-      "step": 13000
-    },
-    {
-      "epoch": 2.61,
-      "learning_rate": 1.4377074488614436e-05,
-      "loss": 0.04,
-      "step": 13500
-    },
-    {
-      "epoch": 2.7,
-      "learning_rate": 1.3798147433423389e-05,
-      "loss": 0.0382,
-      "step": 14000
-    },
-    {
-      "epoch": 2.7,
-      "eval_accuracy": 0.8833404183387756,
-      "eval_loss": 0.5917666554450989,
-      "eval_runtime": 473.8023,
-      "eval_samples_per_second": 124.092,
-      "eval_steps_per_second": 1.94,
-      "step": 14000
-    },
-    {
-      "epoch": 2.8,
-      "learning_rate": 1.3220378232342726e-05,
-      "loss": 0.0392,
-      "step": 14500
-    },
-    {
-      "epoch": 2.89,
-      "learning_rate": 1.2642609031262061e-05,
-      "loss": 0.0367,
-      "step": 15000
-    },
-    {
-      "epoch": 2.89,
-      "eval_accuracy": 0.8793604969978333,
-      "eval_loss": 0.569683313369751,
-      "eval_runtime": 474.4494,
-      "eval_samples_per_second": 123.923,
-      "eval_steps_per_second": 1.937,
-      "step": 15000
-    },
-    {
-      "epoch": 2.99,
-      "learning_rate": 1.2063681976071016e-05,
-      "loss": 0.0358,
-      "step": 15500
-    },
-    {
-      "epoch": 3.09,
-      "learning_rate": 1.148475492087997e-05,
-      "loss": 0.0216,
-      "step": 16000
-    },
-    {
-      "epoch": 3.09,
-      "eval_accuracy": 0.8624373078346252,
-      "eval_loss": 0.8281469345092773,
-      "eval_runtime": 473.8459,
-      "eval_samples_per_second": 124.08,
-      "eval_steps_per_second": 1.939,
-      "step": 16000
-    },
-    {
-      "epoch": 3.18,
-      "learning_rate": 1.0905827865688923e-05,
-      "loss": 0.0211,
-      "step": 16500
-    },
-    {
-      "epoch": 3.28,
-      "learning_rate": 1.032805866460826e-05,
-      "loss": 0.0227,
-      "step": 17000
-    },
-    {
-      "epoch": 3.28,
-      "eval_accuracy": 0.8539331555366516,
-      "eval_loss": 0.9032775163650513,
-      "eval_runtime": 474.5611,
-      "eval_samples_per_second": 123.893,
-      "eval_steps_per_second": 1.937,
-      "step": 17000
-    },
-    {
-      "epoch": 3.38,
-      "learning_rate": 9.749131609417213e-06,
-      "loss": 0.0225,
-      "step": 17500
-    },
-    {
-      "epoch": 3.47,
-      "learning_rate": 9.170204554226166e-06,
-      "loss": 0.0216,
-      "step": 18000
-    },
-    {
-      "epoch": 3.47,
-      "eval_accuracy": 0.888221800327301,
-      "eval_loss": 0.6849754452705383,
-      "eval_runtime": 487.296,
-      "eval_samples_per_second": 120.656,
-      "eval_steps_per_second": 1.886,
-      "step": 18000
-    },
-    {
-      "epoch": 3.57,
-      "learning_rate": 8.591277499035123e-06,
-      "loss": 0.0217,
-      "step": 18500
-    },
-    {
-      "epoch": 3.67,
-      "learning_rate": 8.012350443844076e-06,
-      "loss": 0.0199,
-      "step": 19000
-    },
-    {
-      "epoch": 3.67,
-      "eval_accuracy": 0.8972702026367188,
-      "eval_loss": 0.6052428483963013,
-      "eval_runtime": 490.4243,
-      "eval_samples_per_second": 119.886,
-      "eval_steps_per_second": 1.874,
-      "step": 19000
-    },
-    {
-      "epoch": 3.76,
-      "learning_rate": 7.43342338865303e-06,
-      "loss": 0.0207,
-      "step": 19500
-    },
-    {
-      "epoch": 3.86,
-      "learning_rate": 6.855654187572366e-06,
-      "loss": 0.0206,
-      "step": 20000
-    },
-    {
-      "epoch": 3.86,
-      "eval_accuracy": 0.8873884081840515,
-      "eval_loss": 0.6865639090538025,
-      "eval_runtime": 487.3484,
-      "eval_samples_per_second": 120.643,
-      "eval_steps_per_second": 1.886,
-      "step": 20000
-    },
-    {
-      "epoch": 3.96,
-      "learning_rate": 6.276727132381321e-06,
-      "loss": 0.0187,
-      "step": 20500
-    },
-    {
-      "epoch": 4.05,
-      "learning_rate": 5.697800077190274e-06,
-      "loss": 0.0148,
-      "step": 21000
-    },
-    {
-      "epoch": 4.05,
-      "eval_accuracy": 0.8811633586883545,
-      "eval_loss": 0.9089483618736267,
-      "eval_runtime": 487.5135,
-      "eval_samples_per_second": 120.602,
-      "eval_steps_per_second": 1.885,
-      "step": 21000
-    },
-    {
-      "epoch": 4.15,
-      "learning_rate": 5.118873021999228e-06,
-      "loss": 0.0107,
-      "step": 21500
-    },
-    {
-      "epoch": 4.25,
-      "learning_rate": 4.5399459668081825e-06,
-      "loss": 0.0121,
-      "step": 22000
-    },
-    {
-      "epoch": 4.25,
-      "eval_accuracy": 0.9049409031867981,
-      "eval_loss": 0.641972541809082,
-      "eval_runtime": 487.5431,
-      "eval_samples_per_second": 120.594,
-      "eval_steps_per_second": 1.885,
-      "step": 22000
-    }
-  ],
-  "max_steps": 25910,
-  "num_train_epochs": 5,
-  "total_flos": 1.8496468450847293e+18,
-  "trial_name": null,
-  "trial_params": null
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff464999e57de1ac0bd298f3fa3ce56feaa891ca78b348e781a8786778d55fe3
+size 3362