test

app.py

@@ -0,0 +1,4 @@
+import streamlit as st
+
+st.title('Проект по NLP')
+

pages/gpt.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27c9012fd11ba32a1145f6b167d8cafea4a7208585e102aa283a96c335c0908f
+size 1321

pages/models/.gitattributes

@@ -0,0 +1 @@
+*.bin filter=lfs diff=lfs merge=lfs -text

pages/models/gpt/config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "sberbank-ai/rugpt3small_based_on_gpt2",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 2048,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 2048,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.0",
+  "use_cache": true,
+  "vocab_size": 50264
+}

pages/models/gpt/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.35.0"
+}

pages/models/gpt/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6838ff75c456e4cbc50676a4c6d1f3574a512c555d62d8b753afb4c449a44253
+size 500941440

pages/models/gptmodel.py

@@ -0,0 +1,8 @@
+import transformers
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+import torch
+
+
+model_name_or_path = 'pages/models/gpt'
+tokenizer = GPT2Tokenizer.from_pretrained('sberbank-ai/rugpt3small_based_on_gpt2')
+model = GPT2LMHeadModel.from_pretrained(model_name_or_path).to('cpu')

pages/toxicity.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c47ac7306a6ccb6bba3ad38b72b82cb9a253054dd2160a646414cf6ed41a742c
+size 1880

requirements.txt

@@ -0,0 +1,114 @@
+absl-py==2.0.0
+accelerate==0.24.1
+altair==5.1.2
+astunparse==1.6.3
+attrs==23.1.0
+blinker==1.7.0
+cachetools==5.3.2
+certifi==2023.7.22
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.1.1
+cycler==0.12.1
+filelock==3.13.1
+flatbuffers==23.5.26
+fonttools==4.43.1
+fsspec==2023.10.0
+gast==0.5.4
+gitdb==4.0.11
+GitPython==3.1.40
+google-auth==2.23.4
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
+grpcio==1.59.2
+h5py==3.10.0
+huggingface-hub==0.17.3
+idna==3.4
+importlib-metadata==6.8.0
+Jinja2==3.1.2
+joblib==1.3.2
+jsonschema==4.19.2
+jsonschema-specifications==2023.7.1
+keras==2.14.0
+kiwisolver==1.4.5
+libclang==16.0.6
+Markdown==3.5.1
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+matplotlib==3.8.1
+mdurl==0.1.2
+ml-dtypes==0.2.0
+mpmath==1.3.0
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.1
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.3.52
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+opt-einsum==3.3.0
+packaging==23.2
+pandas==2.1.2
+Pillow==10.1.0
+protobuf==4.25.0
+psutil==5.9.6
+pyarrow==14.0.0
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydeck==0.8.1b0
+Pygments==2.16.1
+pyparsing==3.1.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+rich==13.6.0
+rpds-py==0.10.6
+rsa==4.9
+safetensors==0.4.0
+scikit-learn==1.3.2
+scipy==1.11.3
+six==1.16.0
+smmap==5.0.1
+streamlit==1.28.0
+sympy==1.12
+tenacity==8.2.3
+tensorboard==2.14.1
+tensorboard-data-server==0.7.2
+tensorflow==2.14.0
+tensorflow-estimator==2.14.0
+tensorflow-io-gcs-filesystem==0.34.0
+termcolor==2.3.0
+threadpoolctl==3.2.0
+tokenizers==0.14.1
+toml==0.10.2
+toolz==0.12.0
+torch==2.1.0
+torchaudio==2.1.0
+torchutils==0.0.4
+torchvision==0.16.0
+tornado==6.3.3
+tqdm==4.66.1
+transformers==4.34.1
+triton==2.1.0
+typing_extensions==4.8.0
+tzdata==2023.3
+tzlocal==5.2
+urllib3==2.0.7
+validators==0.22.0
+watchdog==3.0.0
+Werkzeug==3.0.1
+wrapt==1.14.1
+zipp==3.17.0