alexkueck commited on
Commit
7f49dd1
·
1 Parent(s): 3f81b05

Update utils.py

Browse files
Files changed (1) hide show
  1. utils.py +14 -3
utils.py CHANGED
@@ -18,8 +18,8 @@ from pygments import highlight
18
  from pygments.lexers import guess_lexer,get_lexer_by_name
19
  from pygments.formatters import HtmlFormatter
20
  import transformers
21
- from transformers import AutoTokenizer, AutoModelForCausalLM
22
-
23
 
24
  def reset_state():
25
  return [], [], "Reset Done"
@@ -192,4 +192,15 @@ class State:
192
 
193
  def recover(self):
194
  self.interrupted = False
195
- shared_state = State()
 
 
 
 
 
 
 
 
 
 
 
 
18
  from pygments.lexers import guess_lexer,get_lexer_by_name
19
  from pygments.formatters import HtmlFormatter
20
  import transformers
21
+ import datasets
22
+ from datasets import load_dataset
23
 
24
  def reset_state():
25
  return [], [], "Reset Done"
 
192
 
193
  def recover(self):
194
  self.interrupted = False
195
+ shared_state = State()
196
+
197
+
198
+ #######################################################
199
+ #Fine-Tuning
200
+ #######################################################
201
+
202
+ #load Dataset
203
+ def daten_laden(name):
204
+ dataset = load_dataset(name)
205
+
206
+