Chan-Y commited on
Commit
dc3c24e
1 Parent(s): d436c9a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +7 -1
app.py CHANGED
@@ -8,6 +8,7 @@ from pathlib import Path
8
  from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
9
  from langchain_core.output_parsers import JsonOutputParser
10
  from langdetect import detect
 
11
 
12
  llm = HuggingFaceEndpoint(
13
  repo_id="mistralai/Mistral-7B-Instruct-v0.3",
@@ -37,8 +38,13 @@ def summarize(file, n_words):
37
  text = read_pdf(file_path)
38
  else:
39
  text = read_txt(file_path)
 
 
 
 
 
40
 
41
- lang = detect(text[:2000])
42
  template_translate = '''
43
  Please carefully read the following document:
44
  <document>
 
8
  from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
9
  from langchain_core.output_parsers import JsonOutputParser
10
  from langdetect import detect
11
+ CONTEXT_WINDOW = 31_750
12
 
13
  llm = HuggingFaceEndpoint(
14
  repo_id="mistralai/Mistral-7B-Instruct-v0.3",
 
38
  text = read_pdf(file_path)
39
  else:
40
  text = read_txt(file_path)
41
+
42
+ print("Length of text is ", len(text))
43
+ if len(text) > CONTEXT_WINDOW:
44
+ print(f"Slicing the first {CONTEXT_WINDOW} characters")
45
+ text = text[:CONTEXT_WINDOW]
46
 
47
+ lang = detect(text[:CONTEXT_WINDOW])
48
  template_translate = '''
49
  Please carefully read the following document:
50
  <document>