parkerjj commited on
Commit
1caca07
·
1 Parent(s): c439e76

移除情感评分函数中tokenizer调用的线程锁,简化代码逻辑

Browse files
Files changed (1) hide show
  1. preprocess.py +2 -4
preprocess.py CHANGED
@@ -310,8 +310,7 @@ def get_sentiment_score(text):
310
  # 处理每个段落 - 模型一
311
  for segment in segments_one:
312
  with torch.no_grad():
313
- with _tokenizer_lock:
314
- inputs = tokenizer_one(segment, return_tensors="pt", truncation=True, max_length=512)
315
  outputs = sa_model_one(**inputs)
316
  predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
317
 
@@ -334,8 +333,7 @@ def get_sentiment_score(text):
334
  # 处理每个段落 - 模型二
335
  for segment in segments_two:
336
  with torch.no_grad():
337
- with _tokenizer_lock:
338
- inputs = tokenizer_two(segment, return_tensors="pt", truncation=True, max_length=512)
339
  outputs = sa_model_two(**inputs)
340
  predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
341
 
 
310
  # 处理每个段落 - 模型一
311
  for segment in segments_one:
312
  with torch.no_grad():
313
+ inputs = tokenizer_one(segment, return_tensors="pt", truncation=True, max_length=512)
 
314
  outputs = sa_model_one(**inputs)
315
  predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
316
 
 
333
  # 处理每个段落 - 模型二
334
  for segment in segments_two:
335
  with torch.no_grad():
336
+ inputs = tokenizer_two(segment, return_tensors="pt", truncation=True, max_length=512)
 
337
  outputs = sa_model_two(**inputs)
338
  predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
339