modifications

DiseaseAgnostic/edge_to_abstract.py

@@ -162,51 +162,12 @@ if args.mode == 'sentence':
 
     with open(f'generate_abstract/{args.init_mode}{args.reasonable_rate}_sentence.json', 'w') as fl:
         json.dump(single_sentence, fl, indent=4)
-    # with open('generate_abstract/test.txt', 'w') as fl:
-    #     fl.write('\n'.join(test_text))
-    # with open('generate_abstract/dp.txt', 'w') as fl:
-    #     fl.write('\n'.join(test_dp))
+
     with open (f'generate_abstract/path/{args.init_mode}{args.reasonable_rate}_path.json', 'w') as fl:
         fl.write('\n'.join(test_dp))
     with open (f'generate_abstract/path/{args.init_mode}{args.reasonable_rate}_temp.json', 'w') as fl:
         fl.write('\n'.join(test_text))
 
-elif args.mode == 'biogpt':
-    pass
-    # from biogpt_generate import GPT_eval
-    # import spacy
-
-    # model = GPT_eval(args.seed)
-
-    # nlp = spacy.load("en_core_web_sm")
-    # with open(f'generate_abstract/{args.target_split}_{args.reasonable_rate}_sentence.json', 'r') as fl:
-    #     data = json.load(fl)
-
-    # KK = []
-    # input = []
-    # for i,(k, v) in enumerate(data.items()):
-    #     KK.append(k)
-    #     input.append(v)
-    # output = model.eval(input)
-
-    # ret = {}
-    # for i, o in enumerate(output):
-
-    #     o = o.replace('<|abstract|>', '')
-    #     doc = nlp(o)
-    #     sen_list = []
-    #     sen_set = set()
-    #     for sen in doc.sents:
-    #         txt = sen.text
-    #         if not (txt.lower() in sen_set):
-    #             sen_set.add(txt.lower())
-    #             sen_list.append(txt)
-    #     O = ' '.join(sen_list)
-    #     ret[KK[i]] = {'in' : input[i], 'out' : O}
-    
-    # with open(f'generate_abstract/{args.target_split}_{args.reasonable_rate}_biogpt.json', 'w') as fl:
-    #     json.dump(ret, fl, indent=4)
-
 elif args.mode == 'finetune':
 
     import spacy
@@ -260,34 +221,6 @@ elif args.mode == 'finetune':
                     vec[i] = True
             return vec, span
         
-        # def mask_func(tokenized_sen, position):
-
-        #     if len(tokenized_sen) == 0:
-        #         return []
-        #     token_list = []
-        #     # for sen in tokenized_sen:
-        #     #     for token in sen:
-        #     #         token_list.append(token)
-        #     for sen in tokenized_sen:
-        #         token_list += sen.text.split(' ')
-        #     l_p = 0
-        #     r_p = 1
-        #     assert position == 'front' or position == 'back'
-        #     if position == 'back':
-        #         l_p, r_p = r_p, l_p
-        #     P = np.linspace(start = l_p, stop = r_p, num = len(token_list))
-        #     P = (P ** 3) * 0.4
-
-        #     ret_list = []
-        #     for t, p in zip(token_list, list(P)):
-        #         if '.' in t or '(' in t or ')' in t or '[' in t or ']' in t:
-        #             ret_list.append(t)
-        #         else:
-        #             if np.random.rand() < p:
-        #                 ret_list.append('<mask>')
-        #             else:
-        #                 ret_list.append(t)
-        #     return [' '.join(ret_list)]
         def mask_func(tokenized_sen):
 
             if len(tokenized_sen) == 0:
@@ -441,11 +374,7 @@ elif args.mode == 'finetune':
     ret = {}
     case_study = {}
     p_ret = {}
-    add = 0
     dpath_i = 0
-    inner_better = 0
-    outter_better = 0
-    better_than_gpt = 0
     for i,(k, v) in enumerate(tqdm(draft.items())):
 
         span = ret_candidates[str(i)]['span']
@@ -573,80 +502,26 @@ elif args.mode == 'finetune':
         log_Loss = log_Loss[:old_L]
         # sen_list = sen_list[:old_L]
 
-        # mini_span should be preserved
-        # for j in range(len(log_Loss)):
-        #     doc = nlp(sen_list[j])
-        #     sens = [sen.text for sen in doc.sents]
-        #     Len = len(sen_list)
-        #     check_text = ' '.join(sens[j : max(0,len(sens) - Len) + j + 1])
-        #     if span not in check_text:
-        #         log_Loss[j] += 1
-
         p = np.argmin(log_Loss)
-        if p < old_L // 2:
-            inner_better += 1
-        else:
-            outter_better += 1
         content = []
         for i in range(len(real_log_Loss)):
             content.append([sen_list[i], str(real_log_Loss[i])])
         scored[k] = {'path':path_text, 'prompt': prompt, 'in':input, 's':text_s, 'o':text_o, 'out': content, 'bound': boundary}
         p_p = p
-        # print('Old_L:', old_L)
 
         if real_log_Loss[p] > real_log_Loss[p+1+old_L]:
             p_p = p+1+old_L
-        if real_log_Loss[p] > real_log_Loss[p+1+old_L]:
-            add += 1
 
-        if real_log_Loss[p] < real_log_Loss[old_L]:
-            better_than_gpt += 1
-        else:
+        if real_log_Loss[p] > real_log_Loss[old_L]:
             if real_log_Loss[p] > real_log_Loss[p+1+old_L]:
                 p = p+1+old_L
         # case_study[k] = {'path':path_text, 'entity_0': text_s, 'entity_1': text_o, 'GPT_in': input, 'Prompt': prompt, 'GPT_out': {'text': output, 'perplexity': str(np.exp(real_log_Loss[old_L]))}, 'BART_in': BART_in[p], 'BART_out': {'text': sen_list[p], 'perplexity': str(np.exp(real_log_Loss[p]))}, 'Assist': {'text': Assist[p], 'perplexity': str(np.exp(real_log_Loss[p+1+old_L]))}}
         ret[k] = {'prompt': prompt, 'in':input, 'out': sen_list[p]}
-        p_ret[k] = {'prompt': prompt, 'in':input, 'out': sen_list[p_p]}
-    print(add)
-    print('inner_better:', inner_better)
-    print('outter_better:', outter_better)
-    print('better_than_gpt:', better_than_gpt)
-    print('better_than_replace', add)
+
     with open(f'generate_abstract/{args.init_mode}{args.reasonable_rate}{args.ratio}_bioBART_finetune.json', 'w') as fl:
         json.dump(ret, fl, indent=4)
-    # with open(f'generate_abstract/bioBART/case_{args.target_split}_{args.reasonable_rate}_bioBART_finetune.json', 'w') as fl:
-    #     json.dump(case_study, fl, indent=4)
     with open(f'generate_abstract/bioBART/{args.init_mode}{args.reasonable_rate}{args.ratio}_scored.json', 'w') as fl:
         json.dump(scored, fl, indent=4)
-    with open(f'generate_abstract/bioBART/{args.init_mode}{args.reasonable_rate}{args.ratio}_perplexity.json', 'w') as fl:
-        json.dump(p_ret, fl, indent=4)
-
-    # with open(Parameters.GNBRfile+'original_entity_raw_name', 'rb') as fl:
-    #     full_entity_raw_name = pkl.load(fl)
-    # for k, v in entity_raw_name.items():
-    #     assert v in full_entity_raw_name[k]
-
-    # nlp = spacy.load("en_core_web_sm")
-    # type_set = set()
-    # for aa in range(36):
-    #     dependency_sen_dict = retieve_sentence_through_edgetype[aa]['manual']
-    #     tmp_dict = retieve_sentence_through_edgetype[aa]['auto']
-    #     dependencys = list(dependency_sen_dict.keys()) + list(tmp_dict.keys())
-    #     for dependency in dependencys:
-    #         dep_list = dependency.split(' ')
-    #         for sub_dep in dep_list:
-    #             sub_dep_list = sub_dep.split('|')
-    #             assert(len(sub_dep_list) == 3)
-    #             type_set.add(sub_dep_list[1])
-
-    # fine_dict = {}
-    # for k, v_dict in draft.items():
-
-    #     input = v_dict['in']
-    #     output = v_dict['out']
-    #     fine_dict[k] = {'in':input, 'out': input + ' ' + output}
-
-    # with open(f'generate_abstract/{args.target_split}_{args.reasonable_rate}_sentence_finetune.json', 'w') as fl:
-    #     json.dump(fine_dict, fl, indent=4)
+
 else:
     raise Exception('Wrong mode !!')

DiseaseAgnostic/processed_data/attack_edge_distmult_0.7random.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b0ccfbd4d67a60aeef746e45f3e322612f92d2f4ee28f4fe645a84f8284a226
+size 4014

DiseaseAgnostic/processed_data/target_0.7random.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ceda69a4136eb899e6ef21a7ff56ac00d75eac71b056181e7d816532f041634
+size 1214