Spaces:
Running
Running
| from evaluate import Evaluator, ALL_TASKS | |
| from baselines import * | |
| from alignscore.inference import Inferencer | |
| import time | |
| import json | |
| import os | |
| from argparse import ArgumentParser | |
| SAVE_ALL_TABLES = True | |
| SAVE_AND_PRINT_TIMER = False | |
| class Timer(): | |
| def __init__(self) -> None: | |
| self.t0 = time.time() | |
| self.save_path = 'exp_results/time.json' | |
| def finish(self, display_name): | |
| t1 = time.time() | |
| time_pass = t1 - self.t0 | |
| if SAVE_AND_PRINT_TIMER: | |
| print(f"Evalautor {display_name} finished in {time_pass} secs.") | |
| with open(self.save_path, 'a', encoding='utf8') as f: | |
| json.dump({display_name: time_pass}, f) | |
| f.write('\n') | |
| def eval_ctc(model_type, tasks=ALL_TASKS): | |
| ctc_scorer = CTCScorer(model_type) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=ctc_scorer.score, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/CTC-{model_type}" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"CTC-{model_type}") | |
| def eval_simcse(model_type, device, tasks=ALL_TASKS): | |
| simcse_scorer = SimCSEScorer(model_type, device) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=simcse_scorer.score, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/{model_type.split('/')[-1]}_f" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"{model_type.split('/')[-1]}_f") | |
| def eval_bleurt(checkpoint, tasks=ALL_TASKS): | |
| bleurt_scorer = BleurtScorer(checkpoint) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=bleurt_scorer.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/BLEURT" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"BLEURT") | |
| def eval_bertscore(model_type, device, batch_size, tasks=ALL_TASKS): | |
| bertscore_scorer = BertScoreScorer(model_type=model_type, metric='f1', device=device, batch_size=batch_size) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=bertscore_scorer.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/bertscore_{model_type.replace('/', '-')}_f" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"bertscore_{model_type.replace('/', '-')}_f") | |
| def eval_bartscore(checkpoint, device, tasks=ALL_TASKS): | |
| bartscore_scorer = BartScoreScorer(checkpoint, device) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=bartscore_scorer.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/bartscore-{checkpoint.replace('/','-')}" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"bartscore-{checkpoint.replace('/','-')}") | |
| ### Below are Baselines for SummaC | |
| def eval_mnli(model="roberta-large-mnli", device='cuda:0', tasks=ALL_TASKS): | |
| mnli_scorer = MNLIScorer(model=model, device=device) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=mnli_scorer.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/mnli-{model}" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"mnli-{model}") | |
| def eval_ner(tasks=ALL_TASKS): | |
| ner_scorer = NERScorer() | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=ner_scorer.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/NER" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"NER") | |
| def eval_unieval(tasks=ALL_TASKS, device='cuda:0'): | |
| unieval = UniEvalScorer(task='fact', device=device) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=unieval.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/UniEval" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"UniEval") | |
| def eval_feqa(tasks=ALL_TASKS): | |
| feqa = FEQAScorer() | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=feqa.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/FEQA" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"FEQA") | |
| def eval_questeval(tasks=ALL_TASKS): | |
| questeval = QuestEvalScorer() | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=questeval.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/QuestEval" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"QuestEval") | |
| def eval_qafacteval(tasks=ALL_TASKS, device='cuda:0'): | |
| import os, sys | |
| warning("using conda env qaeval!!!") | |
| qafacteval = QAFactEvalScorer(device=device, model_folder=os.path.abspath('../BaselineForNLGEval/QAFactEval/models')) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=qafacteval.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/QAFactEval" | |
| evaluator.evaluate() | |
| def eval_dae(tasks=ALL_TASKS, model_dir=None, device=0): | |
| dae = DAEScorer(model_dir=model_dir, device=device) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=dae.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/DAE" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"DAE") | |
| def eval_bleu(tasks=ALL_TASKS, n_grams=1): | |
| bleu = BLEUScorer(n_grams=n_grams) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=bleu.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/BLEU-{n_grams}" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"BLEU-{n_grams}") | |
| def eval_rouge(tasks=ALL_TASKS, rouge_type='1'): | |
| rouge = ROUGEScorer(rouge_type=rouge_type) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=rouge.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/ROUGE-{rouge_type}" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"ROUGE-{rouge_type}") | |
| def eval_factcc(script_path, test_data_path,result_path, tasks=ALL_TASKS): | |
| factcc = FactCCScorer(script_path=script_path, test_data_path=test_data_path, result_path=result_path) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=factcc.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/FactCC" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"FactCC") | |
| def eval_blanc(tasks=ALL_TASKS, device='cuda:0', batch_size=64): | |
| blanc = BLANCScorer(device=device, batch_size=batch_size) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=blanc.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/BLANC" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"BLANC") | |
| def eval_summac(tasks=ALL_TASKS, summac_type='conv', device='cuda:0'): | |
| summac = SummaCScorer(summac_type=summac_type, device=device) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=summac.scorer, save_all_tables=SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"baselines/SummaC-{summac_type}" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(f"SummaC-{summac_type}") | |
| def eval_align_nlg(ckpt_path, comment='', base_model='roberta-large', batch_size=32, device='cuda:0', tasks=ALL_TASKS, nlg_eval_mode='nli_sp'): | |
| align = Inferencer(ckpt_path=ckpt_path, model=base_model, batch_size=batch_size, device=device) | |
| if 'smart' in nlg_eval_mode: | |
| align.smart_type = nlg_eval_mode | |
| else: | |
| align.nlg_eval_mode = nlg_eval_mode | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=align.nlg_eval, save_all_tables=SAVE_ALL_TABLES) | |
| name = f'AlignScore-{nlg_eval_mode}-{base_model}' | |
| if comment: | |
| name += '_' + comment | |
| evaluator.result_save_name = f"align_eval/{name}" | |
| timer = Timer() | |
| evaluator.evaluate() | |
| timer.finish(name) | |
| def eval_gptscore(api_key, gpt_model='davinci003', tasks=ALL_TASKS): | |
| gptscore = GPTScoreScorer(api_key=api_key, gpt_model=gpt_model) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=gptscore.scorer, is_save_all_tables=IS_SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"nlg_eval_fact/baselines/GPTScore-{gpt_model}" | |
| evaluator.evaluate() | |
| def eval_chatgptluo2023(api_key, chat_model='gpt-3.5-turbo', tasks=['qags_cnndm']): | |
| chatgpt = ChatGPTLuo2023Scorer(task=tasks, api_key=api_key, chat_model=chat_model) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=chatgpt.scorer, is_save_all_tables=IS_SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"nlg_eval_fact/baselines/ChatGPTLuo2023-{chat_model}" | |
| evaluator.evaluate() | |
| def eval_chatgptgao2023(api_key, chat_model='gpt-3.5-turbo', tasks=['qags_cnndm']): | |
| chatgpt = ChatGPTGao2023Scorer(task=tasks, api_key=api_key, chat_model=chat_model) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=chatgpt.scorer, is_save_all_tables=IS_SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"nlg_eval_fact/baselines/ChatGPTGao2023-{chat_model}" | |
| evaluator.evaluate() | |
| def eval_chatgptyichen2023(api_key, chat_model='gpt-3.5-turbo', tasks=['qags_cnndm']): | |
| chatgpt = ChatGPTYiChen2023Scorer(task=tasks, api_key=api_key, chat_model=chat_model) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=chatgpt.scorer, is_save_all_tables=IS_SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"nlg_eval_fact/baselines/ChatGPTYiChen2023-{chat_model}" | |
| evaluator.evaluate() | |
| def eval_chatgptshiqichen2023(api_key, chat_model='gpt-3.5-turbo', tasks=['qags_cnndm']): | |
| chatgpt = ChatGPTShiqiChen2023Scorer(task=tasks, api_key=api_key, chat_model=chat_model) | |
| evaluator = Evaluator(eval_tasks=tasks, align_func=chatgpt.scorer, is_save_all_tables=IS_SAVE_ALL_TABLES) | |
| evaluator.result_save_name = f"nlg_eval_fact/baselines/ChatGPTShiqiChen2023-{chat_model}" | |
| evaluator.evaluate() | |
| def run_benchmarks(args, argugment_error): | |
| os.makedirs('exp_results/baselines', exist_ok=True) | |
| os.makedirs('exp_results/align_eval', exist_ok=True) | |
| if args.alignscore: | |
| if not all((args.alignscore_model, args.alignscore_ckpt, args.alignscore_eval_mode)): | |
| argugment_error('--alignscore-model, --alignscore-model, and --alignscore-ckpt must be specified to run AlignScore') | |
| eval_align_nlg( | |
| nlg_eval_mode=args.alignscore_eval_mode, | |
| ckpt_path=args.alignscore_ckpt, | |
| base_model=args.alignscore_model, | |
| device=args.device, tasks=args.tasks, | |
| comment=args.alignscore_comment | |
| ) | |
| if args.ctc: | |
| if not args.ctc_type: | |
| argugment_error('--ctc-type must be specified to run CTC baseline') | |
| for type in args.ctc_type: | |
| eval_ctc(type, tasks=args.tasks) | |
| if args.simcse: | |
| if not args.simcse_ckpt: | |
| argugment_error('--simcse-ckpt must be specified to run SimCSE baseline') | |
| for ckpt in args.simcse_ckpt: | |
| eval_simcse(ckpt, device=args.device, tasks=args.tasks) | |
| if args.bleurt: | |
| if not args.bleurt_ckpt: | |
| argugment_error('--bleurt-ckpt must be specified to run BLEURT baseline') | |
| eval_bleurt(args.bleurt_ckpt, tasks=args.tasks) | |
| if args.bertscore: | |
| if not args.bertscore_ckpt or not args.bertscore_batch_size: | |
| argugment_error('--bertscore-ckpt and --bertscore-batch-size must be specified to run BERTScore baseline') | |
| for ckpt in args.bertscore_ckpt: | |
| eval_bertscore(ckpt, device=args.device, tasks=args.tasks, batch_size=args.bertscore_batch_size) | |
| if args.bartscore: | |
| if not args.bartscore_ckpt: | |
| argugment_error('--bartscore-ckpt must be specified to run BARTScore baseline') | |
| for ckpt in args.bartscore_ckpt: | |
| eval_bartscore(ckpt, device=args.device, tasks=args.tasks) | |
| if args.mnli: | |
| if not args.mnli_ckpt: | |
| argugment_error('--mnli-ckpt must be specified to run MNLI baseline') | |
| for ckpt in args.mnli_ckpt: | |
| eval_mnli(model=ckpt, device=args.device, tasks=args.tasks) | |
| if args.ner: | |
| eval_ner(tasks=args.tasks) | |
| if args.unieval: | |
| eval_unieval(tasks=args.tasks, device=args.device) | |
| if args.feqa: | |
| eval_feqa(tasks=args.tasks) | |
| if args.questeval: | |
| eval_questeval(tasks=args.tasks) | |
| if args.qafacteval: | |
| eval_qafacteval(tasks=args.tasks) | |
| if args.bleu: | |
| if not args.bleu_ngram: | |
| argugment_error('--bleu-ngram must be specified to run BLEU baseline') | |
| for n in args.bleu_ngram: | |
| eval_bleu(tasks=args.tasks, n_grams=n) | |
| if args.rouge: | |
| if not args.rouge_type: | |
| argugment_error('--rouge-type must be specified to run ROUGE baseline') | |
| for type in args.rouge_type: | |
| eval_rouge(tasks=args.tasks, rouge_type=type) | |
| if args.dae: | |
| if not args.dae_ckpt: | |
| argugment_error('--dae-ckpt must be specified to run DAE baseline') | |
| eval_dae(tasks=args.tasks, model_dir=os.path.abspath(args.dae_ckpt)) | |
| if args.factcc: | |
| if not all((args.factcc_script, args.factcc_test_data, args.factcc_result_path)): | |
| argugment_error('--factcc-script, --factcc-test-data, and --factcc-result-path must be specified to run FactCC baseline') | |
| eval_factcc( | |
| tasks=args.tasks, | |
| script_path=os.path.abspath(args.factcc_script), | |
| test_data_path=os.path.abspath(args.factcc_test_data), | |
| result_path=os.path.abspath(args.factcc_result_path) | |
| ) | |
| if args.blanc: | |
| if not args.blanc_batch_size: | |
| argugment_error('--blanc-batch-size must be specified to run BLANC baseline') | |
| eval_blanc(tasks=args.tasks, device=args.device, batch_size=args.blanc_batch_size) | |
| if args.summac: | |
| if not args.summac_type: | |
| argugment_error('--summac-type must be specified to run SummaC baseline') | |
| for type in args.summac_type: | |
| eval_summac(tasks=args.tasks, device=args.device, summac_type=type) | |
| if __name__ == "__main__": | |
| FACT_EVAL_TASKS = ['summac', 'true','xsumfaith', 'summeval', 'qags_xsum', 'qags_cnndm', 'newsroom', 'rank19', 'frank', 'samsum'] | |
| parser = ArgumentParser() | |
| parser.add_argument('--tasks', nargs='+', type=str, default=FACT_EVAL_TASKS, choices=FACT_EVAL_TASKS) | |
| parser.add_argument('--device', type=str, default='cuda:0') | |
| parser.add_argument('--timer', action='store_true', help='Time all metric runs') | |
| alignscore_parser = parser.add_argument_group('AlignScore') | |
| alignscore_parser.add_argument('--alignscore', action='store_true', help='Run AlignScore benchmark') | |
| alignscore_parser.add_argument('--alignscore-model', type=str, choices=['roberta-base', 'roberta-large']) | |
| alignscore_parser.add_argument('--alignscore-ckpt', type=str) | |
| alignscore_parser.add_argument( | |
| '--alignscore-eval-mode', | |
| type=str, | |
| choices=['bin', 'bin_sp', 'nli', 'nli_sp', 'reg', 'reg_sp', 'smart-n', 'smart-l'], | |
| default='nli_sp' | |
| ) | |
| alignscore_parser.add_argument('--alignscore-comment', type=str, default='') | |
| ctc_parser = parser.add_argument_group('Baseline - CTC') | |
| ctc_parser.add_argument('--ctc', action='store_true', help='Run CTC baseline') | |
| ctc_parser.add_argument( | |
| '--ctc-type', | |
| nargs='*', | |
| type=str, | |
| choices=['D-cnndm', 'E-roberta', 'R-cnndm'], | |
| default=['D-cnndm'] | |
| ) | |
| simcse_parser = parser.add_argument_group('Baseline - SimCSE') | |
| simcse_models = [ | |
| 'princeton-nlp/unsup-simcse-bert-base-uncased', | |
| 'princeton-nlp/unsup-simcse-bert-large-uncased', | |
| 'princeton-nlp/unsup-simcse-roberta-base', | |
| 'princeton-nlp/unsup-simcse-roberta-large', | |
| 'princeton-nlp/sup-simcse-bert-base-uncased', | |
| 'princeton-nlp/sup-simcse-bert-large-uncased', | |
| 'princeton-nlp/sup-simcse-roberta-base', | |
| 'princeton-nlp/sup-simcse-roberta-large' | |
| ] | |
| simcse_parser.add_argument('--simcse', action='store_true', help='Run SimCSE baseline') | |
| simcse_parser.add_argument( | |
| '--simcse-ckpt', | |
| nargs='*', | |
| type=str, | |
| choices=simcse_models, | |
| default=['princeton-nlp/sup-simcse-roberta-large'] | |
| ) | |
| bleurt_parser = parser.add_argument_group('Baseline - BLEURT') | |
| bleurt_parser.add_argument('--bleurt', action='store_true', help='Run BLEURT baseline') | |
| bleurt_parser.add_argument('--bleurt-ckpt', type=str) | |
| bertscore_parser = parser.add_argument_group('Baseline - BERTScore') | |
| bertscore_parser.add_argument('--bertscore', action='store_true', help='Run BERTScore baseline') | |
| bertscore_parser.add_argument( | |
| '--bertscore-ckpt', | |
| nargs='*', | |
| type=str, | |
| default=['microsoft/deberta-xlarge-mnli'] | |
| ) | |
| bertscore_parser.add_argument('--bertscore-batch-size', type=int, default=16) | |
| bartscore_parser = parser.add_argument_group( | |
| 'Baseline - BARTScore', | |
| description='Please clone https://github.com/neulab/BARTScore to baselines/BARTScore.' | |
| ) | |
| bartscore_parser.add_argument('--bartscore', action='store_true', help='Run BARTScore baseline') | |
| bartscore_parser.add_argument( | |
| '--bartscore-ckpt', | |
| type=str, | |
| nargs='*', | |
| default=['facebook/bart-large-cnn'] | |
| ) | |
| mnli_parser = parser.add_argument_group('Baseline - MNLI') | |
| mnli_parser.add_argument('--mnli', action='store_true', help='Run MNLI baseline') | |
| mnli_parser.add_argument( | |
| '--mnli-ckpt', | |
| nargs='*', | |
| type=str, | |
| default=['roberta-large-mnli'] | |
| ) | |
| ner_parser = parser.add_argument_group( | |
| 'Baseline - NER overlap', | |
| description='Please clone https://github.com/tingofurro/summac to baselines/summac.' | |
| ) | |
| ner_parser.add_argument('--ner', action='store_true', help='Run NER overlap baseline') | |
| unieval_parser = parser.add_argument_group( | |
| 'Baseline - UniEval', | |
| description='Please clone https://github.com/maszhongming/UniEval to baselines/UniEval.' | |
| ) | |
| unieval_parser.add_argument('--unieval', action='store_true', help='Run UniEval baseline') | |
| feqa_parser = parser.add_argument_group( | |
| 'Baseline - FEQA', | |
| description='Please clone https://github.com/esdurmus/feqa to baselines/feqa' | |
| ) | |
| feqa_parser.add_argument('--feqa', action='store_true', help='Run FEQA baseline') | |
| questeval_parser = parser.add_argument_group( | |
| 'Baseline - QuestEval', | |
| description='Please clone https://github.com/ThomasScialom/QuestEval to baselines/QuestEval.' | |
| ) | |
| questeval_parser.add_argument('--questeval', action='store_true', help='Run QuestEval baseline') | |
| qafacteval_parser = parser.add_argument_group( | |
| 'Baseline - QAFactEval', | |
| description='Please clone https://github.com/salesforce/QAFactEval to baselines/QAFactEval.' | |
| ) | |
| qafacteval_parser.add_argument('--qafacteval', action='store_true', help='Run QAFactEval baseline') | |
| bleu_parser = parser.add_argument_group('Baseline - BLEU') | |
| bleu_parser.add_argument('--bleu', action='store_true', help='Run BLEU baseline') | |
| bleu_parser.add_argument( | |
| '--bleu-ngram', | |
| nargs='*', | |
| type=int, | |
| choices=[1, 2, 3, 4], | |
| default=[1, 2, 3, 4] | |
| ) | |
| rouge_parser = parser.add_argument_group('Baseline - ROUGE') | |
| rouge_parser.add_argument('--rouge', action='store_true', help='Run ROUGE baseline') | |
| rouge_parser.add_argument( | |
| '--rouge-type', | |
| nargs='*', | |
| type=str, | |
| choices=['1', '2', 'l'], | |
| default=['1', '2', 'l'] | |
| ) | |
| dae_parser = parser.add_argument_group('Baseline - DAE') | |
| dae_parser.add_argument('--dae', action='store_true', help='Run DAE baseline') | |
| dae_parser.add_argument('--dae-ckpt', type=str) | |
| factcc_parser = parser.add_argument_group('Baseline - FactCC') | |
| factcc_parser.add_argument('--factcc', action='store_true', help='Run FactCC baseline') | |
| factcc_parser.add_argument('--factcc-script', type=str) | |
| factcc_parser.add_argument('--factcc-test-data', type=str) | |
| factcc_parser.add_argument('--factcc-result-path', type=str) | |
| blanc_parser = parser.add_argument_group('Baseline - BLANC') | |
| blanc_parser.add_argument('--blanc', action='store_true', help='Run BLANC baseline') | |
| blanc_parser.add_argument('--blanc-batch-size', type=int, default=64) | |
| summac_parser = parser.add_argument_group( | |
| 'Baseline - SummaC', | |
| description='Please clone https://github.com/tingofurro/summac to baselines/summac.' | |
| ) | |
| summac_parser.add_argument('--summac', action='store_true', help='Run SummaC baseline') | |
| summac_parser.add_argument('--summac-type', nargs='*', type=str, choices=['conv', 'zs'], default=['conv', 'zs']) | |
| args = parser.parse_args() | |
| if args.timer: | |
| SAVE_AND_PRINT_TIMER = True | |
| def argugment_error(msg): | |
| parser.error(msg) | |
| run_benchmarks(args, argugment_error) | |