# coding=utf-8 # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This Script is Modified for Natural Language Inference Datasets fine-tuning. All the datasets can be downloaded from this repo. """ import logging import os import sys import json from transformers.data.processors.utils import DataProcessor, InputExample, InputFeatures from transformers.file_utils import is_tf_available if is_tf_available(): import tensorflow as tf logger = logging.getLogger(__name__) def convert_examples_to_features(examples, tokenizer, max_length=512, task=None, label_list=None, output_mode=None, pad_on_left=False, pad_token=0, pad_token_segment_id=0, mask_padding_with_zero=True): is_tf_dataset = False if is_tf_available() and isinstance(examples, tf.data.Dataset): is_tf_dataset = True if task is not None: processor = glue_processors[task]() if label_list is None: label_list = processor.get_labels() logger.info("Using label list %s for task %s" % (label_list, task)) if output_mode is None: output_mode = glue_output_modes[task] logger.info("Using output mode %s for task %s" % (output_mode, task)) label_map = {label: i for i, label in enumerate(label_list)} features = [] for (ex_index, example) in enumerate(examples): if ex_index % 10000 == 0: logger.info("Writing example %d" % (ex_index)) if is_tf_dataset: example = processor.get_example_from_tensor_dict(example) example = processor.tfds_map(example) inputs = tokenizer.encode_plus( example.text_a, example.text_b, add_special_tokens=True, max_length=max_length, ) input_ids, token_type_ids = inputs["input_ids"], inputs["token_type_ids"] # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) # Zero-pad up to the sequence length. padding_length = max_length - len(input_ids) if pad_on_left: input_ids = ([pad_token] * padding_length) + input_ids attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask token_type_ids = ([pad_token_segment_id] * padding_length) + token_type_ids else: input_ids = input_ids + ([pad_token] * padding_length) attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length) token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length) assert len(input_ids) == max_length, "Error with input length {} vs {}".format(len(input_ids), max_length) assert len(attention_mask) == max_length, "Error with input length {} vs {}".format(len(attention_mask), max_length) assert len(token_type_ids) == max_length, "Error with input length {} vs {}".format(len(token_type_ids), max_length) if output_mode == "classification": label = label_map[example.label] elif output_mode == "regression": label = float(example.label) else: raise KeyError(output_mode) if ex_index < 5: logger.info("*** Example ***") logger.info("guid: %s" % (example.guid)) logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids])) logger.info("label: %s (id = %d)" % (example.label, label)) features.append( InputFeatures(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, label=label)) if is_tf_available() and is_tf_dataset: def gen(): for ex in features: yield ({'input_ids': ex.input_ids, 'attention_mask': ex.attention_mask, 'token_type_ids': ex.token_type_ids}, ex.label) return tf.data.Dataset.from_generator(gen, ({'input_ids': tf.int32, 'attention_mask': tf.int32, 'token_type_ids': tf.int32}, tf.int64), ({'input_ids': tf.TensorShape([None]), 'attention_mask': tf.TensorShape([None]), 'token_type_ids': tf.TensorShape([None])}, tf.TensorShape([]))) return features class SnliProcessor(DataProcessor): """Processor for the SNLI dataset (converted).""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['premise'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "train.jsonl")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "dev.jsonl")), "dev") def get_labels(self): """See base class.""" return ["e", "n", "c"] def _read_txt(self, dir): with open(dir, "r", encoding="utf-8") as f: lines = [] for line in f.readlines(): if sys.version_info[0] == 2: line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): dict_line = json.loads(line) guid = "%s-%s" % (set_type, i) label = dict_line['label'] text_a = dict_line['premise'].strip() text_b = dict_line['hypothesis'].strip() examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label) ) return examples class MnliProcessor(DataProcessor): """Processor for the MultiNLI data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['premise'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), "dev_matched") def get_labels(self): """See base class.""" return ["contradiction", "entailment", "neutral"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, line[0]) text_a = line[8] text_b = line[9] label = line[-1] examples.append( InputExample(guid=guid, text_a=text_b, text_b=text_a, label=label)) return examples class MnliMismatchedProcessor(MnliProcessor): """Processor for the MultiNLI Mismatched data set (GLUE version).""" def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/dev_mismatched.tsv")), "dev_matched") class ColaProcessor(DataProcessor): """Processor for the CoLA data set (GLUE version). """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence'].numpy().decode('utf-8'), None, str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") def get_labels(self): """See base class.""" return ["0", "1"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): guid = "%s-%s" % (set_type, i) text_a = line[3] label = line[1] examples.append( InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) return examples class CoodProcessor(DataProcessor): """Processor for the CoLA-ood data set. """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence'].numpy().decode('utf-8'), None, str(tensor_dict['label'].numpy())) def _read_txt(self, dir): with open(dir, "r", encoding="utf-8") as f: lines = [] for line in f.readlines(): if sys.version_info[0] == 2: line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "binary_train.txt")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "binary_dev.txt")), "dev") def get_labels(self): """See base class.""" return [0, 1] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): guid = "%s-%s" % (set_type, i) dict_line = eval(line) print(i) text_a = dict_line['text'] label = dict_line['label'] examples.append( InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) return examples class Sst2Processor(DataProcessor): """Processor for the SST-2 data set (GLUE version). """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence'].numpy().decode('utf-8'), None, str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/dev.tsv")), "dev") def get_labels(self): """See base class.""" return ["0", "1"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, i) text_a = line[0] label = line[1] examples.append( InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) return examples class StsbProcessor(DataProcessor): """Processor for the STS-B data set (GLUE version). """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/dev.tsv")), "dev") def get_labels(self): """See base class.""" return [None] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, line[0]) text_a = line[7] text_b = line[8] label = line[-1] examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples class QqpProcessor(DataProcessor): """Processor for the QQP data set (GLUE version). """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['question1'].numpy().decode('utf-8'), tensor_dict['question2'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/dev.tsv")), "dev") def get_labels(self): """See base class.""" return ["0", "1"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, line[0]) try: text_a = line[3] text_b = line[4] label = line[5] except IndexError: continue examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples class QnliProcessor(DataProcessor): """Processor for the QNLI data set (GLUE version). """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['question'].numpy().decode('utf-8'), tensor_dict['sentence'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev_matched") def get_labels(self): """See base class.""" return ["entailment", "not_entailment"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, line[0]) text_a = line[1] text_b = line[2] label = line[-1] examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples class RteProcessor(DataProcessor): """Processor for the RTE data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/dev.tsv")), "dev") def get_labels(self): """See base class.""" return ["entailment", "not_entailment"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, line[0]) text_a = line[1] text_b = line[2] label = line[-1] examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples class WnliProcessor(DataProcessor): """Processor for the WNLI data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") def get_labels(self): """See base class.""" return ["0", "1"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, line[0]) text_a = line[1] text_b = line[2] label = line[-1] examples.append( InputExample(guid=guid, text_a='', text_b=text_a, label=label)) return examples class PnliProcessor(DataProcessor): """Processor for the ConTRoL dataset (multi-sentence/paragraph/passage level). """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['context'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "train.jsonl")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "dev.jsonl")), "dev") def get_labels(self): """See base class.""" return ["c", "e", "n"] def _read_txt(self, dir): with open(dir, "r", encoding="utf-8") as f: lines = [] for line in f.readlines(): if sys.version_info[0] == 2: line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): dict_line = json.loads(line) guid = "%s-%s" % (set_type, i) label = dict_line['label'] text_a = dict_line['premise'].strip() text_b = dict_line['hypothesis'].strip() examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label) ) return examples """Below is the data reader for long/short segmentation of the ConTRoL data""" # def get_train_examples(self, data_dir): # """See base class.""" # return self._create_examples( # self._read_tsv(os.path.join(data_dir, "short/train.tsv")), "train") # def get_dev_examples(self, data_dir): # """See base class.""" # return self._create_examples( # self._read_tsv(os.path.join(data_dir, "short/dev.tsv")), "dev") # def get_labels(self): # """See base class.""" # return ["c", "e", "n"] # def _create_examples(self, lines, set_type): # """Creates examples for the training and dev sets.""" # examples = [] # for (i, line) in enumerate(lines): # if i == 0: # continue # if len(line) == 3: # guid = "%s-%s" % (set_type, line[0]) # text_a = line[0] # text_b = line[1] # label = line[-1][-1].lower() # examples.append( # InputExample(guid=guid, text_a=text_b, text_b=text_a, label=label)) # return examples class Qa2nliProcessor(DataProcessor): """Processor for the logiqa2nli data set.""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['premise_par_new'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "train.txt")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "dev.txt")), "dev") def get_labels(self): """See base class.""" return ['entailed', 'not entailed'] def _read_txt(self, dir): with open(dir, "r", encoding="utf-8") as f: lines = [] for line in f.readlines(): if sys.version_info[0] == 2: line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): dict_line = json.loads(line) guid = "%s-%s" % (set_type, i) label = dict_line['label'] text_a = "".join(_ for _ in dict_line['major_premise']) + " " + "".join(_ for _ in dict_line['minor_premise']) text_a = text_a.strip() text_b = dict_line['conclusion'].strip() examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label) ) return examples class SciProcessor(DataProcessor): """Processor for the SciTail data set.""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['premise'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "snli_format/train.txt")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "snli_format/dev.txt")), "dev") def get_labels(self): """See base class.""" return ["entailment", "neutral"] def _read_txt(self, dir): with open(dir, "r", encoding="utf-8") as f: lines = [] for line in f.readlines(): if sys.version_info[0] == 2: line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): dict_line = json.loads(line) guid = "%s-%s" % (set_type, i) label = dict_line['gold_label'] text_a = dict_line['sentence1'].strip() text_b = dict_line['sentence2'].strip() examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label) ) return examples class AnliProcessor(DataProcessor): """Processor for the ANLI data set.""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['premise'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "r3/train.jsonl")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "r3/dev.jsonl")), "dev") def get_labels(self): """See base class.""" return ["e", "n", "c"] def _read_txt(self, dir): with open(dir, "r", encoding="utf-8") as f: lines = [] for line in f.readlines(): if sys.version_info[0] == 2: line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): dict_line = json.loads(line) guid = "%s-%s" % (set_type, i) label = dict_line['label'] text_a = dict_line['premise'].strip() text_b = dict_line['hypothesis'].strip() examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label) ) return examples class QoodProcessor(DataProcessor): """Processor for the QNLI-ood data set.""" def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['premise'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "train.txt")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_txt(os.path.join(data_dir, "dev.txt")), "dev") def get_labels(self): """See base class.""" return ["entailment", "not_entailment"] def _read_txt(self, dir): with open(dir, "r", encoding="utf-8") as f: lines = [] for line in f.readlines(): if sys.version_info[0] == 2: line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): dict_line = json.loads(line) guid = "%s-%s" % (set_type, i) label = dict_line['label'] text_a = dict_line['question'].strip() text_b = dict_line['sentence'].strip() examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label) ) return examples class MrpcProcessor(DataProcessor): """Processor for the MRPC data set (GLUE version). """ def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), str(tensor_dict['label'].numpy())) def get_train_examples(self, data_dir): """See base class.""" logger.info("LOOKING AT {}".format(os.path.join(data_dir, "short/train.tsv"))) return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/train.tsv")), "train") def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( self._read_tsv(os.path.join(data_dir, "short/dev.tsv")), "dev") def get_labels(self): """See base class.""" return ["0", "1"] def _create_examples(self, lines, set_type): """Creates examples for the training and dev sets.""" examples = [] for (i, line) in enumerate(lines): if i == 0: continue guid = "%s-%s" % (set_type, i) text_a = line[3] text_b = line[4] label = line[0] examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples try: from scipy.stats import pearsonr, spearmanr from sklearn.metrics import matthews_corrcoef, f1_score, confusion_matrix _has_sklearn = True except (AttributeError, ImportError): _has_sklearn = False def is_sklearn_available(): return _has_sklearn #if _has_sklearn: def simple_accuracy(preds, labels): return (preds == labels).mean() def acc_and_f1(preds, labels): acc = simple_accuracy(preds, labels) f1 = f1_score(y_true=labels, y_pred=preds) return { "acc": acc, "f1": f1, "acc_and_f1": (acc + f1) / 2, } def pearson_and_spearman(preds, labels): pearson_corr = pearsonr(preds, labels)[0] spearman_corr = spearmanr(preds, labels)[0] return { "pearson": pearson_corr, "spearmanr": spearman_corr, "corr": (pearson_corr + spearman_corr) / 2, } def compute_metrics(task_name, preds, labels): assert len(preds) == len(labels) if task_name == "cola": return {"mcc": matthews_corrcoef(labels, preds)} elif task_name == "cood": return {"confusion matrix": confusion_matrix(preds, labels), "mcc": matthews_corrcoef(labels, preds), "f1 score": acc_and_f1(preds, labels)} elif task_name == "sst-2": return {"acc": simple_accuracy(preds, labels)} elif task_name == "mrpc": return acc_and_f1(preds, labels) elif task_name == "sts-b": return pearson_and_spearman(preds, labels) elif task_name == "qqp": return acc_and_f1(preds, labels) elif task_name == "mnli": return {"acc": simple_accuracy(preds, labels)} elif task_name == "mnli-mm": return {"acc": simple_accuracy(preds, labels)} elif task_name == "qnli": return {"acc": simple_accuracy(preds, labels)} elif task_name == "rte": return {"acc": simple_accuracy(preds, labels)} elif task_name == "wnli": return {"acc": simple_accuracy(preds, labels)} elif task_name == "hans": return {"acc": simple_accuracy(preds, labels)} elif task_name == "scitail": return {"acc": simple_accuracy(preds, labels)} elif task_name == "snli": return {"acc": simple_accuracy(preds, labels)} elif task_name == "qa2nli": return {"confusion matrix": confusion_matrix(preds, labels), "mcc": matthews_corrcoef(labels, preds), "f1 score": acc_and_f1(preds, labels)} elif task_name == "anli": return {"acc": simple_accuracy(preds, labels)} elif task_name == "pnli": return {"acc": simple_accuracy(preds, labels)} elif task_name == "qood": return {"acc": simple_accuracy(preds, labels)} else: raise KeyError(task_name) def xnli_compute_metrics(task_name, preds, labels): assert len(preds) == len(labels) if task_name == "xnli": return {"acc": simple_accuracy(preds, labels)} else: raise KeyError(task_name) tasks_num_labels = { "pnli": 3, "cola": 2, "cood": 2, "snli": 3, "mnli": 3, "mrpc": 2, "sst-2": 2, "sts-b": 1, "qqp": 2, "qnli": 2, "rte": 2, "wnli": 2, "qa2nli": 2, "scitail": 2, "anli": 3, "qood": 2, } processors = { "cola": ColaProcessor, "cood": CoodProcessor, "snli": SnliProcessor, "mnli": MnliProcessor, "mnli-mm": MnliMismatchedProcessor, "mrpc": MrpcProcessor, "sst-2": Sst2Processor, "sts-b": StsbProcessor, "qqp": QqpProcessor, "qnli": QnliProcessor, "rte": RteProcessor, "wnli": WnliProcessor, "pnli": PnliProcessor, "qa2nli": Qa2nliProcessor, "scitail": SciProcessor, "anli": AnliProcessor, "qood": QoodProcessor, } output_modes = { "cola": "classification", "cood": "classification", "mnli": "classification", "mnli-mm": "classification", "mrpc": "classification", "sst-2": "classification", "sts-b": "regression", "qqp": "classification", "qnli": "classification", "rte": "classification", "wnli": "classification", "pnli": "classification", "qa2nli": "classification", "scitail": "classification", "snli": "classification", "anli": "classification", "qood": "classification", }