| from inputs.fields.field import Field | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| class MapTokenField(Field): | |
| """Map token field: preocess maping tokens | |
| """ | |
| def __init__(self, namespace, vocab_namespace, source_key, is_counting=True): | |
| """This function sets namesapce of field, vocab namespace for indexing, dataset source key | |
| Arguments: | |
| namespace {str} -- namesapce of field, counter namespace if constructing a counter | |
| vocab_namespace {str} -- vocab namespace for indexing | |
| source_key {str} -- indicate key in text data | |
| Keyword Arguments: | |
| is_counting {bool} -- decide constructing a counter or not (default: {True}) | |
| """ | |
| super().__init__() | |
| self.namespace = str(namespace) | |
| self.counter_namespace = str(namespace) | |
| self.vocab_namespace = str(vocab_namespace) | |
| self.source_key = str(source_key) | |
| self.is_counting = is_counting | |
| def count_vocab_items(self, counter, sentences): | |
| """This function counts dict's values in sentences, | |
| then update counter, each sentence is a dict | |
| Arguments: | |
| counter {dict} -- counter | |
| sentences {list} -- text content after preprocessing, list of dict | |
| """ | |
| if self.is_counting: | |
| for sentence in sentences: | |
| for value in sentence[self.source_key].values(): | |
| counter[self.counter_namespace][str(value)] += 1 | |
| logger.info("Count sentences {} to update counter namespace {} successfully.".format( | |
| self.source_key, self.counter_namespace)) | |
| def index(self, instance, vocab, sentences): | |
| """This function indexes token using vocabulary, then update instance | |
| Arguments: | |
| instance {dict} -- numerical represenration of text data | |
| vocab {Vocabulary} -- vocabulary | |
| sentences {list} -- text content after preprocessing | |
| """ | |
| for sentence in sentences: | |
| instance[self.namespace].append({ | |
| key: vocab.get_token_index(value, self.vocab_namespace) | |
| for key, value in sentence[self.source_key].items() | |
| }) | |
| logger.info("Index sentences {} to construct instance namespace {} successfully.".format( | |
| self.source_key, self.namespace)) | |