File size: 5,681 Bytes
6fd73cf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 |
import pandas as pd
import numpy as np
import torch
import zipfile
import os
from transformers import BertTokenizer, BertForSequenceClassification
import contractions
import re
import nltk
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
from nltk.tokenize import word_tokenize
# Load pre-trained BERT model and tokenizer
# def load_model():
# model_name = "./bert_fine_tuned/bert_fine_tuned"
# tokenizer = BertTokenizer.from_pretrained('./bert_fine_tuned/bert_tokens')
# model = BertForSequenceClassification.from_pretrained(model_name)
# return model, tokenizer
def load_model():
model_name = "azrai99/bert-skills-extraction"
model = BertForSequenceClassification.from_pretrained(model_name)
tokenizer = BertTokenizer.from_pretrained(model_name)
return model,tokenizer
def clean(desc):
desc = contractions.fix(desc)
desc = re.sub("[!@.$\'\'':()]", "", desc)
return desc
def extract_POS(tagged):
#pattern 1
grammar1 = ('''Noun Phrases: {<DT>?<JJ>*<NN|NNS|NNP>+}''')
chunkParser = nltk.RegexpParser(grammar1)
tree1 = chunkParser.parse(tagged)
# typical noun phrase pattern appending to be concatted later
g1_chunks = []
for subtree in tree1.subtrees(filter=lambda t: t.label() == 'Noun Phrases'):
g1_chunks.append(subtree)
#pattern 2
grammar2 = ('''NP2: {<IN>?<JJ|NN>*<NNS|NN>} ''')
chunkParser = nltk.RegexpParser(grammar2)
tree2 = chunkParser.parse(tagged)
# variation of a noun phrase pattern to be pickled for later analyses
g2_chunks = []
for subtree in tree2.subtrees(filter=lambda t: t.label() == 'NP2'):
g2_chunks.append(subtree)
#pattern 3
grammar3 = (''' VS: {<VBG|VBZ|VBP|VBD|VB|VBN><NNS|NN>*}''')
chunkParser = nltk.RegexpParser(grammar3)
tree3 = chunkParser.parse(tagged)
# verb-noun pattern appending to be concatted later
g3_chunks = []
for subtree in tree3.subtrees(filter=lambda t: t.label() == 'VS'):
g3_chunks.append(subtree)
# pattern 4
# any number of a singular or plural noun followed by a comma followed by the same noun, noun, noun pattern
grammar4 = ('''Commas: {<NN|NNS>*<,><NN|NNS>*<,><NN|NNS>*} ''')
chunkParser = nltk.RegexpParser(grammar4)
tree4 = chunkParser.parse(tagged)
# common pattern of listing skills appending to be concatted later
g4_chunks = []
for subtree in tree4.subtrees(filter=lambda t: t.label() == 'Commas'):
g4_chunks.append(subtree)
return g1_chunks, g2_chunks, g3_chunks, g4_chunks
def tokenize_and_tag(desc):
tokens = nltk.word_tokenize(desc.lower())
filtered_tokens = [w for w in tokens if not w in stop_words]
tagged = nltk.pos_tag(filtered_tokens)
return tagged
def training_set(chunks):
'''creates a dataframe that easily parsed with the chunks data '''
df = pd.DataFrame(chunks)
df.fillna('X', inplace = True)
train = []
for row in df.values:
phrase = ''
for tup in row:
# needs a space at the end for seperation
phrase += tup[0] + ' '
phrase = ''.join(phrase)
# could use padding tages but encoder method will provide during
# tokenizing/embeddings; X can replace paddding for now
train.append( phrase.replace('X', '').strip())
df['phrase'] = train
return df.phrase
def strip_commas(df):
'''create new series of individual n-grams'''
grams = []
for sen in df:
sent = sen.split(',')
for word in sent:
grams.append(word)
return pd.Series(grams)
def generate_phrases(desc):
tagged = tokenize_and_tag(desc)
g1_chunks, g2_chunks, g3_chunks, g4_chunks = extract_POS(tagged)
c = training_set(g4_chunks)
separated_chunks4 = strip_commas(c)
phrases = pd.concat([training_set(g1_chunks),
training_set(g2_chunks),
training_set(g3_chunks),
separated_chunks4],
ignore_index = True )
return phrases
def get_predictions(desc, model, tokenizer, threshold=0.6, return_probabilities=False):
# Clean
desc = clean(desc)
phrases = generate_phrases(desc).tolist()
phrases = [phrase.strip() for phrase in phrases]
print(phrases)
# Tokenize and prepare phrases for the model
inputs = tokenizer(phrases, return_tensors="pt", truncation=True, padding=True)
model,tokenizer = load_model()
# Perform inference
with torch.no_grad():
outputs = model(**inputs)
# Get predicted probabilities
probs = torch.nn.functional.softmax(outputs.logits, dim=1)
# Get predicted classes based on the threshold
predictions = (probs[:, 1] > threshold).to(torch.int32)
# Return predicted skills as a list
out = pd.DataFrame({'Phrase': phrases, 'Class': predictions})
skills = out.loc[out['Class'] == 1]
return skills['Phrase'].unique().tolist()
# # Return predicted skills and probabilities as lists
# out = pd.DataFrame({'Phrase': phrases, 'Class': predictions, 'Probability': probs[:, 1]})
# skills = out.loc[out['Class'] == 1]
# if return_probabilities:
# return skills['Phrase'].tolist(), skills['Probability'].tolist()
# else:
# return skills['Phrase'].tolist()
def get_predictions_excel(filename):
"""description column must be titled Job Desc"""
df = pd.read_csv(filename)
df['Extracted skills'] = df['Job Description'].apply(lambda x: get_predictions(x))
return df.to_csv('extracted.csv') |