add bog files
Browse files- .gitignore +2 -0
- README.md +6 -4
- functions.py +86 -0
- main.py +85 -0
.gitignore
ADDED
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
1 |
+
__pycache__
|
2 |
+
data/
|
README.md
CHANGED
@@ -1,9 +1,11 @@
|
|
1 |
-
---
|
2 |
-
license: cc-by-4.0
|
3 |
-
---
|
4 |
-
|
5 |
# Libérez Adam
|
6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
|
8 |
## Objectifs
|
9 |
|
|
|
|
|
|
|
|
|
|
|
1 |
# Libérez Adam
|
2 |
|
3 |
+
TD;DR
|
4 |
+
|
5 |
+
```
|
6 |
+
pip install -r requirements.txt
|
7 |
+
python3 main.py
|
8 |
+
```
|
9 |
|
10 |
## Objectifs
|
11 |
|
functions.py
ADDED
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
import numpy as np
|
3 |
+
import scipy
|
4 |
+
|
5 |
+
# Define the hyperparameters
|
6 |
+
num_layers = 2
|
7 |
+
batch_size = 32
|
8 |
+
hidden_dim = 256
|
9 |
+
|
10 |
+
def random_rotation(inputs):
|
11 |
+
angle = np.random.uniform(-180, 180)
|
12 |
+
inputs = scipy.ndimage.rotate(inputs, angle, reshape=False)
|
13 |
+
return inputs
|
14 |
+
|
15 |
+
def random_scaling(inputs):
|
16 |
+
scale = np.random.uniform(0.8, 1.2)
|
17 |
+
inputs = scipy.ndimage.zoom(inputs, scale)
|
18 |
+
return inputs
|
19 |
+
|
20 |
+
def random_translation(inputs):
|
21 |
+
shift = np.random.uniform(-0.2, 0.2)
|
22 |
+
inputs = scipy.ndimage.shift(inputs, shift)
|
23 |
+
return inputs
|
24 |
+
|
25 |
+
def random_shearing(inputs):
|
26 |
+
shear = np.random.uniform(-0.2, 0.2)
|
27 |
+
inputs = scipy.ndimage.shear(inputs, shear)
|
28 |
+
return inputs
|
29 |
+
|
30 |
+
def random_flipping(inputs):
|
31 |
+
inputs = scipy.ndimage.flip(inputs, axis=1)
|
32 |
+
return inputs
|
33 |
+
|
34 |
+
def data_augmentation(inputs):
|
35 |
+
# Apply random rotation
|
36 |
+
inputs = random_rotation(inputs)
|
37 |
+
# Apply random scaling
|
38 |
+
inputs = random_scaling(inputs)
|
39 |
+
# Apply random translation
|
40 |
+
inputs = random_translation(inputs)
|
41 |
+
# Apply random shearing
|
42 |
+
inputs = random_shearing(inputs)
|
43 |
+
# Apply random flipping
|
44 |
+
inputs = random_flipping(inputs)
|
45 |
+
return inputs
|
46 |
+
|
47 |
+
def evaluate(model, test_data, hyperparameters, recurrent_network=False, pre_trained_model=False, fine_tuning=False):
|
48 |
+
# Use GPU for training if available
|
49 |
+
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
50 |
+
|
51 |
+
# Define the hidden state
|
52 |
+
hidden = (torch.zeros(num_layers, batch_size, hidden_dim).to(device),
|
53 |
+
torch.zeros(num_layers, batch_size, hidden_dim).to(device))
|
54 |
+
model.eval()
|
55 |
+
with torch.no_grad():
|
56 |
+
correct = 0
|
57 |
+
total = 0
|
58 |
+
for data in test_data:
|
59 |
+
inputs, labels = data
|
60 |
+
# Use data augmentation
|
61 |
+
inputs = data_augmentation(inputs)
|
62 |
+
# Use GPU for training
|
63 |
+
inputs = inputs.to(device)
|
64 |
+
labels = labels.to(device)
|
65 |
+
# Use recurrent network
|
66 |
+
if recurrent_network:
|
67 |
+
outputs = model(inputs, hidden)
|
68 |
+
else:
|
69 |
+
outputs = model(inputs)
|
70 |
+
# Use pre-trained model
|
71 |
+
if pre_trained_model:
|
72 |
+
outputs = model.forward_from_pretrained(inputs)
|
73 |
+
# Use fine-tuning
|
74 |
+
if fine_tuning:
|
75 |
+
outputs = model.fine_tune(inputs, hyperparameters)
|
76 |
+
_, predicted = torch.max(outputs.data, 1)
|
77 |
+
total += labels.size(0)
|
78 |
+
correct += (predicted == labels).sum().item()
|
79 |
+
accuracy = 100 * correct / total
|
80 |
+
return accuracy
|
81 |
+
|
82 |
+
def adjust_learning_rate(optimizer, epoch):
|
83 |
+
"""Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
|
84 |
+
lr = 0.001 * (0.1 ** (epoch // 30))
|
85 |
+
for param_group in optimizer.param_groups:
|
86 |
+
param_group['lr'] = lr
|
main.py
ADDED
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
import torch.nn as nn
|
3 |
+
import torch.optim as optim
|
4 |
+
from torchtext import data
|
5 |
+
from gensim.corpora import WikiCorpus
|
6 |
+
from transformers import GPT2Tokenizer, GPT2Model
|
7 |
+
from functions import *
|
8 |
+
|
9 |
+
# Define the model
|
10 |
+
# class GPT(nn.Module):
|
11 |
+
# def __init__(self, vocab_size, embedding_dim, hidden_dim, num_layers):
|
12 |
+
# super().__init__()
|
13 |
+
# self.embedding = nn.Embedding(vocab_size, embedding_dim)
|
14 |
+
# self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers, batch_first=True)
|
15 |
+
# self.fc = nn.Linear(hidden_dim, vocab_size)
|
16 |
+
# self.gpt2 = model
|
17 |
+
|
18 |
+
# def forward(self, x):
|
19 |
+
# # Embed the input
|
20 |
+
# x = self.embedding(x)
|
21 |
+
# # Pass through the GPT2 model
|
22 |
+
# x = self.gpt2(x)
|
23 |
+
# # Pass through the LSTM
|
24 |
+
# x, _ = self.lstm(x)
|
25 |
+
# # Pass through the fully connected layer
|
26 |
+
# x = self.fc(x)
|
27 |
+
# return x
|
28 |
+
|
29 |
+
# Load the GPT2 model
|
30 |
+
print('load gpt2 model')
|
31 |
+
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
|
32 |
+
model = GPT2Model.from_pretrained('gpt2')
|
33 |
+
|
34 |
+
# Load the data
|
35 |
+
print('load custom data')
|
36 |
+
# wiki_corpus_en = WikiCorpus('data/enwiki-latest-pages-articles.xml.bz2')
|
37 |
+
wiki_corpus_fr = WikiCorpus('data/frwiki-latest-pages-articles.xml.bz2')
|
38 |
+
# stackoverflow_corpus = data.TabularDataset('data/stackoverflow.csv', format='csv', fields=['text'])
|
39 |
+
|
40 |
+
# Preprocess the data
|
41 |
+
print('Preprocess the data')
|
42 |
+
# wiki_data_en = [text for text in wiki_corpus_en]
|
43 |
+
wiki_data_fr = [text for text in wiki_corpus_fr]
|
44 |
+
# stackoverflow_data = [text for text in stackoverflow_corpus]
|
45 |
+
|
46 |
+
# Convert the data to a format compatible with PyTorch
|
47 |
+
print('Convert the data to a format compatible with PyTorch')
|
48 |
+
# wiki_data_en = torch.tensor(wiki_data_en)
|
49 |
+
wiki_data_fr = torch.tensor(wiki_data_fr)
|
50 |
+
# stackoverflow_data = torch.tensor(stackoverflow_data)
|
51 |
+
|
52 |
+
# Define the Adam optimizer
|
53 |
+
print('Define the Adam optimizer')
|
54 |
+
optimizer = optim.Adam(model.parameters(), lr=0.001)
|
55 |
+
|
56 |
+
# Define the loss function
|
57 |
+
print('Define the loss function')
|
58 |
+
criterion = nn.CrossEntropyLoss()
|
59 |
+
|
60 |
+
# Train the model
|
61 |
+
print('Train the model')
|
62 |
+
num_epochs=10
|
63 |
+
labels = torch.tensor([0, 1, 1, 0, 0, 1, 0, 1, 0, 1])
|
64 |
+
|
65 |
+
for epoch in range(num_epochs):
|
66 |
+
print('epoch: ' + epoch)
|
67 |
+
# Forward pass
|
68 |
+
# outputs = model(wiki_data, stackoverflow_data)
|
69 |
+
outputs = model(wiki_data_fr)
|
70 |
+
# Calculate the loss
|
71 |
+
loss = criterion(outputs, labels)
|
72 |
+
# Backward pass
|
73 |
+
loss.backward()
|
74 |
+
# Update the parameters
|
75 |
+
optimizer.step()
|
76 |
+
# Reset the gradients
|
77 |
+
optimizer.zero_grad()
|
78 |
+
# Evaluate the model
|
79 |
+
accuracy = evaluate(model, wiki_data_fr)
|
80 |
+
# Save the model weights and states
|
81 |
+
torch.save(model.state_dict(), 'model.pth')
|
82 |
+
# Adjust the learning rate
|
83 |
+
adjust_learning_rate(optimizer, epoch)
|
84 |
+
# Print the loss and accuracy
|
85 |
+
print('Epoch: {}, Loss: {:.4f}, Accuracy: {:.4f}'.format(epoch+1, loss.item(), accuracy))
|