first commit

KZReader.py

@@ -0,0 +1,108 @@
+import os
+import cv2
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont
+from tqdm import tqdm
+import os
+
+import easyocr
+
+models_dir = "./models"
+images_dir = "./images"
+output_dir = "./output"
+dirs = [models_dir, images_dir, output_dir]
+for d in dirs:
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+
+class KZReader:
+    def __init__(self):
+        self.reader = easyocr.Reader(
+                            ['en'],
+                            gpu=True,
+                            recog_network='best_norm_ED',
+                            detect_network="craft",
+                            user_network_directory=models_dir,
+                            model_storage_directory=models_dir,
+                        )  # this needs to run only once to load the model into memory
+    def readtext(self, image,paragraph):
+        
+        result = reader.readtext(image = image, paragraph=True)
+        return result
+
+"""
+Upload easy OCR model files with the same name and font file named Ubuntu-Regular.ttf, examples:
+best_norm_ED.pth
+best_norm_ED.py
+best_norm_ED.yaml
+Ubuntu-Regular.ttf
+
+to models directory
+
+Upload image files you want to test, examples:
+kz_book_simple.jpeg
+kz_blur.jpg
+kz_book_complex.jpg
+
+to images directory
+"""
+
+'''
+font_path = models_dir + "/Ubuntu-Regular.ttf"
+
+reader = easyocr.Reader(
+    ['en'],
+    gpu=True,
+    recog_network='best_norm_ED',
+    detect_network="craft",
+    user_network_directory=models_dir,
+    model_storage_directory=models_dir,
+)  # this needs to run only once to load the model into memory
+
+image_extensions = (".jpg", ".jpeg", ".png")
+'''
+
+'''
+for image_name in tqdm(os.listdir(images_dir)):
+    if not image_name.lower().endswith(image_extensions):
+        print(f'unsupported file {image_name}')
+        continue
+    image_path = f'{images_dir}/{image_name}'
+    print(image_path)
+    # Read image as numpy array
+    image = cv2.imread(image_path)
+
+    # Rotate the image by 270 degrees
+    # image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
+
+    # Convert the image from BGR to RGB (because OpenCV loads images in BGR format)
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    results = reader.readtext(image=image)
+
+    # Load custom font
+    font = ImageFont.truetype(font_path, 32)
+
+    # Display the results
+    for (bbox, text, prob) in results:
+        # Get the bounding box coordinates
+        (top_left, top_right, bottom_right, bottom_left) = bbox
+        top_left = (int(top_left[0]), int(top_left[1]))
+        bottom_right = (int(bottom_right[0]), int(bottom_right[1]))
+
+        # Draw the bounding box on the image
+        cv2.rectangle(image, top_left, bottom_right, (0, 255, 0), 2)
+
+        # Convert the OpenCV image to a PIL image, draw the text, then convert back to an OpenCV image
+        image_pil = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+        draw = ImageDraw.Draw(image_pil)
+        draw.text((top_left[0], top_left[1] - 40), text, font=font, fill=(0, 0, 255))
+        image = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR)
+
+    # Save image
+    cv2.imwrite( f'{output_dir}/{image_name}', image)
+
+    # reader.readtext(image = image, paragraph=True)
+
+
+'''

app.py

@@ -0,0 +1,36 @@
+import pandas as pd
+import numpy as np
+import streamlit as st
+import easyocr
+import PIL
+from PIL import Image, ImageDraw
+from matplotlib import pyplot as plt
+from KZReader import KZReader
+
+
+# main title
+st.set_page_config(layout="wide")
+st.title("Get text from image with EasyOCR") 
+# subtitle
+st.markdown("## EasyOCRR with Streamlit")
+col1, col2 = st.columns(2)
+uploaded_file = col1.file_uploader("Upload your file here ",type=['png','jpeg','jpg','pdf'])
+if uploaded_file is not None:
+    col1.image(uploaded_file) #display
+        #print("GOGO ",type(uploaded_file))
+    
+    image_extensions = (".jpg", ".jpeg", ".png")
+    pdf_extensions = (".pdf")
+    if uploaded_file.name.lower().endswith(image_extensions):
+        image = Image.open(uploaded_file)
+        reader = KZReader() #easyocr.Reader(['tr','en'], gpu=False) 
+        result = reader.readtext(np.array(image),paragraph=True)  # turn image to numpy array
+        #print(len(result))
+        result_text = "\n\n".join([item[1] for item in result])
+        col2.markdown(result_text)
+    elif uploaded_file.name.lower().endswith(pdf_extensions):
+        print("PDF file")
+        col2.markdown("PDF file")
+    else:
+        print(f'unsupported file {image_name}')
+        col2.markdown("ERROR")

models/best_norm_ED.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:927d3bf23c46f9e190819bcc244fd9ba02d8c93aaff28a2c07683130abfd9238
+size 15163627

models/best_norm_ED.py

@@ -0,0 +1,538 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+class TPS_SpatialTransformerNetwork(nn.Module):
+    """ Rectification Network of RARE, namely TPS based STN """
+
+    def __init__(self, F, I_size, I_r_size, I_channel_num=1):
+        """ Based on RARE TPS
+        input:
+            batch_I: Batch Input Image [batch_size x I_channel_num x I_height x I_width]
+            I_size : (height, width) of the input image I
+            I_r_size : (height, width) of the rectified image I_r
+            I_channel_num : the number of channels of the input image I
+        output:
+            batch_I_r: rectified image [batch_size x I_channel_num x I_r_height x I_r_width]
+        """
+        super(TPS_SpatialTransformerNetwork, self).__init__()
+        self.F = F
+        self.I_size = I_size
+        self.I_r_size = I_r_size  # = (I_r_height, I_r_width)
+        self.I_channel_num = I_channel_num
+        self.LocalizationNetwork = LocalizationNetwork(self.F, self.I_channel_num)
+        self.GridGenerator = GridGenerator(self.F, self.I_r_size)
+
+    def forward(self, batch_I):
+        batch_C_prime = self.LocalizationNetwork(batch_I)  # batch_size x K x 2
+        build_P_prime = self.GridGenerator.build_P_prime(batch_C_prime)  # batch_size x n (= I_r_width x I_r_height) x 2
+        build_P_prime_reshape = build_P_prime.reshape([build_P_prime.size(0), self.I_r_size[0], self.I_r_size[1], 2])
+        batch_I_r = F.grid_sample(batch_I, build_P_prime_reshape, padding_mode='border')
+
+        return batch_I_r
+
+
+class LocalizationNetwork(nn.Module):
+    """ Localization Network of RARE, which predicts C' (K x 2) from I (I_width x I_height) """
+
+    def __init__(self, F, I_channel_num):
+        super(LocalizationNetwork, self).__init__()
+        self.F = F
+        self.I_channel_num = I_channel_num
+        self.conv = nn.Sequential(
+            nn.Conv2d(in_channels=self.I_channel_num, out_channels=64, kernel_size=3, stride=1, padding=1,
+                      bias=False), nn.BatchNorm2d(64), nn.ReLU(True),
+            nn.MaxPool2d(2, 2),  # batch_size x 64 x I_height/2 x I_width/2
+            nn.Conv2d(64, 128, 3, 1, 1, bias=False), nn.BatchNorm2d(128), nn.ReLU(True),
+            nn.MaxPool2d(2, 2),  # batch_size x 128 x I_height/4 x I_width/4
+            nn.Conv2d(128, 256, 3, 1, 1, bias=False), nn.BatchNorm2d(256), nn.ReLU(True),
+            nn.MaxPool2d(2, 2),  # batch_size x 256 x I_height/8 x I_width/8
+            nn.Conv2d(256, 512, 3, 1, 1, bias=False), nn.BatchNorm2d(512), nn.ReLU(True),
+            nn.AdaptiveAvgPool2d(1)  # batch_size x 512
+        )
+
+        self.localization_fc1 = nn.Sequential(nn.Linear(512, 256), nn.ReLU(True))
+        self.localization_fc2 = nn.Linear(256, self.F * 2)
+
+        # Init fc2 in LocalizationNetwork
+        self.localization_fc2.weight.data.fill_(0)
+        """ see RARE paper Fig. 6 (a) """
+        ctrl_pts_x = np.linspace(-1.0, 1.0, int(F / 2))
+        ctrl_pts_y_top = np.linspace(0.0, -1.0, num=int(F / 2))
+        ctrl_pts_y_bottom = np.linspace(1.0, 0.0, num=int(F / 2))
+        ctrl_pts_top = np.stack([ctrl_pts_x, ctrl_pts_y_top], axis=1)
+        ctrl_pts_bottom = np.stack([ctrl_pts_x, ctrl_pts_y_bottom], axis=1)
+        initial_bias = np.concatenate([ctrl_pts_top, ctrl_pts_bottom], axis=0)
+        self.localization_fc2.bias.data = torch.from_numpy(initial_bias).float().view(-1)
+
+    def forward(self, batch_I):
+        """
+        input:     batch_I : Batch Input Image [batch_size x I_channel_num x I_height x I_width]
+        output:    batch_C_prime : Predicted coordinates of fiducial points for input batch [batch_size x F x 2]
+        """
+        batch_size = batch_I.size(0)
+        features = self.conv(batch_I).view(batch_size, -1)
+        batch_C_prime = self.localization_fc2(self.localization_fc1(features)).view(batch_size, self.F, 2)
+        return batch_C_prime
+
+
+class GridGenerator(nn.Module):
+    """ Grid Generator of RARE, which produces P_prime by multiplying T with P """
+
+    def __init__(self, F, I_r_size):
+        """ Generate P_hat and inv_delta_C for later """
+        super(GridGenerator, self).__init__()
+        self.eps = 1e-6
+        self.I_r_height, self.I_r_width = I_r_size
+        self.F = F
+        self.C = self._build_C(self.F)  # F x 2
+        self.P = self._build_P(self.I_r_width, self.I_r_height)
+        ## for multi-gpu, you need register buffer
+        self.register_buffer("inv_delta_C", torch.tensor(self._build_inv_delta_C(self.F, self.C)).float())  # F+3 x F+3
+        self.register_buffer("P_hat", torch.tensor(self._build_P_hat(self.F, self.C, self.P)).float())  # n x F+3
+        ## for fine-tuning with different image width, you may use below instead of self.register_buffer
+        # self.inv_delta_C = torch.tensor(self._build_inv_delta_C(self.F, self.C)).float().cuda()  # F+3 x F+3
+        # self.P_hat = torch.tensor(self._build_P_hat(self.F, self.C, self.P)).float().cuda()  # n x F+3
+
+    def _build_C(self, F):
+        """ Return coordinates of fiducial points in I_r; C """
+        ctrl_pts_x = np.linspace(-1.0, 1.0, int(F / 2))
+        ctrl_pts_y_top = -1 * np.ones(int(F / 2))
+        ctrl_pts_y_bottom = np.ones(int(F / 2))
+        ctrl_pts_top = np.stack([ctrl_pts_x, ctrl_pts_y_top], axis=1)
+        ctrl_pts_bottom = np.stack([ctrl_pts_x, ctrl_pts_y_bottom], axis=1)
+        C = np.concatenate([ctrl_pts_top, ctrl_pts_bottom], axis=0)
+        return C  # F x 2
+
+    def _build_inv_delta_C(self, F, C):
+        """ Return inv_delta_C which is needed to calculate T """
+        hat_C = np.zeros((F, F), dtype=float)  # F x F
+        for i in range(0, F):
+            for j in range(i, F):
+                r = np.linalg.norm(C[i] - C[j])
+                hat_C[i, j] = r
+                hat_C[j, i] = r
+        np.fill_diagonal(hat_C, 1)
+        hat_C = (hat_C ** 2) * np.log(hat_C)
+        # print(C.shape, hat_C.shape)
+        delta_C = np.concatenate(  # F+3 x F+3
+            [
+                np.concatenate([np.ones((F, 1)), C, hat_C], axis=1),  # F x F+3
+                np.concatenate([np.zeros((2, 3)), np.transpose(C)], axis=1),  # 2 x F+3
+                np.concatenate([np.zeros((1, 3)), np.ones((1, F))], axis=1)  # 1 x F+3
+            ],
+            axis=0
+        )
+        inv_delta_C = np.linalg.inv(delta_C)
+        return inv_delta_C  # F+3 x F+3
+
+    def _build_P(self, I_r_width, I_r_height):
+        I_r_grid_x = (np.arange(-I_r_width, I_r_width, 2) + 1.0) / I_r_width  # self.I_r_width
+        I_r_grid_y = (np.arange(-I_r_height, I_r_height, 2) + 1.0) / I_r_height  # self.I_r_height
+        P = np.stack(  # self.I_r_width x self.I_r_height x 2
+            np.meshgrid(I_r_grid_x, I_r_grid_y),
+            axis=2
+        )
+        return P.reshape([-1, 2])  # n (= self.I_r_width x self.I_r_height) x 2
+
+    def _build_P_hat(self, F, C, P):
+        n = P.shape[0]  # n (= self.I_r_width x self.I_r_height)
+        P_tile = np.tile(np.expand_dims(P, axis=1), (1, F, 1))  # n x 2 -> n x 1 x 2 -> n x F x 2
+        C_tile = np.expand_dims(C, axis=0)  # 1 x F x 2
+        P_diff = P_tile - C_tile  # n x F x 2
+        rbf_norm = np.linalg.norm(P_diff, ord=2, axis=2, keepdims=False)  # n x F
+        rbf = np.multiply(np.square(rbf_norm), np.log(rbf_norm + self.eps))  # n x F
+        P_hat = np.concatenate([np.ones((n, 1)), P, rbf], axis=1)
+        return P_hat  # n x F+3
+
+    def build_P_prime(self, batch_C_prime):
+        """ Generate Grid from batch_C_prime [batch_size x F x 2] """
+        batch_size = batch_C_prime.size(0)
+        batch_inv_delta_C = self.inv_delta_C.repeat(batch_size, 1, 1)
+        batch_P_hat = self.P_hat.repeat(batch_size, 1, 1)
+        batch_C_prime_with_zeros = torch.cat((batch_C_prime, torch.zeros(
+            batch_size, 3, 2).float().to(device)), dim=1)  # batch_size x F+3 x 2
+        batch_T = torch.bmm(batch_inv_delta_C, batch_C_prime_with_zeros)  # batch_size x F+3 x 2
+        batch_P_prime = torch.bmm(batch_P_hat, batch_T)  # batch_size x n x 2
+        return batch_P_prime  # batch_size x n x 2
+
+
+class VGG_FeatureExtractor(nn.Module):
+    """ FeatureExtractor of CRNN (https://arxiv.org/pdf/1507.05717.pdf) """
+
+    def __init__(self, input_channel, output_channel=512):
+        super(VGG_FeatureExtractor, self).__init__()
+        self.output_channel = [int(output_channel / 8), int(output_channel / 4),
+                               int(output_channel / 2), output_channel]  # [64, 128, 256, 512]
+        self.ConvNet = nn.Sequential(
+            nn.Conv2d(input_channel, self.output_channel[0], 3, 1, 1), nn.ReLU(True),
+            nn.MaxPool2d(2, 2),  # 64x16x50
+            nn.Conv2d(self.output_channel[0], self.output_channel[1], 3, 1, 1), nn.ReLU(True),
+            nn.MaxPool2d(2, 2),  # 128x8x25
+            nn.Conv2d(self.output_channel[1], self.output_channel[2], 3, 1, 1), nn.ReLU(True),  # 256x8x25
+            nn.Conv2d(self.output_channel[2], self.output_channel[2], 3, 1, 1), nn.ReLU(True),
+            nn.MaxPool2d((2, 1), (2, 1)),  # 256x4x25
+            nn.Conv2d(self.output_channel[2], self.output_channel[3], 3, 1, 1, bias=False),
+            nn.BatchNorm2d(self.output_channel[3]), nn.ReLU(True),  # 512x4x25
+            nn.Conv2d(self.output_channel[3], self.output_channel[3], 3, 1, 1, bias=False),
+            nn.BatchNorm2d(self.output_channel[3]), nn.ReLU(True),
+            nn.MaxPool2d((2, 1), (2, 1)),  # 512x2x25
+            nn.Conv2d(self.output_channel[3], self.output_channel[3], 2, 1, 0), nn.ReLU(True))  # 512x1x24
+
+    def forward(self, input):
+        return self.ConvNet(input)
+
+
+class RCNN_FeatureExtractor(nn.Module):
+    """ FeatureExtractor of GRCNN (https://papers.nips.cc/paper/6637-gated-recurrent-convolution-neural-network-for-ocr.pdf) """
+
+    def __init__(self, input_channel, output_channel=512):
+        super(RCNN_FeatureExtractor, self).__init__()
+        self.output_channel = [int(output_channel / 8), int(output_channel / 4),
+                               int(output_channel / 2), output_channel]  # [64, 128, 256, 512]
+        self.ConvNet = nn.Sequential(
+            nn.Conv2d(input_channel, self.output_channel[0], 3, 1, 1), nn.ReLU(True),
+            nn.MaxPool2d(2, 2),  # 64 x 16 x 50
+            GRCL(self.output_channel[0], self.output_channel[0], num_iteration=5, kernel_size=3, pad=1),
+            nn.MaxPool2d(2, 2),  # 64 x 8 x 25
+            GRCL(self.output_channel[0], self.output_channel[1], num_iteration=5, kernel_size=3, pad=1),
+            nn.MaxPool2d(2, (2, 1), (0, 1)),  # 128 x 4 x 26
+            GRCL(self.output_channel[1], self.output_channel[2], num_iteration=5, kernel_size=3, pad=1),
+            nn.MaxPool2d(2, (2, 1), (0, 1)),  # 256 x 2 x 27
+            nn.Conv2d(self.output_channel[2], self.output_channel[3], 2, 1, 0, bias=False),
+            nn.BatchNorm2d(self.output_channel[3]), nn.ReLU(True))  # 512 x 1 x 26
+
+    def forward(self, input):
+        return self.ConvNet(input)
+
+
+class ResNet_FeatureExtractor(nn.Module):
+    """ FeatureExtractor of FAN (http://openaccess.thecvf.com/content_ICCV_2017/papers/Cheng_Focusing_Attention_Towards_ICCV_2017_paper.pdf) """
+
+    def __init__(self, input_channel, output_channel=512):
+        super(ResNet_FeatureExtractor, self).__init__()
+        self.ConvNet = ResNet(input_channel, output_channel, BasicBlock, [1, 2, 5, 3])
+
+    def forward(self, input):
+        return self.ConvNet(input)
+
+
+# For Gated RCNN
+class GRCL(nn.Module):
+
+    def __init__(self, input_channel, output_channel, num_iteration, kernel_size, pad):
+        super(GRCL, self).__init__()
+        self.wgf_u = nn.Conv2d(input_channel, output_channel, 1, 1, 0, bias=False)
+        self.wgr_x = nn.Conv2d(output_channel, output_channel, 1, 1, 0, bias=False)
+        self.wf_u = nn.Conv2d(input_channel, output_channel, kernel_size, 1, pad, bias=False)
+        self.wr_x = nn.Conv2d(output_channel, output_channel, kernel_size, 1, pad, bias=False)
+
+        self.BN_x_init = nn.BatchNorm2d(output_channel)
+
+        self.num_iteration = num_iteration
+        self.GRCL = [GRCL_unit(output_channel) for _ in range(num_iteration)]
+        self.GRCL = nn.Sequential(*self.GRCL)
+
+    def forward(self, input):
+        """ The input of GRCL is consistant over time t, which is denoted by u(0)
+        thus wgf_u / wf_u is also consistant over time t.
+        """
+        wgf_u = self.wgf_u(input)
+        wf_u = self.wf_u(input)
+        x = F.relu(self.BN_x_init(wf_u))
+
+        for i in range(self.num_iteration):
+            x = self.GRCL[i](wgf_u, self.wgr_x(x), wf_u, self.wr_x(x))
+
+        return x
+
+
+class GRCL_unit(nn.Module):
+
+    def __init__(self, output_channel):
+        super(GRCL_unit, self).__init__()
+        self.BN_gfu = nn.BatchNorm2d(output_channel)
+        self.BN_grx = nn.BatchNorm2d(output_channel)
+        self.BN_fu = nn.BatchNorm2d(output_channel)
+        self.BN_rx = nn.BatchNorm2d(output_channel)
+        self.BN_Gx = nn.BatchNorm2d(output_channel)
+
+    def forward(self, wgf_u, wgr_x, wf_u, wr_x):
+        G_first_term = self.BN_gfu(wgf_u)
+        G_second_term = self.BN_grx(wgr_x)
+        G = F.sigmoid(G_first_term + G_second_term)
+
+        x_first_term = self.BN_fu(wf_u)
+        x_second_term = self.BN_Gx(self.BN_rx(wr_x) * G)
+        x = F.relu(x_first_term + x_second_term)
+
+        return x
+
+
+class BasicBlock(nn.Module):
+    expansion = 1
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None):
+        super(BasicBlock, self).__init__()
+        self.conv1 = self._conv3x3(inplanes, planes)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.conv2 = self._conv3x3(planes, planes)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.stride = stride
+
+    def _conv3x3(self, in_planes, out_planes, stride=1):
+        "3x3 convolution with padding"
+        return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
+                         padding=1, bias=False)
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+
+        if self.downsample is not None:
+            residual = self.downsample(x)
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+
+class ResNet(nn.Module):
+
+    def __init__(self, input_channel, output_channel, block, layers):
+        super(ResNet, self).__init__()
+
+        self.output_channel_block = [int(output_channel / 4), int(output_channel / 2), output_channel, output_channel]
+
+        self.inplanes = int(output_channel / 8)
+        self.conv0_1 = nn.Conv2d(input_channel, int(output_channel / 16),
+                                 kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn0_1 = nn.BatchNorm2d(int(output_channel / 16))
+        self.conv0_2 = nn.Conv2d(int(output_channel / 16), self.inplanes,
+                                 kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn0_2 = nn.BatchNorm2d(self.inplanes)
+        self.relu = nn.ReLU(inplace=True)
+
+        self.maxpool1 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
+        self.layer1 = self._make_layer(block, self.output_channel_block[0], layers[0])
+        self.conv1 = nn.Conv2d(self.output_channel_block[0], self.output_channel_block[
+            0], kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(self.output_channel_block[0])
+
+        self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
+        self.layer2 = self._make_layer(block, self.output_channel_block[1], layers[1], stride=1)
+        self.conv2 = nn.Conv2d(self.output_channel_block[1], self.output_channel_block[
+            1], kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(self.output_channel_block[1])
+
+        self.maxpool3 = nn.MaxPool2d(kernel_size=2, stride=(2, 1), padding=(0, 1))
+        self.layer3 = self._make_layer(block, self.output_channel_block[2], layers[2], stride=1)
+        self.conv3 = nn.Conv2d(self.output_channel_block[2], self.output_channel_block[
+            2], kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(self.output_channel_block[2])
+
+        self.layer4 = self._make_layer(block, self.output_channel_block[3], layers[3], stride=1)
+        self.conv4_1 = nn.Conv2d(self.output_channel_block[3], self.output_channel_block[
+            3], kernel_size=2, stride=(2, 1), padding=(0, 1), bias=False)
+        self.bn4_1 = nn.BatchNorm2d(self.output_channel_block[3])
+        self.conv4_2 = nn.Conv2d(self.output_channel_block[3], self.output_channel_block[
+            3], kernel_size=2, stride=1, padding=0, bias=False)
+        self.bn4_2 = nn.BatchNorm2d(self.output_channel_block[3])
+
+    def _make_layer(self, block, planes, blocks, stride=1):
+        downsample = None
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            downsample = nn.Sequential(
+                nn.Conv2d(self.inplanes, planes * block.expansion,
+                          kernel_size=1, stride=stride, bias=False),
+                nn.BatchNorm2d(planes * block.expansion),
+            )
+
+        layers = []
+        layers.append(block(self.inplanes, planes, stride, downsample))
+        self.inplanes = planes * block.expansion
+        for i in range(1, blocks):
+            layers.append(block(self.inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv0_1(x)
+        x = self.bn0_1(x)
+        x = self.relu(x)
+        x = self.conv0_2(x)
+        x = self.bn0_2(x)
+        x = self.relu(x)
+
+        x = self.maxpool1(x)
+        x = self.layer1(x)
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+
+        x = self.maxpool2(x)
+        x = self.layer2(x)
+        x = self.conv2(x)
+        x = self.bn2(x)
+        x = self.relu(x)
+
+        x = self.maxpool3(x)
+        x = self.layer3(x)
+        x = self.conv3(x)
+        x = self.bn3(x)
+        x = self.relu(x)
+
+        x = self.layer4(x)
+        x = self.conv4_1(x)
+        x = self.bn4_1(x)
+        x = self.relu(x)
+        x = self.conv4_2(x)
+        x = self.bn4_2(x)
+        x = self.relu(x)
+
+        return x
+
+
+class BidirectionalLSTM(nn.Module):
+
+    def __init__(self, input_size, hidden_size, output_size):
+        super(BidirectionalLSTM, self).__init__()
+        self.rnn = nn.LSTM(input_size, hidden_size, bidirectional=True, batch_first=True)
+        self.linear = nn.Linear(hidden_size * 2, output_size)
+
+    def forward(self, input):
+        """
+        input : visual feature [batch_size x T x input_size]
+        output : contextual feature [batch_size x T x output_size]
+        """
+        try:
+            self.rnn.flatten_parameters()
+        except:
+            pass
+        recurrent, _ = self.rnn(input)  # batch_size x T x input_size -> batch_size x T x (2*hidden_size)
+        output = self.linear(recurrent)  # batch_size x T x output_size
+        return output
+
+
+class Attention(nn.Module):
+
+    def __init__(self, input_size, hidden_size, num_classes):
+        super(Attention, self).__init__()
+        self.attention_cell = AttentionCell(input_size, hidden_size, num_classes)
+        self.hidden_size = hidden_size
+        self.num_classes = num_classes
+        self.generator = nn.Linear(hidden_size, num_classes)
+
+    def _char_to_onehot(self, input_char, onehot_dim=38):
+        input_char = input_char.unsqueeze(1)
+        batch_size = input_char.size(0)
+        one_hot = torch.FloatTensor(batch_size, onehot_dim).zero_().to(device)
+        one_hot = one_hot.scatter_(1, input_char, 1)
+        return one_hot
+
+    def forward(self, batch_H, text, is_train=True, batch_max_length=25):
+        """
+        input:
+            batch_H : contextual_feature H = hidden state of encoder. [batch_size x num_steps x num_classes]
+            text : the text-index of each image. [batch_size x (max_length+1)]. +1 for [GO] token. text[:, 0] = [GO].
+        output: probability distribution at each step [batch_size x num_steps x num_classes]
+        """
+        batch_size = batch_H.size(0)
+        num_steps = batch_max_length + 1  # +1 for [s] at end of sentence.
+
+        output_hiddens = torch.FloatTensor(batch_size, num_steps, self.hidden_size).fill_(0).to(device)
+        hidden = (torch.FloatTensor(batch_size, self.hidden_size).fill_(0).to(device),
+                  torch.FloatTensor(batch_size, self.hidden_size).fill_(0).to(device))
+
+        if is_train:
+            for i in range(num_steps):
+                # one-hot vectors for a i-th char. in a batch
+                char_onehots = self._char_to_onehot(text[:, i], onehot_dim=self.num_classes)
+                # hidden : decoder's hidden s_{t-1}, batch_H : encoder's hidden H, char_onehots : one-hot(y_{t-1})
+                hidden, alpha = self.attention_cell(hidden, batch_H, char_onehots)
+                output_hiddens[:, i, :] = hidden[0]  # LSTM hidden index (0: hidden, 1: Cell)
+            probs = self.generator(output_hiddens)
+
+        else:
+            targets = torch.LongTensor(batch_size).fill_(0).to(device)  # [GO] token
+            probs = torch.FloatTensor(batch_size, num_steps, self.num_classes).fill_(0).to(device)
+
+            for i in range(num_steps):
+                char_onehots = self._char_to_onehot(targets, onehot_dim=self.num_classes)
+                hidden, alpha = self.attention_cell(hidden, batch_H, char_onehots)
+                probs_step = self.generator(hidden[0])
+                probs[:, i, :] = probs_step
+                _, next_input = probs_step.max(1)
+                targets = next_input
+
+        return probs  # batch_size x num_steps x num_classes
+
+
+class AttentionCell(nn.Module):
+
+    def __init__(self, input_size, hidden_size, num_embeddings):
+        super(AttentionCell, self).__init__()
+        self.i2h = nn.Linear(input_size, hidden_size, bias=False)
+        self.h2h = nn.Linear(hidden_size, hidden_size)  # either i2i or h2h should have bias
+        self.score = nn.Linear(hidden_size, 1, bias=False)
+        self.rnn = nn.LSTMCell(input_size + num_embeddings, hidden_size)
+        self.hidden_size = hidden_size
+
+    def forward(self, prev_hidden, batch_H, char_onehots):
+        # [batch_size x num_encoder_step x num_channel] -> [batch_size x num_encoder_step x hidden_size]
+        batch_H_proj = self.i2h(batch_H)
+        prev_hidden_proj = self.h2h(prev_hidden[0]).unsqueeze(1)
+        e = self.score(torch.tanh(batch_H_proj + prev_hidden_proj))  # batch_size x num_encoder_step * 1
+
+        alpha = F.softmax(e, dim=1)
+        context = torch.bmm(alpha.permute(0, 2, 1), batch_H).squeeze(1)  # batch_size x num_channel
+        concat_context = torch.cat([context, char_onehots], 1)  # batch_size x (num_channel + num_embedding)
+        cur_hidden = self.rnn(concat_context, prev_hidden)
+        return cur_hidden, alpha
+
+
+class Model(nn.Module):
+
+    def __init__(self, input_channel, output_channel, hidden_size, num_class):
+        super(Model, self).__init__()
+
+
+        """ FeatureExtraction """
+        self.FeatureExtraction = VGG_FeatureExtractor(input_channel, output_channel)
+        self.FeatureExtraction_output = output_channel  # int(imgH/16-1) * 512
+        self.AdaptiveAvgPool = nn.AdaptiveAvgPool2d((None, 1))  # Transform final (imgH/16-1) -> 1
+
+        """ Sequence modeling"""
+        self.SequenceModeling = nn.Sequential(
+            BidirectionalLSTM(self.FeatureExtraction_output, hidden_size, hidden_size),
+            BidirectionalLSTM(hidden_size, hidden_size, hidden_size))
+        self.SequenceModeling_output = hidden_size
+
+
+        self.Prediction = nn.Linear(self.SequenceModeling_output, num_class)
+
+    def forward(self, input, text):
+
+        """ Feature extraction stage """
+        visual_feature = self.FeatureExtraction(input)
+        visual_feature = self.AdaptiveAvgPool(visual_feature.permute(0, 3, 1, 2))  # [b, c, h, w] -> [b, w, c, h]
+        visual_feature = visual_feature.squeeze(3)
+
+        """ Sequence modeling stage """
+        contextual_feature = self.SequenceModeling(visual_feature)
+
+        prediction = self.Prediction(contextual_feature.contiguous())
+
+        return prediction

models/best_norm_ED.yaml

@@ -0,0 +1,52 @@
+number: '0123456789'
+symbol: "!?.,:;'#()<>+-/*=%$»« "
+lang_char: 'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯЁабвгдежзийклмнопрстуфхцчшщъыьэюяёӘҒҚҢӨҰҮІҺәғқңөұүіһ'
+experiment_name: 'kz_synthtiger_v6'
+train_data: '../../synthtiger_kz/results/train_v6'
+valid_data: '../../synthtiger_kz/results/test_v6'
+manualSeed: 1111
+workers: 6
+batch_size: 256 #32
+num_iter: 100000
+valInterval: 1000
+saved_model: '' #'saved_models/en_filtered/iter_300000.pth'
+FT: False
+optim: False # default is Adadelta
+lr: 1.
+beta1: 0.9
+rho: 0.95
+eps: 0.00000001
+grad_clip: 5
+#Data processing
+select_data: 'images' # this is dataset folder in train_data
+batch_ratio: '1'
+total_data_usage_ratio: 1.0
+batch_max_length: 34
+imgH: 64
+imgW: 600
+rgb: False
+sensitive: True
+PAD: True
+contrast_adjust: 0.0
+data_filtering_off: False
+# Model Architecture
+Transformation: 'None'
+FeatureExtraction: 'VGG'
+SequenceModeling: 'BiLSTM'
+Prediction: 'CTC'
+num_fiducial: 20
+input_channel: 1
+output_channel: 256
+hidden_size: 256
+decode: 'greedy'
+new_prediction: False
+freeze_FeatureFxtraction: False
+freeze_SequenceModeling: False
+
+network_params:
+  input_channel: 1
+  output_channel: 256
+  hidden_size: 256
+lang_list:
+  - 'en'
+character_list: 0123456789!?.,:;'#()<>+-/*=%$»« АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯЁабвгдежзийклмнопрстуфхцчшщъыьэюяёӘҒҚҢӨҰҮІҺәғқңөұүіһ

models/craft_mlt_25k.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a5efbfb48b4081100544e75e1e2b57f8de3d84f213004b14b85fd4b3748db17
+size 83152330