#!/usr/bin/python """ # ============================== # flowlib.py # library for optical flow processing # Author: Ruoteng Li # Date: 6th Aug 2016 # ============================== """ #import png import numpy as np from PIL import Image import io UNKNOWN_FLOW_THRESH = 1e7 SMALLFLOW = 0.0 LARGEFLOW = 1e8 """ ============= Flow Section ============= """ def write_flow(flow, filename): """ write optical flow in Middlebury .flo format :param flow: optical flow map :param filename: optical flow file path to be saved :return: None """ f = open(filename, 'wb') magic = np.array([202021.25], dtype=np.float32) (height, width) = flow.shape[0:2] w = np.array([width], dtype=np.int32) h = np.array([height], dtype=np.int32) magic.tofile(f) w.tofile(f) h.tofile(f) flow.tofile(f) f.close() def save_flow_image(flow, image_file): """ save flow visualization into image file :param flow: optical flow data :param flow_fil :return: None """ flow_img = flow_to_image(flow) img_out = Image.fromarray(flow_img) img_out.save(image_file) def segment_flow(flow): h = flow.shape[0] w = flow.shape[1] u = flow[:, :, 0] v = flow[:, :, 1] idx = ((abs(u) > LARGEFLOW) | (abs(v) > LARGEFLOW)) idx2 = (abs(u) == SMALLFLOW) class0 = (v == 0) & (u == 0) u[idx2] = 0.00001 tan_value = v / u class1 = (tan_value < 1) & (tan_value >= 0) & (u > 0) & (v >= 0) class2 = (tan_value >= 1) & (u >= 0) & (v >= 0) class3 = (tan_value < -1) & (u <= 0) & (v >= 0) class4 = (tan_value < 0) & (tan_value >= -1) & (u < 0) & (v >= 0) class8 = (tan_value >= -1) & (tan_value < 0) & (u > 0) & (v <= 0) class7 = (tan_value < -1) & (u >= 0) & (v <= 0) class6 = (tan_value >= 1) & (u <= 0) & (v <= 0) class5 = (tan_value >= 0) & (tan_value < 1) & (u < 0) & (v <= 0) seg = np.zeros((h, w)) seg[class1] = 1 seg[class2] = 2 seg[class3] = 3 seg[class4] = 4 seg[class5] = 5 seg[class6] = 6 seg[class7] = 7 seg[class8] = 8 seg[class0] = 0 seg[idx] = 0 return seg def flow_to_image(flow): """ Convert flow into middlebury color code image :param flow: optical flow map :return: optical flow image in middlebury color """ u = flow[:, :, 0] v = flow[:, :, 1] maxu = -999. maxv = -999. minu = 999. minv = 999. idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH) u[idxUnknow] = 0 v[idxUnknow] = 0 maxu = max(maxu, np.max(u)) minu = min(minu, np.min(u)) maxv = max(maxv, np.max(v)) minv = min(minv, np.min(v)) rad = np.sqrt(u ** 2 + v ** 2) maxrad = max(5, np.max(rad)) #maxrad = max(-1, 99) u = u/(maxrad + np.finfo(float).eps) v = v/(maxrad + np.finfo(float).eps) img = compute_color(u, v) idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2) img[idx] = 0 return np.uint8(img) def disp_to_flowfile(disp, filename): """ Read KITTI disparity file in png format :param disp: disparity matrix :param filename: the flow file name to save :return: None """ f = open(filename, 'wb') magic = np.array([202021.25], dtype=np.float32) (height, width) = disp.shape[0:2] w = np.array([width], dtype=np.int32) h = np.array([height], dtype=np.int32) empty_map = np.zeros((height, width), dtype=np.float32) data = np.dstack((disp, empty_map)) magic.tofile(f) w.tofile(f) h.tofile(f) data.tofile(f) f.close() def compute_color(u, v): """ compute optical flow color map :param u: optical flow horizontal map :param v: optical flow vertical map :return: optical flow in color code """ [h, w] = u.shape img = np.zeros([h, w, 3]) nanIdx = np.isnan(u) | np.isnan(v) u[nanIdx] = 0 v[nanIdx] = 0 colorwheel = make_color_wheel() ncols = np.size(colorwheel, 0) rad = np.sqrt(u**2+v**2) a = np.arctan2(-v, -u) / np.pi fk = (a+1) / 2 * (ncols - 1) + 1 k0 = np.floor(fk).astype(int) k1 = k0 + 1 k1[k1 == ncols+1] = 1 f = fk - k0 for i in range(0, np.size(colorwheel,1)): tmp = colorwheel[:, i] col0 = tmp[k0-1] / 255 col1 = tmp[k1-1] / 255 col = (1-f) * col0 + f * col1 idx = rad <= 1 col[idx] = 1-rad[idx]*(1-col[idx]) notidx = np.logical_not(idx) col[notidx] *= 0.75 img[:, :, i] = np.uint8(np.floor(255 * col*(1-nanIdx))) return img def make_color_wheel(): """ Generate color wheel according Middlebury color code :return: Color wheel """ RY = 15 YG = 6 GC = 4 CB = 11 BM = 13 MR = 6 ncols = RY + YG + GC + CB + BM + MR colorwheel = np.zeros([ncols, 3]) col = 0 # RY colorwheel[0:RY, 0] = 255 colorwheel[0:RY, 1] = np.transpose(np.floor(255*np.arange(0, RY) / RY)) col += RY # YG colorwheel[col:col+YG, 0] = 255 - np.transpose(np.floor(255*np.arange(0, YG) / YG)) colorwheel[col:col+YG, 1] = 255 col += YG # GC colorwheel[col:col+GC, 1] = 255 colorwheel[col:col+GC, 2] = np.transpose(np.floor(255*np.arange(0, GC) / GC)) col += GC # CB colorwheel[col:col+CB, 1] = 255 - np.transpose(np.floor(255*np.arange(0, CB) / CB)) colorwheel[col:col+CB, 2] = 255 col += CB # BM colorwheel[col:col+BM, 2] = 255 colorwheel[col:col+BM, 0] = np.transpose(np.floor(255*np.arange(0, BM) / BM)) col += + BM # MR colorwheel[col:col+MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR)) colorwheel[col:col+MR, 0] = 255 return colorwheel def read_flo_file(filename, memcached=False): """ Read from Middlebury .flo file :param flow_file: name of the flow file :return: optical flow data in matrix """ if memcached: filename = io.BytesIO(filename) f = open(filename, 'rb') magic = np.fromfile(f, np.float32, count=1)[0] data2d = None if 202021.25 != magic: print('Magic number incorrect. Invalid .flo file') else: w = np.fromfile(f, np.int32, count=1)[0] h = np.fromfile(f, np.int32, count=1)[0] data2d = np.fromfile(f, np.float32, count=2 * w * h) # reshape data into 3D array (columns, rows, channels) data2d = np.resize(data2d, (h, w, 2)) f.close() return data2d # fast resample layer def resample(img, sz): """ img: flow map to be resampled sz: new flow map size. Must be [height,weight] """ original_image_size = img.shape in_height = img.shape[0] in_width = img.shape[1] out_height = sz[0] out_width = sz[1] out_flow = np.zeros((out_height, out_width, 2)) # find scale height_scale = float(in_height) / float(out_height) width_scale = float(in_width) / float(out_width) [x,y] = np.meshgrid(range(out_width), range(out_height)) xx = x * width_scale yy = y * height_scale x0 = np.floor(xx).astype(np.int32) x1 = x0 + 1 y0 = np.floor(yy).astype(np.int32) y1 = y0 + 1 x0 = np.clip(x0,0,in_width-1) x1 = np.clip(x1,0,in_width-1) y0 = np.clip(y0,0,in_height-1) y1 = np.clip(y1,0,in_height-1) Ia = img[y0,x0,:] Ib = img[y1,x0,:] Ic = img[y0,x1,:] Id = img[y1,x1,:] wa = (y1-yy) * (x1-xx) wb = (yy-y0) * (x1-xx) wc = (y1-yy) * (xx-x0) wd = (yy-y0) * (xx-x0) out_flow[:,:,0] = (Ia[:,:,0]*wa + Ib[:,:,0]*wb + Ic[:,:,0]*wc + Id[:,:,0]*wd) * out_width / in_width out_flow[:,:,1] = (Ia[:,:,1]*wa + Ib[:,:,1]*wb + Ic[:,:,1]*wc + Id[:,:,1]*wd) * out_height / in_height return out_flow