Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 105 additions & 143 deletions ppocr/data/imaug/text_image_aug/warp_mls.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,42 @@
https://github.com/RubanSeven/Text-Image-Augmentation-python/blob/master/warp_mls.py
"""

import cv2
import numpy as np


def _grid_nodes(size, step):
"""Grid coordinates visited by the original loop: 0, step, 2*step, ... and the last pixel."""
nodes, t = [], 0
while True:
if size <= t < size + step - 1:
t = size - 1
elif t >= size:
break
nodes.append(t)
t += step
return np.array(nodes)


def _cells(size, step):
"""Per pixel: start and end node of its grid cell and the position inside it (as in gen_img)."""
t = np.arange(size)
start = (t // step) * step
end = start + step
last = end >= size
length = np.where(last, size - start, step)
end = np.where(last, size - 1, end)
return start, end, (t - start) / length


class WarpMLS:
"""Moving-least-squares warp of text images (TIA distort / stretch / perspective).

Vectorised over the grid nodes and the pixels (the original looped over both in
Python and took most of the data-loading time); results match the loop version
within a level or two (tests/ppocr/test_warp_mls.py).
"""

def __init__(self, src, src_pts, dst_pts, dst_w, dst_h, trans_ratio=1.0):
self.src = src
self.src_pts = src_pts
Expand All @@ -32,156 +64,86 @@ def __init__(self, src, src_pts, dst_pts, dst_w, dst_h, trans_ratio=1.0):
self.rdx = np.zeros((self.dst_h, self.dst_w))
self.rdy = np.zeros((self.dst_h, self.dst_w))

@staticmethod
def __bilinear_interp(x, y, v11, v12, v21, v22):
return (v11 * (1 - y) + v12 * y) * (1 - x) + (v21 * (1 - y) + v22 * y) * x

def generate(self):
self.calc_delta()
return self.gen_img()

def calc_delta(self):
w = np.zeros(self.pt_count, dtype=np.float32)

"""Displacement (rdx, rdy) at the grid nodes by affine-free similarity MLS."""
if self.pt_count < 2:
return

i = 0
while 1:
if self.dst_w <= i < self.dst_w + self.grid_size - 1:
i = self.dst_w - 1
elif i >= self.dst_w:
break

j = 0
while 1:
if self.dst_h <= j < self.dst_h + self.grid_size - 1:
j = self.dst_h - 1
elif j >= self.dst_h:
break

sw = 0
swp = np.zeros(2, dtype=np.float32)
swq = np.zeros(2, dtype=np.float32)
new_pt = np.zeros(2, dtype=np.float32)
cur_pt = np.array([i, j], dtype=np.float32)

k = 0
for k in range(self.pt_count):
if i == self.dst_pts[k][0] and j == self.dst_pts[k][1]:
break

w[k] = 1.0 / (
(i - self.dst_pts[k][0]) * (i - self.dst_pts[k][0])
+ (j - self.dst_pts[k][1]) * (j - self.dst_pts[k][1])
)

sw += w[k]
swp = swp + w[k] * np.array(self.dst_pts[k])
swq = swq + w[k] * np.array(self.src_pts[k])

if k == self.pt_count - 1:
pstar = 1 / sw * swp
qstar = 1 / sw * swq

miu_s = 0
for k in range(self.pt_count):
if i == self.dst_pts[k][0] and j == self.dst_pts[k][1]:
continue
pt_i = self.dst_pts[k] - pstar
miu_s += w[k] * np.sum(pt_i * pt_i)

cur_pt -= pstar
cur_pt_j = np.array([-cur_pt[1], cur_pt[0]])

for k in range(self.pt_count):
if i == self.dst_pts[k][0] and j == self.dst_pts[k][1]:
continue

pt_i = self.dst_pts[k] - pstar
pt_j = np.array([-pt_i[1], pt_i[0]])

tmp_pt = np.zeros(2, dtype=np.float32)
tmp_pt[0] = (
np.sum(pt_i * cur_pt) * self.src_pts[k][0]
- np.sum(pt_j * cur_pt) * self.src_pts[k][1]
)
tmp_pt[1] = (
-np.sum(pt_i * cur_pt_j) * self.src_pts[k][0]
+ np.sum(pt_j * cur_pt_j) * self.src_pts[k][1]
)
tmp_pt *= w[k] / miu_s
new_pt += tmp_pt

new_pt += qstar
else:
new_pt = self.src_pts[k]

self.rdx[j, i] = new_pt[0] - i
self.rdy[j, i] = new_pt[1] - j

j += self.grid_size
i += self.grid_size
p = np.asarray(self.dst_pts, dtype=np.float64)
q = np.asarray(self.src_pts, dtype=np.float64)
ys, xs = np.meshgrid(
_grid_nodes(self.dst_h, self.grid_size),
_grid_nodes(self.dst_w, self.grid_size),
indexing="ij",
)
node = np.stack([xs.ravel(), ys.ravel()], axis=1).astype(np.float64)

d2 = ((node[:, None, :] - p[None, :, :]) ** 2).sum(axis=2)
same = d2 == 0
# the original loop stops at the first control point equal to the node: before the
# last point the node takes that point's source position; equal to the last point only,
# the node is solved from the other points
first = np.where(same.any(axis=1), same.argmax(axis=1), self.pt_count)
snap = first < self.pt_count - 1

with np.errstate(divide="ignore", invalid="ignore"):
w = np.where(same, 0.0, 1.0 / np.where(same, 1.0, d2))
sw = w.sum(axis=1, keepdims=True)
pstar = (w @ p) / sw
qstar = (w @ q) / sw
pt_i = p[None, :, :] - pstar[:, None, :]
miu = (w * (pt_i**2).sum(axis=2)).sum(axis=1, keepdims=True)
cur = node - pstar
cur_j = np.stack([-cur[:, 1], cur[:, 0]], axis=1)
pt_j = np.stack([-pt_i[..., 1], pt_i[..., 0]], axis=2)
a = (pt_i * cur[:, None, :]).sum(axis=2)
b = (pt_j * cur[:, None, :]).sum(axis=2)
c = (pt_i * cur_j[:, None, :]).sum(axis=2)
e = (pt_j * cur_j[:, None, :]).sum(axis=2)
scale = w / miu
new_x = (scale * (a * q[None, :, 0] - b * q[None, :, 1])).sum(
axis=1
) + qstar[:, 0]
new_y = (scale * (-c * q[None, :, 0] + e * q[None, :, 1])).sum(
axis=1
) + qstar[:, 1]

if snap.any():
new_x[snap] = q[first[snap], 0]
new_y[snap] = q[first[snap], 1]
rows, cols = ys.ravel(), xs.ravel()
self.rdx[rows, cols] = new_x - cols
self.rdy[rows, cols] = new_y - rows

def gen_img(self):
"""Bilinear interpolation of the node displacements to every pixel, then bilinear
sampling of the source (cv2.remap; positions clipped to the image as before)."""
src_h, src_w = self.src.shape[:2]
dst = np.zeros_like(self.src, dtype=np.float32)

for i in np.arange(0, self.dst_h, self.grid_size):
for j in np.arange(0, self.dst_w, self.grid_size):
ni = i + self.grid_size
nj = j + self.grid_size
w = h = self.grid_size
if ni >= self.dst_h:
ni = self.dst_h - 1
h = ni - i + 1
if nj >= self.dst_w:
nj = self.dst_w - 1
w = nj - j + 1

di = np.reshape(np.arange(h), (-1, 1))
dj = np.reshape(np.arange(w), (1, -1))
delta_x = self.__bilinear_interp(
di / h,
dj / w,
self.rdx[i, j],
self.rdx[i, nj],
self.rdx[ni, j],
self.rdx[ni, nj],
)
delta_y = self.__bilinear_interp(
di / h,
dj / w,
self.rdy[i, j],
self.rdy[i, nj],
self.rdy[ni, j],
self.rdy[ni, nj],
)
nx = j + dj + delta_x * self.trans_ratio
ny = i + di + delta_y * self.trans_ratio
nx = np.clip(nx, 0, src_w - 1)
ny = np.clip(ny, 0, src_h - 1)
nxi = np.array(np.floor(nx), dtype=np.int32)
nyi = np.array(np.floor(ny), dtype=np.int32)
nxi1 = np.array(np.ceil(nx), dtype=np.int32)
nyi1 = np.array(np.ceil(ny), dtype=np.int32)

if len(self.src.shape) == 3:
x = np.tile(np.expand_dims(ny - nyi, axis=-1), (1, 1, 3))
y = np.tile(np.expand_dims(nx - nxi, axis=-1), (1, 1, 3))
else:
x = ny - nyi
y = nx - nxi
dst[i : i + h, j : j + w] = self.__bilinear_interp(
x,
y,
self.src[nyi, nxi],
self.src[nyi, nxi1],
self.src[nyi1, nxi],
self.src[nyi1, nxi1],
)

dst = np.clip(dst, 0, 255)
dst = np.array(dst, dtype=np.uint8)

return dst
y0, y1, fy = _cells(self.dst_h, self.grid_size)
x0, x1, fx = _cells(self.dst_w, self.grid_size)
fy = fy[:, None]
nx = (
np.arange(self.dst_w)[None, :]
+ self._interp(self.rdx, y0, y1, fy, x0, x1, fx) * self.trans_ratio
)
ny = (
np.arange(self.dst_h)[:, None]
+ self._interp(self.rdy, y0, y1, fy, x0, x1, fx) * self.trans_ratio
)
map_x = np.clip(nx, 0, src_w - 1).astype(np.float32)
map_y = np.clip(ny, 0, src_h - 1).astype(np.float32)
dst = cv2.remap(
self.src, map_x, map_y, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE
)
return np.array(dst, dtype=np.uint8)

@classmethod
def _interp(cls, field, y0, y1, fy, x0, x1, fx):
"""field known at the grid nodes -> every pixel: along x on the node rows, then along y."""
rows = np.unique(np.concatenate([y0, y1]))
along_x = field[rows][:, x0] * (1 - fx) + field[rows][:, x1] * fx
index = np.searchsorted(rows, np.arange(rows.max() + 1))
return along_x[index[y0]] * (1 - fy) + along_x[index[y1]] * fy
Loading