MoMask-test / visualization /InverseKinematics.py
andrewatef's picture
Upload folder using huggingface_hub
823807d verified
import numpy as np
import scipy.linalg as linalg
from visualization import Animation
from visualization import AnimationStructure
from visualization.Quaternions import Quaternions
class BasicInverseKinematics:
"""
Basic Inverse Kinematics Solver
This is an extremely simple full body IK
solver.
It works given the following conditions:
* All joint targets must be specified
* All joint targets must be in reach
* All joint targets must not differ
extremely from the starting pose
* No bone length constraints can be violated
* The root translation and rotation are
set to good initial values
It works under the observation that if the
_directions_ the joints are pointing toward
match the _directions_ of the vectors between
the target joints then the pose should match
that of the target pose.
Therefore it iterates over joints rotating
each joint such that the vectors between it
and it's children match that of the target
positions.
Parameters
----------
animation : Animation
animation input
positions : (F, J, 3) ndarray
target positions for each frame F
and each joint J
iterations : int
Optional number of iterations.
If the above conditions are met
1 iteration should be enough,
therefore the default is 1
silent : bool
Optional if to suppress output
defaults to False
"""
def __init__(self, animation, positions, iterations=1, silent=True):
self.animation = animation
self.positions = positions
self.iterations = iterations
self.silent = silent
def __call__(self):
children = AnimationStructure.children_list(self.animation.parents)
for i in range(self.iterations):
for j in AnimationStructure.joints(self.animation.parents):
c = np.array(children[j])
if len(c) == 0: continue
anim_transforms = Animation.transforms_global(self.animation)
anim_positions = anim_transforms[:, :, :3, 3]
anim_rotations = Quaternions.from_transforms(anim_transforms)
jdirs = anim_positions[:, c] - anim_positions[:, np.newaxis, j]
ddirs = self.positions[:, c] - anim_positions[:, np.newaxis, j]
jsums = np.sqrt(np.sum(jdirs ** 2.0, axis=-1)) + 1e-10
dsums = np.sqrt(np.sum(ddirs ** 2.0, axis=-1)) + 1e-10
jdirs = jdirs / jsums[:, :, np.newaxis]
ddirs = ddirs / dsums[:, :, np.newaxis]
angles = np.arccos(np.sum(jdirs * ddirs, axis=2).clip(-1, 1))
axises = np.cross(jdirs, ddirs)
axises = -anim_rotations[:, j, np.newaxis] * axises
rotations = Quaternions.from_angle_axis(angles, axises)
if rotations.shape[1] == 1:
averages = rotations[:, 0]
else:
averages = Quaternions.exp(rotations.log().mean(axis=-2))
self.animation.rotations[:, j] = self.animation.rotations[:, j] * averages
if not self.silent:
anim_positions = Animation.positions_global(self.animation)
error = np.mean(np.sum((anim_positions - self.positions) ** 2.0, axis=-1) ** 0.5)
print('[BasicInverseKinematics] Iteration %i Error: %f' % (i + 1, error))
return self.animation
class JacobianInverseKinematics:
"""
Jacobian Based Full Body IK Solver
This is a full body IK solver which
uses the dampened least squares inverse
jacobian method.
It should remain fairly stable and effective
even for joint positions which are out of
reach and it can also take any number of targets
to treat as end effectors.
Parameters
----------
animation : Animation
animation to solve inverse problem on
targets : {int : (F, 3) ndarray}
Dictionary of target positions for each
frame F, mapping joint index to
a target position
references : (F, 3)
Optional list of J joint position
references for which the result
should bias toward
iterations : int
Optional number of iterations to
compute. More iterations results in
better accuracy but takes longer to
compute. Default is 10.
recalculate : bool
Optional if to recalcuate jacobian
each iteration. Gives better accuracy
but slower to compute. Defaults to True
damping : float
Optional damping constant. Higher
damping increases stability but
requires more iterations to converge.
Defaults to 5.0
secondary : float
Force, or bias toward secondary target.
Defaults to 0.25
silent : bool
Optional if to suppress output
defaults to False
"""
def __init__(self, animation, targets,
references=None, iterations=10,
recalculate=True, damping=2.0,
secondary=0.25, translate=False,
silent=False, weights=None,
weights_translate=None):
self.animation = animation
self.targets = targets
self.references = references
self.iterations = iterations
self.recalculate = recalculate
self.damping = damping
self.secondary = secondary
self.translate = translate
self.silent = silent
self.weights = weights
self.weights_translate = weights_translate
def cross(self, a, b):
o = np.empty(b.shape)
o[..., 0] = a[..., 1] * b[..., 2] - a[..., 2] * b[..., 1]
o[..., 1] = a[..., 2] * b[..., 0] - a[..., 0] * b[..., 2]
o[..., 2] = a[..., 0] * b[..., 1] - a[..., 1] * b[..., 0]
return o
def jacobian(self, x, fp, fr, ts, dsc, tdsc):
""" Find parent rotations """
prs = fr[:, self.animation.parents]
prs[:, 0] = Quaternions.id((1))
""" Find global positions of target joints """
tps = fp[:, np.array(list(ts.keys()))]
""" Get partial rotations """
qys = Quaternions.from_angle_axis(x[:, 1:prs.shape[1] * 3:3], np.array([[[0, 1, 0]]]))
qzs = Quaternions.from_angle_axis(x[:, 2:prs.shape[1] * 3:3], np.array([[[0, 0, 1]]]))
""" Find axis of rotations """
es = np.empty((len(x), fr.shape[1] * 3, 3))
es[:, 0::3] = ((prs * qzs) * qys) * np.array([[[1, 0, 0]]])
es[:, 1::3] = ((prs * qzs) * np.array([[[0, 1, 0]]]))
es[:, 2::3] = ((prs * np.array([[[0, 0, 1]]])))
""" Construct Jacobian """
j = fp.repeat(3, axis=1)
j = dsc[np.newaxis, :, :, np.newaxis] * (tps[:, np.newaxis, :] - j[:, :, np.newaxis])
j = self.cross(es[:, :, np.newaxis, :], j)
j = np.swapaxes(j.reshape((len(x), fr.shape[1] * 3, len(ts) * 3)), 1, 2)
if self.translate:
es = np.empty((len(x), fr.shape[1] * 3, 3))
es[:, 0::3] = prs * np.array([[[1, 0, 0]]])
es[:, 1::3] = prs * np.array([[[0, 1, 0]]])
es[:, 2::3] = prs * np.array([[[0, 0, 1]]])
jt = tdsc[np.newaxis, :, :, np.newaxis] * es[:, :, np.newaxis, :].repeat(tps.shape[1], axis=2)
jt = np.swapaxes(jt.reshape((len(x), fr.shape[1] * 3, len(ts) * 3)), 1, 2)
j = np.concatenate([j, jt], axis=-1)
return j
# @profile(immediate=True)
def __call__(self, descendants=None, gamma=1.0):
self.descendants = descendants
""" Calculate Masses """
if self.weights is None:
self.weights = np.ones(self.animation.shape[1])
if self.weights_translate is None:
self.weights_translate = np.ones(self.animation.shape[1])
""" Calculate Descendants """
if self.descendants is None:
self.descendants = AnimationStructure.descendants_mask(self.animation.parents)
self.tdescendants = np.eye(self.animation.shape[1]) + self.descendants
self.first_descendants = self.descendants[:, np.array(list(self.targets.keys()))].repeat(3, axis=0).astype(int)
self.first_tdescendants = self.tdescendants[:, np.array(list(self.targets.keys()))].repeat(3, axis=0).astype(
int)
""" Calculate End Effectors """
self.endeff = np.array(list(self.targets.values()))
self.endeff = np.swapaxes(self.endeff, 0, 1)
if not self.references is None:
self.second_descendants = self.descendants.repeat(3, axis=0).astype(int)
self.second_tdescendants = self.tdescendants.repeat(3, axis=0).astype(int)
self.second_targets = dict([(i, self.references[:, i]) for i in range(self.references.shape[1])])
nf = len(self.animation)
nj = self.animation.shape[1]
if not self.silent:
gp = Animation.positions_global(self.animation)
gp = gp[:, np.array(list(self.targets.keys()))]
error = np.mean(np.sqrt(np.sum((self.endeff - gp) ** 2.0, axis=2)))
print('[JacobianInverseKinematics] Start | Error: %f' % error)
for i in range(self.iterations):
""" Get Global Rotations & Positions """
gt = Animation.transforms_global(self.animation)
gp = gt[:, :, :, 3]
gp = gp[:, :, :3] / gp[:, :, 3, np.newaxis]
gr = Quaternions.from_transforms(gt)
x = self.animation.rotations.euler().reshape(nf, -1)
w = self.weights.repeat(3)
if self.translate:
x = np.hstack([x, self.animation.positions.reshape(nf, -1)])
w = np.hstack([w, self.weights_translate.repeat(3)])
""" Generate Jacobian """
if self.recalculate or i == 0:
j = self.jacobian(x, gp, gr, self.targets, self.first_descendants, self.first_tdescendants)
""" Update Variables """
l = self.damping * (1.0 / (w + 0.001))
d = (l * l) * np.eye(x.shape[1])
e = gamma * (self.endeff.reshape(nf, -1) - gp[:, np.array(list(self.targets.keys()))].reshape(nf, -1))
x += np.array(list(map(lambda jf, ef:
linalg.lu_solve(linalg.lu_factor(jf.T.dot(jf) + d), jf.T.dot(ef)), j, e)))
""" Generate Secondary Jacobian """
if self.references is not None:
ns = np.array(list(map(lambda jf:
np.eye(x.shape[1]) - linalg.solve(jf.T.dot(jf) + d, jf.T.dot(jf)), j)))
if self.recalculate or i == 0:
j2 = self.jacobian(x, gp, gr, self.second_targets, self.second_descendants,
self.second_tdescendants)
e2 = self.secondary * (self.references.reshape(nf, -1) - gp.reshape(nf, -1))
x += np.array(list(map(lambda nsf, j2f, e2f:
nsf.dot(linalg.lu_solve(linalg.lu_factor(j2f.T.dot(j2f) + d), j2f.T.dot(e2f))),
ns, j2, e2)))
""" Set Back Rotations / Translations """
self.animation.rotations = Quaternions.from_euler(
x[:, :nj * 3].reshape((nf, nj, 3)), order='xyz', world=True)
if self.translate:
self.animation.positions = x[:, nj * 3:].reshape((nf, nj, 3))
""" Generate Error """
if not self.silent:
gp = Animation.positions_global(self.animation)
gp = gp[:, np.array(list(self.targets.keys()))]
error = np.mean(np.sum((self.endeff - gp) ** 2.0, axis=2) ** 0.5)
print('[JacobianInverseKinematics] Iteration %i | Error: %f' % (i + 1, error))
return self.animation
class BasicJacobianIK:
"""
Same interface as BasicInverseKinematics
but uses the Jacobian IK Solver Instead
"""
def __init__(self, animation, positions, iterations=10, silent=True, **kw):
targets = dict([(i, positions[:, i]) for i in range(positions.shape[1])])
self.ik = JacobianInverseKinematics(animation, targets, iterations=iterations, silent=silent, **kw)
def __call__(self, **kw):
return self.ik(**kw)
class ICP:
def __init__(self,
anim, rest, weights, mesh, goal,
find_closest=True, damping=10,
iterations=10, silent=True,
translate=True, recalculate=True,
weights_translate=None):
self.animation = anim
self.rest = rest
self.vweights = weights
self.mesh = mesh
self.goal = goal
self.find_closest = find_closest
self.iterations = iterations
self.silent = silent
self.translate = translate
self.damping = damping
self.weights = None
self.weights_translate = weights_translate
self.recalculate = recalculate
def cross(self, a, b):
o = np.empty(b.shape)
o[..., 0] = a[..., 1] * b[..., 2] - a[..., 2] * b[..., 1]
o[..., 1] = a[..., 2] * b[..., 0] - a[..., 0] * b[..., 2]
o[..., 2] = a[..., 0] * b[..., 1] - a[..., 1] * b[..., 0]
return o
def jacobian(self, x, fp, fr, goal, weights, des_r, des_t):
""" Find parent rotations """
prs = fr[:, self.animation.parents]
prs[:, 0] = Quaternions.id((1))
""" Get partial rotations """
qys = Quaternions.from_angle_axis(x[:, 1:prs.shape[1] * 3:3], np.array([[[0, 1, 0]]]))
qzs = Quaternions.from_angle_axis(x[:, 2:prs.shape[1] * 3:3], np.array([[[0, 0, 1]]]))
""" Find axis of rotations """
es = np.empty((len(x), fr.shape[1] * 3, 3))
es[:, 0::3] = ((prs * qzs) * qys) * np.array([[[1, 0, 0]]])
es[:, 1::3] = ((prs * qzs) * np.array([[[0, 1, 0]]]))
es[:, 2::3] = ((prs * np.array([[[0, 0, 1]]])))
""" Construct Jacobian """
j = fp.repeat(3, axis=1)
j = des_r[np.newaxis, :, :, :, np.newaxis] * (
goal[:, np.newaxis, :, np.newaxis] - j[:, :, np.newaxis, np.newaxis])
j = np.sum(j * weights[np.newaxis, np.newaxis, :, :, np.newaxis], 3)
j = self.cross(es[:, :, np.newaxis, :], j)
j = np.swapaxes(j.reshape((len(x), fr.shape[1] * 3, goal.shape[1] * 3)), 1, 2)
if self.translate:
es = np.empty((len(x), fr.shape[1] * 3, 3))
es[:, 0::3] = prs * np.array([[[1, 0, 0]]])
es[:, 1::3] = prs * np.array([[[0, 1, 0]]])
es[:, 2::3] = prs * np.array([[[0, 0, 1]]])
jt = des_t[np.newaxis, :, :, :, np.newaxis] * es[:, :, np.newaxis, np.newaxis, :].repeat(goal.shape[1],
axis=2)
jt = np.sum(jt * weights[np.newaxis, np.newaxis, :, :, np.newaxis], 3)
jt = np.swapaxes(jt.reshape((len(x), fr.shape[1] * 3, goal.shape[1] * 3)), 1, 2)
j = np.concatenate([j, jt], axis=-1)
return j
# @profile(immediate=True)
def __call__(self, descendants=None, maxjoints=4, gamma=1.0, transpose=False):
""" Calculate Masses """
if self.weights is None:
self.weights = np.ones(self.animation.shape[1])
if self.weights_translate is None:
self.weights_translate = np.ones(self.animation.shape[1])
nf = len(self.animation)
nj = self.animation.shape[1]
nv = self.goal.shape[1]
weightids = np.argsort(-self.vweights, axis=1)[:, :maxjoints]
weightvls = np.array(list(map(lambda w, i: w[i], self.vweights, weightids)))
weightvls = weightvls / weightvls.sum(axis=1)[..., np.newaxis]
if descendants is None:
self.descendants = AnimationStructure.descendants_mask(self.animation.parents)
else:
self.descendants = descendants
des_r = np.eye(nj) + self.descendants
des_r = des_r[:, weightids].repeat(3, axis=0)
des_t = np.eye(nj) + self.descendants
des_t = des_t[:, weightids].repeat(3, axis=0)
if not self.silent:
curr = Animation.skin(self.animation, self.rest, self.vweights, self.mesh, maxjoints=maxjoints)
error = np.mean(np.sqrt(np.sum((curr - self.goal) ** 2.0, axis=-1)))
print('[ICP] Start | Error: %f' % error)
for i in range(self.iterations):
""" Get Global Rotations & Positions """
gt = Animation.transforms_global(self.animation)
gp = gt[:, :, :, 3]
gp = gp[:, :, :3] / gp[:, :, 3, np.newaxis]
gr = Quaternions.from_transforms(gt)
x = self.animation.rotations.euler().reshape(nf, -1)
w = self.weights.repeat(3)
if self.translate:
x = np.hstack([x, self.animation.positions.reshape(nf, -1)])
w = np.hstack([w, self.weights_translate.repeat(3)])
""" Get Current State """
curr = Animation.skin(self.animation, self.rest, self.vweights, self.mesh, maxjoints=maxjoints)
""" Find Cloest Points """
if self.find_closest:
mapping = np.argmin(
(curr[:, :, np.newaxis] -
self.goal[:, np.newaxis, :]) ** 2.0, axis=2)
e = gamma * (np.array(list(map(lambda g, m: g[m], self.goal, mapping))) - curr).reshape(nf, -1)
else:
e = gamma * (self.goal - curr).reshape(nf, -1)
""" Generate Jacobian """
if self.recalculate or i == 0:
j = self.jacobian(x, gp, gr, self.goal, weightvls, des_r, des_t)
""" Update Variables """
l = self.damping * (1.0 / (w + 1e-10))
d = (l * l) * np.eye(x.shape[1])
if transpose:
x += np.array(list(map(lambda jf, ef: jf.T.dot(ef), j, e)))
else:
x += np.array(list(map(lambda jf, ef:
linalg.lu_solve(linalg.lu_factor(jf.T.dot(jf) + d), jf.T.dot(ef)), j, e)))
""" Set Back Rotations / Translations """
self.animation.rotations = Quaternions.from_euler(
x[:, :nj * 3].reshape((nf, nj, 3)), order='xyz', world=True)
if self.translate:
self.animation.positions = x[:, nj * 3:].reshape((nf, nj, 3))
if not self.silent:
curr = Animation.skin(self.animation, self.rest, self.vweights, self.mesh)
error = np.mean(np.sqrt(np.sum((curr - self.goal) ** 2.0, axis=-1)))
print('[ICP] Iteration %i | Error: %f' % (i + 1, error))
import torch
from torch import nn
class InverseKinematics:
def __init__(self, rotations: torch.Tensor, positions: torch.Tensor, offset, parents, constrains):
self.rotations = rotations.cuda()
self.rotations.requires_grad_(True)
self.position = positions.cuda()
self.position.requires_grad_(True)
self.parents = parents
self.offset = offset.cuda()
self.constrains = constrains.cuda()
# hyper-param to tune
self.optimizer = torch.optim.AdamW([self.position, self.rotations], lr=5e-2, betas=(0.9, 0.999))
self.crit = nn.MSELoss()
self.weights = torch.ones([1,22,1]).cuda()
self.weights[:, [4, 8]] = 0.8
self.weights[:, [1, 5]] = 2.
def step(self):
self.optimizer.zero_grad()
glb = self.forward(self.rotations, self.position, self.offset, order='', quater=True, world=True)
# weighted joint position mse
loss = self.crit(glb*self.weights, self.constrains*self.weights)
# regularization term
loss += 0.5 * self.crit(self.rotations[1:, [3, 7, 12, 16, 20]], self.rotations[:-1, [3, 7, 12, 16, 20]]) + 0.1 * self.crit(self.rotations[1:], self.rotations[:-1])
loss.backward()
self.optimizer.step()
self.glb = glb
return loss.item()
def tloss(self, time):
return self.crit(self.glb[time, :], self.constrains[time, :])
def all_loss(self):
res = [self.tloss(t).detach().numpy() for t in range(self.constrains.shape[0])]
return np.array(res)
'''
rotation should have shape batch_size * Joint_num * (3/4) * Time
position should have shape batch_size * 3 * Time
offset should have shape batch_size * Joint_num * 3
output have shape batch_size * Time * Joint_num * 3
'''
def forward(self, rotation: torch.Tensor, position: torch.Tensor, offset: torch.Tensor, order='xyz', quater=False,
world=True):
'''
if not quater and rotation.shape[-2] != 3: raise Exception('Unexpected shape of rotation')
if quater and rotation.shape[-2] != 4: raise Exception('Unexpected shape of rotation')
rotation = rotation.permute(0, 3, 1, 2)
position = position.permute(0, 2, 1)
'''
result = torch.empty(rotation.shape[:-1] + (3,), device=position.device)
norm = torch.norm(rotation, dim=-1, keepdim=True)
rotation = rotation / norm
# if quater:
transform = self.transform_from_quaternion(rotation)
# else:
# transform = self.transform_from_euler(rotation, order)
offset = offset.reshape((-1, 1, offset.shape[-2], offset.shape[-1], 1))
result[..., 0, :] = position
for i, pi in enumerate(self.parents):
if pi == -1:
assert i == 0
continue
result[..., i, :] = torch.matmul(transform[..., pi, :, :], offset[..., i, :, :]).squeeze()
transform[..., i, :, :] = torch.matmul(transform[..., pi, :, :].clone(), transform[..., i, :, :].clone())
if world: result[..., i, :] += result[..., pi, :]
return result
@staticmethod
def transform_from_axis(euler, axis):
transform = torch.empty(euler.shape[0:3] + (3, 3), device=euler.device)
cos = torch.cos(euler)
sin = torch.sin(euler)
cord = ord(axis) - ord('x')
transform[..., cord, :] = transform[..., :, cord] = 0
transform[..., cord, cord] = 1
if axis == 'x':
transform[..., 1, 1] = transform[..., 2, 2] = cos
transform[..., 1, 2] = -sin
transform[..., 2, 1] = sin
if axis == 'y':
transform[..., 0, 0] = transform[..., 2, 2] = cos
transform[..., 0, 2] = sin
transform[..., 2, 0] = -sin
if axis == 'z':
transform[..., 0, 0] = transform[..., 1, 1] = cos
transform[..., 0, 1] = -sin
transform[..., 1, 0] = sin
return transform
@staticmethod
def transform_from_quaternion(quater: torch.Tensor):
qw = quater[..., 0]
qx = quater[..., 1]
qy = quater[..., 2]
qz = quater[..., 3]
x2 = qx + qx
y2 = qy + qy
z2 = qz + qz
xx = qx * x2
yy = qy * y2
wx = qw * x2
xy = qx * y2
yz = qy * z2
wy = qw * y2
xz = qx * z2
zz = qz * z2
wz = qw * z2
m = torch.empty(quater.shape[:-1] + (3, 3), device=quater.device)
m[..., 0, 0] = 1.0 - (yy + zz)
m[..., 0, 1] = xy - wz
m[..., 0, 2] = xz + wy
m[..., 1, 0] = xy + wz
m[..., 1, 1] = 1.0 - (xx + zz)
m[..., 1, 2] = yz - wx
m[..., 2, 0] = xz - wy
m[..., 2, 1] = yz + wx
m[..., 2, 2] = 1.0 - (xx + yy)
return m