process_snapshot.py

import pickle
import os
import h5py
import sys
import numpy as np
import open3d as o3d
from snapshot_smpl.smpl import Smpl
import cv2
import tqdm


def read_pickle(pkl_path):
    with open(pkl_path, 'rb') as f:
        u = pickle._Unpickler(f)
        u.encoding = 'latin1'
        return u.load()


def get_KRTD(camera):
    K = np.zeros([3, 3])
    K[0, 0] = camera['camera_f'][0]
    K[1, 1] = camera['camera_f'][1]
    K[:2, 2] = camera['camera_c']
    K[2, 2] = 1
    R = np.eye(3)
    T = np.zeros([3])
    D = camera['camera_k']
    return K, R, T, D


def get_o3d_mesh(vertices, faces):
    mesh = o3d.geometry.TriangleMesh()
    mesh.vertices = o3d.utility.Vector3dVector(vertices)
    mesh.triangles = o3d.utility.Vector3iVector(faces)
    mesh.compute_vertex_normals()
    return mesh


def get_smpl(base_smpl, betas, poses, trans):
    base_smpl.betas = betas
    base_smpl.pose = poses
    base_smpl.trans = trans
    vertices = np.array(base_smpl)

    faces = base_smpl.f
    mesh = get_o3d_mesh(vertices, faces)

    return vertices, mesh


def render_smpl(mesh, img, K, R, T):
    vertices = np.array(mesh.vertices)
    rendered_img = renderer.render_multiview(vertices, K[None], R[None],
                                             T[None, None], [img])[0]
    return rendered_img


def extract_image(data_path):
    data_root = os.path.dirname(data_path)
    img_dir = os.path.join(data_root, 'image')
    os.system('mkdir -p {}'.format(img_dir))

    if len(os.listdir(img_dir)) >= 200:
        return

    cap = cv2.VideoCapture(data_path)

    ret, frame = cap.read()
    i = 0

    while ret:
        cv2.imwrite(os.path.join(img_dir, '{}.jpg'.format(i)), frame)
        ret, frame = cap.read()
        i = i + 1

    cap.release()


def extract_mask(masks, mask_dir):
    if len(os.listdir(mask_dir)) >= len(masks):
        return

    for i in tqdm.tqdm(range(len(masks))):
        mask = masks[i].astype(np.uint8)

        # erode the mask
        border = 4
        kernel = np.ones((border, border), np.uint8)
        mask = cv2.erode(mask.copy(), kernel)

        cv2.imwrite(os.path.join(mask_dir, '{}.png'.format(i)), mask)


data_root = 'data/people_snapshot'
# videos = ['female-3-casual']
videos = os.listdir(data_root)

model_paths = [
    'basicModel_f_lbs_10_207_0_v1.0.0.pkl',
    'basicmodel_m_lbs_10_207_0_v1.0.0.pkl'
]

for video in videos:
    camera_path = os.path.join(data_root, video, 'camera.pkl')
    camera = read_pickle(camera_path)
    K, R, T, D = get_KRTD(camera)

    # process video
    video_path = os.path.join(data_root, video, video + '.mp4')
    extract_image(video_path)

    # process mask
    mask_path = os.path.join(data_root, video, 'masks.hdf5')
    masks = h5py.File(mask_path)['masks']
    mask_dir = os.path.join(data_root, video, 'mask')
    os.system('mkdir -p {}'.format(mask_dir))
    extract_mask(masks, mask_dir)

    smpl_path = os.path.join(data_root, video, 'reconstructed_poses.hdf5')
    smpl = h5py.File(smpl_path)
    betas = smpl['betas']
    pose = smpl['pose']
    trans = smpl['trans']

    pose = pose[len(pose) - len(masks):]
    trans = trans[len(trans) - len(masks):]

    # process smpl parameters
    params = {'beta': np.array(betas), 'pose': pose, 'trans': trans}
    params_path = os.path.join(data_root, video, 'params.npy')
    np.save(params_path, params)

    if 'female' in video:
        model_path = model_paths[0]
    else:
        model_path = model_paths[1]
    model_data = read_pickle(model_path)

    img_dir = os.path.join(data_root, video, 'image')
    vertices_dir = os.path.join(data_root, video, 'vertices')
    os.system('mkdir -p {}'.format(vertices_dir))

    num_img = len(os.listdir(img_dir))
    for i in tqdm.tqdm(range(num_img)):
        base_smpl = Smpl(model_data)
        vertices, mesh = get_smpl(base_smpl, betas, pose[i], trans[i])
        np.save(os.path.join(vertices_dir, '{}.npy'.format(i)), vertices)