@article{hewitt2024look,
author = {Hewitt, Charlie and Sadat Saleh, Fatemeh and Aliakbarian, Sadegh and Petikam, Lohit and Rezaeifar, Shideh and Florentin, Louis and Hosenie, Zafiirah and Cashman, Tom and Valentin, Julien and Cosker, Prof. Darren and Baltrusaitis, Tadas},
title = {Look Ma, no markers: holistic performance capture without the hassle},
year = {2024},
month = {December},
abstract = {We tackle the problem of highly-accurate, holistic performance capture for the face, body and hands simultaneously. Motion-capture technologies used in film and game production typically focus only on face, body or hand capture independently, involve complex and expensive hardware and a high degree of manual intervention from skilled operators. While machine-learning-based approaches exist to overcome these problems, they usually only support a single camera, often operate on a single part of the body, do not produce precise world-space results, and rarely generalize outside specific contexts. In this work, we introduce the first technique for marker-free, high-quality reconstruction of the complete human body, including eyes and tongue, without requiring any calibration, manual intervention or custom hardware. Our approach produces stable world-space results from arbitrary camera rigs as well as supporting varied capture environments and clothing. We achieve this through a hybrid approach that leverages machine learning models trained exclusively on synthetic data and powerful parametric models of human shape and motion. We evaluate our method on a number of body, face and hand reconstruction benchmarks and demonstrate state-of-the-art results that generalize on diverse datasets.},
url = {http://approjects.co.za/?big=en-us/research/publication/synthmocap/},
pages = {235:1-235:12},
journal = {ACM Transactions on Graphics},
volume = {36},
number = {6},
}