@inproceedings{jain2024peekaboo,
author = {Jain, Yash and Nasery, Anshul and Vineet, Vibhav and Behl, Harkirat},
title = {Peekaboo: Interactive video generation via masked-diffusion},
booktitle = {2024 Computer Vision and Pattern Recognition},
year = {2024},
month = {June},
abstract = {Modern video generation models like Sora have achieved remarkable success in producing high-quality videos. However, a significant limitation is their inability to offer interactive control to users, a feature that promises to open up unprecedented applications and creativity. In this work, we introduce the first solution to equip diffusion-based video generation models with spatio-temporal control. We present PEEKABOO, a novel masked attention module, which seamlessly integrates with current video generation models offering control without the need for additional training or inference overhead. To facilitate future research, we also introduce a comprehensive benchmark for interactive video generation. This benchmark offers a standardized framework for the community to assess the efficacy of emerging interactive video generation models. Our extensive qualitative and quantitative assessments reveal that PEEKABOO achieves up to a 3.8x improvement in mIoU over baseline models, all while maintaining the same latency. Code and benchmark are available on the webpage.},
url = {http://approjects.co.za/?big=en-us/research/publication/peekaboo-interactive-video-generation-via-masked-diffusion-2/},
pages = {8079-8088},
note = {Invited Talk at Holistic Video Understanding Workshop, CVPR 2024},
}