@inproceedings{pi2022reasoning,
author = {Pi, Xinyu and Liu, Qian and Chen, Bei and Ziyadi, Morteza and Lin, Zeqi and Gao, Yan and Fu, Qiang and Lou, Jian-Guang and Chen, Weizhu},
title = {Reasoning Like Program Executors},
booktitle = {EMNLP2022},
year = {2022},
month = {October},
abstract = {Reasoning over natural language is a long-standing goal for the research community. However, studies have shown that existing language models are inadequate in reasoning. To address the issue, we present POET, a new pre-training paradigm. Through pre-training language models with programs and their execution results, POET empowers language models to harvest the reasoning knowledge possessed in program executors via a data-driven approach. POET is conceptually simple and can be instantiated by different kinds of programs. In this paper, we show three empirically powerful instances, i.e., POET-Math, POET-Logic, and POET-SQL. Experimental results on six benchmarks demonstrate that POET can significantly boost model performance on natural language reasoning, such as numerical reasoning, logical reasoning, and multi-hop reasoning. Taking the DROP benchmark as a representative example, POET improves the F1 metric of BART from 69.2% to 80.6%. Furthermore, POET shines in giant language models, pushing the F1 metric of T5-11B to 87.6% and achieving a new state-of-the-art performance on DROP. POET opens a new gate on reasoning-enhancement pre-training and we hope our analysis would shed light on the future research of reasoning like program executors.},
url = {http://approjects.co.za/?big=en-us/research/publication/reasoning-like-program-executors/},
}