@inproceedings{tang2020aibench,
author = {Tang, Fei and Gao, Wanling and Zhan, Jianfeng and Lan, Chuanxin and Wen, Xu and Wang, Lei and Luo, Chunjie and Dai, Jiahui and Cao, Zheng and Xiong, Xingwang and Jiang, Zihan and Hao, Tianshu and Fan, Fanda and Zhang, Fan and Huang, Yunyou and Chen, Jianan and Du, Mengjia and Ren, Rui and Zheng, Chen and Zheng, Daoyi and Tang, Haoning and Zhan, Kunlin and Wang, Biao and Kong, Defei and Yu, Minghe and Tan, Chongkang and Li, Huan and Tian, Xinhui and Li, Yatao and Lu, Gang and Shao, Junchao and Wang, Zhenyu and Wang, Xiaoyu and Ye, Hainan},
title = {AIBench Training: Balanced Industry-Standard AI Training Benchmarking},
year = {2020},
month = {April},
abstract = {Earlier-stage evaluations of a new AI architecture/system need affordable AI benchmarks, while using a few AI component benchmarks alone in the other stages may lead to misleading conclusions. This paper proposes a balanced benchmarking methodology. Performing an exhaustive survey on Internet service AI domains, we identify and implement seventeen representative AI tasks with the state-of-the-art models to guarantee the diversity and representativeness of the benchmarks. Meanwhile, we keep a benchmark subset to a minimum for affordability. We contribute by far the most comprehensive AI training benchmark suite with seventeen industry partners. The evaluations show: (1) AIBench Training outperforms MLPerf Training in terms of the diversity and representativeness of model complexity, computational cost, convergent rate, computation and memory access patterns, and hotspot functions; (2) With respect to the AIBench full benchmarks, its subset shortens the benchmarking cost by 54%, while maintaining the primary workload characteristics; (3) The performance ranking shows the single-purpose AI accelerator like TPU with the optimized TensorFlow framework performs better than that of GPUs while losing the latters' general support for a variety of AI models. The AIBench Training specifications, source code, testbed, and performance numbers are publicly available from the web site this http URL.},
url = {http://approjects.co.za/?big=en-us/research/publication/aibench-training-balanced-industry-standard-ai-training-benchmarking/},
}