@inproceedings{toneva2019an,
author = {Toneva, Mariya and Sordoni, Alessandro and Tachet des Combes, Remi and Trischler, Adam and Bengio, Yoshua and Gordon, Geoff},
title = {An Empirical Study of Example Forgetting during Deep Neural Network Learning},
booktitle = {ICLR 2019},
year = {2019},
month = {April},
abstract = {Inspired by the phenomenon of catastrophic forgetting, we investigate the learning dynamics of neural networks as they train on single classification tasks. Our goal is to understand whether a related phenomenon occurs when data does not undergo a clear distributional shift. We define a ``forgetting event'' to have occurred when an individual training example transitions from being classified correctly to incorrectly over the course of learning. Across several benchmark data sets, we find that: (i) certain examples are forgotten with high frequency, and some not at all; (ii) a data set's (un)forgettable examples generalize across neural architectures; and (iii) based on forgetting dynamics, a significant fraction of examples can be omitted from the training data set while still maintaining state-of-the-art generalization performance.},
url = {http://approjects.co.za/?big=en-us/research/publication/an-empirical-study-of-example-forgetting-during-deep-neural-network-learning/},
}