bibtext: @inproceedings{
  wu2024do,
  title={Do Language Models Plan Ahead for Future Tokens?},
  author={Wilson Wu and John Xavier Morris and Lionel Levine},
  booktitle={First Conference on Language Modeling},
  year={2024},
  url={https://openreview.net/forum?id=BaOAvPUyBO}
  }
abstract: Do transformers ``think ahead'' during inference at a given position? It is known transformers prepare information in the hidden states of the forward pass at time step $t$ that is then used in future forward passes $t+\tau$. We posit two explanations for this phenomenon: pre-caching, in which off-diagonal gradient terms present during training result in the model computing features at $t$ irrelevant to the present inference task but useful for the future, and breadcrumbs, in which features most relevant to time step $t$ are already the same as those that would most benefit inference at time $t+\tau$. We test these hypotheses by training language models without propagating gradients to past timesteps, a scheme we formalize as myopic training. In a constructed synthetic data setting, we find clear evidence for pre-caching. In the autoregressive language modeling setting, our experiments are more suggestive of the breadcrumbs hypothesis, though pre-caching increases with model scale.
title: Do Language Models Plan Ahead for Future Tokens?
authors: Wilson Wu John Xavier Morris Lionel Levine