bibtext: @inproceedings{
  peng2024eagle,
  title={Eagle and Finch: {RWKV} with Matrix-Valued States and Dynamic Recurrence},
  author={Bo Peng and Daniel Goldstein and Quentin Gregory Anthony and Alon Albalak and Eric Alcaide and Stella Biderman and Eugene Cheah and Teddy Ferdinan and Kranthi Kiran GV and Haowen Hou and Satyapriya Krishna and Ronald McClelland Jr. and Niklas Muennighoff and Fares Obeid and Atsushi Saito and Guangyu Song and Haoqin Tu and Ruichong Zhang and Bingchen Zhao and Qihang Zhao and Jian Zhu and Rui-Jie Zhu},
  booktitle={First Conference on Language Modeling},
  year={2024},
  url={https://openreview.net/forum?id=soz1SEiPeq}
  }
abstract: We present Eagle (RWKV-5) and Finch (RWKV-6), sequence models improving upon the RWKV architecture. Our architectural design
  advancements include multi-headed matrix-valued states and a dynamic recurrence mechanism that improve expressivity while maintaining the inference efficiency characteristics of RNNs. We introduce a new multilingual corpus with 1.12 trillion tokens and a fast tokenizer based on greedy matching for enhanced multilinguality. We trained four Eagle models, ranging from 0.46 to 7.5 billion parameters, and two Finch models with 1.6 and 3.1 billion parameters and find that they achieve competitive performance across a wide variety of benchmarks.
title: Eagle and Finch: RWKV with Matrix-Valued States and Dynamic Recurrence
authors: Bo Peng Daniel Goldstein Quentin Gregory Anthony Alon Albalak Eric Alcaide Stella Biderman Eugene Cheah Teddy Ferdinan Kranthi Kiran GV Haowen Hou Satyapriya Krishna Ronald McClelland Jr. Niklas Muennighoff Fares Obeid Atsushi Saito Guangyu Song Haoqin Tu Ruichong Zhang Bingchen Zhao Qihang Zhao Jian Zhu Rui-Jie Zhu
Models: BlinkDL/rwkv-6-world TimeMobius/Mobius-RWKV-r5-chat-12B-8k TimeMobius/Mobius-RWKV-r6-12B xiaol/mobius-rwkv-r6-12B xiaol/Mobius-RWKV-r5-chat-12B-8k