bibtext: @inproceedings{
  fujii2024continual,
  title={Continual Pre-Training for Cross-Lingual {LLM} Adaptation: Enhancing Japanese Language Capabilities},
  author={Kazuki Fujii and Taishi Nakamura and Mengsay Loem and Hiroki Iida and Masanari Ohi and Kakeru Hattori and Hirai Shota and Sakae Mizuki and Rio Yokota and Naoaki Okazaki},
  booktitle={First Conference on Language Modeling},
  year={2024},
  url={https://openreview.net/forum?id=TQdd1VhWbe}
  }
abstract: Cross-lingual continual pre-training of large language models (LLMs) initially trained on English corpus allows us to leverage the vast amount of English language resources and reduce the pre-training cost. In this study, we constructed Swallow, an LLM with enhanced Japanese capability, by extending the vocabulary of Llama 2 to include Japanese characters and conducting continual pre-training on a large Japanese web corpus. Experimental results confirmed that the performance on Japanese tasks drastically improved through continual pre-training, and the performance monotonically increased with the amount of training data up to 100B tokens. Consequently, Swallow achieved superior performance compared to other LLMs that were trained from scratch in English and Japanese. An analysis of the effects of continual pre-training revealed that it was particularly effective for Japanese question answering tasks. Furthermore, to elucidate effective methodologies for cross-lingual continual pre-training from English to Japanese, we investigated the impact of vocabulary expansion and the effectiveness of incorporating parallel corpora. The results showed that the efficiency gained through vocabulary expansion had no negative impact on performance, except for the summarization task, and that the combined use of parallel corpora enhanced translation ability.
title: Continual Pre-Training for Cross-Lingual LLM Adaptation: Enhancing Japanese Language Capabilities
authors: Kazuki Fujii Taishi Nakamura Mengsay Loem Hiroki Iida Masanari Ohi Kakeru Hattori Hirai Shota Sakae Mizuki Rio Yokota Naoaki Okazaki
Models: tokyotech-llm/Swallow-7b-instruct-hf tokyotech-llm/Swallow-70b-instruct-hf tokyotech-llm/Swallow-13b-instruct-hf tokyotech-llm/Swallow-7b-hf tokyotech-llm/Swallow-13b-hf tokyotech-llm/Swallow-7b-plus-hf tokyotech-llm/Swallow-70b-hf tokyotech-llm/Swallow-7b-NVE-instruct-hf tokyotech-llm/Swallow-70b-NVE-instruct-hf tokyotech-llm/Swallow-7b-NVE-hf tokyotech-llm/Swallow-70b-NVE-hf tokyotech-llm/Swallow-13b-NVE-hf RichardErkhov/tokyotech-llm_-_Swallow-7b-NVE-instruct-hf-4bits RichardErkhov/tokyotech-llm_-_Swallow-7b-NVE-instruct-hf-gguf RichardErkhov/tokyotech-llm_-_Swallow-7b-NVE-instruct-hf-8bits RichardErkhov/tokyotech-llm_-_Swallow-7b-instruct-hf-4bits RichardErkhov/tokyotech-llm_-_Swallow-7b-instruct-hf-gguf RichardErkhov/tokyotech-llm_-_Swallow-7b-instruct-hf-8bits RichardErkhov/tokyotech-llm_-_Swallow-13b-instruct-hf-gguf RichardErkhov/tokyotech-llm_-_Swallow-70b-NVE-instruct-hf-gguf RichardErkhov/tokyotech-llm_-_Swallow-70b-instruct-v0.1-gguf RichardErkhov/tokyotech-llm_-_Swallow-13b-instruct-v0.1-gguf RichardErkhov/tokyotech-llm_-_Swallow-7b-hf-gguf