bibtext: @inproceedings{
  chen2024measuring,
  title={Measuring Taiwanese Mandarin Language Understanding},
  author={Po-Heng Chen and Sijia Cheng and Wei-Lin Chen and Yen-Ting Lin and Yun-Nung Chen},
  booktitle={First Conference on Language Modeling},
  year={2024},
  url={https://openreview.net/forum?id=7jSMMvXLri}
  }
abstract: The evaluation of large language models (LLMs) has drawn substantial attention in the field recently.
  This work focuses on evaluating LLMs in a Chinese context, specifically, for Traditional Chinese which has been largely underrepresented in existing benchmarks.
  We present TMLU, a comprehensive evaluation suit tailored for assessing the advanced knowledge and reasoning capability in LLMs, under the context of Taiwanese Mandarin.
  TMLU consists of an array of 37 subjects across social science, STEM, humanities, Taiwan-specific content, and others, ranging from middle school to professional levels.
  In addition, we curate chain-of-thought-like few-shot explanations for each subject to facilitate the evaluation of complex reasoning skills.
  To establish a comprehensive baseline, we conduct extensive experiments and analysis on 24 advanced LLMs.
  The results suggest that Chinese open-weight models demonstrate inferior performance comparing to multilingual proprietary ones, and open-weight models tailored for Taiwanese Mandarin lag behind the Simplified-Chinese counterparts.
  The findings indicate great headrooms for improvement, and emphasize the goal of TMLU to foster the development of localized Taiwanese-Mandarin LLMs.
  We release the benchmark and evaluation scripts for the community to promote future research.
title: Measuring Taiwanese Mandarin Language Understanding
authors: Po-Heng Chen Sijia Cheng Wei-Lin Chen Yen-Ting Lin Yun-Nung Chen
Models: yentinglin/Llama-3-Taiwan-70B-Instruct yentinglin/Llama-3-Taiwan-8B-Instruct yentinglin/Llama-3-Taiwan-8B-Instruct-128k yentinglin/Llama-3-Taiwan-70B-Instruct-DPO yentinglin/Llama-3-Taiwan-70B-Instruct-128k chienweichang/Llama-3-Taiwan-8B-Instruct-128k-GGUF chienweichang/Llama-3-Taiwan-70B-Instruct-GGUF nihaomur/Llama-3-Taiwan-8B-Instruct-AWQ-4bit yentinglin/Llama-3-Taiwan-8B-Instruct-DPO chienweichang/Llama-3-Taiwan-8B-Instruct-DPO-GGUF chienweichang/Llama-3-Taiwan-8B-Instruct-GGUF RichardErkhov/yentinglin_-_Llama-3-Taiwan-8B-Instruct-gguf pigfoot/Llama-3-Taiwan-8B-Instruct-V1-5bpw-exl2