bibtext: @inproceedings{
  xu2024rejection,
  title={Rejection Improves Reliability: Training {LLM}s to Refuse Unknown Questions Using {RL} from Knowledge Feedback},
  author={Hongshen Xu and Zichen Zhu and Situo Zhang and Da Ma and Shuai Fan and Lu Chen and Kai Yu},
  booktitle={First Conference on Language Modeling},
  year={2024},
  url={https://openreview.net/forum?id=lJMioZBoR8}
  }
abstract: Large Language Models (LLMs) often generate erroneous outputs, known as hallucinations, due to their limitations in discerning questions beyond their knowledge scope. While addressing hallucination has been a focal point in research, previous efforts primarily concentrate on enhancing correctness without giving due consideration to the significance of rejection mechanisms. In this paper, we conduct a comprehensive examination of the role of rejection, introducing the alignment goal of model reliability along with corresponding metrics. This goal requires the model to provide accurate responses while adeptly rejecting questions exceeding its knowledge boundaries, thereby minimizing hallucinations. To improve the inherent reliability of LLMs, we present a novel alignment framework called Reinforcement Learning from Knowledge Feedback (RLKF). RLKF leverages knowledge feedback to dynamically determine the model's knowledge boundary and trains a reliable reward model to encourage the rejection of out-of-knowledge questions. Experimental results on mathematical and question answering datasets affirm the substantial efficacy of RLKF in significantly enhancing LLM reliability.
title: Rejection Improves Reliability: Training LLMs to Refuse Unknown Questions Using RL from Knowledge Feedback
authors: Hongshen Xu Zichen Zhu Situo Zhang Da Ma Shuai Fan Lu Chen Kai Yu