bibtext: @inproceedings{
  ahrabian2024the,
  title={The Curious Case of Nonverbal Abstract Reasoning with Multi-Modal Large Language Models},
  author={Kian Ahrabian and Zhivar Sourati and Kexuan Sun and Jiarui Zhang and Yifan Jiang and Fred Morstatter and Jay Pujara},
  booktitle={First Conference on Language Modeling},
  year={2024},
  url={https://openreview.net/forum?id=eDWcNqiQWW}
  }
abstract: While large language models (LLMs) are still being adopted to new domains and utilized in novel applications, we are experiencing an influx of the new generation of foundation models, namely multi-modal large language models (MLLMs). These models integrate verbal and visual information, opening new possibilities to demonstrate more complex reasoning abilities at the intersection of the two modalities. However, despite the revolutionizing prospect of MLLMs, our understanding of their reasoning abilities is limited. In this study, we assess the nonverbal abstract reasoning abilities of open-source and closed-source MLLMs using variations of Raven's Progressive Matrices. Our experiments reveal the challenging nature of such problems for MLLMs while showcasing the immense gap between open-source and closed-source models. We also uncover critical shortcomings of visual and textual perceptions, subjecting the models to low-performance ceilings. Finally, to improve MLLMs' performance, we experiment with different methods, such as Chain-of-Thought prompting, leading to a significant (up to 100\%) boost in performance.
title: The Curious Case of Nonverbal Abstract Reasoning with Multi-Modal Large Language Models
authors: Kian Ahrabian Zhivar Sourati Kexuan Sun Jiarui Zhang Yifan Jiang Fred Morstatter Jay Pujara