2026
Zhang, Sicheng; Naseer, Muhammad Muzammal; Xie, Binzhu; Suryanto, Naufal; Qiu, Shi; Bentahar, Jamal; Akhtar, Naveed; Shah, Mubarak
ReasonCLIP-58M: Visually Grounded Commonsense Reasoning Supervision for CLIP Conference
European Conference on Computer Vision (ECCV), Springer, 2026.
Abstract | Tags: Commonsense Reasoning, ECCV, Vision-Language Models | Links:
@conference{zhang2026reasonclipb,
title = {ReasonCLIP-58M: Visually Grounded Commonsense Reasoning Supervision for CLIP},
author = {Sicheng Zhang and Muhammad Muzammal Naseer and Binzhu Xie and Naufal Suryanto and Shi Qiu and Jamal Bentahar and Naveed Akhtar and Mubarak Shah},
url = {https://arxiv.org/abs/2606.26794},
year = {2026},
date = {2026-09-07},
urldate = {2026-01-01},
booktitle = {European Conference on Computer Vision (ECCV)},
publisher = {Springer},
abstract = {CLIP and its variants are widely adopted visual backbones in multimodal systems, but their pretraining remains dominated by descriptive image-text alignment. As downstream applications increasingly demand visually grounded commonsense inference and compositional reasoning, it remains unclear whether CLIP-style encoders can support such reasoning without architectural changes. To address this, we present ReasonCLIP-58M, a continual pretraining framework that integrates large-scale reasoning supervision into CLIP-style models through our two-stage strategy, which progressively integrates reasoning signals while preserving descriptive alignment, followed by category-structured reasoning supervision. To support this framework, we construct two complementary datasets and a benchmark: ReasonLite-42M, with open-form, visually verifiable reasoning captions; ReasonPro-16M, with category-specific reasoning supervision; and RCLIP-Bench for diagnostic evaluation of visually grounded reasoning. We train a family of ReasonCLIP that improves visually grounded commonsense and compositional reasoning while also enhancing zero-shot retrieval performance. As a drop-in visual encoder for multimodal large language models such as LLaVA-NeXT, ReasonCLIP delivers consistent gains without additional inference cost, demonstrating that structured reasoning supervision enhances the expressive capacity of CLIP-style visual representations. All datasets, models, and training code are available at this https URL.},
keywords = {Commonsense Reasoning, ECCV, Vision-Language Models},
pubstate = {published},
tppubtype = {conference}
}
CLIP and its variants are widely adopted visual backbones in multimodal systems, but their pretraining remains dominated by descriptive image-text alignment. As downstream applications increasingly demand visually grounded commonsense inference and compositional reasoning, it remains unclear whether CLIP-style encoders can support such reasoning without architectural changes. To address this, we present ReasonCLIP-58M, a continual pretraining framework that integrates large-scale reasoning supervision into CLIP-style models through our two-stage strategy, which progressively integrates reasoning signals while preserving descriptive alignment, followed by category-structured reasoning supervision. To support this framework, we construct two complementary datasets and a benchmark: ReasonLite-42M, with open-form, visually verifiable reasoning captions; ReasonPro-16M, with category-specific reasoning supervision; and RCLIP-Bench for diagnostic evaluation of visually grounded reasoning. We train a family of ReasonCLIP that improves visually grounded commonsense and compositional reasoning while also enhancing zero-shot retrieval performance. As a drop-in visual encoder for multimodal large language models such as LLaVA-NeXT, ReasonCLIP delivers consistent gains without additional inference cost, demonstrating that structured reasoning supervision enhances the expressive capacity of CLIP-style visual representations. All datasets, models, and training code are available at this https URL.
Mollah, Sabbir; Gupta, Rohit; Sirnam, Swetha; Liu, Qingyang; Munir, Ahnaf; Shah, Mubarak
The Telephone Game: Evaluating Semantic Drift in Unified Models Conference
European Conference on Computer Vision (ECCV), Springer, 2026.
Abstract | Tags: ECCV, Model Evaluation, Vision-Language Models | Links:
@conference{mollah2026telephonegameb,
title = {The Telephone Game: Evaluating Semantic Drift in Unified Models},
author = {Sabbir Mollah and Rohit Gupta and Swetha Sirnam and Qingyang Liu and Ahnaf Munir and Mubarak Shah},
url = {https://arxiv.org/abs/2509.04438
https://ai.ucf.edu/wp-content/uploads/2026/09/Poster__Telephone_Game.pdf
https://youtu.be/1y_Pn6xD0fw?si=Oab_zk-25L591fLO},
year = {2026},
date = {2026-09-07},
urldate = {2026-09-07},
booktitle = {European Conference on Computer Vision (ECCV)},
publisher = {Springer},
abstract = {Employing a single, unified model (UM) for both visual understanding (image-to-text: I2T) and visual generation (text-to-image: T2I) has opened a new direction in Visual Language Model (VLM) research. While UMs can also support broader unimodal tasks (e.g., text-to-text, image-to-image), we focus on the core cross-modal pair T2I and I2T. Existing evaluation benchmarks consider these capabilities in isolation: FID and GenEval for T2I, and benchmarks such as MME, MMBench for I2T. These isolated single-pass metrics do not reveal cross-consistency: whether a model that "understands" a concept can also "render" it, nor whether semantic meaning is preserved when cycling between image and text modalities. To address this, we introduce the Semantic Drift Protocol (SDP) for Unified Models, a cyclic evaluation protocol that alternates I2T and T2I over multiple generations to quantify semantic drift. We propose two metrics: (i) Mean Cumulative Drift (MCD), an embedding-based measure of overall semantic drift; and (ii) Multi-Generation GenEval (MGG), an object-level compliance score extending GenEval. To assess generalization beyond COCO dataset, which is widely used in training; we create a new benchmark Nocaps+Docci400, sampled from NoCaps and DOCCI and evaluated on seven recent models. SDP reveals substantial variation in cross-modal stability: some models like BAGEL maintain semantic meaning over many alternations, whereas others like VILA-U drift quickly despite strong single-pass scores. Our results highlight SDP as a necessary complement to standard I2T and T2I evaluations. Code is available at this https URL},
keywords = {ECCV, Model Evaluation, Vision-Language Models},
pubstate = {published},
tppubtype = {conference}
}
Employing a single, unified model (UM) for both visual understanding (image-to-text: I2T) and visual generation (text-to-image: T2I) has opened a new direction in Visual Language Model (VLM) research. While UMs can also support broader unimodal tasks (e.g., text-to-text, image-to-image), we focus on the core cross-modal pair T2I and I2T. Existing evaluation benchmarks consider these capabilities in isolation: FID and GenEval for T2I, and benchmarks such as MME, MMBench for I2T. These isolated single-pass metrics do not reveal cross-consistency: whether a model that "understands" a concept can also "render" it, nor whether semantic meaning is preserved when cycling between image and text modalities. To address this, we introduce the Semantic Drift Protocol (SDP) for Unified Models, a cyclic evaluation protocol that alternates I2T and T2I over multiple generations to quantify semantic drift. We propose two metrics: (i) Mean Cumulative Drift (MCD), an embedding-based measure of overall semantic drift; and (ii) Multi-Generation GenEval (MGG), an object-level compliance score extending GenEval. To assess generalization beyond COCO dataset, which is widely used in training; we create a new benchmark Nocaps+Docci400, sampled from NoCaps and DOCCI and evaluated on seven recent models. SDP reveals substantial variation in cross-modal stability: some models like BAGEL maintain semantic meaning over many alternations, whereas others like VILA-U drift quickly despite strong single-pass scores. Our results highlight SDP as a necessary complement to standard I2T and T2I evaluations. Code is available at this https URL