Publications

@conference{nokey,

title = {A3Bench: A Benchmark for Compositional Reasoning over Aggressive Interactions in Videos},

author = {Austin C Baker and Raiyaan Abdullah and Daniel Z Yoffe and Shruti Vyas and Yogesh Rawat

},

url = {https://openreview.net/forum?id=iWKda5S9fT

https://neurips.cc/virtual/2026/loc/atlanta/poster/139138},

year  = {2026},

date = {2026-12-01},

urldate = {2026-12-01},

publisher = {NeurIPS 2026},

abstract = {Recent Multimodal Large Language Models (MLLMs) have achieved strong performance on general video understanding benchmarks, yet their ability to reason about participant roles in aggressive interactions remains largely unexplored. Existing violence and crime datasets primarily evaluate whether aggression occurs, without testing whether models can identify who performed the action, who received it, and how participants are related within the scene. To address this gap, we introduce **A

Bench**, a benchmark to study *aggressive action and association* in videos. The benchmark evaluates whether models can jointly recognize aggressive actions, distinguish aggressors from victims and bystanders, and correctly bind actions to participants under challenging distractor settings. We benchmark 15 recent MLLMs and find that current models struggle substantially on fine-grained aggressive-scene understanding. Performance degrades consistently as reasoning complexity increases, with models frequently confusing aggressors with victims or bystanders, revealing a systematic failure in role-action association rather than coarse aggression recognition alone. We further observe that increasing model scale or applying generic reasoning prompts provides limited improvement. Finally, we explore a simple role-graph prompting strategy that encourages structured reasoning over participant interactions, leading to consistent gains on compositional reasoning. Our findings highlight that reliable aggressive-scene understanding remains a major challenge for current MLLMs and motivate future research on role-aware video reasoning beyond coarse violence detection.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {MaRiO: Multi-agent Collaborative Reasoning via Shared Observations in MLLMs},

author = {Nathaniel Redmond and Fidel Omar Tito Cruz and Devansh Sharma and Shehreen Azad and Sirshapan Mitra and Shruti Vyas and Yogesh Rawat

},

url = {https://neurips.cc/virtual/2026/loc/atlanta/poster/139083},

year  = {2026},

date = {2026-12-01},

urldate = {2026-12-01},

publisher = {NeurIPS 2026},

abstract = {Multimodal Large Language Models (MLLMs) show strong visual perception capabilities, but existing evaluations largely focus on single-view or multi-view settings with shared camera parameters, leaving the challenge of {multi-agent collaborative} scenarios underexplored. In such settings, multiple agents observe a shared environment from independent viewpoints, and reliable decisions require reasoning across these independent observations of the same environment. To study this problem, we introduce MaRiO, a benchmark for evaluating multi-agent collaborative reasoning, and evaluate 23 state-of-the-art MLLMs on it. Our analysis reveals three key findings: (1) spatial tasks are generally easier than geometric reasoning tasks; (2) most models struggle to resolve cross-agent correspondences, with errors increasing with scene complexity; and (3) models show limited ability to leverage geometric cues present in the scene. As an initial step toward addressing these challenges, we explore geometry-guided synthetic view augmentation that generates intermediate transition frames between agent observations, providing auxiliary spatial context that improves performance in certain categories. Overall, our results highlight that multi-agent collaborative reasoning remains an open challenge and an important direction for multimodal embodied systems.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PHOEBI: An Open-World Benchmark for Bacterial Identification in Phase-Contrast Microscopy},

author = {Aaditya Baranwal and MD JAHID HASAN and Shruti Vyas},

url = {https://arxiv.org/abs/2606.22890},

year  = {2026},

date = {2026-12-01},

publisher = {NeurIPS 2026},

abstract = {Optical microscopy lets us watch living bacteria in real time without stains or labels, and it remains the standard instrument for identifying species across clinical, environmental and industrial microbiology. Yet real samples rarely hold a single species. They arrive as mixtures and sometimes carry an organism no model has ever seen. What happens when a model trained on familiar mixtures meets one it has never encountered? No benchmark has asked this of phase-contrast images, so we built one. textbf{PHOEBI} brings together 120,000 phase-contrast images of 40 mixtures of six bacterial species that we cultured and imaged ourselves, and it holds whole mixtures out of training so that every test is a combination the model has never seen. The answer is sobering. Classifiers that are nearly perfect on the mixtures they know collapse on new ones, and no choice of backbone or pretraining rescues them. The features turn out to be sound and the failure lies in how the evidence within each image is pooled. A culture looks the same everywhere, and decoders built on this simple fact hold steady where large networks fail even though one of them is never trained by gradient descent. The same features go further still. They notice when a species outside the catalogue appears, and the simplest way of giving it a class of its own proves the most effective while leaving the known species almost untouched. PHOEBI turns the mixed culture, the everyday case at the bench, into a measurable target for open-world recognition.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Re:Cognize: A Framework for Open-Set Sequential Character Re-Identification},

author = {Aaditya Baranwal and Madhav Kataria and Yogesh Rawat and Shruti Vyas},

url = {https://arxiv.org/abs/2609.34032},

year  = {2026},

date = {2026-12-01},

publisher = {NeurIPS 2026},

abstract = {Comics and manga tell stories through characters whose appearances change from panel to panel, yet readers recognise them hundreds of pages later without ever consulting a cast list. Character re-identification models rarely face the same challenge: existing evaluations give them a fixed set of identities before the story begins. textbf{Re:Cognize} instead asks models to recognise characters as the story unfolds and learn who is who along the way. Across three manga corpora, we evaluate five Re-ID backbones and textbf{MECHA}, a memory-augmented baseline we introduce, and find that the central difficulty lies somewhere unexpected. Recognising familiar characters is nearly solved. Knowing which matches to trust is not. When models build a gallery and grow it from their own predictions, their performance deteriorates, even though correct identity labels would improve it by over twenty points. This raises a simple question: when should a model trust its own match? We derive a single comparison that determines when gallery updates improve recognition and use it to design textbf{VOUCH}, which updates its memory only when the page provides supporting evidence. Without fitting additional parameters, VOUCH recovers between one-third and two-thirds of the improvement available from perfect labels. RE:COGNIZE measures whether a model can recognise characters as it reads; VOUCH shows how to remember them reliably. We focus our methodological claims on identity maintenance for previously encountered characters, while separately evaluating new-character emergence as a diagnostic under a fixed reference rule to distinguish recognition from discovery.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{mcgee2026trust,

title = {Trust the Direction, Search the Step: Zero-and-First-Order Methods for LLM Fine-Tuning},

author = {Cristian McGee and El Houcine Bergou and Aritra Dutta},

url = {https://openreview.net/forum?id=ZKbACxi4Qb},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Step-size selection remains a central challenge in large-scale neural network optimization: conservative steps slow convergence, while aggressive steps can destabilize training. We propose Zero-and-First-Order optimization (ZFO), a lightweight framework that decouples direction selection from step-size selection. ZFO uses a trusted first-order optimizer to determine the direction and performs zeroth-order evaluations only along this one-dimensional subspace to choose how far to move. Using the current forward-backward pass and two additional fixed-batch function evaluations, ZFO constructs a local model of the objective along the proposed direction and selects a curvature-aware step within a bounded search interval. This yields an adaptive step-selection mechanism with lower cost than full line search. We provide theoretical guarantees showing that shared-sample evaluations produce reliable finite-difference curvature estimates, that the induced local model selects a near-optimal step along the search interval, and that ZFO converges to a neighborhood of a stationary point. Across multiple language models and datasets, ZFO improves accuracy and stability over standard first-order fine-tuning algorithms.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{li2026attend,

title = {Attend Locally, Remember Linearly: Linear Attention as Cross-Frame Memory for Autoregressive Video Diffusion},

author = {Kunyang Li and Mubarak Shah and Yuzhang Shang},

url = {https://openreview.net/forum?id=BScHl5MZ8h},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Autoregressive (AR) video diffusion is a powerful paradigm for streaming and interactive video generation. However, its reliance on softmax self-attention leads to quadratic compute complexity in sequence length and memory usage due to key-value caching, which limits its scalability to long video horizons. Existing remedies (e.g., sparse attention and KV-cache compression) reduce per-step cost but still rely on a linearly growing cache or irreversibly discard past context, and thus fail to address linear memory growth and streaming context management. To address this scalability bottleneck, we propose ARL2 (Attend Locally, Remember Linearly), a hybrid attention module that replaces quadratic cross-frame attention with a fixed-size recurrent state. We decompose self-attention into two branches: an intra-frame softmax branch for spatial detail and local dependencies, and an inter-frame gated recurrent linear branch that maintains a fixed-size state for streaming context. Our key insight is that softmax attention captures fine-grained local interactions, while a recurrent state provides controllable long-range memory. This design achieves linear-time scaling with constant memory while improving temporal consistency over the full-softmax model. To prevent noisy intermediate states from corrupting memory, we update the recurrent state only after the denoised pass. To avoid within-frame information asymmetry, all tokens share the same pre-update state rather than sequential updates. To the best of our knowledge, this is the first work to convert a pretrained AR video diffusion model into a hybrid linear attention architecture, through an efficient two-stage training scheme for AR video. With 75% of layers replaced by hybrid linear attention, the model achieves up to 2.26 wall-clock speedup and 54% memory reduction, while maintaining comparable quality with improving temporal consistency.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{sun2026dynamically,

title = {Dynamically Structured Diffusion Language Model Decoding via Bayesian Inference},

author = {Bian Sun and Kevin Zhai and Mubarak Shah and Zhenyi Wang},

url = {https://openreview.net/forum?id=Z6jLmoD11G

https://arxiv.org/abs/2605.09820},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Diffusion language models (DLMs) have recently emerged as a promising alternative to autoregressive models, primarily due to their ability to enable parallel decoding. Despite this advantage, most existing DLMs rely on a fixed generation length specified prior to decoding, which restricts their flexibility in real-world applications. While a few recent works attempt to support flexible-length generation, they typically suffer from notable limitations: some require costly retraining to accommodate variable-length outputs, while others depend solely on local confidence signals during decoding. Such local criteria fail to capture the evolving structure of the sequence, often resulting in suboptimal generation quality. In this paper, we propose a training-free, Bayesian structured decoding framework that formulates flexible-length generation as a dynamic structural inference problem. Our approach formulates flexible-length generation as a dynamic structural inference problem, jointly computing the expansion length, the block boundaries, and the decoding schedule. At each window expansion step, the method integrates local uncertainty with structural signals via a unified mechanism that supports dynamic structured generation, including both flexible block expansion and block organization, while maintaining coherence. Extensive experiments across multiple benchmarks demonstrate that our approach significantly improves generation quality and flexibility over existing fixed-length and flexible-length baselines. These results highlight the advantage of Bayesian structured decoding for diffusion language model, providing a principled and efficient solution for structured text generation.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{siddiqui2026flashssm,

title = {FlashSSM-3D: A Distilled State Space Model for Lightning-Fast Dense 3D Reconstruction},

author = {Nyle Siddiqui and Mubarak Shah},

url = {https://openreview.net/forum?id=qrND6erSwi

https://nylesiddiqui.github.io/FlashSSM-3D-prefinal.pdf},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {3D geometric understanding is undergoing a major transition where rigid, deterministic, optimization-based pipelines are being replaced by large-scale, deep learning models. While recent methods achieve impressive performance on tasks such as multi-view 3D reconstruction and camera pose estimation, they suffer from two critical limitations: (i): current state-of-the-art (SOTA) models require industrylevel compute and training corpora, severely limiting accessibility, and (ii): recent approaches remain constrained to modifying existing pretrained transformer models, since substantially changing their architecture would require retraining from scratch. We instead study a complementary direction: training an attention-free state space model for dense 3D understanding. We introduce FlashSSM-3D, the first 3D reconstruction state space model capable of performing accurate geometric reasoning over hundreds of images with linear computational complexity. However, training such a model from scratch with competitive performance is impractical under typical academic compute constraints. To address this, we train FlashSSM-3D by distilling knowledge from VGGT, avoiding the need to learn 3D geometric priors entirely from scratch. However, standard distillation losses are poorly suited for dense 3D reconstruction, where preserving real-world spatial structure is essential. We therefore propose a geometry-aware knowledge distillation method specifically tailored for large-scale 3D reconstruction models. Our method introduces two novel losses: a Procrustes-based alignment loss that enforces global structural consistency between student and teacher 3D point predictions, and a camera reprojection loss that directly supervises view-consistent camera geometry between teacher and student. Extensive experiments across multi-view 3D reconstruction and camera pose estimation benchmarks demonstrate that FlashSSM-3D achieves competitive performance while requiring only ∼ 25% of the training data and less than 10% of the training compute used by VGGT. At inference time, FlashSSM-3D benefits from the practical efficiency of state space models, reconstructing scenes from 500+ views in under 10 seconds while maintaining competitive accuracy.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{shatwell2026georeferenced,

title = {Georeferenced Cross-View 3D Reconstruction from Ground and Satellite Images},

author = {David G Shatwell and Mubarak Shah},

url = {https://openreview.net/forum?id=h1zgNdJwcF

https://dshatwell23.github.io/documents/shatwell2026georeferenced.pdf},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Georeferenced 3D reconstruction from ground images requires recovering not only the local scene geometry, but also localizing every camera center and 3D point in a geographic coordinate frame. Existing cross-view geo-localization methods are able to estimate the GPS coordinate of a single ground camera, but do not reconstruct dense 3D geometry, while recent feed-forward multi-view 3D reconstruction models predict point maps and camera poses only in a local coordinate frame. We introduce X-VGGT, a feed-forward cross-view geometry transformer that unifies these tasks by jointly processing a set of ground-level images and a georeferenced satellite image to predict ground-view camera poses, dense 3D point maps, and a scene-level transform parameterized by gravity, scale, yaw and translation. This transform projects the local reconstruction into the satellite plane, resulting in the geo-registration of all ground images with the georeferenced satellite image. We further propose an evaluation protocol for this joint task and show that, across multiple cross-view datasets, X-VGGT produces accurate georeferenced reconstructions and outperforms prior cross-view geo-localization and multi-view 3D reconstruction models.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{bawatneh2026omnitom,

title = {OmniToM: Benchmarking Theory of Mind in LLMs via Explicit Belief Modeling},

author = {Adam Bawatneh and Sagar Sapkota and Amrit Singh Bedi and Santu Karmaker and Mubarak Shah},

url = {https://openreview.net/forum?id=wm4vgQFJmq

https://arxiv.org/abs/2605.26322},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Theory of Mind (ToM), the ability to infer others' knowledge, intentions, and emotions, is commonly evaluated in large language models (LLMs) using end-point question answering, where performance is judged solely by the final answer to a social reasoning query. This paradigm obscures whether the model actually constructs the underlying mental-state representations required for robust reasoning, particularly in scenarios involving divergent, evolving, or mistaken beliefs. In order to address this research gap, we introduce OmniToM, a benchmark that directly evaluates these representations by requiring explicit modeling of belief structures for all relevant actors within a narrative. These structures are composed of belief propositions: minimal statements of what an actor takes to be true about the world or another actor's mental state, allowing knowledge, intentions, emotions, and false beliefs to be analyzed in a common format. Models are evaluated in two stages: Stage 1: Belief Extraction, which extracts from the story the beliefs relevant to its social dynamics, and Stage 2: Belief Labeling, which assigns each belief a seven-dimensional schema label covering recursive order, truth status, knowledge access, explicitness, content type, mental source, and context. Built from 895 stories from the existing ToMBench story corpus and augmented with 22,343 labeled belief propositions, OmniToM uses a human-calibrated LLM-assisted annotation pipeline. Across diverse models in zero-shot evaluation, OmniToM reveals an actor-specific belief-tracking bottleneck: current LLMs struggle with the knowledge-access and representational decisions required to transform narrative facts into actors' beliefs and shared mental states.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{yousaf2026safe,

title = {Safe in Its Own Words: Self-Guided Safety Alignment for Multimodal Reasoning Models},

author = {Adeel Yousaf and Souradip Chakraborty and Mubarak Shah and Amrit Singh Bedi},

url = {https://openreview.net/forum?id=PFyto9A2fG

https://neurips.cc/virtual/2026/loc/atlanta/poster/152939},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Multimodal large reasoning models can reason over images and text, but they remain vulnerable to multimodal jailbreaks. Recent safety-alignment methods reduce attack success rate (ASR) by fine-tuning on reasoning traces from stronger teacher models. We show that this has a hidden cost of over-refusal. The aligned model learns to reject benign inputs that contain sensitive words or visual cues. This is especially harmful for boundary-safe inputs, where the prompt looks risky on the surface but is safe in context. We identify external teacher supervision as a key factor in this behavior. It introduces a distributional mismatch and shifts supervision away from the base model's own reasoning distribution. To address this, we propose SAGE (Safety-Aware Guided Elicitation), a self-guided data generation framework for multimodal safety alignment. SAGE guides generation in two stages: a reasoning-level guide first elicits a safety-aware trace, and a decision-level guide then elicits the final response conditioned on that trace. This factorized guidance lets SAGE shape both the reasoning process and the final safety decision without relying on external teacher traces. On LLaVA-CoT, SAGE reduces FigStep ASR from 86.2% to 2.1% and achieves an average over-refusal of 43.1, compared with 66.9--71.1 for prior reasoning-based safety-alignment methods, while preserving general utility. We will release the SAGE training dataset.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{lin2026efficient,

title = {Efficient and Transferable Agentic Knowledge Graph RAG via Reinforcement Learning},

author = {Junhong Lin and Shicheng Liu and Jinyeop Song and Song Wang and Julian Shun and Yada Zhu},

url = {https://arxiv.org/abs/2509.26383

http://github.com/junhongmit/KG-R1/},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Knowledge-graph retrieval-augmented generation (KG-RAG) couples large language models (LLMs) with structured, verifiable knowledge graphs (KGs) to reduce hallucination and provide reasoning traces. However, current KG-RAG systems often rely on fixed pipelines of multiple LLM modules (e.g., planning, reasoning, and responding), which inflate inference costs and tie performance to specific graph schemas. To address this, we introduce KG-R1, an agentic framework that optimizes KG-RAG through reinforcement learning (RL). Unlike modular workflows, KG-R1 uses a single agent that interacts with KGs as its environment, learning to retrieve information at each step and incorporating it into its reasoning and generation in a unified process. Across Knowledge-Graph Question Answering (KGQA) benchmarks, KG-R1 demonstrates both efficiency and transferability-using Qwen 2.5-3B, KG-R1 improves answer accuracy with fewer generation tokens than prior multi-module workflow methods that use much larger foundation or fine-tuned models. Furthermore, KG-R1 exhibits strong plug-and-play capability: after training, maintaining accuracy on unseen KGs without retraining. These properties make KG-R1 a promising KG-RAG framework for real-world deployment. Our code is publicly available at this http URL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{zhang2026jarvis,

title = {JARVIS-Bench: Benchmarking Personal Intelligence Agents on Long-Horizon Real-User Daily Traces},

author = {Weizhi Zhang and Wei-Chieh Huang and Yueqing Liang and Liwei Jiang and Zhengxiang Wang and Liangwei Yang and Zechen Li and Yuchen Wu and Haozhen Zhang and Yu Wang and Yuanchen Bei and Yue Zhou and Siqi Zhu and Henry P Zou and Jiahong Liu and Xinni Zhang and Paul Martin and Joseph Marvin Imperial and Yuyang Luo and Xiongxiao Xu and Baixiang Huang and Shanglin Wu and Lucas Resck and Yibo Wang and Yuqing Liu and Langzhou He and Chengze Li and Yuxin Tian and Kening Zheng and Enze Ma and Boi Huynh and Zheng Hui and Rui Yang and Tao Feng and Jie Yang and Shanghao Li and Haoran Wang and Wooseong Yang and Hanrong Zhang and Huanhuan Ma and Daye Yoon and Yaozu Wu and Shikan Lian and Xiaoqian Ruan and Fangxin Wang and Hyeonjeong Park and Hins Hu and Wenzhe Fan and Chen Wang and Yifan Gu and Dongyuan Li and Song Wang and Aylin Caliskan and Jindong Wang and Xiao Luo and Yankai Chen and Kai Shu and Steve Liu and Philip S Yu},

url = {https://openreview.net/forum?id=yGQt9wAGkS},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Personal intelligence agents aim to support users across daily-life tasks by learning their preferences, routines, needs, and emotional states from long-term interaction. They are expected to anticipate user needs, provide proactive assistance, and infer preferences that users have never explicitly articulated. However, existing benchmarks for LLM personalization lack real user data spanning extended periods and diverse daily-life domains, and typically reduce personalization to simplified settings such as recommendation or fact retrieval. We introduce JARVIS-Bench, a new benchmark for evaluating personal intelligence agents, curated from real users’ daily traces and augmented with synthetic trajectory expansions grounded in implicit-preference insights from surveys and structured personas. JARVIS-Bench is built from participants who contributed 28 to 44 days of multimodal self-tracking logs, structured personas, and a 100-question implicit-preference survey spanning eight domains. It supports four downstream task surfaces: implicit preference reasoning, proactive-help prediction, emotion-aware modeling, and cross-domain transfer, all evaluated through a unified event iterator and long-horizon user-memory pipeline. We benchmark 28 LLMs from 10 model families, including Gemini, GPT, Qwen, Gemma, and Claude, across a five-condition input ladder ranging from no memory to lifelong memory. Results show that memory and model scaling improve personalization, yet current frontier models remain far from achieving robust real-world personal intelligence.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{liu2026mitigating,

title = {Mitigating Object Hallucination in Large Vision-Language Models via False Discovery Controlled Visual Data Splitting},

author = {Chang Liu and Yu Tian and Rui Xie},

url = {https://openreview.net/forum?id=ek5KY0D3CL

https://changliu1993-cl.github.io/CORAL/

https://arxiv.org/abs/2609.38979},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Multiple object hallucination, where large vision-language models (LVLMs) generate objects not supported by the visual input, is a persistent challenge caused by visual uncertainty during decoding. Existing methods reduce hallucinations using contrastive signals, but they rely on heuristics and lack principled control of false positives at the image level. To address this, we propose False Discovery Rate-COntRol of HALlucination (CORAL), a training-free framework that models visual uncertainty using an uncertainty-aware visual data splitting strategy and leverages mirror statistics to quantify visual contrast during decoding. By computing mirror statistics from paired, symmetrically perturbed visual inputs, CORAL estimates spurious object predictions and sets a data-driven threshold to control the expected fraction of false discoveries per image, suppressing hallucinations while retaining high power for truly grounded objects. The framework is flexible, supports multiple LVLMs, and mitigates hallucinations without retraining or supervision. Extensive experiments on multiple benchmarks with several evaluation metrics demonstrate that CORAL consistently outperforms state-of-the-art methods, providing more reliable and robust hallucination control. Code is available at: this https URL},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{li2026dd,

title = {DD-Ranking: Rethinking the Evaluation of Dataset Distillation},

author = {Zekai Li and Xinhao Zhong and Samir Khaki and Zhiyuan Liang and Yuhao Zhou and Mingjia Shi and Ziqiao Wang and Xuanlei Zhao and Wangbo Zhao and Ziheng Qin and Mengxuan Wu and Pengfei Zhou and Haonan Wang and David Junhao Zhang and Jia-Wei Liu and Shaobo Wang and Dai Liu and Linfeng Zhang and Guang Li and Kun Wang and Zheng Zhu and Zhiheng Ma and Joey Tianyi Zhou and Jiancheng Lv and Yaochu Jin and Peihao Wang and Kaipeng Zhang and Yiran Huang and Zhiwei Deng and Xindi Wu and George Cazenavette and Yuzhang Shang and Justin CUI and Jindong Gu and Qian Zheng and Hao Ye and Shuo Wang and Xiaobo Wang and Yan Yan and Angela Yao and Mike Zheng Shou and Tianlong Chen and Hakan Bilen and Baharan Mirzasoleiman and Manolis Kellis and Konstantinos N Plataniotis and Zhangyang ""Atlas"" Wang and Bo Zhao and Yang You and Kai Wang},

url = {https://arxiv.org/abs/2505.13300},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {In recent years, dataset distillation has provided a reliable solution for data compression, where models trained on the resulting smaller synthetic datasets achieve performance comparable to those trained on the original datasets. To further improve the performance of synthetic datasets, various training pipelines and optimization objectives have been proposed, greatly advancing the field of dataset distillation. Recent decoupled dataset distillation methods introduce soft labels and stronger data augmentation during the post-evaluation phase and scale dataset distillation up to larger datasets (e.g., ImageNet-1K). However, this raises a question: Is accuracy still a reliable metric to fairly evaluate dataset distillation methods? Our empirical findings suggest that the performance improvements of these methods often stem from additional techniques rather than the inherent quality of the images themselves, with even randomly sampled images achieving superior results. Such misaligned evaluation settings severely hinder the development of DD. Therefore, we propose DD-Ranking, a unified evaluation framework, along with new general evaluation metrics to uncover the true performance improvements achieved by different methods. By refocusing on the actual information enhancement of distilled datasets, DD-Ranking provides a more comprehensive and fair evaluation standard for future research advancements.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{buck2026topological,

title = {TopoGED: A Topological Encoder Decoder Framework for Temporal Graph Learning},

author = {Ronan Buck and Kiarash Shamsi and Tran Gia Bao Ngo and Astrit Tola and Baris Coskunuzer and Cuneyt Akcora},

url = {https://openreview.net/forum?id=cZEj9pgOP7},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{uddin2026nodes,

title = {From Nodes to Pixels: Topological and Structural Two-View Graph Imaging},

author = {Md Joshem Uddin and Soham Changani and Sai K Navuluru and Cuneyt Akcora and Baris Coskunuzer},

url = {https://github.com/sohamchangani/G2Image},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {G2Image is a deep learning framework that transforms graph-structured data into grid-based image representations. It leverages a dual-view approach—combining structural information (via Adjacency Grid Images) and topological features (via Multipersistence Toposurfaces)—to classify graphs using a CNN-Transformer architecture with Supervised Contrastive Learning.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{shamsi2026hydra,

title = {Hydra: Towards Transferable Multi-Task Learning on Temporal Graphs},

author = {Kiarash Shamsi and Farimah Poursafaei and Tran Gia Bao Ngo and Reihaneh Rabbany and Baris Coskunuzer and Guillaume Rabusseau and Michael Bronstein and Shenyang Huang and Cuneyt Akcora},

url = {

https://openreview.net/forum?id=prXn189GaI},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Real-world evolving networks are naturally modeled as temporal graphs (TGs), where capturing temporal dynamics is essential for predicting future graph properties that support downstream decision-making. Existing temporal graph methods have been developed primarily for single-task prediction, and little is known about their generalization across tasks or transfer to unseen networks. This leaves the challenge of multi-task graph property prediction in TGs largely open. We address this challenge by introducing Hydra, a novel architecture that integrates local connectivity features from temporal GNNs with a spectral learning module that captures global connectivity patterns. This design enables joint learning of local and global information under a multi-task objective. In multi-task classification, Hydra achieves an 8.9% relative gain in AUC over the strongest competitor. In multi-task regression, Hydra achieves competitive results in all three tasks, while obtaining the best results in two tasks with a 8.2% relative gain in MAE compared to the strongest baseline. Moreover, Hydra delivers these gains with a 22× reduction in training time compared to temporal transfer models. These results provide the first systematic evidence that multi-task transferable learning on temporal graphs is effective. By delivering consistent top-ranked performance, Hydra highlights multi-task training on temporal graphs as a promising direction toward adaptable foundation models for temporal graphs.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{zou2026conditional,

title = {Conditional Multi-Event Temporal Grounding in Long-Form Video},

author = {Yuanhao Zou and Arthad Kulkarni and Lucas Toñanez and Lincoln Spencer and Guangyu Sun and Tianxingjian Ding and Andong Deng and Yi Li and Shuangjun Liu and Yuan Li and Dashan Gao and Ning Bi and TAOTAO JING and Shuai Zhang and Chen Chen},

url = {https://openreview.net/forum?id=i6iM7B1Koa

https://arxiv.org/html/2606.15320v1},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Multimodal large language models have made rapid progress in video temporal grounding, yet real-world applications routinely require localizing every event that satisfies compositional temporal and spatial conditions. Existing benchmarks fall short: they localize only a single moment per query, count without temporal conditions, or treat grounding and counting as disjoint tasks. We introduce CoMET-Bench for Conditional Multi-Event Temporal Grounding in long-form video, comprising 2,789 queries over 600 videos averaging 33.8 minutes across five real-world domains, with each query composed from 4 temporal conditions, 3 spatial conditions, and a dedicated negative-query subset. We further propose a unified evaluation protocol jointly measuring counting, grounding, and negative-query recognition, including a new Rejection-F1 metric that prevents trivial gaming by lazy “always-empty” models. Benchmarking a broad suite of MLLMs, agent-based, and grounding-specialized methods reveals that existing approaches remain far from solving this task. Building on these findings, we propose CoMET-Agent, a training-free agentic framework that reformulates the task as structured search-and-aggregate, improving F1@0.5 by 6.1% over GPT-5 purely through structural reasoning. Failure analysis further surfaces three open directions: fine-grained entity tracking, position-uniform retrieval, and causal event pairing.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{kwon2026factorizedhmr,

title = {FactorizedHMR: A Hybrid Framework for Video Human Mesh Recovery},

author = {Patrick Kwon and Chen Chen},

url = {https://openreview.net/forum?id=7ERjBgSwbw

https://arxiv.org/abs/2605.14854},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Human Mesh Recovery (HMR) is fundamentally ambiguous: under occlusion or weak depth cues, multiple 3D bodies can explain the same image evidence. This ambiguity is not uniform across the body, as torso pose and root structure are often relatively well constrained, whereas distal articulations such as the arms and legs are more uncertain. Building on this observation, we propose FactorizedHMR, a two-stage framework that treats these two regimes differently. A deterministic regression module first recovers a stable torso-root anchor, and a probabilistic flow-matching module then completes the remaining non-torso articulation. To make this completion reliable, we combine a composite target representation with geometry-aware supervision and feature-aware classifier-free guidance, preserving the torso-root anchor while improving single-reference recovery of ambiguity-prone articulation. We also introduce a synthetic data pipeline that provides the paired image-camera-motion supervision under diverse viewpoints. Across camera-space and world-space benchmarks, FactorizedHMR remains competitive with strong baselines, with the clearest gains in occlusion-heavy recovery and drift-sensitive world-space metrics.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{dong2026rethinking,

title = {Rethinking Long-Video Efficiency: A Joint Allocation Perspective on Frames, Pixels, and Front-End Latency},

author = {Sixun Dong and Wei Li and Andong Deng and Qi Qian and Victor Zhu and Zhengping Ji and Chen Chen},

url = {https://sixundong.com/projects/lohi},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Efficient long-video understanding with vision-language models (VLMs) is often framed as selecting informative frames or visual tokens at a fixed native resolution. We show that per-frame resolution can instead be traded for denser temporal coverage, while front-end decoding latency depends on the size of the candidate pool rather than the final token budget. An empirical study across multiple VLMs and long-video benchmarks yields three findings: dense low-resolution sampling outperforms sparse native-resolution sampling at matched token budgets; resolution-sensitive tasks benefit from selected high-resolution frames; and front-end decoding dominates wall time for hour-long videos. Motivated by these findings, we introduce LoHi, a training-free, single-pass framework that combines a dense low-resolution video stream with sparse high-resolution image frames through the VLM's native video and image pathways. LoHi-Anchor selects high-resolution frames using codec I-frame metadata, while LoHi-SemDiv uses query relevance and visual diversity over CLIP features. Across three long-video benchmarks, LoHi improves average accuracy by 10.6 percentage points over the native-resolution baseline at a matched token budget and by 5.2 percentage points over the strongest prior efficiency method. It also reduces front-end decoding latency by up to 7x on hour-long videos. Project page: this https URL},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{kowsher2026where,

title = {Where You Backpropagate Matters: Token Hypothesis for Memory-Efficient Fine-Tuning},

author = {Md Kowsher and Chen Chen},

url = {https://openreview.net/forum?id=8D8sa1Nuh0},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Fine-tuning large transformers routes gradients through every token position, paying the cost in activation memory that scales linearly with sequence length and dominates the GPU budget. Existing efficient methods reduce the parameter side—low-rank adapters, prefix tuning, weight-decomposed updates—but leave the activation footprint untouched. We present the Token Hypothesis, a memory-efficient fine-tuning procedure motivated by a Neural Tangent Kernel analysis of how fine-tuning signal is distributed across token positions. We prove that every position is a low-dimensional capacity bottleneck whose width does not grow with the model, that positions with aligned hidden-state geometry are interchangeable in bidirectional models, and that causal positions form a monotone capacity chain in which later positions strictly dominate earlier ones. The analysis provides an assumption-qualified criterion for estimating how many positions are needed and which ones, using quantities computable from forward passes alone: in bidirectional models, small arbitrary subsets are predicted to suffice when position-wise kernels are aligned, while in causal models, later positions are predicted to carry the richest fine-tuning signal. We translate this into a procedure that backpropagates through only a small subset of positions while running the full forward pass over the rest. Empirically, the procedure matches or exceeds full-sequence fine-tuning across six task families—commonsense reasoning, mathematical reasoning, visual question answering, video question answering, GLUE, and image classification—and five backbones spanning bidirectional and causal architectures. It reduces activation memory substantially, translating to up to ~20% lower total GPU memory (~33 GB) on long-context tasks, and composes cleanly with parameter-efficient methods such as LoRA and DoRA.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{lyu2026mls,

title = {MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI},

author = {Bohan Lyu and Yucheng Yang and Siqiao Huang and Jiaru Zhang and Qixin Xu and Xinghan Li and Xinyang Han and Huaqing Zhang and Yicheng Zhang and Runhan Huang and Kaicheng Yang and Zitao Chen and Wentao Guo and Junlin Yang and Xinyue Ai and Wenhao Chai and Yadi Cao and Ziran Yang and Kun Wang and Dapeng Jiang and Huan-ang Gao and Shange Tang and Chengshuai Shi and Simon Du and Max Simchowitz and Jiantao Jiao and Dawn Song and Chi Jin},

url = {https://openreview.net/forum?id=EaF7FneBfZ

https://mls-bench.com/},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Modern AI progress has been driven by ML methods that are generalizable across settings and scalable to larger regimes. As large language models demonstrate advanced capabilities in reasoning, coding, and engineering tasks, it is increasingly important to understand whether they can discover such methods rather than only apply existing ones. We introduce MLS-Bench, a benchmark for evaluating whether AI systems can invent generalizable and scalable ML methods. MLS-Bench contains 140 tasks across 12 domains, each requiring an agent to improve one targeted component of an ML system or algorithm and demonstrate that the improvement generalizes across controlled settings and scales. We find that current agents remain far from reliably surpassing human-designed methods, and that engineering-style tuning is easier for them than genuine method invention. We further study the effects of test-time scaling, adaptive compute allocation, and context provision on agents' discovery performance, together with case studies of their behavior. Our analyses suggest that the bottleneck is not only in proposing new methods, but also in the scientific insight needed to plan, validate, and scale claims about them. More search, compute, or context alone does not remove this bottleneck. We build and maintain a community platform for cumulative and comparable iteration, and release the data and code at this https URL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{jore2026bifurcation,

title = {Bifurcation Models: Learning Set-Valued Solution Maps with Weight-Tied Dynamics},

author = {Caleb Jore and Jialin Liu},

url = {https://openreview.net/forum?id=tVZohLsVHC},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Many scientific and combinatorial problems admit multiple correct solutions, not a single label. Standard supervised learning resolves this ambiguity by choosing one solution as the target, but this hidden selector can be arbitrary, discontinuous, and harder to learn than the underlying solution set. We study bifurcation models, a weight-tied dynamical view in which different initializations can converge to different stable equilibria, so the model represents an attractor landscape rather than one chosen branch. We prove that broad set-valued maps with locally Lipschitz branches can be represented by regular equilibrium dynamics and that the induced selectors are almost everywhere regular, while manual selectors can be arbitrarily irregular. Experiments on frustrated Ising models show that such dynamics can discover multiple valid equilibria without branch labels and outperform single-branch supervision. Allen--Cahn experiments further show that diversity is not automatic: it can be encouraged explicitly, but with an accuracy--diversity tradeoff.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{do2026simulating,

title = {Simulating Students or Sycophantic Problem Solving? On Misconception Faithfulness of LLM Simulators},

author = {Heejin Do and Shashank Sonkar and Mrinmaya Sachan},

url = {https://openreview.net/forum?id=t2Q9ciLFrV

https://arxiv.org/abs/2605.12748},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Large language models (LLMs) can fluently generate student-like responses, making them attractive as simulated students for training and evaluating AI tutors and human educators. Yet such simulators are typically evaluated by output similarity to real students, not by whether they behave like students with coherent misconceptions during interaction. We introduce a controlled framework for evaluating misconception faithfulness, whether a simulator maintains a misconception-driven belief state and updates selectively when feedback addresses the underlying misconception. Central to our framework is a misconception-contrastive feedback protocol that compares targeted feedback against two controls: misaligned feedback (targeting a different but plausible misconception) and generic feedback (only identifying answer is wrong). We propose Selective Flip Score (SFS), which quantifies how much more often a simulator flips its answer under targeted feedback than under contrastive controls. Across seven LLMs (4B-120B), multiple datasets, and prompting strategies, simulators exhibit near-zero SFS, correcting their answers at similarly high rates regardless of feedback relevance. Further analyses reveal a sycophantic failure mode: models behave less like students with misconceptions but more like problem-solvers who treat any corrective signal as a cue to abandon the simulated belief and re-solve from internal knowledge. To address this, we develop a post-training pipeline spanning supervised fine-tuning (SFT), preference optimization, and reinforcement learning (RL) with an SFS-aligned reward; SFT yields notable gains up to +0.56, and SFS-aligned RL provides more consistent improvements than preference optimization. Our results establish misconception faithfulness as a challenging yet trainable property, motivating a shift from static output matching toward interactive, belief-aware student modeling.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{ahmed2026robust,

title = {Robust Nash Alignment under Preference Uncertainty},

author = {Shihab Ahmed and Debamita Ghosh and David Tang and Yudan Wang and Alvaro Velasquez and Yue Wang},

url = {https://openreview.net/forum?id=CiLFISXIZl},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Preference-based alignment methods typically optimize against a single preference model, and can therefore be brittle when pairwise preferences are uncertain: noisy, heterogeneous, or shift after deployment. To address these issues, we propose Robust Nash Alignment, a game-theoretic framework for alignment to uncertain pairwise preferences. Our formulation has a major learner seeking a policy with a large worst-case win rate against both an adversarial competitor and any preference kernel lying in an ambiguity set around a nominal preference. When the ambiguity set captures the uncertainty in preferences, the resulting robust objective of the game directly yields a certified lower bound on worst-case performance. However, we note this problem is computationally challenging to optimize, and to address this, we introduce a four-player primal-dual proxy game involving the leader policy, follower policy, adversarial kernel, and dual variable, and develop a single-loop optimistic mirror descent-ascent algorithm for it. We show that the proxy always lower-bounds the truncated hard-constrained objective, quantify the proxy-to-hard gap, and characterize an exactness condition under which the proxy recovers the robust objective. We then prove an (mathcal{O}(1/sqrt{T})) average-iteration convergence for the proxy-game duality gap, which implies a near-optimal robust policy for the original robust objective. Experiments on controlled tabular games and LLM alignment with uncertain preference further validate the convergence theory and show improved performance over nominal baselines.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{sklaviadis2026bayesian,

title = {The Bayesian Learning Rule Beyond KL Geometry},

author = {Sophia Sklaviadis and Wu Lin and Mario A T Figueiredo and Andre Martins and Thomas Möllenhoff and Mohammad Emtiyaz Khan},

url = {https://openreview.net/forum?id=iMEeZOdaUi

https://arxiv.org/abs/2610.31308},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {The Bayesian learning rule (BLR) exploits the relation between Bayesian updating and the Kullback-Leibler divergence (KLD) to derive efficient additive updates for exponential family variational posteriors. The KL geometry is powerful but restrictive: combined with posterior families with parameter-dependent support it can lead to undefined objectives. We ask which non-KL geometries preserve the additive BLR computation. Previous works that use f-divergences suggest that this is only possible with the KLD. Here, we show that this is not true. We show that additive information-geometric optimization algorithms can be derived for the class of mixed-space Bregman or Fenchel-Young discrepancies. These discrepancies induce generalized maximum-entropy families, dual coordinates, and generalized-BLR canonical gradient updates, recovering the classical BLR. The generalized BLR leads to new algorithms that textit{cannot} be derived from the KL-based BLR. For the Tsallis negentropy with quadratic statistics, the generalized BLR yields bounded-support and heavy-tailed q-Gaussian posteriors and an easy Newton-like implementation. Experiments up to CIFAR-100 illustrate controllable support and tail behavior and competitive performance.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{minut2026soap,

title = {SOAP-Bubbles: Effective and Scalable Variational Learning with Structured Covariances},

author = {Adrian Robert Minut and Nico Daheim and Marco Miani and Mohammad Emtiyaz Khan and Wu Lin and Thomas Möllenhoff},

url = {https://openreview.net/forum?id=YBJkwSJVbo

https://arxiv.org/abs/2606.23357},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Structured weight-uncertainty can improve many aspects of deep learning, but it remains costly to estimate and difficult to implement. Here, we show that these issues can be addressed by adapting the SOAP optimizer. Our key idea is to run IVON, an existing diagonal-covariance variational method, in the eigenspace of SOAP's preconditioner and then use the preconditioner to transform the diagonal estimate into a non-diagonal covariance. The resulting method has costs similar to those of SOAP and requires no drastic changes to training pipelines. We call the posteriors obtained in this way SOAP-Bubbles and our new optimizer Eigenspace-VON (EVON). We show that, for logistic regression, EVON recovers the exact Gaussian covariance and that, for language model pretraining, it yields significantly better results than existing diagonal-covariance methods. Our work makes it easier to estimate more expressive posterior distributions for deep learning at scale.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{persaud2026brute,

title = {Brute-Force Jailbreaks and Codon-Aware Watermarking for DNA Foundation Models},

author = {Munish B Persaud and Amrit Singh Bedi and Souradip Chakraborty},

url = {https://openreview.net/forum?id=6JUgH59aJ1

https://neurips.cc/virtual/2026/loc/atlanta/poster/155344},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{chehade2026test,

title = {Test-time Risk Adaptation with Mixture of Agents},

author = {Mohamad Chehade and Amrit Singh Bedi and Souradip Chakraborty and Amy Zhang and Hao Zhu},

url = {https://openreview.net/forum?id=xjpWCBDACk},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Deployed reinforcement learning agents must satisfy safety requirements that emerge only at test time—evolving regulations, unexpected hazards, or shifted operational priorities. Current risk-aware methods embed fixed risk models (typically return variance) during training, but this approach suffers from two fundamental limitations: it restricts risk expressiveness to trajectory-level statistics, and it induces uniform conservatism that reduces behavioral coverage needed for effective deployment adaptation. We propose TRAM (Test-time Risk Adaptation via Mixture of Agents), a deployment-time framework that composes risk-neutral source policies to satisfy arbitrary risk specifications without retraining. TRAM represents risk through occupancy-based functionals that capture spatial constraints, behavioral drift, and local volatility—risk types that trajectory variance cannot encode. Our theoretical analysis provides localized performance bounds that cleanly separate reward transfer quality from risk alignment costs, and proves that risk-neutral source training is minimax optimal for deployment risk adaptation. Empirically, TRAM delivers superior safety-performance trade-offs across gridworld, continuous control, and large language model domains while maintaining computational efficiency through successor feature implementation.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{barakat2026why,

title = {Why Pass@k Optimization Can Degrade Pass@1: Prompt Interference in LLM Post-Training},

author = {Anas Barakat and Souradip Chakraborty and Khushbu Pahwa and Amrit Singh Bedi},

url = {https://openreview.net/forum?id=TCynOWswES

https://arxiv.org/abs/2602.21189},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Pass@k is a widely used performance metric for verifiable large language model tasks, including mathematical reasoning, code generation, and short-answer reasoning. It defines success if any of k independently sampled solutions passes a verifier. This multi-sample inference metric has motivated inference-aware fine-tuning methods that directly optimize pass@k. However, prior work reports a trade-off: pass@k improves while pass@1 degrades under such methods. This trade-off is practically important because pass@1 often remains a hard operational constraint due to latency and cost budgets, imperfect verifier coverage, and the need for a reliable single-shot fallback. We study the origin of this trade-off and provide a theoretical characterization of when pass@k policy optimization can reduce pass@1 through gradient conflict induced by prompt interference. This interference arises because shared model parameters couple prompts, so improving performance on some prompts can adversely affect others. We show that pass@k policy gradients can conflict with pass@1 gradients because pass@k optimization implicitly reweights prompts toward low-success prompts; when these prompts are negatively interfering, their upweighting can rotate the pass@k update direction away from the pass@1 direction. We illustrate our findings with large language model experiments on verifiable mathematical reasoning and coding tasks.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{cui2026closer,

title = {A Closer Look at Dynamic Scene Graph Generation in the Era of Multimodal Large Language Models},

author = {Xuanming Cui and Jaiminkumar A Bhoi and Chionh W Peng and Adriel Kuek and Ser Nam Lim},

url = {https://openreview.net/forum?id=Y1D9E4ZhwT

https://arxiv.org/abs/2503.15846},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

abstract = {Dynamic Scene Graph Generation (DSGG) aims to capture objects and their evolving relations in videos. Despite recent progress, the practicality and quality of generated scene graphs remain limited compared to the rapid advances in Multimodal Large Language Models (MLLMs). In this work, we revisit DSGG from two fundamental perspectives: task setup and model design. From the task setup perspective, we identify two key limitations of the current recall-oriented evaluation protocol: (i) a severe precision-recall trade-off, and (ii) uninformative and redundant relation generation. To better assess practical usefulness, we introduce five additional metrics that provide a more comprehensive evaluation of the quality of generated dynamic scene graphs. From the model design perspective, we explore directly using MLLMs for scene graph generation and establish a strong MLLM-based DSGG baseline through three design changes. First, we replace the conventional bottom-up pipeline with a top-down reason-then-locate strategy. Second, we reformulate frame-wise dynamic graphs as Temporal Relation Set (TRS) prediction, improving both efficiency and performance. Third, we introduce Importance-Aware Finetuning (IAF) to encourage more relevant and diverse relation generation. Extensive experiments on Action Genome, VidVRD, and PVSG show that our approach consistently achieves state-of-the-art performance.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{hossain2026evaluating,

title = {Evaluating Whether LLMs Can Reliably Connect the DOTs?},

author = {Eftekhar Hossain and John Salvador and Santu Karmaker},

url = {https://openreview.net/forum?id=St4No81OAc},

year  = {2026},

date = {2026-01-01},

urldate = {2026-01-01},

booktitle = {Advances in Neural Information Processing Systems},

publisher = {NeurIPS 2026},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025,

title = {Adaptive Defense against Harmful Fine-Tuning via Bayesian Data Scheduler},

author = {Zixuan Hu and Li Shen and Zhenyi Wang and Yongxian Wei and Dacheng Tao},

url = {https://neurips.cc/virtual/2025/poster/115659},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Harmful fine-tuning poses critical safety risks to fine-tuning-as-a-service for large language models. Existing defense strategies preemptively build robustness via attack simulation but suffer from fundamental limitations: (i) the infeasibility of performing attack simulation due to lacking prior knowledge about potential attack data, and (ii) limited adaptability to varying attack settings, as simulation fails to capture their variability and complexity. To address these challenges, we propose Bayesian Data Scheduler (BDS), an adaptive tuning-stage defense strategy with no need for attack simulation. BDS formulates harmful fine-tuning defense as a Bayesian inference problem, learning the posterior distribution of each data point's safety attribute, conditioned on the fine-tuning and alignment datasets. The fine-tuning process is then constrained by weighting data with their safety attributes sampled from the posterior, thus mitigating the influence of harmful data. By leveraging the post hoc nature of Bayesian inference, the posterior is conditioned on the fine-tuning dataset, enabling BDS to tailor its defense to the specific dataset, thereby achieving adaptive defense. Furthermore, we introduce a neural scheduler based on amortized Bayesian learning, enabling efficient transfer to new data without retraining. Comprehensive results across diverse attack and defense settings demonstrate the state-of-the-art performance of our approach.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tola2025,

title = {TopER: Topological Embeddings in Graph Representation Learning},

author = {Astrit Tola and Funmilola Mary Taiwo and Cuneyt Gurcan Akcora and Baris Coskunuzer},

url = {https://arxiv.org/abs/2410.01778},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Graph embeddings play a critical role in graph representation learning, allowing machine learning models to explore and interpret graph-structured data. However, existing methods often rely on opaque, high-dimensional embeddings, limiting interpretability and practical visualization.

In this work, we introduce Topological Evolution Rate (TopER), a novel, low-dimensional embedding approach grounded in topological data analysis. TopER simplifies a key topological approach, Persistent Homology, by calculating the evolution rate of graph substructures, resulting in intuitive and interpretable visualizations of graph data. This approach not only enhances the exploration of graph datasets but also delivers competitive performance in graph clustering and classification tasks. Our TopER-based models achieve or surpass state-of-the-art results across molecular, biological, and social network datasets in tasks such as classification, clustering, and visualization.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shamsi2025,

title = {MiNT: Multi-Network Transfer Benchmark for Temporal Graph Learning},

author = {Kiarash Shamsi and Tran Gia Bao Ngo and Razieh Shirzadkhani and Shenyang Huang and Farimah Poursafaei and Poupak Azad and Reihaneh Rabbany and Baris Coskunuzer and Guillaume Rabusseau and Cuneyt Gurcan Akcora},

url = {https://neurips.cc/virtual/2025/poster/121574

https://github.com/benjaminnNgo/ScalingTGNs},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Temporal Graph Learning (TGL) aims to discover patterns in evolving networks or temporal graphs and leverage these patterns to predict future interactions. However, most existing research focuses on learning from a single network in isolation, leaving the challenges of within-domain and cross-domain generalization largely unaddressed. In this study, we introduce a new benchmark of 84 real-world temporal transaction networks and propose Temporal Multi-network Transfer (MiNT), a pre-training framework designed to capture transferable temporal dynamics across diverse networks. We train MiNT models on up to 64 transaction networks and evaluate their generalization ability on 20 held-out, unseen networks. Our results show that MiNT consistently outperforms individually trained models, revealing a strong relation between the number of pre-training networks and transfer performance. These findings highlight scaling trends in temporal graph learning and underscore the importance of network diversity in improving generalization. This work establishes the first large-scale benchmark for studying transferability in TGL and lays the groundwork for developing Temporal Graph Foundation Models. Our code is available at url{https://github.com/benjaminnNgo/ScalingTGNs}},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Lyu2025b,

title = {CPO: Condition Preference Optimization for Controllable Image Generation},

author = {Zonglin Lyu and Ming Li and Xinxin Liu and Chen Chen},

url = {https://neurips.cc/virtual/2025/poster/117815},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {To enhance controllability in text-to-image generation, ControlNet introduces image-based control signals, while ControlNet++ improves pixel-level cycle consistency between generated images and the input control signal. To avoid the prohibitive cost of back-propagating through the sampling process, ControlNet++ optimizes only low-noise timesteps (e.g., 

) using a single-step approximation, which not only ignores the contribution of high-noise timesteps but also introduces additional approximation errors. A straightforward alternative for optimizing controllability across all timesteps is Direct Preference Optimization (DPO), a fine-tuning method that increases model preference for more controllable images (

) over less controllable ones (

). However, due to uncertainty in generative models, it is difficult to ensure that win--lose image pairs differ only in controllability while keeping other factors, such as image quality, fixed. To address this, we propose performing preference learning over control conditions rather than generated images. Specifically, we construct winning and losing control signals, 

 and 

, and train the model to prefer 

. This method, which we term textit{Condition Preference Optimization} (CPO), eliminates confounding factors and yields a low-variance training objective. Our approach theoretically exhibits lower contrastive loss variance than DPO and empirically achieves superior results. Moreover, CPO requires less computation and storage for dataset curation. Extensive experiments show that CPO significantly improves controllability over the state-of-the-art ControlNet++ across multiple control types: over 

% error rate reduction in segmentation, 

-

% in human pose, and consistent 

%-

% reductions in edge and depth maps. Here, the error rate is defined as the difference between evaluated controllability and oracle results.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025b,

title = {Multi-Agent Debate for LLM Judges with Adaptive Stability Detection},

author = {Tianyu Hu and Zhen Tan and Song Wang and Huaizhi Qu and Tianlong Chen},

url = {https://neurips.cc/virtual/2025/poster/117644},

year  = {2025},

date = {2025-11-30},

abstract = {With advancements in reasoning capabilities, Large Language Models (LLMs) are increasingly employed for automated judgment tasks.While LLMs-as-Judges offer promise in automating evaluations, current approaches often rely on simplistic aggregation methods (e.g., majority voting), which can fail even when individual agents provide correct answers.To address this, we propose a multi-agent debate judge framework where agents collaboratively reason and iteratively refine their responses. We formalize the debate process mathematically, analyzing agent interactions and proving that debate amplifies correctness compared to static ensembles.To enhance efficiency, we introduce a stability detection mechanism that models judge consensus dynamics via a time-varying Beta-Binomial mixture, with adaptive stopping based on distributional similarity (Kolmogorov-Smirnov test).This mechanism models the judges' collective correct rate dynamics using a time-varying mixture of Beta-Binomial distributions and employs an adaptive stopping criterion based on distributional similarity (Kolmogorov-Smirnov statistic). Experiments across multiple benchmarks and models demonstrate that our framework improves judgment accuracy over majority voting while maintaining computational efficiency.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{He2025,

title = {Hierarchical Demonstration Order Optimization for Many-shot In-Context Learning},

author = {Yinhan He and Wendy Zheng and Song Wang and Zaiyi Zheng and Yushun Dong and Yaochen Zhu and Jundong Li},

url = {https://neurips.cc/virtual/2025/poster/119561

https://anonymous.4open.science/r/HIDO-B2DE/},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {In-Context Learning (ICL) is a technique where large language models (LLMs) leverage multiple demonstrations (i.e., examples) to perform tasks. With the recent expansion of LLM context windows, many-shot ICL (generally with more than 50 demonstrations) can lead to significant performance improvements on a variety of language tasks such as text classification and question answering.Nevertheless, ICL faces the issue of demonstration order instability (ICL-DOI), which means that performance varies significantly depending on the order of demonstrations. Moreover, ICL-DOI persists in many-shot ICL, validated by our thorough experimental investigation.Current strategies for handling ICL-DOI are not applicable to many-shot ICL due to two critical challenges: (1) Most existing methods assess demonstration order quality by first prompting the LLM, then using heuristic metrics based on the LLM's predictions. In the many-shot scenarios, these metrics without theoretical grounding become unreliable, where the LLMs struggle to effectively utilize information from long input contexts, making order distinctions less clear.(2) The requirement to examine all orders for the large number of demonstrations is computationally infeasible due to the super-exponential complexity of the order space in many-shot ICL. To tackle the first challenge, we design a demonstration order evaluation metric based on information theory for measuring order quality, which effectively quantifies the usable information gain of a given demonstration order.To address the second challenge, we propose a hierarchical demonstration order optimization method named HIDO that enables a more refined exploration of the order space, achieving high ICL performance without the need to evaluate all possible orders. Extensive experiments on multiple LLMs and real-world datasets demonstrate that our HIDO method consistently and efficiently outperforms other baselines. Our code project can be found at https://anonymous.4open.science/r/HIDO-B2DE/.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2025,

title = {From Play to Replay: Composed Video Retrieval for Sports Highlights},

author = {Animesh Gupta and Jay Parmar and Ishan Rajendrakumar Dave and Mubarak Shah},

url = {https://neurips.cc/virtual/2025/poster/121717

https://animesh-007.github.io/TF-CoVR-WEBSITE/},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Composed Video Retrieval (CoVR) retrieves a target video given a query video and a modification text describing the intended change. Existing CoVR benchmarks emphasize appearance shifts or coarse event changes and therefore do not test the ability to capture subtle, fast-paced temporal differences. We introduce TF-CoVR, the first large-scale benchmark dedicated to temporally fine-grained CoVR. TF-CoVR focuses on gymnastics and diving and provides 1.8 M triplets drawn from FineGym and FineDiving. Previous CoVR benchmarks focusing on temporal aspect, link each query to a single target segment taken from the same video, limiting practical usefulness. In TF-CoVR, we instead construct each pair by prompting an LLM with the label differences between clips drawn from different videos; every pair is thus associated with multiple valid target videos (3.9 on average), reflecting real-world tasks such as sports-highlight generation. To model these temporal dynamics we propose TF-CoVR-Base, a concise two-stage training framework: (i) pre-train a video encoder on fine-grained action classification to obtain temporally discriminative embeddings; (ii) align the composed query with candidate videos using contrastive learning. We conduct the first comprehensive study of image, video, and general multimodal embedding (GME) models on temporally fine-grained composed retrieval in both zero-shot and fine-tuning regimes. On TF-CoVR, TF-CoVR-Base improves zero-shot mAP@50 from 5.92 (LanguageBind) to 7.51, and after fine-tuning raises the state of the art from 19.83 to 25.82.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhao2025,

title = {Efficient Multimodal Dataset Distillation via Generative Models},

author = {Zhenghao Zhao and Haoxuan Wang and Junyi Wu and Yuzhang Shang and Gaowen Liu and Yan Yan},

url = {https://neurips.cc/virtual/2025/poster/119089},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Dataset distillation aims to synthesize a small dataset from a large dataset, enabling the model trained on it to perform well on the original dataset. With the blooming of large language models and multimodal large language models, the importance of multimodal datasets, particularly image-text datasets, has grown significantly. However, existing multimodal dataset distillation methods are constrained by the Matching Training Trajectories algorithm, which significantly increases the computing resource requirement, and takes days to process the distillation. In this work, we introduce EDGE, a generative distillation method for efficient multimodal dataset distillation. Specifically, we identify two key challenges of distilling multimodal datasets with generative models: 1) The lack of correlation between generated images and captions.2) The lack of diversity among generated samples.To address the aforementioned issues, we propose a novel generative model training workflow with a bi-directional contrastive loss and a diversity loss. Furthermore, we propose a caption synthesis strategy to further improve text-to-image retrieval performance by introducing more text information. Our method is evaluated on Flickr30K, COCO, and CC3M datasets, demonstrating superior performance and efficiency compared to existing approaches. Notably, our method achieves results 18

 faster than the state-of-the-art method.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xue2025,

title = {DictPFL: Efficient and Private Federated Learning on Encrypted Gradients},

author = {Jiaqi Xue and Mayank Kumar and Yuzhang Shang and Shangqian Gao and Rui Ning and Mengxin Zheng and Xiaoqian Jiang and Qian Lou},

url = {https://neurips.cc/virtual/2025/poster/119806},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Federated learning (FL) enables institutions to collaboratively train machine learning models by aggregating local gradients without sharing sensitive data. However, sharing gradients still poses privacy risks, e.g., gradient inversion attacks. Homomorphic encryption (HE) can be used in FL to encrypt gradients at the data owner's side, enabling secure aggregation without decryption on the server. Existing HE approaches to FL lie at two extremes. One encrypts every gradient update, providing strong privacy but incurring prohibitive computation and bandwidth costs. The other encrypts only a subset of gradients, reducing overhead yet leaving the remaining plaintext updates vulnerable to privacy attacks. Our proposed DictPFL bridges this gap. It encrypts every gradient that must be transmitted to the server—protecting all shared information—while keeping the rest of the (unencrypted) gradients on the client, where they never leave the device. By safeguarding every transmitted update, DictPFL achieves the same privacy guarantees as fully encrypted FL, but its selective-encryption strategy slashes computational and communication overhead. DictPFL comprises two modules: Decompose-for-Partial-Encrypt (DePE) and Prune-for-Minimum-Encrypt (PrME). In DePE, we decompose model weights to be trained into a dictionary and a lookup table. Only the gradients of the lookup table are encrypted and aggregated securely while the dictionary remains fixed and is not transmitted for aggregation. In PrME, we aim to further minimize the encrypted parameters with an encryption-aware pruning technique that ensures a consistent pruning mask across clients by leveraging the history of global gradients. Experimental results demonstrate that DictPFL significantly reduces communication overhead by 402 to 748 times and speeds training by 28 to 65 times compared to fully encrypted method. It also outperforms the state-of-the-art selectively encrypted gradient by lowering overhead by 51 to 155 times and accelerating training by 4 to 19 times. DictPFL increases training time by even less than a 2

 factor compared with its plaintext counterpart without gradients protection, demonstrating—for the first time—that HE–based private federated learning is practical for real-world deployment.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Adak2025,

title = {MolVision: Molecular Property Prediction with Vision Language Models},

author = {Deepan Adak and Yogesh Singh Rawat and Shruti Vyas},

url = {https://neurips.cc/virtual/2025/poster/121822

https://chemvision.github.io/chemvision/},

year  = {2025},

date = {2025-11-30},

urldate = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Molecular property prediction is a fundamental task in computational chemistry with critical applications in drug discovery and materials science. While recent works have explored Large Language Models (LLMs) for this task, they primarily rely on textual molecular representations such as SMILES/SELFIES, which can be ambiguous and structurally uninformative. In this work, we introduce MolVision, a novel approach that leverages Vision-Language Models (VLMs) by integrating both molecular structure images and textual descriptions to enhance property prediction. We construct a benchmark spanning nine diverse datasets, covering both classification and regression tasks. Evaluating nine different VLMs in zero-shot, few-shot, and fine-tuned settings, we find that visual information improves prediction performance, particularly when combined with efficient fine-tuning strategies such as LoRA. Our results reveal that while visual information alone is insufficient, multimodal fusion significantly enhances generalization across molecular properties. Adaptation of vision encoder for molecular images in conjunction with LoRA further improves the performance. The code and data is available at : https://chemvision.github.io/chemvision/.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025d,

title = {Hierarchical Fine-grained Preference Optimization for Physically Plausible Video Generation},

author = {Haodong Chen and Haojian Huang and Qifeng Chen and Harry Yang and Ser Nam Lim},

url = {https://neurips.cc/virtual/2025/poster/115193},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Recent advancements in video generation have enabled the creation of high-quality, visually compelling videos. However, generating videos that adhere to the laws of physics remains a critical challenge for applications requiring realism and accuracy. In this work, we propose PhysHPO, a novel framework for Hierarchical Cross-Modal Direct Preference Optimization, to tackle this challenge by enabling fine-grained preference alignment for physically plausible video generation. PhysHPO optimizes video alignment across four hierarchical granularities: a) Instance Level, aligning the overall video content with the input prompt; b) State Level, ensuring temporal consistency using boundary frames as anchors; c) Motion Level, modeling motion trajectories for realistic dynamics; and d) Semantic Level, maintaining logical consistency between narrative and visuals. Recognizing that real-world videos are the best reflections of physical phenomena, we further introduce an automated data selection pipeline to efficiently identify and utilize "good data" from existing large-scale text-video datasets, thereby eliminating the need for costly and time-intensive dataset construction. Extensive experiments on both physics-focused and general capability benchmarks demonstrate that PhysHPO significantly improves physical plausibility and overall video generation quality of advanced models. To the best of our knowledge, this is the first work to explore fine-grained preference alignment and data selection for video generation, paving the way for more realistic and human-preferred video generation paradigms.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Huang2025b,

title = {ESCA: Contextualizing Embodied Agents via Scene-Graph Generation},

author = {Jiani Huang and Mayank Keoliya and Matthew Kuo and Neelay Velingker and Amish Sethi and JungHo Jung and Ziyang Li and Ser Nam Lim and Mayur Naik},

url = {https://neurips.cc/virtual/2025/poster/117064},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Multi-modal large language models (MLLMs) are making rapid progress toward general-purpose embodied agents. However, current training pipelines primarily rely on high-level vision-sound-text pairs and lack fine-grained, structured alignment between pixel-level visual content and textual semantics. To overcome this challenge, we propose ESCA, a new framework for contextualizing embodied agents through structured spatial-temporal understanding. At its core is SGClip, a novel CLIP-based, open-domain, and promptable model for generating scene graphs. SGClip is trained on 87K+ open-domain videos via a neurosymbolic learning pipeline, which harnesses model-driven self-supervision from video-caption pairs and structured reasoning, thereby eliminating the need for human-labeled scene graph annotations. We demonstrate that SGClip supports both prompt-based inference and task-specific fine-tuning, excelling in scene graph generation and action localization benchmarks. ESCA with SGClip consistently improves both open-source and commercial MLLMs, achieving state-of-the-art performance across two embodied environments. Notably, it significantly reduces agent perception errors and enables open-source models to surpass proprietary baselines.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shu2025,

title = {When Semantics Mislead Vision: Mitigating Large Multimodal Models Hallucinations in Scene Text Spotting and Understanding},

author = {Yan Shu and Hangui Lin and Yexin Liu and Yan Zhang and Gangyan Zeng and Yan Li and Yu Zhou and Ser Nam Lim and Harry Yang and Nicu Sebe},

url = {https://neurips.cc/virtual/2025/poster/119366},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Large Multimodal Models (LMMs) have achieved impressive progress in visual perception and reasoning. However, when confronted with visually ambiguous or non-semantic scene text, they often struggle to accurately spot and understand the content, frequently generating semantically plausible yet visually incorrect answers, which we refer to as semantic hallucination.In this work, we investigate the underlying causes of semantic hallucination and identify a key finding: Transformer layers in LLM with stronger attention focus on scene text regions are less prone to producing semantic hallucinations. Thus, we propose a training-free semantic hallucination mitigation framework comprising two key components: (1) ZoomText, a coarse-to-fine strategy that identifies potential text regions without external detectors; and (2) Grounded Layer Correction, which adaptively leverages the internal representations from layers less prone to hallucination to guide decoding, correcting hallucinated outputs for non-semantic samples while preserving the semantics of meaningful ones. To enable rigorous evaluation, we introduce TextHalu-Bench, a benchmark of over 1,730 samples spanning both semantic and non-semantic cases, with manually curated question–answer pairs designed to probe model hallucinations.Extensive experiments demonstrate that our method not only effectively mitigates semantic hallucination but also achieves strong performance on public benchmarks for scene text spotting and understanding.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ghosal2025b,

title = {Does Thinking More Always Help? Understanding Test-Time Scaling in Reasoning Models},

author = {Soumya Suvra Ghosal and Souradip Chakraborty and Avinash Reddy and Yifu Lu and Mengdi Wang and Dinesh Manocha and Furong Huang and Mohammad Ghavamzadeh and Amrit Singh Bedi},

url = {https://neurips.cc/virtual/2025/poster/115605},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Recent trends in test-time scaling for reasoning models (e.g., OpenAI o1, DeepSeek R1) have led to a popular belief that extending thinking traces using prompts like “Wait” or “Let me rethink” can improve performance. This raises a natural question: Does thinking more at test-time truly lead to better reasoning? To answer this question, we perform a detailed empirical study across models and benchmarks, which reveals a consistent pattern of initial performance improvements from additional thinking followed by a decline, due to "overthinking". To understand this non-monotonic trend, we consider a simple probabilistic model, which reveals that additional thinking increases output variance—creating an illusion of improved reasoning while ultimately undermining precision. Thus, observed gains from "more thinking" are not true indicators of improved reasoning, but artifacts stemming from the connection between model uncertainty and evaluation metric. This suggests that test-time scaling through extended thinking is not an effective way to utilize the inference thinking budget. Recognizing these limitations, we introduce an alternative test-time scaling approach, parallel thinking, inspired by Best-of-N sampling. Our method generates multiple independent reasoning paths within the same inference budget and selects the most consistent response via majority vote, achieving up to 20% higher accuracy compared to extended thinking. This provides a simple yet effective mechanism for test-time scaling of reasoning models.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Mudit2025,

title = {On the Sample Complexity Bounds of Bilevel Reinforcement Learning},

author = {Mudit and Utsav Singh and Amrit Singh Bedi and Raghu Pasupathy and Vaneet Aggarwal},

url = {https://neurips.cc/virtual/2025/poster/116657},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Bilevel reinforcement learning (BRL) has emerged as a powerful framework for aligning generative models, yet its theoretical foundations, especially sample complexity bounds, remain underexplored. In this work, we present the first sample complexity bound for BRL, establishing a rate of 

 in continuous state-action spaces. Traditional MDP analysis techniques do not extend to BRL due to its nested structure and non-convex lower-level problems. We overcome these challenges by leveraging the Polyak-Łojasiewicz (PL) condition and the MDP structure to obtain closed-form gradients, enabling tight sample complexity analysis. Our analysis also extends to general bi-level optimization settings with non-convex lower levels, where we achieve state-of-the-art sample complexity results of 

 improving upon existing bounds of 

. Additionally, we address the computational bottleneck of hypergradient estimation by proposing a fully first-order, Hessian-free algorithm suitable for large-scale problems.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Barakat2025,

title = {On the Global Optimality of Policy Gradient Methods in General Utility Reinforcement Learning},

author = {Anas Barakat and Souradip Chakraborty and Peihong Yu and Pratap Tokekar and Amrit Singh Bedi},

url = {https://neurips.cc/virtual/2025/poster/117237},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Reinforcement learning with general utilities (RLGU) offers a unifying framework to capture several problems beyond standard expected returns, including imitation learning, pure exploration, and safe RL. Despite recent fundamental advances in the theoretical analysis of policy gradient (PG) for standard RL and recent efforts in RLGU, the understanding of PG methods and their scope of application in RLGU still remain limited. In this work, we establish global optimality guarantees of PG methods for RLGU in which the objective is a general concave utility function of the state-action occupancy measure. In the tabular setting, we provide global optimality results using a new proof technique building on recent theoretical developments on the convergence of PG methods for standard RL using gradient domination. Our proof technique opens avenues for analyzing policy parameterizations beyond the direct policy parameterization for RLGU. In addition, we provide global optimality results for large state action space settings beyond prior work which has mostly focused on the tabular setting. In this large scale setting, we adapt PG methods by approximating occupancy measures within a function approximation class using maximum likelihood estimation. Our sample complexity only scales with the dimension of our function approximation class rather than the size of the state action space.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rai2025,

title = {Failure by Interference: Language Models Make Balanced Parentheses Errors When Faulty Mechanisms Overshadow Sound Ones},

author = {Daking Rai and Samuel Miller and Kevin Moran and Ziyu Yao},

url = {https://neurips.cc/virtual/2025/poster/120187},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Despite remarkable advances in coding capabilities, language models (LMs) still struggle with simple syntactic tasks such as generating balanced parentheses. In this study, we investigate the underlying mechanisms behind the persistence of these errors across LMs of varying sizes (124M–7B) to both understand and mitigate the errors. Our study reveals that LMs rely on a number of components (attention heads and FF neurons) that independently make their own predictions. While some components reliably promote correct answers across a generalized range of inputs (i.e., implementing "sound mechanisms''), others are less reliable and introduce noise by promoting incorrect tokens (i.e., implementing "faulty mechanisms''). Errors occur when the faulty mechanisms overshadow the sound ones and dominantly affect the predictions. Motivated by this insight, we introduce RASteer, a steering method to systematically identify and increase the contribution of reliable components for improving model performance. RASteer substantially improves performance on balanced parentheses tasks, boosting accuracy of some models from 0% to around 100% without impairing the models' general coding ability. We further demonstrate its broader applicability in arithmetic reasoning tasks, achieving performance gains of up to around 20%.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025bb,

title = {Adaptive Defense against Harmful Fine-Tuning via Bayesian Data Scheduler},

author = {Zixuan Hu and Li Shen and Zhenyi Wang and Yongxian Wei and Dacheng Tao},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Harmful fine-tuning poses critical safety risks to fine-tuning-as-a-service for large language models. Existing defense strategies preemptively build robustness via attack simulation but suffer from fundamental limitations: (i) the infeasibility of performing attack simulation due to lacking prior knowledge about potential attack data, and (ii) limited adaptability to varying attack settings, as simulation fails to capture their variability and complexity. To address these challenges, we propose Bayesian Data Scheduler (BDS), an adaptive tuning-stage defense strategy with no need for attack simulation. BDS formulates harmful fine-tuning defense as a Bayesian inference problem, learning the posterior distribution of each data point's safety attribute, conditioned on the fine-tuning and alignment datasets. The fine-tuning process is then constrained by weighting data with their safety attributes sampled from the posterior, thus mitigating the influence of harmful data. By leveraging the post hoc nature of Bayesian inference, the posterior is conditioned on the fine-tuning dataset, enabling BDS to tailor its defense to the specific dataset, thereby achieving adaptive defense. Furthermore, we introduce a neural scheduler based on amortized Bayesian learning, enabling efficient transfer to new data without retraining. Comprehensive results across diverse attack and defense settings demonstrate the state-of-the-art performance of our approach.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhang2025db,

title = {EGGS: Exchangeable 2D/3D Gaussian Splatting for Geometry-Appearance Balanced Novel View Synthesis},

author = {Yancheng Zhang and Guangyu Sun and Chen Chen},

url = {https://neurips.cc/virtual/2025/poster/120173},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Novel view synthesis (NVS) is crucial in computer vision and graphics, with wide applications in AR, VR, and autonomous driving. While 3D Gaussian Splatting (3DGS) enables real-time rendering with high appearance fidelity, it suffers from multi-view inconsistencies, limiting geometric accuracy. In contrast, 2D Gaussian Splatting (2DGS) enforces multi-view consistency but compromises texture details. To address these limitations, we propose Exchangeable Gaussian Splatting (EGGS), a hybrid representation that integrates 2D and 3D Gaussians to balance appearance and geometry. To achieve this, we introduce Hybrid Gaussian Rasterization for unified rendering, Adaptive Type Exchange for dynamic adaptation between 2D and 3D Gaussians, and Frequency-Decoupled Optimization that effectively exploits the strengths of each type of Gaussian representation. Our CUDA-accelerated implementation ensures efficient training and inference. Extensive experiments demonstrate that EGGS outperforms existing methods in rendering quality, geometric accuracy, and efficiency, providing a practical solution for high-quality NVS.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Qu2025b,

title = {BetaConform: Efficient MAP Estimation of LLM Ensemble Judgment Performance with Prior Transfer},

author = {Huaizhi Qu and Inyoung Choi and Zhen Tan and Song Wang and Sukwon Yun and Qi Long and Faizan Siddiqui and Kwonjoon Lee and Tianlong Chen},

url = {https://arxiv.org/abs/2504.12589},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {LLM ensembles are widely used for LLM judges. However, how to estimate their accuracy, especially in an efficient way, is unknown. In this paper, we present a principled maximum a posteriori (MAP) framework for an economical and precise estimation of the performance of LLM ensemble judgment. We first propose a mixture of Beta-Binomial distributions to model the judgment distribution, revising from the vanilla Binomial distribution. Next, we introduce a conformal prediction-driven approach that enables adaptive stopping during iterative sampling to balance accuracy with efficiency. Furthermore, we design a prior transfer mechanism that utilizes learned distributions on open-source datasets to improve estimation on a target dataset when only scarce annotations are available. Finally, we present BetaConform, a framework that integrates our distribution assumption, adaptive stopping, and the prior transfer mechanism to deliver a theoretically guaranteed distribution estimation of LLM ensemble judgment with minimum labeled samples. BetaConform is also validated empirically. For instance, with only 10 samples from the TruthfulQA dataset, for a Llama ensembled judge, BetaConform gauges its performance with error margin as small as 3.37%.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_76,

title = {DEXTER: Diffusion-Guided EXplanations with TExtual Reasoning for Vision Models},

author = {Simone Carnemolla and Matteo Pennisi and Sarinda Samarasinghe and Giovanni Bellitto and Simone Palazzo and Daniela Giordano and Mubarak Shah and Concetto Spampinato},

url = {https://neurips.cc/virtual/2025/poster/117167},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Understanding and explaining the behavior of machine learning models is essential for building transparent and trustworthy AI systems. We introduce DEXTER, a data-free framework that combines diffusion models and large language models to generate global, textual explanations of visual classifiers. DEXTER operates by optimizing text prompts to synthesize class-conditional images that strongly activate a target classifier. These synthetic samples are then used to elicit detailed natural language reports that describe class-specific decision patterns and biases. Unlike prior work, DEXTER enables natural language reasoning about a classifier's decision process without access to training data or ground-truth labels. We demonstrate DEXTER's flexibility across three tasks—activation maximization, slice discovery and debiasing, and bias explanation—each illustrating its ability to uncover the internal mechanisms of visual classifiers. Quantitative and qualitative evaluations, including a user study, show that DEXTER produces accurate, interpretable outputs. Experiments on ImageNet, Waterbirds, CelebA, and FairFaces confirm that DEXTER outperforms existing approaches in global model explanation and class-level bias reporting.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025bc,

title = {Adaptive Defense against Harmful Fine-Tuning via Bayesian Data Scheduler},

author = {Zixuan Hu and Li Shen and Zhenyi Wang and Yongxian Wei and Dacheng Tao},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Harmful fine-tuning poses critical safety risks to fine-tuning-as-a-service for large language models. Existing defense strategies preemptively build robustness via attack simulation but suffer from fundamental limitations: (i) the infeasibility of performing attack simulation due to lacking prior knowledge about potential attack data, and (ii) limited adaptability to varying attack settings, as simulation fails to capture their variability and complexity. To address these challenges, we propose Bayesian Data Scheduler (BDS), an adaptive tuning-stage defense strategy with no need for attack simulation. BDS formulates harmful fine-tuning defense as a Bayesian inference problem, learning the posterior distribution of each data point's safety attribute, conditioned on the fine-tuning and alignment datasets. The fine-tuning process is then constrained by weighting data with their safety attributes sampled from the posterior, thus mitigating the influence of harmful data. By leveraging the post hoc nature of Bayesian inference, the posterior is conditioned on the fine-tuning dataset, enabling BDS to tailor its defense to the specific dataset, thereby achieving adaptive defense. Furthermore, we introduce a neural scheduler based on amortized Bayesian learning, enabling efficient transfer to new data without retraining. Comprehensive results across diverse attack and defense settings demonstrate the state-of-the-art performance of our approach.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Qu2025c,

title = {BetaConform: Efficient MAP Estimation of LLM Ensemble Judgment Performance with Prior Transfer},

author = {Huaizhi Qu and Inyoung Choi and Zhen Tan and Song Wang and Sukwon Yun and Qi Long and Faizan Siddiqui and Kwonjoon Lee and Tianlong Chen},

url = {https://arxiv.org/abs/2504.12589},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {LLM ensembles are widely used for LLM judges. However, how to estimate their accuracy, especially in an efficient way, is unknown. In this paper, we present a principled maximum a posteriori (MAP) framework for an economical and precise estimation of the performance of LLM ensemble judgment. We first propose a mixture of Beta-Binomial distributions to model the judgment distribution, revising from the vanilla Binomial distribution. Next, we introduce a conformal prediction-driven approach that enables adaptive stopping during iterative sampling to balance accuracy with efficiency. Furthermore, we design a prior transfer mechanism that utilizes learned distributions on open-source datasets to improve estimation on a target dataset when only scarce annotations are available. Finally, we present BetaConform, a framework that integrates our distribution assumption, adaptive stopping, and the prior transfer mechanism to deliver a theoretically guaranteed distribution estimation of LLM ensemble judgment with minimum labeled samples. BetaConform is also validated empirically. For instance, with only 10 samples from the TruthfulQA dataset, for a Llama ensembled judge, BetaConform gauges its performance with error margin as small as 3.37%.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wu2024,

title = {PTQ4DiT: Post-training Quantization for Diffusion Transformers},

author = {Junyi Wu and Haoxuan Wang and Yuzhang Shang and Mubarak Shah and Yan Yan},

url = {https://nips.cc/virtual/2024/poster/95445 

https://arxiv.org/pdf/2405.16005 

https://github.com/adreamwu/PTQ4DiT},

year  = {2024},

date = {2024-12-13},

urldate = {2024-12-13},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {The recent introduction of Diffusion Transformers (DiTs) has demonstrated exceptional capabilities in image generation by using a different backbone architecture, departing from traditional U-Nets and embracing the scalable nature of transformers. Despite their advanced capabilities, the wide deployment of DiTs, particularly for real-time applications, is currently hampered by considerable computational demands at the inference stage. Post-training Quantization (PTQ) has emerged as a fast and data-efficient solution that can significantly reduce computation and memory footprint by using low-bit weights and activations. However, its applicability to DiTs has not yet been explored and faces non-trivial difficulties due to the unique design of DiTs. In this paper, we propose PTQ4DiT, a specifically designed PTQ method for DiTs. We discover two primary quantization challenges inherent in DiTs, notably the presence of salient channels with extreme magnitudes and the temporal variability in distributions of salient activation over multiple timesteps. To tackle these challenges, we propose Channel-wise Salience Balancing (CSB) and Spearmen's 

ρ 

-guided Salience Calibration (SSC). CSB leverages the complementarity property of channel magnitudes to redistribute the extremes, alleviating quantization errors for both activations and weights. SSC extends this approach by dynamically adjusting the balanced salience to capture the temporal variations in activation. Additionally, to eliminate extra computational costs caused by PTQ4DiT during inference, we design an offline re-parameterization strategy for DiTs. Experiments demonstrate that our PTQ4DiT successfully quantizes DiTs to 8-bit precision (W8A8) while preserving comparable generation ability and further enables effective quantization to 4-bit weight precision (W4A8) for the first time.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wu2024b,

title = {PTQ4DiT: Post-training Quantization for Diffusion Transformers},

author = {Junyi Wu and Haoxuan Wang and Yuzhang Shang and Mubarak Shah and Yan Yan},

url = {https://nips.cc/virtual/2024/poster/95445 

https://arxiv.org/pdf/2405.16005 

https://github.com/adreamwu/PTQ4DiT},

year  = {2024},

date = {2024-12-13},

urldate = {2024-12-13},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {The recent introduction of Diffusion Transformers (DiTs) has demonstrated exceptional capabilities in image generation by using a different backbone architecture, departing from traditional U-Nets and embracing the scalable nature of transformers. Despite their advanced capabilities, the wide deployment of DiTs, particularly for real-time applications, is currently hampered by considerable computational demands at the inference stage. Post-training Quantization (PTQ) has emerged as a fast and data-efficient solution that can significantly reduce computation and memory footprint by using low-bit weights and activations. However, its applicability to DiTs has not yet been explored and faces non-trivial difficulties due to the unique design of DiTs. In this paper, we propose PTQ4DiT, a specifically designed PTQ method for DiTs. We discover two primary quantization challenges inherent in DiTs, notably the presence of salient channels with extreme magnitudes and the temporal variability in distributions of salient activation over multiple timesteps. To tackle these challenges, we propose Channel-wise Salience Balancing (CSB) and Spearmen's 

ρ 

-guided Salience Calibration (SSC). CSB leverages the complementarity property of channel magnitudes to redistribute the extremes, alleviating quantization errors for both activations and weights. SSC extends this approach by dynamically adjusting the balanced salience to capture the temporal variations in activation. Additionally, to eliminate extra computational costs caused by PTQ4DiT during inference, we design an offline re-parameterization strategy for DiTs. Experiments demonstrate that our PTQ4DiT successfully quantizes DiTs to 8-bit precision (W8A8) while preserving comparable generation ability and further enables effective quantization to 4-bit weight precision (W4A8) for the first time.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2024,

title = {A Unified Principle of Pessimism for Offline Reinforcement Learning under Model Mismatch},

author = {Yue Wang and Zhongchang Sun and Shaofeng Zou},

url = {https://nips.cc/virtual/2024/poster/94438},

year  = {2024},

date = {2024-12-12},

urldate = {2024-12-12},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Abstract: In this paper, we address the challenges of offline reinforcement learning (RL) under model mismatch, where the agent aims to optimize its performance through an offline dataset that may not accurately represent the deployment environment. We identify two primary challenges under the setting: inaccurate model estimation due to limited data and performance degradation caused by the model mismatch between the dataset-collecting environment and the target deployment one. To tackle these issues, we propose a unified principle of pessimism using distributionally robust Markov decision processes. We carefully construct a robust MDP with a single uncertainty set to tackle both data sparsity and model mismatch, and demonstrate that the optimal robust policy enjoys a near-optimal sub-optimality gap under the target environment across three widely used uncertainty models: total variation, χ^2 divergence, and KL divergence. Our results improve upon or match the state-of-the-art performance under the total variation and KL divergence models, and provide the first result for the χ^2 divergence model.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Nguyen2024,

title = {Noisy Label Learning with Instance-Dependent Outliers: Identifiability via Crowd Wisdom},

author = {Tri Nguyen and Shahana Ibrahim and Xiao Fu},

url = {https://nips.cc/virtual/2024/poster/95831},

year  = {2024},

date = {2024-12-12},

urldate = {2024-12-12},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {The generation of label noise is often modeled as a process involving a probability transition matrix (often interpreted as the {it annotator confusion matrix}) imposed onto the ground-truth label distribution.Under this model, rectifying the label noise and learning the target classifier boil down to identifying the confusion matrix. This line of work demonstrated appealing empirical performance, yet identifiability of the model was mostly established by assuming an instance-invariant confusion matrix. Having an (occasionally) instance-dependent confusion matrix across data samples is apparently more realistic, but inevitably introduces outliers to the model.Our interest lies in confusion matrix-based noisy label learning with such outliers taken into consideration.We begin with pointing out that under the model of interest, detecting the outliers in the presence of a single confusion matrix is fundamentally insufficient.Then, we prove that by employing a crowdsourcing strategy involving multiple annotators, a carefully designed loss function can detect the outliers and identify the desired classifier under reasonable conditions.Our development builds upon a link between the noisy label model and a column-corrupted matrix factorization model---which turns out attesting to the importance of crowdsourced data annotation. Experiments show that our learning scheme substantially improves the outlier detection probability and the learned neural systems' testing accuracy.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chakraborty2024b,

title = {Transfer Q-star : Principled Decoding for LLM Alignment},

author = {Souradip Chakraborty and Soumya Suvra Ghosal and Ming Yin and Dinesh Manocha and Mengdi Wang and Amrit Singh Bedi and Furong Huang},

url = {https://neurips.cc/virtual/2024/poster/96588},

year  = {2024},

date = {2024-12-12},

urldate = {2024-12-12},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Aligning foundation models is essential for their safe and trustworthy deployment. However, traditional fine-tuning methods are computationally intensive and require updating billions of model parameters. A promising alternative, alignment via decoding, adjusts the response distribution directly without model updates to maximize a target reward r, thus providing a lightweight and adaptable framework for alignment. However, principled decoding methods rely on oracle access to an optimal Q-function (Q∗), which is often unavailable in practice. Hence, prior SoTA methods either approximate this Q∗ using Qπsft (derived from the reference SFT model) or rely on short-term rewards, resulting in sub-optimal decoding performance. In this work, we propose Transfer Q∗, which implicitly estimates the optimal value function for a target reward 

r through a baseline model ρBL aligned with a baseline reward rBL (which can be different from the target reward r). Theoretical analyses of Transfer Q∗ provide a rigorous characterization of its optimality, deriving an upper bound on the sub-optimality gap and identifying a hyperparameter to control the deviation from the pre-trained reference SFT model based on user needs. Our approach significantly reduces the sub-optimality gap observed in prior SoTA methods and demonstrates superior empirical performance across key metrics such as coherence, diversity, and quality in extensive tests on several synthetic and real datasets. },

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Bornstein2024,

title = {FACT or Fiction: Can Truthful Mechanisms Eliminate Federated Free Riding?},

author = {Marco Bornstein and Amrit Singh Bedi and Abdirisak Mohamed and Furong Huang},

url = {https://nips.cc/virtual/2024/poster/95703},

year  = {2024},

date = {2024-12-12},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Standard federated learning (FL) approaches are vulnerable to the free-rider dilemma: participating agents can contribute little to nothing yet receive a well-trained aggregated model. While prior mechanisms attempt to solve the free-rider dilemma, none have addressed the issue of truthfulness. In practice, adversarial agents can provide false information to the server in order to cheat its way out of contributing to federated training. In an effort to make free-riding-averse federated mechanisms truthful, and consequently less prone to breaking down in practice, we propose FACT. FACT is the first federated mechanism that: (1) eliminates federated free riding by using a penalty system, (2) ensures agents provide truthful information by creating a competitive environment, and (3) encourages agent participation by offering better performance than training alone. Empirically, FACT avoids free-riding when agents are untruthful, and reduces agent loss by over 4x.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024,

title = {Rethinking the Capacity of Graph Neural Networks for Branching Strategy},

author = {Ziang Chen, Jialin Liu, Xiaohan Chen, Xinshang Wang, Wotao Yin},

url = {https://neurips.cc/virtual/2024/poster/95991

https://arxiv.org/pdf/2402.07099},

year  = {2024},

date = {2024-12-11},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Graph neural networks (GNNs) have been widely used to predict properties and heuristics of mixed-integer linear programs (MILPs) and hence accelerate MILP solvers. This paper investigates the capacity of GNNs to represent strong branching (SB), the most effective yet computationally expensive heuristic employed in the branch-and-bound algorithm. In the literature, message-passing GNN (MP-GNN), as the simplest GNN structure, is frequently used as a fast approximation of SB and we find that not all MILPs's SB can be represented with MP-GNN. We precisely define a class of 

MP-tractable" MILPs for which MP-GNNs can accurately approximate SB scores. Particularly, we establish a universal approximation theorem: for any data distribution over the MP-tractable class, there always exists an MP-GNN that can approximate the SB score with arbitrarily high accuracy and arbitrarily high probability, which lays a theoretical foundation of the existing works on imitating SB with MP-GNN. For MILPs without the MP-tractability, unfortunately, a similar result is impossible, which can be illustrated by two MILP instances with different SB scores that cannot be distinguished by any MP-GNN, regardless of the number of parameters. Recognizing this, we explore another GNN structure called the second-order folklore GNN (2-FGNN) that overcomes this limitation, and the aforementioned universal approximation theorem can be extended to the entire MILP space using 2-FGNN, regardless of the MP-tractability. A small-scale numerical experiment is conducted to directly validate our theoretical findings.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2024,

title = {Asynchronous Perception Machine for Test Time Training},

author = {Rajat Modi and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/apm_final.pdf 

https://www.crcv.ucf.edu/person/rawat/},

year  = {2024},

date = {2024-12-11},

urldate = {2024-12-11},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {In this work, we propose Asynchronous Perception Machine (APM), a computationally-efficient architecture for test-time-training (TTT). APM can process patches of an image one at a time in any order asymmetrically and still encode semantic-awareness. We demonstrate APM's ability to recognize out-of-distribution images without dataset-specific pre-training, and its competitive classification performance over existing TTT approaches. To perform TTT, APM just distills test sample's representation once. APM possesses a unique property: it can learn using just this single representation and starts predicting semantically-aware features. We demonstrate APM's potential application beyond test-time-training: APM can scale up to a dataset of 2D images and yield semantic-clusterings in a single forward pass. APM also provides first empirical evidence of GLOM's insight, i.e. percept is really a field. Therefore, APM helps us converge towards an implementation which can do both interpolation and perception on a shared-connectionist hardware. Our codebase has been provided for review and will be made publicly-available. ——–It now appears that some of the ideas in GLOM could be made to work.https://www.technologyreview.com/2021/04/16/1021871/geoffrey-hinton-glom-godfather-ai-neural-networks/},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2024b,

title = {Asynchronous Perception Machine for Test Time Training},

author = {Rajat Modi and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/apm_final.pdf 

https://www.crcv.ucf.edu/person/rawat/},

year  = {2024},

date = {2024-12-11},

urldate = {2024-12-11},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {In this work, we propose Asynchronous Perception Machine (APM), a computationally-efficient architecture for test-time-training (TTT). APM can process patches of an image one at a time in any order asymmetrically and still encode semantic-awareness. We demonstrate APM's ability to recognize out-of-distribution images without dataset-specific pre-training, and its competitive classification performance over existing TTT approaches. To perform TTT, APM just distills test sample's representation once. APM possesses a unique property: it can learn using just this single representation and starts predicting semantically-aware features. We demonstrate APM's potential application beyond test-time-training: APM can scale up to a dataset of 2D images and yield semantic-clusterings in a single forward pass. APM also provides first empirical evidence of GLOM's insight, i.e. percept is really a field. Therefore, APM helps us converge towards an implementation which can do both interpolation and perception on a shared-connectionist hardware. Our codebase has been provided for review and will be made publicly-available. ——–It now appears that some of the ideas in GLOM could be made to work.https://www.technologyreview.com/2021/04/16/1021871/geoffrey-hinton-glom-godfather-ai-neural-networks/},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Csaba2024,

title = {Label Delay in Online Continual Learning},

author = {Botos Csaba and Wenxuan Zhang and Matthias Müller and Ser-Nam Lim and Philip Torr and Adel Bibi},

url = {https://arxiv.org/abs/2312.00923},

year  = {2024},

date = {2024-12-09},

urldate = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Online continual learning, the process of training models on streaming data, has gained increasing attention in recent years. However, a critical aspect often overlooked is the label delay, where new data may not be labeled due to slow and costly annotation processes. We introduce a new continual learning framework with explicit modeling of the label delay between data and label streams over time steps. In each step, the framework reveals both unlabeled data from the current time step t and labels delayed with d steps, from the time step t−d. In our extensive experiments amounting to 1060 GPU days, we show that merely augmenting the computational resources is insufficient to tackle this challenge. Our findings underline a notable performance decline when solely relying on labeled data when the label delay becomes significant. More surprisingly, when using state-of-the-art SSL and TTA techniques to utilize the newer, unlabeled data, they fail to surpass the performance of a naïve method that simply trains on the delayed supervised stream. To this end, we introduce a simple, efficient baseline that rehearses from the labeled memory samples that are most similar to the new unlabeled samples. This method bridges the accuracy gap caused by label delay without significantly increasing computational complexity. We show experimentally that our method is the least affected by the label delay factor and in some cases successfully recovers the accuracy of the non-delayed counterpart. We conduct various ablations and sensitivity experiments, demonstrating the effectiveness of our approach.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Lim2024,

title = {AirSketch: Generative Motion to Sketch},

author = {Hui Xian Grace Lim and Xuanming Cui and Yogesh Singh Rawat and Ser-Nam Lim },

url = {https://arxiv.org/pdf/2407.08906

https://www.crcv.ucf.edu/person/rawat/},

year  = {2024},

date = {2024-12-09},

urldate = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Illustration is a fundamental mode of human expression and communication. Certain types of motion that accompany speech can provide this illustrative mode of communication. While Augmented and Virtual Reality technologies (AR/VR) have introduced tools for producing drawings with hand motions (air drawing), they typically require costly hardware and additional digital markers, thereby limiting their accessibility and portability. Furthermore, air drawing demands considerable skill to achieve aesthetic results. To address these challenges, we introduce the concept of AirSketch, aimed at generating faithful and visually coherent sketches directly from hand motions, eliminating the need for complicated headsets or markers. We devise a simple augmentation-based self-supervised training procedure, enabling a controllable image diffusion model to learn to translate from highly noisy hand tracking images to clean, aesthetically pleasing sketches, while preserving the essential visual cues from the original tracking data. We present two air drawing datasets to study this problem. Our findings demonstrate that beyond producing photo-realistic images from precise spatial inputs, controllable image diffusion can effectively produce a refined, clear sketch from a noisy input. Our work serves as an initial step towards marker-less air drawing and reveals distinct applications of controllable diffusion models to AirSketch and AR/VR in general.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Lim2024b,

title = {AirSketch: Generative Motion to Sketch},

author = {Hui Xian Grace Lim and Xuanming Cui and Yogesh Singh Rawat and Ser-Nam Lim},

url = {https://arxiv.org/pdf/2407.08906 

https://www.crcv.ucf.edu/person/rawat/},

year  = {2024},

date = {2024-12-09},

urldate = {2024-07-12},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Illustration is a fundamental mode of human expression and communication. Certain types of motion that accompany speech can provide this illustrative mode of communication. While Augmented and Virtual Reality technologies (AR/VR) have introduced tools for producing drawings with hand motions (air drawing), they typically require costly hardware and additional digital markers, thereby limiting their accessibility and portability. Furthermore, air drawing demands considerable skill to achieve aesthetic results. To address these challenges, we introduce the concept of AirSketch, aimed at generating faithful and visually coherent sketches directly from hand motions, eliminating the need for complicated headsets or markers. We devise a simple augmentation-based self-supervised training procedure, enabling a controllable image diffusion model to learn to translate from highly noisy hand tracking images to clean, aesthetically pleasing sketches, while preserving the essential visual cues from the original tracking data. We present two air drawing datasets to study this problem. Our findings demonstrate that beyond producing photo-realistic images from precise spatial inputs, controllable image diffusion can effectively produce a refined, clear sketch from a noisy input. Our work serves as an initial step towards marker-less air drawing and reveals distinct applications of controllable diffusion models to AirSketch and AR/VR in general.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2024,

title = {Advancing Video Anomaly Detection: A Concise Review and a New Dataset},

author = {Arjun Raj Lei Wang Liyun Zhu},

url = {https://arxiv.org/pdf/2402.04857 

https://arxiv.org/abs/2402.04857 

https://msad-dataset.github.io/},

year  = {2024},

date = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Video Anomaly Detection (VAD) finds widespread applications in security surveillance, traffic monitoring, industrial monitoring, and healthcare. Despite extensive research efforts, there remains a lack of concise reviews that provide insightful guidance for researchers. Such reviews would serve as quick references to grasp current challenges, research trends, and future directions. In this paper, we present such a review, examining models and datasets from various perspectives. We emphasize the critical relationship between model and dataset, where the quality and diversity of datasets profoundly influence model performance, and dataset development adapts to the evolving needs of emerging approaches. Our review identifies practical issues, including the absence of comprehensive datasets with diverse scenarios. To address this, we introduce a new dataset, Multi-Scenario Anomaly Detection (MSAD), comprising 14 distinct scenarios captured from various camera views. Our dataset has diverse motion patterns and challenging variations, such as different lighting and weather conditions, providing a robust foundation for training superior models. We conduct an in-depth analysis of recent representative models using MSAD and highlight its potential in addressing the challenges of detecting anomalies across diverse and evolving surveillance scenarios. Our dataset is available here.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2024c,

title = {Taming Cross-Domain Representation Variance in Federated Prototype Learning with Heterogeneous Data Domains},

author = {Lei Wang and Jieming Bian and Letian Zhang and Chen Chen and Jie Xu},

url = {https://arxiv.org/pdf/2403.09048 

https://arxiv.org/abs/2403.09048},

year  = {2024},

date = {2024-12-09},

urldate = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Federated learning (FL) allows collaborative machine learning training without sharing private data. While most FL methods assume identical data domains across clients, real-world scenarios often involve heterogeneous data domains. Federated Prototype Learning (FedPL) addresses this issue, using mean feature vectors as prototypes to enhance model generalization. However, existing FedPL methods create the same number of prototypes for each client, leading to cross-domain performance gaps and disparities for clients with varied data distributions. To mitigate cross-domain feature representation variance, we introduce FedPLVM, which establishes variance-aware dual-level prototypes clustering and employs a novel α-sparsity prototype loss. The dual-level prototypes clustering strategy creates local clustered prototypes based on private data features, then performs global prototypes clustering to reduce communication complexity and preserve local data privacy. The α-sparsity prototype loss aligns samples from underrepresented domains, enhancing intra-class similarity and reducing inter-class similarity. Evaluations on Digit-5, Office-10, and DomainNet datasets demonstrate our method's superiority over existing approaches.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Morafah2024,

title = {Towards Diverse Device Heterogeneous Federated Learning via Task Arithmetic Knowledge Integration},

author = {Mahdi Morafah and Vyacheslav Kungurtsev and Hojin Matthew Chang and Chen Chen and Bill Lin},

url = {https://arxiv.org/pdf/2409.18461 

https://mmorafah.github.io/takflpage/ 

https://github.com/MMorafah/TAKFL},

year  = {2024},

date = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Federated Learning (FL) has emerged as a promising paradigm for collaborative machine learning, while preserving user data privacy. Despite its potential, standard FL algorithms lack support for diverse heterogeneous device prototypes, which vary significantly in model and dataset sizes—from small IoT devices to large workstations. This limitation is only partially addressed by existing knowledge distillation (KD) techniques, which often fail to transfer knowledge effectively across a broad spectrum of device prototypes with varied capabilities. This failure primarily stems from two issues: the dilution of informative logits from more capable devices by those from less capable ones, and the use of a single integrated logits as the distillation target across all devices, which neglects their individual learning capacities and and the unique contributions of each device. To address these challenges, we introduce TAKFL, a novel KD-based framework that treats the knowledge transfer from each device prototype’s ensemble as a separate task, independently distilling each to preserve its unique contributions and avoid dilution. TAKFL also incorporates a KD-based self-regularization technique to mitigate the issues related to the noisy and unsupervised ensemble distillation process. To integrate the separately distilled knowledge, we introduce an adaptive task arithmetic knowledge integration process, allowing each student model to customize the knowledge integration for optimal performance. Additionally, we present theoretical results demonstrating the effectiveness of task arithmetic in transferring knowledge across heterogeneous device prototypes with varying capacities. Comprehensive evaluations of our method across both computer vision (CV) and natural language processing (NLP) tasks demonstrate that TAKFL achieves state-of-the-art results in a variety of datasets and settings, significantly outperforming existing KD-based methods. Our code is released at https://github.com/MMorafah/TAKFL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Lim2024c,

title = {AirSketch: Generative Motion to Sketch},

author = {Hui Xian Grace Lim and Xuanming Cui and Yogesh Singh Rawat and Ser-Nam Lim},

url = {https://arxiv.org/pdf/2407.08906 

https://www.crcv.ucf.edu/person/rawat/},

year  = {2024},

date = {2024-12-09},

urldate = {2024-07-12},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Illustration is a fundamental mode of human expression and communication. Certain types of motion that accompany speech can provide this illustrative mode of communication. While Augmented and Virtual Reality technologies (AR/VR) have introduced tools for producing drawings with hand motions (air drawing), they typically require costly hardware and additional digital markers, thereby limiting their accessibility and portability. Furthermore, air drawing demands considerable skill to achieve aesthetic results. To address these challenges, we introduce the concept of AirSketch, aimed at generating faithful and visually coherent sketches directly from hand motions, eliminating the need for complicated headsets or markers. We devise a simple augmentation-based self-supervised training procedure, enabling a controllable image diffusion model to learn to translate from highly noisy hand tracking images to clean, aesthetically pleasing sketches, while preserving the essential visual cues from the original tracking data. We present two air drawing datasets to study this problem. Our findings demonstrate that beyond producing photo-realistic images from precise spatial inputs, controllable image diffusion can effectively produce a refined, clear sketch from a noisy input. Our work serves as an initial step towards marker-less air drawing and reveals distinct applications of controllable diffusion models to AirSketch and AR/VR in general.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2024b,

title = {Advancing Video Anomaly Detection: A Concise Review and a New Dataset},

author = {Arjun Raj Lei Wang Liyun Zhu},

url = {https://arxiv.org/pdf/2402.04857 

https://arxiv.org/abs/2402.04857 

https://msad-dataset.github.io/},

year  = {2024},

date = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Video Anomaly Detection (VAD) finds widespread applications in security surveillance, traffic monitoring, industrial monitoring, and healthcare. Despite extensive research efforts, there remains a lack of concise reviews that provide insightful guidance for researchers. Such reviews would serve as quick references to grasp current challenges, research trends, and future directions. In this paper, we present such a review, examining models and datasets from various perspectives. We emphasize the critical relationship between model and dataset, where the quality and diversity of datasets profoundly influence model performance, and dataset development adapts to the evolving needs of emerging approaches. Our review identifies practical issues, including the absence of comprehensive datasets with diverse scenarios. To address this, we introduce a new dataset, Multi-Scenario Anomaly Detection (MSAD), comprising 14 distinct scenarios captured from various camera views. Our dataset has diverse motion patterns and challenging variations, such as different lighting and weather conditions, providing a robust foundation for training superior models. We conduct an in-depth analysis of recent representative models using MSAD and highlight its potential in addressing the challenges of detecting anomalies across diverse and evolving surveillance scenarios. Our dataset is available here.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2024cb,

title = {Taming Cross-Domain Representation Variance in Federated Prototype Learning with Heterogeneous Data Domains},

author = {Lei Wang and Jieming Bian and Letian Zhang and Chen Chen and Jie Xu},

url = {https://arxiv.org/pdf/2403.09048 

https://arxiv.org/abs/2403.09048},

year  = {2024},

date = {2024-12-09},

urldate = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Federated learning (FL) allows collaborative machine learning training without sharing private data. While most FL methods assume identical data domains across clients, real-world scenarios often involve heterogeneous data domains. Federated Prototype Learning (FedPL) addresses this issue, using mean feature vectors as prototypes to enhance model generalization. However, existing FedPL methods create the same number of prototypes for each client, leading to cross-domain performance gaps and disparities for clients with varied data distributions. To mitigate cross-domain feature representation variance, we introduce FedPLVM, which establishes variance-aware dual-level prototypes clustering and employs a novel α-sparsity prototype loss. The dual-level prototypes clustering strategy creates local clustered prototypes based on private data features, then performs global prototypes clustering to reduce communication complexity and preserve local data privacy. The α-sparsity prototype loss aligns samples from underrepresented domains, enhancing intra-class similarity and reducing inter-class similarity. Evaluations on Digit-5, Office-10, and DomainNet datasets demonstrate our method's superiority over existing approaches.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Morafah2024b,

title = {Towards Diverse Device Heterogeneous Federated Learning via Task Arithmetic Knowledge Integration},

author = {Mahdi Morafah and Vyacheslav Kungurtsev and Hojin Matthew Chang and Chen Chen and Bill Lin},

url = {https://arxiv.org/pdf/2409.18461 

https://mmorafah.github.io/takflpage/ 

https://github.com/MMorafah/TAKFL},

year  = {2024},

date = {2024-12-09},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {Federated Learning (FL) has emerged as a promising paradigm for collaborative machine learning, while preserving user data privacy. Despite its potential, standard FL algorithms lack support for diverse heterogeneous device prototypes, which vary significantly in model and dataset sizes—from small IoT devices to large workstations. This limitation is only partially addressed by existing knowledge distillation (KD) techniques, which often fail to transfer knowledge effectively across a broad spectrum of device prototypes with varied capabilities. This failure primarily stems from two issues: the dilution of informative logits from more capable devices by those from less capable ones, and the use of a single integrated logits as the distillation target across all devices, which neglects their individual learning capacities and and the unique contributions of each device. To address these challenges, we introduce TAKFL, a novel KD-based framework that treats the knowledge transfer from each device prototype’s ensemble as a separate task, independently distilling each to preserve its unique contributions and avoid dilution. TAKFL also incorporates a KD-based self-regularization technique to mitigate the issues related to the noisy and unsupervised ensemble distillation process. To integrate the separately distilled knowledge, we introduce an adaptive task arithmetic knowledge integration process, allowing each student model to customize the knowledge integration for optimal performance. Additionally, we present theoretical results demonstrating the effectiveness of task arithmetic in transferring knowledge across heterogeneous device prototypes with varying capacities. Comprehensive evaluations of our method across both computer vision (CV) and natural language processing (NLP) tasks demonstrate that TAKFL achieves state-of-the-art results in a variety of datasets and settings, significantly outperforming existing KD-based methods. Our code is released at https://github.com/MMorafah/TAKFL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2023,

title = {On Occlusions in Video Action Detection: Benchmark Datasets And Training Recipes},

author = {Rajat Modi and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/nips_23.pdf},

year  = {2023},

date = {2023-12-10},

publisher = {NeurIPS 2023.},

abstract = {This paper explores the impact of occlusions in video action detection. We facilitate this study by introducing five new benchmark datasets namely O-UCF and OJHMDB consisting of synthetically controlled static/dynamic occlusions, OVISUCF and OVIS-JHMDB consisting of occlusions with realistic motions and Real-OUCF for occlusions in realistic-world scenarios. We formally confirm an intuitive expectation: existing models suffer a lot as occlusion severity is increased and exhibit different behaviours when occluders are static vs when they are moving. We discover several curious phenomenon emerging in neural nets: 1) transformers can naturally outperform CNN models which might have even used occlusion as a form of data augmentation during training 2) incorporating symbolic-components like capsules to such backbones allows them to bind to occluders never even seen during training and 3) Islands of agreement (similar to the ones hypothesized in Hinton et Al’s GLOM) can emerge in realistic images/videos without instance-level supervision, distillation or contrastive-based objectives2(eg. video-textual training). Such emergent properties allow us to derive simple yet effective training recipes which lead to robust occlusion models inductively satisfying the first two stages of the binding mechanism (grouping/segregation). Models leveraging these recipes outperform existing video action-detectors under occlusion by 32.3% on O-UCF, 32.7% on O-JHMDB & 2.6% on Real-OUCF in terms of the vMAP metric.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Modi2023b,

title = {On Occlusions in Video Action Detection: Benchmark Datasets And Training Recipes},

author = {Rajat Modi and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/nips_23.pdf},

year  = {2023},

date = {2023-12-10},

publisher = {NeurIPS 2023.},

abstract = {This paper explores the impact of occlusions in video action detection. We facilitate this study by introducing five new benchmark datasets namely O-UCF and OJHMDB consisting of synthetically controlled static/dynamic occlusions, OVISUCF and OVIS-JHMDB consisting of occlusions with realistic motions and Real-OUCF for occlusions in realistic-world scenarios. We formally confirm an intuitive expectation: existing models suffer a lot as occlusion severity is increased and exhibit different behaviours when occluders are static vs when they are moving. We discover several curious phenomenon emerging in neural nets: 1) transformers can naturally outperform CNN models which might have even used occlusion as a form of data augmentation during training 2) incorporating symbolic-components like capsules to such backbones allows them to bind to occluders never even seen during training and 3) Islands of agreement (similar to the ones hypothesized in Hinton et Al’s GLOM) can emerge in realistic images/videos without instance-level supervision, distillation or contrastive-based objectives2(eg. video-textual training). Such emergent properties allow us to derive simple yet effective training recipes which lead to robust occlusion models inductively satisfying the first two stages of the binding mechanism (grouping/segregation). Models leveraging these recipes outperform existing video action-detectors under occlusion by 32.3% on O-UCF, 32.7% on O-JHMDB & 2.6% on Real-OUCF in terms of the vMAP metric.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_100,

title = {Are all Frames Equal? Active Sparse Labeling for Video Action Detection},

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/active-sparse-labeling-for-video-action-detection/ 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/neurips_poster_ASL_upload.png 

https://github.com/aayushjr/ASL-video},

year  = {2022},

date = {2022-11-28},

urldate = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Video action detection requires annotations at every frame, which drastically increases the labeling cost. In this work, we focus on efficient labeling of videos for action detection to minimize this cost. We propose active sparse labeling (ASL), a novel active learning strategy for video action detection. We propose a novel frame-level scoring mechanism aimed at selecting the most informative frames in a video. We also introduce a novel loss formulation which enables training of action detection model with these sparsely selected frames. We evaluated the proposed approach on two different action detection benchmark datasets, UCF-101-24 and J-HMDB-21, and observed that active sparse labeling can be very effective in saving annotation costs. We demonstrate that the proposed approach performs better than random selection, outperforming all other baselines, with performance comparable to supervised approach using merely 10% annotations.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schiappa2022,

title = {Robustness Analysis of Video-Language Models Against Visual and Language Perturbations},

author = {Madeline Chantry Schiappa and Shruti Vyas and Hamid Palangi and Yogesh Rawat and Vibhav Vineet},

url = {https://sites.google.com/view/videolanguagerobustness/home https://openreview.net/forum?id=A79jAS4MeW9 

https://github.com/Maddy12/VideoLanguageModelRobustness/tree/master},

year  = {2022},

date = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Joint visual and language modeling on large-scale datasets has recently shown good progress in multi-modal tasks when compared to single modal learning. However, robustness of these approaches against real-world perturbations has not been studied. In this work, we perform the first extensive robustness study of video-language models against various real-world perturbations. We focus on text-to-video retrieval and propose two large-scale benchmark datasets, MSRVTT-P and YouCook2-P, which utilize 90 different visual and 35 different text perturbations. The study reveals some interesting initial findings from the studied models: 1) models are more robust when text is perturbed versus when video is perturbed, 2) models that are pre-trained are more robust than those trained from scratch, 3) models attend more to scene and objects rather than motion and action. We hope this study will serve as a benchmark and guide future research in robust video-language learning. The benchmark introduced in this study along with the code and datasets is available at https://bit.ly/3CNOly4.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_148,

title = {Are all Frames Equal? Active Sparse Labeling for Video Action Detection},

author = {Aayush Rana and Yogesh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/active-sparse-labeling-for-video-action-detection/ 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/neurips_poster_ASL_upload.png 

https://github.com/aayushjr/ASL-video},

year  = {2022},

date = {2022-11-28},

urldate = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Video action detection requires annotations at every frame, which drastically increases the labeling cost. In this work, we focus on efficient labeling of videos for action detection to minimize this cost. We propose active sparse labeling (ASL), a novel active learning strategy for video action detection. We propose a novel frame-level scoring mechanism aimed at selecting the most informative frames in a video. We also introduce a novel loss formulation which enables training of action detection model with these sparsely selected frames. We evaluated the proposed approach on two different action detection benchmark datasets, UCF-101-24 and J-HMDB-21, and observed that active sparse labeling can be very effective in saving annotation costs. We demonstrate that the proposed approach performs better than random selection, outperforming all other baselines, with performance comparable to supervised approach using merely 10% annotations.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schiappa2022b,

title = {Robustness Analysis of Video-Language Models Against Visual and Language Perturbations},

author = {Madeline Chantry Schiappa and Shruti Vyas and Hamid Palangi and Yogesh Rawat and Vibhav Vineet},

url = {https://sites.google.com/view/videolanguagerobustness/home https://openreview.net/forum?id=A79jAS4MeW9 

https://github.com/Maddy12/VideoLanguageModelRobustness/tree/master},

year  = {2022},

date = {2022-11-28},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {Joint visual and language modeling on large-scale datasets has recently shown good progress in multi-modal tasks when compared to single modal learning. However, robustness of these approaches against real-world perturbations has not been studied. In this work, we perform the first extensive robustness study of video-language models against various real-world perturbations. We focus on text-to-video retrieval and propose two large-scale benchmark datasets, MSRVTT-P and YouCook2-P, which utilize 90 different visual and 35 different text perturbations. The study reveals some interesting initial findings from the studied models: 1) models are more robust when text is perturbed versus when video is perturbed, 2) models that are pre-trained are more robust than those trained from scratch, 3) models attend more to scene and objects rather than motion and action. We hope this study will serve as a benchmark and guide future research in robust video-language learning. The benchmark introduced in this study along with the code and datasets is available at https://bit.ly/3CNOly4.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xu2022,

title = {Don’t Pour Cereal into Coffee: Differentiable Temporal Logic for Temporal Action Segmentation},

author = {Ziwei Xu and Yogesh Rawat and Yongkang Wong and Mohan Kankanhalli and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ziwei_neurips2022.pdf 

https://diff-tl.github.io/ 

https://github.com/ZiweiXU/DTL-action-segmentation},

year  = {2022},

date = {2022-11-09},

urldate = {2022-11-09},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {We propose Differentiable Temporal Logic (DTL), a model-agnostic framework that introduces temporal constraints to deep networks. DTL treats the outputs of a network as a truth assignment of a temporal logic formula, and computes a temporal logic loss reflecting the consistency between the output and the constraints. We propose a comprehensive set of constraints, which are implicit in data annotations, and incorporate them with deep networks via DTL. We evaluate the effectiveness of DTL on the temporal action segmentation task and observe improved performance and reduced logical errors in the output of different task models. Furthermore, we provide an extensive analysis to visualize the desirable effects of DTL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xu2022b,

title = {Don’t Pour Cereal into Coffee: Differentiable Temporal Logic for Temporal Action Segmentation},

author = {Ziwei Xu and Yogesh Rawat and Yongkang Wong and Mohan Kankanhalli and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ziwei_neurips2022.pdf 

https://diff-tl.github.io/ 

https://github.com/ZiweiXU/DTL-action-segmentation},

year  = {2022},

date = {2022-11-09},

urldate = {2022-11-09},

publisher = {36th Conference on Neural Information Processing Systems (NeurIPS 2022)},

abstract = {We propose Differentiable Temporal Logic (DTL), a model-agnostic framework that introduces temporal constraints to deep networks. DTL treats the outputs of a network as a truth assignment of a temporal logic formula, and computes a temporal logic loss reflecting the consistency between the output and the constraints. We propose a comprehensive set of constraints, which are implicit in data annotations, and incorporate them with deep networks via DTL. We evaluate the effectiveness of DTL on the temporal action segmentation task and observe improved performance and reduced logical errors in the output of different task models. Furthermore, we provide an extensive analysis to visualize the desirable effects of DTL.},

keywords = {NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kerrigan2021,

title = {Reformulating Zero-shot Action Recognition for Multi-label Actions},

author = {Alec Kerrigan and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/reformulating_zero_shot_action2.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ZSL-Supp.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Reformulating-Zero-shot-Action-Recognition-forMulti-label-Actions.pptx},

year  = {2021},

date = {2021-12-06},

urldate = {2021-12-06},

booktitle = {Thirty-fifth Conference on Neural Information Processing Systems},

keywords = {Action Recognition, NeurIPS, Zero-Shot Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kerrigan2021b,

title = {Reformulating Zero-shot Action Recognition for Multi-label Actions},

author = {Alec Kerrigan and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/reformulating_zero_shot_action2.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ZSL-Supp.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Reformulating-Zero-shot-Action-Recognition-forMulti-label-Actions.pptx},

year  = {2021},

date = {2021-12-06},

urldate = {2021-12-06},

booktitle = {Thirty-fifth Conference on Neural Information Processing Systems},

keywords = {Action Recognition, NeurIPS, Zero-Shot Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khodadadeh2019,

title = {Unsupervised Meta-Learning for Few-Shot Image Classification},

author = {Siavash Khodadadeh and Ladislau Bölöni and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/01/Publications_unsupervised-meta-learning-for-few-shot-image-classification.pdf 

https://www.crcv.ucf.edu/research/projects/unsupervised-meta-learning-for-few-shot-image-and-video-classification/},

year  = {2019},

date = {2019-11-30},

urldate = {2019-11-30},

booktitle = {33rd Conference on Neural Information Processing Systems (NeurIPS 2019), Vancouver, Canada},

journal = {33rd Conference on Neural Information Processing Systems (NeurIPS 2019), Vancouver, Canada},

keywords = {Meta-Learning, NeurIPS},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khodadadeh2019b,

title = {Unsupervised Meta-Learning for Few-Shot Image Classification},

author = {Siavash Khodadadeh and Ladislau Bölöni and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/01/Publications_unsupervised-meta-learning-for-few-shot-image-classification.pdf 

https://www.crcv.ucf.edu/research/projects/unsupervised-meta-learning-for-few-shot-image-and-video-classification/},

year  = {2019},

date = {2019-11-30},

urldate = {2019-11-30},

booktitle = {33rd Conference on Neural Information Processing Systems (NeurIPS 2019), Vancouver, Canada},

journal = {33rd Conference on Neural Information Processing Systems (NeurIPS 2019), Vancouver, Canada},

keywords = {Meta-Learning, NeurIPS},

pubstate = {published},

tppubtype = {conference}

}