Publications

@conference{nokey,

title = {Monocular Models are Strong Learners for Multi-View Human Mesh Recovery},

author = {Haoyu Xie; Shengkai Xu; Cheng Guo; Muhammad Usama Saleem; Wenhan Wu; Chen Chen; Ahmed Helmy; Pu Wang; Hongfei Xue},

url = {https://arxiv.org/abs/2603.20391},

year  = {2026},

date = {2026-09-08},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {Multi-view human mesh recovery (HMR) is broadly deployed in diverse domains where high accuracy and strong generalization are essential. Existing approaches can be broadly grouped into geometry-based and learning-based methods. However, geometry-based methods (e.g., triangulation) rely on cumbersome camera calibration, while learning-based approaches often generalize poorly to unseen camera configurations due to the lack of multi-view training data, limiting their performance in real-world scenarios. To enable calibration-free reconstruction that generalizes to arbitrary camera setups, we propose a training-free framework that leverages pretrained single-view HMR models as strong priors, eliminating the need for multi-view training data. Our method first constructs a robust and consistent multi-view initialization from single-view predictions, and then refines it via test-time optimization guided by multi-view consistency and anatomical constraints. Extensive experiments demonstrate state-of-the-art performance on standard benchmarks, surpassing multi-view models trained with explicit multi-view supervision.},

keywords = {ECCV, Human Pose Estimation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tian2026,

title = {CLARITY: Medical World Model for Guiding Treatment Decisions by Modeling Context-Aware Disease Trajectories in Latent Space},

author = {Tianxingjian Ding and Yuanhao Zou and Chen Chen and Mubarak Shah and Yu Tian},

url = {https://arxiv.org/abs/2512.08029

https://ai.ucf.edu/wp-content/uploads/2026/09/CLARITY-1.pdf

https://youtu.be/4Gvaj6ppPzY?si=YYxdnLW-V4JhfGkx},

year  = {2026},

date = {2026-09-08},

urldate = {2026-09-08},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {Clinical decision-making in oncology requires predicting dynamic disease evolution, a task current static AI predictors cannot perform. While world models (WMs) offer a paradigm for generative prediction, existing medical applications remain limited. Existing methods often rely on stochastic diffusion models, focusing on visual reconstruction rather than causal, physiological transitions. Furthermore, in medical domain, models like MeWM typically ignore patient-specific temporal and clinical contexts and lack a feedback mechanism to link predictions to treatment decisions. To address these gaps, we introduce CLARITY, a medical world model that forecasts disease evolution directly within a structured latent space. It explicitly integrates time intervals (temporal context) and patient-specific data (clinical context) to model treatment-conditioned progression as a smooth, interpretable trajectory, and thus generate physiologically faithful, individualized treatment plans. Finally, CLARITY introduces a novel prediction-to-decision framework, translating latent rollouts into transparent, actionable recommendations. CLARITY demonstrates state-of-the-art performance in treatment planning. On the MU-Glioma-Post dataset, our approach outperforms recent MeWM by 12%, and significantly surpasses all other medical-specific large language models.},

keywords = {ECCV, Medical AI, World Models},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {SAF3R: Dynamic Sparse Attention for Feed-Forward 3D Reconstruction Transformers},

author = {Jianing Deng; Yuanzhe LI; Jialu Wang; Song Wang; Tianlong Chen; Huanrui Yang; Jingtong Hu},

url = {https://arxiv.org/abs/2607.03612},

year  = {2026},

date = {2026-09-07},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {Feed-forward 3D reconstruction (F3R) transformers have recently achieved remarkable success. However, scaling them to long image sequences remains challenging, as the quadratic complexity of cross-view global attention quickly becomes the dominant computational bottleneck. While recent efforts attempt to improve efficiency through compressed or sparse attention, they fail to fully exploit the inherent sparsity and dynamic behavior of global attention. In this work, we present a comprehensive analysis of global attention across multiple F3R transformers and reveal that attention patterns are highly heterogeneous, dynamic, and extremely sparse across layers and attention heads. Motivated by these findings, we propose SAF3R, a training-free dynamic sparse attention framework tailored to F3R transformers. SAF3R integrates tailored sparse attention mechanisms with offline head profiling and an efficient online adaptation strategy to match input-dependent attention behaviors. Extensive experiments demonstrate that SAF3R achieves high sparsity ratios while preserving camera pose estimation and 3D reconstruction quality, translating into substantial end-to-end speedup on F3R transformers compared to existing methods. },

keywords = {3D Reconstruction, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Forget, Anticipate and Adapt: Test Time Training for Long Videos},

author = {Rajat Modi and S. Noel and Xin Liang and Yogesh Singh Rawat

},

url = {https://arxiv.org/abs/2606.26515},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {Test Time Training (TTT) is a mechanism in which a model adapts to an incoming test-sample by performing some self-supervised (SSL) task and updating its weights even during inference. This procedure does not require labels at test-time. This paper focuses on TTT for long-videos. A major concern with existing approaches is: 1) they perform TTT updates using a sliding window containing frames in the past, whose compute increases linearly with the size of window. This becomes computationally intractable when the videos are hours long. 2) TTT is performed even when temporally close frames look similar, thereby consuming a lot of compute.

We present the Frame Forgetting Network (FFN) that: 1) operates on only three frames within the sliding window, namely the frame that exits, the current frame and the frame after that. The model still manages to retain temporal context and work for hours long-videos; 2) mathematically define a surprise metric: how much new information the incoming frame contains with respect to the past seen frame. This facilitates determining how to modify the effective window size during TTT and constitutes the core mechanism of an adaptive windowing algorithm. Additionally, we curate a dataset EpicTours containing up to 3 hour long videos of walking city-tours, whereas earlier datasets on this problem were only 5 min long. We demonstrate FFNs empirical effectiveness on dense-segmentation, video classification tasks, generalization to depth-estimation, and multi-hour long videos.},

keywords = {ECCV, Long Video Understanding},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Robust Onion: Peeling Open Vocab Object Detectors Under Noise},

author = {Priyank Pathak; M. Karuppasamy, A. Baranwal; Shruti Vyas; Yogesh Singh Rawat

},

url = {https://arxiv.org/abs/2606.26734},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {The impact of real-world noise on Open Vocabulary Object Detectors (OV-ODs) remains poorly understood due to their architectural complexity. We present our comprehensive analysis Robust Onion, an empirical study that uses controlled synthetic visual degradations to peel OV-ODs layer-by-layer, revealing how, why, and where robustness degrades, systematically analyzing feature collapse. Our findings reveal that models with similar vision backbones exhibit comparable robustness, driven by similar feature collapse at similar layers, while factors such as pretraining strategy, architectural nuances, and caption supervision contribute little. Robustness is primarily governed by the image domain rather than annotations, explaining the similar robustness impact on COCO and LVIS, and why datasets like ODinW-13 can give an impression of inflated robustness due to large, isolated objects. Finally, we validate our insights by improving robustness on real-world BDD100K, WiderFace, and VisDRONE via our lightweight plug-and-play NN & TK0 approach, using 96x fewer trainable parameters than end-to-end training. We also explain the prior works' robustness observations.},

keywords = {ECCV, Explainability, Interpretability},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {CoSPlan: Corrective Sequential Planning via Scene Graph Incremental Updates},

author = {Shresth Grover and Priyank Pathak and Akash Kumar and Yogesh Singh Rawat

},

url = {https://arxiv.org/abs/2512.10342

https://shroglck.github.io/cos_plan/},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {Vision Language Models (VLMs) have shown promising planning capabilities, yet their success remains confined to the text domain, leaving visual decision-making relatively underexplored. Addressing this gap, we introduce Corrective Sequence Planning (CoSPlan) benchmark, where VLMs must plan a sequence of visual actions from an initial scene to a target scene. CoSPlan evaluates models on their ability to imagine and execute a coherent set of visual steps required to reach the goal (Step Completion). To prevent any shortcuts that simply describe the final scene, we introduce an erroneous action in decision-making, which must be detected (Error Detection) and corrected to reach the goal, enabling a deeper understanding of the task. CoSPlan spans across 4 tasks: maze navigation, block re-arrangement, image reconstruction, and object re-organization. Despite using advanced reasoning strategies such as Chain-of-Thought and Scene Graphs, VLMs struggle on CoSPlan, while still showing promising performance in the text domain. Addressing this, we propose Scene Graph Incremental updates (SGI), a novel training-free method to transform images into `textual' scene graphs, enabling step-by-step reasoning through iterative scene graph refinement. SGI yields an average of ~4.4% improvement on CoSPlan w/ generalization on PlanBench and VQA. Link for solving puzzles on the project page.},

keywords = {ECCV, VLM},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Learning to Deny: Action Denial in Multimodal Large Language Models},

author = {Raiyaan Abdullah and Shehreen Azad and Yogesh Singh Rawat

},

url = {https://arxiv.org/abs/2606.31187},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {Multimodal large language models (MLLMs) have rapidly advanced video understanding, achieving strong zero-shot and few-shot recognition across standard benchmarks. Yet their ability to deny an action by recognizing when an activity is not happening despite strong contextual cues remains largely unexplored. We introduce UCF101-AD, a large-scale benchmark consisting of paired Action-Presence and Action-Denial clips, designed to evaluate this capacity for denial. Each negative video in UCF101-AD preserves the same contextual and motion cues, including persons, objects, and locations, as its positive counterpart, but the defining action itself is explicitly absent. Evaluating 20 state-of-the-art MLLMs reveals a consistent failure: models that exceed 85% accuracy on the positive action classes collapse below 50% on their action-denial counterparts, indicating a strong inclination to affirm plausible actions rather than verify that they truly occur. This exposes a critical blind spot in modern video understanding: the inability to reason causally about whether a motion actually happens. To probe this issue, we explore a causal graph formulation, CausalAct, which expresses scene structure through natural-language prompts linking context, interaction, and motion. Incorporating such causal cues substantially reduces false positives, demonstrating that denial is a learnable reasoning skill. UCF101-AD provides a new lens for diagnosing and improving causal reasoning in multimodal models. },

keywords = {ECCV, MLLM},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {DnA: Denoising Attention for Visual Tasks},

author = {Ron Campos and Subhajit Maity and Xin Li and Srijan Das and Aritra Dutta},

url = {https://rjccv.github.io/DnA/},

year  = {2026},

date = {2026-09-07},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {The softmax activation in multihead attention (MHA) is the de facto standard for attention-based models in visual perception tasks. However, standard softmax can produce noisy attention patterns that dilute relevant features and degrade its performance. In this paper, we propose Denoising Attention or DnA, in which, first, a positive query identifies which image features belong to the correct class, and a negative query identifies closely associated but irrelevant image features. DnA then projects these interactions into two distinct subspaces with larger principal angles, promoting subspace separation and improved discriminability. Using a ViT-B backbone, our proposed DnA achieves a 0.8% absolute gain on ImageNet-1K compared to the baseline. We further show improvements across multiple visual understanding tasks, including video understanding with video transformers (1.8%) and video LLMs (0.5%). Our extensive empirical analyses justify the design choices involving two interacting subspaces and the denoising effect of DnA.},

keywords = {ECCV, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{fioresi2026controlembedb,

title = {Controlling Embedding Spaces with Text-Conditioned Transformations},

author = {Joseph Fioresi and Fabian Caba Heilbron and Pankaj Nathani and Mubarak Shah and Kushal Kafle},

url = {https://arxiv.org/abs/2607.22919 

https://ai.ucf.edu/wp-content/uploads/2026/09/ControlEmbed_ECCV_poster.pdf

https://youtu.be/n0Hy5GSCrtw?si=9n-OekUWO_aRF2cN},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

booktitle = {European Conference on Computer Vision (ECCV)},

publisher = {Springer},

abstract = {Multimodal embedding spaces in models like CLIP enable powerful capabilities such as semantic similarity retrieval and cross-modal zero-shot classification. These embeddings compress high-level semantics into a single vector, which comes at the cost of primarily expressing a dominant semantics like main object while suppressing other important attributes such as camera angle or color tone. We propose a text-conditioned transformation of visual embeddings that makes such attributes explicitly accessible. Given a natural language description of an attribute category (e.g., "color" or "art style"), a network generates an affine transformation that emphasizes the specified attribute. Conditioning on text enables it to learn many attributes simultaneously, accessing them at inference time through an intuitive interface. The network is trained to align transformed embeddings with the frozen latent space, enabling retrieval using existing large-scale embeddings without any re-encoding. When applied to a full set, the same mechanism transforms the latent space for attribute disentanglement tasks such as multi-clustering. By operating directly in latent space, our method provides a unified and efficient framework for controlling embedding spaces, demonstrating state-of-the-art performance across both attribute-based retrieval and multi-attribute organization tasks with near-zero inference cost. Project page: this https URL},

keywords = {CLIP, ECCV, Multimodal Retrieval},

pubstate = {published},

tppubtype = {conference}

}

@conference{yousaf2026sageb,

title = {The Illusion of High Utility in Safety Alignment of Text-to-Image Diffusion Models},

author = {Adeel Yousaf and Soumik Ghosh and James Beetham and Amrit Singh Bedi and Mubarak Shah},

url = {https://arxiv.org/abs/2607.00402 

https://ai.ucf.edu/wp-content/uploads/2026/09/Illusion-of-High-Utility-in-Safety-JPG-1.jpg

https://youtu.be/BVw6WnfG2iw?si=WtAX_hPSnDRR0-1u},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

booktitle = {European Conference on Computer Vision (ECCV)},

publisher = {Springer},

abstract = {Safety alignment of text-to-image (T2I) diffusion models aims to suppress harmful generations while preserving utility on benign prompts. Recent methods often appear to deliver high safety with high utility, but this conclusion rests largely on coarse global utility metrics (e.g., FID, CLIPScore) that are insensitive to fine-grained semantic correctness, creating an illusion of high utility. We show that when utility is measured with structured evaluation, this illusion breaks: on TIFA (Text-to-Image Faithfulness evaluation with Question Answering), safety-aligned models suffer substantial drops in semantic fidelity, including failures in object counts, attributes, and relationships. To diagnose the source of this gap, we analyze the text-encoder prompt embedding space and uncover semantic collapse, a contraction of embedding spread coupled with distortion of inter-prompt similarity structure, which strongly correlates with structured utility loss. Guided by this insight, we propose StructureAware Geometric Regularization (SAGE), a safety alignment objective that explicitly preserves embedding spread and inter-prompt relational structure during adaptation. Our method restores structured utility (TIFA +5.0% over prior state-of-the-art) while maintaining strong safety performance and competitive coarse-grained utility scores. Our source code and trained models are available at this https URL.},

keywords = {AI Safety, Diffusion Models, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{zhang2026reasonclipb,

title = {ReasonCLIP-58M: Visually Grounded Commonsense Reasoning Supervision for CLIP},

author = {Sicheng Zhang and Muhammad Muzammal Naseer and Binzhu Xie and Naufal Suryanto and Shi Qiu and Jamal Bentahar and Naveed Akhtar and Mubarak Shah},

url = {https://arxiv.org/abs/2606.26794},

year  = {2026},

date = {2026-09-07},

urldate = {2026-01-01},

booktitle = {European Conference on Computer Vision (ECCV)},

publisher = {Springer},

abstract = {CLIP and its variants are widely adopted visual backbones in multimodal systems, but their pretraining remains dominated by descriptive image-text alignment. As downstream applications increasingly demand visually grounded commonsense inference and compositional reasoning, it remains unclear whether CLIP-style encoders can support such reasoning without architectural changes. To address this, we present ReasonCLIP-58M, a continual pretraining framework that integrates large-scale reasoning supervision into CLIP-style models through our two-stage strategy, which progressively integrates reasoning signals while preserving descriptive alignment, followed by category-structured reasoning supervision. To support this framework, we construct two complementary datasets and a benchmark: ReasonLite-42M, with open-form, visually verifiable reasoning captions; ReasonPro-16M, with category-specific reasoning supervision; and RCLIP-Bench for diagnostic evaluation of visually grounded reasoning. We train a family of ReasonCLIP that improves visually grounded commonsense and compositional reasoning while also enhancing zero-shot retrieval performance. As a drop-in visual encoder for multimodal large language models such as LLaVA-NeXT, ReasonCLIP delivers consistent gains without additional inference cost, demonstrating that structured reasoning supervision enhances the expressive capacity of CLIP-style visual representations. All datasets, models, and training code are available at this https URL.},

keywords = {Commonsense Reasoning, ECCV, Vision-Language Models},

pubstate = {published},

tppubtype = {conference}

}

@conference{narnaware2026visageb,

title = {Seeing to Ground: Visual Attention for Hallucination-Resilient MDLLMs},

author = {Vishal Narnaware and Animesh Gupta and Kevin Zhai and Zhenyi Wang and Mubarak Shah},

url = {https://arxiv.org/abs/2603.25711

https://ai.ucf.edu/wp-content/uploads/2026/09/Seeing-to-Ground-ECCV.jpg

https://youtu.be/60Acip-ylyM?si=WtRA0tTr-quzOOCO},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

booktitle = {European Conference on Computer Vision (ECCV)},

publisher = {Springer},

abstract = {Multimodal Diffusion Large Language Models (MDLLMs) achieve high-concurrency generation through parallel masked decoding, yet the architectures remain prone to multimodal hallucinations. This structural vulnerability stems from an algorithmic flaw: the decoder ranks candidate tokens based on textual likelihood without verifying localized visual support. We establish that this language-only ranking induces an objective mismatch, where language probability mass acts as a misspecified proxy for the intended multimodal task. Consequently, we reinterpret hallucination as a localized optimization error, a phenomenon where the decoder exploits language shortcuts to maximize a proxy score at the expense of visual grounding. To address this objective mismatch, we introduce VISAGE, a training-free decoding framework that calibrates the objective at inference time. VISAGE estimates the proxy discrepancy by quantifying the spatial entropy of cross-attention distributions. By enforcing a localization consensus across attention heads, the method penalizes spatially uniform distributions and re-ranks token commitments to favor visually grounded outcomes. We provide an analytical stability guarantee establishing that VISAGE maintains a bounded objective loss under estimation error. Evaluations across hallucination-sensitive and general-purpose benchmarks demonstrate the robustness of the framework, yielding relative gains of 8.59% on MMMU-val and 7.75% on HallusionBench.},

keywords = {ECCV, Hallucination Mitigation, Multimodal LLMs},

pubstate = {published},

tppubtype = {conference}

}

@conference{liu2026skyregb,

title = {Pixel-wise Geo-registration of Drone Images},

author = {Qingyang Liu and David G. Shatwell and Parth Parag Kulkarni and Mubarak Shah},

url = {https://qingyangliu.github.io/qingyang.github.io/static/Georegistration.pdf

https://ai.ucf.edu/wp-content/uploads/2026/09/Pixel-Wise-Geo-Registration-JPG.png

https://youtu.be/jgVuME3oFJ4?si=R-AkoBd3rlG_791Z},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

booktitle = {European Conference on Computer Vision (ECCV)},

publisher = {Springer},

abstract = {Cross-view geo-registration is the task of aligning a query image to a geodetically accurate reference (e.g., satellite image), so that each query pixel maps to a real-world GPS coordinate. Most existing work addresses the related problem of cross-view geo-localization, where the goal is typically to estimate the camera center using retrieval, classification, matching, or regression. Because these approaches do not provide dense supervision, they are poorly suited for learning and evaluating pixel-wise alignment. We introduce SkyReg, a geometry-aware geo-registration model that estimates the transformation between the query and reference images by explicitly modeling the 3D scene geometry. Applying this transformation to warp the query into the reference frame yields pixel-wise geo-localization, without relying on 2D point matches, and remains robust to occlusions and large viewpoint changes. To enable training and standardized evaluation, we release (i) SkyReg-Train, a large-scale dataset of drone–satellite images annotated with per-pixel GPS coordinates, depth maps, and camera parameters derived from LiDAR and structure-from-motion, and (ii) SkyReg-Bench, a held-out benchmark of unseen Urban and Suburban scenes with the same dense annotations. SkyReg achieves state-of-the-art performance against strong retrieval and homography baselines, demonstrating the value of geometry-aware models and dense geodetic benchmarks for cross-view geo-registration. Dataset available at https://parthpk.github.io/sk yreg-webpage.},

keywords = {ECCV, Geo-Localization, Remote Sensing},

pubstate = {published},

tppubtype = {conference}

}

@conference{mollah2026telephonegameb,

title = {The Telephone Game: Evaluating Semantic Drift in Unified Models},

author = {Sabbir Mollah and Rohit Gupta and Swetha Sirnam and Qingyang Liu and Ahnaf Munir and Mubarak Shah},

url = {https://arxiv.org/abs/2509.04438

https://ai.ucf.edu/wp-content/uploads/2026/09/Poster__Telephone_Game.pdf

https://youtu.be/1y_Pn6xD0fw?si=Oab_zk-25L591fLO},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

booktitle = {European Conference on Computer Vision (ECCV)},

publisher = {Springer},

abstract = {Employing a single, unified model (UM) for both visual understanding (image-to-text: I2T) and visual generation (text-to-image: T2I) has opened a new direction in Visual Language Model (VLM) research. While UMs can also support broader unimodal tasks (e.g., text-to-text, image-to-image), we focus on the core cross-modal pair T2I and I2T. Existing evaluation benchmarks consider these capabilities in isolation: FID and GenEval for T2I, and benchmarks such as MME, MMBench for I2T. These isolated single-pass metrics do not reveal cross-consistency: whether a model that "understands" a concept can also "render" it, nor whether semantic meaning is preserved when cycling between image and text modalities. To address this, we introduce the Semantic Drift Protocol (SDP) for Unified Models, a cyclic evaluation protocol that alternates I2T and T2I over multiple generations to quantify semantic drift. We propose two metrics: (i) Mean Cumulative Drift (MCD), an embedding-based measure of overall semantic drift; and (ii) Multi-Generation GenEval (MGG), an object-level compliance score extending GenEval. To assess generalization beyond COCO dataset, which is widely used in training; we create a new benchmark Nocaps+Docci400, sampled from NoCaps and DOCCI and evaluated on seven recent models. SDP reveals substantial variation in cross-modal stability: some models like BAGEL maintain semantic meaning over many alternations, whereas others like VILA-U drift quickly despite strong single-pass scores. Our results highlight SDP as a necessary complement to standard I2T and T2I evaluations. Code is available at this https URL},

keywords = {ECCV, Model Evaluation, Vision-Language Models},

pubstate = {published},

tppubtype = {conference}

}

@conference{narnaware2026bbqvb,

title = {BBQ-V: Benchmarking Visual Stereotype Bias in Large Multimodal Models},

author = {Vishal Narnaware and Ashmal Vayani and Rohit Gupta and Swetha Sirnam and Mubarak Shah},

url = {https://arxiv.org/abs/2502.08779

https://ai.ucf.edu/wp-content/uploads/2026/09/BBQ-V-ECCV.jpg

https://youtu.be/y4bLLiA_gGA?si=J61V2M-LkFjyrHMy},

year  = {2026},

date = {2026-09-07},

urldate = {2026-09-07},

booktitle = {European Conference on Computer Vision (ECCV)},

publisher = {Springer},

abstract = {Stereotype biases in Large Multimodal Models (LMMs) perpetuate harmful societal prejudices, undermining the fairness and equity of AI applications. As LMMs grow increasingly influential, addressing and mitigating inherent biases related to stereotypes, harmful generations, and ambiguous assumptions in real-world scenarios has become essential. However, existing datasets evaluating stereotype biases in LMMs often lack diversity, rely on synthetic images, and often have single-actor images, leaving a gap in bias evaluation for real-world visual contexts. To address the gap in bias evaluation using real images, we introduce the BBQ-Vision (BBQ-V), the most comprehensive framework for assessing stereotype biases across nine diverse categories and 50 sub-categories with real and multi-actor images. BBQ-V benchmark contains 14,144 image-question pairs and rigorously evaluates LMMs through carefully curated, visually grounded scenarios, challenging them to reason accurately about visual stereotypes. It offers a robust evaluation framework featuring real-world visual samples, image variations, and open-ended question formats. BBQ-V enables a precise and nuanced assessment of a model's reasoning capabilities across varying levels of difficulty. Through rigorous testing of 19 state-of-the-art open-source (general-purpose and reasoning) and closed-source LMMs, we highlight that these top-performing models are often biased on several social stereotypes, and demonstrate that the thinking models induce more bias in the reasoning chains. This benchmark represents a significant step toward fostering fairness in AI systems and reducing harmful biases, laying the groundwork for more equitable and socially responsible LMMs. Our dataset and evaluation code are publicly available.},

keywords = {AI Fairness, ECCV, Multimodal Evaluation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2024,

title = {Towards Multi-modal Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Aritra Dutta and Xin Li and Chen Chen},

url = {https://arxiv.org/pdf/2404.12467.pdf

https://github.com/imguangyu/FedCola},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_83,

title = {Möbius Transform for Mitigating Perspective Distortions in Representation Learning},

author = {Prakash Chandra Chhipa and Meenakshi Subhash Chippa and Kanjar De and Rajkumar Saini and Marcus Liwicki and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_ECCV2024_CameraReady.pdf 

https://prakashchhipa.github.io/projects/mpd 

https://youtu.be/MKh9NE_XEMY 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_presentation_ECCV2024_final.pdf},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Perspective distortion (PD) causes unprecedented changes in shape, size, orientation, angles, and other spatial relationships of visual concepts in images. Precisely estimating camera intrinsic and extrinsic parameters is a challenging task that prevents synthesizing perspective distortion. Non-availability of dedicated training data poses a critical barrier to developing robust computer vision methods. Additionally, distortion correction methods make other computer vision tasks a multi-step approach and lack performance. In this work, we propose 

mitigating perspective distortion (MPD) by employing a fine-grained parameter control on a specific family of Möbius transform to model real-world distortion without estimating camera intrinsic and extrinsic parameters and without the need for actual distorted data. Also, we present a dedicated perspectively distorted benchmark dataset, ImageNet-PD, to benchmark the robustness of deep learning models against this new dataset. The proposed method outperforms existing benchmarks, ImageNet-E and ImageNet-X. Additionally, it significantly 

improves performance on ImageNet-PD while consistently performing on standard data distribution. Notably, our method shows improved performance on three PD-affected real-world applications—crowd counting, fisheye image recognition, and person re-identification—and one PD-affected challenging CV task: object detection. The source code, dataset, and models are available on the project webpage at https://prakashchhipa.github.io/projects/mpd.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kang2024,

title = {SegVG: Transferring Object Bounding Box to Segmentation for Visual Grounding},

author = {Weitai Kang and Gaowen Liu and Mubarak Shah and Yan Yan},

url = {https://arxiv.org/pdf/2407.03200},

doi = {https://doi.org/10.48550/arXiv.2407.03200},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Different from Object Detection, Visual Grounding deals with detecting a bounding box for each text-image pair. This one box for each text-image data provides sparse supervision signals. Although previous works achieve impressive results, their passive utilization of annotation, i.e. the sole use of the box annotation as regression ground truth, results in a suboptimal performance. In this paper, we present SegVG, a novel method transfers the box-level annotation as Segmentation signals to provide an additional pixel-level supervision for Visual Grounding. Specifically, we propose the Multi-layer Multi-task Encoder-Decoder as the target grounding stage, where we learn a regression query and multiple segmentation queries to ground the target by regression and segmentation of the box in each decoding layer, respectively. This approach allows us to iteratively exploit the annotation as signals for both box-level regression and pixel-level segmentation. Moreover, as the backbones are typically initialized by pretrained parameters learned from unimodal tasks and the queries for both regression and segmentation are static learnable embeddings, a domain discrepancy remains among these three types of features, which impairs subsequent target grounding. To mitigate this discrepancy, we introduce the Triple Alignment module, where the query, text, and vision tokens are triangularly updated to share the same space by triple attention mechanism. Extensive experiments on five widely used datasets validate our state-of-the-art (SOTA) performance.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024,

title = {FinePseudo: Improving Pseudo-Labelling through Temporal-Alignablity for Semi-Supervised Fine-Grained Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/finepsuedo_eccv24_dave.pdf 

https://daveishan.github.io/finepsuedo-webpage/ https://youtu.be/bWOd8_JpjQs?si=WWRDhdg5ADWL0uwB},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Real-life applications of action recognition often require a 

fine-grained understanding of subtle movements, e.g., in sports analytics, user interactions in AR/VR, and surgical videos. Although fine-grained actions are more costly to annotate, existing semi-supervised action recognition has mainly focused on coarse-grained action recognition. Since fine-grained actions are more challenging due to the absence of scene bias, classifying these actions requires an understanding of action-phases. Hence, existing coarse-grained semi-supervised methods 

do not work effectively. In this work, we for the first time thoroughly investigate semi-supervised fine-grained action recognition (FGAR). We observe that alignment distances like dynamic time warping (DTW) provide a suitable action-phase-aware measure for comparing fine-grained actions, a concept previously unexploited in FGAR. However, since regular DTW distance is pairwise and assumes strict alignment between pairs, it is not directly suitable for classifying fine-grained actions. To 

utilize such alignment distances in a limited-label setting, we propose an Alignability-Verification-based Metric learning technique to effectively discriminate between fine-grained action pairs. Our learnable alignability score provides a better phase-aware measure, which we use to refine the pseudo-labels of the primary video encoder. Our collaborative pseudolabeling-based framework ‘FinePseudo’ significantly outperforms prior methods on four fine-grained action recognition datasets: Diving48, FineGym99, FineGym288, and FineDiving, and shows improvement on existing coarse-grained datasets: Kinetics400 and Something-SomethingV2. We also demonstrate the robustness of our collaborative pseudo-labeling in handling novel unlabeled classes in open-world semi-supervised setups.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2024,

title = {X-Former: Unifying Contrastive and Reconstruction Learning for MLLMs},

author = {Sirnam Swetha and Jinyu Yang and Tal Neiman and Mamshad Nayeem Rizve and Son Tran and Benjamin Yao and Trishul Chilimbi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2407.13851v1.pdf 

https://arxiv.org/abs/2407.13851 

https://swetha5.github.io/XFormer/},

doi = {https://doi.org/10.48550/arXiv.2407.13851},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent advancements in Multimodal Large Language Models (MLLMs) have revolutionized the field of vision-language understanding by integrating visual perception capabilities into Large Language Models (LLMs). The prevailing trend in this field involves the utilization of a vision encoder derived from vision-language contrastive learning (CL), showing expertise in capturing overall representations while facing difficulties in capturing detailed local patterns. In this work, we focus on enhancing the visual representations for MLLMs by combining high-frequency and detailed visual representations, obtained through masked image modeling (MIM), with semantically-enriched low-frequency representations captured by CL. To achieve this goal, we introduce X-Former which is a lightweight transformer module designed to exploit the complementary strengths of CL and MIM through an innovative interaction mechanism. Specifically, X-Former first bootstraps vision-language representation learning and multimodal-to-multimodal generative learning from two frozen vision encoders, i.e., CLIP-ViT (CL-based) and MAE-ViT (MIM-based). It further bootstraps vision-to-language generative learning from a frozen LLM to ensure visual features from X-Former can be interpreted by the LLM. To demonstrate the effectiveness of our approach, we assess its performance on tasks demanding detailed visual understanding. Extensive evaluations indicate that X-Former excels in visual reasoning tasks involving both structural and semantic categories in the GQA dataset. Assessment on fine-grained visual perception benchmark further confirms its superior capabilities in visual understanding.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2024,

title = {Open Vocabulary Multi-Label Video Classification},

author = {Rohit Gupta and Mamshad Nayeem Rizve and Ashish Tawari and Jayakrishnan Unnikrishnan and Son Tran and Mubarak Shah and Yao and Trishul Chilimbi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_CameraReady-2.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_Supplementary.pdf 

https://arxiv.org/html/2407.09073v1#S1},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

abstract = {Pre-trained vision-language models (VLMs) have enabled significant progress in open vocabulary computer vision tasks such as image classification, object detection and image segmentation. Some recent works have focused on extending VLMs to open vocabulary single label action classification in videos. However, previous methods fall short in holistic video understanding which requires the ability to simultaneously recognize multiple actions and entities e.g., objects in the video in an open vocabulary setting. We formulate this problem as open vocabulary multi-label video classification and propose a method to adapt a pre-trained VLM such as CLIP to solve this task. We leverage large language models (LLMs) to provide semantic guidance to the VLM about class labels to improve its open vocabulary performance with two key contributions. First, we propose an end-to-end trainable architecture that learns to prompt an LLM to generate soft attributes for the CLIP text-encoder to enable it to recognize novel classes. Second, we integrate a temporal modeling module into CLIP’s vision encoder to effectively model the spatio-temporal dynamics of video concepts as well as propose a novel regularized finetuning technique to ensure strong open vocabulary classification performance in the video domain. Our extensive experimentation showcases the efficacy of our approach on multiple benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024b,

title = {Sync from the Sea: Retrieving Alignable Videos from Large-Scale Datasets},

author = {Ishan Rajendrakumar Dave and Fabian Caba Heilbron and Mubarak Shah and Simon Jenni},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/avr_eccv24_dave.pdf 

https://daveishan.github.io/avr-webpage/ https://youtu.be/6euQwz7XdQk?si=v12dQH4e7UrTUrIU},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024, Oral (Top 3%)},

abstract = {Temporal video alignment aims to synchronize the key events 

like object interactions or action phase transitions in two videos. Such methods could benefit various video editing, processing, and understanding tasks. However, existing approaches operate under the restrictive assumption that a suitable video pair for alignment is given, significantly limiting their broader applicability. To address this, we re-pose temporal alignment as a search problem and introduce the task of Alignable Video Retrieval (AVR). Given a query video, our approach can identify well-alignable videos from a large collection of clips and temporally synchronize them to the query. To achieve this, we make three key contributions: 1) we introduce DRAQ, a video alignability indicator to identify and re-rank the best alignable video from a set of candidates; 2) we propose an effective and generalizable frame-level video feature design to improve the alignment performance of several off-the-shelf feature representations, and 3) we propose a novel benchmark and evaluation protocol for AVR using cycle-consistency metrics. Our experiments on 3 datasets, including large-scale Kinetics700, demonstrate the effectiveness of our approach in identifying alignable video pairs from diverse datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2024,

title = {Regulating Model Reliance on Non-Robust Features by Smoothing Input Marginal Density},

author = {Paiyu Yang and Naveed Akhtar and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/main_paper-1.pdf 

https://arxiv.org/pdf/2407.04370 

https://github.com/ypeiyu/input_density_reg},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tai2024,

title = {uCAP: An Unsupervised Prompting Method for Vision-Language Models},

author = {Kai Sheng Tai and Sirius Chen and Satya Narayan Shukla and Hanchao Yu and Philip Torr and Taipeng Tian and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses a significant limitation that prevents Contrastive Language-Image Pretrained Models (CLIP) from achieving optimal performance on downstream image classification tasks. The key problem with CLIP-style zero-shot classification is that it requires domain-specific context in the form of prompts to better align the class descriptions to the downstream data distribution. In particular, prompts for vision-language models are domain-level texts (e.g., “a centered satellite image of ...”) which, together with the class names, are fed into the text encoder to provide more context for the downstream dataset. These prompts are typically manually tuned, which is time consuming and often sub-optimal. To overcome this bottleneck, this paper proposes uCAP, a method to automatically learn domain-specific prompts/contexts using only unlabeled in-domain images. We achieve this by modeling the generation of images given the class names and a domain-specific prompt with an unsupervised likelihood distribution, and then performing inference of the prompts. We validate the proposed method across various models and datasets, showing that uCAP consistently outperforms manually tuned prompts and related baselines on the evaluated datasets: ImageNet, CIFAR-10, CIFAR-100, OxfordPets (up to 2%), SUN397 (up to 5%), and Caltech101 (up to 3%).},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024b,

title = {Fast Encoding and Decoding for Implicit Video Representation},

author = {Hao Chen and Saining Xie and Ser-Nam Lim and Abhinav Shrivastava},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Jang2024b,

title = {Spherical Linear Interpolation and Text-Anchoring for Zero-shot Composed Retrieval},

author = {Young Kyun Jang and Dat Huynh and Ashish Shah and Wen-Kai Chen and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2024c,

title = {Towards Multi-modal Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Aritra Dutta and Xin Li and Chen Chen},

url = {https://arxiv.org/pdf/2404.12467.pdf 

https://github.com/imguangyu/FedCola},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Li2024b,

title = {ControlNet++: Improving Conditional Controls with Efficient Consistency Feedback},

author = {Ming Li and Taojiannan Yang and Huafeng Kuang and Jie Wu and Zhaoning Wang and Xuefeng Xiao and Chen Chen},

url = {https://arxiv.org/pdf/2404.07987.pdf 

https://liming-ai.github.io/ControlNet_Plus_Plus/ 

https://github.com/liming-ai/ControlNet_Plus_Plus 

https://huggingface.co/spaces/limingcv/ControlNet-Plus-Plus},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {To enhance the controllability of text-to-image diffusion models, existing efforts like ControlNet incorporated image-based conditional controls. In this paper, we reveal that existing methods still face significant challenges in generating images that align with the image conditional controls. To this end, we propose ControlNet++, a novel approach that improves controllable generation by explicitly optimizing pixel-level cycle consistency between generated images and conditional controls. Specifically, for an input conditional control, we use a pre-trained discriminative reward model to extract the corresponding condition of the generated images, and then optimize the consistency loss between the input conditional control and extracted condition. A straightforward implementation would be generating images from random noises and then calculating the consistency loss, but such an approach requires storing gradients for multiple sampling timesteps, leading to considerable time and memory costs. To address this, we introduce an efficient reward strategy that deliberately disturbs the input images by adding noise, and then uses the single-step denoised images for reward fine-tuning. This avoids the extensive costs associated with image sampling, allowing for more efficient reward fine-tuning. Extensive experiments show that ControlNet++ significantly improves controllability under various conditional controls. For example, it achieves improvements over ControlNet by 7.9% mIoU, 13.4% SSIM, and 7.6% RMSE, respectively, for segmentation mask, line-art edge, and depth conditions.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_84,

title = {BAMM: Bidirectional Autoregressive Motion Model},

author = {Ekkasit Pinyoanuntapong and Muhammad Usama Saleem and Pu Wang and Minwoo Lee and Srijan Das and Chen Chen},

url = {https://arxiv.org/pdf/2403.19435.pdf 

https://exitudio.github.io/BAMM-page/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Generating human motion from text has been dominated by denoising motion models either through diffusion or generative masking process. However, these models face great limitations in usability by requiring prior knowledge of the motion length. Conversely, autoregressive motion models address this limitation by adaptively predicting motion endpoints, at the cost of degraded generation quality and editing capabilities. To address these challenges, we propose Bidirectional Autoregressive Motion Model (BAMM), a novel text-to-motion generation framework. BAMM consists of two key components: (1) a motion tokenizer that transforms 3D human motion into discrete tokens in latent space, and (2) a masked self-attention transformer that autoregressively predicts randomly masked tokens via a hybrid attention masking strategy. By unifying generative masked modeling and autoregressive modeling, BAMM captures rich and bidirectional dependencies among motion tokens, while learning the probabilistic mapping from textual inputs to motion outputs with dynamically-adjusted motion sequence length. This feature enables BAMM to simultaneously achieving high-quality motion generation with enhanced usability and built-in motion editability. Extensive experiments on HumanML3D and KIT-ML datasets demonstrate that BAMM surpasses current state-of-the-art methods in both qualitative and quantitative measures.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khalid2024,

title = {3DEgo: 3D Editing on the Go!},

author = {Umar Khalid and Hasan Iqbal and Azib Farooq and Jing Hua and Chen Chen},

url = {https://arxiv.org/pdf/2407.10102 

https://3dego.github.io/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {We introduce 3DEgo to address a novel problem of directly synthesizing photorealistic 3D scenes from monocular videos guided by textual prompts. Conventional methods construct a text-conditioned 3D scene through a three-stage process, involving pose estimation using Structure-from-Motion (SfM) libraries like COLMAP, initializing the 3D model with unedited images, and iteratively updating the dataset with edited images to achieve a 3D scene with text fidelity. Our framework streamlines the conventional multi-stage 3D editing process into a single-stage workflow by overcoming the reliance on COLMAP and eliminating the cost of model initialization. We apply a diffusion model to edit video frames prior to 3D scene creation by incorporating our designed noise blender module for enhancing multi-view editing consistency, a step that does not require additional training or fine-tuning of T2I diffusion models. 3DEgo utilizes 3D Gaussian Splatting to create 3D scenes from the multi-view consistent edited frames, capitalizing on the inherent temporal continuity and explicit point cloud data. 3DEgo demonstrates remarkable editing precision, speed, and adaptability across a variety of video sources, as validated by extensive evaluations on six datasets, including our own prepared GS25 dataset.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khalid2024b,

title = {LatentEditor: Text Driven Local Editing of 3D Scenes},

author = {Umar Khalid and Hasan Iqbal and Muhammad Tayyab and Md Nazmul Karim and Jing Hua and Chen Chen},

url = {https://arxiv.org/pdf/2312.09313.pdf 

https://latenteditor.github.io/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {While neural fields have made significant strides in view synthesis and scene reconstruction, editing them poses a formidable challenge due to their implicit encoding of geometry and texture information from multi-view inputs. In this paper, we introduce LatenEditor, an innovative framework designed to empower users with the ability to perform precise and locally controlled editing of neural fields using text prompts. Leveraging denoising diffusion models, we successfully embed real-world scenes into the latent space, resulting in a faster and more adaptable NeRF backbone for editing compared to traditional methods. To enhance editing precision, we introduce a delta score to calculate the 2D mask in the latent space that serves as a guide for local modifications while preserving irrelevant regions. Our novel pixel-level scoring approach harnesses the power of InstructPix2Pix (IP2P) to discern the disparity between IP2P conditional and unconditional noise predictions in the latent space. The edited latents conditioned on the 2D masks are then iteratively updated in the training set to achieve 3D local editing. Our approach achieves faster editing speeds and superior output quality compared to existing 3D editing models, bridging the gap between textual instructions and high-quality 3D scene editing in latent space.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Karim2024,

title = {Free-Editor: Zero-shot Text-driven 3D Scene Editing},

author = {Md Nazmul Karim and Hasan Iqbal and Umar Khalid and Chen Chen and Jing Hua},

url = {https://arxiv.org/pdf/2312.13663.pdf 

https://free-editor.github.io/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Text-to-Image (T2I) diffusion models have gained popularity recently due to their multipurpose and easy-to-use nature, e.g. image and video generation as well as editing. However, training a diffusion model specifically for 3D scene editing is not straightforward due to the lack of large-scale datasets. To date, editing 3D scenes requires either re-training the model to adapt to various 3D edited scenes or design-specific methods for each special editing type. Furthermore, state-of-the-art (SOTA) methods require multiple synchronized edited images from the same scene to facilitate the scene editing. Due to the current limitations of T2I models, it is very challenging to apply consis- tent editing effects to multiple images, i.e. multi-view inconsistency in editing. This in turn compromises the desired 3D scene editing performance if these images are used. In our work, we propose a novel training-free 3D scene editing technique, FREE-EDITOR, which allows users to edit 3D scenes without further re-training the model during test time. Our proposed method successfully avoids the multi- view style inconsistency issue in SOTA methods with the help of a “single-view editing” scheme. Specifically, we show that editing a particular 3D scene can be performed by only modifying a single view. To this end, we introduce an Edit Transformer that enforces intra-view consistency and inter-view style transfer by utilizing self- and cross- attention, respectively. Since it is no longer required to re-train the model and edit every view in a scene, the editing time, as well as memory resources, are reduced significantly, e.g., the runtime being ∼ 20× faster than SOTA. We have conducted extensive experiments on a wide range of benchmark datasets and achieve diverse editing capabilities with our proposed technique.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fang2024,

title = {Rethinking Weakly-supervised Video Temporal Grounding From a Game Perspective},

author = {Xiang Fang and Zeyu Xiong and Wanlong Fang and Xiaoye Qu and Chen Chen and Jianfeng Dong and Keke Tang and Pan Zhou and Yu Cheng and Daizong Liu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ECCV2024_Grounding_camera.pdf 

https://eccv2024.ecva.net/virtual/2024/poster/1833},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses the challenging task of weakly-supervised video temporal grounding. Existing approaches are generally based on the moment candidate selection pipeline that utilizes contrastive learning and reconstruction paradigm for scoring the pre-defined moments. Although they have achieved significant progress, we argue that their current frameworks have overlooked two indispensable issues: (1) Coarse-grained cross-modal learning: previous methods solely capture the global video-level alignment with the query, failing to model the detailed consistency between video frames and query words for accurately grounding the moment boundaries. (2) Complex moment candidates: the performance of these methods severely relies on the quality of moment candidates, which are also time-consuming and complicated for selection. To this end, in this paper, we make the first attempt to tackle this task from a novel game perspective, which effectively learns the uncertain relationship between each frame-word pair with diverse granularity and flexible combination for fine-grained cross-modal interaction. Specifically, we creatively model each video frame and query word as game players with multivariate cooperative game theory to learn their contribution to the cross-modal similarity score. By quantifying the trend of frame-word cooperation within a coalition via the game-theoretic interaction, we are able to value all uncertain but possible correspondence between frames and words. At last, instead of using moment proposals, we utilize the learned query-guided frame-wise scores for fine-grained moment boundary grounding. Experiments show that our method achieves superior performance on both Charades-STA and ActivityNet Caption datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2024,

title = {GAReT: Cross-view Video Geolocalization with Adapters and Auto-Regressive Transformers},

author = {Manu S Pillai and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07875-supp.pdf 

https://arxiv.org/abs/2408.02840 

https://github.com/manupillai308/GAReT},

doi = {https://doi.org/10.48550/arXiv.2408.02840},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Cross-view video geo-localization (CVGL) aims to derive GPS trajectories from street-view videos by aligning them with aerial-view images. Despite their promising performance, current CVGL methods face significant challenges. These methods use camera and odometry data, typically absent in real-world scenarios. They utilize multiple adjacent frames and various encoders for feature extraction, resulting in high computational costs. Moreover, these approaches independently predict each street-view frame's location, resulting in temporally inconsistent GPS trajectories. To address these challenges, in this work, we propose GAReT, a fully transformer-based method for CVGL that does not require camera and odometry data. We introduce GeoAdapter, a transformer-adapter module designed to efficiently aggregate image-level representations and adapt them for video inputs. Specifically, we train a transformer encoder on video frames and aerial images, then freeze the encoder to optimize the GeoAdapter module to obtain video-level representation. To address temporally inconsistent trajectories, we introduce TransRetriever, an encoder-decoder transformer model that predicts GPS locations of street-view frames by encoding top-k nearest neighbor predictions per frame and auto-regressively decoding the best neighbor based on the previous frame's predictions. Our method's effectiveness is validated through extensive experiments, demonstrating state-of-the-art performance on benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2024,

title = {CityGuessr: City-Level Video Geo-Localization on a Global Scale},

author = {Parth Parag Kulkarni and Guarav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CityGuessr.pdf 

https://parthpk.github.io/cityguessr-webpage/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Video geolocalization is a crucial problem in current times. Given just a video, ascertaining where it was captured from can have a plethora of advantages. The problem of worldwide geolocalization has been tackled before, but only using the image modality. Its video counterpart remains relatively unexplored. Meanwhile, video geolocalization has also garnered some attention in the recent past, but the existing methods are all restricted to specific regions. This motivates us to explore the problem of video geolocalization at a global scale. Hence, we propose a novel problem of worldwide video geolocalization with the objective of hierarchically predicting the correct city, state/province, country, and continent, given a video. However, no large scale video datasets that have extensive worldwide coverage exist, to train models for solving this problem. To this end, we introduce a new dataset, “CityGuessr68k” comprising of 68,269 videos from 166 cities all over the world. We also propose a novel baseline approach to this problem, by designing a transformerbased architecture comprising of an elegant “Self-Cross Attention” module for incorporating scenes as well as a “TextLabel Alignment” strategy for distilling knowledge from textlabels in feature space. To further enhance our location prediction, we also utilize soft-scene labels. Finally we demonstrate the performance of our method on our new dataset as well as Mapillary(MSLS) [38].},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_85,

title = {Augmented Neural Fine-tuning for Efficient Backdoor Purification},

author = {Nazmul Karim and Abdullah Al Arafat and Umar Khalid and Zhishan Guo and Nazanin Rahnavard},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2024_ECCV_NFT.pdf 

https://arxiv.org/pdf/2407.10052 

https://github.com/nazmul-karim170/NFT-Augmented-Backdoor-Purification},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent studies have revealed the vulnerability of deep neural networks (DNNs) to various backdoor attacks, where the behavior of DNNs can be compromised by utilizing certain types of triggers or poisoning mechanisms. State-of-the-art (SOTA) defenses employ too sophisticated mechanisms that require either a computationally expensive adversarial search module for reverse-engineering the trigger distribution or an over-sensitive hyper-parameter selection module. Moreover, they offer sub-par performance in challenging scenarios, e.g., limited validation data and strong attacks. In this paper, we propose—Neural mask Fine-Tuning (NFT)—with an aim to optimally re-organize the neuron activities in a way that the effect of the backdoor is removed. Utilizing a simple data augmentation like MixUp, NFT relaxes the trigger synthesis process and eliminates the requirement of the adversarial search module. Our study further reveals that direct weight fine-tuning under limited validation data results in poor post-purification clean test accuracy, primarily due to overfitting issue. To overcome this, we propose to fine-tune neural masks instead of model weights. In addition, a mask regularizer 

has been devised to further mitigate the model drift during the purification process. The distinct characteristics of NFT render it highly efficient in both runtime and sample usage, as it can remove the backdoor even when a single sample is available from each class. We validate the effectiveness of NFT through extensive experiments covering the tasks of image classification, object detection, video action recognition, 3D point cloud, and natural language processing. We evaluate our method against 14 different attacks (LIRA, WaNet, etc.) on 11 benchmark data sets (ImageNet, UCF101, Pascal VOC, ModelNet, OpenSubtitles2012, etc.).},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_130,

title = {Möbius Transform for Mitigating Perspective Distortions in Representation Learning},

author = {Prakash Chandra Chhipa and Meenakshi Subhash Chippa and Kanjar De and Rajkumar Saini and Marcus Liwicki and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_ECCV2024_CameraReady.pdf 

https://prakashchhipa.github.io/projects/mpd 

https://youtu.be/MKh9NE_XEMY 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_presentation_ECCV2024_final.pdf},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Perspective distortion (PD) causes unprecedented changes in shape, size, orientation, angles, and other spatial relationships of visual concepts in images. Precisely estimating camera intrinsic and extrinsic parameters is a challenging task that prevents synthesizing perspective distortion. Non-availability of dedicated training data poses a critical barrier to developing robust computer vision methods. Additionally, distortion correction methods make other computer vision tasks a multi-step approach and lack performance. In this work, we propose 

mitigating perspective distortion (MPD) by employing a fine-grained parameter control on a specific family of Möbius transform to model real-world distortion without estimating camera intrinsic and extrinsic parameters and without the need for actual distorted data. Also, we present a dedicated perspectively distorted benchmark dataset, ImageNet-PD, to benchmark the robustness of deep learning models against this new dataset. The proposed method outperforms existing benchmarks, ImageNet-E and ImageNet-X. Additionally, it significantly 

improves performance on ImageNet-PD while consistently performing on standard data distribution. Notably, our method shows improved performance on three PD-affected real-world applications—crowd counting, fisheye image recognition, and person re-identification—and one PD-affected challenging CV task: object detection. The source code, dataset, and models are available on the project webpage at https://prakashchhipa.github.io/projects/mpd.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kang2024b,

title = {SegVG: Transferring Object Bounding Box to Segmentation for Visual Grounding},

author = {Weitai Kang and Gaowen Liu and Mubarak Shah and Yan Yan},

url = {https://arxiv.org/pdf/2407.03200},

doi = {https://doi.org/10.48550/arXiv.2407.03200},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Different from Object Detection, Visual Grounding deals with detecting a bounding box for each text-image pair. This one box for each text-image data provides sparse supervision signals. Although previous works achieve impressive results, their passive utilization of annotation, i.e. the sole use of the box annotation as regression ground truth, results in a suboptimal performance. In this paper, we present SegVG, a novel method transfers the box-level annotation as Segmentation signals to provide an additional pixel-level supervision for Visual Grounding. Specifically, we propose the Multi-layer Multi-task Encoder-Decoder as the target grounding stage, where we learn a regression query and multiple segmentation queries to ground the target by regression and segmentation of the box in each decoding layer, respectively. This approach allows us to iteratively exploit the annotation as signals for both box-level regression and pixel-level segmentation. Moreover, as the backbones are typically initialized by pretrained parameters learned from unimodal tasks and the queries for both regression and segmentation are static learnable embeddings, a domain discrepancy remains among these three types of features, which impairs subsequent target grounding. To mitigate this discrepancy, we introduce the Triple Alignment module, where the query, text, and vision tokens are triangularly updated to share the same space by triple attention mechanism. Extensive experiments on five widely used datasets validate our state-of-the-art (SOTA) performance.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024f,

title = {FinePseudo: Improving Pseudo-Labelling through Temporal-Alignablity for Semi-Supervised Fine-Grained Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/finepsuedo_eccv24_dave.pdf 

https://daveishan.github.io/finepsuedo-webpage/ https://youtu.be/bWOd8_JpjQs?si=WWRDhdg5ADWL0uwB},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Real-life applications of action recognition often require a 

fine-grained understanding of subtle movements, e.g., in sports analytics, user interactions in AR/VR, and surgical videos. Although fine-grained actions are more costly to annotate, existing semi-supervised action recognition has mainly focused on coarse-grained action recognition. Since fine-grained actions are more challenging due to the absence of scene bias, classifying these actions requires an understanding of action-phases. Hence, existing coarse-grained semi-supervised methods 

do not work effectively. In this work, we for the first time thoroughly investigate semi-supervised fine-grained action recognition (FGAR). We observe that alignment distances like dynamic time warping (DTW) provide a suitable action-phase-aware measure for comparing fine-grained actions, a concept previously unexploited in FGAR. However, since regular DTW distance is pairwise and assumes strict alignment between pairs, it is not directly suitable for classifying fine-grained actions. To 

utilize such alignment distances in a limited-label setting, we propose an Alignability-Verification-based Metric learning technique to effectively discriminate between fine-grained action pairs. Our learnable alignability score provides a better phase-aware measure, which we use to refine the pseudo-labels of the primary video encoder. Our collaborative pseudolabeling-based framework ‘FinePseudo’ significantly outperforms prior methods on four fine-grained action recognition datasets: Diving48, FineGym99, FineGym288, and FineDiving, and shows improvement on existing coarse-grained datasets: Kinetics400 and Something-SomethingV2. We also demonstrate the robustness of our collaborative pseudo-labeling in handling novel unlabeled classes in open-world semi-supervised setups.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2024b,

title = {X-Former: Unifying Contrastive and Reconstruction Learning for MLLMs},

author = {Sirnam Swetha and Jinyu Yang and Tal Neiman and Mamshad Nayeem Rizve and Son Tran and Benjamin Yao and Trishul Chilimbi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2407.13851v1.pdf 

https://arxiv.org/abs/2407.13851 

https://swetha5.github.io/XFormer/},

doi = {https://doi.org/10.48550/arXiv.2407.13851},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent advancements in Multimodal Large Language Models (MLLMs) have revolutionized the field of vision-language understanding by integrating visual perception capabilities into Large Language Models (LLMs). The prevailing trend in this field involves the utilization of a vision encoder derived from vision-language contrastive learning (CL), showing expertise in capturing overall representations while facing difficulties in capturing detailed local patterns. In this work, we focus on enhancing the visual representations for MLLMs by combining high-frequency and detailed visual representations, obtained through masked image modeling (MIM), with semantically-enriched low-frequency representations captured by CL. To achieve this goal, we introduce X-Former which is a lightweight transformer module designed to exploit the complementary strengths of CL and MIM through an innovative interaction mechanism. Specifically, X-Former first bootstraps vision-language representation learning and multimodal-to-multimodal generative learning from two frozen vision encoders, i.e., CLIP-ViT (CL-based) and MAE-ViT (MIM-based). It further bootstraps vision-to-language generative learning from a frozen LLM to ensure visual features from X-Former can be interpreted by the LLM. To demonstrate the effectiveness of our approach, we assess its performance on tasks demanding detailed visual understanding. Extensive evaluations indicate that X-Former excels in visual reasoning tasks involving both structural and semantic categories in the GQA dataset. Assessment on fine-grained visual perception benchmark further confirms its superior capabilities in visual understanding.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2024b,

title = {Open Vocabulary Multi-Label Video Classification},

author = {Rohit Gupta and Mamshad Nayeem Rizve and Ashish Tawari and Jayakrishnan Unnikrishnan and Son Tran and Mubarak Shah and Yao and Trishul Chilimbi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_CameraReady-2.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_Supplementary.pdf 

https://arxiv.org/html/2407.09073v1#S1},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

abstract = {Pre-trained vision-language models (VLMs) have enabled significant progress in open vocabulary computer vision tasks such as image classification, object detection and image segmentation. Some recent works have focused on extending VLMs to open vocabulary single label action classification in videos. However, previous methods fall short in holistic video understanding which requires the ability to simultaneously recognize multiple actions and entities e.g., objects in the video in an open vocabulary setting. We formulate this problem as open vocabulary multi-label video classification and propose a method to adapt a pre-trained VLM such as CLIP to solve this task. We leverage large language models (LLMs) to provide semantic guidance to the VLM about class labels to improve its open vocabulary performance with two key contributions. First, we propose an end-to-end trainable architecture that learns to prompt an LLM to generate soft attributes for the CLIP text-encoder to enable it to recognize novel classes. Second, we integrate a temporal modeling module into CLIP’s vision encoder to effectively model the spatio-temporal dynamics of video concepts as well as propose a novel regularized finetuning technique to ensure strong open vocabulary classification performance in the video domain. Our extensive experimentation showcases the efficacy of our approach on multiple benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024bb,

title = {Sync from the Sea: Retrieving Alignable Videos from Large-Scale Datasets},

author = {Ishan Rajendrakumar Dave and Fabian Caba Heilbron and Mubarak Shah and Simon Jenni},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/avr_eccv24_dave.pdf 

https://daveishan.github.io/avr-webpage/ https://youtu.be/6euQwz7XdQk?si=v12dQH4e7UrTUrIU},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024, Oral (Top 3%)},

abstract = {Temporal video alignment aims to synchronize the key events 

like object interactions or action phase transitions in two videos. Such methods could benefit various video editing, processing, and understanding tasks. However, existing approaches operate under the restrictive assumption that a suitable video pair for alignment is given, significantly limiting their broader applicability. To address this, we re-pose temporal alignment as a search problem and introduce the task of Alignable Video Retrieval (AVR). Given a query video, our approach can identify well-alignable videos from a large collection of clips and temporally synchronize them to the query. To achieve this, we make three key contributions: 1) we introduce DRAQ, a video alignability indicator to identify and re-rank the best alignable video from a set of candidates; 2) we propose an effective and generalizable frame-level video feature design to improve the alignment performance of several off-the-shelf feature representations, and 3) we propose a novel benchmark and evaluation protocol for AVR using cycle-consistency metrics. Our experiments on 3 datasets, including large-scale Kinetics700, demonstrate the effectiveness of our approach in identifying alignable video pairs from diverse datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2024b,

title = {Regulating Model Reliance on Non-Robust Features by Smoothing Input Marginal Density},

author = {Paiyu Yang and Naveed Akhtar and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/main_paper-1.pdf 

https://arxiv.org/pdf/2407.04370 

https://github.com/ypeiyu/input_density_reg},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tai2024b,

title = {uCAP: An Unsupervised Prompting Method for Vision-Language Models},

author = {Kai Sheng Tai and Sirius Chen and Satya Narayan Shukla and Hanchao Yu and Philip Torr and Taipeng Tian and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses a significant limitation that prevents Contrastive Language-Image Pretrained Models (CLIP) from achieving optimal performance on downstream image classification tasks. The key problem with CLIP-style zero-shot classification is that it requires domain-specific context in the form of prompts to better align the class descriptions to the downstream data distribution. In particular, prompts for vision-language models are domain-level texts (e.g., “a centered satellite image of ...”) which, together with the class names, are fed into the text encoder to provide more context for the downstream dataset. These prompts are typically manually tuned, which is time consuming and often sub-optimal. To overcome this bottleneck, this paper proposes uCAP, a method to automatically learn domain-specific prompts/contexts using only unlabeled in-domain images. We achieve this by modeling the generation of images given the class names and a domain-specific prompt with an unsupervised likelihood distribution, and then performing inference of the prompts. We validate the proposed method across various models and datasets, showing that uCAP consistently outperforms manually tuned prompts and related baselines on the evaluated datasets: ImageNet, CIFAR-10, CIFAR-100, OxfordPets (up to 2%), SUN397 (up to 5%), and Caltech101 (up to 3%).},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024bb,

title = {Fast Encoding and Decoding for Implicit Video Representation},

author = {Hao Chen and Saining Xie and Ser-Nam Lim and Abhinav Shrivastava},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Jang2024c,

title = {Spherical Linear Interpolation and Text-Anchoring for Zero-shot Composed Retrieval},

author = {Young Kyun Jang and Dat Huynh and Ashish Shah and Wen-Kai Chen and Ser-Nam Lim},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2024e,

title = {Towards Multi-modal Transformers in Federated Learning},

author = {Guangyu Sun and Matias Mendieta and Aritra Dutta and Xin Li and Chen Chen},

url = {https://arxiv.org/pdf/2404.12467.pdf 

https://github.com/imguangyu/FedCola},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Li2024c,

title = {ControlNet++: Improving Conditional Controls with Efficient Consistency Feedback},

author = {Ming Li and Taojiannan Yang and Huafeng Kuang and Jie Wu and Zhaoning Wang and Xuefeng Xiao and Chen Chen},

url = {https://arxiv.org/pdf/2404.07987.pdf 

https://liming-ai.github.io/ControlNet_Plus_Plus/ 

https://github.com/liming-ai/ControlNet_Plus_Plus 

https://huggingface.co/spaces/limingcv/ControlNet-Plus-Plus},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {To enhance the controllability of text-to-image diffusion models, existing efforts like ControlNet incorporated image-based conditional controls. In this paper, we reveal that existing methods still face significant challenges in generating images that align with the image conditional controls. To this end, we propose ControlNet++, a novel approach that improves controllable generation by explicitly optimizing pixel-level cycle consistency between generated images and conditional controls. Specifically, for an input conditional control, we use a pre-trained discriminative reward model to extract the corresponding condition of the generated images, and then optimize the consistency loss between the input conditional control and extracted condition. A straightforward implementation would be generating images from random noises and then calculating the consistency loss, but such an approach requires storing gradients for multiple sampling timesteps, leading to considerable time and memory costs. To address this, we introduce an efficient reward strategy that deliberately disturbs the input images by adding noise, and then uses the single-step denoised images for reward fine-tuning. This avoids the extensive costs associated with image sampling, allowing for more efficient reward fine-tuning. Extensive experiments show that ControlNet++ significantly improves controllability under various conditional controls. For example, it achieves improvements over ControlNet by 7.9% mIoU, 13.4% SSIM, and 7.6% RMSE, respectively, for segmentation mask, line-art edge, and depth conditions.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_131,

title = {BAMM: Bidirectional Autoregressive Motion Model},

author = {Ekkasit Pinyoanuntapong and Muhammad Usama Saleem and Pu Wang and Minwoo Lee and Srijan Das and Chen Chen},

url = {https://arxiv.org/pdf/2403.19435.pdf 

https://exitudio.github.io/BAMM-page/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Generating human motion from text has been dominated by denoising motion models either through diffusion or generative masking process. However, these models face great limitations in usability by requiring prior knowledge of the motion length. Conversely, autoregressive motion models address this limitation by adaptively predicting motion endpoints, at the cost of degraded generation quality and editing capabilities. To address these challenges, we propose Bidirectional Autoregressive Motion Model (BAMM), a novel text-to-motion generation framework. BAMM consists of two key components: (1) a motion tokenizer that transforms 3D human motion into discrete tokens in latent space, and (2) a masked self-attention transformer that autoregressively predicts randomly masked tokens via a hybrid attention masking strategy. By unifying generative masked modeling and autoregressive modeling, BAMM captures rich and bidirectional dependencies among motion tokens, while learning the probabilistic mapping from textual inputs to motion outputs with dynamically-adjusted motion sequence length. This feature enables BAMM to simultaneously achieving high-quality motion generation with enhanced usability and built-in motion editability. Extensive experiments on HumanML3D and KIT-ML datasets demonstrate that BAMM surpasses current state-of-the-art methods in both qualitative and quantitative measures.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khalid2024bb,

title = {LatentEditor: Text Driven Local Editing of 3D Scenes},

author = {Umar Khalid and Hasan Iqbal and Muhammad Tayyab and Md Nazmul Karim and Jing Hua and Chen Chen},

url = {https://arxiv.org/pdf/2312.09313.pdf 

https://latenteditor.github.io/},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {While neural fields have made significant strides in view synthesis and scene reconstruction, editing them poses a formidable challenge due to their implicit encoding of geometry and texture information from multi-view inputs. In this paper, we introduce LatenEditor, an innovative framework designed to empower users with the ability to perform precise and locally controlled editing of neural fields using text prompts. Leveraging denoising diffusion models, we successfully embed real-world scenes into the latent space, resulting in a faster and more adaptable NeRF backbone for editing compared to traditional methods. To enhance editing precision, we introduce a delta score to calculate the 2D mask in the latent space that serves as a guide for local modifications while preserving irrelevant regions. Our novel pixel-level scoring approach harnesses the power of InstructPix2Pix (IP2P) to discern the disparity between IP2P conditional and unconditional noise predictions in the latent space. The edited latents conditioned on the 2D masks are then iteratively updated in the training set to achieve 3D local editing. Our approach achieves faster editing speeds and superior output quality compared to existing 3D editing models, bridging the gap between textual instructions and high-quality 3D scene editing in latent space.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Karim2024b,

title = {Free-Editor: Zero-shot Text-driven 3D Scene Editing},

author = {Md Nazmul Karim and Hasan Iqbal and Umar Khalid and Chen Chen and Jing Hua},

url = {https://arxiv.org/pdf/2312.13663.pdf 

https://free-editor.github.io/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Text-to-Image (T2I) diffusion models have gained popularity recently due to their multipurpose and easy-to-use nature, e.g. image and video generation as well as editing. However, training a diffusion model specifically for 3D scene editing is not straightforward due to the lack of large-scale datasets. To date, editing 3D scenes requires either re-training the model to adapt to various 3D edited scenes or design-specific methods for each special editing type. Furthermore, state-of-the-art (SOTA) methods require multiple synchronized edited images from the same scene to facilitate the scene editing. Due to the current limitations of T2I models, it is very challenging to apply consis- tent editing effects to multiple images, i.e. multi-view inconsistency in editing. This in turn compromises the desired 3D scene editing performance if these images are used. In our work, we propose a novel training-free 3D scene editing technique, FREE-EDITOR, which allows users to edit 3D scenes without further re-training the model during test time. Our proposed method successfully avoids the multi- view style inconsistency issue in SOTA methods with the help of a “single-view editing” scheme. Specifically, we show that editing a particular 3D scene can be performed by only modifying a single view. To this end, we introduce an Edit Transformer that enforces intra-view consistency and inter-view style transfer by utilizing self- and cross- attention, respectively. Since it is no longer required to re-train the model and edit every view in a scene, the editing time, as well as memory resources, are reduced significantly, e.g., the runtime being ∼ 20× faster than SOTA. We have conducted extensive experiments on a wide range of benchmark datasets and achieve diverse editing capabilities with our proposed technique.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fang2024b,

title = {Rethinking Weakly-supervised Video Temporal Grounding From a Game Perspective},

author = {Xiang Fang and Zeyu Xiong and Wanlong Fang and Xiaoye Qu and Chen Chen and Jianfeng Dong and Keke Tang and Pan Zhou and Yu Cheng and Daizong Liu},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/ECCV2024_Grounding_camera.pdf 

https://eccv2024.ecva.net/virtual/2024/poster/1833},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {This paper addresses the challenging task of weakly-supervised video temporal grounding. Existing approaches are generally based on the moment candidate selection pipeline that utilizes contrastive learning and reconstruction paradigm for scoring the pre-defined moments. Although they have achieved significant progress, we argue that their current frameworks have overlooked two indispensable issues: (1) Coarse-grained cross-modal learning: previous methods solely capture the global video-level alignment with the query, failing to model the detailed consistency between video frames and query words for accurately grounding the moment boundaries. (2) Complex moment candidates: the performance of these methods severely relies on the quality of moment candidates, which are also time-consuming and complicated for selection. To this end, in this paper, we make the first attempt to tackle this task from a novel game perspective, which effectively learns the uncertain relationship between each frame-word pair with diverse granularity and flexible combination for fine-grained cross-modal interaction. Specifically, we creatively model each video frame and query word as game players with multivariate cooperative game theory to learn their contribution to the cross-modal similarity score. By quantifying the trend of frame-word cooperation within a coalition via the game-theoretic interaction, we are able to value all uncertain but possible correspondence between frames and words. At last, instead of using moment proposals, we utilize the learned query-guided frame-wise scores for fine-grained moment boundary grounding. Experiments show that our method achieves superior performance on both Charades-STA and ActivityNet Caption datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2024b,

title = {GAReT: Cross-view Video Geolocalization with Adapters and Auto-Regressive Transformers},

author = {Manu S Pillai and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07875-supp.pdf 

https://arxiv.org/abs/2408.02840 

https://github.com/manupillai308/GAReT},

doi = {https://doi.org/10.48550/arXiv.2408.02840},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Cross-view video geo-localization (CVGL) aims to derive GPS trajectories from street-view videos by aligning them with aerial-view images. Despite their promising performance, current CVGL methods face significant challenges. These methods use camera and odometry data, typically absent in real-world scenarios. They utilize multiple adjacent frames and various encoders for feature extraction, resulting in high computational costs. Moreover, these approaches independently predict each street-view frame's location, resulting in temporally inconsistent GPS trajectories. To address these challenges, in this work, we propose GAReT, a fully transformer-based method for CVGL that does not require camera and odometry data. We introduce GeoAdapter, a transformer-adapter module designed to efficiently aggregate image-level representations and adapt them for video inputs. Specifically, we train a transformer encoder on video frames and aerial images, then freeze the encoder to optimize the GeoAdapter module to obtain video-level representation. To address temporally inconsistent trajectories, we introduce TransRetriever, an encoder-decoder transformer model that predicts GPS locations of street-view frames by encoding top-k nearest neighbor predictions per frame and auto-regressively decoding the best neighbor based on the previous frame's predictions. Our method's effectiveness is validated through extensive experiments, demonstrating state-of-the-art performance on benchmark datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2024b,

title = {CityGuessr: City-Level Video Geo-Localization on a Global Scale},

author = {Parth Parag Kulkarni and Guarav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CityGuessr.pdf 

https://parthpk.github.io/cityguessr-webpage/},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Video geolocalization is a crucial problem in current times. Given just a video, ascertaining where it was captured from can have a plethora of advantages. The problem of worldwide geolocalization has been tackled before, but only using the image modality. Its video counterpart remains relatively unexplored. Meanwhile, video geolocalization has also garnered some attention in the recent past, but the existing methods are all restricted to specific regions. This motivates us to explore the problem of video geolocalization at a global scale. Hence, we propose a novel problem of worldwide video geolocalization with the objective of hierarchically predicting the correct city, state/province, country, and continent, given a video. However, no large scale video datasets that have extensive worldwide coverage exist, to train models for solving this problem. To this end, we introduce a new dataset, “CityGuessr68k” comprising of 68,269 videos from 166 cities all over the world. We also propose a novel baseline approach to this problem, by designing a transformerbased architecture comprising of an elegant “Self-Cross Attention” module for incorporating scenes as well as a “TextLabel Alignment” strategy for distilling knowledge from textlabels in feature space. To further enhance our location prediction, we also utilize soft-scene labels. Finally we demonstrate the performance of our method on our new dataset as well as Mapillary(MSLS) [38].},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey_132,

title = {Augmented Neural Fine-tuning for Efficient Backdoor Purification},

author = {Nazmul Karim and Abdullah Al Arafat and Umar Khalid and Zhishan Guo and Nazanin Rahnavard},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2024_ECCV_NFT.pdf 

https://arxiv.org/pdf/2407.10052 

https://github.com/nazmul-karim170/NFT-Augmented-Backdoor-Purification},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent studies have revealed the vulnerability of deep neural networks (DNNs) to various backdoor attacks, where the behavior of DNNs can be compromised by utilizing certain types of triggers or poisoning mechanisms. State-of-the-art (SOTA) defenses employ too sophisticated mechanisms that require either a computationally expensive adversarial search module for reverse-engineering the trigger distribution or an over-sensitive hyper-parameter selection module. Moreover, they offer sub-par performance in challenging scenarios, e.g., limited validation data and strong attacks. In this paper, we propose—Neural mask Fine-Tuning (NFT)—with an aim to optimally re-organize the neuron activities in a way that the effect of the backdoor is removed. Utilizing a simple data augmentation like MixUp, NFT relaxes the trigger synthesis process and eliminates the requirement of the adversarial search module. Our study further reveals that direct weight fine-tuning under limited validation data results in poor post-purification clean test accuracy, primarily due to overfitting issue. To overcome this, we propose to fine-tune neural masks instead of model weights. In addition, a mask regularizer 

has been devised to further mitigate the model drift during the purification process. The distinct characteristics of NFT render it highly efficient in both runtime and sample usage, as it can remove the backdoor even when a single sample is available from each class. We validate the effectiveness of NFT through extensive experiments covering the tasks of image classification, object detection, video action recognition, 3D point cloud, and natural language processing. We evaluate our method against 14 different attacks (LIRA, WaNet, etc.) on 11 benchmark data sets (ImageNet, UCF101, Pascal VOC, ModelNet, OpenSubtitles2012, etc.).},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2022,

title = {GAMa: Cross-view Video Geo-localization},

author = {Shruti Vyas and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512-supp.pdf 

https://youtu.be/KSHuer_VXJo},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {The existing work in cross-view geo-localization is based on images where a ground panorama is matched to an aerial image. In this work, we focus on ground videos instead of images which provides ad-ditional contextual cues which are important for this task. There are no existing datasets for this problem, therefore we propose GAMa dataset, a large-scale dataset with ground videos and corresponding aerial images. We also propose a novel approach to solve this problem. At clip-level, a short video clip is matched with corresponding aerial image and is later used to get video-level geo-localization of a long video. Moreover, we propose a hierarchical approach to further improve the clip-level geo-localization. On this challenging dataset, with unaligned images and lim-ited field of view, our proposed method achieves a Top-1 recall rate of 19.4% and 45.1% @1.0mile. Code & dataset are available at this link.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2022,

title = {Med-DANet: Dynamic Architecture Network for Efficient Medical Volumetric Segmentation},

author = {Wenxuan Wang and Chen Chen and Jing Wang and Sen Zha and Yan Zhang and Jiangyun Li},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2206.06575.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {For 3D medical image (e.g. CT and MRI) segmentation, the difficulty of segmenting each slice in a clinical case varies greatly. Previous research on volumetric medical image segmentation in a slice-byslice manner conventionally use the identical 2D deep neural network to segment all the slices of the same case, ignoring the data heterogeneity among image slices. In this paper, we focus on multi-modal 3D MRI brain tumor segmentation and propose a dynamic architecture network named Med-DANet based on adaptive model selection to achieve effective accuracy and efficiency trade-off. For each slice of the input 3D MRI volume, our proposed method learns a slice-specific decision by the Decision Network to dynamically select a suitable model from the predefined Model Bank for the subsequent 2D segmentation task. Extensive experimental results on both BraTS 2019 and 2020 datasets show that our proposed method achieves comparable or better results than previous state-of-the art methods for 3D MRI brain tumor segmentation with much less model complexity. Compared with the state-of-the-art 3D method TransBTS, the proposed framework improves the model efficiency by up to 3.5 × without sacrificing the accuracy. Our code will be publicly available at https://github.com/Wenxuan-1119/Med-DANet.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2022,

title = {GALA: Toward Geometry-and-Lighting-Aware Object Search for Compositing},

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.00125.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GALA_supplementary.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Compositing-aware object search aims to find the most compatible objects for compositing given a background image and a query bounding box. Previous works focus on learning compatibility between the foreground object and background, but fail to learn other important factors from large-scale data, i.e. geometry and lighting. To move a step further, this paper proposes GALA (Geometry-and-Lighting-Aware), a generic foreground object search method with discriminative modeling on geometry and lighting compatibility for open-world image compositing. Remarkably, it achieves state-of-the-art results on the CAIS dataset and generalizes well on large-scale open-world datasets, i.e. Pixabay and 

Open Images. In addition, our method can effectively handle non-box scenarios, where users only provide background images without any input bounding box. A web demo (see supplementary materials) is built to showcase applications of the proposed method for compositing-aware search and automatic location/scale prediction for the foreground object.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2022,

title = {Weakly Supervised Grounding for VQA in Vision-Language Transformers},

author = {Aisha Urooj Khan and Hilde Kuehne and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011-supp.pdf 

https://github.com/aurooj/WSG-VQA-VLTransformers 

https://youtu.be/dekmVb6lq3I},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Transformers for visual-language representation learning have been getting a lot of interest and shown tremendous performance on visual question answering (VQA) and grounding. However, most systems that show good performance of those tasks still rely on pre-trained object 

detectors during training, which limits their applicability to the object classes available for those detectors. To mitigate this limitation, this paper 

focuses on the problem of weakly supervised grounding in the context of visual question answering in transformers. Our approach leverages 

capsules by transforming each visual token into a capsule representation in the visual encoder; it then uses activations from language self-attention layers as a text-guided selection module to mask those capsules before they are forwarded to the next layer. We evaluate our approach on the challenging GQA as well as VQA-HAT dataset for VQA grounding. Our experiments show that: while removing the information of masked 

objects from standard transformer architectures leads to a significant drop in performance, the integration of capsules significantly improves the grounding ability of such systems and provides new state-of-the-art results compared to other approaches in the field.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022,

title = {OpenLDN: Learning to Discover Novel Classes for Open-World Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665-supp.pdf 

https://github.com/nayeemrizve/OpenLDN 

https://youtu.be/p2lYqvklcjA},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Semi-supervised learning (SSL) is one of the dominant approaches to address the annotation bottleneck of supervised learning. Recent SSL methods can effectively leverage a large repository of unlabeled data to improve performance while relying on a small set of labeled data. One common assumption in most SSL methods is that the labeled and unlabeled data are from the same data distribution. However, this is hardly the case in many real-world scenarios, which limits their applicability. In this work, instead, we attempt to solve the challenging open-world SSL problem that does not make such an assumption. In the open-world SSL problem, the objective is to recognize samples of known classes, and simultaneously detect and cluster samples belonging to novel classes present in unlabeled data. This work introduces OpenLDN that utilizes a pairwise similarity loss to discover novel classes. Using a bi-level optimization rule this pairwise similarity loss exploits the information available in the labeled set to implicitly cluster novel class samples, while simultaneously recognizing samples from known classes. After discovering novel classes, OpenLDN transforms the open-world SSL problem into a standard SSL problem to achieve additional performance gains using existing SSL methods. Our extensive experiments demonstrate that OpenLDN outperforms the current state-of-the-art methods on multiple popular classification benchmarks while providing a better accuracy/training time trade-off. Code: https://github.com/nayeemrizve/OpenLDN},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022b,

title = {Towards Realistic Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402-supp.pdf 

https://github.com/nayeemrizve/TRSSL 

https://youtu.be/mE7GeQ35WyY},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Deep learning is pushing the state-of-the-art in many computer 

vision applications. However, it relies on large annotated data 

repositories, and capturing the unconstrained nature of the real-world 

data is yet to be solved. Semi-supervised learning (SSL) complements 

the annotated training data with a large corpus of unlabeled data to 

reduce annotation cost. The standard SSL approach assumes unlabeled 

data are from the same distribution as annotated data. Recently, a more 

realistic SSL problem, called open-world SSL, is introduced, where the 

unannotated data might contain samples from unknown classes. In this paper, we propose a novel pseudo-label based approach to tackle SSL in 

open-world setting. At the core of our method, we utilize sample uncertainty and incorporate prior knowledge about class distribution to generate reliable class-distribution-aware pseudo-labels for unlabeled data belonging to both known and unknown classes. Our extensive experimentation showcases the effectiveness of our approach on several benchmark datasets, where it substantially outperforms the existing state-of-the art on seven diverse datasets including CIFAR-100 (∼17%), ImageNet-100 (∼5%), and Tiny ImageNet (∼9%). We also highlight the flexibility of our approach in solving novel class discovery task, demonstrate its stability in dealing with imbalanced data, and complement our approach with a technique to estimate the number of novel classes. Code: https://github.com/nayeemrizve/TRSSL},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2022b,

title = {GAMa: Cross-view Video Geo-localization},

author = {Shruti Vyas and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1512-supp.pdf 

https://youtu.be/KSHuer_VXJo},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {The existing work in cross-view geo-localization is based on images where a ground panorama is matched to an aerial image. In this work, we focus on ground videos instead of images which provides ad-ditional contextual cues which are important for this task. There are no existing datasets for this problem, therefore we propose GAMa dataset, a large-scale dataset with ground videos and corresponding aerial images. We also propose a novel approach to solve this problem. At clip-level, a short video clip is matched with corresponding aerial image and is later used to get video-level geo-localization of a long video. Moreover, we propose a hierarchical approach to further improve the clip-level geo-localization. On this challenging dataset, with unaligned images and lim-ited field of view, our proposed method achieves a Top-1 recall rate of 19.4% and 45.1% @1.0mile. Code & dataset are available at this link.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2022b,

title = {Med-DANet: Dynamic Architecture Network for Efficient Medical Volumetric Segmentation},

author = {Wenxuan Wang and Chen Chen and Jing Wang and Sen Zha and Yan Zhang and Jiangyun Li},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2206.06575.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {For 3D medical image (e.g. CT and MRI) segmentation, the difficulty of segmenting each slice in a clinical case varies greatly. Previous research on volumetric medical image segmentation in a slice-byslice manner conventionally use the identical 2D deep neural network to segment all the slices of the same case, ignoring the data heterogeneity among image slices. In this paper, we focus on multi-modal 3D MRI brain tumor segmentation and propose a dynamic architecture network named Med-DANet based on adaptive model selection to achieve effective accuracy and efficiency trade-off. For each slice of the input 3D MRI volume, our proposed method learns a slice-specific decision by the Decision Network to dynamically select a suitable model from the predefined Model Bank for the subsequent 2D segmentation task. Extensive experimental results on both BraTS 2019 and 2020 datasets show that our proposed method achieves comparable or better results than previous state-of-the art methods for 3D MRI brain tumor segmentation with much less model complexity. Compared with the state-of-the-art 3D method TransBTS, the proposed framework improves the model efficiency by up to 3.5 × without sacrificing the accuracy. Our code will be publicly available at https://github.com/Wenxuan-1119/Med-DANet.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2022b,

title = {GALA: Toward Geometry-and-Lighting-Aware Object Search for Compositing},

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2204.00125.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/GALA_supplementary.pdf},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Compositing-aware object search aims to find the most compatible objects for compositing given a background image and a query bounding box. Previous works focus on learning compatibility between the foreground object and background, but fail to learn other important factors from large-scale data, i.e. geometry and lighting. To move a step further, this paper proposes GALA (Geometry-and-Lighting-Aware), a generic foreground object search method with discriminative modeling on geometry and lighting compatibility for open-world image compositing. Remarkably, it achieves state-of-the-art results on the CAIS dataset and generalizes well on large-scale open-world datasets, i.e. Pixabay and 

Open Images. In addition, our method can effectively handle non-box scenarios, where users only provide background images without any input bounding box. A web demo (see supplementary materials) is built to showcase applications of the proposed method for compositing-aware search and automatic location/scale prediction for the foreground object.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2022b,

title = {Weakly Supervised Grounding for VQA in Vision-Language Transformers},

author = {Aisha Urooj Khan and Hilde Kuehne and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/1011-supp.pdf 

https://github.com/aurooj/WSG-VQA-VLTransformers 

https://youtu.be/dekmVb6lq3I},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Transformers for visual-language representation learning have been getting a lot of interest and shown tremendous performance on visual question answering (VQA) and grounding. However, most systems that show good performance of those tasks still rely on pre-trained object 

detectors during training, which limits their applicability to the object classes available for those detectors. To mitigate this limitation, this paper 

focuses on the problem of weakly supervised grounding in the context of visual question answering in transformers. Our approach leverages 

capsules by transforming each visual token into a capsule representation in the visual encoder; it then uses activations from language self-attention layers as a text-guided selection module to mask those capsules before they are forwarded to the next layer. We evaluate our approach on the challenging GQA as well as VQA-HAT dataset for VQA grounding. Our experiments show that: while removing the information of masked 

objects from standard transformer architectures leads to a significant drop in performance, the integration of capsules significantly improves the grounding ability of such systems and provides new state-of-the-art results compared to other approaches in the field.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022c,

title = {OpenLDN: Learning to Discover Novel Classes for Open-World Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/6665-supp.pdf 

https://github.com/nayeemrizve/OpenLDN 

https://youtu.be/p2lYqvklcjA},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Semi-supervised learning (SSL) is one of the dominant approaches to address the annotation bottleneck of supervised learning. Recent SSL methods can effectively leverage a large repository of unlabeled data to improve performance while relying on a small set of labeled data. One common assumption in most SSL methods is that the labeled and unlabeled data are from the same data distribution. However, this is hardly the case in many real-world scenarios, which limits their applicability. In this work, instead, we attempt to solve the challenging open-world SSL problem that does not make such an assumption. In the open-world SSL problem, the objective is to recognize samples of known classes, and simultaneously detect and cluster samples belonging to novel classes present in unlabeled data. This work introduces OpenLDN that utilizes a pairwise similarity loss to discover novel classes. Using a bi-level optimization rule this pairwise similarity loss exploits the information available in the labeled set to implicitly cluster novel class samples, while simultaneously recognizing samples from known classes. After discovering novel classes, OpenLDN transforms the open-world SSL problem into a standard SSL problem to achieve additional performance gains using existing SSL methods. Our extensive experiments demonstrate that OpenLDN outperforms the current state-of-the-art methods on multiple popular classification benchmarks while providing a better accuracy/training time trade-off. Code: https://github.com/nayeemrizve/OpenLDN},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2022bb,

title = {Towards Realistic Semi-Supervised Learning},

author = {Mamshad Nayeem Rizve and Navid Kardan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7402-supp.pdf 

https://github.com/nayeemrizve/TRSSL 

https://youtu.be/mE7GeQ35WyY},

year  = {2022},

date = {2022-10-23},

urldate = {2022-10-23},

booktitle = {European Conference on Computer Vision},

abstract = {Deep learning is pushing the state-of-the-art in many computer 

vision applications. However, it relies on large annotated data 

repositories, and capturing the unconstrained nature of the real-world 

data is yet to be solved. Semi-supervised learning (SSL) complements 

the annotated training data with a large corpus of unlabeled data to 

reduce annotation cost. The standard SSL approach assumes unlabeled 

data are from the same distribution as annotated data. Recently, a more 

realistic SSL problem, called open-world SSL, is introduced, where the 

unannotated data might contain samples from unknown classes. In this paper, we propose a novel pseudo-label based approach to tackle SSL in 

open-world setting. At the core of our method, we utilize sample uncertainty and incorporate prior knowledge about class distribution to generate reliable class-distribution-aware pseudo-labels for unlabeled data belonging to both known and unknown classes. Our extensive experimentation showcases the effectiveness of our approach on several benchmark datasets, where it substantially outperforms the existing state-of-the art on seven diverse datasets including CIFAR-100 (∼17%), ImageNet-100 (∼5%), and Tiny ImageNet (∼9%). We also highlight the flexibility of our approach in solving novel class discovery task, demonstrate its stability in dealing with imbalanced data, and complement our approach with a technique to estimate the number of novel classes. Code: https://github.com/nayeemrizve/TRSSL},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2020,

title = {Simultaneous Detection and Tracking with Motion Modelling for Multiple Object Tracking},

author = {ShiJie Sun and Naveed Akhtar and XiangYu Song and HuanSheng Song and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Simultaneous-Detection-and-Tracking-with-Motion-Modelling-for-Multiple-Object-Tracking.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Simultaneous-Detection-and-Tracking-with-Motion-Modelling-for-Multiple-Object-Tracking_Supp.pdf 

https://shijies.github.io/DMMN_Page/},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Deep learning based Multiple Object Tracking (MOT) currently relies on off-the-shelf detectors for tracking-by-detection. This results in deep models that are detector biased and evaluations that are detector influenced. To resolve this issue, we introduce Deep Motion Modeling Network (DMM-Net) that can estimate multiple objects’ motion parameters to perform joint detection and association in an end-to-end manner. DMM-Net models object features over multiple frames and simultaneously infers object classes, visibility and their motion parameters. These outputs are readily used to update the tracklets for efficient MOT. DMM-Net achieves PR-MOTA score of 12.80 @ 120+ fps for the popular UA-DETRAC challenge - which is better performance and orders of magnitude faster. We also contribute a synthetic large-scale public dataset Omni-MOT for vehicle tracking that provides precise ground-truth annotations to eliminate the detector influence in MOT evaluation. This 14M+ frames dataset is extendable with our public script (Code at Dataset, Dataset Recorder, Omni-MOT Source). We demonstrate the suitability of Omni-MOT for deep learning with DMM-Net, and also make the source code of our network public.},

keywords = {ECCV, Tracking},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xie2020,

title = {Count- and Similarity-aware R-CNN for Pedestrian Detection},

author = {Jin Xie and Hisham Cholakkal and Rao Muhammad Anwer and Fahad Shahbaz Khan and Yanwei Pang and Ling Shao and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Count-and-Similarity-aware-R-CNN-for-Pedestrian-Detection.pdf},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Recent pedestrian detection methods generally rely on additional supervision, such as visible bounding-box annotations, to handle heavy occlusions. We propose an approach that leverages pedestrian count and proposal similarity information within a two-stage pedestrian detection framework. Both pedestrian count and proposal similarity are derived from standard full-body annotations commonly used to train pedestrian detectors. We introduce a count-weighted detection loss function that assigns higher weights to the detection errors occurring at highly overlapping pedestrians. The proposed loss function is utilized at both stages of the two-stage detector. We further introduce a count-andsimilarity branch within the two-stage detection framework, which predicts pedestrian count as well as proposal similarity to identify distinct proposals. Our approach requires neither part information nor visible bounding-box annotations. Experiments are performed on the CityPersons and CrowdHuman datasets. Our method sets a new state-of-the-art on both datasets. Further, it achieves an absolute gain of 2.4% over the current state-of-the-art, in terms of log-average miss rate, on the heavily occluded (HO) set of CityPersons test set, without using additional visible bounding-box supervision. Finally, we demonstrate the applicability of our approach for the problem of human instance segmentation. Code and models are available at: https://github.com/Leotju/CaSe.},

keywords = {Detection, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schatz2020,

title = {A Recurrent Transformer Network for Novel View Action Synthesis},

author = {Kara Marie Schatz and Erik Quintanilla and Shruti Vyas and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_A-Recurrent-Transformer-Network-for-Novel-View-Action-Synthesis.pdf 

https://www.crcv.ucf.edu/research/projects/a-recurrent-transformer-network-for-novel-view-action-synthesis/},

year  = {2020},

date = {2020-08-23},

urldate = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {In this work, we address the problem of synthesizing human actions from novel views. Given an input video of an actor performing some action, we aim to synthesize a video with the same action performed from a novel view with the help of an appearance prior. We propose an end-to-end deep network to solve this problem. The proposed network utilizes the change in viewpoint to transform the action from the input view to the novel view in feature space. The transformed action is integrated with the target appearance using the proposed recurrent transformer network, which provides a transformed appearance for each time-step in the action sequence. The recurrent transformer network utilize action key-points which are determined in an unsupervised approach using the encoded action features. We also propose a hierarchical structure for the recurrent transformation which further improves the performance. We demonstrate the effectiveness of the proposed method through extensive experiments conducted on a large-scale multi-view action recognition NTU-RGB+D dataset. In addition, we show that the proposed method can transform the action to a novel viewpoint with an entirely different scene or actor. The code is publicly available at https://github.com/schatzkara/cross-view-video.},

keywords = {Cross-View, ECCV, REU, View Action Synthesis, View Synthesis},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2020,

title = {Multi-view Action Recognition using Cross-view Video Prediction},

author = {Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_Multi-view-Action-Recognition-using-Cross-view-Video-Prediction.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_Multi-view-Action-Recognition-using-Cross-view-Video-Prediction_Supp.pdf 

https://www.crcv.ucf.edu/research/projects/multi-view-action-recognition-using-cross-view-video-prediction/},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {In this work, we address the problem of action recognition in a multi-view environment. Most of the existing approaches utilize pose information for multi-view action recognition. We focus on RGB modality instead and propose an unsupervised representation learning framework, which encodes the scene dynamics in videos captured from multiple viewpoints via predicting actions from unseen views. The framework takes multiple short video clips from different viewpoints and time as input and learns an holistic internal representation which is used to predict a video clip from an unseen viewpoint and time. The ability of the proposed network to render unseen video frames enables it to learn a meaningful and robust representation of the scene dynamics. We evaluate the effectiveness of the learned representation for multiview video action recognition in a supervised approach. We observe a significant improvement in the performance with RGB modality on NTU-RGB+D dataset, which is the largest dataset for multi-view action recognition. The proposed framework also achieves state-of-the-art results with depth modality, which validates the generalization capability of the approach to other data modalities. The code is publicly available at https://github.com/svyas23/cross-view-action.},

keywords = {Cross-View Video Prediction, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Venkataramanan2020,

title = {Attention Guided Anomaly Localization in Images},

author = {Shashanka Venkataramanan and Kuan-Chuan Peng and Rajat Vikram Singh and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/2813.pdf 

https://youtu.be/b-EQr-fGPWo},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Anomaly localization is an important problem in computer vision which involves localizing anomalous regions within images with applications in industrial inspection, surveillance, and medical imaging. This task is challenging due to the small sample size and pixel coverage of the anomaly in real-world scenarios. Most prior works need to use anomalous training images to compute a class-specific threshold to localize anomalies. Without the need of anomalous training images, we propose Convolutional Adversarial Variational autoencoder with Guided Attention (CAVGA), which localizes the anomaly with a convolutional latent variable to preserve the spatial information. In the unsupervised setting, we propose an attention expansion loss where we encourage CAVGA to focus on all normal regions in the image. Furthermore, in the weakly supervised setting we propose a complementary guided attention loss, where we encourage the attention map to focus on all normal regions while minimizing the attention map corresponding to anomalous regions in the image. CAVGA outperforms the state-of-the-art (SOTA) anomaly localization methods on MVTec Anomaly Detection (MVTAD), modified ShanghaiTech Campus (mSTC) and Large-scale Attention based Glaucoma (LAG) datasets in the unsupervised setting and when using only 2% anomalous images in the weakly-supervised setting. CAVGA also outperforms SOTA anomaly detection methods on the MNIST, CIFAR-10, Fashion-MNIST, MVTAD, mSTC and LAG datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sun2020b,

title = {Simultaneous Detection and Tracking with Motion Modelling for Multiple Object Tracking},

author = {ShiJie Sun and Naveed Akhtar and XiangYu Song and HuanSheng Song and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Simultaneous-Detection-and-Tracking-with-Motion-Modelling-for-Multiple-Object-Tracking.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Simultaneous-Detection-and-Tracking-with-Motion-Modelling-for-Multiple-Object-Tracking_Supp.pdf 

https://shijies.github.io/DMMN_Page/},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Deep learning based Multiple Object Tracking (MOT) currently relies on off-the-shelf detectors for tracking-by-detection. This results in deep models that are detector biased and evaluations that are detector influenced. To resolve this issue, we introduce Deep Motion Modeling Network (DMM-Net) that can estimate multiple objects’ motion parameters to perform joint detection and association in an end-to-end manner. DMM-Net models object features over multiple frames and simultaneously infers object classes, visibility and their motion parameters. These outputs are readily used to update the tracklets for efficient MOT. DMM-Net achieves PR-MOTA score of 12.80 @ 120+ fps for the popular UA-DETRAC challenge - which is better performance and orders of magnitude faster. We also contribute a synthetic large-scale public dataset Omni-MOT for vehicle tracking that provides precise ground-truth annotations to eliminate the detector influence in MOT evaluation. This 14M+ frames dataset is extendable with our public script (Code at Dataset, Dataset Recorder, Omni-MOT Source). We demonstrate the suitability of Omni-MOT for deep learning with DMM-Net, and also make the source code of our network public.},

keywords = {ECCV, Tracking},

pubstate = {published},

tppubtype = {conference}

}

@conference{Xie2020b,

title = {Count- and Similarity-aware R-CNN for Pedestrian Detection},

author = {Jin Xie and Hisham Cholakkal and Rao Muhammad Anwer and Fahad Shahbaz Khan and Yanwei Pang and Ling Shao and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Publications_Count-and-Similarity-aware-R-CNN-for-Pedestrian-Detection.pdf},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Recent pedestrian detection methods generally rely on additional supervision, such as visible bounding-box annotations, to handle heavy occlusions. We propose an approach that leverages pedestrian count and proposal similarity information within a two-stage pedestrian detection framework. Both pedestrian count and proposal similarity are derived from standard full-body annotations commonly used to train pedestrian detectors. We introduce a count-weighted detection loss function that assigns higher weights to the detection errors occurring at highly overlapping pedestrians. The proposed loss function is utilized at both stages of the two-stage detector. We further introduce a count-andsimilarity branch within the two-stage detection framework, which predicts pedestrian count as well as proposal similarity to identify distinct proposals. Our approach requires neither part information nor visible bounding-box annotations. Experiments are performed on the CityPersons and CrowdHuman datasets. Our method sets a new state-of-the-art on both datasets. Further, it achieves an absolute gain of 2.4% over the current state-of-the-art, in terms of log-average miss rate, on the heavily occluded (HO) set of CityPersons test set, without using additional visible bounding-box supervision. Finally, we demonstrate the applicability of our approach for the problem of human instance segmentation. Code and models are available at: https://github.com/Leotju/CaSe.},

keywords = {Detection, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Schatz2020b,

title = {A Recurrent Transformer Network for Novel View Action Synthesis},

author = {Kara Marie Schatz and Erik Quintanilla and Shruti Vyas and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_A-Recurrent-Transformer-Network-for-Novel-View-Action-Synthesis.pdf 

https://www.crcv.ucf.edu/research/projects/a-recurrent-transformer-network-for-novel-view-action-synthesis/},

year  = {2020},

date = {2020-08-23},

urldate = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {In this work, we address the problem of synthesizing human actions from novel views. Given an input video of an actor performing some action, we aim to synthesize a video with the same action performed from a novel view with the help of an appearance prior. We propose an end-to-end deep network to solve this problem. The proposed network utilizes the change in viewpoint to transform the action from the input view to the novel view in feature space. The transformed action is integrated with the target appearance using the proposed recurrent transformer network, which provides a transformed appearance for each time-step in the action sequence. The recurrent transformer network utilize action key-points which are determined in an unsupervised approach using the encoded action features. We also propose a hierarchical structure for the recurrent transformation which further improves the performance. We demonstrate the effectiveness of the proposed method through extensive experiments conducted on a large-scale multi-view action recognition NTU-RGB+D dataset. In addition, we show that the proposed method can transform the action to a novel viewpoint with an entirely different scene or actor. The code is publicly available at https://github.com/schatzkara/cross-view-video.},

keywords = {Cross-View, ECCV, REU, View Action Synthesis, View Synthesis},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vyas2020b,

title = {Multi-view Action Recognition using Cross-view Video Prediction},

author = {Shruti Vyas and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_Multi-view-Action-Recognition-using-Cross-view-Video-Prediction.pdf 

https://www.crcv.ucf.edu/wp-content/uploads/2020/07/Projects_Multi-view-Action-Recognition-using-Cross-view-Video-Prediction_Supp.pdf 

https://www.crcv.ucf.edu/research/projects/multi-view-action-recognition-using-cross-view-video-prediction/},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {In this work, we address the problem of action recognition in a multi-view environment. Most of the existing approaches utilize pose information for multi-view action recognition. We focus on RGB modality instead and propose an unsupervised representation learning framework, which encodes the scene dynamics in videos captured from multiple viewpoints via predicting actions from unseen views. The framework takes multiple short video clips from different viewpoints and time as input and learns an holistic internal representation which is used to predict a video clip from an unseen viewpoint and time. The ability of the proposed network to render unseen video frames enables it to learn a meaningful and robust representation of the scene dynamics. We evaluate the effectiveness of the learned representation for multiview video action recognition in a supervised approach. We observe a significant improvement in the performance with RGB modality on NTU-RGB+D dataset, which is the largest dataset for multi-view action recognition. The proposed framework also achieves state-of-the-art results with depth modality, which validates the generalization capability of the approach to other data modalities. The code is publicly available at https://github.com/svyas23/cross-view-action.},

keywords = {Cross-View Video Prediction, ECCV},

pubstate = {published},

tppubtype = {conference}

}

@conference{Venkataramanan2020b,

title = {Attention Guided Anomaly Localization in Images},

author = {Shashanka Venkataramanan and Kuan-Chuan Peng and Rajat Vikram Singh and Abhijit Mahalanobis},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/07/2813.pdf 

https://youtu.be/b-EQr-fGPWo},

year  = {2020},

date = {2020-08-23},

booktitle = {16th European Conference on Computer Vision},

abstract = {Anomaly localization is an important problem in computer vision which involves localizing anomalous regions within images with applications in industrial inspection, surveillance, and medical imaging. This task is challenging due to the small sample size and pixel coverage of the anomaly in real-world scenarios. Most prior works need to use anomalous training images to compute a class-specific threshold to localize anomalies. Without the need of anomalous training images, we propose Convolutional Adversarial Variational autoencoder with Guided Attention (CAVGA), which localizes the anomaly with a convolutional latent variable to preserve the spatial information. In the unsupervised setting, we propose an attention expansion loss where we encourage CAVGA to focus on all normal regions in the image. Furthermore, in the weakly supervised setting we propose a complementary guided attention loss, where we encourage the attention map to focus on all normal regions while minimizing the attention map corresponding to anomalous regions in the image. CAVGA outperforms the state-of-the-art (SOTA) anomaly localization methods on MVTec Anomaly Detection (MVTAD), modified ShanghaiTech Campus (mSTC) and Large-scale Attention based Glaucoma (LAG) datasets in the unsupervised setting and when using only 2% anomalous images in the weakly-supervised setting. CAVGA also outperforms SOTA anomaly detection methods on the MNIST, CIFAR-10, Fashion-MNIST, MVTAD, mSTC and LAG datasets.},

keywords = {ECCV},

pubstate = {published},

tppubtype = {conference}

}