2026
Campos, Ron; Maity, Subhajit; Li, Xin; Das, Srijan; Dutta, Aritra
DnA: Denoising Attention for Visual Tasks Conference
European Conference on Computer Vision -- ECCV 2026, 2026.
Abstract | Tags: ECCV, Transformers | Links:
@conference{nokey,
title = {DnA: Denoising Attention for Visual Tasks},
author = {Ron Campos and Subhajit Maity and Xin Li and Srijan Das and Aritra Dutta},
url = {https://rjccv.github.io/DnA/},
year = {2026},
date = {2026-09-07},
publisher = {European Conference on Computer Vision -- ECCV 2026},
abstract = {The softmax activation in multihead attention (MHA) is the de facto standard for attention-based models in visual perception tasks. However, standard softmax can produce noisy attention patterns that dilute relevant features and degrade its performance. In this paper, we propose Denoising Attention or DnA, in which, first, a positive query identifies which image features belong to the correct class, and a negative query identifies closely associated but irrelevant image features. DnA then projects these interactions into two distinct subspaces with larger principal angles, promoting subspace separation and improved discriminability. Using a ViT-B backbone, our proposed DnA achieves a 0.8% absolute gain on ImageNet-1K compared to the baseline. We further show improvements across multiple visual understanding tasks, including video understanding with video transformers (1.8%) and video LLMs (0.5%). Our extensive empirical analyses justify the design choices involving two interacting subspaces and the denoising effect of DnA.},
keywords = {ECCV, Transformers},
pubstate = {published},
tppubtype = {conference}
}
The softmax activation in multihead attention (MHA) is the de facto standard for attention-based models in visual perception tasks. However, standard softmax can produce noisy attention patterns that dilute relevant features and degrade its performance. In this paper, we propose Denoising Attention or DnA, in which, first, a positive query identifies which image features belong to the correct class, and a negative query identifies closely associated but irrelevant image features. DnA then projects these interactions into two distinct subspaces with larger principal angles, promoting subspace separation and improved discriminability. Using a ViT-B backbone, our proposed DnA achieves a 0.8% absolute gain on ImageNet-1K compared to the baseline. We further show improvements across multiple visual understanding tasks, including video understanding with video transformers (1.8%) and video LLMs (0.5%). Our extensive empirical analyses justify the design choices involving two interacting subspaces and the denoising effect of DnA.
2022
Zhu, Sijie; Shah, Mubarak; Chen, Chen
TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization Conference
IEEE Computer Vision and Pattern Recognition, 2022.
Tags: Cross-View, CVPR, Geo-Localization, Transformers | Links:
@conference{nokey_107,
title = {TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization},
author = {Sijie Zhu and Mubarak Shah and Chen Chen},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695-supp.pdf},
year = {2022},
date = {2022-06-19},
urldate = {2022-06-19},
publisher = {IEEE Computer Vision and Pattern Recognition},
keywords = {Cross-View, CVPR, Geo-Localization, Transformers},
pubstate = {published},
tppubtype = {conference}
}
Cao, Jiale; Pang, Yenwai; Anwer, Rao Muhammad; Cholakkal, Hisham; Xie, Jin; Shah, Mubarak; Khan, Fahad Shahbaz
PSTR: End-to-End One-Step Person Search With Transformers Conference
IEEE Computer Vision and Pattern Recognition, 2022.
Abstract | Tags: CVPR, Re-Identification, Transformers, Visual Search | Links:
@conference{nokey_108,
title = {PSTR: End-to-End One-Step Person Search With Transformers},
author = {Jiale Cao and Yenwai Pang and Rao Muhammad Anwer and Hisham Cholakkal and Jin Xie and Mubarak Shah and Fahad Shahbaz Khan},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/05237-2.pdf
https://github.com/JialeCao001/PSTR},
year = {2022},
date = {2022-06-19},
urldate = {2022-06-19},
publisher = {IEEE Computer Vision and Pattern Recognition},
abstract = {We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for
discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step
transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging
PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.},
keywords = {CVPR, Re-Identification, Transformers, Visual Search},
pubstate = {published},
tppubtype = {conference}
}
We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for
discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step
transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging
PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.
discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step
transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging
PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.
Gupta, Akshita; Narayan, Sanath; Joseph, K J; Khan, Salman; Khan, Fahad Shahbaz; Shah, Mubarak
OW-DETR: Open-world Detection Transformer Conference
IEEE Computer Vision and Pattern Recognition, 2022.
Abstract | Tags: CVPR, Object Detection, Open World, Transformers | Links:
@conference{nokey_109,
title = {OW-DETR: Open-world Detection Transformer},
author = {Akshita Gupta and Sanath Narayan and K J Joseph and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815-supp.pdf
https://github.com/akshitac8/OW-DETR.},
year = {2022},
date = {2022-06-19},
urldate = {2022-06-19},
publisher = {IEEE Computer Vision and Pattern Recognition},
abstract = {Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components
namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental
object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.},
keywords = {CVPR, Object Detection, Open World, Transformers},
pubstate = {published},
tppubtype = {conference}
}
Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components
namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental
object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.
namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental
object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.
Zhu, Sijie; Shah, Mubarak; Chen, Chen
TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization Conference
IEEE Computer Vision and Pattern Recognition, 2022.
Tags: Cross-View, CVPR, Geo-Localization, Transformers | Links:
@conference{nokey_155,
title = {TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization},
author = {Sijie Zhu and Mubarak Shah and Chen Chen},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695-supp.pdf},
year = {2022},
date = {2022-06-19},
urldate = {2022-06-19},
publisher = {IEEE Computer Vision and Pattern Recognition},
keywords = {Cross-View, CVPR, Geo-Localization, Transformers},
pubstate = {published},
tppubtype = {conference}
}
Cao, Jiale; Pang, Yenwai; Anwer, Rao Muhammad; Cholakkal, Hisham; Xie, Jin; Shah, Mubarak; Khan, Fahad Shahbaz
PSTR: End-to-End One-Step Person Search With Transformers Conference
IEEE Computer Vision and Pattern Recognition, 2022.
Abstract | Tags: CVPR, Re-Identification, Transformers, Visual Search | Links:
@conference{nokey_156,
title = {PSTR: End-to-End One-Step Person Search With Transformers},
author = {Jiale Cao and Yenwai Pang and Rao Muhammad Anwer and Hisham Cholakkal and Jin Xie and Mubarak Shah and Fahad Shahbaz Khan},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/05237-2.pdf
https://github.com/JialeCao001/PSTR},
year = {2022},
date = {2022-06-19},
urldate = {2022-06-19},
publisher = {IEEE Computer Vision and Pattern Recognition},
abstract = {We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for
discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step
transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging
PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.},
keywords = {CVPR, Re-Identification, Transformers, Visual Search},
pubstate = {published},
tppubtype = {conference}
}
We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for
discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step
transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging
PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.
discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step
transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging
PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.
Gupta, Akshita; Narayan, Sanath; Joseph, K J; Khan, Salman; Khan, Fahad Shahbaz; Shah, Mubarak
OW-DETR: Open-world Detection Transformer Conference
IEEE Computer Vision and Pattern Recognition, 2022.
Abstract | Tags: CVPR, Object Detection, Open World, Transformers | Links:
@conference{nokey_157,
title = {OW-DETR: Open-world Detection Transformer},
author = {Akshita Gupta and Sanath Narayan and K J Joseph and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815-supp.pdf
https://github.com/akshitac8/OW-DETR.},
year = {2022},
date = {2022-06-19},
urldate = {2022-06-19},
publisher = {IEEE Computer Vision and Pattern Recognition},
abstract = {Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components
namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental
object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.},
keywords = {CVPR, Object Detection, Open World, Transformers},
pubstate = {published},
tppubtype = {conference}
}
Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components
namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental
object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.
namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental
object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.
2021
Regmi, Krishna; Shah, Mubarak
Video Geo-Localization Employing Geo-Temporal Feature Learning and GPS Trajectory Smoothing Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: Geo-Localization, ICCV, Transformers, Video Geo-localization | Links:
@conference{RegmiICCV2021,
title = {Video Geo-Localization Employing Geo-Temporal Feature Learning and GPS Trajectory Smoothing},
author = {Krishna Regmi and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/video_geolocalization_ICCV2021.pdf
https://github.com/kregmi/VTE},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {In this paper, we address the problem of video geolocalization by proposing a Geo-Temporal Feature Learning (GTFL) Network to simultaneously learn the discriminative features for the query video frames and the gallery images for estimating the geo-spatial trajectory of a query video. Based on a transformer encoder architecture, our
GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive
evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at
https://github.com/kregmi/VTE.},
keywords = {Geo-Localization, ICCV, Transformers, Video Geo-localization},
pubstate = {published},
tppubtype = {conference}
}
In this paper, we address the problem of video geolocalization by proposing a Geo-Temporal Feature Learning (GTFL) Network to simultaneously learn the discriminative features for the query video frames and the gallery images for estimating the geo-spatial trajectory of a query video. Based on a transformer encoder architecture, our
GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive
evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at
https://github.com/kregmi/VTE.
GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive
evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at
https://github.com/kregmi/VTE.
Bhunia, Ankan Kumar; Khan, Salman; Cholakkal, Hisham; Anwer, Rao Muhammad; Khan, Fahad Shahbaz; Shah, Mubarak
Handwriting Transformers Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: ICCV, Transformers | Links:
@conference{BhuniaICCV2021,
title = {Handwriting Transformers},
author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Fahad Shahbaz Khan and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21_supp.pdf},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {We propose a novel transformer-based styled handwritten text image generation approach, HWT, that strives to learn both style-content entanglement as well as global and local writing style patterns. The proposed HWT captures the long and short range relationships within the style examples through a self-attention mechanism, thereby encoding both global and local style patterns. Further, the proposed transformer-based HWT comprises an encoder-decoder attention that enables style-content entanglement by gathering the style representation of each query character. To the best of our knowledge, we are the first to introduce a transformer-based generative network for styled handwritten text generation.
Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.},
keywords = {ICCV, Transformers},
pubstate = {published},
tppubtype = {conference}
}
We propose a novel transformer-based styled handwritten text image generation approach, HWT, that strives to learn both style-content entanglement as well as global and local writing style patterns. The proposed HWT captures the long and short range relationships within the style examples through a self-attention mechanism, thereby encoding both global and local style patterns. Further, the proposed transformer-based HWT comprises an encoder-decoder attention that enables style-content entanglement by gathering the style representation of each query character. To the best of our knowledge, we are the first to introduce a transformer-based generative network for styled handwritten text generation.
Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.
Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.
Narayan, Sanath; Gupta, Akshita; Khan, Salman; Khan, Fahad Shahbaz; Shao, Ling; Shah, Mubarak
Discriminative Region-based Multi-Label Zero-Shot Learning Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: ICCV, Transformers, Zero-Shot Learning | Links:
@conference{NarayanICCV2021,
title = {Discriminative Region-based Multi-Label Zero-Shot Learning},
author = {Sanath Narayan and Akshita Gupta and Salman Khan and Fahad Shahbaz Khan and Ling Shao and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617-supp.pdf},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {Multi-label zero-shot learning (ZSL) is a more realistic counter-part of standard single-label ZSL since several objects can co-exist in a natural image. However, the occurrence of multiple objects complicates the reasoning and re-quires region-specific processing of visual features to pre-serve their contextual cues. We note that the best existing multi-label ZSL method takes a shared approach towards attending to region features with a common set of attention maps for all the classes. Such shared maps lead to diffused attention, which does not discriminatively focus on relevant locations when the number of classes are large. Moreover, mapping spatially-pooled visual features to the class semantics leads to inter-class feature entanglement, thus hampering the classification. Here, we propose an alternate approach towards region-based discriminability-preserving multi-label zero-shot classification. Our approach maintains the spatial resolution to preserve region-level characteristics and utilizes a bi-level attention module (BiAM) to enrich the features by incorporating both region and scene context information. The enriched region-level features are then mapped to the class semantics and only their class predictions are spatially pooled to obtain image-level predictions, thereby keeping the multi-class features disentangled. Our approach sets a new state of the art on two large-scale multi-label zero-shot benchmarks: NUS-WIDE and Open Images. On NUS-WIDE, our approach achieves an absolute gain of 6.9% mAP for ZSL, compared to the best published results. Source code is available at <a href="https://github.com/akshitac8/BiAM" target="_blank">https://github.com/akshitac8/BiAM</a>.},
keywords = {ICCV, Transformers, Zero-Shot Learning},
pubstate = {published},
tppubtype = {conference}
}
Multi-label zero-shot learning (ZSL) is a more realistic counter-part of standard single-label ZSL since several objects can co-exist in a natural image. However, the occurrence of multiple objects complicates the reasoning and re-quires region-specific processing of visual features to pre-serve their contextual cues. We note that the best existing multi-label ZSL method takes a shared approach towards attending to region features with a common set of attention maps for all the classes. Such shared maps lead to diffused attention, which does not discriminatively focus on relevant locations when the number of classes are large. Moreover, mapping spatially-pooled visual features to the class semantics leads to inter-class feature entanglement, thus hampering the classification. Here, we propose an alternate approach towards region-based discriminability-preserving multi-label zero-shot classification. Our approach maintains the spatial resolution to preserve region-level characteristics and utilizes a bi-level attention module (BiAM) to enrich the features by incorporating both region and scene context information. The enriched region-level features are then mapped to the class semantics and only their class predictions are spatially pooled to obtain image-level predictions, thereby keeping the multi-class features disentangled. Our approach sets a new state of the art on two large-scale multi-label zero-shot benchmarks: NUS-WIDE and Open Images. On NUS-WIDE, our approach achieves an absolute gain of 6.9% mAP for ZSL, compared to the best published results. Source code is available at <a href="https://github.com/akshitac8/BiAM" target="_blank">https://github.com/akshitac8/BiAM</a>.
Regmi, Krishna; Shah, Mubarak
Video Geo-Localization Employing Geo-Temporal Feature Learning and GPS Trajectory Smoothing Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: Geo-Localization, ICCV, Transformers, Video Geo-localization | Links:
@conference{RegmiICCV2021b,
title = {Video Geo-Localization Employing Geo-Temporal Feature Learning and GPS Trajectory Smoothing},
author = {Krishna Regmi and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/video_geolocalization_ICCV2021.pdf
https://github.com/kregmi/VTE},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {In this paper, we address the problem of video geolocalization by proposing a Geo-Temporal Feature Learning (GTFL) Network to simultaneously learn the discriminative features for the query video frames and the gallery images for estimating the geo-spatial trajectory of a query video. Based on a transformer encoder architecture, our
GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive
evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at
https://github.com/kregmi/VTE.},
keywords = {Geo-Localization, ICCV, Transformers, Video Geo-localization},
pubstate = {published},
tppubtype = {conference}
}
In this paper, we address the problem of video geolocalization by proposing a Geo-Temporal Feature Learning (GTFL) Network to simultaneously learn the discriminative features for the query video frames and the gallery images for estimating the geo-spatial trajectory of a query video. Based on a transformer encoder architecture, our
GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive
evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at
https://github.com/kregmi/VTE.
GTFL model encodes query and gallery data separately, via two dedicated branches. The proposed GPS Loss and Clip Triplet Loss exploit the geographical and temporal proximity between the frames and the clips to jointly learn the query and the gallery features. We also propose a deep learning approach to trajectory smoothing by predicting the outliers in the estimated GPS positions and learning the offsets to smooth the trajectory. We build a large dataset from four different regions of USA; New York, San Francisco, Berkeley and Bay Area using BDD driving videos as query, and by collecting corresponding Google StreetView (GSV) Images for gallery. Extensive
evaluations of proposed method on this new dataset are provided. Code and dataset details is publicly available at
https://github.com/kregmi/VTE.
Bhunia, Ankan Kumar; Khan, Salman; Cholakkal, Hisham; Anwer, Rao Muhammad; Khan, Fahad Shahbaz; Shah, Mubarak
Handwriting Transformers Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: ICCV, Transformers | Links:
@conference{BhuniaICCV2021b,
title = {Handwriting Transformers},
author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Fahad Shahbaz Khan and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Handwriting_Generation_ICCV21_supp.pdf},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {We propose a novel transformer-based styled handwritten text image generation approach, HWT, that strives to learn both style-content entanglement as well as global and local writing style patterns. The proposed HWT captures the long and short range relationships within the style examples through a self-attention mechanism, thereby encoding both global and local style patterns. Further, the proposed transformer-based HWT comprises an encoder-decoder attention that enables style-content entanglement by gathering the style representation of each query character. To the best of our knowledge, we are the first to introduce a transformer-based generative network for styled handwritten text generation.
Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.},
keywords = {ICCV, Transformers},
pubstate = {published},
tppubtype = {conference}
}
We propose a novel transformer-based styled handwritten text image generation approach, HWT, that strives to learn both style-content entanglement as well as global and local writing style patterns. The proposed HWT captures the long and short range relationships within the style examples through a self-attention mechanism, thereby encoding both global and local style patterns. Further, the proposed transformer-based HWT comprises an encoder-decoder attention that enables style-content entanglement by gathering the style representation of each query character. To the best of our knowledge, we are the first to introduce a transformer-based generative network for styled handwritten text generation.
Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.
Our proposed HWT generates realistic styled handwritten text images and significantly outperforms the state-of-the-art demonstrated through extensive qualitative, quantitative and human-based evaluations. The proposed HWT can handle arbitrary length of text and any desired writing style in a few-shot setting. Further, our HWT generalizes well to the challenging scenario where both words and writing style are unseen during training, generating realistic styled handwritten text images.
Narayan, Sanath; Gupta, Akshita; Khan, Salman; Khan, Fahad Shahbaz; Shao, Ling; Shah, Mubarak
Discriminative Region-based Multi-Label Zero-Shot Learning Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: ICCV, Transformers, Zero-Shot Learning | Links:
@conference{NarayanICCV2021b,
title = {Discriminative Region-based Multi-Label Zero-Shot Learning},
author = {Sanath Narayan and Akshita Gupta and Salman Khan and Fahad Shahbaz Khan and Ling Shao and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02617-supp.pdf},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {Multi-label zero-shot learning (ZSL) is a more realistic counter-part of standard single-label ZSL since several objects can co-exist in a natural image. However, the occurrence of multiple objects complicates the reasoning and re-quires region-specific processing of visual features to pre-serve their contextual cues. We note that the best existing multi-label ZSL method takes a shared approach towards attending to region features with a common set of attention maps for all the classes. Such shared maps lead to diffused attention, which does not discriminatively focus on relevant locations when the number of classes are large. Moreover, mapping spatially-pooled visual features to the class semantics leads to inter-class feature entanglement, thus hampering the classification. Here, we propose an alternate approach towards region-based discriminability-preserving multi-label zero-shot classification. Our approach maintains the spatial resolution to preserve region-level characteristics and utilizes a bi-level attention module (BiAM) to enrich the features by incorporating both region and scene context information. The enriched region-level features are then mapped to the class semantics and only their class predictions are spatially pooled to obtain image-level predictions, thereby keeping the multi-class features disentangled. Our approach sets a new state of the art on two large-scale multi-label zero-shot benchmarks: NUS-WIDE and Open Images. On NUS-WIDE, our approach achieves an absolute gain of 6.9% mAP for ZSL, compared to the best published results. Source code is available at <a href="https://github.com/akshitac8/BiAM" target="_blank">https://github.com/akshitac8/BiAM</a>.},
keywords = {ICCV, Transformers, Zero-Shot Learning},
pubstate = {published},
tppubtype = {conference}
}
Multi-label zero-shot learning (ZSL) is a more realistic counter-part of standard single-label ZSL since several objects can co-exist in a natural image. However, the occurrence of multiple objects complicates the reasoning and re-quires region-specific processing of visual features to pre-serve their contextual cues. We note that the best existing multi-label ZSL method takes a shared approach towards attending to region features with a common set of attention maps for all the classes. Such shared maps lead to diffused attention, which does not discriminatively focus on relevant locations when the number of classes are large. Moreover, mapping spatially-pooled visual features to the class semantics leads to inter-class feature entanglement, thus hampering the classification. Here, we propose an alternate approach towards region-based discriminability-preserving multi-label zero-shot classification. Our approach maintains the spatial resolution to preserve region-level characteristics and utilizes a bi-level attention module (BiAM) to enrich the features by incorporating both region and scene context information. The enriched region-level features are then mapped to the class semantics and only their class predictions are spatially pooled to obtain image-level predictions, thereby keeping the multi-class features disentangled. Our approach sets a new state of the art on two large-scale multi-label zero-shot benchmarks: NUS-WIDE and Open Images. On NUS-WIDE, our approach achieves an absolute gain of 6.9% mAP for ZSL, compared to the best published results. Source code is available at <a href="https://github.com/akshitac8/BiAM" target="_blank">https://github.com/akshitac8/BiAM</a>.
Tirupattur, Praveen; Duarte, Kevin; Rawat, Yogesh Singh; Shah, Mubarak
Modeling Multi-Label Action Dependencies for Temporal Action Localization Conference
IEEE Conference on Computer Vision and Pattern Recognition (Oral), 2021.
Tags: CVPR, Transformers | Links:
@conference{Tirupattur2021,
title = {Modeling Multi-Label Action Dependencies for Temporal Action Localization},
author = {Praveen Tirupattur and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization_Supp.zip
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization-CVPR-2021-ORAL.mp4},
year = {2021},
date = {2021-06-19},
urldate = {2021-06-19},
publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},
keywords = {CVPR, Transformers},
pubstate = {published},
tppubtype = {conference}
}
Tirupattur, Praveen; Duarte, Kevin; Rawat, Yogesh Singh; Shah, Mubarak
Modeling Multi-Label Action Dependencies for Temporal Action Localization Conference
IEEE Conference on Computer Vision and Pattern Recognition (Oral), 2021.
Tags: CVPR, Transformers | Links:
@conference{Tirupattur2021b,
title = {Modeling Multi-Label Action Dependencies for Temporal Action Localization},
author = {Praveen Tirupattur and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization_Supp.zip
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization-CVPR-2021-ORAL.mp4},
year = {2021},
date = {2021-06-19},
urldate = {2021-06-19},
publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},
keywords = {CVPR, Transformers},
pubstate = {published},
tppubtype = {conference}
}
Khan, Salman; Naseer, Muzammal; Hayat, Munawar; Zamir, Syed Waqas; Khan, Fahad Shahbaz; Shah, Mubarak
Transformers in Vision: A Survey Technical Report
no. arXiv:2101.01169, 2021.
Tags: Survey, Transformers | Links:
@techreport{Khan2021,
title = {Transformers in Vision: A Survey},
author = {Salman Khan and Muzammal Naseer and Munawar Hayat and Syed Waqas Zamir and Fahad Shahbaz Khan and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_TransformersSurvey.pdf},
year = {2021},
date = {2021-02-02},
urldate = {2021-02-02},
number = {arXiv:2101.01169},
keywords = {Survey, Transformers},
pubstate = {published},
tppubtype = {techreport}
}
Khan, Salman; Naseer, Muzammal; Hayat, Munawar; Zamir, Syed Waqas; Khan, Fahad Shahbaz; Shah, Mubarak
Transformers in Vision: A Survey Technical Report
no. arXiv:2101.01169, 2021.
Tags: Survey, Transformers | Links:
@techreport{Khan2021d,
title = {Transformers in Vision: A Survey},
author = {Salman Khan and Muzammal Naseer and Munawar Hayat and Syed Waqas Zamir and Fahad Shahbaz Khan and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Publications_TransformersSurvey.pdf},
year = {2021},
date = {2021-02-02},
urldate = {2021-02-02},
number = {arXiv:2101.01169},
keywords = {Survey, Transformers},
pubstate = {published},
tppubtype = {techreport}
}