2021
Duarte, Kevin
Capsule Networks for Video Understanding PhD Thesis
University of Central Florida, 2021.
Tags: Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video | Links:
@phdthesis{Duarte2021b,
title = {Capsule Networks for Video Understanding},
author = {Kevin Duarte},
url = {https://www.crcv.ucf.edu/people/alumni/#:~:text=Capsule%20Networks%20for%20Video%20Understanding},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
school = {University of Central Florida},
keywords = {Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video},
pubstate = {published},
tppubtype = {phdthesis}
}
Duarte, Kevin; Chen, Brian; Shvetsova, Nina; Rouditchenko, Andrew; Thomas, Samuel; Liu, Alexander; Harwath, David; Glass, James; Kuehne, Hilde; Shah, Mubarak
Routing with Self-Attention for Multimodal Capsule Networks Unpublished
arXiv preprint arXiv:2112.00775, 2021.
Abstract | Tags: Audio, Capsule Networks, Multi-Modal Learning, Text and Video | Links:
@unpublished{nokey_114,
title = {Routing with Self-Attention for Multimodal Capsule Networks},
author = {Kevin Duarte and Brian Chen and Nina Shvetsova and Andrew Rouditchenko and Samuel Thomas and Alexander Liu and David Harwath and James Glass and Hilde Kuehne and Mubarak Shah},
editor = {arXiv},
url = {https://arxiv.org/pdf/2112.00775.pdf
https://arxiv.org/abs/2112.00775},
doi = {https://doi.org/10.48550/arXiv.2112.00775},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
abstract = {The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.},
howpublished = {arXiv preprint arXiv:2112.00775},
keywords = {Audio, Capsule Networks, Multi-Modal Learning, Text and Video},
pubstate = {published},
tppubtype = {unpublished}
}
The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.
Duarte, Kevin
Capsule Networks for Video Understanding PhD Thesis
University of Central Florida, 2021.
Tags: Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video | Links:
@phdthesis{Duarte2021bb,
title = {Capsule Networks for Video Understanding},
author = {Kevin Duarte},
url = {https://www.crcv.ucf.edu/people/alumni/#:~:text=Capsule%20Networks%20for%20Video%20Understanding},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
school = {University of Central Florida},
keywords = {Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video},
pubstate = {published},
tppubtype = {phdthesis}
}
Duarte, Kevin; Chen, Brian; Shvetsova, Nina; Rouditchenko, Andrew; Thomas, Samuel; Liu, Alexander; Harwath, David; Glass, James; Kuehne, Hilde; Shah, Mubarak
Routing with Self-Attention for Multimodal Capsule Networks Unpublished
arXiv preprint arXiv:2112.00775, 2021.
Abstract | Tags: Audio, Capsule Networks, Multi-Modal Learning, Text and Video | Links:
@unpublished{nokey_162,
title = {Routing with Self-Attention for Multimodal Capsule Networks},
author = {Kevin Duarte and Brian Chen and Nina Shvetsova and Andrew Rouditchenko and Samuel Thomas and Alexander Liu and David Harwath and James Glass and Hilde Kuehne and Mubarak Shah},
editor = {arXiv},
url = {https://arxiv.org/pdf/2112.00775.pdf
https://arxiv.org/abs/2112.00775},
doi = {https://doi.org/10.48550/arXiv.2112.00775},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
abstract = {The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.},
howpublished = {arXiv preprint arXiv:2112.00775},
keywords = {Audio, Capsule Networks, Multi-Modal Learning, Text and Video},
pubstate = {published},
tppubtype = {unpublished}
}
The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.
Khan, Aisha Urooj; Kuehne, Hilde; Duarte, Kevin; Gan, Chuang; Lobo, Niels Da Vitoria; Shah, Mubarak
Found a Reason for me? Weakly-supervised Grounded Visual Question Answering using Capsules Conference
IEEE Conference on Computer Vision and Pattern Recognition, 2021.
Tags: Capsule Networks, CVPR, Grounding, VQA | Links:
@conference{Khan2021b,
title = {Found a Reason for me? Weakly-supervised Grounded Visual Question Answering using Capsules},
author = {Aisha Urooj Khan and Hilde Kuehne and Kevin Duarte and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me_Supp.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr21_poster_v2.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr_2021_5min.mp4},
year = {2021},
date = {2021-06-19},
urldate = {2021-06-19},
publisher = {IEEE Conference on Computer Vision and Pattern Recognition},
keywords = {Capsule Networks, CVPR, Grounding, VQA},
pubstate = {published},
tppubtype = {conference}
}
Khan, Aisha Urooj; Kuehne, Hilde; Duarte, Kevin; Gan, Chuang; Lobo, Niels Da Vitoria; Shah, Mubarak
Found a Reason for me? Weakly-supervised Grounded Visual Question Answering using Capsules Conference
IEEE Conference on Computer Vision and Pattern Recognition, 2021.
Tags: Capsule Networks, CVPR, Grounding, VQA | Links:
@conference{Khan2021bb,
title = {Found a Reason for me? Weakly-supervised Grounded Visual Question Answering using Capsules},
author = {Aisha Urooj Khan and Hilde Kuehne and Kevin Duarte and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me_Supp.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr21_poster_v2.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr_2021_5min.mp4},
year = {2021},
date = {2021-06-19},
urldate = {2021-06-19},
publisher = {IEEE Conference on Computer Vision and Pattern Recognition},
keywords = {Capsule Networks, CVPR, Grounding, VQA},
pubstate = {published},
tppubtype = {conference}
}
2020
McIntosh, Bruce; Duarte, Kevin; Rawat, Yogesh Singh; Shah, Mubarak
Visual-textual Capsule Routing for Text-based Video Segmentation Conference
IEEE Conference on Computer Vision and Pattern Recognition (Oral), 2020.
Abstract | Tags: Capsule Networks, CVPR, Video Object Segmentation | Links:
@conference{Duarte2020,
title = {Visual-textual Capsule Routing for Text-based Video Segmentation},
author = {Bruce McIntosh and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation_Supplementary.zip
https://www.crcv.ucf.edu/research/projects/visual-textual-capsule-routing-for-text-based-video-segmentation/},
year = {2020},
date = {2020-06-14},
publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},
edition = {(Oral)},
abstract = {Joint understanding of vision and natural language is a challenging problem with a wide range of applications in artificial intelligence. In this work, we focus on integration of video and text for the task of actor and action video segmentation from a sentence. We propose a capsule-based approach which performs pixel-level localization based on a natural language query describing the actor of interest. We encode both the video and textual input in the form of capsules, which provide a more effective representation in comparison with standard convolution based features. Our novel visual-textual routing mechanism allows for the fusion of video and text capsules to successfully localize the actor and action. The existing works on actor-action localization are mainly focused on localization in a single frame instead of the full video. Different from existing works, we propose to perform the localization on all frames of the video. To validate the potential of the proposed network for actor and action video localization, we extend an existing actor-action dataset (A2D) with annotations for all the frames. The experimental evaluation demonstrates the effectiveness of our capsule network for text selective actor and action localization in videos. The proposed method also improves upon the performance of the existing state-of-the art works on single frame-based localization.},
keywords = {Capsule Networks, CVPR, Video Object Segmentation},
pubstate = {published},
tppubtype = {conference}
}
Joint understanding of vision and natural language is a challenging problem with a wide range of applications in artificial intelligence. In this work, we focus on integration of video and text for the task of actor and action video segmentation from a sentence. We propose a capsule-based approach which performs pixel-level localization based on a natural language query describing the actor of interest. We encode both the video and textual input in the form of capsules, which provide a more effective representation in comparison with standard convolution based features. Our novel visual-textual routing mechanism allows for the fusion of video and text capsules to successfully localize the actor and action. The existing works on actor-action localization are mainly focused on localization in a single frame instead of the full video. Different from existing works, we propose to perform the localization on all frames of the video. To validate the potential of the proposed network for actor and action video localization, we extend an existing actor-action dataset (A2D) with annotations for all the frames. The experimental evaluation demonstrates the effectiveness of our capsule network for text selective actor and action localization in videos. The proposed method also improves upon the performance of the existing state-of-the art works on single frame-based localization.
McIntosh, Bruce; Duarte, Kevin; Rawat, Yogesh Singh; Shah, Mubarak
Visual-textual Capsule Routing for Text-based Video Segmentation Conference
IEEE Conference on Computer Vision and Pattern Recognition (Oral), 2020.
Abstract | Tags: Capsule Networks, CVPR, Video Object Segmentation | Links:
@conference{Duarte2020b,
title = {Visual-textual Capsule Routing for Text-based Video Segmentation},
author = {Bruce McIntosh and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation_Supplementary.zip
https://www.crcv.ucf.edu/research/projects/visual-textual-capsule-routing-for-text-based-video-segmentation/},
year = {2020},
date = {2020-06-14},
publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},
edition = {(Oral)},
abstract = {Joint understanding of vision and natural language is a challenging problem with a wide range of applications in artificial intelligence. In this work, we focus on integration of video and text for the task of actor and action video segmentation from a sentence. We propose a capsule-based approach which performs pixel-level localization based on a natural language query describing the actor of interest. We encode both the video and textual input in the form of capsules, which provide a more effective representation in comparison with standard convolution based features. Our novel visual-textual routing mechanism allows for the fusion of video and text capsules to successfully localize the actor and action. The existing works on actor-action localization are mainly focused on localization in a single frame instead of the full video. Different from existing works, we propose to perform the localization on all frames of the video. To validate the potential of the proposed network for actor and action video localization, we extend an existing actor-action dataset (A2D) with annotations for all the frames. The experimental evaluation demonstrates the effectiveness of our capsule network for text selective actor and action localization in videos. The proposed method also improves upon the performance of the existing state-of-the art works on single frame-based localization.},
keywords = {Capsule Networks, CVPR, Video Object Segmentation},
pubstate = {published},
tppubtype = {conference}
}
Joint understanding of vision and natural language is a challenging problem with a wide range of applications in artificial intelligence. In this work, we focus on integration of video and text for the task of actor and action video segmentation from a sentence. We propose a capsule-based approach which performs pixel-level localization based on a natural language query describing the actor of interest. We encode both the video and textual input in the form of capsules, which provide a more effective representation in comparison with standard convolution based features. Our novel visual-textual routing mechanism allows for the fusion of video and text capsules to successfully localize the actor and action. The existing works on actor-action localization are mainly focused on localization in a single frame instead of the full video. Different from existing works, we propose to perform the localization on all frames of the video. To validate the potential of the proposed network for actor and action video localization, we extend an existing actor-action dataset (A2D) with annotations for all the frames. The experimental evaluation demonstrates the effectiveness of our capsule network for text selective actor and action localization in videos. The proposed method also improves upon the performance of the existing state-of-the art works on single frame-based localization.
Edraki, Marzieh; Rahnavard, Nazanin; Shah, Mubarak
Subspace Capsule Network Conference
34th Conference on Association for the Advancement of Artificial Intelligence (AAAI 2020), New York, USA, 2020.
Tags: Capsule Networks | Links:
@conference{Edraki2019,
title = {Subspace Capsule Network},
author = {Marzieh Edraki and Nazanin Rahnavard and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/03/Projects_SubspaceCapsuleNetwork.pdf
https://www.crcv.ucf.edu/research/projects/subspace-capsule-network/},
year = {2020},
date = {2020-02-07},
booktitle = {34th Conference on Association for the Advancement of Artificial Intelligence (AAAI 2020), New York, USA},
keywords = {Capsule Networks},
pubstate = {published},
tppubtype = {conference}
}
Edraki, Marzieh; Rahnavard, Nazanin; Shah, Mubarak
Subspace Capsule Network Conference
34th Conference on Association for the Advancement of Artificial Intelligence (AAAI 2020), New York, USA, 2020.
Tags: Capsule Networks | Links:
@conference{Edraki2019b,
title = {Subspace Capsule Network},
author = {Marzieh Edraki and Nazanin Rahnavard and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/03/Projects_SubspaceCapsuleNetwork.pdf
https://www.crcv.ucf.edu/research/projects/subspace-capsule-network/},
year = {2020},
date = {2020-02-07},
booktitle = {34th Conference on Association for the Advancement of Artificial Intelligence (AAAI 2020), New York, USA},
keywords = {Capsule Networks},
pubstate = {published},
tppubtype = {conference}
}