2021
Duarte, Kevin
Capsule Networks for Video Understanding PhD Thesis
University of Central Florida, 2021.
Tags: Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video | Links:
@phdthesis{Duarte2021b,
title = {Capsule Networks for Video Understanding},
author = {Kevin Duarte},
url = {https://www.crcv.ucf.edu/people/alumni/#:~:text=Capsule%20Networks%20for%20Video%20Understanding},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
school = {University of Central Florida},
keywords = {Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video},
pubstate = {published},
tppubtype = {phdthesis}
}
Duarte, Kevin; Chen, Brian; Shvetsova, Nina; Rouditchenko, Andrew; Thomas, Samuel; Liu, Alexander; Harwath, David; Glass, James; Kuehne, Hilde; Shah, Mubarak
Routing with Self-Attention for Multimodal Capsule Networks Unpublished
arXiv preprint arXiv:2112.00775, 2021.
Abstract | Tags: Audio, Capsule Networks, Multi-Modal Learning, Text and Video | Links:
@unpublished{nokey_114,
title = {Routing with Self-Attention for Multimodal Capsule Networks},
author = {Kevin Duarte and Brian Chen and Nina Shvetsova and Andrew Rouditchenko and Samuel Thomas and Alexander Liu and David Harwath and James Glass and Hilde Kuehne and Mubarak Shah},
editor = {arXiv},
url = {https://arxiv.org/pdf/2112.00775.pdf
https://arxiv.org/abs/2112.00775},
doi = {https://doi.org/10.48550/arXiv.2112.00775},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
abstract = {The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.},
howpublished = {arXiv preprint arXiv:2112.00775},
keywords = {Audio, Capsule Networks, Multi-Modal Learning, Text and Video},
pubstate = {published},
tppubtype = {unpublished}
}
The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.
Duarte, Kevin
Capsule Networks for Video Understanding PhD Thesis
University of Central Florida, 2021.
Tags: Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video | Links:
@phdthesis{Duarte2021bb,
title = {Capsule Networks for Video Understanding},
author = {Kevin Duarte},
url = {https://www.crcv.ucf.edu/people/alumni/#:~:text=Capsule%20Networks%20for%20Video%20Understanding},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
school = {University of Central Florida},
keywords = {Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video},
pubstate = {published},
tppubtype = {phdthesis}
}
Duarte, Kevin; Chen, Brian; Shvetsova, Nina; Rouditchenko, Andrew; Thomas, Samuel; Liu, Alexander; Harwath, David; Glass, James; Kuehne, Hilde; Shah, Mubarak
Routing with Self-Attention for Multimodal Capsule Networks Unpublished
arXiv preprint arXiv:2112.00775, 2021.
Abstract | Tags: Audio, Capsule Networks, Multi-Modal Learning, Text and Video | Links:
@unpublished{nokey_162,
title = {Routing with Self-Attention for Multimodal Capsule Networks},
author = {Kevin Duarte and Brian Chen and Nina Shvetsova and Andrew Rouditchenko and Samuel Thomas and Alexander Liu and David Harwath and James Glass and Hilde Kuehne and Mubarak Shah},
editor = {arXiv},
url = {https://arxiv.org/pdf/2112.00775.pdf
https://arxiv.org/abs/2112.00775},
doi = {https://doi.org/10.48550/arXiv.2112.00775},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
abstract = {The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.},
howpublished = {arXiv preprint arXiv:2112.00775},
keywords = {Audio, Capsule Networks, Multi-Modal Learning, Text and Video},
pubstate = {published},
tppubtype = {unpublished}
}
The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.