2021
Duarte, Kevin
Capsule Networks for Video Understanding PhD Thesis
University of Central Florida, 2021.
Tags: Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video | Links:
@phdthesis{Duarte2021b,
title = {Capsule Networks for Video Understanding},
author = {Kevin Duarte},
url = {https://www.crcv.ucf.edu/people/alumni/#:~:text=Capsule%20Networks%20for%20Video%20Understanding},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
school = {University of Central Florida},
keywords = {Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video},
pubstate = {published},
tppubtype = {phdthesis}
}
Duarte, Kevin; Chen, Brian; Shvetsova, Nina; Rouditchenko, Andrew; Thomas, Samuel; Liu, Alexander; Harwath, David; Glass, James; Kuehne, Hilde; Shah, Mubarak
Routing with Self-Attention for Multimodal Capsule Networks Unpublished
arXiv preprint arXiv:2112.00775, 2021.
Abstract | Tags: Audio, Capsule Networks, Multi-Modal Learning, Text and Video | Links:
@unpublished{nokey_114,
title = {Routing with Self-Attention for Multimodal Capsule Networks},
author = {Kevin Duarte and Brian Chen and Nina Shvetsova and Andrew Rouditchenko and Samuel Thomas and Alexander Liu and David Harwath and James Glass and Hilde Kuehne and Mubarak Shah},
editor = {arXiv},
url = {https://arxiv.org/pdf/2112.00775.pdf
https://arxiv.org/abs/2112.00775},
doi = {https://doi.org/10.48550/arXiv.2112.00775},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
abstract = {The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.},
howpublished = {arXiv preprint arXiv:2112.00775},
keywords = {Audio, Capsule Networks, Multi-Modal Learning, Text and Video},
pubstate = {published},
tppubtype = {unpublished}
}
The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.
Duarte, Kevin
Capsule Networks for Video Understanding PhD Thesis
University of Central Florida, 2021.
Tags: Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video | Links:
@phdthesis{Duarte2021bb,
title = {Capsule Networks for Video Understanding},
author = {Kevin Duarte},
url = {https://www.crcv.ucf.edu/people/alumni/#:~:text=Capsule%20Networks%20for%20Video%20Understanding},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
school = {University of Central Florida},
keywords = {Action Detection, Capsule Networks, Multi-Modal Learning, Ph.D. Dissertation, Text and Video},
pubstate = {published},
tppubtype = {phdthesis}
}
Duarte, Kevin; Chen, Brian; Shvetsova, Nina; Rouditchenko, Andrew; Thomas, Samuel; Liu, Alexander; Harwath, David; Glass, James; Kuehne, Hilde; Shah, Mubarak
Routing with Self-Attention for Multimodal Capsule Networks Unpublished
arXiv preprint arXiv:2112.00775, 2021.
Abstract | Tags: Audio, Capsule Networks, Multi-Modal Learning, Text and Video | Links:
@unpublished{nokey_162,
title = {Routing with Self-Attention for Multimodal Capsule Networks},
author = {Kevin Duarte and Brian Chen and Nina Shvetsova and Andrew Rouditchenko and Samuel Thomas and Alexander Liu and David Harwath and James Glass and Hilde Kuehne and Mubarak Shah},
editor = {arXiv},
url = {https://arxiv.org/pdf/2112.00775.pdf
https://arxiv.org/abs/2112.00775},
doi = {https://doi.org/10.48550/arXiv.2112.00775},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
abstract = {The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.},
howpublished = {arXiv preprint arXiv:2112.00775},
keywords = {Audio, Capsule Networks, Multi-Modal Learning, Text and Video},
pubstate = {published},
tppubtype = {unpublished}
}
The task of multimodal learning has seen a growing interest recently as it allows for training neural architectures based on different modalities such as vision, text, and audio. One challenge in training such models is that they need to jointly learn semantic concepts and their relationships across different input representations. Capsule networks have been shown to perform well in context of capturing the relation between low-level input features and higher-level concepts. However, capsules have so far mainly been used only in small-scale fully supervised settings due to the resource demand of conventional routing algorithms. We present a new multimodal capsule network that allows us to leverage the strength of capsules in the context of a multimodal learning framework on large amounts of video data. To adapt the capsules to large-scale input data, we propose a novel routing by self-attention mechanism that selects relevant capsules which are then used to generate a final joint multimodal feature representation. This allows not only for robust training with noisy video data, but also to scale up the size of the capsule network compared to traditional routing methods while still being computationally efficient. We evaluate the proposed architecture by pretraining it on a large-scale multimodal video dataset and applying it on four datasets in two challenging downstream tasks. Results show that the proposed multimodal capsule network is not only able to improve results compared to other routing techniques, but also achieves competitive performance on the task of multimodal learning.
Chen, Brian; Rouditchenko, Andrew; Duarte, Kevin; Kuehne, Hilde; Thomas, Samuel; Boggust, Angie; Panda, Rameswar; Kingsbury, Brian; Feris, Rogerio; Harwatch, David; Glass, James; Picheny, Michael; Chang, Shih-Fu
Multimodal Clustering Networks for Self-supervised Learning from Unlabeled Videos Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: ICCV, Multi-Modal Learning | Links:
@conference{ChenICCV2021,
title = {Multimodal Clustering Networks for Self-supervised Learning from Unlabeled Videos},
author = {Brian Chen and Andrew Rouditchenko and Kevin Duarte and Hilde Kuehne and Samuel Thomas and Angie Boggust and Rameswar Panda and Brian Kingsbury and Rogerio Feris and David Harwatch and James Glass and Michael Picheny and Shih-Fu Chang},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965-supp.pdf},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {Multimodal self-supervised learning is getting more and more attention as it allows not only to train large networks without human supervision but also to search and retrieve data across various modalities. In this context, this paper proposes a framework that, starting from a pre- trained backbone, learns a common multimodal embedding space that, in addition to sharing representations across different modalities, enforces a grouping of semantically similar instances.
To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,
namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets.},
keywords = {ICCV, Multi-Modal Learning},
pubstate = {published},
tppubtype = {conference}
}
Multimodal self-supervised learning is getting more and more attention as it allows not only to train large networks without human supervision but also to search and retrieve data across various modalities. In this context, this paper proposes a framework that, starting from a pre- trained backbone, learns a common multimodal embedding space that, in addition to sharing representations across different modalities, enforces a grouping of semantically similar instances.
To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,
namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets.
To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,
namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets.
Chen, Brian; Rouditchenko, Andrew; Duarte, Kevin; Kuehne, Hilde; Thomas, Samuel; Boggust, Angie; Panda, Rameswar; Kingsbury, Brian; Feris, Rogerio; Harwatch, David; Glass, James; Picheny, Michael; Chang, Shih-Fu
Multimodal Clustering Networks for Self-supervised Learning from Unlabeled Videos Conference
International Conference on Computer Vision, 2021.
Abstract | Tags: ICCV, Multi-Modal Learning | Links:
@conference{ChenICCV2021b,
title = {Multimodal Clustering Networks for Self-supervised Learning from Unlabeled Videos},
author = {Brian Chen and Andrew Rouditchenko and Kevin Duarte and Hilde Kuehne and Samuel Thomas and Angie Boggust and Rameswar Panda and Brian Kingsbury and Rogerio Feris and David Harwatch and James Glass and Michael Picheny and Shih-Fu Chang},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/02965-supp.pdf},
year = {2021},
date = {2021-10-11},
urldate = {2021-10-11},
booktitle = {International Conference on Computer Vision},
abstract = {Multimodal self-supervised learning is getting more and more attention as it allows not only to train large networks without human supervision but also to search and retrieve data across various modalities. In this context, this paper proposes a framework that, starting from a pre- trained backbone, learns a common multimodal embedding space that, in addition to sharing representations across different modalities, enforces a grouping of semantically similar instances.
To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,
namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets.},
keywords = {ICCV, Multi-Modal Learning},
pubstate = {published},
tppubtype = {conference}
}
Multimodal self-supervised learning is getting more and more attention as it allows not only to train large networks without human supervision but also to search and retrieve data across various modalities. In this context, this paper proposes a framework that, starting from a pre- trained backbone, learns a common multimodal embedding space that, in addition to sharing representations across different modalities, enforces a grouping of semantically similar instances.
To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,
namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets.
To this end, we extend the concept of instance-level contrastive learning with a multimodal lustering step in the training pipeline to capture semantic similarities across modalities. The resulting embedding space enables retrieval of samples across all modalities, even from unseen datasets and different domains. To evaluate our approach, we train our model on the HowTo100M dataset and evaluate its zero-shot retrieval capabilities in two challenging domains,
namely text-to-video retrieval, and temporal action localization, showing state-of-the-art results on four different datasets.
0000
Test Bachelor Thesis
0000.
Tags: Crowd Analysis, Damage Assessment, DIVA, Drone Video Analysis, EEG/Brain, Incremental Learning, Learning with Less Labels, LIDAR, Multi-Modal Learning, Noisy Labels, Open World, Out of Distribution (OOD), Privacy Preservation, Pseudo-Labeling, SAR, Self-Joint Learning, Semi-supervised learning, Tiny Actions, Video Synopsis, Video Synthesis, View-Invariance, Weakly Supervised Learning, Wildlife Preservation
@bachelorthesis{nokey_135,
title = {Test},
keywords = {Crowd Analysis, Damage Assessment, DIVA, Drone Video Analysis, EEG/Brain, Incremental Learning, Learning with Less Labels, LIDAR, Multi-Modal Learning, Noisy Labels, Open World, Out of Distribution (OOD), Privacy Preservation, Pseudo-Labeling, SAR, Self-Joint Learning, Semi-supervised learning, Tiny Actions, Video Synopsis, Video Synthesis, View-Invariance, Weakly Supervised Learning, Wildlife Preservation},
pubstate = {published},
tppubtype = {bachelorthesis}
}
Test Bachelor Thesis
0000.
Tags: Crowd Analysis, Damage Assessment, DIVA, Drone Video Analysis, EEG/Brain, Incremental Learning, Learning with Less Labels, LIDAR, Multi-Modal Learning, Noisy Labels, Open World, Out of Distribution (OOD), Privacy Preservation, Pseudo-Labeling, SAR, Self-Joint Learning, Semi-supervised learning, Tiny Actions, Video Synopsis, Video Synthesis, View-Invariance, Weakly Supervised Learning, Wildlife Preservation
@bachelorthesis{nokey_171,
title = {Test},
keywords = {Crowd Analysis, Damage Assessment, DIVA, Drone Video Analysis, EEG/Brain, Incremental Learning, Learning with Less Labels, LIDAR, Multi-Modal Learning, Noisy Labels, Open World, Out of Distribution (OOD), Privacy Preservation, Pseudo-Labeling, SAR, Self-Joint Learning, Semi-supervised learning, Tiny Actions, Video Synopsis, Video Synthesis, View-Invariance, Weakly Supervised Learning, Wildlife Preservation},
pubstate = {published},
tppubtype = {bachelorthesis}
}