Publications

@misc{kwon_are_2024,

title = {Are LLMs Effective Negotiators? Systematic Evaluation of the Multifaceted Capabilities of LLMs in Negotiation Dialogues},

author = {Deuksin Kwon and Emily Weiss and Tara Kulshrestha and Kushal Chawla and Gale M. Lucas and Jonathan Gratch},

url = {http://arxiv.org/abs/2402.13550},

year  = {2024},

date = {2024-02-01},

urldate = {2024-03-14},

publisher = {arXiv},

abstract = {A successful negotiation demands a deep comprehension of the conversation context, Theory-of-Mind (ToM) skills to infer the partner's motives, as well as strategic reasoning and effective communication, making it challenging for automated systems. Given the remarkable performance of LLMs across a variety of NLP tasks, in this work, we aim to understand how LLMs can advance different aspects of negotiation research, ranging from designing dialogue systems to providing pedagogical feedback and scaling up data collection practices. To this end, we devise a methodology to analyze the multifaceted capabilities of LLMs across diverse dialogue scenarios covering all the time stages of a typical negotiation interaction. Our analysis adds to the increasing evidence for the superiority of GPT-4 across various tasks while also providing insights into specific tasks that remain difficult for LLMs. For instance, the models correlate poorly with human players when making subjective assessments about the negotiation dialogues and often struggle to generate responses that are contextually appropriate as well as strategically advantageous.},

note = {arXiv:2402.13550 [cs]},

keywords = {AI, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{yu_tokenmotion_2024,

title = {TokenMotion: Motion-Guided Vision Transformer for Video Camouflaged Object Detection Via Learnable Token Selection},

author = {Zifan Yu and Erfan Bank Tavakoli and Meida Chen and Suya You and Raghuveer Rao and Sanjeev Agarwal and Fengbo Ren},

url = {http://arxiv.org/abs/2311.02535},

year  = {2024},

date = {2024-02-01},

urldate = {2024-02-21},

publisher = {arXiv},

abstract = {The area of Video Camouflaged Object Detection (VCOD) presents unique challenges in the field of computer vision due to texture similarities between target objects and their surroundings, as well as irregular motion patterns caused by both objects and camera movement. In this paper, we introduce TokenMotion (TMNet), which employs a transformer-based model to enhance VCOD by extracting motion-guided features using a learnable token selection. Evaluated on the challenging MoCA-Mask dataset, TMNet achieves state-of-the-art performance in VCOD. It outperforms the existing state-of-the-art method by a 12.8% improvement in weighted F-measure, an 8.4% enhancement in S-measure, and a 10.7% boost in mean IoU. The results demonstrate the benefits of utilizing motion-guided features via learnable token selection within a transformer-based framework to tackle the intricate task of VCOD.},

note = {arXiv:2311.02535 [cs]},

keywords = {Narrative},

pubstate = {published},

tppubtype = {misc}

}

@misc{ehsanpour_social-mae_2024,

title = {Social-MAE: Social Masked Autoencoder for Multi-person Motion Representation Learning},

author = {Mahsa Ehsanpour and Ian Reid and Hamid Rezatofighi},

url = {https://arxiv.org/abs/2404.05578},

doi = {10.48550/ARXIV.2404.05578},

year  = {2024},

date = {2024-01-01},

urldate = {2024-07-12},

publisher = {arXiv},

abstract = {For a complete comprehension of multi-person scenes, it is essential to go beyond basic tasks like detection and tracking. Higher-level tasks, such as understanding the interactions and social activities among individuals, are also crucial. Progress towards models that can fully understand scenes involving multiple people is hindered by a lack of sufficient annotated data for such high-level tasks. To address this challenge, we introduce Social-MAE, a simple yet effective transformer-based masked autoencoder framework for multi-person human motion data. The framework uses masked modeling to pre-train the encoder to reconstruct masked human joint trajectories, enabling it to learn generalizable and data efficient representations of motion in human crowded scenes. Social-MAE comprises a transformer as the MAE encoder and a lighter-weight transformer as the MAE decoder which operates on multi-person joints' trajectory in the frequency domain. After the reconstruction task, the MAE decoder is replaced with a task-specific decoder and the model is fine-tuned end-to-end for a variety of high-level social tasks. Our proposed model combined with our pre-training approach achieves the state-of-the-art results on various high-level social tasks, including multi-person pose forecasting, social grouping, and social action understanding. These improvements are demonstrated across four popular multi-person datasets encompassing both human 2D and 3D body pose.},

note = {Version Number: 1},

keywords = {Social Simulation},

pubstate = {published},

tppubtype = {misc}

}

@misc{shi_build_2024,

title = {Build Your Own Robot Friend: An Open-Source Learning Module for Accessible and Engaging AI Education},

author = {Zhonghao Shi and Allison O'Connell and Zongjian Li and Siqi Liu and Jennifer Ayissi and Guy Hoffman and Mohammad Soleymani and Maja J. Matarić},

url = {http://arxiv.org/abs/2402.01647},

year  = {2024},

date = {2024-01-01},

urldate = {2024-02-21},

publisher = {arXiv},

abstract = {As artificial intelligence (AI) is playing an increasingly important role in our society and global economy, AI education and literacy have become necessary components in college and K-12 education to prepare students for an AI-powered society. However, current AI curricula have not yet been made accessible and engaging enough for students and schools from all socio-economic backgrounds with different educational goals. In this work, we developed an open-source learning module for college and high school students, which allows students to build their own robot companion from the ground up. This open platform can be used to provide hands-on experience and introductory knowledge about various aspects of AI, including robotics, machine learning (ML), software engineering, and mechanical engineering. Because of the social and personal nature of a socially assistive robot companion, this module also puts a special emphasis on human-centered AI, enabling students to develop a better understanding of human-AI interaction and AI ethics through hands-on learning activities. With open-source documentation, assembling manuals and affordable materials, students from different socio-economic backgrounds can personalize their learning experience based on their individual educational goals. To evaluate the student-perceived quality of our module, we conducted a usability testing workshop with 15 college students recruited from a minority-serving institution. Our results indicate that our AI module is effective, easy-to-follow, and engaging, and it increases student interest in studying AI/ML and robotics in the future. We hope that this work will contribute toward accessible and engaging AI education in human-AI interaction for college and high school students.},

note = {arXiv:2402.01647 [cs]},

keywords = {Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{cho_can_2023,

title = {Can Language Model Moderators Improve the Health of Online Discourse?},

author = {Hyundong Cho and Shuai Liu and Taiwei Shi and Darpan Jain and Basem Rizk and Yuyang Huang and Zixun Lu and Nuan Wen and Jonathan Gratch and Emilio Ferrara and Jonathan May},

url = {http://arxiv.org/abs/2311.10781},

year  = {2023},

date = {2023-11-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {Human moderation of online conversation is essential to maintaining civility and focus in a dialogue, but is challenging to scale and harmful to moderators. The inclusion of sophisticated natural language generation modules as a force multiplier aid moderators is a tantalizing prospect, but adequate evaluation approaches have so far been elusive. In this paper, we establish a systematic definition of conversational moderation effectiveness through a multidisciplinary lens that incorporates insights from social science. We then propose a comprehensive evaluation framework that uses this definition to asses models' moderation capabilities independently of human intervention. With our framework, we conduct the first known study of conversational dialogue models as moderators, finding that appropriately prompted models can provide specific and fair feedback on toxic behavior but struggle to influence users to increase their levels of respect and cooperation.},

note = {arXiv:2311.10781 [cs]},

keywords = {AI, Dialogue, DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{yang_context_2023,

title = {Context Unlocks Emotions: Text-based Emotion Classification Dataset Auditing with Large Language Models},

author = {Daniel Yang and Aditya Kommineni and Mohammad Alshehri and Nilamadhab Mohanty and Vedant Modi and Jonathan Gratch and Shrikanth Narayanan},

url = {http://arxiv.org/abs/2311.03551},

year  = {2023},

date = {2023-11-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {The lack of contextual information in text data can make the annotation process of text-based emotion classification datasets challenging. As a result, such datasets often contain labels that fail to consider all the relevant emotions in the vocabulary. This misalignment between text inputs and labels can degrade the performance of machine learning models trained on top of them. As re-annotating entire datasets is a costly and time-consuming task that cannot be done at scale, we propose to use the expressive capabilities of large language models to synthesize additional context for input text to increase its alignment with the annotated emotional labels. In this work, we propose a formal definition of textual context to motivate a prompting strategy to enhance such contextual information. We provide both human and empirical evaluation to demonstrate the efficacy of the enhanced context. Our method improves alignment between inputs and their human-annotated labels from both an empirical and human-evaluated standpoint.},

note = {arXiv:2311.03551 [cs]},

keywords = {AI, DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{chang_magicdance_2023,

title = {MagicDance: Realistic Human Dance Video Generation with Motions & Facial Expressions Transfer},

author = {Di Chang and Yichun Shi and Quankai Gao and Jessica Fu and Hongyi Xu and Guoxian Song and Qing Yan and Xiao Yang and Mohammad Soleymani},

url = {http://arxiv.org/abs/2311.12052},

year  = {2023},

date = {2023-11-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {In this work, we propose MagicDance, a diffusion-based model for 2D human motion and facial expression transfer on challenging human dance videos. Specifically, we aim to generate human dance videos of any target identity driven by novel pose sequences while keeping the identity unchanged. To this end, we propose a two-stage training strategy to disentangle human motions and appearance (e.g., facial expressions, skin tone and dressing), consisting of the pretraining of an appearance-control block and fine-tuning of an appearance-pose-joint-control block over human dance poses of the same dataset. Our novel design enables robust appearance control with temporally consistent upper body, facial attributes, and even background. The model also generalizes well on unseen human identities and complex motion sequences without the need for any fine-tuning with additional data with diverse human attributes by leveraging the prior knowledge of image diffusion models. Moreover, the proposed model is easy to use and can be considered as a plug-in module/extension to Stable Diffusion. We also demonstrate the model's ability for zero-shot 2D animation generation, enabling not only the appearance transfer from one identity to another but also allowing for cartoon-like stylization given only pose inputs. Extensive experiments demonstrate our superior performance on the TikTok dataset.},

note = {arXiv:2311.12052 [cs]},

keywords = {DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{prinzing_can_2023-1,

title = {Can AI Agents Help Humans to Connect?},

author = {Michael Prinzing and Catherine Garton and Catherine J. Berman and Jieni Zhou and Taylor Nicole West and Jonathan Gratch and Barbara Fredrickson},

url = {https://osf.io/muq6s},

doi = {10.31234/osf.io/muq6s},

year  = {2023},

date = {2023-10-01},

urldate = {2024-08-13},

abstract = {This paper reports on a pre-registered experiment designed to test whether artificial agents can help people to create more moments of high-quality connection with other humans. Of four pre-registered hypotheses, we found (partial) support for only one.},

keywords = {AI},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_instant_2023,

title = {Instant Photorealistic Style Transfer: A Lightweight and Adaptive Approach},

author = {Rong Liu and Enyu Zhao and Zhiyuan Liu and Andrew Feng and Scott John Easley},

url = {http://arxiv.org/abs/2309.10011},

year  = {2023},

date = {2023-10-01},

urldate = {2024-05-14},

publisher = {arXiv},

abstract = {In this paper, we propose an Instant Photorealistic Style Transfer (IPST) approach, designed to achieve instant photorealistic style transfer on super-resolution inputs without the need for pre-training on pair-wise datasets or imposing extra constraints. Our method utilizes a lightweight StyleNet to enable style transfer from a style image to a content image while preserving non-color information. To further enhance the style transfer process, we introduce an instance-adaptive optimization to prioritize the photorealism of outputs and accelerate the convergence of the style network, leading to a rapid training completion within seconds. Moreover, IPST is well-suited for multi-frame style transfer tasks, as it retains temporal and multi-view consistency of the multi-frame inputs such as video and Neural Radiance Field (NeRF). Experimental results demonstrate that IPST requires less GPU memory usage, offers faster multi-frame transfer speed, and generates photorealistic outputs, making it a promising solution for various photorealistic transfer applications.},

note = {arXiv:2309.10011 [cs, eess]},

keywords = {DTIC, UARC},

pubstate = {published},

tppubtype = {misc}

}

@misc{lukin_navigating_2023,

title = {Navigating to Success in Multi-Modal Human-Robot Collaboration: Analysis and Corpus Release},

author = {Stephanie M. Lukin and Kimberly A. Pollard and Claire Bonial and Taylor Hudson and Ron Arstein and Clare Voss and David Traum},

url = {http://arxiv.org/abs/2310.17568},

year  = {2023},

date = {2023-10-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {Human-guided robotic exploration is a useful approach to gathering information at remote locations, especially those that might be too risky, inhospitable, or inaccessible for humans. Maintaining common ground between the remotely-located partners is a challenge, one that can be facilitated by multi-modal communication. In this paper, we explore how participants utilized multiple modalities to investigate a remote location with the help of a robotic partner. Participants issued spoken natural language instructions and received from the robot: text-based feedback, continuous 2D LIDAR mapping, and upon-request static photographs. We noticed that different strategies were adopted in terms of use of the modalities, and hypothesize that these differences may be correlated with success at several exploration sub-tasks. We found that requesting photos may have improved the identification and counting of some key entities (doorways in particular) and that this strategy did not hinder the amount of overall area exploration. Future work with larger samples may reveal the effects of more nuanced photo and dialogue strategies, which can inform the training of robotic agents. Additionally, we announce the release of our unique multi-modal corpus of human-robot communication in an exploration context: SCOUT, the Situated Corpus on Understanding Transactions.},

note = {arXiv:2310.17568 [cs]},

keywords = {DTIC, Natural Language, UARC},

pubstate = {published},

tppubtype = {misc}

}

@misc{chawla_be_2023,

title = {Be Selfish, But Wisely: Investigating the Impact of Agent Personality in Mixed-Motive Human-Agent Interactions},

author = {Kushal Chawla and Ian Wu and Yu Rong and Gale M. Lucas and Jonathan Gratch},

url = {http://arxiv.org/abs/2310.14404},

year  = {2023},

date = {2023-10-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {A natural way to design a negotiation dialogue system is via self-play RL: train an agent that learns to maximize its performance by interacting with a simulated user that has been designed to imitate human-human dialogue data. Although this procedure has been adopted in prior work, we find that it results in a fundamentally flawed system that fails to learn the value of compromise in a negotiation, which can often lead to no agreements (i.e., the partner walking away without a deal), ultimately hurting the model's overall performance. We investigate this observation in the context of the DealOrNoDeal task, a multi-issue negotiation over books, hats, and balls. Grounded in negotiation theory from Economics, we modify the training procedure in two novel ways to design agents with diverse personalities and analyze their performance with human partners. We find that although both techniques show promise, a selfish agent, which maximizes its own performance while also avoiding walkaways, performs superior to other variants by implicitly learning to generate value for both itself and the negotiation partner. We discuss the implications of our findings for what it means to be a successful negotiation dialogue system and how these systems should be designed in the future.},

note = {arXiv:2310.14404 [cs]},

keywords = {Dialogue, DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{tran_privacy-preserving_2023,

title = {Privacy-preserving Representation Learning for Speech Understanding},

author = {Minh Tran and Mohammad Soleymani},

url = {http://arxiv.org/abs/2310.17194},

year  = {2023},

date = {2023-10-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {Existing privacy-preserving speech representation learning methods target a single application domain. In this paper, we present a novel framework to anonymize utterance-level speech embeddings generated by pre-trained encoders and show its effectiveness for a range of speech classification tasks. Specifically, given the representations from a pre-trained encoder, we train a Transformer to estimate the representations for the same utterances spoken by other speakers. During inference, the extracted representations can be converted into different identities to preserve privacy. We compare the results with the voice anonymization baselines from the VoicePrivacy 2022 challenge. We evaluate our framework on speaker identification for privacy and emotion recognition, depression classification, and intent classification for utility. Our method outperforms the baselines on privacy and utility in paralinguistic tasks and achieves comparable performance for intent classification.},

note = {arXiv:2310.17194 [eess]},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{yin_fg-net_2023,

title = {FG-Net: Facial Action Unit Detection with Generalizable Pyramidal Features},

author = {Yufeng Yin and Di Chang and Guoxian Song and Shen Sang and Tiancheng Zhi and Jing Liu and Linjie Luo and Mohammad Soleymani},

url = {http://arxiv.org/abs/2308.12380},

year  = {2023},

date = {2023-08-01},

urldate = {2024-02-21},

publisher = {arXiv},

abstract = {Automatic detection of facial Action Units (AUs) allows for objective facial expression analysis. Due to the high cost of AU labeling and the limited size of existing benchmarks, previous AU detection methods tend to overfit the dataset, resulting in a significant performance loss when evaluated across corpora. To address this problem, we propose FG-Net for generalizable facial action unit detection. Specifically, FG-Net extracts feature maps from a StyleGAN2 model pre-trained on a large and diverse face image dataset. Then, these features are used to detect AUs with a Pyramid CNN Interpreter, making the training efficient and capturing essential local features. The proposed FG-Net achieves a strong generalization ability for heatmap-based AU detection thanks to the generalizable and semantic-rich features extracted from the pre-trained generative model. Extensive experiments are conducted to evaluate within- and cross-corpus AU detection with the widely-used DISFA and BP4D datasets. Compared with the state-of-the-art, the proposed method achieves superior cross-domain performance while maintaining competitive within-domain performance. In addition, FG-Net is data-efficient and achieves competitive performance even when trained on 1000 samples. Our code will be released at textbackslashurlhttps://github.com/ihp-lab/FG-Net},

note = {arXiv:2308.12380 [cs]},

keywords = {DTIC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{chang_libreface_2023,

title = {LibreFace: An Open-Source Toolkit for Deep Facial Expression Analysis},

author = {Di Chang and Yufeng Yin and Zongjian Li and Minh Tran and Mohammad Soleymani},

url = {http://arxiv.org/abs/2308.10713},

year  = {2023},

date = {2023-08-01},

urldate = {2024-02-21},

publisher = {arXiv},

abstract = {Facial expression analysis is an important tool for human-computer interaction. In this paper, we introduce LibreFace, an open-source toolkit for facial expression analysis. This open-source toolbox offers real-time and offline analysis of facial behavior through deep learning models, including facial action unit (AU) detection, AU intensity estimation, and facial expression recognition. To accomplish this, we employ several techniques, including the utilization of a large-scale pre-trained network, feature-wise knowledge distillation, and task-specific fine-tuning. These approaches are designed to effectively and accurately analyze facial expressions by leveraging visual information, thereby facilitating the implementation of real-time interactive applications. In terms of Action Unit (AU) intensity estimation, we achieve a Pearson Correlation Coefficient (PCC) of 0.63 on DISFA, which is 7% higher than the performance of OpenFace 2.0 while maintaining highly-efficient inference that runs two times faster than OpenFace 2.0. Despite being compact, our model also demonstrates competitive performance to state-of-the-art facial expression analysis methods on AffecNet, FFHQ, and RAF-DB. Our code will be released at https://github.com/ihp-lab/LibreFace},

note = {arXiv:2308.10713 [cs]},

keywords = {DTIC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}