Publications

@misc{jalal-kamali_predicting_2025,

title = {Predicting Team Performance from Communications in Simulated Search-and-Rescue},

author = {Ali Jalal-Kamali and Nikolos Gurney and David Pynadath},

url = {http://arxiv.org/abs/2503.03791},

doi = {10.48550/arXiv.2503.03791},

year  = {2025},

date = {2025-03-01},

urldate = {2025-03-18},

publisher = {arXiv},

abstract = {Understanding how individual traits influence team performance is valuable, but these traits are not always directly observable. Prior research has inferred traits like trust from behavioral data. We analyze conversational data to identify team traits and their correlation with teaming outcomes. Using transcripts from a Minecraft-based search-and-rescue experiment, we apply topic modeling and clustering to uncover key interaction patterns. Our findings show that variations in teaming outcomes can be explained through these inferences, with different levels of predictive power derived from individual traits and team dynamics.},

note = {arXiv:2503.03791 [cs]},

keywords = {AI, DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_investigating_2025,

title = {Investigating Role of Personal Factors in Shaping Responses to Active Shooter Incident using Machine Learning},

author = {Ruying Liu and Burçin Becerik-Gerber and Gale M. Lucas},

url = {http://arxiv.org/abs/2503.05719},

doi = {10.48550/arXiv.2503.05719},

year  = {2025},

date = {2025-02-01},

urldate = {2025-03-18},

publisher = {arXiv},

abstract = {This study bridges the knowledge gap on how personal factors affect building occupants' responses in active shooter situations by applying interpretable machine learning methods to data from 107 participants. The personal factors studied are training methods, prior training experience, sense of direction, and gender. The response performance measurements consist of decisions (run, hide, multiple), vulnerability (corresponding to the time a participant is visible to a shooter), and pre-evacuation time. The results indicate that the propensity to run significantly determines overall response strategies, overshadowing vulnerability, and pre-evacuation time. The training method is a critical factor where VR-based training leads to better responses than video-based training. A better sense of direction and previous training experience are correlated with a greater propensity to run and less vulnerability. Gender slightly influences decisions and vulnerability but significantly impacts pre-evacuation time, with females evacuating slower, potentially due to higher risk perception. This study underscores the importance of personal factors in shaping responses to active shooter incidents.},

note = {arXiv:2503.05719 [cs]},

keywords = {DTIC, Social Simulation, VR},

pubstate = {published},

tppubtype = {misc}

}

@misc{siniukov_towards_2025,

title = {Towards a Generalizable Speech Marker for Parkinson's Disease Diagnosis},

author = {Maksim Siniukov and Ellie Xing and Sanaz and Attaripour Isfahani and Mohammad Soleymani},

url = {https://arxiv.org/abs/2501.03581},

doi = {10.48550/ARXIV.2501.03581},

year  = {2025},

date = {2025-01-01},

urldate = {2025-01-14},

publisher = {arXiv},

abstract = {Parkinson's Disease (PD) is a neurodegenerative disorder characterized by motor symptoms, including altered voice production in the early stages. Early diagnosis is crucial not only to improve PD patients' quality of life but also to enhance the efficacy of potential disease-modifying therapies during early neurodegeneration, a window often missed by current diagnostic tools. In this paper, we propose a more generalizable approach to PD recognition through domain adaptation and self-supervised learning. We demonstrate the generalization capabilities of the proposed approach across diverse datasets in different languages. Our approach leverages HuBERT, a large deep neural network originally trained for speech recognition and further trains it on unlabeled speech data from a population that is similar to the target group, i.e., the elderly, in a self-supervised manner. The model is then fine-tuned and adapted for use across different datasets in multiple languages, including English, Italian, and Spanish. Evaluations on four publicly available PD datasets demonstrate the model's efficacy, achieving an average specificity of 92.1% and an average sensitivity of 91.2%. This method offers objective and consistent evaluations across large populations, addressing the variability inherent in human assessments and providing a non-invasive, cost-effective and accessible diagnostic option.},

note = {Version Number: 1},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_deformable_2025,

title = {Deformable Beta Splatting},

author = {Rong Liu and Dylan Sun and Meida Chen and Yue Wang and Andrew Feng},

url = {http://arxiv.org/abs/2501.18630},

doi = {10.48550/arXiv.2501.18630},

year  = {2025},

date = {2025-01-01},

urldate = {2025-02-20},

publisher = {arXiv},

abstract = {3D Gaussian Splatting (3DGS) has advanced radiance field reconstruction by enabling real-time rendering. However, its reliance on Gaussian kernels for geometry and low-order Spherical Harmonics (SH) for color encoding limits its ability to capture complex geometries and diverse colors. We introduce Deformable Beta Splatting (DBS), a deformable and compact approach that enhances both geometry and color representation. DBS replaces Gaussian kernels with deformable Beta Kernels, which offer bounded support and adaptive frequency control to capture fine geometric details with higher fidelity while achieving better memory efficiency. In addition, we extended the Beta Kernel to color encoding, which facilitates improved representation of diffuse and specular components, yielding superior results compared to SH-based methods. Furthermore, Unlike prior densification techniques that depend on Gaussian properties, we mathematically prove that adjusting regularized opacity alone ensures distribution-preserved Markov chain Monte Carlo (MCMC), independent of the splatting kernel type. Experimental results demonstrate that DBS achieves state-of-the-art visual quality while utilizing only 45% of the parameters and rendering 1.5x faster than 3DGS-based methods. Notably, for the first time, splatting-based methods outperform state-of-the-art Neural Radiance Fields, highlighting the superior performance and efficiency of DBS for real-time radiance field rendering.},

note = {arXiv:2501.18630 [cs]},

keywords = {DTIC, Narrative},

pubstate = {published},

tppubtype = {misc}

}

@misc{roemmele_test-taking_2024-1,

title = {From Test-Taking to Test-Making: Examining LLM Authoring of Commonsense Assessment Items},

author = {Melissa Roemmele and Andrew S. Gordon},

url = {https://arxiv.org/abs/2410.14897},

doi = {10.48550/ARXIV.2410.14897},

year  = {2024},

date = {2024-10-01},

urldate = {2024-12-05},

publisher = {arXiv},

abstract = {LLMs can now perform a variety of complex writing tasks. They also excel in answering questions pertaining to natural language inference and commonsense reasoning. Composing these questions is itself a skilled writing task, so in this paper we consider LLMs as authors of commonsense assessment items. We prompt LLMs to generate items in the style of a prominent benchmark for commonsense reasoning, the Choice of Plausible Alternatives (COPA). We examine the outcome according to analyses facilitated by the LLMs and human annotation. We find that LLMs that succeed in answering the original COPA benchmark are also more successful in authoring their own items.},

note = {Version Number: 1},

keywords = {DTIC, Learning Sciences},

pubstate = {published},

tppubtype = {misc}

}

@misc{gao_skyeyes_2024,

title = {Skyeyes: Ground Roaming using Aerial View Images},

author = {Zhiyuan Gao and Wenbin Teng and Gonglin Chen and Jinsen Wu and Ningli Xu and Rongjun Qin and Andrew Feng and Yajie Zhao},

url = {https://arxiv.org/abs/2409.16685},

doi = {10.48550/ARXIV.2409.16685},

year  = {2024},

date = {2024-09-01},

urldate = {2025-01-16},

publisher = {arXiv},

abstract = {Integrating aerial imagery-based scene generation into applications like autonomous driving and gaming enhances realism in 3D environments, but challenges remain in creating detailed content for occluded areas and ensuring real-time, consistent rendering. In this paper, we introduce Skyeyes, a novel framework that can generate photorealistic sequences of ground view images using only aerial view inputs, thereby creating a ground roaming experience. More specifically, we combine a 3D representation with a view consistent generation model, which ensures coherence between generated images. This method allows for the creation of geometrically consistent ground view images, even with large view gaps. The images maintain improved spatial-temporal coherence and realism, enhancing scene comprehension and visualization from aerial perspectives. To the best of our knowledge, there are no publicly available datasets that contain pairwise geo-aligned aerial and ground view imagery. Therefore, we build a large, synthetic, and geo-aligned dataset using Unreal Engine. Both qualitative and quantitative analyses on this synthetic dataset display superior results compared to other leading synthesis approaches. See the project page for more results: https://chaoren2357.github.io/website-skyeyes/.},

note = {Version Number: 1},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{chen_geometry-aware_2024,

title = {Geometry-aware Feature Matching for Large-Scale Structure from Motion},

author = {Gonglin Chen and Jinsen Wu and Haiwei Chen and Wenbin Teng and Zhiyuan Gao and Andrew Feng and Rongjun Qin and Yajie Zhao},

url = {https://arxiv.org/abs/2409.02310},

doi = {10.48550/ARXIV.2409.02310},

year  = {2024},

date = {2024-09-01},

urldate = {2025-01-16},

publisher = {arXiv},

abstract = {Establishing consistent and dense correspondences across multiple images is crucial for Structure from Motion (SfM) systems. Significant view changes, such as air-to-ground with very sparse view overlap, pose an even greater challenge to the correspondence solvers. We present a novel optimization-based approach that significantly enhances existing feature matching methods by introducing geometry cues in addition to color cues. This helps fill gaps when there is less overlap in large-scale scenarios. Our method formulates geometric verification as an optimization problem, guiding feature matching within detector-free methods and using sparse correspondences from detector-based methods as anchor points. By enforcing geometric constraints via the Sampson Distance, our approach ensures that the denser correspondences from detector-free methods are geometrically consistent and more accurate. This hybrid strategy significantly improves correspondence density and accuracy, mitigates multi-view inconsistencies, and leads to notable advancements in camera pose accuracy and point cloud density. It outperforms state-of-the-art feature matching methods on benchmark datasets and enables feature matching in challenging extreme large-scale settings.},

note = {Version Number: 3},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{tak_gpt-4_2024,

title = {GPT-4 Emulates Average-Human Emotional Cognition from a Third-Person Perspective},

author = {Ala N. Tak and Jonathan Gratch},

url = {http://arxiv.org/abs/2408.13718},

year  = {2024},

date = {2024-08-01},

urldate = {2024-09-17},

publisher = {arXiv},

abstract = {This paper extends recent investigations on the emotional reasoning abilities of Large Language Models (LLMs). Current research on LLMs has not directly evaluated the distinction between how LLMs predict the self-attribution of emotions and the perception of others' emotions. We first look at carefully crafted emotion-evoking stimuli, originally designed to find patterns of brain neural activity representing fine-grained inferred emotional attributions of others. We show that GPT-4 is especially accurate in reasoning about such stimuli. This suggests LLMs agree with humans' attributions of others' emotions in stereotypical scenarios remarkably more than self-attributions of emotions in idiosyncratic situations. To further explore this, our second study utilizes a dataset containing annotations from both the author and a third-person perspective. We find that GPT-4's interpretations align more closely with human judgments about the emotions of others than with self-assessments. Notably, conventional computational models of emotion primarily rely on self-reported ground truth as the gold standard. However, an average observer's standpoint, which LLMs appear to have adopted, might be more relevant for many downstream applications, at least in the absence of individual information and adequate safety considerations.},

note = {arXiv:2408.13718 [cs]},

keywords = {DTIC, Emotions},

pubstate = {published},

tppubtype = {misc}

}

@misc{han_knowledge-based_2024,

title = {Knowledge-based Emotion Recognition using Large Language Models},

author = {Bin Han and Cleo Yau and Su Lei and Jonathan Gratch},

url = {http://arxiv.org/abs/2408.04123},

year  = {2024},

date = {2024-08-01},

urldate = {2024-08-15},

publisher = {arXiv},

abstract = {Emotion recognition in social situations is a complex task that requires integrating information from both facial expressions and the situational context. While traditional approaches to automatic emotion recognition have focused on decontextualized signals, recent research emphasizes the importance of context in shaping emotion perceptions. This paper contributes to the emerging field of context-based emotion recognition by leveraging psychological theories of human emotion perception to inform the design of automated methods. We propose an approach that combines emotion recognition methods with Bayesian Cue Integration (BCI) to integrate emotion inferences from decontextualized facial expressions and contextual knowledge inferred via Large-language Models. We test this approach in the context of interpreting facial expressions during a social task, the prisoner's dilemma. Our results provide clear support for BCI across a range of automatic emotion recognition methods. The best automated method achieved results comparable to human observers, suggesting the potential for this approach to advance the field of affective computing.},

note = {arXiv:2408.04123 [cs]},

keywords = {DTIC, Emotions},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_enhancing_2024,

title = {Enhancing Building Safety Design for Active Shooter Incidents: Exploration of Building Exit Parameters using Reinforcement Learning-Based Simulations},

author = {Ruying Liu and Wanjing Wu and Burcin Becerik-Gerber and Gale M. Lucas},

url = {http://arxiv.org/abs/2407.10441},

year  = {2024},

date = {2024-07-01},

urldate = {2024-09-17},

publisher = {arXiv},

abstract = {With the alarming rise in active shooter incidents (ASIs) in the United States, enhancing public safety through building design has become a pressing need. This study proposes a reinforcement learning-based simulation approach addressing gaps in existing research that has neglected the dynamic behaviours of shooters. We developed an autonomous agent to simulate an active shooter within a realistic office environment, aiming to offer insights into the interactions between building design parameters and ASI outcomes. A case study is conducted to quantitatively investigate the impact of building exit numbers (total count of accessible exits) and configuration (arrangement of which exits are available or not) on evacuation and harm rates. Findings demonstrate that greater exit availability significantly improves evacuation outcomes and reduces harm. Exits nearer to the shooter's initial position hold greater importance for accessibility than those farther away. By encompassing dynamic shooter behaviours, this study offers preliminary insights into effective building safety design against evolving threats.},

note = {arXiv:2407.10441 [cs]},

keywords = {DTIC, Virtual Worlds},

pubstate = {published},

tppubtype = {misc}

}

@misc{xiao_localized_2024,

title = {Localized Gaussian Splatting Editing with Contextual Awareness},

author = {Hanyuan Xiao and Yingshu Chen and Huajian Huang and Haolin Xiong and Jing Yang and Pratusha Prasad and Yajie Zhao},

url = {http://arxiv.org/abs/2408.00083},

year  = {2024},

date = {2024-07-01},

urldate = {2024-08-16},

publisher = {arXiv},

abstract = {Recent text-guided generation of individual 3D object has achieved great success using diffusion priors. However, these methods are not suitable for object insertion and replacement tasks as they do not consider the background, leading to illumination mismatches within the environment. To bridge the gap, we introduce an illumination-aware 3D scene editing pipeline for 3D Gaussian Splatting (3DGS) representation. Our key observation is that inpainting by the state-of-the-art conditional 2D diffusion model is consistent with background in lighting. To leverage the prior knowledge from the well-trained diffusion models for 3D object generation, our approach employs a coarse-to-fine objection optimization pipeline with inpainted views. In the first coarse step, we achieve image-to-3D lifting given an ideal inpainted view. The process employs 3D-aware diffusion prior from a view-conditioned diffusion model, which preserves illumination present in the conditioning image. To acquire an ideal inpainted image, we introduce an Anchor View Proposal (AVP) algorithm to find a single view that best represents the scene illumination in target region. In the second Texture Enhancement step, we introduce a novel Depth-guided Inpainting Score Distillation Sampling (DI-SDS), which enhances geometry and texture details with the inpainting diffusion prior, beyond the scope of the 3D-aware diffusion prior knowledge in the first coarse step. DI-SDS not only provides fine-grained texture enhancement, but also urges optimization to respect scene lighting. Our approach efficiently achieves local editing with global illumination consistency without explicitly modeling light transport. We demonstrate robustness of our method by evaluating editing in real scenes containing explicit highlight and shadows, and compare against the state-of-the-art text-to-3D editing methods.},

note = {arXiv:2408.00083 [cs]},

keywords = {DTIC, VGL},

pubstate = {published},

tppubtype = {misc}

}

@misc{huang_psyborg_2024,

title = {PsybORG+: Modeling and Simulation for Detecting Cognitive Biases in Advanced Persistent Threats},

author = {Shuo Huang and Fred Jones and Nikolos Gurney and David Pynadath and Kunal Srivastava and Stoney Trent and Peggy Wu and Quanyan Zhu},

url = {https://arxiv.org/abs/2408.01310},

doi = {10.48550/ARXIV.2408.01310},

year  = {2024},

date = {2024-07-01},

urldate = {2024-12-05},

publisher = {arXiv},

abstract = {Advanced Persistent Threats (APTs) bring significant challenges to cybersecurity due to their sophisticated and stealthy nature. Traditional cybersecurity measures fail to defend against APTs. Cognitive vulnerabilities can significantly influence attackers' decision-making processes, which presents an opportunity for defenders to exploit. This work introduces PsybORG$ˆ+$, a multi-agent cybersecurity simulation environment designed to model APT behaviors influenced by cognitive vulnerabilities. A classification model is built for cognitive vulnerability inference and a simulator is designed for synthetic data generation. Results show that PsybORG$ˆ+$ can effectively model APT attackers with different loss aversion and confirmation bias levels. The classification model has at least a 0.83 accuracy rate in predicting cognitive vulnerabilities.},

note = {Version Number: 3},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{yin_validation_2024,

title = {Validation and Evacuee Modeling of Virtual Robot-guided Emergency Evacuation Experiments},

author = {Yinxuan Yin and Mollik Nayyar and Daniel Holman and Gale Lucas and Colin Holbrook and Alan Wagner},

url = {https://osf.io/mr78s},

doi = {10.31234/osf.io/mr78s},

year  = {2024},

date = {2024-06-01},

urldate = {2024-09-17},

publisher = {Center for Open Science},

abstract = {Virtual Reality (VR) is an increasingly common tool for investigating human responses to emergency situations. Nonetheless, studies validating and comparing human subject behavior during real world emergencies to their responses in VR are notably rare, and no prior studies have validated whether human emergency responses to guidance from a robot are comparable in VR versus the real world. In the present pre-registered study, we used VR to replicate a previous robot- guided emergency evacuation study conducted in the real world and compared human subject behavior in matched physical and virtual environments. In both environments, human subjects were asked to follow a robot to a location and to then read an article. While reading, a fire alarm sounds. The robot then attempted to guide them to a distant, unfamiliar exit rather than nearby and familiar exits. We observed close correspondences between evacuee exit choice (the robot’s distant exit versus closer exits), evacuation time, and trust in the robot between the VR and physical environments. We further demonstrate that data collected in virtual reality can be used to create accurate motion models (mean error of 0.42 centimeters) predicting evacuee trajectories and locations in real life. Taken together, the results provide evidence for the ecological validity of VR approaches to studying human-robot interaction, particularly robot- guided emergency evacuation.},

keywords = {DTIC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{tran_dyadic_2024,

title = {Dyadic Interaction Modeling for Social Behavior Generation},

author = {Minh Tran and Di Chang and Maksim Siniukov and Mohammad Soleymani},

url = {http://arxiv.org/abs/2403.09069},

year  = {2024},

date = {2024-03-01},

urldate = {2024-03-19},

publisher = {arXiv},

abstract = {Human-human communication is like a delicate dance where listeners and speakers concurrently interact to maintain conversational dynamics. Hence, an effective model for generating listener nonverbal behaviors requires understanding the dyadic context and interaction. In this paper, we present an effective framework for creating 3D facial motions in dyadic interactions. Existing work consider a listener as a reactive agent with reflexive behaviors to the speaker's voice and facial motions. The heart of our framework is Dyadic Interaction Modeling (DIM), a pre-training approach that jointly models speakers' and listeners' motions through masking and contrastive learning to learn representations that capture the dyadic context. To enable the generation of non-deterministic behaviors, we encode both listener and speaker motions into discrete latent representations, through VQ-VAE. The pre-trained model is further fine-tuned for motion generation. Extensive experiments demonstrate the superiority of our framework in generating listener motions, establishing a new state-of-the-art according to the quantitative measures capturing the diversity and realism of generated motions. Qualitative results demonstrate the superior capabilities of the proposed approach in generating diverse and realistic expressions, eye blinks and head gestures.},

note = {arXiv:2403.09069 [cs]},

keywords = {DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{lu_leveraging_2024,

title = {Leveraging Synthetic Data for Generalizable and Fair Facial Action Unit Detection},

author = {Liupei Lu and Yufeng Yin and Yuming Gu and Yizhen Wu and Pratusha Prasad and Yajie Zhao and Mohammad Soleymani},

url = {http://arxiv.org/abs/2403.10737},

year  = {2024},

date = {2024-03-01},

urldate = {2024-04-16},

publisher = {arXiv},

abstract = {Facial action unit (AU) detection is a fundamental block for objective facial expression analysis. Supervised learning approaches require a large amount of manual labeling which is costly. The limited labeled data are also not diverse in terms of gender which can affect model fairness. In this paper, we propose to use synthetically generated data and multi-source domain adaptation (MSDA) to address the problems of the scarcity of labeled data and the diversity of subjects. Specifically, we propose to generate a diverse dataset through synthetic facial expression re-targeting by transferring the expressions from real faces to synthetic avatars. Then, we use MSDA to transfer the AU detection knowledge from a real dataset and the synthetic dataset to a target dataset. Instead of aligning the overall distributions of different domains, we propose Paired Moment Matching (PM2) to align the features of the paired real and synthetic data with the same facial expression. To further improve gender fairness, PM2 matches the features of the real data with a female and a male synthetic image. Our results indicate that synthetic data and the proposed model improve both AU detection performance and fairness across genders, demonstrating its potential to solve AU detection in-the-wild.},

note = {arXiv:2403.10737 [cs]},

keywords = {DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{cho_can_2023,

title = {Can Language Model Moderators Improve the Health of Online Discourse?},

author = {Hyundong Cho and Shuai Liu and Taiwei Shi and Darpan Jain and Basem Rizk and Yuyang Huang and Zixun Lu and Nuan Wen and Jonathan Gratch and Emilio Ferrara and Jonathan May},

url = {http://arxiv.org/abs/2311.10781},

year  = {2023},

date = {2023-11-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {Human moderation of online conversation is essential to maintaining civility and focus in a dialogue, but is challenging to scale and harmful to moderators. The inclusion of sophisticated natural language generation modules as a force multiplier aid moderators is a tantalizing prospect, but adequate evaluation approaches have so far been elusive. In this paper, we establish a systematic definition of conversational moderation effectiveness through a multidisciplinary lens that incorporates insights from social science. We then propose a comprehensive evaluation framework that uses this definition to asses models' moderation capabilities independently of human intervention. With our framework, we conduct the first known study of conversational dialogue models as moderators, finding that appropriately prompted models can provide specific and fair feedback on toxic behavior but struggle to influence users to increase their levels of respect and cooperation.},

note = {arXiv:2311.10781 [cs]},

keywords = {AI, Dialogue, DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{yang_context_2023,

title = {Context Unlocks Emotions: Text-based Emotion Classification Dataset Auditing with Large Language Models},

author = {Daniel Yang and Aditya Kommineni and Mohammad Alshehri and Nilamadhab Mohanty and Vedant Modi and Jonathan Gratch and Shrikanth Narayanan},

url = {http://arxiv.org/abs/2311.03551},

year  = {2023},

date = {2023-11-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {The lack of contextual information in text data can make the annotation process of text-based emotion classification datasets challenging. As a result, such datasets often contain labels that fail to consider all the relevant emotions in the vocabulary. This misalignment between text inputs and labels can degrade the performance of machine learning models trained on top of them. As re-annotating entire datasets is a costly and time-consuming task that cannot be done at scale, we propose to use the expressive capabilities of large language models to synthesize additional context for input text to increase its alignment with the annotated emotional labels. In this work, we propose a formal definition of textual context to motivate a prompting strategy to enhance such contextual information. We provide both human and empirical evaluation to demonstrate the efficacy of the enhanced context. Our method improves alignment between inputs and their human-annotated labels from both an empirical and human-evaluated standpoint.},

note = {arXiv:2311.03551 [cs]},

keywords = {AI, DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{chang_magicdance_2023,

title = {MagicDance: Realistic Human Dance Video Generation with Motions & Facial Expressions Transfer},

author = {Di Chang and Yichun Shi and Quankai Gao and Jessica Fu and Hongyi Xu and Guoxian Song and Qing Yan and Xiao Yang and Mohammad Soleymani},

url = {http://arxiv.org/abs/2311.12052},

year  = {2023},

date = {2023-11-01},

urldate = {2023-12-07},

publisher = {arXiv},

abstract = {In this work, we propose MagicDance, a diffusion-based model for 2D human motion and facial expression transfer on challenging human dance videos. Specifically, we aim to generate human dance videos of any target identity driven by novel pose sequences while keeping the identity unchanged. To this end, we propose a two-stage training strategy to disentangle human motions and appearance (e.g., facial expressions, skin tone and dressing), consisting of the pretraining of an appearance-control block and fine-tuning of an appearance-pose-joint-control block over human dance poses of the same dataset. Our novel design enables robust appearance control with temporally consistent upper body, facial attributes, and even background. The model also generalizes well on unseen human identities and complex motion sequences without the need for any fine-tuning with additional data with diverse human attributes by leveraging the prior knowledge of image diffusion models. Moreover, the proposed model is easy to use and can be considered as a plug-in module/extension to Stable Diffusion. We also demonstrate the model's ability for zero-shot 2D animation generation, enabling not only the appearance transfer from one identity to another but also allowing for cartoon-like stylization given only pose inputs. Extensive experiments demonstrate our superior performance on the TikTok dataset.},

note = {arXiv:2311.12052 [cs]},

keywords = {DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}