Publications

@misc{roth_supercharging_2024,

title = {Supercharging Federated Learning with Flower and NVIDIA FLARE},

author = {Holger R. Roth and Daniel J. Beutel and Yan Cheng and Javier Fernandez Marques and Heng Pan and Chester Chen and Zhihong Zhang and Yuhong Wen and Sean Yang and Isaac and Yang and Yuan-Ting Hsieh and Ziyue Xu and Daguang Xu and Nicholas D. Lane and Andrew Feng},

url = {http://arxiv.org/abs/2407.00031},

doi = {10.48550/arXiv.2407.00031},

year  = {2024},

date = {2024-07-01},

urldate = {2025-01-16},

publisher = {arXiv},

abstract = {Several open-source systems, such as Flower and NVIDIA FLARE, have been developed in recent years while focusing on different aspects of federated learning (FL). Flower is dedicated to implementing a cohesive approach to FL, analytics, and evaluation. Over time, Flower has cultivated extensive strategies and algorithms tailored for FL application development, fostering a vibrant FL community in research and industry. Conversely, FLARE has prioritized the creation of an enterprise-ready, resilient runtime environment explicitly designed for FL applications in production environments. In this paper, we describe our initial integration of both frameworks and show how they can work together to supercharge the FL ecosystem as a whole. Through the seamless integration of Flower and FLARE, applications crafted within the Flower framework can effortlessly operate within the FLARE runtime environment without necessitating any modifications. This initial integration streamlines the process, eliminating complexities and ensuring smooth interoperability between the two platforms, thus enhancing the overall efficiency and accessibility of FL applications.},

note = {arXiv:2407.00031 [cs]},

keywords = {},

pubstate = {published},

tppubtype = {misc}

}

@misc{huang_psyborg_2024,

title = {PsybORG+: Modeling and Simulation for Detecting Cognitive Biases in Advanced Persistent Threats},

author = {Shuo Huang and Fred Jones and Nikolos Gurney and David Pynadath and Kunal Srivastava and Stoney Trent and Peggy Wu and Quanyan Zhu},

url = {https://arxiv.org/abs/2408.01310},

doi = {10.48550/ARXIV.2408.01310},

year  = {2024},

date = {2024-07-01},

urldate = {2024-12-05},

publisher = {arXiv},

abstract = {Advanced Persistent Threats (APTs) bring significant challenges to cybersecurity due to their sophisticated and stealthy nature. Traditional cybersecurity measures fail to defend against APTs. Cognitive vulnerabilities can significantly influence attackers' decision-making processes, which presents an opportunity for defenders to exploit. This work introduces PsybORG$ˆ+$, a multi-agent cybersecurity simulation environment designed to model APT behaviors influenced by cognitive vulnerabilities. A classification model is built for cognitive vulnerability inference and a simulator is designed for synthetic data generation. Results show that PsybORG$ˆ+$ can effectively model APT attackers with different loss aversion and confirmation bias levels. The classification model has at least a 0.83 accuracy rate in predicting cognitive vulnerabilities.},

note = {Version Number: 3},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{han_-depth_2024,

title = {In-Depth Analysis of Emotion Recognition through Knowledge-Based Large Language Models},

author = {Bin Han and Cleo Yau and Su Lei and Jonathan Gratch},

url = {http://arxiv.org/abs/2408.00780},

year  = {2024},

date = {2024-07-01},

urldate = {2024-08-15},

publisher = {arXiv},

abstract = {Emotion recognition in social situations is a complex task that requires integrating information from both facial expressions and the situational context. While traditional approaches to automatic emotion recognition have focused on decontextualized signals, recent research emphasizes the importance of context in shaping emotion perceptions. This paper contributes to the emerging field of context-based emotion recognition by leveraging psychological theories of human emotion perception to inform the design of automated methods. We propose an approach that combines emotion recognition methods with Bayesian Cue Integration (BCI) to integrate emotion inferences from decontextualized facial expressions and contextual knowledge inferred via Large-language Models. We test this approach in the context of interpreting facial expressions during a social task, the prisoner's dilemma. Our results provide clear support for BCI across a range of automatic emotion recognition methods. The best automated method achieved results comparable to human observers, suggesting the potential for this approach to advance the field of affective computing.},

note = {arXiv:2408.00780 [cs]},

keywords = {Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{xiao_localized_2024,

title = {Localized Gaussian Splatting Editing with Contextual Awareness},

author = {Hanyuan Xiao and Yingshu Chen and Huajian Huang and Haolin Xiong and Jing Yang and Pratusha Prasad and Yajie Zhao},

url = {http://arxiv.org/abs/2408.00083},

year  = {2024},

date = {2024-07-01},

urldate = {2024-08-16},

publisher = {arXiv},

abstract = {Recent text-guided generation of individual 3D object has achieved great success using diffusion priors. However, these methods are not suitable for object insertion and replacement tasks as they do not consider the background, leading to illumination mismatches within the environment. To bridge the gap, we introduce an illumination-aware 3D scene editing pipeline for 3D Gaussian Splatting (3DGS) representation. Our key observation is that inpainting by the state-of-the-art conditional 2D diffusion model is consistent with background in lighting. To leverage the prior knowledge from the well-trained diffusion models for 3D object generation, our approach employs a coarse-to-fine objection optimization pipeline with inpainted views. In the first coarse step, we achieve image-to-3D lifting given an ideal inpainted view. The process employs 3D-aware diffusion prior from a view-conditioned diffusion model, which preserves illumination present in the conditioning image. To acquire an ideal inpainted image, we introduce an Anchor View Proposal (AVP) algorithm to find a single view that best represents the scene illumination in target region. In the second Texture Enhancement step, we introduce a novel Depth-guided Inpainting Score Distillation Sampling (DI-SDS), which enhances geometry and texture details with the inpainting diffusion prior, beyond the scope of the 3D-aware diffusion prior knowledge in the first coarse step. DI-SDS not only provides fine-grained texture enhancement, but also urges optimization to respect scene lighting. Our approach efficiently achieves local editing with global illumination consistency without explicitly modeling light transport. We demonstrate robustness of our method by evaluating editing in real scenes containing explicit highlight and shadows, and compare against the state-of-the-art text-to-3D editing methods.},

note = {arXiv:2408.00083 [cs]},

keywords = {DTIC, VGL},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_enhancing_2024,

title = {Enhancing Building Safety Design for Active Shooter Incidents: Exploration of Building Exit Parameters using Reinforcement Learning-Based Simulations},

author = {Ruying Liu and Wanjing Wu and Burcin Becerik-Gerber and Gale M. Lucas},

url = {http://arxiv.org/abs/2407.10441},

year  = {2024},

date = {2024-07-01},

urldate = {2024-09-17},

publisher = {arXiv},

abstract = {With the alarming rise in active shooter incidents (ASIs) in the United States, enhancing public safety through building design has become a pressing need. This study proposes a reinforcement learning-based simulation approach addressing gaps in existing research that has neglected the dynamic behaviours of shooters. We developed an autonomous agent to simulate an active shooter within a realistic office environment, aiming to offer insights into the interactions between building design parameters and ASI outcomes. A case study is conducted to quantitatively investigate the impact of building exit numbers (total count of accessible exits) and configuration (arrangement of which exits are available or not) on evacuation and harm rates. Findings demonstrate that greater exit availability significantly improves evacuation outcomes and reduces harm. Exits nearer to the shooter's initial position hold greater importance for accessibility than those farther away. By encompassing dynamic shooter behaviours, this study offers preliminary insights into effective building safety design against evolving threats.},

note = {arXiv:2407.10441 [cs]},

keywords = {DTIC, Virtual Worlds},

pubstate = {published},

tppubtype = {misc}

}

@misc{yin_validation_2024,

title = {Validation and Evacuee Modeling of Virtual Robot-guided Emergency Evacuation Experiments},

author = {Yinxuan Yin and Mollik Nayyar and Daniel Holman and Gale Lucas and Colin Holbrook and Alan Wagner},

url = {https://osf.io/mr78s},

doi = {10.31234/osf.io/mr78s},

year  = {2024},

date = {2024-06-01},

urldate = {2024-09-17},

publisher = {Center for Open Science},

abstract = {Virtual Reality (VR) is an increasingly common tool for investigating human responses to emergency situations. Nonetheless, studies validating and comparing human subject behavior during real world emergencies to their responses in VR are notably rare, and no prior studies have validated whether human emergency responses to guidance from a robot are comparable in VR versus the real world. In the present pre-registered study, we used VR to replicate a previous robot- guided emergency evacuation study conducted in the real world and compared human subject behavior in matched physical and virtual environments. In both environments, human subjects were asked to follow a robot to a location and to then read an article. While reading, a fire alarm sounds. The robot then attempted to guide them to a distant, unfamiliar exit rather than nearby and familiar exits. We observed close correspondences between evacuee exit choice (the robot’s distant exit versus closer exits), evacuation time, and trust in the robot between the VR and physical environments. We further demonstrate that data collected in virtual reality can be used to create accurate motion models (mean error of 0.42 centimeters) predicting evacuee trajectories and locations in real life. Taken together, the results provide evidence for the ecological validity of VR approaches to studying human-robot interaction, particularly robot- guided emergency evacuation.},

keywords = {DTIC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_atomgs_2024,

title = {AtomGS: Atomizing Gaussian Splatting for High-Fidelity Radiance Field},

author = {Rong Liu and Rui Xu and Yue Hu and Meida Chen and Andrew Feng},

url = {https://arxiv.org/abs/2405.12369},

doi = {10.48550/ARXIV.2405.12369},

year  = {2024},

date = {2024-05-01},

urldate = {2024-07-11},

publisher = {arXiv},

abstract = {3D Gaussian Splatting (3DGS) has recently advanced radiance field reconstruction by offering superior capabilities for novel view synthesis and real-time rendering speed. However, its strategy of blending optimization and adaptive density control might lead to sub-optimal results; it can sometimes yield noisy geometry and blurry artifacts due to prioritizing optimizing large Gaussians at the cost of adequately densifying smaller ones. To address this, we introduce AtomGS, consisting of Atomized Proliferation and Geometry-Guided Optimization. The Atomized Proliferation constrains ellipsoid Gaussians of various sizes into more uniform-sized Atom Gaussians. The strategy enhances the representation of areas with fine features by placing greater emphasis on densification in accordance with scene details. In addition, we proposed a Geometry-Guided Optimization approach that incorporates an Edge-Aware Normal Loss. This optimization method effectively smooths flat surfaces while preserving intricate details. Our evaluation shows that AtomGS outperforms existing state-of-the-art methods in rendering quality. Additionally, it achieves competitive accuracy in geometry reconstruction and offers a significant improvement in training speed over other SDF-based methods. More interactive demos can be found in our website (https://rongliu-leo.github.io/AtomGS/).},

note = {Version Number: 2},

keywords = {Graphics, VGL},

pubstate = {published},

tppubtype = {misc}

}

@misc{chang_magicpose_2024,

title = {MagicPose: Realistic Human Poses and Facial Expressions Retargeting with Identity-aware Diffusion},

author = {Di Chang and Yichun Shi and Quankai Gao and Jessica Fu and Hongyi Xu and Guoxian Song and Qing Yan and Yizhe Zhu and Xiao Yang and Mohammad Soleymani},

url = {http://arxiv.org/abs/2311.12052},

year  = {2024},

date = {2024-05-01},

urldate = {2024-07-18},

publisher = {arXiv},

abstract = {In this work, we propose MagicPose, a diffusion-based model for 2D human pose and facial expression retargeting. Specifically, given a reference image, we aim to generate a person's new images by controlling the poses and facial expressions while keeping the identity unchanged. To this end, we propose a two-stage training strategy to disentangle human motions and appearance (e.g., facial expressions, skin tone and dressing), consisting of (1) the pre-training of an appearance-control block and (2) learning appearance-disentangled pose control. Our novel design enables robust appearance control over generated human images, including body, facial attributes, and even background. By leveraging the prior knowledge of image diffusion models, MagicPose generalizes well to unseen human identities and complex poses without the need for additional fine-tuning. Moreover, the proposed model is easy to use and can be considered as a plug-in module/extension to Stable Diffusion. The code is available at: https://github.com/Boese0601/MagicDance},

note = {arXiv:2311.12052 [cs]},

keywords = {},

pubstate = {published},

tppubtype = {misc}

}

@misc{west_improving_2024,

title = {Improving Social Connection with Weak Ties and Strangers: Effects of a New Micro-Intervention on Interaction Quality and Social Behavior},

author = {Taylor Nicole West and Michael Prinzing and Catherine Garton and Catherine J. Berman and Jieni Zhou and James Hale and Jonathan Gratch and Barbara Fredrickson},

url = {https://osf.io/ytjr6},

doi = {10.31234/osf.io/ytjr6},

year  = {2024},

date = {2024-05-01},

urldate = {2024-06-25},

abstract = {We propose that the emotional quality of people’s interactions with acquaintances (i.e., weak ties) and strangers contributes to well-being. We test whether a new micro-intervention can raise the quality of these interactions. We randomized young adults (N = 335) to this connectedness micro-intervention or a control intervention. Both interventions were delivered via a psychoeducational video followed by a brief conversation with a virtual human, with whom participants developed if-then plans to carry out their assigned behavioral goal. Pre-intervention, high-quality weak-tie and stranger interactions were associated with lower loneliness and greater mental health independent of strong-tie interaction quality. Experimental data showed the connectedness intervention improved the emotional quality of participants' interactions with weak ties and strangers over two days, evident in participants’ episodic self-reports and faster in-lab conversational response time. Discussion centers on implications for developing scalable behavioral interventions to improve well-being.},

keywords = {Emotions, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{zhang_magicpose4d_2024,

title = {MagicPose4D: Crafting Articulated Models with Appearance and Motion Control},

author = {Hao Zhang and Di Chang and Fang Li and Mohammad Soleymani and Narendra Ahuja},

url = {https://arxiv.org/abs/2405.14017},

doi = {10.48550/ARXIV.2405.14017},

year  = {2024},

date = {2024-05-01},

urldate = {2024-06-25},

publisher = {arXiv},

abstract = {With the success of 2D and 3D visual generative models, there is growing interest in generating 4D content. Existing methods primarily rely on text prompts to produce 4D content, but they often fall short of accurately defining complex or rare motions. To address this limitation, we propose MagicPose4D, a novel framework for refined control over both appearance and motion in 4D generation. Unlike traditional methods, MagicPose4D accepts monocular videos as motion prompts, enabling precise and customizable motion generation. MagicPose4D comprises two key modules: 

 i) Dual-Phase 4D Reconstruction Modulevphantom which operates in two phases. The first phase focuses on capturing the model's shape using accurate 2D supervision and less accurate but geometrically informative 3D pseudo-supervision without imposing skeleton constraints. The second phase refines the model using more accurate pseudo-3D supervision, obtained in the first phase and introduces kinematic chain-based skeleton constraints to ensure physical plausibility. Additionally, we propose a Global-local Chamfer loss that aligns the overall distribution of predicted mesh vertices with the supervision while maintaining part-level alignment without extra annotations. 

 ii) Cross-category Motion Transfer Modulevphantom leverages the predictions from the 4D reconstruction module and uses a kinematic-chain-based skeleton to achieve cross-category motion transfer. It ensures smooth transitions between frames through dynamic rigidity, facilitating robust generalization without additional training. 

 Through extensive experiments, we demonstrate that MagicPose4D significantly improves the accuracy and consistency of 4D content generation, outperforming existing methods in various benchmarks.},

note = {Version Number: 1},

keywords = {VGL, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{ehsanpour_social-mae_2024,

title = {Social-MAE: Social Masked Autoencoder for Multi-person Motion Representation Learning},

author = {Mahsa Ehsanpour and Ian Reid and Hamid Rezatofighi},

url = {https://arxiv.org/abs/2404.05578},

doi = {10.48550/ARXIV.2404.05578},

year  = {2024},

date = {2024-04-01},

urldate = {2024-07-12},

publisher = {arXiv},

abstract = {For a complete comprehension of multi-person scenes, it is essential to go beyond basic tasks like detection and tracking. Higher-level tasks, such as understanding the interactions and social activities among individuals, are also crucial. Progress towards models that can fully understand scenes involving multiple people is hindered by a lack of sufficient annotated data for such high-level tasks. To address this challenge, we introduce Social-MAE, a simple yet effective transformer-based masked autoencoder framework for multi-person human motion data. The framework uses masked modeling to pre-train the encoder to reconstruct masked human joint trajectories, enabling it to learn generalizable and data efficient representations of motion in human crowded scenes. Social-MAE comprises a transformer as the MAE encoder and a lighter-weight transformer as the MAE decoder which operates on multi-person joints' trajectory in the frequency domain. After the reconstruction task, the MAE decoder is replaced with a task-specific decoder and the model is fine-tuned end-to-end for a variety of high-level social tasks. Our proposed model combined with our pre-training approach achieves the state-of-the-art results on various high-level social tasks, including multi-person pose forecasting, social grouping, and social action understanding. These improvements are demonstrated across four popular multi-person datasets encompassing both human 2D and 3D body pose.},

note = {Version Number: 1},

keywords = {Social Simulation},

pubstate = {published},

tppubtype = {misc}

}

@misc{chen_dont_2024,

title = {Don't Look into the Dark: Latent Codes for Pluralistic Image Inpainting},

author = {Haiwei Chen and Yajie Zhao},

url = {http://arxiv.org/abs/2403.18186},

year  = {2024},

date = {2024-03-01},

urldate = {2024-08-15},

publisher = {arXiv},

abstract = {We present a method for large-mask pluralistic image inpainting based on the generative framework of discrete latent codes. Our method learns latent priors, discretized as tokens, by only performing computations at the visible locations of the image. This is realized by a restrictive partial encoder that predicts the token label for each visible block, a bidirectional transformer that infers the missing labels by only looking at these tokens, and a dedicated synthesis network that couples the tokens with the partial image priors to generate coherent and pluralistic complete image even under extreme mask settings. Experiments on public benchmarks validate our design choices as the proposed method outperforms strong baselines in both visual quality and diversity metrics.},

note = {arXiv:2403.18186 [cs]},

keywords = {VGL},

pubstate = {published},

tppubtype = {misc}

}

@misc{singh_twostep_2024,

title = {TwoStep: Multi-agent Task Planning using Classical Planners and Large Language Models},

author = {Ishika Singh and David Traum and Jesse Thomason},

url = {http://arxiv.org/abs/2403.17246},

year  = {2024},

date = {2024-03-01},

urldate = {2024-08-15},

publisher = {arXiv},

abstract = {Classical planning formulations like the Planning Domain Definition Language (PDDL) admit action sequences guaranteed to achieve a goal state given an initial state if any are possible. However, reasoning problems defined in PDDL do not capture temporal aspects of action taking, for example that two agents in the domain can execute an action simultaneously if postconditions of each do not interfere with preconditions of the other. A human expert can decompose a goal into largely independent constituent parts and assign each agent to one of these subgoals to take advantage of simultaneous actions for faster execution of plan steps, each using only single agent planning. By contrast, large language models (LLMs) used for directly inferring plan steps do not guarantee execution success, but do leverage commonsense reasoning to assemble action sequences. We combine the strengths of classical planning and LLMs by approximating human intuitions for two-agent planning goal decomposition. We demonstrate that LLM-based goal decomposition leads to faster planning times than solving multi-agent PDDL problems directly while simultaneously achieving fewer plan execution steps than a single agent plan alone and preserving execution success. Additionally, we find that LLM-based approximations of subgoals can achieve similar multi-agent execution steps than those specified by human experts. Website and resources at https://glamor-usc.github.io/twostep},

note = {arXiv:2403.17246 [cs]},

keywords = {},

pubstate = {published},

tppubtype = {misc}

}

@misc{lu_leveraging_2024,

title = {Leveraging Synthetic Data for Generalizable and Fair Facial Action Unit Detection},

author = {Liupei Lu and Yufeng Yin and Yuming Gu and Yizhen Wu and Pratusha Prasad and Yajie Zhao and Mohammad Soleymani},

url = {http://arxiv.org/abs/2403.10737},

year  = {2024},

date = {2024-03-01},

urldate = {2024-04-16},

publisher = {arXiv},

abstract = {Facial action unit (AU) detection is a fundamental block for objective facial expression analysis. Supervised learning approaches require a large amount of manual labeling which is costly. The limited labeled data are also not diverse in terms of gender which can affect model fairness. In this paper, we propose to use synthetically generated data and multi-source domain adaptation (MSDA) to address the problems of the scarcity of labeled data and the diversity of subjects. Specifically, we propose to generate a diverse dataset through synthetic facial expression re-targeting by transferring the expressions from real faces to synthetic avatars. Then, we use MSDA to transfer the AU detection knowledge from a real dataset and the synthetic dataset to a target dataset. Instead of aligning the overall distributions of different domains, we propose Paired Moment Matching (PM2) to align the features of the paired real and synthetic data with the same facial expression. To further improve gender fairness, PM2 matches the features of the real data with a female and a male synthetic image. Our results indicate that synthetic data and the proposed model improve both AU detection performance and fairness across genders, demonstrating its potential to solve AU detection in-the-wild.},

note = {arXiv:2403.10737 [cs]},

keywords = {DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{tran_dyadic_2024,

title = {Dyadic Interaction Modeling for Social Behavior Generation},

author = {Minh Tran and Di Chang and Maksim Siniukov and Mohammad Soleymani},

url = {http://arxiv.org/abs/2403.09069},

year  = {2024},

date = {2024-03-01},

urldate = {2024-03-19},

publisher = {arXiv},

abstract = {Human-human communication is like a delicate dance where listeners and speakers concurrently interact to maintain conversational dynamics. Hence, an effective model for generating listener nonverbal behaviors requires understanding the dyadic context and interaction. In this paper, we present an effective framework for creating 3D facial motions in dyadic interactions. Existing work consider a listener as a reactive agent with reflexive behaviors to the speaker's voice and facial motions. The heart of our framework is Dyadic Interaction Modeling (DIM), a pre-training approach that jointly models speakers' and listeners' motions through masking and contrastive learning to learn representations that capture the dyadic context. To enable the generation of non-deterministic behaviors, we encode both listener and speaker motions into discrete latent representations, through VQ-VAE. The pre-trained model is further fine-tuned for motion generation. Extensive experiments demonstrate the superiority of our framework in generating listener motions, establishing a new state-of-the-art according to the quantitative measures capturing the diversity and realism of generated motions. Qualitative results demonstrate the superior capabilities of the proposed approach in generating diverse and realistic expressions, eye blinks and head gestures.},

note = {arXiv:2403.09069 [cs]},

keywords = {DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}