Publications

@misc{roemmele_test-taking_2024-1,

title = {From Test-Taking to Test-Making: Examining LLM Authoring of Commonsense Assessment Items},

author = {Melissa Roemmele and Andrew S. Gordon},

url = {https://arxiv.org/abs/2410.14897},

doi = {10.48550/ARXIV.2410.14897},

year  = {2024},

date = {2024-10-01},

urldate = {2024-12-05},

publisher = {arXiv},

abstract = {LLMs can now perform a variety of complex writing tasks. They also excel in answering questions pertaining to natural language inference and commonsense reasoning. Composing these questions is itself a skilled writing task, so in this paper we consider LLMs as authors of commonsense assessment items. We prompt LLMs to generate items in the style of a prominent benchmark for commonsense reasoning, the Choice of Plausible Alternatives (COPA). We examine the outcome according to analyses facilitated by the LLMs and human annotation. We find that LLMs that succeed in answering the original COPA benchmark are also more successful in authoring their own items.},

note = {Version Number: 1},

keywords = {DTIC, Learning Sciences},

pubstate = {published},

tppubtype = {misc}

}

@misc{lin_estuary_2024,

title = {Estuary: A Framework For Building Multimodal Low-Latency Real-Time Socially Interactive Agents},

author = {Spencer Lin and Basem Rizk and Miru Jun and Andy Artze and Caitlin Sullivan and Sharon Mozgai and Scott Fisher},

url = {http://arxiv.org/abs/2410.20116},

doi = {10.1145/3652988.3696198},

year  = {2024},

date = {2024-10-01},

urldate = {2024-12-06},

abstract = {The rise in capability and ubiquity of generative artificial intelligence (AI) technologies has enabled its application to the field of Socially Interactive Agents (SIAs). Despite rising interest in modern AI-powered components used for real-time SIA research, substantial friction remains due to the absence of a standardized and universal SIA framework. To target this absence, we developed Estuary: a multimodal (text, audio, and soon video) framework which facilitates the development of low-latency, real-time SIAs. Estuary seeks to reduce repeat work between studies and to provide a flexible platform that can be run entirely off-cloud to maximize configurability, controllability, reproducibility of studies, and speed of agent response times. We are able to do this by constructing a robust multimodal framework which incorporates current and future components seamlessly into a modular and interoperable architecture.},

note = {arXiv:2410.20116 [cs]},

keywords = {Virtual Agents},

pubstate = {published},

tppubtype = {misc}

}

@misc{chen_geometry-aware_2024,

title = {Geometry-aware Feature Matching for Large-Scale Structure from Motion},

author = {Gonglin Chen and Jinsen Wu and Haiwei Chen and Wenbin Teng and Zhiyuan Gao and Andrew Feng and Rongjun Qin and Yajie Zhao},

url = {https://arxiv.org/abs/2409.02310},

doi = {10.48550/ARXIV.2409.02310},

year  = {2024},

date = {2024-09-01},

urldate = {2025-01-16},

publisher = {arXiv},

abstract = {Establishing consistent and dense correspondences across multiple images is crucial for Structure from Motion (SfM) systems. Significant view changes, such as air-to-ground with very sparse view overlap, pose an even greater challenge to the correspondence solvers. We present a novel optimization-based approach that significantly enhances existing feature matching methods by introducing geometry cues in addition to color cues. This helps fill gaps when there is less overlap in large-scale scenarios. Our method formulates geometric verification as an optimization problem, guiding feature matching within detector-free methods and using sparse correspondences from detector-based methods as anchor points. By enforcing geometric constraints via the Sampson Distance, our approach ensures that the denser correspondences from detector-free methods are geometrically consistent and more accurate. This hybrid strategy significantly improves correspondence density and accuracy, mitigates multi-view inconsistencies, and leads to notable advancements in camera pose accuracy and point cloud density. It outperforms state-of-the-art feature matching methods on benchmark datasets and enables feature matching in challenging extreme large-scale settings.},

note = {Version Number: 3},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{gao_skyeyes_2024,

title = {Skyeyes: Ground Roaming using Aerial View Images},

author = {Zhiyuan Gao and Wenbin Teng and Gonglin Chen and Jinsen Wu and Ningli Xu and Rongjun Qin and Andrew Feng and Yajie Zhao},

url = {https://arxiv.org/abs/2409.16685},

doi = {10.48550/ARXIV.2409.16685},

year  = {2024},

date = {2024-09-01},

urldate = {2025-01-16},

publisher = {arXiv},

abstract = {Integrating aerial imagery-based scene generation into applications like autonomous driving and gaming enhances realism in 3D environments, but challenges remain in creating detailed content for occluded areas and ensuring real-time, consistent rendering. In this paper, we introduce Skyeyes, a novel framework that can generate photorealistic sequences of ground view images using only aerial view inputs, thereby creating a ground roaming experience. More specifically, we combine a 3D representation with a view consistent generation model, which ensures coherence between generated images. This method allows for the creation of geometrically consistent ground view images, even with large view gaps. The images maintain improved spatial-temporal coherence and realism, enhancing scene comprehension and visualization from aerial perspectives. To the best of our knowledge, there are no publicly available datasets that contain pairwise geo-aligned aerial and ground view imagery. Therefore, we build a large, synthetic, and geo-aligned dataset using Unreal Engine. Both qualitative and quantitative analyses on this synthetic dataset display superior results compared to other leading synthesis approaches. See the project page for more results: https://chaoren2357.github.io/website-skyeyes/.},

note = {Version Number: 1},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_alignbench_2024,

title = {AlignBench: Benchmarking Chinese Alignment of Large Language Models},

author = {Xiao Liu and Xuanyu Lei and Shengyuan Wang and Yue Huang and Zhuoer Feng and Bosi Wen and Jiale Cheng and Pei Ke and Yifan Xu and Weng Lam Tam and Xiaohan Zhang and Lichao Sun and Xiaotao Gu and Hongning Wang and Jing Zhang and Minlie Huang and Yuxiao Dong and Jie Tang},

url = {http://arxiv.org/abs/2311.18743},

doi = {10.48550/arXiv.2311.18743},

year  = {2024},

date = {2024-08-01},

urldate = {2025-01-16},

publisher = {arXiv},

abstract = {Alignment has become a critical step for instruction-tuned Large Language Models (LLMs) to become helpful assistants. However, the effective evaluation of alignment for emerging Chinese LLMs is still largely unexplored. To fill in this gap, we introduce AlignBench, a comprehensive multi-dimensional benchmark for evaluating LLMs' alignment in Chinese. We design a human-in-the-loop data curation pipeline, containing eight main categories, 683 real-scenario rooted queries and corresponding human verified references. To ensure the correctness of references, each knowledge-intensive query is accompanied with evidences collected from reliable web sources (including URLs and quotations) by our annotators. For automatic evaluation, our benchmark employs a rule-calibrated multi-dimensional LLM-as-Judgetextasciitildetextbackslashcitezheng2023judging approach with Chain-of-Thought to generate explanations and final ratings, ensuring high reliability and interpretability. All evaluation code, data, and LLM generations are available at textbackslashurlhttps://github.com/THUDM/AlignBench. Since its release, AlignBench has been adopted by top (Chinese) LLMs for evaluating their alignment capabilities in Chinese, including ChatGLM, Qwen, DeepSeek, Yi, Baichuan, and Abab.},

note = {arXiv:2311.18743 [cs]},

keywords = {Machine Learning},

pubstate = {published},

tppubtype = {misc}

}

@misc{han_knowledge-based_2024,

title = {Knowledge-based Emotion Recognition using Large Language Models},

author = {Bin Han and Cleo Yau and Su Lei and Jonathan Gratch},

url = {http://arxiv.org/abs/2408.04123},

year  = {2024},

date = {2024-08-01},

urldate = {2024-08-15},

publisher = {arXiv},

abstract = {Emotion recognition in social situations is a complex task that requires integrating information from both facial expressions and the situational context. While traditional approaches to automatic emotion recognition have focused on decontextualized signals, recent research emphasizes the importance of context in shaping emotion perceptions. This paper contributes to the emerging field of context-based emotion recognition by leveraging psychological theories of human emotion perception to inform the design of automated methods. We propose an approach that combines emotion recognition methods with Bayesian Cue Integration (BCI) to integrate emotion inferences from decontextualized facial expressions and contextual knowledge inferred via Large-language Models. We test this approach in the context of interpreting facial expressions during a social task, the prisoner's dilemma. Our results provide clear support for BCI across a range of automatic emotion recognition methods. The best automated method achieved results comparable to human observers, suggesting the potential for this approach to advance the field of affective computing.},

note = {arXiv:2408.04123 [cs]},

keywords = {DTIC, Emotions},

pubstate = {published},

tppubtype = {misc}

}

@misc{tak_gpt-4_2024,

title = {GPT-4 Emulates Average-Human Emotional Cognition from a Third-Person Perspective},

author = {Ala N. Tak and Jonathan Gratch},

url = {http://arxiv.org/abs/2408.13718},

year  = {2024},

date = {2024-08-01},

urldate = {2024-09-17},

publisher = {arXiv},

abstract = {This paper extends recent investigations on the emotional reasoning abilities of Large Language Models (LLMs). Current research on LLMs has not directly evaluated the distinction between how LLMs predict the self-attribution of emotions and the perception of others' emotions. We first look at carefully crafted emotion-evoking stimuli, originally designed to find patterns of brain neural activity representing fine-grained inferred emotional attributions of others. We show that GPT-4 is especially accurate in reasoning about such stimuli. This suggests LLMs agree with humans' attributions of others' emotions in stereotypical scenarios remarkably more than self-attributions of emotions in idiosyncratic situations. To further explore this, our second study utilizes a dataset containing annotations from both the author and a third-person perspective. We find that GPT-4's interpretations align more closely with human judgments about the emotions of others than with self-assessments. Notably, conventional computational models of emotion primarily rely on self-reported ground truth as the gold standard. However, an average observer's standpoint, which LLMs appear to have adopted, might be more relevant for many downstream applications, at least in the absence of individual information and adequate safety considerations.},

note = {arXiv:2408.13718 [cs]},

keywords = {DTIC, Emotions},

pubstate = {published},

tppubtype = {misc}

}

@misc{roth_supercharging_2024,

title = {Supercharging Federated Learning with Flower and NVIDIA FLARE},

author = {Holger R. Roth and Daniel J. Beutel and Yan Cheng and Javier Fernandez Marques and Heng Pan and Chester Chen and Zhihong Zhang and Yuhong Wen and Sean Yang and Isaac and Yang and Yuan-Ting Hsieh and Ziyue Xu and Daguang Xu and Nicholas D. Lane and Andrew Feng},

url = {http://arxiv.org/abs/2407.00031},

doi = {10.48550/arXiv.2407.00031},

year  = {2024},

date = {2024-07-01},

urldate = {2025-01-16},

publisher = {arXiv},

abstract = {Several open-source systems, such as Flower and NVIDIA FLARE, have been developed in recent years while focusing on different aspects of federated learning (FL). Flower is dedicated to implementing a cohesive approach to FL, analytics, and evaluation. Over time, Flower has cultivated extensive strategies and algorithms tailored for FL application development, fostering a vibrant FL community in research and industry. Conversely, FLARE has prioritized the creation of an enterprise-ready, resilient runtime environment explicitly designed for FL applications in production environments. In this paper, we describe our initial integration of both frameworks and show how they can work together to supercharge the FL ecosystem as a whole. Through the seamless integration of Flower and FLARE, applications crafted within the Flower framework can effortlessly operate within the FLARE runtime environment without necessitating any modifications. This initial integration streamlines the process, eliminating complexities and ensuring smooth interoperability between the two platforms, thus enhancing the overall efficiency and accessibility of FL applications.},

note = {arXiv:2407.00031 [cs]},

keywords = {},

pubstate = {published},

tppubtype = {misc}

}

@misc{huang_psyborg_2024,

title = {PsybORG+: Modeling and Simulation for Detecting Cognitive Biases in Advanced Persistent Threats},

author = {Shuo Huang and Fred Jones and Nikolos Gurney and David Pynadath and Kunal Srivastava and Stoney Trent and Peggy Wu and Quanyan Zhu},

url = {https://arxiv.org/abs/2408.01310},

doi = {10.48550/ARXIV.2408.01310},

year  = {2024},

date = {2024-07-01},

urldate = {2024-12-05},

publisher = {arXiv},

abstract = {Advanced Persistent Threats (APTs) bring significant challenges to cybersecurity due to their sophisticated and stealthy nature. Traditional cybersecurity measures fail to defend against APTs. Cognitive vulnerabilities can significantly influence attackers' decision-making processes, which presents an opportunity for defenders to exploit. This work introduces PsybORG$ˆ+$, a multi-agent cybersecurity simulation environment designed to model APT behaviors influenced by cognitive vulnerabilities. A classification model is built for cognitive vulnerability inference and a simulator is designed for synthetic data generation. Results show that PsybORG$ˆ+$ can effectively model APT attackers with different loss aversion and confirmation bias levels. The classification model has at least a 0.83 accuracy rate in predicting cognitive vulnerabilities.},

note = {Version Number: 3},

keywords = {DTIC},

pubstate = {published},

tppubtype = {misc}

}

@misc{han_-depth_2024,

title = {In-Depth Analysis of Emotion Recognition through Knowledge-Based Large Language Models},

author = {Bin Han and Cleo Yau and Su Lei and Jonathan Gratch},

url = {http://arxiv.org/abs/2408.00780},

year  = {2024},

date = {2024-07-01},

urldate = {2024-08-15},

publisher = {arXiv},

abstract = {Emotion recognition in social situations is a complex task that requires integrating information from both facial expressions and the situational context. While traditional approaches to automatic emotion recognition have focused on decontextualized signals, recent research emphasizes the importance of context in shaping emotion perceptions. This paper contributes to the emerging field of context-based emotion recognition by leveraging psychological theories of human emotion perception to inform the design of automated methods. We propose an approach that combines emotion recognition methods with Bayesian Cue Integration (BCI) to integrate emotion inferences from decontextualized facial expressions and contextual knowledge inferred via Large-language Models. We test this approach in the context of interpreting facial expressions during a social task, the prisoner's dilemma. Our results provide clear support for BCI across a range of automatic emotion recognition methods. The best automated method achieved results comparable to human observers, suggesting the potential for this approach to advance the field of affective computing.},

note = {arXiv:2408.00780 [cs]},

keywords = {Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{xiao_localized_2024,

title = {Localized Gaussian Splatting Editing with Contextual Awareness},

author = {Hanyuan Xiao and Yingshu Chen and Huajian Huang and Haolin Xiong and Jing Yang and Pratusha Prasad and Yajie Zhao},

url = {http://arxiv.org/abs/2408.00083},

year  = {2024},

date = {2024-07-01},

urldate = {2024-08-16},

publisher = {arXiv},

abstract = {Recent text-guided generation of individual 3D object has achieved great success using diffusion priors. However, these methods are not suitable for object insertion and replacement tasks as they do not consider the background, leading to illumination mismatches within the environment. To bridge the gap, we introduce an illumination-aware 3D scene editing pipeline for 3D Gaussian Splatting (3DGS) representation. Our key observation is that inpainting by the state-of-the-art conditional 2D diffusion model is consistent with background in lighting. To leverage the prior knowledge from the well-trained diffusion models for 3D object generation, our approach employs a coarse-to-fine objection optimization pipeline with inpainted views. In the first coarse step, we achieve image-to-3D lifting given an ideal inpainted view. The process employs 3D-aware diffusion prior from a view-conditioned diffusion model, which preserves illumination present in the conditioning image. To acquire an ideal inpainted image, we introduce an Anchor View Proposal (AVP) algorithm to find a single view that best represents the scene illumination in target region. In the second Texture Enhancement step, we introduce a novel Depth-guided Inpainting Score Distillation Sampling (DI-SDS), which enhances geometry and texture details with the inpainting diffusion prior, beyond the scope of the 3D-aware diffusion prior knowledge in the first coarse step. DI-SDS not only provides fine-grained texture enhancement, but also urges optimization to respect scene lighting. Our approach efficiently achieves local editing with global illumination consistency without explicitly modeling light transport. We demonstrate robustness of our method by evaluating editing in real scenes containing explicit highlight and shadows, and compare against the state-of-the-art text-to-3D editing methods.},

note = {arXiv:2408.00083 [cs]},

keywords = {DTIC, VGL},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_enhancing_2024,

title = {Enhancing Building Safety Design for Active Shooter Incidents: Exploration of Building Exit Parameters using Reinforcement Learning-Based Simulations},

author = {Ruying Liu and Wanjing Wu and Burcin Becerik-Gerber and Gale M. Lucas},

url = {http://arxiv.org/abs/2407.10441},

year  = {2024},

date = {2024-07-01},

urldate = {2024-09-17},

publisher = {arXiv},

abstract = {With the alarming rise in active shooter incidents (ASIs) in the United States, enhancing public safety through building design has become a pressing need. This study proposes a reinforcement learning-based simulation approach addressing gaps in existing research that has neglected the dynamic behaviours of shooters. We developed an autonomous agent to simulate an active shooter within a realistic office environment, aiming to offer insights into the interactions between building design parameters and ASI outcomes. A case study is conducted to quantitatively investigate the impact of building exit numbers (total count of accessible exits) and configuration (arrangement of which exits are available or not) on evacuation and harm rates. Findings demonstrate that greater exit availability significantly improves evacuation outcomes and reduces harm. Exits nearer to the shooter's initial position hold greater importance for accessibility than those farther away. By encompassing dynamic shooter behaviours, this study offers preliminary insights into effective building safety design against evolving threats.},

note = {arXiv:2407.10441 [cs]},

keywords = {DTIC, Virtual Worlds},

pubstate = {published},

tppubtype = {misc}

}

@misc{yin_validation_2024,

title = {Validation and Evacuee Modeling of Virtual Robot-guided Emergency Evacuation Experiments},

author = {Yinxuan Yin and Mollik Nayyar and Daniel Holman and Gale Lucas and Colin Holbrook and Alan Wagner},

url = {https://osf.io/mr78s},

doi = {10.31234/osf.io/mr78s},

year  = {2024},

date = {2024-06-01},

urldate = {2024-09-17},

publisher = {Center for Open Science},

abstract = {Virtual Reality (VR) is an increasingly common tool for investigating human responses to emergency situations. Nonetheless, studies validating and comparing human subject behavior during real world emergencies to their responses in VR are notably rare, and no prior studies have validated whether human emergency responses to guidance from a robot are comparable in VR versus the real world. In the present pre-registered study, we used VR to replicate a previous robot- guided emergency evacuation study conducted in the real world and compared human subject behavior in matched physical and virtual environments. In both environments, human subjects were asked to follow a robot to a location and to then read an article. While reading, a fire alarm sounds. The robot then attempted to guide them to a distant, unfamiliar exit rather than nearby and familiar exits. We observed close correspondences between evacuee exit choice (the robot’s distant exit versus closer exits), evacuation time, and trust in the robot between the VR and physical environments. We further demonstrate that data collected in virtual reality can be used to create accurate motion models (mean error of 0.42 centimeters) predicting evacuee trajectories and locations in real life. Taken together, the results provide evidence for the ecological validity of VR approaches to studying human-robot interaction, particularly robot- guided emergency evacuation.},

keywords = {DTIC, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{liu_atomgs_2024,

title = {AtomGS: Atomizing Gaussian Splatting for High-Fidelity Radiance Field},

author = {Rong Liu and Rui Xu and Yue Hu and Meida Chen and Andrew Feng},

url = {https://arxiv.org/abs/2405.12369},

doi = {10.48550/ARXIV.2405.12369},

year  = {2024},

date = {2024-05-01},

urldate = {2024-07-11},

publisher = {arXiv},

abstract = {3D Gaussian Splatting (3DGS) has recently advanced radiance field reconstruction by offering superior capabilities for novel view synthesis and real-time rendering speed. However, its strategy of blending optimization and adaptive density control might lead to sub-optimal results; it can sometimes yield noisy geometry and blurry artifacts due to prioritizing optimizing large Gaussians at the cost of adequately densifying smaller ones. To address this, we introduce AtomGS, consisting of Atomized Proliferation and Geometry-Guided Optimization. The Atomized Proliferation constrains ellipsoid Gaussians of various sizes into more uniform-sized Atom Gaussians. The strategy enhances the representation of areas with fine features by placing greater emphasis on densification in accordance with scene details. In addition, we proposed a Geometry-Guided Optimization approach that incorporates an Edge-Aware Normal Loss. This optimization method effectively smooths flat surfaces while preserving intricate details. Our evaluation shows that AtomGS outperforms existing state-of-the-art methods in rendering quality. Additionally, it achieves competitive accuracy in geometry reconstruction and offers a significant improvement in training speed over other SDF-based methods. More interactive demos can be found in our website (https://rongliu-leo.github.io/AtomGS/).},

note = {Version Number: 2},

keywords = {Graphics, VGL},

pubstate = {published},

tppubtype = {misc}

}

@misc{chang_magicpose_2024,

title = {MagicPose: Realistic Human Poses and Facial Expressions Retargeting with Identity-aware Diffusion},

author = {Di Chang and Yichun Shi and Quankai Gao and Jessica Fu and Hongyi Xu and Guoxian Song and Qing Yan and Yizhe Zhu and Xiao Yang and Mohammad Soleymani},

url = {http://arxiv.org/abs/2311.12052},

year  = {2024},

date = {2024-05-01},

urldate = {2024-07-18},

publisher = {arXiv},

abstract = {In this work, we propose MagicPose, a diffusion-based model for 2D human pose and facial expression retargeting. Specifically, given a reference image, we aim to generate a person's new images by controlling the poses and facial expressions while keeping the identity unchanged. To this end, we propose a two-stage training strategy to disentangle human motions and appearance (e.g., facial expressions, skin tone and dressing), consisting of (1) the pre-training of an appearance-control block and (2) learning appearance-disentangled pose control. Our novel design enables robust appearance control over generated human images, including body, facial attributes, and even background. By leveraging the prior knowledge of image diffusion models, MagicPose generalizes well to unseen human identities and complex poses without the need for additional fine-tuning. Moreover, the proposed model is easy to use and can be considered as a plug-in module/extension to Stable Diffusion. The code is available at: https://github.com/Boese0601/MagicDance},

note = {arXiv:2311.12052 [cs]},

keywords = {},

pubstate = {published},

tppubtype = {misc}

}

@misc{west_improving_2024,

title = {Improving Social Connection with Weak Ties and Strangers: Effects of a New Micro-Intervention on Interaction Quality and Social Behavior},

author = {Taylor Nicole West and Michael Prinzing and Catherine Garton and Catherine J. Berman and Jieni Zhou and James Hale and Jonathan Gratch and Barbara Fredrickson},

url = {https://osf.io/ytjr6},

doi = {10.31234/osf.io/ytjr6},

year  = {2024},

date = {2024-05-01},

urldate = {2024-06-25},

abstract = {We propose that the emotional quality of people’s interactions with acquaintances (i.e., weak ties) and strangers contributes to well-being. We test whether a new micro-intervention can raise the quality of these interactions. We randomized young adults (N = 335) to this connectedness micro-intervention or a control intervention. Both interventions were delivered via a psychoeducational video followed by a brief conversation with a virtual human, with whom participants developed if-then plans to carry out their assigned behavioral goal. Pre-intervention, high-quality weak-tie and stranger interactions were associated with lower loneliness and greater mental health independent of strong-tie interaction quality. Experimental data showed the connectedness intervention improved the emotional quality of participants' interactions with weak ties and strangers over two days, evident in participants’ episodic self-reports and faster in-lab conversational response time. Discussion centers on implications for developing scalable behavioral interventions to improve well-being.},

keywords = {Emotions, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}

@misc{zhang_magicpose4d_2024,

title = {MagicPose4D: Crafting Articulated Models with Appearance and Motion Control},

author = {Hao Zhang and Di Chang and Fang Li and Mohammad Soleymani and Narendra Ahuja},

url = {https://arxiv.org/abs/2405.14017},

doi = {10.48550/ARXIV.2405.14017},

year  = {2024},

date = {2024-05-01},

urldate = {2024-06-25},

publisher = {arXiv},

abstract = {With the success of 2D and 3D visual generative models, there is growing interest in generating 4D content. Existing methods primarily rely on text prompts to produce 4D content, but they often fall short of accurately defining complex or rare motions. To address this limitation, we propose MagicPose4D, a novel framework for refined control over both appearance and motion in 4D generation. Unlike traditional methods, MagicPose4D accepts monocular videos as motion prompts, enabling precise and customizable motion generation. MagicPose4D comprises two key modules: 

 i) Dual-Phase 4D Reconstruction Modulevphantom which operates in two phases. The first phase focuses on capturing the model's shape using accurate 2D supervision and less accurate but geometrically informative 3D pseudo-supervision without imposing skeleton constraints. The second phase refines the model using more accurate pseudo-3D supervision, obtained in the first phase and introduces kinematic chain-based skeleton constraints to ensure physical plausibility. Additionally, we propose a Global-local Chamfer loss that aligns the overall distribution of predicted mesh vertices with the supervision while maintaining part-level alignment without extra annotations. 

 ii) Cross-category Motion Transfer Modulevphantom leverages the predictions from the 4D reconstruction module and uses a kinematic-chain-based skeleton to achieve cross-category motion transfer. It ensures smooth transitions between frames through dynamic rigidity, facilitating robust generalization without additional training. 

 Through extensive experiments, we demonstrate that MagicPose4D significantly improves the accuracy and consistency of 4D content generation, outperforming existing methods in various benchmarks.},

note = {Version Number: 1},

keywords = {VGL, Virtual Humans},

pubstate = {published},

tppubtype = {misc}

}