Publications
Search
1.
Liu, Ruying; Becerik-Gerber, Burcin; Lucas, Gale M.
In: International Journal of Human–Computer Interaction, pp. 1–20, 2026, ISSN: 1044-7318, 1532-7590.
@article{liu_technology_2026,
title = {Technology Acceptance in High-Stakes Serious Games: Extending Technology Acceptance Model (TAM) with Affective and Experiential Predictors in Active Shooter Preparedness},
author = {Ruying Liu and Burcin Becerik-Gerber and Gale M. Lucas},
url = {https://www.tandfonline.com/doi/full/10.1080/10447318.2026.2635682},
doi = {10.1080/10447318.2026.2635682},
issn = {1044-7318, 1532-7590},
year = {2026},
date = {2026-03-01},
urldate = {2026-06-18},
journal = {International Journal of Human–Computer Interaction},
pages = {1–20},
abstract = {Active shooter incidents (ASIs) represent extreme threats to public safety. Virtual reality (VR) serious games offer promising training potential, but user acceptance in high-stakes and emotionally intense contexts is underexplored. This study extends the Technology Acceptance Model (TAM) by incorporating affective (perceived enjoyment) and experiential (self-reported engagement) factors. Using a desktop-based VR ASI preparedness simulation with 50 healthcare workers, we tested the extended TAM model. Core TAM relationships hold for ease of use and usefulness, but enjoyment and engagement play distinct roles. Enjoyment strongly predicted adoption, fully mediating the effect of ease of use and partially mediating usefulness. It was associated with high-arousal positive emotions(e.g., inspiration, pride, enthusiasm), reflecting eudaimonic states rather than hedonic fun. Engagement also significantly promoted adoption, though with weaker mediation effects. These findings extend TAM to high-stakes serious games and suggest that effective design should cultivate meaningful enjoyment and engagement without undermining scenario seriousness.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Active shooter incidents (ASIs) represent extreme threats to public safety. Virtual reality (VR) serious games offer promising training potential, but user acceptance in high-stakes and emotionally intense contexts is underexplored. This study extends the Technology Acceptance Model (TAM) by incorporating affective (perceived enjoyment) and experiential (self-reported engagement) factors. Using a desktop-based VR ASI preparedness simulation with 50 healthcare workers, we tested the extended TAM model. Core TAM relationships hold for ease of use and usefulness, but enjoyment and engagement play distinct roles. Enjoyment strongly predicted adoption, fully mediating the effect of ease of use and partially mediating usefulness. It was associated with high-arousal positive emotions(e.g., inspiration, pride, enthusiasm), reflecting eudaimonic states rather than hedonic fun. Engagement also significantly promoted adoption, though with weaker mediation effects. These findings extend TAM to high-stakes serious games and suggest that effective design should cultivate meaningful enjoyment and engagement without undermining scenario seriousness.
2.
Jin, Zhangyu; Feng, Andrew; Chemburkar, Ankur; Melo, Celso M. De
PromptGAR: Flexible Promptive Group Activity Recognition Proceedings Article
In: Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision, pp. 4461–4471, 2026.
@inproceedings{jin_promptgar_2026,
title = {PromptGAR: Flexible Promptive Group Activity Recognition},
author = {Zhangyu Jin and Andrew Feng and Ankur Chemburkar and Celso M. De Melo},
url = {https://openaccess.thecvf.com/content/WACV2026/papers/Jin_PromptGAR_Flexible_Promptive_Group_Activity_Recognition_WACV_2026_paper.pdf},
year = {2026},
date = {2026-03-01},
booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision},
pages = {4461–4471},
abstract = {We present PromptGAR, a novel framework for Group Activity Recognition (GAR) that offering both input flexibility and high recognition accuracy. The existing approaches suffer from limited real-world applicability due to their reliance on full prompt annotations, fixed number of frames and instances, and the lack of actor consistency. To bridge the gap, we proposed PromptGAR, which is the first GAR model to provide input flexibility across prompts, frames, and instances without the need for retraining. We leverage diverse visual prompts–like bounding boxes, skeletal keypoints, and instance identities–by unifying them as point prompts. A recognition decoder then cross-updates class and prompt tokens for enhanced performance. To ensure actor consistency for extended activity durations, we also introduce a relative instance attention mechanism that directly encodes instance identities. Comprehensive evaluations demonstrate that PromptGAR achieves competitive performances both on full prompts and partial prompt inputs, establishing its effectiveness on input flexibility and generalization ability for real-world applications.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We present PromptGAR, a novel framework for Group Activity Recognition (GAR) that offering both input flexibility and high recognition accuracy. The existing approaches suffer from limited real-world applicability due to their reliance on full prompt annotations, fixed number of frames and instances, and the lack of actor consistency. To bridge the gap, we proposed PromptGAR, which is the first GAR model to provide input flexibility across prompts, frames, and instances without the need for retraining. We leverage diverse visual prompts–like bounding boxes, skeletal keypoints, and instance identities–by unifying them as point prompts. A recognition decoder then cross-updates class and prompt tokens for enhanced performance. To ensure actor consistency for extended activity durations, we also introduce a relative instance attention mechanism that directly encodes instance identities. Comprehensive evaluations demonstrate that PromptGAR achieves competitive performances both on full prompts and partial prompt inputs, establishing its effectiveness on input flexibility and generalization ability for real-world applications.
3.
Chaubey, Ashutosh; Guan, Xulang; Soleymani, Mohammad
Face-LLaVA: Facial Expression and Attribute Understanding through Instruction Tuning Proceedings Article
In: Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision, pp. 2648–2660, 2026.
@inproceedings{chaubey_face-llava_2026,
title = {Face-LLaVA: Facial Expression and Attribute Understanding through Instruction Tuning},
author = {Ashutosh Chaubey and Xulang Guan and Mohammad Soleymani},
year = {2026},
date = {2026-03-01},
booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision},
pages = {2648–2660},
abstract = {The human face plays a central role in social communication, necessitating the use of performant computer vision tools for human-centered applications. We propose Face-LLaVA, a multimodal large language model for face-centered, in-context learning, including facial expression and attribute recognition. Additionally, Face-LLaVA is able to generate natural language descriptions that can be used for reasoning. Leveraging existing visual databases, we first developed FaceInstruct-1M, a face-centered database for instruction tuning MLLMs for face processing. We then developed a novel face-specific visual encoder powered by Face-Region Guided Cross-Attention that integrates face geometry with local visual features. We evaluated the proposed method across nine different datasets and five different face processing tasks, including facial expression recognition, action unit detection, facial attribute detection, age estimation and deepfake detection. Face-LLaVA achieves superior results compared to existing open-source MLLMs and competitive performance compared to commercial solutions. Our model output also receives a higher reasoning rating by GPT under a zero-shot setting across all the tasks. Both our dataset and model will be made public to support future advancements in social AI and foundational vision-language research.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The human face plays a central role in social communication, necessitating the use of performant computer vision tools for human-centered applications. We propose Face-LLaVA, a multimodal large language model for face-centered, in-context learning, including facial expression and attribute recognition. Additionally, Face-LLaVA is able to generate natural language descriptions that can be used for reasoning. Leveraging existing visual databases, we first developed FaceInstruct-1M, a face-centered database for instruction tuning MLLMs for face processing. We then developed a novel face-specific visual encoder powered by Face-Region Guided Cross-Attention that integrates face geometry with local visual features. We evaluated the proposed method across nine different datasets and five different face processing tasks, including facial expression recognition, action unit detection, facial attribute detection, age estimation and deepfake detection. Face-LLaVA achieves superior results compared to existing open-source MLLMs and competitive performance compared to commercial solutions. Our model output also receives a higher reasoning rating by GPT under a zero-shot setting across all the tasks. Both our dataset and model will be made public to support future advancements in social AI and foundational vision-language research.
4.
Tran, Minh; Siniukov, Maksim; Jin, Zhangyu; Soleymani, Mohammad
Discrete Facial Encoding: A Framework for the Data-driven Facial Display Discovery Proceedings Article
In: Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision, pp. 2348–2358, 2026.
@inproceedings{tran_discrete_2026,
title = {Discrete Facial Encoding: A Framework for the Data-driven Facial Display Discovery},
author = {Minh Tran and Maksim Siniukov and Zhangyu Jin and Mohammad Soleymani},
year = {2026},
date = {2026-03-01},
booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision},
pages = {2348–2358},
abstract = {Facial expression analysis is central to understanding human behavior, yet existing coding systems such as the Facial Action Coding System (FACS) are constrained by limited coverage and costly manual annotation. In this work,
we introduce Discrete Facial Encoding (DFE), an unsupervised, data-driven alternative of compact and interpretable dictionary of facial expressions from 3D mesh sequences learned through a Residual Vector Quantized Variational Autoencoder (RVQ-VAE). Our approach first extracts identityinvariant expression features from images using a 3D Morphable Model (3DMM), effectively disentangling factors such as head pose and facial geometry. We then encode these features using an RVQ-VAE, producing a sequence of discrete tokens from a shared codebook, where each token captures a specific, reusable facial deformation patter that contributes to the overall expression. Through extensive experiments, we demonstrate that Discrete Facial Encoding captures more precise facial behaviors than FACS
and other facial encoding alternatives. We evaluate the
utility of our representation across three high-level psychological tasks: stress detection, personality prediction, and depression detection. Using a simple Bag-of-Words model built on top of the learned tokens, our system consistently outperforms both FACS-based pipelines and strong image and video representation learning models such as Masked Autoencoders. Further analysis reveals that our representation covers a wider variety of facial displays, highlighting its potential as a scalable and effective alternative to FACS for psychological and affective computing applications. We released the source code and model weights at https://github.com/ihp-lab/vqface.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Facial expression analysis is central to understanding human behavior, yet existing coding systems such as the Facial Action Coding System (FACS) are constrained by limited coverage and costly manual annotation. In this work,
we introduce Discrete Facial Encoding (DFE), an unsupervised, data-driven alternative of compact and interpretable dictionary of facial expressions from 3D mesh sequences learned through a Residual Vector Quantized Variational Autoencoder (RVQ-VAE). Our approach first extracts identityinvariant expression features from images using a 3D Morphable Model (3DMM), effectively disentangling factors such as head pose and facial geometry. We then encode these features using an RVQ-VAE, producing a sequence of discrete tokens from a shared codebook, where each token captures a specific, reusable facial deformation patter that contributes to the overall expression. Through extensive experiments, we demonstrate that Discrete Facial Encoding captures more precise facial behaviors than FACS
and other facial encoding alternatives. We evaluate the
utility of our representation across three high-level psychological tasks: stress detection, personality prediction, and depression detection. Using a simple Bag-of-Words model built on top of the learned tokens, our system consistently outperforms both FACS-based pipelines and strong image and video representation learning models such as Masked Autoencoders. Further analysis reveals that our representation covers a wider variety of facial displays, highlighting its potential as a scalable and effective alternative to FACS for psychological and affective computing applications. We released the source code and model weights at https://github.com/ihp-lab/vqface.
we introduce Discrete Facial Encoding (DFE), an unsupervised, data-driven alternative of compact and interpretable dictionary of facial expressions from 3D mesh sequences learned through a Residual Vector Quantized Variational Autoencoder (RVQ-VAE). Our approach first extracts identityinvariant expression features from images using a 3D Morphable Model (3DMM), effectively disentangling factors such as head pose and facial geometry. We then encode these features using an RVQ-VAE, producing a sequence of discrete tokens from a shared codebook, where each token captures a specific, reusable facial deformation patter that contributes to the overall expression. Through extensive experiments, we demonstrate that Discrete Facial Encoding captures more precise facial behaviors than FACS
and other facial encoding alternatives. We evaluate the
utility of our representation across three high-level psychological tasks: stress detection, personality prediction, and depression detection. Using a simple Bag-of-Words model built on top of the learned tokens, our system consistently outperforms both FACS-based pipelines and strong image and video representation learning models such as Masked Autoencoders. Further analysis reveals that our representation covers a wider variety of facial displays, highlighting its potential as a scalable and effective alternative to FACS for psychological and affective computing applications. We released the source code and model weights at https://github.com/ihp-lab/vqface.
5.
Hale, James; Dorley, Emmanuel J.; Kim, Peter H.; Gratch, Jonathan
IVAs to Study Gender Bias in Negotiation Journal Article
In: J Multimodal User Interfaces, vol. 20, no. 1, pp. 33–42, 2026, ISSN: 1783-7677, 1783-8738.
@article{hale_ivas_2026,
title = {IVAs to Study Gender Bias in Negotiation},
author = {James Hale and Emmanuel J. Dorley and Peter H. Kim and Jonathan Gratch},
url = {https://link.springer.com/10.1007/s12193-026-00476-5},
doi = {10.1007/s12193-026-00476-5},
issn = {1783-7677, 1783-8738},
year = {2026},
date = {2026-03-01},
urldate = {2026-06-18},
journal = {J Multimodal User Interfaces},
volume = {20},
number = {1},
pages = {33–42},
abstract = {Abstract
Virtual agents offer a powerful and underutilized methodology for studying social interaction, particularly how bias and stereotypes operate. Drawing on theoretical frameworks such as the Media Equation and supported by empirical studies, we demonstrate how virtual agents can serve as controlled proxies for human social actors to study social bias. Through two empirical studies in the negotiation domain, we show how gender cues can be studied in negotiation contexts. Study 1 examines the effects of gendered job descriptions on negotiation behavior — finding that gendered language and gender interact to affect participants’ minimally acceptable salary. Study 2 explores how agent embodiment and emotional tone influence user perceptions across gender lines — finding that agent embodiment and one’s gender interact to affect participants’ subjective feelings about the dispute. Together, these findings imply the need for greater use of virtual agents in research, as they help inform the development of more inclusive and equitable AI systems.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Abstract
Virtual agents offer a powerful and underutilized methodology for studying social interaction, particularly how bias and stereotypes operate. Drawing on theoretical frameworks such as the Media Equation and supported by empirical studies, we demonstrate how virtual agents can serve as controlled proxies for human social actors to study social bias. Through two empirical studies in the negotiation domain, we show how gender cues can be studied in negotiation contexts. Study 1 examines the effects of gendered job descriptions on negotiation behavior — finding that gendered language and gender interact to affect participants’ minimally acceptable salary. Study 2 explores how agent embodiment and emotional tone influence user perceptions across gender lines — finding that agent embodiment and one’s gender interact to affect participants’ subjective feelings about the dispute. Together, these findings imply the need for greater use of virtual agents in research, as they help inform the development of more inclusive and equitable AI systems.
Virtual agents offer a powerful and underutilized methodology for studying social interaction, particularly how bias and stereotypes operate. Drawing on theoretical frameworks such as the Media Equation and supported by empirical studies, we demonstrate how virtual agents can serve as controlled proxies for human social actors to study social bias. Through two empirical studies in the negotiation domain, we show how gender cues can be studied in negotiation contexts. Study 1 examines the effects of gendered job descriptions on negotiation behavior — finding that gendered language and gender interact to affect participants’ minimally acceptable salary. Study 2 explores how agent embodiment and emotional tone influence user perceptions across gender lines — finding that agent embodiment and one’s gender interact to affect participants’ subjective feelings about the dispute. Together, these findings imply the need for greater use of virtual agents in research, as they help inform the development of more inclusive and equitable AI systems.
6.
Zhang, Tianyi; Traum, David
Rethinking Evaluation in Retrieval-Augmented Personalized Dialogue: A Cognitive and Linguistic Perspective Miscellaneous
2026, (arXiv:2603.14217 [cs.CL]).
@misc{zhang_rethinking_2026,
title = {Rethinking Evaluation in Retrieval-Augmented Personalized Dialogue: A Cognitive and Linguistic Perspective},
author = {Tianyi Zhang and David Traum},
url = {http://arxiv.org/abs/2603.14217},
doi = {10.48550/arXiv.2603.14217},
year = {2026},
date = {2026-03-01},
urldate = {2026-06-18},
publisher = {arXiv},
abstract = {In cognitive science and linguistic theory, dialogue is not seen as a chain of independent utterances but rather as a joint activity sustained by coherence, consistency, and shared understanding. However, many systems for open-domain and personalized dialogue use surface-level similarity metrics (e.g., BLEU, ROUGE, F1) as one of their main reporting measures, which fail to capture these deeper aspects of conversational quality. We re-examine a notable retrieval-augmented framework for personalized dialogue, LAPDOG, as a case study for evaluation methodology. Using both human and LLM-based judges, we identify limitations in current evaluation practices, including corrupted dialogue histories, contradictions between retrieved stories and persona, and incoherent response generation. Our results show that human and LLM judgments align closely but diverge from lexical similarity metrics, underscoring the need for cognitively grounded evaluation methods. Broadly, this work charts a path toward more reliable assessment frameworks for retrieval-augmented dialogue systems that better reflect the principles of natural human communication.},
note = {arXiv:2603.14217 [cs.CL]},
keywords = {},
pubstate = {published},
tppubtype = {misc}
}
In cognitive science and linguistic theory, dialogue is not seen as a chain of independent utterances but rather as a joint activity sustained by coherence, consistency, and shared understanding. However, many systems for open-domain and personalized dialogue use surface-level similarity metrics (e.g., BLEU, ROUGE, F1) as one of their main reporting measures, which fail to capture these deeper aspects of conversational quality. We re-examine a notable retrieval-augmented framework for personalized dialogue, LAPDOG, as a case study for evaluation methodology. Using both human and LLM-based judges, we identify limitations in current evaluation practices, including corrupted dialogue histories, contradictions between retrieved stories and persona, and incoherent response generation. Our results show that human and LLM judgments align closely but diverge from lexical similarity metrics, underscoring the need for cognitively grounded evaluation methods. Broadly, this work charts a path toward more reliable assessment frameworks for retrieval-augmented dialogue systems that better reflect the principles of natural human communication.
7.
Kwon, Deuksin; Shrestha, Kaleen; Han, Bin; Lin, Spencer; Hale, James; Gratch, Jonathan; Matarić, Maja; Lucas, Gale M.
Can LLMs Truly Embody Human Personality? Analyzing AI and Human Behavior Alignment in Dispute Resolution Miscellaneous
2026, (arXiv:2602.07414 [cs.AI]).
@misc{kwon_can_2026,
title = {Can LLMs Truly Embody Human Personality? Analyzing AI and Human Behavior Alignment in Dispute Resolution},
author = {Deuksin Kwon and Kaleen Shrestha and Bin Han and Spencer Lin and James Hale and Jonathan Gratch and Maja Matarić and Gale M. Lucas},
url = {http://arxiv.org/abs/2602.07414},
doi = {10.48550/arXiv.2602.07414},
year = {2026},
date = {2026-02-01},
urldate = {2026-06-18},
publisher = {arXiv},
abstract = {Large language models (LLMs) are increasingly used to simulate human behavior in social settings such as legal mediation, negotiation, and dispute resolution. However, it remains unclear whether these simulations reproduce the personality-behavior patterns observed in humans. Human personality, for instance, shapes how individuals navigate social interactions, including strategic choices and behaviors in emotionally charged interactions. This raises the question: Can LLMs, when prompted with personality traits, reproduce personality-driven differences in human conflict behavior? To explore this, we introduce an evaluation framework that enables direct comparison of human-human and LLM-LLM behaviors in dispute resolution dialogues with respect to Big Five Inventory (BFI) personality traits. This framework provides a set of interpretable metrics related to strategic behavior and conflict outcomes. We additionally contribute a novel dataset creation methodology for LLM dispute resolution dialogues with matched scenarios and personality traits with respect to human conversations. Finally, we demonstrate the use of our evaluation framework with three contemporary closed-source LLMs and show significant divergences in how personality manifests in conflict across different LLMs compared to human data, challenging the assumption that personality-prompted agents can serve as reliable behavioral proxies in socially impactful applications. Our work highlights the need for psychological grounding and validation in AI simulations before real-world use.},
note = {arXiv:2602.07414 [cs.AI]},
keywords = {},
pubstate = {published},
tppubtype = {misc}
}
Large language models (LLMs) are increasingly used to simulate human behavior in social settings such as legal mediation, negotiation, and dispute resolution. However, it remains unclear whether these simulations reproduce the personality-behavior patterns observed in humans. Human personality, for instance, shapes how individuals navigate social interactions, including strategic choices and behaviors in emotionally charged interactions. This raises the question: Can LLMs, when prompted with personality traits, reproduce personality-driven differences in human conflict behavior? To explore this, we introduce an evaluation framework that enables direct comparison of human-human and LLM-LLM behaviors in dispute resolution dialogues with respect to Big Five Inventory (BFI) personality traits. This framework provides a set of interpretable metrics related to strategic behavior and conflict outcomes. We additionally contribute a novel dataset creation methodology for LLM dispute resolution dialogues with matched scenarios and personality traits with respect to human conversations. Finally, we demonstrate the use of our evaluation framework with three contemporary closed-source LLMs and show significant divergences in how personality manifests in conflict across different LLMs compared to human data, challenging the assumption that personality-prompted agents can serve as reliable behavioral proxies in socially impactful applications. Our work highlights the need for psychological grounding and validation in AI simulations before real-world use.
8.
Rodrigues, Patrick Borges; Lucas, Gale M.; Fang, Yiyang; Wang, Zihao; Roll, Shawn C.; Becerik-Gerber, Burcin; Soibelman, Lucio
Understanding Operators’ Sensory Needs for Human–Robot Interaction in Teleoperated Demolition Journal Article
In: J. Constr. Eng. Manage., vol. 151, no. 12, pp. 04025207, 2025, ISSN: 0733-9364, 1943-7862.
@article{rodrigues_understanding_2025-1,
title = {Understanding Operators’ Sensory Needs for Human–Robot Interaction in Teleoperated Demolition},
author = {Patrick Borges Rodrigues and Gale M. Lucas and Yiyang Fang and Zihao Wang and Shawn C. Roll and Burcin Becerik-Gerber and Lucio Soibelman},
url = {https://ascelibrary.org/doi/10.1061/JCEMD4.COENG-16911},
doi = {10.1061/JCEMD4.COENG-16911},
issn = {0733-9364, 1943-7862},
year = {2025},
date = {2025-12-01},
urldate = {2026-06-22},
journal = {J. Constr. Eng. Manage.},
volume = {151},
number = {12},
pages = {04025207},
abstract = {Despite the benefits of remotely operated compact demolition machines in constrained and hazardous environments, many safety and occupational risks remain. Operators often stay close to the machines, exposing themselves to the risks of structural collapses, falling debris, dust, and noise. In this study, we employed a generative user experience (UX) framework with six demolition industry professionals to inform the design considerations for a sensory-enhanced teleoperation interface for demolition robots. Through narrative interviews and focus groups, we identified demolition operators’ sensory needs during traditional demolition tasks involving manned and remotely operated machines and explored potential sensory features for a teleoperation interface for demolition robots. To increase the trustworthiness of the findings, we subsequently conducted a verification strategy where over 60 demolition stakeholders tested and provided feedback on the proposed teleoperation interface features within a virtual environment. Findings indicate that demolition operators use multiple sensory channels to perceive and interact with the environment during traditional demolition tasks. However, certain site conditions (e.g., noise, constrained spaces, clutter) still compromise their performance, safety, and well-being. Multimodal, simple, safe, and personalizable teleoperation interfaces that provide appropriate levels of control over the robots are preferred. Key suggestions included improvements in the robot’s controllers and the inclusion of mechanisms to facilitate communications with on-site workers and the perception of hazards. The findings inform the design considerations for a sensory-enhanced teleoperation interface for compact demolition robots. Additionally, the proposed generative UX framework can be applied to other construction applications to facilitate identifying operators’ needs and designing teleoperation interfaces.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Despite the benefits of remotely operated compact demolition machines in constrained and hazardous environments, many safety and occupational risks remain. Operators often stay close to the machines, exposing themselves to the risks of structural collapses, falling debris, dust, and noise. In this study, we employed a generative user experience (UX) framework with six demolition industry professionals to inform the design considerations for a sensory-enhanced teleoperation interface for demolition robots. Through narrative interviews and focus groups, we identified demolition operators’ sensory needs during traditional demolition tasks involving manned and remotely operated machines and explored potential sensory features for a teleoperation interface for demolition robots. To increase the trustworthiness of the findings, we subsequently conducted a verification strategy where over 60 demolition stakeholders tested and provided feedback on the proposed teleoperation interface features within a virtual environment. Findings indicate that demolition operators use multiple sensory channels to perceive and interact with the environment during traditional demolition tasks. However, certain site conditions (e.g., noise, constrained spaces, clutter) still compromise their performance, safety, and well-being. Multimodal, simple, safe, and personalizable teleoperation interfaces that provide appropriate levels of control over the robots are preferred. Key suggestions included improvements in the robot’s controllers and the inclusion of mechanisms to facilitate communications with on-site workers and the perception of hazards. The findings inform the design considerations for a sensory-enhanced teleoperation interface for compact demolition robots. Additionally, the proposed generative UX framework can be applied to other construction applications to facilitate identifying operators’ needs and designing teleoperation interfaces.
9.
Gordon, Andrew S.
Automated Formalization of William Shakespeare’s Venus and Adonis Using Large Language Models Proceedings Article
In: 12th Annual Conference on Advances in Cognitive Systems, pp. 259–274, Atlanta, GA, 2025.
@inproceedings{gordon_automated_2025,
title = {Automated Formalization of William Shakespeare’s Venus and Adonis Using Large Language Models},
author = {Andrew S. Gordon},
url = {https://asgordon.github.io/publications/ACS2025.PDF},
year = {2025},
date = {2025-10-01},
booktitle = {12th Annual Conference on Advances in Cognitive Systems},
volume = {ACS-2025 (259-274)},
pages = {259–274},
address = {Atlanta, GA},
abstract = {Advances in large language models have created new opportunities for the translation of natural
language into formal representations for use in symbolic reasoning systems, particularly when the
formal vocabulary is well-specified with sufficient training examples. In this paper, we explore the
more challenging problem of automating the knowledge engineering task of authoring the formal
vocabulary itself, based on the representational needs of the input text. We describe a methodology
that incrementally builds a formal vocabulary of first-order predicate-argument definitions and constant terms by prompting a large language model and restricting its output generations to adhere to
tightly constrained formalisms. We evaluate this methodology in its application to a challenging literary text, William Shakespeare’s narrative poem “Venus and Adonis” (1593), with a focus on how
well new vocabulary is reused by the large language model in after it has been introduced. We then
investigate the semantic content of the generated formalisms by manually sorting the vocabulary
into a taxonomy of 48 foundational areas used in knowledge representation research. Our results
point to several challenges in fully-automated knowledge engineering pipelines, but also point to
new opportunities in using large language models to support axiomization of foundational theories.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Advances in large language models have created new opportunities for the translation of natural
language into formal representations for use in symbolic reasoning systems, particularly when the
formal vocabulary is well-specified with sufficient training examples. In this paper, we explore the
more challenging problem of automating the knowledge engineering task of authoring the formal
vocabulary itself, based on the representational needs of the input text. We describe a methodology
that incrementally builds a formal vocabulary of first-order predicate-argument definitions and constant terms by prompting a large language model and restricting its output generations to adhere to
tightly constrained formalisms. We evaluate this methodology in its application to a challenging literary text, William Shakespeare’s narrative poem “Venus and Adonis” (1593), with a focus on how
well new vocabulary is reused by the large language model in after it has been introduced. We then
investigate the semantic content of the generated formalisms by manually sorting the vocabulary
into a taxonomy of 48 foundational areas used in knowledge representation research. Our results
point to several challenges in fully-automated knowledge engineering pipelines, but also point to
new opportunities in using large language models to support axiomization of foundational theories.
language into formal representations for use in symbolic reasoning systems, particularly when the
formal vocabulary is well-specified with sufficient training examples. In this paper, we explore the
more challenging problem of automating the knowledge engineering task of authoring the formal
vocabulary itself, based on the representational needs of the input text. We describe a methodology
that incrementally builds a formal vocabulary of first-order predicate-argument definitions and constant terms by prompting a large language model and restricting its output generations to adhere to
tightly constrained formalisms. We evaluate this methodology in its application to a challenging literary text, William Shakespeare’s narrative poem “Venus and Adonis” (1593), with a focus on how
well new vocabulary is reused by the large language model in after it has been introduced. We then
investigate the semantic content of the generated formalisms by manually sorting the vocabulary
into a taxonomy of 48 foundational areas used in knowledge representation research. Our results
point to several challenges in fully-automated knowledge engineering pipelines, but also point to
new opportunities in using large language models to support axiomization of foundational theories.
10.
Marti, Deniz; Budathoki, Anjila; Ding, Yi; Lucas, Gale; Nelson, David
How Does Acknowledging Users’ Preferences Impact AI’s Ability to Make Conflicting Recommendations? Journal Article
In: International Journal of Human–Computer Interaction, vol. 41, no. 15, pp. 9466–9477, 2025, ISSN: 1044-7318, 1532-7590.
@article{marti_how_2025,
title = {How Does Acknowledging Users’ Preferences Impact AI’s Ability to Make Conflicting Recommendations?},
author = {Deniz Marti and Anjila Budathoki and Yi Ding and Gale Lucas and David Nelson},
url = {https://www.tandfonline.com/doi/full/10.1080/10447318.2024.2426035},
doi = {10.1080/10447318.2024.2426035},
issn = {1044-7318, 1532-7590},
year = {2025},
date = {2025-08-01},
urldate = {2026-05-22},
journal = {International Journal of Human–Computer Interaction},
volume = {41},
number = {15},
pages = {9466–9477},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Filter
2026
Liu, Ruying; Becerik-Gerber, Burcin; Lucas, Gale M.
In: International Journal of Human–Computer Interaction, pp. 1–20, 2026, ISSN: 1044-7318, 1532-7590.
Abstract | Links | BibTeX | Tags: ARO, DTIC?, VR
@article{liu_technology_2026,
title = {Technology Acceptance in High-Stakes Serious Games: Extending Technology Acceptance Model (TAM) with Affective and Experiential Predictors in Active Shooter Preparedness},
author = {Ruying Liu and Burcin Becerik-Gerber and Gale M. Lucas},
url = {https://www.tandfonline.com/doi/full/10.1080/10447318.2026.2635682},
doi = {10.1080/10447318.2026.2635682},
issn = {1044-7318, 1532-7590},
year = {2026},
date = {2026-03-01},
urldate = {2026-06-18},
journal = {International Journal of Human–Computer Interaction},
pages = {1–20},
abstract = {Active shooter incidents (ASIs) represent extreme threats to public safety. Virtual reality (VR) serious games offer promising training potential, but user acceptance in high-stakes and emotionally intense contexts is underexplored. This study extends the Technology Acceptance Model (TAM) by incorporating affective (perceived enjoyment) and experiential (self-reported engagement) factors. Using a desktop-based VR ASI preparedness simulation with 50 healthcare workers, we tested the extended TAM model. Core TAM relationships hold for ease of use and usefulness, but enjoyment and engagement play distinct roles. Enjoyment strongly predicted adoption, fully mediating the effect of ease of use and partially mediating usefulness. It was associated with high-arousal positive emotions(e.g., inspiration, pride, enthusiasm), reflecting eudaimonic states rather than hedonic fun. Engagement also significantly promoted adoption, though with weaker mediation effects. These findings extend TAM to high-stakes serious games and suggest that effective design should cultivate meaningful enjoyment and engagement without undermining scenario seriousness.},
keywords = {ARO, DTIC?, VR},
pubstate = {published},
tppubtype = {article}
}
Active shooter incidents (ASIs) represent extreme threats to public safety. Virtual reality (VR) serious games offer promising training potential, but user acceptance in high-stakes and emotionally intense contexts is underexplored. This study extends the Technology Acceptance Model (TAM) by incorporating affective (perceived enjoyment) and experiential (self-reported engagement) factors. Using a desktop-based VR ASI preparedness simulation with 50 healthcare workers, we tested the extended TAM model. Core TAM relationships hold for ease of use and usefulness, but enjoyment and engagement play distinct roles. Enjoyment strongly predicted adoption, fully mediating the effect of ease of use and partially mediating usefulness. It was associated with high-arousal positive emotions(e.g., inspiration, pride, enthusiasm), reflecting eudaimonic states rather than hedonic fun. Engagement also significantly promoted adoption, though with weaker mediation effects. These findings extend TAM to high-stakes serious games and suggest that effective design should cultivate meaningful enjoyment and engagement without undermining scenario seriousness.
Jin, Zhangyu; Feng, Andrew; Chemburkar, Ankur; Melo, Celso M. De
PromptGAR: Flexible Promptive Group Activity Recognition Proceedings Article
In: Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision, pp. 4461–4471, 2026.
Abstract | Links | BibTeX | Tags: ARO, DTIC
@inproceedings{jin_promptgar_2026,
title = {PromptGAR: Flexible Promptive Group Activity Recognition},
author = {Zhangyu Jin and Andrew Feng and Ankur Chemburkar and Celso M. De Melo},
url = {https://openaccess.thecvf.com/content/WACV2026/papers/Jin_PromptGAR_Flexible_Promptive_Group_Activity_Recognition_WACV_2026_paper.pdf},
year = {2026},
date = {2026-03-01},
booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision},
pages = {4461–4471},
abstract = {We present PromptGAR, a novel framework for Group Activity Recognition (GAR) that offering both input flexibility and high recognition accuracy. The existing approaches suffer from limited real-world applicability due to their reliance on full prompt annotations, fixed number of frames and instances, and the lack of actor consistency. To bridge the gap, we proposed PromptGAR, which is the first GAR model to provide input flexibility across prompts, frames, and instances without the need for retraining. We leverage diverse visual prompts–like bounding boxes, skeletal keypoints, and instance identities–by unifying them as point prompts. A recognition decoder then cross-updates class and prompt tokens for enhanced performance. To ensure actor consistency for extended activity durations, we also introduce a relative instance attention mechanism that directly encodes instance identities. Comprehensive evaluations demonstrate that PromptGAR achieves competitive performances both on full prompts and partial prompt inputs, establishing its effectiveness on input flexibility and generalization ability for real-world applications.},
keywords = {ARO, DTIC},
pubstate = {published},
tppubtype = {inproceedings}
}
We present PromptGAR, a novel framework for Group Activity Recognition (GAR) that offering both input flexibility and high recognition accuracy. The existing approaches suffer from limited real-world applicability due to their reliance on full prompt annotations, fixed number of frames and instances, and the lack of actor consistency. To bridge the gap, we proposed PromptGAR, which is the first GAR model to provide input flexibility across prompts, frames, and instances without the need for retraining. We leverage diverse visual prompts–like bounding boxes, skeletal keypoints, and instance identities–by unifying them as point prompts. A recognition decoder then cross-updates class and prompt tokens for enhanced performance. To ensure actor consistency for extended activity durations, we also introduce a relative instance attention mechanism that directly encodes instance identities. Comprehensive evaluations demonstrate that PromptGAR achieves competitive performances both on full prompts and partial prompt inputs, establishing its effectiveness on input flexibility and generalization ability for real-world applications.
Chaubey, Ashutosh; Guan, Xulang; Soleymani, Mohammad
Face-LLaVA: Facial Expression and Attribute Understanding through Instruction Tuning Proceedings Article
In: Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision, pp. 2648–2660, 2026.
Abstract | BibTeX | Tags: ARO, DTIC, LLM, Natural Language, UARC
@inproceedings{chaubey_face-llava_2026,
title = {Face-LLaVA: Facial Expression and Attribute Understanding through Instruction Tuning},
author = {Ashutosh Chaubey and Xulang Guan and Mohammad Soleymani},
year = {2026},
date = {2026-03-01},
booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision},
pages = {2648–2660},
abstract = {The human face plays a central role in social communication, necessitating the use of performant computer vision tools for human-centered applications. We propose Face-LLaVA, a multimodal large language model for face-centered, in-context learning, including facial expression and attribute recognition. Additionally, Face-LLaVA is able to generate natural language descriptions that can be used for reasoning. Leveraging existing visual databases, we first developed FaceInstruct-1M, a face-centered database for instruction tuning MLLMs for face processing. We then developed a novel face-specific visual encoder powered by Face-Region Guided Cross-Attention that integrates face geometry with local visual features. We evaluated the proposed method across nine different datasets and five different face processing tasks, including facial expression recognition, action unit detection, facial attribute detection, age estimation and deepfake detection. Face-LLaVA achieves superior results compared to existing open-source MLLMs and competitive performance compared to commercial solutions. Our model output also receives a higher reasoning rating by GPT under a zero-shot setting across all the tasks. Both our dataset and model will be made public to support future advancements in social AI and foundational vision-language research.},
keywords = {ARO, DTIC, LLM, Natural Language, UARC},
pubstate = {published},
tppubtype = {inproceedings}
}
The human face plays a central role in social communication, necessitating the use of performant computer vision tools for human-centered applications. We propose Face-LLaVA, a multimodal large language model for face-centered, in-context learning, including facial expression and attribute recognition. Additionally, Face-LLaVA is able to generate natural language descriptions that can be used for reasoning. Leveraging existing visual databases, we first developed FaceInstruct-1M, a face-centered database for instruction tuning MLLMs for face processing. We then developed a novel face-specific visual encoder powered by Face-Region Guided Cross-Attention that integrates face geometry with local visual features. We evaluated the proposed method across nine different datasets and five different face processing tasks, including facial expression recognition, action unit detection, facial attribute detection, age estimation and deepfake detection. Face-LLaVA achieves superior results compared to existing open-source MLLMs and competitive performance compared to commercial solutions. Our model output also receives a higher reasoning rating by GPT under a zero-shot setting across all the tasks. Both our dataset and model will be made public to support future advancements in social AI and foundational vision-language research.
Tran, Minh; Siniukov, Maksim; Jin, Zhangyu; Soleymani, Mohammad
Discrete Facial Encoding: A Framework for the Data-driven Facial Display Discovery Proceedings Article
In: Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision, pp. 2348–2358, 2026.
Abstract | BibTeX | Tags: ARO, DTIC?, Virtual Humans
@inproceedings{tran_discrete_2026,
title = {Discrete Facial Encoding: A Framework for the Data-driven Facial Display Discovery},
author = {Minh Tran and Maksim Siniukov and Zhangyu Jin and Mohammad Soleymani},
year = {2026},
date = {2026-03-01},
booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision},
pages = {2348–2358},
abstract = {Facial expression analysis is central to understanding human behavior, yet existing coding systems such as the Facial Action Coding System (FACS) are constrained by limited coverage and costly manual annotation. In this work,
we introduce Discrete Facial Encoding (DFE), an unsupervised, data-driven alternative of compact and interpretable dictionary of facial expressions from 3D mesh sequences learned through a Residual Vector Quantized Variational Autoencoder (RVQ-VAE). Our approach first extracts identityinvariant expression features from images using a 3D Morphable Model (3DMM), effectively disentangling factors such as head pose and facial geometry. We then encode these features using an RVQ-VAE, producing a sequence of discrete tokens from a shared codebook, where each token captures a specific, reusable facial deformation patter that contributes to the overall expression. Through extensive experiments, we demonstrate that Discrete Facial Encoding captures more precise facial behaviors than FACS
and other facial encoding alternatives. We evaluate the
utility of our representation across three high-level psychological tasks: stress detection, personality prediction, and depression detection. Using a simple Bag-of-Words model built on top of the learned tokens, our system consistently outperforms both FACS-based pipelines and strong image and video representation learning models such as Masked Autoencoders. Further analysis reveals that our representation covers a wider variety of facial displays, highlighting its potential as a scalable and effective alternative to FACS for psychological and affective computing applications. We released the source code and model weights at https://github.com/ihp-lab/vqface.},
keywords = {ARO, DTIC?, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Facial expression analysis is central to understanding human behavior, yet existing coding systems such as the Facial Action Coding System (FACS) are constrained by limited coverage and costly manual annotation. In this work,
we introduce Discrete Facial Encoding (DFE), an unsupervised, data-driven alternative of compact and interpretable dictionary of facial expressions from 3D mesh sequences learned through a Residual Vector Quantized Variational Autoencoder (RVQ-VAE). Our approach first extracts identityinvariant expression features from images using a 3D Morphable Model (3DMM), effectively disentangling factors such as head pose and facial geometry. We then encode these features using an RVQ-VAE, producing a sequence of discrete tokens from a shared codebook, where each token captures a specific, reusable facial deformation patter that contributes to the overall expression. Through extensive experiments, we demonstrate that Discrete Facial Encoding captures more precise facial behaviors than FACS
and other facial encoding alternatives. We evaluate the
utility of our representation across three high-level psychological tasks: stress detection, personality prediction, and depression detection. Using a simple Bag-of-Words model built on top of the learned tokens, our system consistently outperforms both FACS-based pipelines and strong image and video representation learning models such as Masked Autoencoders. Further analysis reveals that our representation covers a wider variety of facial displays, highlighting its potential as a scalable and effective alternative to FACS for psychological and affective computing applications. We released the source code and model weights at https://github.com/ihp-lab/vqface.
we introduce Discrete Facial Encoding (DFE), an unsupervised, data-driven alternative of compact and interpretable dictionary of facial expressions from 3D mesh sequences learned through a Residual Vector Quantized Variational Autoencoder (RVQ-VAE). Our approach first extracts identityinvariant expression features from images using a 3D Morphable Model (3DMM), effectively disentangling factors such as head pose and facial geometry. We then encode these features using an RVQ-VAE, producing a sequence of discrete tokens from a shared codebook, where each token captures a specific, reusable facial deformation patter that contributes to the overall expression. Through extensive experiments, we demonstrate that Discrete Facial Encoding captures more precise facial behaviors than FACS
and other facial encoding alternatives. We evaluate the
utility of our representation across three high-level psychological tasks: stress detection, personality prediction, and depression detection. Using a simple Bag-of-Words model built on top of the learned tokens, our system consistently outperforms both FACS-based pipelines and strong image and video representation learning models such as Masked Autoencoders. Further analysis reveals that our representation covers a wider variety of facial displays, highlighting its potential as a scalable and effective alternative to FACS for psychological and affective computing applications. We released the source code and model weights at https://github.com/ihp-lab/vqface.
Hale, James; Dorley, Emmanuel J.; Kim, Peter H.; Gratch, Jonathan
IVAs to Study Gender Bias in Negotiation Journal Article
In: J Multimodal User Interfaces, vol. 20, no. 1, pp. 33–42, 2026, ISSN: 1783-7677, 1783-8738.
Abstract | Links | BibTeX | Tags: ARO, LLM, Social, Virtual Agents
@article{hale_ivas_2026,
title = {IVAs to Study Gender Bias in Negotiation},
author = {James Hale and Emmanuel J. Dorley and Peter H. Kim and Jonathan Gratch},
url = {https://link.springer.com/10.1007/s12193-026-00476-5},
doi = {10.1007/s12193-026-00476-5},
issn = {1783-7677, 1783-8738},
year = {2026},
date = {2026-03-01},
urldate = {2026-06-18},
journal = {J Multimodal User Interfaces},
volume = {20},
number = {1},
pages = {33–42},
abstract = {Abstract
Virtual agents offer a powerful and underutilized methodology for studying social interaction, particularly how bias and stereotypes operate. Drawing on theoretical frameworks such as the Media Equation and supported by empirical studies, we demonstrate how virtual agents can serve as controlled proxies for human social actors to study social bias. Through two empirical studies in the negotiation domain, we show how gender cues can be studied in negotiation contexts. Study 1 examines the effects of gendered job descriptions on negotiation behavior — finding that gendered language and gender interact to affect participants’ minimally acceptable salary. Study 2 explores how agent embodiment and emotional tone influence user perceptions across gender lines — finding that agent embodiment and one’s gender interact to affect participants’ subjective feelings about the dispute. Together, these findings imply the need for greater use of virtual agents in research, as they help inform the development of more inclusive and equitable AI systems.},
keywords = {ARO, LLM, Social, Virtual Agents},
pubstate = {published},
tppubtype = {article}
}
Abstract
Virtual agents offer a powerful and underutilized methodology for studying social interaction, particularly how bias and stereotypes operate. Drawing on theoretical frameworks such as the Media Equation and supported by empirical studies, we demonstrate how virtual agents can serve as controlled proxies for human social actors to study social bias. Through two empirical studies in the negotiation domain, we show how gender cues can be studied in negotiation contexts. Study 1 examines the effects of gendered job descriptions on negotiation behavior — finding that gendered language and gender interact to affect participants’ minimally acceptable salary. Study 2 explores how agent embodiment and emotional tone influence user perceptions across gender lines — finding that agent embodiment and one’s gender interact to affect participants’ subjective feelings about the dispute. Together, these findings imply the need for greater use of virtual agents in research, as they help inform the development of more inclusive and equitable AI systems.
Virtual agents offer a powerful and underutilized methodology for studying social interaction, particularly how bias and stereotypes operate. Drawing on theoretical frameworks such as the Media Equation and supported by empirical studies, we demonstrate how virtual agents can serve as controlled proxies for human social actors to study social bias. Through two empirical studies in the negotiation domain, we show how gender cues can be studied in negotiation contexts. Study 1 examines the effects of gendered job descriptions on negotiation behavior — finding that gendered language and gender interact to affect participants’ minimally acceptable salary. Study 2 explores how agent embodiment and emotional tone influence user perceptions across gender lines — finding that agent embodiment and one’s gender interact to affect participants’ subjective feelings about the dispute. Together, these findings imply the need for greater use of virtual agents in research, as they help inform the development of more inclusive and equitable AI systems.
Zhang, Tianyi; Traum, David
Rethinking Evaluation in Retrieval-Augmented Personalized Dialogue: A Cognitive and Linguistic Perspective Miscellaneous
2026, (arXiv:2603.14217 [cs.CL]).
Abstract | Links | BibTeX | Tags: ARO, Dialogue, DTIC, Natural Language, UARC
@misc{zhang_rethinking_2026,
title = {Rethinking Evaluation in Retrieval-Augmented Personalized Dialogue: A Cognitive and Linguistic Perspective},
author = {Tianyi Zhang and David Traum},
url = {http://arxiv.org/abs/2603.14217},
doi = {10.48550/arXiv.2603.14217},
year = {2026},
date = {2026-03-01},
urldate = {2026-06-18},
publisher = {arXiv},
abstract = {In cognitive science and linguistic theory, dialogue is not seen as a chain of independent utterances but rather as a joint activity sustained by coherence, consistency, and shared understanding. However, many systems for open-domain and personalized dialogue use surface-level similarity metrics (e.g., BLEU, ROUGE, F1) as one of their main reporting measures, which fail to capture these deeper aspects of conversational quality. We re-examine a notable retrieval-augmented framework for personalized dialogue, LAPDOG, as a case study for evaluation methodology. Using both human and LLM-based judges, we identify limitations in current evaluation practices, including corrupted dialogue histories, contradictions between retrieved stories and persona, and incoherent response generation. Our results show that human and LLM judgments align closely but diverge from lexical similarity metrics, underscoring the need for cognitively grounded evaluation methods. Broadly, this work charts a path toward more reliable assessment frameworks for retrieval-augmented dialogue systems that better reflect the principles of natural human communication.},
note = {arXiv:2603.14217 [cs.CL]},
keywords = {ARO, Dialogue, DTIC, Natural Language, UARC},
pubstate = {published},
tppubtype = {misc}
}
In cognitive science and linguistic theory, dialogue is not seen as a chain of independent utterances but rather as a joint activity sustained by coherence, consistency, and shared understanding. However, many systems for open-domain and personalized dialogue use surface-level similarity metrics (e.g., BLEU, ROUGE, F1) as one of their main reporting measures, which fail to capture these deeper aspects of conversational quality. We re-examine a notable retrieval-augmented framework for personalized dialogue, LAPDOG, as a case study for evaluation methodology. Using both human and LLM-based judges, we identify limitations in current evaluation practices, including corrupted dialogue histories, contradictions between retrieved stories and persona, and incoherent response generation. Our results show that human and LLM judgments align closely but diverge from lexical similarity metrics, underscoring the need for cognitively grounded evaluation methods. Broadly, this work charts a path toward more reliable assessment frameworks for retrieval-augmented dialogue systems that better reflect the principles of natural human communication.
Kwon, Deuksin; Shrestha, Kaleen; Han, Bin; Lin, Spencer; Hale, James; Gratch, Jonathan; Matarić, Maja; Lucas, Gale M.
Can LLMs Truly Embody Human Personality? Analyzing AI and Human Behavior Alignment in Dispute Resolution Miscellaneous
2026, (arXiv:2602.07414 [cs.AI]).
Abstract | Links | BibTeX | Tags: ARO, DTIC, LLM, Simulation, UARC
@misc{kwon_can_2026,
title = {Can LLMs Truly Embody Human Personality? Analyzing AI and Human Behavior Alignment in Dispute Resolution},
author = {Deuksin Kwon and Kaleen Shrestha and Bin Han and Spencer Lin and James Hale and Jonathan Gratch and Maja Matarić and Gale M. Lucas},
url = {http://arxiv.org/abs/2602.07414},
doi = {10.48550/arXiv.2602.07414},
year = {2026},
date = {2026-02-01},
urldate = {2026-06-18},
publisher = {arXiv},
abstract = {Large language models (LLMs) are increasingly used to simulate human behavior in social settings such as legal mediation, negotiation, and dispute resolution. However, it remains unclear whether these simulations reproduce the personality-behavior patterns observed in humans. Human personality, for instance, shapes how individuals navigate social interactions, including strategic choices and behaviors in emotionally charged interactions. This raises the question: Can LLMs, when prompted with personality traits, reproduce personality-driven differences in human conflict behavior? To explore this, we introduce an evaluation framework that enables direct comparison of human-human and LLM-LLM behaviors in dispute resolution dialogues with respect to Big Five Inventory (BFI) personality traits. This framework provides a set of interpretable metrics related to strategic behavior and conflict outcomes. We additionally contribute a novel dataset creation methodology for LLM dispute resolution dialogues with matched scenarios and personality traits with respect to human conversations. Finally, we demonstrate the use of our evaluation framework with three contemporary closed-source LLMs and show significant divergences in how personality manifests in conflict across different LLMs compared to human data, challenging the assumption that personality-prompted agents can serve as reliable behavioral proxies in socially impactful applications. Our work highlights the need for psychological grounding and validation in AI simulations before real-world use.},
note = {arXiv:2602.07414 [cs.AI]},
keywords = {ARO, DTIC, LLM, Simulation, UARC},
pubstate = {published},
tppubtype = {misc}
}
Large language models (LLMs) are increasingly used to simulate human behavior in social settings such as legal mediation, negotiation, and dispute resolution. However, it remains unclear whether these simulations reproduce the personality-behavior patterns observed in humans. Human personality, for instance, shapes how individuals navigate social interactions, including strategic choices and behaviors in emotionally charged interactions. This raises the question: Can LLMs, when prompted with personality traits, reproduce personality-driven differences in human conflict behavior? To explore this, we introduce an evaluation framework that enables direct comparison of human-human and LLM-LLM behaviors in dispute resolution dialogues with respect to Big Five Inventory (BFI) personality traits. This framework provides a set of interpretable metrics related to strategic behavior and conflict outcomes. We additionally contribute a novel dataset creation methodology for LLM dispute resolution dialogues with matched scenarios and personality traits with respect to human conversations. Finally, we demonstrate the use of our evaluation framework with three contemporary closed-source LLMs and show significant divergences in how personality manifests in conflict across different LLMs compared to human data, challenging the assumption that personality-prompted agents can serve as reliable behavioral proxies in socially impactful applications. Our work highlights the need for psychological grounding and validation in AI simulations before real-world use.
2025
Rodrigues, Patrick Borges; Lucas, Gale M.; Fang, Yiyang; Wang, Zihao; Roll, Shawn C.; Becerik-Gerber, Burcin; Soibelman, Lucio
Understanding Operators’ Sensory Needs for Human–Robot Interaction in Teleoperated Demolition Journal Article
In: J. Constr. Eng. Manage., vol. 151, no. 12, pp. 04025207, 2025, ISSN: 0733-9364, 1943-7862.
Abstract | Links | BibTeX | Tags: ARO
@article{rodrigues_understanding_2025-1,
title = {Understanding Operators’ Sensory Needs for Human–Robot Interaction in Teleoperated Demolition},
author = {Patrick Borges Rodrigues and Gale M. Lucas and Yiyang Fang and Zihao Wang and Shawn C. Roll and Burcin Becerik-Gerber and Lucio Soibelman},
url = {https://ascelibrary.org/doi/10.1061/JCEMD4.COENG-16911},
doi = {10.1061/JCEMD4.COENG-16911},
issn = {0733-9364, 1943-7862},
year = {2025},
date = {2025-12-01},
urldate = {2026-06-22},
journal = {J. Constr. Eng. Manage.},
volume = {151},
number = {12},
pages = {04025207},
abstract = {Despite the benefits of remotely operated compact demolition machines in constrained and hazardous environments, many safety and occupational risks remain. Operators often stay close to the machines, exposing themselves to the risks of structural collapses, falling debris, dust, and noise. In this study, we employed a generative user experience (UX) framework with six demolition industry professionals to inform the design considerations for a sensory-enhanced teleoperation interface for demolition robots. Through narrative interviews and focus groups, we identified demolition operators’ sensory needs during traditional demolition tasks involving manned and remotely operated machines and explored potential sensory features for a teleoperation interface for demolition robots. To increase the trustworthiness of the findings, we subsequently conducted a verification strategy where over 60 demolition stakeholders tested and provided feedback on the proposed teleoperation interface features within a virtual environment. Findings indicate that demolition operators use multiple sensory channels to perceive and interact with the environment during traditional demolition tasks. However, certain site conditions (e.g., noise, constrained spaces, clutter) still compromise their performance, safety, and well-being. Multimodal, simple, safe, and personalizable teleoperation interfaces that provide appropriate levels of control over the robots are preferred. Key suggestions included improvements in the robot’s controllers and the inclusion of mechanisms to facilitate communications with on-site workers and the perception of hazards. The findings inform the design considerations for a sensory-enhanced teleoperation interface for compact demolition robots. Additionally, the proposed generative UX framework can be applied to other construction applications to facilitate identifying operators’ needs and designing teleoperation interfaces.},
keywords = {ARO},
pubstate = {published},
tppubtype = {article}
}
Despite the benefits of remotely operated compact demolition machines in constrained and hazardous environments, many safety and occupational risks remain. Operators often stay close to the machines, exposing themselves to the risks of structural collapses, falling debris, dust, and noise. In this study, we employed a generative user experience (UX) framework with six demolition industry professionals to inform the design considerations for a sensory-enhanced teleoperation interface for demolition robots. Through narrative interviews and focus groups, we identified demolition operators’ sensory needs during traditional demolition tasks involving manned and remotely operated machines and explored potential sensory features for a teleoperation interface for demolition robots. To increase the trustworthiness of the findings, we subsequently conducted a verification strategy where over 60 demolition stakeholders tested and provided feedback on the proposed teleoperation interface features within a virtual environment. Findings indicate that demolition operators use multiple sensory channels to perceive and interact with the environment during traditional demolition tasks. However, certain site conditions (e.g., noise, constrained spaces, clutter) still compromise their performance, safety, and well-being. Multimodal, simple, safe, and personalizable teleoperation interfaces that provide appropriate levels of control over the robots are preferred. Key suggestions included improvements in the robot’s controllers and the inclusion of mechanisms to facilitate communications with on-site workers and the perception of hazards. The findings inform the design considerations for a sensory-enhanced teleoperation interface for compact demolition robots. Additionally, the proposed generative UX framework can be applied to other construction applications to facilitate identifying operators’ needs and designing teleoperation interfaces.
Gordon, Andrew S.
Automated Formalization of William Shakespeare’s Venus and Adonis Using Large Language Models Proceedings Article
In: 12th Annual Conference on Advances in Cognitive Systems, pp. 259–274, Atlanta, GA, 2025.
Abstract | Links | BibTeX | Tags: ARO, Cognitive Architecture, DTIC, LLM, UARC
@inproceedings{gordon_automated_2025,
title = {Automated Formalization of William Shakespeare’s Venus and Adonis Using Large Language Models},
author = {Andrew S. Gordon},
url = {https://asgordon.github.io/publications/ACS2025.PDF},
year = {2025},
date = {2025-10-01},
booktitle = {12th Annual Conference on Advances in Cognitive Systems},
volume = {ACS-2025 (259-274)},
pages = {259–274},
address = {Atlanta, GA},
abstract = {Advances in large language models have created new opportunities for the translation of natural
language into formal representations for use in symbolic reasoning systems, particularly when the
formal vocabulary is well-specified with sufficient training examples. In this paper, we explore the
more challenging problem of automating the knowledge engineering task of authoring the formal
vocabulary itself, based on the representational needs of the input text. We describe a methodology
that incrementally builds a formal vocabulary of first-order predicate-argument definitions and constant terms by prompting a large language model and restricting its output generations to adhere to
tightly constrained formalisms. We evaluate this methodology in its application to a challenging literary text, William Shakespeare’s narrative poem “Venus and Adonis” (1593), with a focus on how
well new vocabulary is reused by the large language model in after it has been introduced. We then
investigate the semantic content of the generated formalisms by manually sorting the vocabulary
into a taxonomy of 48 foundational areas used in knowledge representation research. Our results
point to several challenges in fully-automated knowledge engineering pipelines, but also point to
new opportunities in using large language models to support axiomization of foundational theories.},
keywords = {ARO, Cognitive Architecture, DTIC, LLM, UARC},
pubstate = {published},
tppubtype = {inproceedings}
}
Advances in large language models have created new opportunities for the translation of natural
language into formal representations for use in symbolic reasoning systems, particularly when the
formal vocabulary is well-specified with sufficient training examples. In this paper, we explore the
more challenging problem of automating the knowledge engineering task of authoring the formal
vocabulary itself, based on the representational needs of the input text. We describe a methodology
that incrementally builds a formal vocabulary of first-order predicate-argument definitions and constant terms by prompting a large language model and restricting its output generations to adhere to
tightly constrained formalisms. We evaluate this methodology in its application to a challenging literary text, William Shakespeare’s narrative poem “Venus and Adonis” (1593), with a focus on how
well new vocabulary is reused by the large language model in after it has been introduced. We then
investigate the semantic content of the generated formalisms by manually sorting the vocabulary
into a taxonomy of 48 foundational areas used in knowledge representation research. Our results
point to several challenges in fully-automated knowledge engineering pipelines, but also point to
new opportunities in using large language models to support axiomization of foundational theories.
language into formal representations for use in symbolic reasoning systems, particularly when the
formal vocabulary is well-specified with sufficient training examples. In this paper, we explore the
more challenging problem of automating the knowledge engineering task of authoring the formal
vocabulary itself, based on the representational needs of the input text. We describe a methodology
that incrementally builds a formal vocabulary of first-order predicate-argument definitions and constant terms by prompting a large language model and restricting its output generations to adhere to
tightly constrained formalisms. We evaluate this methodology in its application to a challenging literary text, William Shakespeare’s narrative poem “Venus and Adonis” (1593), with a focus on how
well new vocabulary is reused by the large language model in after it has been introduced. We then
investigate the semantic content of the generated formalisms by manually sorting the vocabulary
into a taxonomy of 48 foundational areas used in knowledge representation research. Our results
point to several challenges in fully-automated knowledge engineering pipelines, but also point to
new opportunities in using large language models to support axiomization of foundational theories.
Marti, Deniz; Budathoki, Anjila; Ding, Yi; Lucas, Gale; Nelson, David
How Does Acknowledging Users’ Preferences Impact AI’s Ability to Make Conflicting Recommendations? Journal Article
In: International Journal of Human–Computer Interaction, vol. 41, no. 15, pp. 9466–9477, 2025, ISSN: 1044-7318, 1532-7590.
Links | BibTeX | Tags: AI, ARL, ARO, LLM, UARC
@article{marti_how_2025,
title = {How Does Acknowledging Users’ Preferences Impact AI’s Ability to Make Conflicting Recommendations?},
author = {Deniz Marti and Anjila Budathoki and Yi Ding and Gale Lucas and David Nelson},
url = {https://www.tandfonline.com/doi/full/10.1080/10447318.2024.2426035},
doi = {10.1080/10447318.2024.2426035},
issn = {1044-7318, 1532-7590},
year = {2025},
date = {2025-08-01},
urldate = {2026-05-22},
journal = {International Journal of Human–Computer Interaction},
volume = {41},
number = {15},
pages = {9466–9477},
keywords = {AI, ARL, ARO, LLM, UARC},
pubstate = {published},
tppubtype = {article}
}