Publications

@inproceedings{mozgai_accelerating_2024,

title = {Accelerating Scoping Reviews: A Case Study in the User-Centered Design of an AI-Enabled Interdisciplinary Research Tool},

author = {Sharon A Mozgai and Cari Kaurloto and Jade G Winn and Andrew Leeds and Sarah Beland and Arman Sookiassian and Arno Hartholt},

url = {https://dl.acm.org/doi/10.1145/3613905.3637110},

doi = {10.1145/3613905.3637110},

isbn = {9798400703317},

year  = {2024},

date = {2024-05-01},

urldate = {2024-06-18},

booktitle = {Extended Abstracts of the CHI Conference on Human Factors in Computing Systems},

pages = {1–8},

publisher = {ACM},

address = {Honolulu HI USA},

keywords = {AI, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_multidisciplinary_2024,

title = {Multidisciplinary Research & Development of Multi-Agents and Virtual Humans Leveraging Integrated Middleware Platforms},

author = {Arno Hartholt and Andrew Leeds and Ed Fast and Edwin Sookiassian and Kevin Kim and Sarah Beland and Pranav Kulkarni and Sharon Mozgai},

url = {https://openaccess.cms-conferences.org/publications/book/978-1-958651-95-7/article/978-1-958651-95-7_33},

doi = {10.54941/ahfe1004497},

year  = {2024},

date = {2024-01-01},

urldate = {2024-04-16},

abstract = {The current pace of technological advancements has led to an ever-increasing availability of technologies to investigate and help address the challenges that contemporary society faces today. However, while this trend increases the potential for creating more relevant, effective, and efficient solutions, it also inherently increases the complexity of realizing that potential. Our work aims to manage this complexity through the creation and dissemination of integrated middleware platforms that enable researchers and developers to rapidly prototype novel solutions within the areas of modelling & simulation, virtual humans, and virtual worlds. In this paper, we discuss two related platforms: the Rapid Integration & Development Environment (RIDE) and the Virtual Human Toolkit (VHToolkit). Specifically, we explore two use cases: 1) the development of an authoring tool aimed at domain experts to rapidly create low-echelon military training scenarios, and 2) the development of a virtual human led mHealth wellness and suicide prevention app for veterans.},

keywords = {DTIC, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_creating_2023,

title = {Creating Virtual Worlds with the Virtual Human Toolkit and the Rapid Integration & Development Environment},

author = {Arno Hartholt and Sharon Mozgai},

url = {https://openaccess.cms-conferences.org/publications/book/978-1-958651-45-2/article/978-1-958651-45-2_41},

doi = {10.54941/ahfe1002856},

isbn = {978-1-958651-45-2},

year  = {2023},

date = {2023-01-01},

urldate = {2023-03-31},

booktitle = {Intelligent Human Systems Integration (IHSI 2023): Integrating People and Intelligent Systems},

volume = {69},

publisher = {AHFE Open Acces},

abstract = {The research and development of virtual humans, and the virtual worlds they inhabit, is inherently complex, requiring interdisciplinary approaches that combine social sciences, computer science, design, art, production, and domain expertise. Our previous work in managing this complexity has resulted in the release of the Virtual Human Toolkit (VHToolkit), aimed at lowering the burden of creating embodied conversational agents. In our current efforts, we are integrating the VHToolkit with the Rapid Integration & Development Environment (RIDE), a rapid prototyping modeling and simulation middleware platform that leverages real-time game engines. This integration results in the ability to mix and match commercial AI services from AWS, Azure, and Google, as well as leverage novel 3D geospatial terrain creation pipelines. Combined with dedicated authoring tools that have been developed through human-centered design processes, the platform enables researchers, developers, and domain experts to rapidly create digital worlds with virtual humans for both military and civilian contexts. Our approach is highly interdisciplinary, including academia, government, and industry collaborators. The demonstration shows a user interacting with an embodied conversational agent embedded within real-world captured and virtualized terrain. Further research and development features of the platform are shown, including scripted agent behaviors, networked team play, and machine learning interfaces.},

note = {ISSN: 27710718 

Issue: 69},

keywords = {DTIC, UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mozgai_toward_2023,

title = {Toward a Scoping Review of Social Intelligence in Virtual Humans},

author = {Sharon Mozgai and Sarah Beland and Andrew Leeds and Jade G. Winn and Cari Kaurloto and Dirk Heylen and Arno Hartholt},

url = {https://ieeexplore.ieee.org/abstract/document/10042532},

doi = {10.1109/FG57933.2023.10042532},

year  = {2023},

date = {2023-01-01},

booktitle = {2023 IEEE 17th International Conference on Automatic Face and Gesture Recognition (FG)},

pages = {1–6},

abstract = {As the demand for socially intelligent Virtual Humans (VHs) increases, so follows the demand for effective and efficient cross-discipline collaboration that is required to bring these VHs “to life”. One avenue for increasing cross-discipline fluency is the aggregation and organization of seemingly disparate areas of research and development (e.g., graphics and emotion models) that are essential to the field of VH research. Our initial investigation (1) identifies and catalogues research streams concentrated in three multidisciplinary VH topic clusters within the domain of social intelligence, Emotion, Social Behavior, and The Face, (2) brings to the forefront key themes and prolific authors within each topic cluster, and (3) provides evidence that a full scoping review is warranted to further map the field, aggregate research findings, and identify gaps in the research. To enable collaboration, we provide full access to the refined VH cluster datasets, key word and author word clouds, as well as interactive evidence maps.},

keywords = {DTIC, UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_re-architecting_2022,

title = {Re-architecting the virtual human toolkit: towards an interoperable platform for embodied conversational agent research and development},

author = {Arno Hartholt and Ed Fast and Zongjian Li and Kevin Kim and Andrew Leeds and Sharon Mozgai},

url = {https://dl.acm.org/doi/10.1145/3514197.3549671},

doi = {10.1145/3514197.3549671},

isbn = {978-1-4503-9248-8},

year  = {2022},

date = {2022-09-01},

urldate = {2022-09-15},

booktitle = {Proceedings of the 22nd ACM International Conference on Intelligent Virtual Agents},

pages = {1–8},

publisher = {ACM},

address = {Faro Portugal},

keywords = {DTIC, UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mozgai_toward_2022,

title = {Toward a Semi-Automated Scoping Review of Virtual Human Smiles},

author = {Sharon Mozgai and Jade Winn and Cari Kaurloto and Andrew Leeds and Dirk Heylen and Arno Hartholt},

url = {http://www.lrec-conf.org/proceedings/lrec2022/workshops/SmiLa/index.html},

year  = {2022},

date = {2022-06-01},

booktitle = {Proceedings of the Smiling and Laughter across Contexts and the Life-span Workshop},

keywords = {DTIC, UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_demonstrating_2022,

title = {Demonstrating the Rapid Integration & Development Environment (RIDE): Embodied Conversational Agent (ECA) and Multiagent Capabilities},

author = {Arno Hartholt and Ed Fast and Andrew Leeds and Kevin Kim and Andrew Gordon and Kyle McCullough and Volkan Ustun and Sharon Mozgai},

isbn = {978-1-4503-9213-6},

year  = {2022},

date = {2022-05-01},

urldate = {2022-09-20},

booktitle = {Proceedings of the 21st International Conference on Autonomous Agents and Multiagent Systems},

pages = {1902–1904},

publisher = {International Foundation for Autonomous Agents and Multiagent Systems},

address = {Richland, SC},

series = {AAMAS '22},

abstract = {We demonstrate the Rapid Integration & Development Environment (RIDE), a research and development platform that enables rapid prototyping in support of multiagents and embodied conversational agents. RIDE is based on commodity game engines and includes a flexible architecture, system interoperability, and native support for artificial intelligence and machine learning frameworks.},

keywords = {AI, DTIC, Integration Technology, Machine Learning, UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_rapid_2021,

title = {Rapid Prototyping for Simulation and Training with the Rapid Integration & Development Environment (RIDE)},

author = {Arno Hartholt and Kyle McCullough and Ed Fast and Andrew Leeds and Sharon Mozgai and Tim Aris and Volkan Ustun and Andrew Gordon and Christopher McGroarty},

year  = {2021},

date = {2021-11-01},

keywords = {AI, DTIC, Integration Technology, Machine Learning, Simulation, UARC, VHTL},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mozgai_building_2021,

title = {Building BRAVEMIND Vietnam: User-Centered Design for Virtual Reality Exposure Therapy},

author = {Sharon Mozgai and Andrew Leeds and David Kwok and Ed Fast and Albert Skip Rizzo and Arno Hartholt},

doi = {10.1109/AIVR52153.2021.00056},

year  = {2021},

date = {2021-11-01},

booktitle = {2021 IEEE International Conference on Artificial Intelligence and Virtual Reality (AIVR)},

pages = {247–250},

abstract = {There has been a recent resurgence of Vietnam-era Veterans seeking Veteran Affairs (VA) services for post-traumatic stress disorder (PTSD). Multiple studies have reported positive outcomes using Virtual Reality Exposure Therapy (VRET) with Iraq/Afghanistan Veteran PTSD patients, but there have been fewer investigations into the acceptability and feasibility of VRET with older Veterans. We have extended an established VRET system, BRAVEMIND, to include Vietnam relevant content following a user-centered iterative design methodology. This paper will present the BRAVEMIND Vietnam VRET system, including setup, content, and technical architecture along with the findings of an initial focus group with Vietnam Veterans. These Veterans rated this system to be acceptable as a treatment tool for combat-related PTSD and provided valuable first-person accounts of their time in Vietnam to help guide the evolution of the VR content.},

keywords = {DTIC, MedVR, VHTL, VR},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_introducing_2021-1,

title = {Introducing VHMason: A Visual, Integrated, Multimodal Virtual Human Authoring Tool},

author = {Arno Hartholt and Ed Fast and Andrew Leeds and Sharon Mozgai},

url = {https://dl.acm.org/doi/10.1145/3472306.3478363},

doi = {10.1145/3472306.3478363},

isbn = {978-1-4503-8619-7},

year  = {2021},

date = {2021-09-01},

urldate = {2023-03-31},

booktitle = {Proceedings of the 21st ACM International Conference on Intelligent Virtual Agents},

pages = {109–111},

publisher = {Association for Computing Machinery},

address = {New York, NY, USA},

series = {IVA '21},

abstract = {A major impediment to the success of virtual agents is the inability of non-technical experts to easily author content. To address this barrier we present VHMason, a multimodal authoring tool designed to help creative authors build embodied conversational agents. We introduce the novel aspects of this authoring tool and explore a use case of the creation of an agent-led educational experience implemented at Children's Hospital Los Angeles (CHLA).},

keywords = {UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_introducing_2020,

title = {Introducing Canvas: Combining Nonverbal Behavior Generation with User-Generated Content to Rapidly Create Educational Videos},

author = {Arno Hartholt and Adam Reilly and Ed Fast and Sharon Mozgai},

url = {https://dl.acm.org/doi/10.1145/3383652.3423880},

doi = {10.1145/3383652.3423880},

isbn = {978-1-4503-7586-3},

year  = {2020},

date = {2020-10-01},

booktitle = {Proceedings of the 20th ACM International Conference on Intelligent Virtual Agents},

pages = {1–3},

publisher = {ACM},

address = {Virtual Event Scotland UK},

abstract = {Rapidly creating educational content that is effective, engaging, and low-cost is a challenge. We present Canvas, a tool for educators that addresses this challenge by enabling the generation of educational video, led by an intelligent virtual agent, that combines rapid nonverbal behavior generation techniques with end-user facing authoring tools. With Canvas, educators can easily produce compelling educational videos with a minimum of investment by leveraging existing content provided by the tool (e.g., characters and environments) while incorporating their own custom content (e.g., images and video clips). Canvas has been delivered to the Smithsonian Science Education Center and is currently being evaluated internally before wider release. We discuss the system, feature set, design process, and lessons learned.},

keywords = {VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mozgai_iterative_2020,

title = {Iterative Participatory Design for VRET Domain Transfer: From Combat Exposure to Military Sexual Trauma},

author = {Sharon Mozgai and Arno Hartholt and Andrew Leeds and Albert ‘Skip’ Rizzo},

url = {https://dl.acm.org/doi/abs/10.1145/3334480.3375219},

doi = {10.1145/3334480.3375219},

year  = {2020},

date = {2020-04-01},

booktitle = {Proceedings of the 2020 CHI Conference of Human Factors in Computing Systems},

pages = {8},

publisher = {ACM},

address = {Honolulu, HI},

abstract = {This case study describes the expansion of the BRAVEMIND virtual reality exposure therapy (VRET) system from the domain of combat-related posttraumatic stress disorder (PTSD) to the domain of PTSD due to Military Sexual Trauma (MST). As VRET continues to demonstrate efficacy in treating PTSD across multiple trauma types and anxiety disorders, adapting existing systems and content to new domains while simultaneously maintaining clinical integrity is becoming a high priority. To develop BRAVEMIND-MST we engaged in an iterative participatory design process with psychologists, engineers, and artists. This first-person account of our collaborative development process focuses on three key areas (1) VR Environment, (2) User-Avatar State, and (3) Events, while detailing the challenges we encountered and lessons learned. This process culminated in eight design guidelines as a first-step in defining a VRET domain transfer methodology.},

keywords = {MedVR, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mozgai_passive_2020,

title = {The Passive Sensing Agent: A Multimodal Adaptive mHealth Application},

author = {Sharon Mozgai and Arno Hartholt and Albert Rizzo},

url = {https://ieeexplore.ieee.org/document/9156177/},

doi = {10.1109/PerComWorkshops48775.2020.9156177},

isbn = {978-1-72814-716-1},

year  = {2020},

date = {2020-03-01},

booktitle = {Proceedings of the 2020 IEEE International Conference on Pervasive Computing and Communications Workshops (PerCom Workshops)},

pages = {1–3},

publisher = {IEEE},

address = {Austin, TX, USA},

abstract = {We are demoing the Passive Sensing Agent (PSA), an mHealth virtual human coach, that collects multimodal data through passive sensors native to popular wearables (e.g., Apple Watch, FitBit, and Garmin). This virtual human interface delivers adaptive multi-media content via smartphone application that is specifically tailored to the user in the interdependent domains of physical, cognitive, and emotional health. Initially developed for the military, the PSA delivers health interventions (e.g., educational exercises, physical challenges, and performance feedback) matched to the individual user via novel adaptive logic-based algorithms while employing various behavior change techniques (e.g., goal-setting, barrier identification, rewards, modeling, etc.). A virtual human coach leads all interactions including the firsttime user experience and the brief daily sessions. All interactions were specifically designed to engage and motivate the user while continuously collecting data on their cognitive, emotional, and physical fitness. This multi-component application is integrated and deployed on an iPhone and Apple Watch prototype; a civilian version is currently in-development.},

keywords = {MedVR, UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mozgai_adaptive_2020,

title = {An Adaptive Agent-Based Interface for Personalized Health Interventions},

author = {Sharon Mozgai and Arno Hartholt and Albert "Skip" Rizzo},

url = {https://dl.acm.org/doi/10.1145/3379336.3381467},

doi = {10.1145/3379336.3381467},

isbn = {978-1-4503-7513-9},

year  = {2020},

date = {2020-03-01},

booktitle = {Proceedings of the 25th International Conference on Intelligent User Interfaces Companion},

pages = {118–119},

publisher = {ACM},

address = {Cagliari Italy},

abstract = {This demo introduces a novel mHealth application with an agentbased interface designed to collect multimodal data with passive sensors native to popular wearables (e.g., Apple Watch, FitBit, and Garmin) as well as through user self-report. This mHealth application delivers personalized and adaptive multimedia content via smartphone application specifically tailored to the user in the interdependent domains of physical, cognitive, and emotional health via novel adaptive logic-based algorithms while employing behavior change techniques (e.g., goal-setting, barrier identification, etc.). A virtual human coach leads all interactions to improve adherence.},

keywords = {MedVR, UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_ubiquitous_2019,

title = {Ubiquitous Virtual Humans: A Multi-platform Framework for Embodied AI Agents in XR},

author = {Arno Hartholt and Ed Fast and Adam Reilly and Wendy Whitcup and Matt Liewer and Sharon Mozgai},

url = {https://ieeexplore.ieee.org/document/8942321/},

doi = {10.1109/AIVR46125.2019.00072},

isbn = {978-1-72815-604-0},

year  = {2019},

date = {2019-12-01},

booktitle = {Proceedings of the 2019 IEEE International Conference on Artificial Intelligence and Virtual Reality (AIVR)},

pages = {308–3084},

publisher = {IEEE},

address = {San Diego, CA, USA},

abstract = {We present an architecture and framework for the development of virtual humans for a range of computing platforms, including mobile, web, Virtual Reality (VR) and Augmented Reality (AR). The framework uses a mix of in-house and commodity technologies to support audio-visual sensing, speech recognition, natural language processing, nonverbal behavior generation and realization, text-to-speech generation, and rendering. This work builds on the Virtual Human Toolkit, which has been extended to support computing platforms beyond Windows. The resulting framework maintains the modularity of the underlying architecture, allows re-use of both logic and content through cloud services, and is extensible by porting lightweight clients. We present the current state of the framework, discuss how we model and animate our characters, and offer lessons learned through several use cases, including expressive character animation in seated VR, shared space and navigation in roomscale VR, autonomous AI in mobile AR, and real-time user performance feedback based on mobile sensors in headset AR.},

keywords = {UARC, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_virtual_2019,

title = {Virtual Humans in Augmented Reality: A First Step towards Real-World Embedded Virtual Roleplayers},

author = {Arno Hartholt and Sharon Mozgai and Ed Fast and Matt Liewer and Adam Reilly and Wendy Whitcup and Albert "Skip" Rizzo},

url = {http://dl.acm.org/citation.cfm?doid=3349537.3352766},

doi = {10.1145/3349537.3352766},

isbn = {978-1-4503-6922-0},

year  = {2019},

date = {2019-10-01},

booktitle = {Proceedings of the 7th International Conference on Human-Agent Interaction - HAI '19},

pages = {205–207},

publisher = {ACM Press},

address = {Kyoto, Japan},

abstract = {We present one of the first applications of virtual humans in Augmented Reality (AR), which allows young adults with Autism Spectrum Disorder (ASD) the opportunity to practice job interviews. It uses the Magic Leap’s AR hardware sensors to provide users with immediate feedback on six different metrics, including eye gaze, blink rate and head orientation. The system provides two characters, with three conversational modes each. Ported from an existing desktop application, the main development lessons learned were: 1) provide users with navigation instructions in the user interface, 2) avoid dark colors as they are rendered transparently, 3) use dynamic gaze so characters maintain eye contact with the user, 4) use hardware sensors like eye gaze to provide user feedback, and 5) use surface detection to place characters dynamically in the world.},

keywords = {MedVR, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_virtual_2019-1,

title = {Virtual Job Interviewing Practice for High-Anxiety Populations},

author = {Arno Hartholt and Sharon Mozgai and Albert "Skip" Rizzo},

url = {http://dl.acm.org/citation.cfm?doid=3308532.3329417},

doi = {10.1145/3308532.3329417},

isbn = {978-1-4503-6672-4},

year  = {2019},

date = {2019-07-01},

booktitle = {Proceedings of the 19th ACM International Conference on Intelligent Virtual Agents - IVA '19},

pages = {238–240},

publisher = {ACM Press},

address = {Paris, France},

abstract = {We present a versatile system for training job interviewing skills that focuses specifically on segments of the population facing increased challenges during the job application process. In particular, we target those with Autism Spectrum Disorder (ADS), veterans transitioning to civilian life, and former convicts integrating back into society. The system itself follows the SAIBA framework and contains several interviewer characters, who each represent a different type of vocational field, (e.g. service industry, retail, office, etc.) Each interviewer can be set to one of three conversational modes, which not only affects what they say and how they say it, but also their supporting body language. This approach offers varying difficulties, allowing users to start practicing with interviewers who are more encouraging and accommodating before moving on to personalities that are more direct and indifferent. Finally, the user can place the interviewers in different environmental settings (e.g. conference room, restaurant, executive office, etc.), allowing for many different combinations in which to practice.},

keywords = {MedVR, VHTL, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{gordon_primer_2019,

title = {PRIMER: An Emotionally Aware Virtual Agent},

author = {Carla Gordon and Anton Leuski and Grace Benn and Eric Klassen and Edward Fast and Matt Liewer and Arno Hartholt and David Traum},

url = {https://www.research.ibm.com/haifa/Workshops/user2agent2019/},

year  = {2019},

date = {2019-03-01},

booktitle = {Proceedings of the 24th International Conference on Intelligent User Interfaces},

pages = {10},

publisher = {ACM},

address = {Los Angeles, CA},

abstract = {PRIMER is a proof-of-concept system designed to show the potential of immersive dialogue agents and virtual environments that adapt and respond to both direct verbal input and indirect emotional input. The system has two novel interfaces: (1) for the user, an immersive VR environment and an animated virtual agent both of which adapt and react to the user’s direct input as well as the user’s perceived emotional state, and (2) for an observer, an interface that helps track the perceived emotional state of the user, with visualizations to provide insight into the system’s decision making process. While the basic system architecture can be adapted for many potential real world applications, the initial version of this system was designed to assist clinical social workers in helping children cope with bullying. The virtual agent produces verbal and non-verbal behaviors guided by a plan for the counseling session, based on in-depth discussions with experienced counselors, but is also reactive to both initiatives that the user takes, e.g. asking their own questions, and the user’s perceived emotional state.},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_building_2017,

title = {Building a Backbone for Multi-Agent Tutoring in GIFT (Work in Progress)},

author = {Benjamin D. Nye and Daniel Auerbach and Tirth R. Mehta and Arno Hartholt},

url = {https://books.google.com/books?id=PwMtDwAAQBAJ&printsec=copyright&source=gbs_pub_info_r#v=onepage&q&f=false},

year  = {2017},

date = {2017-05-01},

booktitle = {Proceedings of the GIFTSym5},

pages = {23–35},

publisher = {ARL},

address = {Orlando, Florida},

abstract = {As intelligent tutoring systems (ITS) increasingly need to interoperate and co-exist, emerging systems have transitioned toward service-oriented designs to enable modularity and composability of tutoring components made and/or maintained by different research and development groups. However, as a research community, we have still not reached a point where it is trivial for a new service to be added into a system like the Generalized Intelligent Framework for Tutoring (GIFT; Sottilare, Goldberg, Brawner, & Holden, 2012). In an early paper considering this issue with respect to the GIFT architecture (Nye & Morrison, 2013), we proposed addressing this issue by building toward a lightweight multi-agent archi-tecture where certain services act as autonomous agents: “a system situated within and a part of an environment that senses that environment and acts on it, over time, in pursuit of its own agenda and so as to affect what it senses in the future” (Franklin & Graesser, 1997; p. 25). In our work in progress described here, we discuss how we are approaching the opportunity to build such capabilities into GIFT. The high level goals of our work are targeting two core goals for GIFT: A) to be a lightweight framework that will expand access to and use of ITS and B) to help GIFT to increase the intelligence and effectiveness of its services based on data over time. We are currently targeting the first goal, which will underpin the second goal. However, what does it mean to be a lightweight framework? In this context, a “lightweight framework” is framed as minimizing the following criteria: (1) hardware requirements, (2) software expertise to design services, (3) software expertise to use existing services, (4) software expertise to stand up the message-passing layer between agents, and (5) a minimal working message ontology (Nye & Morrison, 2013). Since our original paper four years ago, GIFT has made significant strides in reducing barriers related to hardware by building a cloud-based version and software expertise to use GIFT services through authoring tools. It has also developed a growing ontology of messages (e.g., https://gifttutoring.org/projects/gift/wiki/Interface_Control_Document_2016-1). With that said, despite now-extensive documentation, designing new services for GIFT is still not trivial and strong expertise is required to pass messages between GIFT modules and agents (either internal or external). To address these issues, the Building a Backbone project is working toward agent-oriented designs that build on GIFT's existing service-oriented framework. By moving from services toward agents, modules will be able to act more autonomously, enabling capabilities such as plug-and-play, hotswapping, and selecting between multiple services providing the same capabilities. These new capabilities are intended to reduce barriers to building new GIFT-compatible services and also to integrating GIFT with other service-oriented ecosystems. The first steps toward these capabilities are an ontology mapping service and an initial integration that combines GIFT, the Virtual Human Toolkit core framework for agents, and the SuperGLU framework for adding agent-oriented capabilities for coordinating services. This paper reports on work to date, with an emphasis on target capabilities, design decisions, challenges, and open research questions for this work.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{swartout_designing_2016,

title = {Designing a Personal Assistant for Life-Long Learning (PAL3)},

author = {William Swartout and Benjamin D. Nye and Arno Hartholt and Adam Reilly and Arthur C. Graesser and Kurt VanLehn and Jon Wetzel and Matt Liewer and Fabrizio Morbini and Brent Morgan and Lijia Wang and Grace Benn and Milton Rosenberg},

url = {http://www.aaai.org/ocs/index.php/FLAIRS/FLAIRS16/paper/view/12793},

isbn = {978-1-57735-756-8},

year  = {2016},

date = {2016-05-01},

booktitle = {Proceedings of The Twenty-Ninth International Flairs Conference},

pages = {491–496},

publisher = {AAAI Press},

address = {Key Largo, FL},

abstract = {Learners’ skills decay during gaps in instruction, since they lack the structure and motivation to continue studying. To meet this challenge, the PAL3 system was designed to accompany a learner throughout their career and mentor them to build and maintain skills through: 1) the use of an embodied pedagogical agent (Pal), 2) a persistent learning record that drives a student model which estimates forgetting, 3) an adaptive recommendation engine linking to both intelligent tutors and traditional learning resources, and 4) game-like mechanisms to promote engagement (e.g., leaderboards, effort-based point rewards, unlocking customizations). The design process for PAL3 is discussed, from the perspective of insights and revisions based on a series of formative feedback and evaluation sessions.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{stratou_demonstration_2015,

title = {A Demonstration of the Perception System in SimSensei, a Virtual Human Application for Healthcare Interviews},

author = {Giota Stratou and Louis-Philippe Morency and David DeVault and Arno Hartholt and Edward Fast and Margaux Lhommet and Gale Lucas and Fabrizio Morbini and Kallirroi Georgila and Stefan Scherer and Jonathan Gratch and Marcella Stacy and David Traum and Albert Rizzo},

url = {http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7344661},

doi = {10.1109/ACII.2015.7344661},

year  = {2015},

date = {2015-09-01},

booktitle = {Affective Computing and Intelligent Interaction (ACII), 2015 International Conference on},

pages = {787–789},

publisher = {IEEE},

address = {Xi'an, China},

abstract = {We present the SimSensei system, a fully automatic virtual agent that conducts interviews to assess indicators of psychological distress. With this demo, we focus our attention on the perception part of the system, a multimodal framework which captures and analyzes user state behavior for both behavioral understanding and interactional purposes. We will demonstrate real-time user state sensing as a part of the SimSensei architecture and discuss how this technology enabled automatic analysis of behaviors related to psychological distress.},

keywords = {MedVR, Social Simulation, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{bruijnes_virtual_2015,

title = {Virtual Suspect William},

author = {Merijn Bruijnes and Rieks Akker and Arno Hartholt and Dirk Heylen},

url = {http://ict.usc.edu/pubs/Virtual%20Suspect%20William.pdf},

year  = {2015},

date = {2015-08-01},

booktitle = {Intelligent Virtual Agents},

pages = {67–76},

publisher = {Springer},

abstract = {We evaluate an algorithm which computes the responses of an agent that plays the role of a suspect in simulations of police interrogations. The algorithm is based on a cognitive model - the response model - that is centred around keeping track of interpersonal relations. The model is parametrized in such a way that different personalities of the virtual suspect can be defined. In the evaluation we defined three different personalities and had participants guess the personality based on the responses the model provided in an interaction with the participant. We investigate what factors contributed to the ability of a virtual agent to show behaviour that was recognized by participants as belonging to a persona.},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{morency_simsensei_2015,

title = {SimSensei Demonstration: A Perceptive Virtual Human Interviewer for Healthcare Applications},

author = {Louis-Philippe Morency and Giota Stratou and David DeVault and Arno Hartholt and Margaux Lhommet and Gale Lucas and Fabrizio Morbini and Kallirroi Georgila and Stefan Scherer and Jonathan Gratch and Stacy Marsella and David Traum and Albert "Skip" Rizzo},

url = {http://ict.usc.edu/pubs/SimSensei%20Demonstration%20A%20Perceptive%20Virtual%20Human%20Interviewer%20for%20Healthcare%20Applications.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 29th AAAI Conference on Artificial Intelligence (AAAI)},

address = {Austin, Texas},

abstract = {We present the SimSensei system, a fully automatic virtual agent that conducts interviews to assess indicators of psychological distress. We emphasize on the perception part of the system, a multimodal framework which captures and analyzes user state for both behavioral understanding and interactional purposes.},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{rizzo_detection_2014,

title = {Detection and Computational Analysis of Psychological Signals Using a Virtual Human Interviewing Agent},

author = {Albert Rizzo and Stefan Scherer and David DeVault and Jonathan Gratch and Ron Artstein and Arno Hartholt and Gale Lucas and Stacy Marsella and Fabrizio Morbini and Angela Nazarian and Giota Stratou and David Traum and Rachel Wood and Jill Boberg and Louis-Philippe Morency},

url = {http://ict.usc.edu/pubs/Detection%20and%20Computational%20Analysis%20of%20Psychological%20Signals%20Using%20a%20Virtual%20Human%20Interviewing%20Agent.pdf},

year  = {2014},

date = {2014-12-01},

booktitle = {Proceedings of ICDVRAT 2014},

publisher = {International Journal of Disability and Human Development},

address = {Gothenburg, Sweden},

abstract = {It has long been recognized that facial expressions, body posture/gestures and vocal parameters play an important role in human communication and the implicit signalling of emotion. Recent advances in low cost computer vision and behavioral sensing technologies can now be applied to the process of making meaningful inferences as to user state when a person interacts with a computational device. Effective use of this additive information could serve to promote human interaction with virtual human (VH) agents that may enhance diagnostic assessment. This paper will focus on our current research in these areas within the DARPA-funded “Detection and Computational Analysis of Psychological Signals” project, with specific attention to the SimSensei application use case. SimSensei is a virtual human interaction platform that is able to sense and interpret real-time audiovisual behavioral signals from users interacting with the system. It is specifically designed for health care support and leverages years of virtual human research and development at USC-ICT. The platform enables an engaging face-to-face interaction where the virtual human automatically reacts to the state and inferred intent of the user through analysis of behavioral signals gleaned from facial expressions, body gestures and vocal parameters. Akin to how non-verbal behavioral signals have an impact on human to human interaction and communication, SimSensei aims to capture and infer from user non-verbal communication to improve engagement between a VH and a user. The system can also quantify and interpret sensed behavioral signals.},

keywords = {MedVR, Social Simulation, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_shared_2014,

title = {A Shared, Modular Architecture for Developing Virtual Humans},

author = {Arno Hartholt and David Traum and Stacy Marsella and Louis-Philippe Morency and Ari Shapiro and Jonathan Gratch},

url = {http://ict.usc.edu/pubs/A%20Shared%20Modular%20Architecture%20for%20Developing%20Virtual%20Humans.pdf},

doi = {10.2390/biecoll-wasiva2014-02},

year  = {2014},

date = {2014-08-01},

booktitle = {Proceedings of the Workshop on Architectures and Standards for Intelligent Virtual Agents at IVA 2014},

pages = {4–7},

address = {Boston, MA},

abstract = {Realizing the full potential of intelligent virtual agents requires compelling characters that can engage users in meaningful and realistic social interactions, and an ability to develop these characters effectively and efficiently. Advances are needed in individual capabilities, but perhaps more importantly, fundamental questions remain as to how best to integrate these capabilities into a single framework that allows us to efficiently create characters that can engage users in meaningful and realistic social interactions. This integration requires in-depth, inter-disciplinary understanding few individuals, or even teams of individuals, possess.},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{devault_simsensei_2014,

title = {SimSensei Kiosk: A Virtual Human Interviewer for Healthcare Decision Support},

author = {David DeVault and Ron Artstein and Grace Benn and Teresa Dey and Edward Fast and Alesia Gainer and Kallirroi Georgila and Jonathan Gratch and Arno Hartholt and Margaux Lhommet and Gale Lucas and Stacy C. Marsella and Morbini Fabrizio and Angela Nazarian and Stefan Scherer and Giota Stratou and Apar Suri and David Traum and Rachel Wood and Yuyu Xu and Albert Rizzo and Louis-Philippe Morency},

url = {https://dl.acm.org/citation.cfm?id=2617415},

year  = {2014},

date = {2014-05-01},

booktitle = {Proceedings of the 13th Inter-national Conference on Autonomous Agents and Multiagent Systems (AAMAS 2014)},

pages = {1061–1068},

publisher = {International Foundation for Autonomous Agents and Multiagent Systems},

address = {Paris, France},

abstract = {We present SimSensei Kiosk, an implemented virtual human interviewer designed to create an engaging face-to-face inter-action where the user feels comfortable talking and sharing information. SimSensei Kiosk is also designed to create in- teractional situations favorable to the automatic assessment of distress indicators, de_ned as verbal and nonverbal behav- iors correlated with depression, anxiety or post-traumatic stress disorder (PTSD). In this paper, we summarize the de- sign methodology, performed over the past two years, which is based on three main development cycles: (1) analysis of face-to-face human interactions to identify potential distress indicators, dialogue policies and virtual human gestures, (2) development and analysis of a Wizard-of-Oz prototype sys- tem where two human operators were deciding the spoken and gestural responses, and (3) development of a fully au- tomatic virtual interviewer able to engage users in 15-25 minute interactions. We show the potential of our fully auto- matic virtual human interviewer in a user study, and situate its performance in relation to the Wizard-of-Oz prototype.},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{gratch_virtual_2013,

title = {Virtual Humans: A New Toolkit for Cognitive Science Research},

author = {Jonathan Gratch and Arno Hartholt and Morteza Dehghani and Stacy C. Marsella},

url = {http://ict.usc.edu/pubs/Virtual%20Humans-%20A%20New%20Toolkit%20for%20Cognitive%20Science%20Research.pdf},

year  = {2013},

date = {2013-08-01},

booktitle = {Cognitive Science},

address = {Berlin, Germany},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_all_2013,

title = {All Together Now: Introducing the Virtual Human Toolkit},

author = {Arno Hartholt and David Traum and Stacy C. Marsella and Ari Shapiro and Giota Stratou and Anton Leuski and Louis-Philippe Morency and Jonathan Gratch},

url = {http://ict.usc.edu/pubs/All%20Together%20Now.pdf},

year  = {2013},

date = {2013-08-01},

booktitle = {13th International Conference on Intelligent Virtual Agents},

address = {Edinburgh, UK},

abstract = {While virtual humans are proven tools for training, education and re- search, they are far from realizing their full potential. Advances are needed in indi- vidual capabilities, such as character animation and speech synthesis, but perhaps more importantly, fundamental questions remain as to how best to integrate these capabilities into a single framework that allows us to efficiently create characters that can engage users in meaningful and realistic social interactions. This integration re- quires in-depth, inter-disciplinary understanding few individuals, or even teams of individuals, possess. We help address this challenge by introducing the ICT Virtual Human Toolkit1, which offers a flexible framework for exploring a variety of differ- ent types of virtual human systems, from virtual listeners and question-answering characters to virtual role-players. We show that due to its modularity, the Toolkit allows researchers to mix and match provided capabilities with their own, lowering the barrier of entry to this multi-disciplinary research challenge.},

keywords = {UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{wilson_facial_2011,

title = {Facial Cartography: Interactive Scan Correspondence},

author = {Cyrus A. Wilson and Oleg Alexander and Borom Tunwattanapong and Pieter Peers and Abhijeet Ghosh and Jay Busch and Arno Hartholt and Paul Debevec},

url = {http://ict.usc.edu/pubs/Facial%20Cartography-%20Interactive%20Scan%20Correspondence.pdf},

year  = {2011},

date = {2011-08-01},

booktitle = {ACM/Eurographics Symposium on Computer Animation},

abstract = {We present a semi-automatic technique for computing surface correspondences between 3D facial scans in different expressions, such that scan data can be mapped into a common domain for facial animation. The technique can accurately correspond high-resolution scans of widely differing expressions – without requiring intermediate pose sequences – such that they can be used, together with reflectance maps, to create high-quality blendshape-based facial animation. We optimize correspondences through a combination of Image, Shape, and Internal forces, as well as Directable forces to allow a user to interactively guide and refine the solution. Key to our method is a novel representation, called an Active Visage, that balances the advantages of both deformable templates and correspondence computation in a 2D canonical domain. We show that our semi-automatic technique achieves more robust results than automated correspondence alone, and is more precise than is practical with unaided manual input.},

keywords = {Graphics, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{campbell_developing_2011,

title = {Developing INOTS to Support Interpersonal Skills Practice},

author = {Julia Campbell and Mark Core and Ron Artstein and Lindsay Armstrong and Arno Hartholt and Cyrus A. Wilson and Kallirroi Georgila and Fabrizio Morbini and Kip Haynes and Dave Gomboc and Mike Birch and Jonathan Bobrow and H. Chad Lane and Jillian Gerten and Anton Leuski and David Traum and Matthew Trimmer and Rich DiNinni and Matthew Bosack and Timothy Jones and Richard E. Clark and Kenneth A. Yates},

url = {http://ict.usc.edu/pubs/Developing%20INOTS%20to%20Support%20Interpersonal%20Skills%20Practice.pdf},

year  = {2011},

date = {2011-03-01},

booktitle = {IEEE Aerospace Conference},

abstract = {The Immersive Naval Officer Training System (INOTS) is a blended learning environment that merges traditional classroom instruction with a mixed reality training setting. INOTS supports the instruction, practice and assessment of interpersonal communication skills. The goal of INOTS is to provide a consistent training experience to supplement interpersonal skills instruction for Naval officer candidates without sacrificing trainee throughput and instructor control. We developed an instructional design from cognitive task analysis interviews with experts to serve as a framework for system development. We also leveraged commercial student response technology and research technologies including natural language recognition, virtual humans, realistic graphics, intelligent tutoring and automated instructor support tools. In this paper, we describe our methodologies for developing a blended learning environment, and our challenges adding mixed reality and virtual human technologies to a traditional classroom to support interpersonal skills training.},

keywords = {Learning Sciences, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_at_2009,

title = {At the Virtual Frontier: Introducing Gunslinger, a Multi- Character, Mixed-Reality, Story-Driven Experience},

author = {Arno Hartholt and Jonathan Gratch and Anton Leuski and Louis-Philippe Morency and Stacy C. Marsella and Matt Liewer and Prathibha Doraiswamy and Lori Weiss and Kim LeMasters and Edward Fast and Ramy Sadek and Andrew Marshall and Jina Lee and Marcus Thiebaux and Andreas Tsiartas},

url = {http://ict.usc.edu/pubs/At%20the%20Virtual%20Frontier-%20Introducing%20Gunslinger%20a%20Multi-%20Character%20Mixed-Reality%20Story-Driven%20Experience.pdf},

year  = {2009},

date = {2009-01-01},

booktitle = {Proceedings of the 9th International Conference on Intelligent Virtual Agents (IVA)},

address = {Amsterdam, The Netherlands},

abstract = {We describe an application of intelligent virtual agents to the domain of mixed-reality interactive entertainment. Gunslinger allows users to interact with life-sized virtual humans within the context of a wild west story world. The application incorporates a novel integration of capabilities including gesture and spoken language recognition, story and dialogue reasoning, and multi-character, multi-modal behavior generation and synthesis. The article describes our design process, technological innovations, and initial feedback from user interactions with the system.},

keywords = {Social Simulation, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{traum_multi-party_2008,

title = {Multi-party, Multi-issue, Multi-strategy Negotiation for Multi-modal Virtual Agents},

author = {David Traum and Jonathan Gratch and Arno Hartholt and Stacy C. Marsella and Jina Lee},

url = {http://ict.usc.edu/pubs/Multi-party,%20Multi-issue,%20Multi-strategy%20Negotiation.pdf},

year  = {2008},

date = {2008-09-01},

booktitle = {Proceedings of the 8th International Conference on Intelligent Virtual Agents},

pages = {117–130},

address = {Tokyo, Japan},

abstract = {We present a model of negotiation for virtual agents that extends previous work to be more human-like and applicable to a broader range of situations, including more than two negotiators with different goals, and negotiating over multiple options. The agents can dynamically change their negotiating strategies based on the current values of several parameters and factors that can be updated in the course of the negotiation.We have implemented this model and done preliminary evaluation within a prototype training system and a three-party negotiation with two virtual humans and one human.},

keywords = {Social Simulation, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hartholt_common_2008,

title = {A Common Ground for Virtual Humans: Using an Ontology in a Natural Language Oriented Virtual Human Architecture},

author = {Arno Hartholt and Thomas Russ and David Traum and Eduard Hovy and Susan Robinson},

url = {http://ict.usc.edu/pubs/A%20Common%20Ground%20for%20Virtual%20Humans-%20Using%20an%20Ontology%20in%20a%20Natural%20Language%20Oriented%20Virtual%20Human%20Architecture.pdf},

year  = {2008},

date = {2008-05-01},

booktitle = {International Conference on Language Resources and Evaluation (LREC)},

address = {Marrakech, Morocco},

abstract = {When dealing with large, distributed systems that use state-of-the-art components, individual components are usually developed in parallel. As development continues, the decoupling invariably leads to a mismatch between how these components internally represent concepts and how they communicate these representations to other components: representations can get out of synch, contain localized errors, or become manageable only by a small group of experts for each module. In this paper, we describe the use of an ontology as part of a complex distributed virtual human architecture in order to enable better communication between modules while improving the overall flexibility needed to change or extend the system. We focus on the natural language understanding capabilities of this architecture and the relationship between language and concepts within the entire system in general and the ontology in particular.},

keywords = {Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{kenny_building_2007,

title = {Building Interactive Virtual Humans for Training Environments},

author = {Patrick G. Kenny and Arno Hartholt and Jonathan Gratch and William Swartout and David Traum and Stacy C. Marsella and Diane Piepol},

url = {http://ict.usc.edu/pubs/Building%20Interactive%20Virtual%20Humans%20for%20Training%20Environments.pdf},

year  = {2007},

date = {2007-11-01},

booktitle = {Interservice/Industry Training, Simulation and Education Conference (I/ITSEC)},

address = {Orlando, FL},

abstract = {There is a great need in the Joint Forces to have human to human interpersonal training for skills such as negotiation, leadership, interviewing and cultural training. Virtual environments can be incredible training tools if used properly and used for the correct training application. Virtual environments have already been very successful in training Warfighters how to operate vehicles and weapons systems. At the Institute for Creative Technologies (ICT) we have been exploring a new question: can virtual environments be used to train Warfighters in interpersonal skills such as negotiation, tactical questioning and leadership that are so critical for success in the contemporary operating environment? Using embodied conversational agents to create this type of training system has been one of the goals of the Virtual Humans project at the institute. ICT has a great deal of experience building complex, integrated and immersive training systems that address the human factor needs for training experiences. This paper will address the research, technology and value of developing virtual humans for training environments. This research includes speech recognition, natural language understanding & generation, dialogue management, cognitive agents, emotion modeling, question response managers, speech generation and non-verbal behavior. Also addressed will be the diverse set of training environments we have developed for the system, from single computer laptops to multi-computer immersive displays to real and virtual integrated environments. This paper will also discuss the problems, issues and solutions we encountered while building these systems. The paper will recount subject testing we have performed in these environments and results we have obtained from users. Finally the future of this type of Virtual Humans technology and training applications will be discussed.},

keywords = {MedVR, Social Simulation, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}