Publications
Search
221.
Song, Yale; Soleymani, Mohammad
Polysemous Visual-Semantic Embedding for Cross-Modal Retrieval Proceedings Article
In: Proceedings of the 2019 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pp. 10, IEEE, Long Beach, CA, 2019.
@inproceedings{song_polysemous_2019,
title = {Polysemous Visual-Semantic Embedding for Cross-Modal Retrieval},
author = {Yale Song and Mohammad Soleymani},
url = {https://arxiv.org/abs/1906.04402},
year = {2019},
date = {2019-06-01},
booktitle = {Proceedings of the 2019 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
pages = {10},
publisher = {IEEE},
address = {Long Beach, CA},
abstract = {Visual-semantic embedding aims to find a shared latent space where related visual and textual instances are close to each other. Most current methods learn injective embedding functions that map an instance to a single point in the shared space. Unfortunately, injective embedding cannot effectively handle polysemous instances with multiple possible meanings; at best, it would find an average representation of different meanings. This hinders its use in real-world scenarios where individual instances and their cross-modal associations are often ambiguous. In this work, we introduce Polysemous Instance Embedding Networks (PIE-Nets) that compute multiple and diverse representations of an instance by combining global context with locally-guided features via multi-head self-attention and residual learning. To learn visual-semantic embedding, we tie-up two PIE-Nets and optimize them jointly in the multiple instance learning framework. Most existing work on cross-modal retrieval focus on image-text pairs of data. Here, we also tackle a more challenging case of video-text retrieval. To facilitate further research in video-text retrieval, we release a new dataset of 50K video-sentence pairs collected from social media, dubbed MRW (my reaction when). We demonstrate our approach on both image-text and video-text retrieval scenarios using MS-COCO, TGIF, and our new MRW dataset.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Visual-semantic embedding aims to find a shared latent space where related visual and textual instances are close to each other. Most current methods learn injective embedding functions that map an instance to a single point in the shared space. Unfortunately, injective embedding cannot effectively handle polysemous instances with multiple possible meanings; at best, it would find an average representation of different meanings. This hinders its use in real-world scenarios where individual instances and their cross-modal associations are often ambiguous. In this work, we introduce Polysemous Instance Embedding Networks (PIE-Nets) that compute multiple and diverse representations of an instance by combining global context with locally-guided features via multi-head self-attention and residual learning. To learn visual-semantic embedding, we tie-up two PIE-Nets and optimize them jointly in the multiple instance learning framework. Most existing work on cross-modal retrieval focus on image-text pairs of data. Here, we also tackle a more challenging case of video-text retrieval. To facilitate further research in video-text retrieval, we release a new dataset of 50K video-sentence pairs collected from social media, dubbed MRW (my reaction when). We demonstrate our approach on both image-text and video-text retrieval scenarios using MS-COCO, TGIF, and our new MRW dataset.
222.
Editor, Managing; Adler, Aaron; Dasgupta, Prithviraj; DePalma, Nick; Eslami, Mohammed; Freedman, Richard; Laird, John; Lebiere, Christian; Lohan, Katrin; Mead, Ross; Roberts, Mark; Rosenbloom, Paul; Senft, Emmanuel; Stein, Frank; Williams, Tom; Wray, Kyle Hollins; Yaman, Fusun; Zilberstein, Shlomo
Reports of the 2018 AAAI Fall Symposium Journal Article
In: AI Magazine, vol. 40, no. 2, pp. 66–72, 2019, ISSN: 2371-9621, 0738-4602.
@article{editor_reports_2019,
title = {Reports of the 2018 AAAI Fall Symposium},
author = {Managing Editor and Aaron Adler and Prithviraj Dasgupta and Nick DePalma and Mohammed Eslami and Richard Freedman and John Laird and Christian Lebiere and Katrin Lohan and Ross Mead and Mark Roberts and Paul Rosenbloom and Emmanuel Senft and Frank Stein and Tom Williams and Kyle Hollins Wray and Fusun Yaman and Shlomo Zilberstein},
url = {http://www.aaai.org/ojs/index.php/aimagazine/article/view/2887},
doi = {10.1609/aimag.v40i2.2887},
issn = {2371-9621, 0738-4602},
year = {2019},
date = {2019-06-01},
journal = {AI Magazine},
volume = {40},
number = {2},
pages = {66–72},
abstract = {The AAAI 2018 Fall Symposium Series was held Thursday through Saturday, October 18–20, at the Westin Arlington Gateway in Arlington, Virginia, adjacent to Washington, D.C. The titles of the eight symposia were Adversary-Aware Learning Techniques and Trends in Cybersecurity; Artificial Intelligence for Synthetic Biology; Artificial Intelligence in Government and Public Sector; A Common Model of Cognition; Gathering for Artificial Intelligence and Natural System; Integrating Planning, Diagnosis, and Causal Reasoning; Interactive Learning in Artificial Intelligence for HumanRobot Interaction; and Reasoning and Learning in Real-World Systems for Long-Term Autonomy. The highlights of each symposium (except the Gathering for Artificial Intelligence and Natural System symposium, whose organizers failed to submit a summary) are presented in this report.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The AAAI 2018 Fall Symposium Series was held Thursday through Saturday, October 18–20, at the Westin Arlington Gateway in Arlington, Virginia, adjacent to Washington, D.C. The titles of the eight symposia were Adversary-Aware Learning Techniques and Trends in Cybersecurity; Artificial Intelligence for Synthetic Biology; Artificial Intelligence in Government and Public Sector; A Common Model of Cognition; Gathering for Artificial Intelligence and Natural System; Integrating Planning, Diagnosis, and Causal Reasoning; Interactive Learning in Artificial Intelligence for HumanRobot Interaction; and Reasoning and Learning in Real-World Systems for Long-Term Autonomy. The highlights of each symposium (except the Gathering for Artificial Intelligence and Natural System symposium, whose organizers failed to submit a summary) are presented in this report.
223.
Zhou, Yi; Barnes, Connelly; Lu, Jingwan; Yang, Jimei; Li, Hao
On the Continuity of Rotation Representations in Neural Networks Proceedings Article
In: Proceedings of CVPR, pp. 9, IEEE, Long Beach, CA, 2019.
@inproceedings{zhou_continuity_2019,
title = {On the Continuity of Rotation Representations in Neural Networks},
author = {Yi Zhou and Connelly Barnes and Jingwan Lu and Jimei Yang and Hao Li},
url = {http://openaccess.thecvf.com/content_CVPR_2019/html/Zhou_On_the_Continuity_of_Rotation_Representations_in_Neural_Networks_CVPR_2019_paper.html},
year = {2019},
date = {2019-06-01},
booktitle = {Proceedings of CVPR},
pages = {9},
publisher = {IEEE},
address = {Long Beach, CA},
abstract = {In neural networks, it is often desirable to work with various representations of the same space. For example, 3D rotations can be represented with quaternions or Euler angles. In this paper, we advance a definition of a continuous representation, which can be helpful for training deep neural networks. We relate this to topological concepts such as homeomorphism and embedding. We then investigate what are continuous and discontinuous representations for 2D, 3D, and n-dimensional rotations. We demonstrate that for 3D rotations, all representations are discontinuous in the real Euclidean spaces of four or fewer dimensions. Thus, widely used representations such as quaternions and Euler angles are discontinuous and difficult for neural networks to learn. We show that the 3D rotations have continuous representations in 5D and 6D, which are more suitable for learning. We also present continuous representations for the general case of the n dimensional rotation group SO(n). While our main focus is on rotations, we also show that our constructions apply to other groups such as the orthogonal group and similarity transforms. We finally present empirical results, which show that our continuous rotation representations outperform discontinuous ones for several practical problems in graphics and vision, including a simple autoencoder sanity test, a rotation estimator for 3D point clouds, and an inverse kinematics solver for 3D human poses.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In neural networks, it is often desirable to work with various representations of the same space. For example, 3D rotations can be represented with quaternions or Euler angles. In this paper, we advance a definition of a continuous representation, which can be helpful for training deep neural networks. We relate this to topological concepts such as homeomorphism and embedding. We then investigate what are continuous and discontinuous representations for 2D, 3D, and n-dimensional rotations. We demonstrate that for 3D rotations, all representations are discontinuous in the real Euclidean spaces of four or fewer dimensions. Thus, widely used representations such as quaternions and Euler angles are discontinuous and difficult for neural networks to learn. We show that the 3D rotations have continuous representations in 5D and 6D, which are more suitable for learning. We also present continuous representations for the general case of the n dimensional rotation group SO(n). While our main focus is on rotations, we also show that our constructions apply to other groups such as the orthogonal group and similarity transforms. We finally present empirical results, which show that our continuous rotation representations outperform discontinuous ones for several practical problems in graphics and vision, including a simple autoencoder sanity test, a rotation estimator for 3D point clouds, and an inverse kinematics solver for 3D human poses.
224.
Natsume, Ryota; Saito, Shunsuke; Huang, Zeng; Chen, Weikai; Ma, Chongyang; Li, Hao; Morishima, Shigeo
SiCloPe: Silhouette-Based Clothed People Proceedings Article
In: Proceedings of CVPR, pp. 11, IEEE, Long Beach, CA, 2019.
@inproceedings{natsume_siclope_2019,
title = {SiCloPe: Silhouette-Based Clothed People},
author = {Ryota Natsume and Shunsuke Saito and Zeng Huang and Weikai Chen and Chongyang Ma and Hao Li and Shigeo Morishima},
url = {http://openaccess.thecvf.com/content_CVPR_2019/html/Natsume_SiCloPe_Silhouette-Based_Clothed_People_CVPR_2019_paper.html},
year = {2019},
date = {2019-06-01},
booktitle = {Proceedings of CVPR},
pages = {11},
publisher = {IEEE},
address = {Long Beach, CA},
abstract = {We introduce a new silhouette-based representation for modeling clothed human bodies using deep generative models. Our method can reconstruct a complete and textured 3D model of a person wearing clothes from a single input picture. Inspired by the visual hull algorithm, our implicit representation uses 2D silhouettes and 3D joints of a body pose to describe the immense shape complexity and variations of clothed people. Given a segmented 2D silhouette of a person and its inferred 3D joints from the input picture, we first synthesize consistent silhouettes from novel view points around the subject. The synthesized silhouettes which are the most consistent with the input segmentation are fed into a deep visual hull algorithm for robust 3D shape prediction. We then infer the texture of the subject’s back view using the frontal image and segmentation mask as input to a conditional generative adversarial network. Our experiments demonstrate that our silhouette-based model is an effective representation and the appearance of the back view can be predicted reliably using an image-to-image translation network. While classic methods based on parametric models often fail for single-view images of subjects with challenging clothing, our approach can still produce successful results, which are comparable to those obtained from multi-view input.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We introduce a new silhouette-based representation for modeling clothed human bodies using deep generative models. Our method can reconstruct a complete and textured 3D model of a person wearing clothes from a single input picture. Inspired by the visual hull algorithm, our implicit representation uses 2D silhouettes and 3D joints of a body pose to describe the immense shape complexity and variations of clothed people. Given a segmented 2D silhouette of a person and its inferred 3D joints from the input picture, we first synthesize consistent silhouettes from novel view points around the subject. The synthesized silhouettes which are the most consistent with the input segmentation are fed into a deep visual hull algorithm for robust 3D shape prediction. We then infer the texture of the subject’s back view using the frontal image and segmentation mask as input to a conditional generative adversarial network. Our experiments demonstrate that our silhouette-based model is an effective representation and the appearance of the back view can be predicted reliably using an image-to-image translation network. While classic methods based on parametric models often fail for single-view images of subjects with challenging clothing, our approach can still produce successful results, which are comparable to those obtained from multi-view input.
225.
Georgila, Kallirroi; Core, Mark G; Nye, Benjamin D; Karumbaiah, Shamya; Auerbach, Daniel; Ram, Maya
Using Reinforcement Learning to Optimize the Policies of an Intelligent Tutoring System for Interpersonal Skills Training Proceedings Article
In: Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems, pp. 9, IFAAMAS, Montreal, Canada, 2019.
@inproceedings{georgila_using_2019,
title = {Using Reinforcement Learning to Optimize the Policies of an Intelligent Tutoring System for Interpersonal Skills Training},
author = {Kallirroi Georgila and Mark G Core and Benjamin D Nye and Shamya Karumbaiah and Daniel Auerbach and Maya Ram},
url = {http://www.ifaamas.org/Proceedings/aamas2019/pdfs/p737.pdf},
year = {2019},
date = {2019-05-01},
booktitle = {Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems},
pages = {9},
publisher = {IFAAMAS},
address = {Montreal, Canada},
abstract = {Reinforcement Learning (RL) has been applied successfully to Intelligent Tutoring Systems (ITSs) in a limited set of well-defined domains such as mathematics and physics. This work is unique in using a large state space and for applying RL to tutoring interpersonal skills. Interpersonal skills are increasingly recognized as critical to both social and economic development. In particular, this work enhances an ITS designed to teach basic counseling skills that can be applied to challenging issues such as sexual harassment and workplace conflict. An initial data collection was used to train RL policies for the ITS, and an evaluation with human participants compared a hand-crafted ITS which had been used for years with students (control) versus the new ITS guided by RL policies. The RL condition differed from the control condition most notably in the strikingly large quantity of guidance it provided to learners. Both systems were effective and there was an overall significant increase from pre- to post-test scores. Although learning gains did not differ significantly between conditions, learners had a significantly higher self-rating of confidence in the RL condition. Confidence and learning gains were both part of the reward function used to train the RL policies, and it could be the case that there was the most room for improvement in confidence, an important learner emotion. Thus, RL was successful in improving an ITS for teaching interpersonal skills without the need to prune the state space (as previously done).},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Reinforcement Learning (RL) has been applied successfully to Intelligent Tutoring Systems (ITSs) in a limited set of well-defined domains such as mathematics and physics. This work is unique in using a large state space and for applying RL to tutoring interpersonal skills. Interpersonal skills are increasingly recognized as critical to both social and economic development. In particular, this work enhances an ITS designed to teach basic counseling skills that can be applied to challenging issues such as sexual harassment and workplace conflict. An initial data collection was used to train RL policies for the ITS, and an evaluation with human participants compared a hand-crafted ITS which had been used for years with students (control) versus the new ITS guided by RL policies. The RL condition differed from the control condition most notably in the strikingly large quantity of guidance it provided to learners. Both systems were effective and there was an overall significant increase from pre- to post-test scores. Although learning gains did not differ significantly between conditions, learners had a significantly higher self-rating of confidence in the RL condition. Confidence and learning gains were both part of the reward function used to train the RL policies, and it could be the case that there was the most room for improvement in confidence, an important learner emotion. Thus, RL was successful in improving an ITS for teaching interpersonal skills without the need to prune the state space (as previously done).
226.
Chaffey, Patricia; Artstein, Ron; Georgila, Kallirroi; Pollard, Kimberly A.; Gilani, Setareh Nasihati; Krum, David M.; Nelson, David; Huynh, Kevin; Gainer, Alesia; Alavi, Seyed Hossein; Yahata, Rhys; Traum, David
Developing a Virtual Reality Wildfire Simulation to Analyze Human Communication and Interaction with a Robotic Swarm During Emergencies Proceedings Article
In: Proceedings of the 9th Language and Technology Conference, LTC, Poznań, Poland, 2019.
@inproceedings{chaffey_developing_2019,
title = {Developing a Virtual Reality Wildfire Simulation to Analyze Human Communication and Interaction with a Robotic Swarm During Emergencies},
author = {Patricia Chaffey and Ron Artstein and Kallirroi Georgila and Kimberly A. Pollard and Setareh Nasihati Gilani and David M. Krum and David Nelson and Kevin Huynh and Alesia Gainer and Seyed Hossein Alavi and Rhys Yahata and David Traum},
url = {http://www-scf.usc.edu/ nasihati/publications/HLTCEM_2019.pdf},
year = {2019},
date = {2019-05-01},
booktitle = {Proceedings of the 9th Language and Technology Conference},
publisher = {LTC},
address = {Poznań, Poland},
abstract = {Search and rescue missions involving robots face multiple challenges. The ratio of operators to robots is frequently one to one or higher, operators tasked with robots must contend with cognitive overload for long periods, and the robots themselves may be discomfiting to located survivors. To improve on the current state, we propose a swarm of robots equipped with natural language abilities and guided by a central virtual “spokesperson” able to access “plays”. The spokesperson may assist the operator with tasking the robots in their exploration of a zone, which allows the operator to maintain a safe distance. The use of multiple robots enables rescue personnel to cover a larger swath of ground, and the natural language component allows the robots to communicate with survivors located on site. This capability frees the operator to handle situations requiring personal attention, and overall can accelerate the location and assistance of survivors. In order to develop this system, we are creating a virtual reality simulation, in order to conduct a study and analysis of how humans communicate with these swarms of robots. The data collected from this experiment will inform how to best design emergency response swarm robots that are effectively able to communicate with the humans around them.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Search and rescue missions involving robots face multiple challenges. The ratio of operators to robots is frequently one to one or higher, operators tasked with robots must contend with cognitive overload for long periods, and the robots themselves may be discomfiting to located survivors. To improve on the current state, we propose a swarm of robots equipped with natural language abilities and guided by a central virtual “spokesperson” able to access “plays”. The spokesperson may assist the operator with tasking the robots in their exploration of a zone, which allows the operator to maintain a safe distance. The use of multiple robots enables rescue personnel to cover a larger swath of ground, and the natural language component allows the robots to communicate with survivors located on site. This capability frees the operator to handle situations requiring personal attention, and overall can accelerate the location and assistance of survivors. In order to develop this system, we are creating a virtual reality simulation, in order to conduct a study and analysis of how humans communicate with these swarms of robots. The data collected from this experiment will inform how to best design emergency response swarm robots that are effectively able to communicate with the humans around them.
227.
Gervits, Felix; Leuski, Anton; Bonial, Claire; Gordon, Carla; Traum, David
A Classification-Based Approach to Automating Human-Robot Dialogue Proceedings Article
In: Proceedings of IWSDS 2019, pp. 12, Siracusa, Italy, 2019.
@inproceedings{gervits_classication-based_2019,
title = {A Classification-Based Approach to Automating Human-Robot Dialogue},
author = {Felix Gervits and Anton Leuski and Claire Bonial and Carla Gordon and David Traum},
url = {https://www.semanticscholar.org/paper/A-Classification-Based-Approach-to-Automating-Gervits-Leuski/262cf9e3a14e370d46a5e65f7872b32482d9ea69?tab=abstract&citingPapersSort=is-influential&citingPapersLimit=10&citingPapersOffset=0&year%5B0%5D=&year%5B1%5D=&citedPapersSort=is-influential&citedPapersLimit=10&citedPapersOffset=10},
year = {2019},
date = {2019-04-01},
booktitle = {Proceedings of IWSDS 2019},
pages = {12},
address = {Siracusa, Italy},
abstract = {We present a dialogue system based on statistical classification which was used to automate human-robot dialogue in a collaborative navigation domain. The classifier was trained on a small corpus of multi-floor Wizard-of-Oz dialogue including two wizards: one standing in for dialogue capabilities and another for navigation. Below, we describe the implementation details of the classifier and show how it was used to automate the dialogue wizard. We evaluate our system on several sets of source data from the corpus and find that response accuracy is generally high, even with very limited training data. Another contribution of this work is the novel demonstration of a dialogue manager that uses the classifier to engage in multi-floor dialogue with two different human roles. Overall, this approach is useful for enabling spoken dialogue systems to produce robust and accurate responses to natural language input, and for robots that need to interact with humans in a team setting.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We present a dialogue system based on statistical classification which was used to automate human-robot dialogue in a collaborative navigation domain. The classifier was trained on a small corpus of multi-floor Wizard-of-Oz dialogue including two wizards: one standing in for dialogue capabilities and another for navigation. Below, we describe the implementation details of the classifier and show how it was used to automate the dialogue wizard. We evaluate our system on several sets of source data from the corpus and find that response accuracy is generally high, even with very limited training data. Another contribution of this work is the novel demonstration of a dialogue manager that uses the classifier to engage in multi-floor dialogue with two different human roles. Overall, this approach is useful for enabling spoken dialogue systems to produce robust and accurate responses to natural language input, and for robots that need to interact with humans in a team setting.
228.
Liu, Shichen; Li, Tianye; Chen, Weikai; Li, Hao
Soft Rasterizer: A Differentiable Renderer for Image-based 3D Reasoning Journal Article
In: arXiv:1904.01786 [cs], 2019.
@article{liu_soft_2019,
title = {Soft Rasterizer: A Differentiable Renderer for Image-based 3D Reasoning},
author = {Shichen Liu and Tianye Li and Weikai Chen and Hao Li},
url = {http://arxiv.org/abs/1904.01786},
year = {2019},
date = {2019-04-01},
journal = {arXiv:1904.01786 [cs]},
abstract = {Rendering bridges the gap between 2D vision and 3D scenes by simulating the physical process of image formation. By inverting such renderer, one can think of a learning approach to infer 3D information from 2D images. However, standard graphics renderers involve a fundamental discretization step called rasterization, which prevents the rendering process to be differentiable, hence able to be learned. Unlike the state-of-the-art differentiable renderers [29, 19], which only approximate the rendering gradient in the back propagation, we propose a truly differentiable rendering framework that is able to (1) directly render colorized mesh using differentiable functions and (2) back-propagate efficient supervision signals to mesh vertices and their attributes from various forms of image representations, including silhouette, shading and color images. The key to our framework is a novel formulation that views rendering as an aggregation function that fuses the probabilistic contributions of all mesh triangles with respect to the rendered pixels. Such formulation enables our framework to flow gradients to the occluded and far-range vertices, which cannot be achieved by the previous state-of-thearts. We show that by using the proposed renderer, one can achieve significant improvement in 3D unsupervised singleview reconstruction both qualitatively and quantitatively. Experiments also demonstrate that our approach is able to handle the challenging tasks in image-based shape fitting, which remain nontrivial to existing differentiable renderers. Code is available at https://github.com/ ShichenLiu/SoftRas.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Rendering bridges the gap between 2D vision and 3D scenes by simulating the physical process of image formation. By inverting such renderer, one can think of a learning approach to infer 3D information from 2D images. However, standard graphics renderers involve a fundamental discretization step called rasterization, which prevents the rendering process to be differentiable, hence able to be learned. Unlike the state-of-the-art differentiable renderers [29, 19], which only approximate the rendering gradient in the back propagation, we propose a truly differentiable rendering framework that is able to (1) directly render colorized mesh using differentiable functions and (2) back-propagate efficient supervision signals to mesh vertices and their attributes from various forms of image representations, including silhouette, shading and color images. The key to our framework is a novel formulation that views rendering as an aggregation function that fuses the probabilistic contributions of all mesh triangles with respect to the rendered pixels. Such formulation enables our framework to flow gradients to the occluded and far-range vertices, which cannot be achieved by the previous state-of-thearts. We show that by using the proposed renderer, one can achieve significant improvement in 3D unsupervised singleview reconstruction both qualitatively and quantitatively. Experiments also demonstrate that our approach is able to handle the challenging tasks in image-based shape fitting, which remain nontrivial to existing differentiable renderers. Code is available at https://github.com/ ShichenLiu/SoftRas.
229.
Dennison, Mark S.; Krum, David M.
Unifying Research to Address Motion Sickness Proceedings Article
In: Proceedings of the 2019 IEEE Conference on Virtual Reality and 3D User Interfaces (VR), pp. 1858–1859, IEEE, Osaka, Japan, 2019, ISBN: 978-1-72811-377-7.
@inproceedings{dennison_unifying_2019,
title = {Unifying Research to Address Motion Sickness},
author = {Mark S. Dennison and David M. Krum},
url = {https://ieeexplore.ieee.org/document/8798297/},
doi = {10.1109/VR.2019.8798297},
isbn = {978-1-72811-377-7},
year = {2019},
date = {2019-03-01},
booktitle = {Proceedings of the 2019 IEEE Conference on Virtual Reality and 3D User Interfaces (VR)},
pages = {1858–1859},
publisher = {IEEE},
address = {Osaka, Japan},
abstract = {Be it discussed as cybersickness, immersive sickness, simulator sickness, or virtual reality sickness, the ill effects of visuo-vestibular mismatch in immersive environments are of great concern for the wider adoption of virtual reality and related technologies. In this position paper, we discuss a unified research approach that may address motion sickness and identify critical research topics.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Be it discussed as cybersickness, immersive sickness, simulator sickness, or virtual reality sickness, the ill effects of visuo-vestibular mismatch in immersive environments are of great concern for the wider adoption of virtual reality and related technologies. In this position paper, we discuss a unified research approach that may address motion sickness and identify critical research topics.
230.
Bernardet, Ulysses; Kanq, Sin-Hwa; Feng, Andrew; DiPaola, Steve; Shapiro, Ari
Speech Breathing in Virtual Humans: An Interactive Model and Empirical Study Proceedings Article
In: 2019 IEEE Virtual Humans and Crowds for Immersive Environments (VHCIE), pp. 1–9, IEEE, Osaka, Japan, 2019, ISBN: 978-1-72813-219-8.
@inproceedings{bernardet_speech_2019,
title = {Speech Breathing in Virtual Humans: An Interactive Model and Empirical Study},
author = {Ulysses Bernardet and Sin-Hwa Kanq and Andrew Feng and Steve DiPaola and Ari Shapiro},
url = {https://ieeexplore.ieee.org/document/8714737/},
doi = {10.1109/VHCIE.2019.8714737},
isbn = {978-1-72813-219-8},
year = {2019},
date = {2019-03-01},
booktitle = {2019 IEEE Virtual Humans and Crowds for Immersive Environments (VHCIE)},
pages = {1–9},
publisher = {IEEE},
address = {Osaka, Japan},
abstract = {Human speech production requires the dynamic regulation of air through the vocal system. While virtual character systems commonly are capable of speech output, they rarely take breathing during speaking – speech breathing – into account. We believe that integrating dynamic speech breathing systems in virtual characters can significantly contribute to augmenting their realism. Here, we present a novel control architecture aimed at generating speech breathing in virtual characters. This architecture is informed by behavioral, linguistic and anatomical knowledge of human speech breathing. Based on textual input and controlled by a set of lowand high-level parameters, the system produces dynamic signals in real-time that control the virtual character’s anatomy (thorax, abdomen, head, nostrils, and mouth) and sound production (speech and breathing).},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Human speech production requires the dynamic regulation of air through the vocal system. While virtual character systems commonly are capable of speech output, they rarely take breathing during speaking – speech breathing – into account. We believe that integrating dynamic speech breathing systems in virtual characters can significantly contribute to augmenting their realism. Here, we present a novel control architecture aimed at generating speech breathing in virtual characters. This architecture is informed by behavioral, linguistic and anatomical knowledge of human speech breathing. Based on textual input and controlled by a set of lowand high-level parameters, the system produces dynamic signals in real-time that control the virtual character’s anatomy (thorax, abdomen, head, nostrils, and mouth) and sound production (speech and breathing).
231.
Kang, Sinhwa; Chanenson, Jake; Cowal, Peter; Weaver, Madeleine
Advancing Ethical Decision Making in Virtual Reality Proceedings Article
In: Proceedings of the 2019 IEEE Conference on Virtual Reality and 3D User Interfaces (VR), pp. 2, IEEE, Osaka, Japan, 2019.
@inproceedings{kang_advancing_2019,
title = {Advancing Ethical Decision Making in Virtual Reality},
author = {Sinhwa Kang and Jake Chanenson and Peter Cowal and Madeleine Weaver},
url = {https://ieeexplore.ieee.org/document/8798151},
doi = {10.1109/VR.2019.8798151},
year = {2019},
date = {2019-03-01},
booktitle = {Proceedings of the 2019 IEEE Conference on Virtual Reality and 3D User Interfaces (VR)},
pages = {2},
publisher = {IEEE},
address = {Osaka, Japan},
abstract = {Virtual reality (VR) has been widely utilized for training and education purposes because of pedagogical, safety, and economic benefits. The investigation of moral judgment is a particularly interesting VR application, related to training. For this study, we designed a withinsubject experiment manipulating the role of study participants in a Trolley Dilemma scenario: either victim or driver. We conducted a pilot study with four participants and describe preliminary results and implications in this poster.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Virtual reality (VR) has been widely utilized for training and education purposes because of pedagogical, safety, and economic benefits. The investigation of moral judgment is a particularly interesting VR application, related to training. For this study, we designed a withinsubject experiment manipulating the role of study participants in a Trolley Dilemma scenario: either victim or driver. We conducted a pilot study with four participants and describe preliminary results and implications in this poster.
232.
Gordon, Carla; Leuski, Anton; Benn, Grace; Klassen, Eric; Fast, Edward; Liewer, Matt; Hartholt, Arno; Traum, David
PRIMER: An Emotionally Aware Virtual Agent Proceedings Article
In: Proceedings of the 24th International Conference on Intelligent User Interfaces, pp. 10, ACM, Los Angeles, CA, 2019.
@inproceedings{gordon_primer_2019,
title = {PRIMER: An Emotionally Aware Virtual Agent},
author = {Carla Gordon and Anton Leuski and Grace Benn and Eric Klassen and Edward Fast and Matt Liewer and Arno Hartholt and David Traum},
url = {https://www.research.ibm.com/haifa/Workshops/user2agent2019/},
year = {2019},
date = {2019-03-01},
booktitle = {Proceedings of the 24th International Conference on Intelligent User Interfaces},
pages = {10},
publisher = {ACM},
address = {Los Angeles, CA},
abstract = {PRIMER is a proof-of-concept system designed to show the potential of immersive dialogue agents and virtual environments that adapt and respond to both direct verbal input and indirect emotional input. The system has two novel interfaces: (1) for the user, an immersive VR environment and an animated virtual agent both of which adapt and react to the user’s direct input as well as the user’s perceived emotional state, and (2) for an observer, an interface that helps track the perceived emotional state of the user, with visualizations to provide insight into the system’s decision making process. While the basic system architecture can be adapted for many potential real world applications, the initial version of this system was designed to assist clinical social workers in helping children cope with bullying. The virtual agent produces verbal and non-verbal behaviors guided by a plan for the counseling session, based on in-depth discussions with experienced counselors, but is also reactive to both initiatives that the user takes, e.g. asking their own questions, and the user’s perceived emotional state.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
PRIMER is a proof-of-concept system designed to show the potential of immersive dialogue agents and virtual environments that adapt and respond to both direct verbal input and indirect emotional input. The system has two novel interfaces: (1) for the user, an immersive VR environment and an animated virtual agent both of which adapt and react to the user’s direct input as well as the user’s perceived emotional state, and (2) for an observer, an interface that helps track the perceived emotional state of the user, with visualizations to provide insight into the system’s decision making process. While the basic system architecture can be adapted for many potential real world applications, the initial version of this system was designed to assist clinical social workers in helping children cope with bullying. The virtual agent produces verbal and non-verbal behaviors guided by a plan for the counseling session, based on in-depth discussions with experienced counselors, but is also reactive to both initiatives that the user takes, e.g. asking their own questions, and the user’s perceived emotional state.
233.
Artstein, Ron; Gordon, Carla; Sohail, Usman; Merchant, Chirag; Jones, Andrew; Campbell, Julia; Trimmer, Matthew; Bevington, Jeffrey; Engen, COL Christopher; Traum, David
Digital Survivor of Sexual Assault Proceedings Article
In: Proceedings of the 24th International Conference on Intelligent User Interfaces, pp. 417–425, ACM, Marina del Rey, California, 2019, ISBN: 978-1-4503-6272-6.
@inproceedings{artstein_digital_2019,
title = {Digital Survivor of Sexual Assault},
author = {Ron Artstein and Carla Gordon and Usman Sohail and Chirag Merchant and Andrew Jones and Julia Campbell and Matthew Trimmer and Jeffrey Bevington and COL Christopher Engen and David Traum},
url = {https://doi.org/10.1145/3301275.3302303},
doi = {10.1145/3301275.3302303},
isbn = {978-1-4503-6272-6},
year = {2019},
date = {2019-03-01},
booktitle = {Proceedings of the 24th International Conference on Intelligent User Interfaces},
pages = {417–425},
publisher = {ACM},
address = {Marina del Rey, California},
abstract = {The Digital Survivor of Sexual Assault (DS2A) is an interface that allows a user to have a conversational experience with a survivor of sexual assault, using Artificial Intelligence technology and recorded videos. The application uses a statistical classifier to retrieve contextually appropriate pre-recorded video utterances by the survivor, together with dialogue management policies which enable users to conduct simulated conversations with the survivor about the sexual assault, its aftermath, and other pertinent topics. The content in the application has been specifically elicited to support the needs for the training of U.S. Army professionals in the Sexual Harassment/Assault Response and Prevention (SHARP) Program, and the application comes with an instructional support package. The system has been tested with approximately 200 users, and is presently being used in the SHARP Academy's capstone course.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The Digital Survivor of Sexual Assault (DS2A) is an interface that allows a user to have a conversational experience with a survivor of sexual assault, using Artificial Intelligence technology and recorded videos. The application uses a statistical classifier to retrieve contextually appropriate pre-recorded video utterances by the survivor, together with dialogue management policies which enable users to conduct simulated conversations with the survivor about the sexual assault, its aftermath, and other pertinent topics. The content in the application has been specifically elicited to support the needs for the training of U.S. Army professionals in the Sexual Harassment/Assault Response and Prevention (SHARP) Program, and the application comes with an instructional support package. The system has been tested with approximately 200 users, and is presently being used in the SHARP Academy's capstone course.
234.
Bönsch, Andrea; Feng, Andrew; Patel, Parth; Shapiro, Ari
Volumetric Video Capture using Unsynchronized, Low-cost Cameras: Proceedings Article
In: Proceedings of the 14th International Joint Conference on Computer Vision, Imaging and Computer Graphics Theory and Applications, pp. 255–261, SCITEPRESS - Science and Technology Publications, Prague, Czech Republic, 2019, ISBN: 978-989-758-354-4.
@inproceedings{bonsch_volumetric_2019,
title = {Volumetric Video Capture using Unsynchronized, Low-cost Cameras:},
author = {Andrea Bönsch and Andrew Feng and Parth Patel and Ari Shapiro},
url = {http://www.scitepress.org/DigitalLibrary/Link.aspx?doi=10.5220/0007373202550261},
doi = {10.5220/0007373202550261},
isbn = {978-989-758-354-4},
year = {2019},
date = {2019-02-01},
booktitle = {Proceedings of the 14th International Joint Conference on Computer Vision, Imaging and Computer Graphics Theory and Applications},
pages = {255–261},
publisher = {SCITEPRESS - Science and Technology Publications},
address = {Prague, Czech Republic},
abstract = {Volumetric video can be used in virtual and augmented reality applications to show detailed animated performances by human actors. In this paper, we describe a volumetric capture system based on a photogrammetry cage with unsynchronized, low-cost cameras which is able to generate high-quality geometric data for animated avatars. This approach requires, inter alia, a subsequent synchronization of the captured videos.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Volumetric video can be used in virtual and augmented reality applications to show detailed animated performances by human actors. In this paper, we describe a volumetric capture system based on a photogrammetry cage with unsynchronized, low-cost cameras which is able to generate high-quality geometric data for animated avatars. This approach requires, inter alia, a subsequent synchronization of the captured videos.
235.
Rizzo, Albert; Koenig, Sebastian Thomas; Talbot, Thomas B.
Clinical Results Using Virtual Reality Journal Article
In: Journal of Technology in Human Services, vol. 37, no. 1, pp. 51–74, 2019, ISSN: 1522-8835, 1522-8991.
@article{rizzo_clinical_2019-1,
title = {Clinical Results Using Virtual Reality},
author = {Albert Rizzo and Sebastian Thomas Koenig and Thomas B. Talbot},
editor = {Gabor F. Fulop and Charles M. Hanson and Bjørn F. Andresen},
url = {https://www.tandfonline.com/doi/full/10.1080/15228835.2019.1604292},
doi = {10.1080/15228835.2019.1604292},
issn = {1522-8835, 1522-8991},
year = {2019},
date = {2019-02-01},
booktitle = {Infrared Technology and Applications XLV},
journal = {Journal of Technology in Human Services},
volume = {37},
number = {1},
pages = {51–74},
publisher = {SPIE},
address = {Baltimore, United States},
abstract = {Virtual Reality (VR) technology offers new opportunities for the development of innovative clinical research, assessment, and intervention tools. VR-based testing, training, teaching, and treatment approaches that would be difficult, if not impossible to deliver using traditional methods are now being developed that take advantage of the assets that are available with VR technology. As research evidence continues to indicate clinical efficacy, VR applications are being increasingly regarded as providing innovative options for targeting the cognitive, psychological, motor, and functional impairments that result from various clinical health conditions. VR allows for the precise presentation and control of stimuli within dynamic multisensory 3-D computer generated simulations as well as providing advanced methods for capturing and quantifying behavioral responses. These characteristics support the rationale for the use of VR applications in clinical assessment, intervention, and training. This article begins with a brief review of the history and rationale for the use of VR with clinical populations. We then detail one use-case for the clinical application of VR—the exposure therapy treatment of anxiety disorders and post-traumatic stress disorder. While significant work is cited in other areas of Clinical VR (e.g., pain management, cognitive/physical assessment/rehabilitation, eating disorders, social skills/clinical training, etc.), a full overview of such a broad literature is beyond the scope of this article. Thus, we have opted to provide more in-depth analysis of one specific clinical area that clearly illustrates how VR has been successfully applied and is supported by an encouraging and evolving scientific literature.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Virtual Reality (VR) technology offers new opportunities for the development of innovative clinical research, assessment, and intervention tools. VR-based testing, training, teaching, and treatment approaches that would be difficult, if not impossible to deliver using traditional methods are now being developed that take advantage of the assets that are available with VR technology. As research evidence continues to indicate clinical efficacy, VR applications are being increasingly regarded as providing innovative options for targeting the cognitive, psychological, motor, and functional impairments that result from various clinical health conditions. VR allows for the precise presentation and control of stimuli within dynamic multisensory 3-D computer generated simulations as well as providing advanced methods for capturing and quantifying behavioral responses. These characteristics support the rationale for the use of VR applications in clinical assessment, intervention, and training. This article begins with a brief review of the history and rationale for the use of VR with clinical populations. We then detail one use-case for the clinical application of VR—the exposure therapy treatment of anxiety disorders and post-traumatic stress disorder. While significant work is cited in other areas of Clinical VR (e.g., pain management, cognitive/physical assessment/rehabilitation, eating disorders, social skills/clinical training, etc.), a full overview of such a broad literature is beyond the scope of this article. Thus, we have opted to provide more in-depth analysis of one specific clinical area that clearly illustrates how VR has been successfully applied and is supported by an encouraging and evolving scientific literature.
236.
Melo, Celso M.; Marsella, Stacy; Gratch, Jonathan
Human Cooperation When Acting Through Autonomous Machines Journal Article
In: Proceedings of the National Academy of Sciences, vol. 116, no. 9, pp. 3482–3487, 2019, ISSN: 0027-8424, 1091-6490.
@article{de_melo_human_2019,
title = {Human Cooperation When Acting Through Autonomous Machines},
author = {Celso M. Melo and Stacy Marsella and Jonathan Gratch},
url = {http://www.pnas.org/lookup/doi/10.1073/pnas.1817656116},
doi = {10.1073/pnas.1817656116},
issn = {0027-8424, 1091-6490},
year = {2019},
date = {2019-02-01},
journal = {Proceedings of the National Academy of Sciences},
volume = {116},
number = {9},
pages = {3482–3487},
abstract = {Recent times have seen an emergence of intelligent machines that act autonomously on our behalf, such as autonomous vehicles. Despite promises of increased efficiency, it is not clear whether this paradigm shift will change how we decide when our self-interest (e.g., comfort) is pitted against the collective interest (e.g., environment). Here we show that acting through machines changes the way people solve these social dilemmas and we present experimental evidence showing that participants program their autonomous vehicles to act more cooperatively than if they were driving themselves. We show that this happens because programming causes selfish short-term rewards to become less salient, leading to considerations of broader societal goals. We also show that the programmed behavior is influenced by past experience. Finally, we report evidence that the effect generalizes beyond the domain of autonomous vehicles. We discuss implications for designing autonomous machines that contribute to a more cooperative society},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Recent times have seen an emergence of intelligent machines that act autonomously on our behalf, such as autonomous vehicles. Despite promises of increased efficiency, it is not clear whether this paradigm shift will change how we decide when our self-interest (e.g., comfort) is pitted against the collective interest (e.g., environment). Here we show that acting through machines changes the way people solve these social dilemmas and we present experimental evidence showing that participants program their autonomous vehicles to act more cooperatively than if they were driving themselves. We show that this happens because programming causes selfish short-term rewards to become less salient, leading to considerations of broader societal goals. We also show that the programmed behavior is influenced by past experience. Finally, we report evidence that the effect generalizes beyond the domain of autonomous vehicles. We discuss implications for designing autonomous machines that contribute to a more cooperative society
237.
Chu, Veronica C.; Lucas, Gale M.; Lei, Su; Mozgai, Sharon; Khooshabeh, Peter; Gratch, Jonathan
Emotion Regulation in the Prisoner’s Dilemma: Effects of Reappraisal on Behavioral Measures and Cardiovascular Measures of Challenge and Threat Journal Article
In: Frontiers in Human Neuroscience, vol. 13, 2019, ISSN: 1662-5161.
@article{chu_emotion_2019,
title = {Emotion Regulation in the Prisoner’s Dilemma: Effects of Reappraisal on Behavioral Measures and Cardiovascular Measures of Challenge and Threat},
author = {Veronica C. Chu and Gale M. Lucas and Su Lei and Sharon Mozgai and Peter Khooshabeh and Jonathan Gratch},
url = {https://www.frontiersin.org/article/10.3389/fnhum.2019.00050/full},
doi = {10.3389/fnhum.2019.00050},
issn = {1662-5161},
year = {2019},
date = {2019-02-01},
journal = {Frontiers in Human Neuroscience},
volume = {13},
abstract = {The current study examines cooperation and cardiovascular responses in individuals that were defected on by their opponent in the first round of an iterated Prisoner’s Dilemma. In this scenario, participants were either primed with the emotion regulation strategy of reappraisal or no emotion regulation strategy, and their opponent either expressed an amused smile or a polite smile after the results were presented. We found that cooperation behavior decreased in the no emotion regulation group when the opponent expressed an amused smile compared to a polite smile. In the cardiovascular measures, we found significant differences between the emotion regulation conditions using the biopsychosocial (BPS) model of challenge and threat. However, the cardiovascular measures of participants instructed with the reappraisal strategy were only weakly comparable with a threat state of the BPS model, which involves decreased blood flow and perception of greater task demands than resources to cope with those demands. Conversely, the cardiovascular measures of participants without an emotion regulation were only weakly comparable with a challenge state of the BPS model, which involves increased blood flow and perception of having enough or more resources to cope with task demands.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The current study examines cooperation and cardiovascular responses in individuals that were defected on by their opponent in the first round of an iterated Prisoner’s Dilemma. In this scenario, participants were either primed with the emotion regulation strategy of reappraisal or no emotion regulation strategy, and their opponent either expressed an amused smile or a polite smile after the results were presented. We found that cooperation behavior decreased in the no emotion regulation group when the opponent expressed an amused smile compared to a polite smile. In the cardiovascular measures, we found significant differences between the emotion regulation conditions using the biopsychosocial (BPS) model of challenge and threat. However, the cardiovascular measures of participants instructed with the reappraisal strategy were only weakly comparable with a threat state of the BPS model, which involves decreased blood flow and perception of greater task demands than resources to cope with those demands. Conversely, the cardiovascular measures of participants without an emotion regulation were only weakly comparable with a challenge state of the BPS model, which involves increased blood flow and perception of having enough or more resources to cope with task demands.
238.
Talbot, Thomas B.; Rizzo, Albert Skip
Virtual Standardized Patients for Interactive Conversational Training: A Grand Experiment and New Approach Book Section
In: Exploring the Cognitive, Social, Cultural, and Psychological Aspects of Gaming and Simulations:, IGI Global, Hershey, PA, 2019, ISBN: 978-1-5225-7461-3 978-1-5225-7462-0.
@incollection{talbot_virtual_2019,
title = {Virtual Standardized Patients for Interactive Conversational Training: A Grand Experiment and New Approach},
author = {Thomas B. Talbot and Albert Skip Rizzo},
url = {http://services.igi-global.com/resolvedoi/resolve.aspx?doi=10.4018/978-1-5225-7461-3},
doi = {10.4018/978-1-5225-7461-3},
isbn = {978-1-5225-7461-3 978-1-5225-7462-0},
year = {2019},
date = {2019-01-01},
booktitle = {Exploring the Cognitive, Social, Cultural, and Psychological Aspects of Gaming and Simulations:},
publisher = {IGI Global},
address = {Hershey, PA},
series = {Advances in Game-Based Learning},
abstract = {The USC Standard Patient is a virtual human-based conversational agent serving in the role of a simulated medical patient, also known as a virtual standardized patient (VSP). This research identified deficiencies of extant VSP systems, defined a robust set of requirements, and successfully achieved nearly all of them. Markedly impressive advancements were made in virtual human technology, techniques to apply natural language processing, automated assessment artificial intelligence, and pedagogical design. The effort succeeded with performance parameters of high conversational performance, accurate assessment, and strongly demonstrated user training effect. Although working well within its confined are of expertise, the ability for computers to create authentic mixed initiative conversations remains elusive. This effort leaves behind many lessons for interactive serious games, clinical virtual humans, and conversational virtual human training applications.},
keywords = {},
pubstate = {published},
tppubtype = {incollection}
}
The USC Standard Patient is a virtual human-based conversational agent serving in the role of a simulated medical patient, also known as a virtual standardized patient (VSP). This research identified deficiencies of extant VSP systems, defined a robust set of requirements, and successfully achieved nearly all of them. Markedly impressive advancements were made in virtual human technology, techniques to apply natural language processing, automated assessment artificial intelligence, and pedagogical design. The effort succeeded with performance parameters of high conversational performance, accurate assessment, and strongly demonstrated user training effect. Although working well within its confined are of expertise, the ability for computers to create authentic mixed initiative conversations remains elusive. This effort leaves behind many lessons for interactive serious games, clinical virtual humans, and conversational virtual human training applications.
239.
Chen, Meida; Astani, Sonny; McAlinden, Ryan; Spicer, Ryan
Semantic Modeling of Outdoor Scenes for the Creation of Virtual Environments and Simulations Proceedings Article
In: Proceedings of the 52nd Hawaii International Conference on System Sciences, pp. 10, IEEE, Maui, Hawaii, 2019, ISBN: 978-0-9981331-2-6.
@inproceedings{chen_semantic_2019,
title = {Semantic Modeling of Outdoor Scenes for the Creation of Virtual Environments and Simulations},
author = {Meida Chen and Sonny Astani and Ryan McAlinden and Ryan Spicer},
url = {https://scholarspace.manoa.hawaii.edu/handle/10125/59634},
doi = {10.24251/HICSS.2019.236},
isbn = {978-0-9981331-2-6},
year = {2019},
date = {2019-01-01},
booktitle = {Proceedings of the 52nd Hawaii International Conference on System Sciences},
pages = {10},
publisher = {IEEE},
address = {Maui, Hawaii},
abstract = {Efforts from both academia and industry have adopted photogrammetric techniques to generate visually compelling 3D models for the creation of virtual environments and simulations. However, such generated meshes do not contain semantic information for distinguishing between objects. To allow both user- and system-level interaction with the meshes, and enhance the visual acuity of the scene, classifying the generated point clouds and associated meshes is a necessary step. This paper presents a point cloud/mesh classification and segmentation framework. The proposed framework provides a novel way of extracting object information – i.e., individual tree locations and related features while considering the data quality issues presented in a photogrammetric-generated point cloud. A case study has been conducted using data that were collected at the University of Southern California to evaluate the proposed framework.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Efforts from both academia and industry have adopted photogrammetric techniques to generate visually compelling 3D models for the creation of virtual environments and simulations. However, such generated meshes do not contain semantic information for distinguishing between objects. To allow both user- and system-level interaction with the meshes, and enhance the visual acuity of the scene, classifying the generated point clouds and associated meshes is a necessary step. This paper presents a point cloud/mesh classification and segmentation framework. The proposed framework provides a novel way of extracting object information – i.e., individual tree locations and related features while considering the data quality issues presented in a photogrammetric-generated point cloud. A case study has been conducted using data that were collected at the University of Southern California to evaluate the proposed framework.
240.
Joshi, Himanshu; Rosenbloom, Paul S; Ustun, Volkan
Exact, Tractable Inference in the Sigma Cognitive Architecture via Sum-Product Networks Journal Article
In: Advances in Cognitive Systems, pp. 31–47, 2018.
@article{joshi_exact_2018,
title = {Exact, Tractable Inference in the Sigma Cognitive Architecture via Sum-Product Networks},
author = {Himanshu Joshi and Paul S Rosenbloom and Volkan Ustun},
url = {http://www.cogsys.org/papers/ACSvol7/papers/paper-7-4.pdf},
year = {2018},
date = {2018-12-01},
journal = {Advances in Cognitive Systems},
pages = {31–47},
abstract = {Sum-product networks (SPNs) are a new kind of deep architecture that support exact, tractable inference over a large class of problems for which traditional graphical models cannot. The Sigma cognitive architecture is based on graphical models, posing a challenge for it to handle problems within this class, such as parsing with probabilistic grammars, a potentially important aspect of language processing. This work proves that an early unidirectional extension to Sigma’s graphical architecture, originally added in service of rule-like behavior but later also shown to support neural networks, can be leveraged to yield exact, tractable computations across this class of problems, and further demonstrates this tractability experimentally for probabilistic parsing. It thus shows that Sigma is able to specify any valid SPN and, despite its grounding in graphical models, retain the desirable inference properties of SPNs when solving them.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Sum-product networks (SPNs) are a new kind of deep architecture that support exact, tractable inference over a large class of problems for which traditional graphical models cannot. The Sigma cognitive architecture is based on graphical models, posing a challenge for it to handle problems within this class, such as parsing with probabilistic grammars, a potentially important aspect of language processing. This work proves that an early unidirectional extension to Sigma’s graphical architecture, originally added in service of rule-like behavior but later also shown to support neural networks, can be leveraged to yield exact, tractable computations across this class of problems, and further demonstrates this tractability experimentally for probabilistic parsing. It thus shows that Sigma is able to specify any valid SPN and, despite its grounding in graphical models, retain the desirable inference properties of SPNs when solving them.
Filter
2014
Gordon, Andrew; Core, Mark; Kang, Sin-Hwa; Wang, Catherine; Wienberg, Christopher
Civilian Analogs of Army Tasks: Supporting Pedagogical Storytelling Across Domains Journal Article
In: Proceedings of the 11th International Conference of the Learning Sciences, 2014.
Abstract | Links | BibTeX | Tags: Learning Sciences, MedVR, The Narrative Group, UARC
@article{gordon_civilian_2014,
title = {Civilian Analogs of Army Tasks: Supporting Pedagogical Storytelling Across Domains},
author = {Andrew Gordon and Mark Core and Sin-Hwa Kang and Catherine Wang and Christopher Wienberg},
url = {http://ict.usc.edu/pubs/Civilian%20Analogs%20of%20Army%20Tasks%20-%20Supporting%20Pedagogical%20Storytelling%20Across%20Domains.pdf},
year = {2014},
date = {2014-06-01},
journal = {Proceedings of the 11th International Conference of the Learning Sciences},
abstract = {Storytelling is the most basic means by which people learn from the experiences of others. Advances in educational technologies offer new opportunities and experiences for learners, but risk losing the natural forms of pedagogical storytelling afforded by face-to-face teacher-student discussion. In this paper, we present a technology-supported solution to the problem of curating and algorithmically delivering relevant stories to learners in computer-based learning environments. Our approach is to mine public weblogs for textual narratives related to specific activity contexts, both inside and outside the domain of the target skillset. These stories are then linked directly to task representations in the learner model of an intelligent tutoring system, and delivered to learners along with other tutoring guidance. We demonstrate our approach to curating stories by creating collections of narratives that are analogous to tactical tasks of the U.S. Army, and evaluate the difficulty of incorporating these stories into intelligent tutoring systems.},
keywords = {Learning Sciences, MedVR, The Narrative Group, UARC},
pubstate = {published},
tppubtype = {article}
}
Storytelling is the most basic means by which people learn from the experiences of others. Advances in educational technologies offer new opportunities and experiences for learners, but risk losing the natural forms of pedagogical storytelling afforded by face-to-face teacher-student discussion. In this paper, we present a technology-supported solution to the problem of curating and algorithmically delivering relevant stories to learners in computer-based learning environments. Our approach is to mine public weblogs for textual narratives related to specific activity contexts, both inside and outside the domain of the target skillset. These stories are then linked directly to task representations in the learner model of an intelligent tutoring system, and delivered to learners along with other tutoring guidance. We demonstrate our approach to curating stories by creating collections of narratives that are analogous to tactical tasks of the U.S. Army, and evaluate the difficulty of incorporating these stories into intelligent tutoring systems.
Core, Mark; Lane, H. Chad; Traum, David
Intelligent Tutoring Support for Learners Interacting with Virtual Humans Book Section
In: Design Recommendations for Intelligent Tutoring Systems, vol. 2, pp. 249 – 257, 2014, ISBN: 978-0-9893923-2-7.
Links | BibTeX | Tags: Learning Sciences, UARC
@incollection{core_intelligent_2014,
title = {Intelligent Tutoring Support for Learners Interacting with Virtual Humans},
author = {Mark Core and H. Chad Lane and David Traum},
url = {http://books.google.com/books?hl=en&lr=&id=BNWEBAAAQBAJ&oi=fnd&pg=PR2&dq=+Design+Recommendations+for+Intelligent+Tutoring+Systems,+volume+2&ots=jIk3zyGi4M&sig=qb_hc4KKE3-rMh2mrs8WkxBicG4#v=onepage&q&f=false},
isbn = {978-0-9893923-2-7},
year = {2014},
date = {2014-06-01},
booktitle = {Design Recommendations for Intelligent Tutoring Systems},
volume = {2},
pages = {249 – 257},
keywords = {Learning Sciences, UARC},
pubstate = {published},
tppubtype = {incollection}
}
Morbini, Fabrizio; Forbell, Eric; Sagae, Kenji
Improving Classification-Based Natural Language Understanding with Non-Expert Annotation Proceedings Article
In: Proceedings of SIGDIAL 2014, pp. 69–73, Philadelphia, PA, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{morbini_improving_2014,
title = {Improving Classification-Based Natural Language Understanding with Non-Expert Annotation},
author = {Fabrizio Morbini and Eric Forbell and Kenji Sagae},
url = {http://ict.usc.edu/pubs/Improved%20Classification-based%20Natural%20Language%20Understanding%20with%20Non-Expert%20Annotation.pdf},
year = {2014},
date = {2014-06-01},
booktitle = {Proceedings of SIGDIAL 2014},
pages = {69–73},
address = {Philadelphia, PA},
abstract = {Although data-driven techniques are commonly used for Natural Language Understanding in dialogue systems, their efficacy is often hampered by the lack of appropriate annotated training data in sufficient amounts. We present an approach for rapid and cost-effective annotation of training data for classification-based language understanding in conversational dialogue systems. Experiments using a webaccessible conversational character that interacts with a varied user population show that a dramatic improvement in natural language understanding and a substantial reduction in expert annotation effort can be achieved by leveraging non-expert annotation.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Although data-driven techniques are commonly used for Natural Language Understanding in dialogue systems, their efficacy is often hampered by the lack of appropriate annotated training data in sufficient amounts. We present an approach for rapid and cost-effective annotation of training data for classification-based language understanding in conversational dialogue systems. Experiments using a webaccessible conversational character that interacts with a varied user population show that a dramatic improvement in natural language understanding and a substantial reduction in expert annotation effort can be achieved by leveraging non-expert annotation.
Stratou, Giota; Scherer, Stefan; Gratch, Jonathan; Morency, Louis-Philippe
Automatic nonverbal behavior indicators of depression and PTSD: the effect of gender Journal Article
In: Journal on Multimodal User Interfaces, vol. 9, no. 1, pp. 17 –29, 2014, ISSN: 1783-7677, 1783-8738.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@article{stratou_automatic_2014,
title = {Automatic nonverbal behavior indicators of depression and PTSD: the effect of gender},
author = {Giota Stratou and Stefan Scherer and Jonathan Gratch and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Automatic%20Nonverbal%20Behavior%20Indicators%20of%20Depression%20and%20PTSD%20-%20The%20Effect%20of%20Gender.pdf},
doi = {10.1007/s12193-014-0161-4},
issn = {1783-7677, 1783-8738},
year = {2014},
date = {2014-06-01},
journal = {Journal on Multimodal User Interfaces},
volume = {9},
number = {1},
pages = {17 –29},
abstract = {Recently there has been arising interest in automatically recognizing nonverbal behaviors that are linked with psychological conditions. Work in this direction has shown great potential for cases such as depression and post-traumatic stress disorder (PTSD), however most of the times gender differences have not been explored. In this paper, we show that gender plays an important role in the automatic assessment of psychological conditions such as depression and PTSD. We identify a directly interpretable and intuitive set of predictive indicators, selected from three general categories of nonverbal behaviors: affect, expression variability and motor variability. For the analysis, we employ a semi-structured virtual human interview dataset which includes 53 video recorded interactions. Our experiments on automatic classification of psychological conditions show that a gender-dependent approach significantly improves the performance over a gender agnostic one.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {article}
}
Recently there has been arising interest in automatically recognizing nonverbal behaviors that are linked with psychological conditions. Work in this direction has shown great potential for cases such as depression and post-traumatic stress disorder (PTSD), however most of the times gender differences have not been explored. In this paper, we show that gender plays an important role in the automatic assessment of psychological conditions such as depression and PTSD. We identify a directly interpretable and intuitive set of predictive indicators, selected from three general categories of nonverbal behaviors: affect, expression variability and motor variability. For the analysis, we employ a semi-structured virtual human interview dataset which includes 53 video recorded interactions. Our experiments on automatic classification of psychological conditions show that a gender-dependent approach significantly improves the performance over a gender agnostic one.
Vigil, Jesse; Tait, Asa Shumskas; Wienberg, Christopher; Gordon, Andrew S.
Friends You Haven’t Met Yet: A Documentary Short Film Proceedings Article
In: Proceedings of the 2014 ACM conference on Web science, pp. 176–176, ACM Press, Bloomington, IN, 2014, ISBN: 978-1-4503-2622-3.
Abstract | Links | BibTeX | Tags: The Narrative Group, UARC
@inproceedings{vigil_friends_2014,
title = {Friends You Haven’t Met Yet: A Documentary Short Film},
author = {Jesse Vigil and Asa Shumskas Tait and Christopher Wienberg and Andrew S. Gordon},
url = {http://dl.acm.org/citation.cfm?doid=2615569.2617797},
doi = {10.1145/2615569.2617797},
isbn = {978-1-4503-2622-3},
year = {2014},
date = {2014-06-01},
booktitle = {Proceedings of the 2014 ACM conference on Web science},
pages = {176–176},
publisher = {ACM Press},
address = {Bloomington, IN},
abstract = {"Friends You Haven't Met Yet" is a documentary short film that chronicles encounters between extremely prolific bloggers and a computer scientist who uses their personal narratives for research. It explores issues related to public sharing of personal stories, the ethical obligations of researchers who use web data, and the changing nature of online privacy. The film was conceived by Andrew Gordon and Christopher Wienberg at the University of Southern California, whose research involves the collection of millions of personal stories posted to internet weblogs. In analyzing their data, these researchers discovered an unusual population of extremely prolific bloggers, people who post personal stories about their daily lives everyday over the course of many years. They posed three questions about this population: 1. What motivates these people to post so frequently and publicly about their personal life? 2. To what degree do these people embellish their stories to make them more interesting than reality? 3. What expectations do these authors have about their readers, and what are the ethical implications for researchers like us who analyze their posts? To answer these questions, PhD Student Christopher Wienberg contacted many of these bloggers directly and set up face-to-face interviews at their homes. Accompanied by a documentary film crew, Christopher traveled to locations around California, in both urban and rural settings, to better understand the people whose contributions on the web serve as data in social media research.},
keywords = {The Narrative Group, UARC},
pubstate = {published},
tppubtype = {inproceedings}
}
"Friends You Haven't Met Yet" is a documentary short film that chronicles encounters between extremely prolific bloggers and a computer scientist who uses their personal narratives for research. It explores issues related to public sharing of personal stories, the ethical obligations of researchers who use web data, and the changing nature of online privacy. The film was conceived by Andrew Gordon and Christopher Wienberg at the University of Southern California, whose research involves the collection of millions of personal stories posted to internet weblogs. In analyzing their data, these researchers discovered an unusual population of extremely prolific bloggers, people who post personal stories about their daily lives everyday over the course of many years. They posed three questions about this population: 1. What motivates these people to post so frequently and publicly about their personal life? 2. To what degree do these people embellish their stories to make them more interesting than reality? 3. What expectations do these authors have about their readers, and what are the ethical implications for researchers like us who analyze their posts? To answer these questions, PhD Student Christopher Wienberg contacted many of these bloggers directly and set up face-to-face interviews at their homes. Accompanied by a documentary film crew, Christopher traveled to locations around California, in both urban and rural settings, to better understand the people whose contributions on the web serve as data in social media research.
DeVault, David; Artstein, Ron; Benn, Grace; Dey, Teresa; Fast, Edward; Gainer, Alesia; Georgila, Kallirroi; Gratch, Jonathan; Hartholt, Arno; Lhommet, Margaux; Lucas, Gale; Marsella, Stacy C.; Fabrizio, Morbini; Nazarian, Angela; Scherer, Stefan; Stratou, Giota; Suri, Apar; Traum, David; Wood, Rachel; Xu, Yuyu; Rizzo, Albert; Morency, Louis-Philippe
SimSensei Kiosk: A Virtual Human Interviewer for Healthcare Decision Support Proceedings Article
In: Proceedings of the 13th Inter-national Conference on Autonomous Agents and Multiagent Systems (AAMAS 2014), pp. 1061–1068, International Foundation for Autonomous Agents and Multiagent Systems, Paris, France, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{devault_simsensei_2014,
title = {SimSensei Kiosk: A Virtual Human Interviewer for Healthcare Decision Support},
author = {David DeVault and Ron Artstein and Grace Benn and Teresa Dey and Edward Fast and Alesia Gainer and Kallirroi Georgila and Jonathan Gratch and Arno Hartholt and Margaux Lhommet and Gale Lucas and Stacy C. Marsella and Morbini Fabrizio and Angela Nazarian and Stefan Scherer and Giota Stratou and Apar Suri and David Traum and Rachel Wood and Yuyu Xu and Albert Rizzo and Louis-Philippe Morency},
url = {https://dl.acm.org/citation.cfm?id=2617415},
year = {2014},
date = {2014-05-01},
booktitle = {Proceedings of the 13th Inter-national Conference on Autonomous Agents and Multiagent Systems (AAMAS 2014)},
pages = {1061–1068},
publisher = {International Foundation for Autonomous Agents and Multiagent Systems},
address = {Paris, France},
abstract = {We present SimSensei Kiosk, an implemented virtual human interviewer designed to create an engaging face-to-face inter-action where the user feels comfortable talking and sharing information. SimSensei Kiosk is also designed to create in- teractional situations favorable to the automatic assessment of distress indicators, de_ned as verbal and nonverbal behav- iors correlated with depression, anxiety or post-traumatic stress disorder (PTSD). In this paper, we summarize the de- sign methodology, performed over the past two years, which is based on three main development cycles: (1) analysis of face-to-face human interactions to identify potential distress indicators, dialogue policies and virtual human gestures, (2) development and analysis of a Wizard-of-Oz prototype sys- tem where two human operators were deciding the spoken and gestural responses, and (3) development of a fully au- tomatic virtual interviewer able to engage users in 15-25 minute interactions. We show the potential of our fully auto- matic virtual human interviewer in a user study, and situate its performance in relation to the Wizard-of-Oz prototype.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
We present SimSensei Kiosk, an implemented virtual human interviewer designed to create an engaging face-to-face inter-action where the user feels comfortable talking and sharing information. SimSensei Kiosk is also designed to create in- teractional situations favorable to the automatic assessment of distress indicators, de_ned as verbal and nonverbal behav- iors correlated with depression, anxiety or post-traumatic stress disorder (PTSD). In this paper, we summarize the de- sign methodology, performed over the past two years, which is based on three main development cycles: (1) analysis of face-to-face human interactions to identify potential distress indicators, dialogue policies and virtual human gestures, (2) development and analysis of a Wizard-of-Oz prototype sys- tem where two human operators were deciding the spoken and gestural responses, and (3) development of a fully au- tomatic virtual interviewer able to engage users in 15-25 minute interactions. We show the potential of our fully auto- matic virtual human interviewer in a user study, and situate its performance in relation to the Wizard-of-Oz prototype.
Gratch, Jonathan; Artstein, Ron; Lucas, Gale; Stratou, Giota; Scherer, Stefan; Nazarian, Angela; Wood, Rachel; Boberg, Jill; DeVault, David; Marsella, Stacy; Traum, David; Rizzo, Albert; Morency, Louis-Philippe
The Distress Analysis Interview Corpus of human and computer interviews Proceedings Article
In: Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC 2014), pp. 3123–3128, LREC, Reykjavik, Iceland, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{gratch_distress_2014,
title = {The Distress Analysis Interview Corpus of human and computer interviews},
author = {Jonathan Gratch and Ron Artstein and Gale Lucas and Giota Stratou and Stefan Scherer and Angela Nazarian and Rachel Wood and Jill Boberg and David DeVault and Stacy Marsella and David Traum and Albert Rizzo and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/The%20Distress%20Analysis%20Interview%20Corpus%20of%20human%20and%20computer%20interviews.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC 2014)},
pages = {3123–3128},
publisher = {LREC},
address = {Reykjavik, Iceland},
abstract = {The Distress Analysis Interview Corpus (DAIC) contains clinical interviews designed to support the diagnosis of psychological distress conditions such as anxiety, depression, and post traumatic stress disorder. The interviews are conducted by humans, human controlled agents and autonomous agents, and the participants include both distressed and non-distressed individuals. Data collected include audio and video recordings and extensive questionnaire responses; parts of the corpus have been transcribed and annotated for a variety of verbal and non-verbal features. The corpus has been used to support the creation of an automated interviewer agent, and for research on the automatic identification of psychological distress.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
The Distress Analysis Interview Corpus (DAIC) contains clinical interviews designed to support the diagnosis of psychological distress conditions such as anxiety, depression, and post traumatic stress disorder. The interviews are conducted by humans, human controlled agents and autonomous agents, and the participants include both distressed and non-distressed individuals. Data collected include audio and video recordings and extensive questionnaire responses; parts of the corpus have been transcribed and annotated for a variety of verbal and non-verbal features. The corpus has been used to support the creation of an automated interviewer agent, and for research on the automatic identification of psychological distress.
Shapiro, Ari; Feng, Andrew; Wang, Ruizhe; Li, Hao; Bolas, Mark; Medioni, Gerard; Suma, Evan
Rapid avatar capture and simulation using commodity depth sensors Journal Article
In: Computer Animation and Virtual Worlds, vol. 25, no. 3-4, pp. 201–211, 2014, ISSN: 15464261.
Abstract | Links | BibTeX | Tags: MxR, UARC, Virtual Humans
@article{shapiro_rapid_2014,
title = {Rapid avatar capture and simulation using commodity depth sensors},
author = {Ari Shapiro and Andrew Feng and Ruizhe Wang and Hao Li and Mark Bolas and Gerard Medioni and Evan Suma},
url = {http://ict.usc.edu/pubs/Rapid%20Avatar%20Capture%20and%20Simulation%20Using%20Commodity%20Depth%20Sensors.pdf},
doi = {10.1002/cav.1579},
issn = {15464261},
year = {2014},
date = {2014-05-01},
journal = {Computer Animation and Virtual Worlds},
volume = {25},
number = {3-4},
pages = {201–211},
abstract = {We demonstrate a method of acquiring a 3D model of a human using commodity scanning hardware and then controlling that 3D figure in a simulated environment in only a few minutes. The model acquisition requires four static poses taken at 90 degree angles relative to each other. The 3D model is then given a skeleton and smooth binding information necessary for control and simulation. The 3D models that are captured are suitable for use in applications where recognition and distinction among characters by shape, form, or clothing is important, such as small group or crowd simulations or other socially oriented applications. Because of the speed at which a human figure can be captured and the low hardware requirements, this method can be used to capture, track, and model human figures as their appearances change over time.},
keywords = {MxR, UARC, Virtual Humans},
pubstate = {published},
tppubtype = {article}
}
We demonstrate a method of acquiring a 3D model of a human using commodity scanning hardware and then controlling that 3D figure in a simulated environment in only a few minutes. The model acquisition requires four static poses taken at 90 degree angles relative to each other. The 3D model is then given a skeleton and smooth binding information necessary for control and simulation. The 3D models that are captured are suitable for use in applications where recognition and distinction among characters by shape, form, or clothing is important, such as small group or crowd simulations or other socially oriented applications. Because of the speed at which a human figure can be captured and the low hardware requirements, this method can be used to capture, track, and model human figures as their appearances change over time.
Krum, David M.; Kang, Sin-Hwa; Bolas, Mark
Virtual Coaches over Mobile Video Proceedings Article
In: Proceedingsof International Conference on Computer Animation and Social Agents (CASA), 2014.
Abstract | Links | BibTeX | Tags: MedVR, MxR, UARC
@inproceedings{krum_virtual_2014,
title = {Virtual Coaches over Mobile Video},
author = {David M. Krum and Sin-Hwa Kang and Mark Bolas},
url = {http://ict.usc.edu/pubs/Virtual%20Coaches%20over%20Mobile%20Video.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {Proceedingsof International Conference on Computer Animation and Social Agents (CASA)},
abstract = {We hypothesize that the context of a smartphone, how a virtual human is presented within a smartphone app, and indeed, the nature of that app, can profoundly affect how the virtual human is perceived by a real human. We believe that virtual humans, presented over video chat services (such as Skype) and delivered using mobile phones, can be an effective way to deliver coaching applications. We propose to build a prototype system that allows virtual humans to initiate and receive Skype calls. This hardware will enable broadcast of the audio and video imagery of a character. Using this platform and a virtual human, we will conduct two user studies. The first study will examine factors involved in making a mobile video based character seem engaging and “real”. This study will examine how character appearance and the artifacts of the communication channel, such as video and audio quality, can affect rapport with a virtual human. The second study will examine ways to maintain a long-term relationship with a character, leveraging the character’s ability to call and interact with a real human over a longer period of time. These studies will help develop design guidelines for presenting virtual humans over mobile video.},
keywords = {MedVR, MxR, UARC},
pubstate = {published},
tppubtype = {inproceedings}
}
We hypothesize that the context of a smartphone, how a virtual human is presented within a smartphone app, and indeed, the nature of that app, can profoundly affect how the virtual human is perceived by a real human. We believe that virtual humans, presented over video chat services (such as Skype) and delivered using mobile phones, can be an effective way to deliver coaching applications. We propose to build a prototype system that allows virtual humans to initiate and receive Skype calls. This hardware will enable broadcast of the audio and video imagery of a character. Using this platform and a virtual human, we will conduct two user studies. The first study will examine factors involved in making a mobile video based character seem engaging and “real”. This study will examine how character appearance and the artifacts of the communication channel, such as video and audio quality, can affect rapport with a virtual human. The second study will examine ways to maintain a long-term relationship with a character, leveraging the character’s ability to call and interact with a real human over a longer period of time. These studies will help develop design guidelines for presenting virtual humans over mobile video.
Kang, Sin-Hwa; Gratch, Jonathan
Exploring Users' Social Responses to Computer Counseling Interviewers' Behavior Journal Article
In: Journal of Computers in Human Behavior, vol. 34, pp. 120–130, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@article{kang_exploring_2014,
title = {Exploring Users' Social Responses to Computer Counseling Interviewers' Behavior},
author = {Sin-Hwa Kang and Jonathan Gratch},
url = {http://linkinghub.elsevier.com/retrieve/pii/S0747563214000119},
doi = {10.1016/j.chb.2014.01.006},
year = {2014},
date = {2014-05-01},
journal = {Journal of Computers in Human Behavior},
volume = {34},
pages = {120–130},
abstract = {We explore the effect of behavioral realism and reciprocal self-disclosure from computer interviewers on the social responses of human users in simulated psychotherapeutic counseling interactions. To investigate this subject, we designed a 3x3 factorial between-subjects experiment involving three conditions of behavioral realism: high realism, low realism, and audio-only (displaying no behavior at all) and three conditions of reciprocal self-disclosure: high disclosure, low disclosure, and no disclosure. We measured users' feelings of social presence (Copresence, Social Attraction, and Emotional Credibility), rapport, perception of the quality of users' own responses (Embarrassment and Self-Performance), emotional state (PANAS), perception of an interaction partner (Personal Perception), self-reported self-disclosure. We observed some contradictory outcomes in users' subjective reports. However, the results of objective data analysis demonstrated that users disclosed greater verbal self-disclosure (medium level of intimacy) when interacting with computer interviewers that displayed high behavioral realism and high self-disclosure. Users also delivered more fluent speech when interacting with computer interviewers that displayed high behaviorla realism. The results are described in detail and implications of the findings are discussed in this paper.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {article}
}
We explore the effect of behavioral realism and reciprocal self-disclosure from computer interviewers on the social responses of human users in simulated psychotherapeutic counseling interactions. To investigate this subject, we designed a 3x3 factorial between-subjects experiment involving three conditions of behavioral realism: high realism, low realism, and audio-only (displaying no behavior at all) and three conditions of reciprocal self-disclosure: high disclosure, low disclosure, and no disclosure. We measured users' feelings of social presence (Copresence, Social Attraction, and Emotional Credibility), rapport, perception of the quality of users' own responses (Embarrassment and Self-Performance), emotional state (PANAS), perception of an interaction partner (Personal Perception), self-reported self-disclosure. We observed some contradictory outcomes in users' subjective reports. However, the results of objective data analysis demonstrated that users disclosed greater verbal self-disclosure (medium level of intimacy) when interacting with computer interviewers that displayed high behavioral realism and high self-disclosure. Users also delivered more fluent speech when interacting with computer interviewers that displayed high behaviorla realism. The results are described in detail and implications of the findings are discussed in this paper.
Miguel, Eder; Feng, Andrew; Xu, Yuyu; Shapiro, Ari
Towards Cloth-Manipulating Characters Proceedings Article
In: CASA 2014, Houston, Texas, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{miguel_towards_2014,
title = {Towards Cloth-Manipulating Characters},
author = {Eder Miguel and Andrew Feng and Yuyu Xu and Ari Shapiro},
url = {http://ict.usc.edu/pubs/Towards%20Cloth-Manipulating%20Characters.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {CASA 2014},
address = {Houston, Texas},
abstract = {Cloth manipulation is a common action in humans that current animated virtual characters are not able to perform due to its complexity. In this paper we focus on dressing-up, which is probably the most common action involving cloth. We identify the steps required to perform the task and describe the systems responsible for each of them. Our results show a character that is able to put on a scarf and react to cloth collision and over-stretching events.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Cloth manipulation is a common action in humans that current animated virtual characters are not able to perform due to its complexity. In this paper we focus on dressing-up, which is probably the most common action involving cloth. We identify the steps required to perform the task and describe the systems responsible for each of them. Our results show a character that is able to put on a scarf and react to cloth collision and over-stretching events.
Chatterjee, Moitreya; Stratou, Giota; Scherer, Stefan; Morency, Louis-Philippe
CONTEXT-BASED SIGNAL DESCRIPTORS OF HEART-RATE VARIABILITY FOR ANXIETY ASSESSMENT Proceedings Article
In: Acoustics, Speech and Signal Processing (ICASSP), 2014 IEEE International Conference on, pp. 3631–3635, IEEE, Florence, Italy, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{chatterjee_context-based_2014,
title = {CONTEXT-BASED SIGNAL DESCRIPTORS OF HEART-RATE VARIABILITY FOR ANXIETY ASSESSMENT},
author = {Moitreya Chatterjee and Giota Stratou and Stefan Scherer and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Context-based%20signal%20descriptors%20of%20heart-rate%20variability%20for%20anxiety%20assessment.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {Acoustics, Speech and Signal Processing (ICASSP), 2014 IEEE International Conference on},
pages = {3631–3635},
publisher = {IEEE},
address = {Florence, Italy},
abstract = {In this paper, we investigate the role of multiple context-based heart-rate variability descriptors for evaluating a person’s psychological health, specifically anxiety disorders. The descriptors are extracted from visually sensed heart-rate signals obtained during the course of a semi-structured interview with a virtual human and can potentially integrate question context as well. The proposed descriptors are motivated by prior related work and are constructed based on histogram-based approaches, time and frequency domain analysis of heart-rate variability. In order to contextualize our descriptors, we use information about the polarity and intimacy levels of the questions asked. Our experiments reveal that the descriptors, both with and without context, perform far better than chance in predicting anxiety. Further on, we perform at-a-par with the state-of-the-art in predicting anxiety and other psychological disorders when we integrate the question context information into the descriptors.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
In this paper, we investigate the role of multiple context-based heart-rate variability descriptors for evaluating a person’s psychological health, specifically anxiety disorders. The descriptors are extracted from visually sensed heart-rate signals obtained during the course of a semi-structured interview with a virtual human and can potentially integrate question context as well. The proposed descriptors are motivated by prior related work and are constructed based on histogram-based approaches, time and frequency domain analysis of heart-rate variability. In order to contextualize our descriptors, we use information about the polarity and intimacy levels of the questions asked. Our experiments reveal that the descriptors, both with and without context, perform far better than chance in predicting anxiety. Further on, we perform at-a-par with the state-of-the-art in predicting anxiety and other psychological disorders when we integrate the question context information into the descriptors.
Chiu, Chung-Cheng; Marsella, Stacy C.
Gesture Generation with Low-Dimensional Embeddings Proceedings Article
In: Proceedings of the 13th International Conference on Autonomous Agents and Multiagent Systems, pp. 781–788, Paris, France, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{chiu_gesture_2014,
title = {Gesture Generation with Low-Dimensional Embeddings},
author = {Chung-Cheng Chiu and Stacy C. Marsella},
url = {http://ict.usc.edu/pubs/Gesture%20generation%20with%20low-dimensional%20embeddings.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {Proceedings of the 13th International Conference on Autonomous Agents and Multiagent Systems},
pages = {781–788},
address = {Paris, France},
abstract = {There is a growing demand for embodied agents capable of engaging in face-to-face dialog using the same verbal and nonverbal behavior that people use. The focus of our work is generating coverbal hand gestures for these agents, gestures coupled to the content and timing of speech. A common approach to achieve this is to use motion capture of an actor or hand-crafted animations for each utterance. An alternative machine learning approach that saves development effort is to learn a general gesture controller that can generate behavior for novel utterances. However learning a direct mapping from speech to gesture movement faces the complexity of inferring the relation between the two time series of speech and gesture motion. We present a novel machine learning approach that decomposes the overall learning problem into learning two mappings: from speech to a gestural annotation and from gestural annotation to gesture motion. The combined model learns to synthesize natural gesture animation from speech audio. We assess the quality of generated animations by comparing them with the result generated by a previous approach that learns a direct mapping. Results from a human subject study show that our framework is perceived to be significantly better.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
There is a growing demand for embodied agents capable of engaging in face-to-face dialog using the same verbal and nonverbal behavior that people use. The focus of our work is generating coverbal hand gestures for these agents, gestures coupled to the content and timing of speech. A common approach to achieve this is to use motion capture of an actor or hand-crafted animations for each utterance. An alternative machine learning approach that saves development effort is to learn a general gesture controller that can generate behavior for novel utterances. However learning a direct mapping from speech to gesture movement faces the complexity of inferring the relation between the two time series of speech and gesture motion. We present a novel machine learning approach that decomposes the overall learning problem into learning two mappings: from speech to a gestural annotation and from gestural annotation to gesture motion. The combined model learns to synthesize natural gesture animation from speech audio. We assess the quality of generated animations by comparing them with the result generated by a previous approach that learns a direct mapping. Results from a human subject study show that our framework is perceived to be significantly better.
Gratch, Jonathan; Lucas, Gale M.; King, Aisha Aisha; Morency, Louis-Philippe
It’s Only a Computer: The Impact of Human-agent Interaction in Clinical Interviews Proceedings Article
In: Proceedings of 13th International Conference on Autonomous Agents and Multiagent Systems, pp. 85–92, International Foundation for Autonomous Agents and Multiagent Systems, Paris, France, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{gratch_its_2014,
title = {It’s Only a Computer: The Impact of Human-agent Interaction in Clinical Interviews},
author = {Jonathan Gratch and Gale M. Lucas and Aisha Aisha King and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/It%E2%80%99s%20only%20a%20computer%20-%20The%20impact%20of%20human-agent%20interaction%20in%20clinical%20interviews.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {Proceedings of 13th International Conference on Autonomous Agents and Multiagent Systems},
pages = {85–92},
publisher = {International Foundation for Autonomous Agents and Multiagent Systems},
address = {Paris, France},
abstract = {Research has begun to explore the use of virtual humans (VHs) in medical interviews [1]. When designed as supportive and “safe” interaction partners, VHs may improve such screenings by encouraging patients to disclose more personal information [2-3]. In medical contexts, patients often feel resistance to selfdisclosure and engage in impression management to be viewed more positively by healthcare providers. This paper provides the first empirical evidence that VHs can reduce such resistance and impression management. In the context of health-screening interviews, we report a study in which participants interacted with a VH that was either teleo-operated by humans (Wizard-of-Oz) or fully-automated (AI). Independently, we manipulated whether participants believed the VH was controlled by humans or automation. As predicted, participants who believed they were interacting with a computer reported lower resistance to selfdisclosure, lower impression management and higher system usability than those who believed they were interacting with a human operator. Whether the virtual human was actually operated by a human or AI only affected ratings of the system’s usability. These results suggest that automated VHs can help overcome a significant barrier to obtaining truthful patient information in medical domains.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Research has begun to explore the use of virtual humans (VHs) in medical interviews [1]. When designed as supportive and “safe” interaction partners, VHs may improve such screenings by encouraging patients to disclose more personal information [2-3]. In medical contexts, patients often feel resistance to selfdisclosure and engage in impression management to be viewed more positively by healthcare providers. This paper provides the first empirical evidence that VHs can reduce such resistance and impression management. In the context of health-screening interviews, we report a study in which participants interacted with a VH that was either teleo-operated by humans (Wizard-of-Oz) or fully-automated (AI). Independently, we manipulated whether participants believed the VH was controlled by humans or automation. As predicted, participants who believed they were interacting with a computer reported lower resistance to selfdisclosure, lower impression management and higher system usability than those who believed they were interacting with a human operator. Whether the virtual human was actually operated by a human or AI only affected ratings of the system’s usability. These results suggest that automated VHs can help overcome a significant barrier to obtaining truthful patient information in medical domains.
Malandrakis, Nikolaos; Potamianos, Alexandros; Hsu, Kean J.; Babeva, Kalina N.; Feng, Michelle C.; Davison, Gerald C.; Narayanan, Shrikanth
AFFECTIVE LANGUAGE MODEL ADAPTATION VIA CORPUS SELECTION Proceedings Article
In: proceedings of ICASSP, Florence, Italy, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{malandrakis_affective_2014,
title = {AFFECTIVE LANGUAGE MODEL ADAPTATION VIA CORPUS SELECTION},
author = {Nikolaos Malandrakis and Alexandros Potamianos and Kean J. Hsu and Kalina N. Babeva and Michelle C. Feng and Gerald C. Davison and Shrikanth Narayanan},
url = {http://ict.usc.edu/pubs/Affective%20language%20model%20adaptation%20via%20corpus%20selection.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {proceedings of ICASSP},
address = {Florence, Italy},
abstract = {Motivated by methods used in language modeling and grammar induction, we propose the use of pragmatic constraints and perplexity as criteria to filter the unlabeled data used to generate the semantic similarity model. We investigate unsupervised adaptation algorithms of the semantic-affective models proposed in [1, 2]. Affective ratings at the utterance level are generated based on an emotional lexicon, which in turn is created using a semantic (similarity) model estimated over raw, unlabeled text. The proposed adaptation method creates task-dependent semantic similarity models and task- dependent word/term affective ratings. The proposed adaptation algorithms are tested on anger/distress detection of transcribed speech data and sentiment analysis in tweets showing significant relative classification error reduction of up to 10%.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Motivated by methods used in language modeling and grammar induction, we propose the use of pragmatic constraints and perplexity as criteria to filter the unlabeled data used to generate the semantic similarity model. We investigate unsupervised adaptation algorithms of the semantic-affective models proposed in [1, 2]. Affective ratings at the utterance level are generated based on an emotional lexicon, which in turn is created using a semantic (similarity) model estimated over raw, unlabeled text. The proposed adaptation method creates task-dependent semantic similarity models and task- dependent word/term affective ratings. The proposed adaptation algorithms are tested on anger/distress detection of transcribed speech data and sentiment analysis in tweets showing significant relative classification error reduction of up to 10%.
Vaz, Colin; Tsiartas, Andreas; Narayanan, Shrikanth
ENERGY-CONSTRAINED MINIMUM VARIANCE RESPONSE FILTER FOR ROBUST VOWEL SPECTRAL ESTIMATION Proceedings Article
In: Acoustics, Speech and Signal Processing (ICASSP), 2014 IEEE International Conference on, pp. 6275–6279, IEEE, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{vaz_energy-constrained_2014,
title = {ENERGY-CONSTRAINED MINIMUM VARIANCE RESPONSE FILTER FOR ROBUST VOWEL SPECTRAL ESTIMATION},
author = {Colin Vaz and Andreas Tsiartas and Shrikanth Narayanan},
url = {http://ict.usc.edu/pubs/Energy-Constrained%20Minimum%20Variance%20Response%20Filter%20for%20Robust%20Vowel%20Spectral%20Estimation.pdf},
year = {2014},
date = {2014-05-01},
booktitle = {Acoustics, Speech and Signal Processing (ICASSP), 2014 IEEE International Conference on},
pages = {6275–6279},
publisher = {IEEE},
abstract = {We propose the energy-constrained minimum-variance response (ECMVR) filter to perform robust spectral estimation of vowels. We modify the distortionless constraint of the minimum-variance distortionless response (MVDR) filter and add an energy constraint to its formulation to mitigate the influence of noise on the speech spectrum. We test our ECMVR filter on a vowel classification task with different background noises at various SNR levels. Results show that vowels are classified more accurately in certain noises using MFCC and PLP features extracted from the ECMVR spectrum compared to using features extracted from the FFT and MVDR spectra. Index Terms: frequency estimation, MVDR, robust signal processing, spectral estimation.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
We propose the energy-constrained minimum-variance response (ECMVR) filter to perform robust spectral estimation of vowels. We modify the distortionless constraint of the minimum-variance distortionless response (MVDR) filter and add an energy constraint to its formulation to mitigate the influence of noise on the speech spectrum. We test our ECMVR filter on a vowel classification task with different background noises at various SNR levels. Results show that vowels are classified more accurately in certain noises using MFCC and PLP features extracted from the ECMVR spectrum compared to using features extracted from the FFT and MVDR spectra. Index Terms: frequency estimation, MVDR, robust signal processing, spectral estimation.
Miller, Chreston; Quek, Francis; Morency, Louis-Philippe
Search Strategies for Pattern Identification in Multimodal Data: Three Case Studies Proceedings Article
In: pp. 273–280, ACM Press, 2014, ISBN: 978-1-4503-2782-4.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{miller_search_2014,
title = {Search Strategies for Pattern Identification in Multimodal Data: Three Case Studies},
author = {Chreston Miller and Francis Quek and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Search%20Strategies%20for%20Pattern%20Identification%20in%20Multimodal%20Data%20Three%20Case%20Studies.pdf},
doi = {10.1145/2578726.2578761},
isbn = {978-1-4503-2782-4},
year = {2014},
date = {2014-04-01},
pages = {273–280},
publisher = {ACM Press},
abstract = {The analysis of multimodal data benefits from meaningful search and retrieval. This paper investigates strategies of searching multimodal data for event patterns. Through three longitudinal case studies, we observed researchers exploring and identifying event patterns in multimodal data. The events were extracted from different multimedia signal sources ranging from annotated video transcripts to interaction logs. Each researcher’s data has varying temporal characteristics (e.g., sparse, dense, or clustered) that posed several challenges for identifying relevant patterns. We identify unique search strategies and better understand the aspects that contributed to each.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
The analysis of multimodal data benefits from meaningful search and retrieval. This paper investigates strategies of searching multimodal data for event patterns. Through three longitudinal case studies, we observed researchers exploring and identifying event patterns in multimodal data. The events were extracted from different multimedia signal sources ranging from annotated video transcripts to interaction logs. Each researcher’s data has varying temporal characteristics (e.g., sparse, dense, or clustered) that posed several challenges for identifying relevant patterns. We identify unique search strategies and better understand the aspects that contributed to each.
McAlinden, Ryan; Pynadath, David V.; Hill, Randall W.
UrbanSim: Using Social Simulation to Train for Stability Operations Book Section
In: Understanding Megacities with the Reconnaissance, Surveillance, and Intelligence Paradigm, 2014.
Abstract | Links | BibTeX | Tags: Social Simulation, STG, UARC
@incollection{mcalinden_urbansim_2014,
title = {UrbanSim: Using Social Simulation to Train for Stability Operations},
author = {Ryan McAlinden and David V. Pynadath and Randall W. Hill},
url = {http://ict.usc.edu/pubs/UrbanSim%20-%20Using%20Social%20Simulation%20to%20Train%20for%20Stability%20Operations.pdf},
year = {2014},
date = {2014-04-01},
booktitle = {Understanding Megacities with the Reconnaissance, Surveillance, and Intelligence Paradigm},
abstract = {As the United States reorients itself towards to a period of reduced military capacity and away from large‐footprint military engagements, there is an imperative to keep commanders and decision‐makers mentally sharp and prepared for the next ‘hot spot.’ One potential hot spot, megacities, presents a unique set of challenges due to their expansive, often interwoven ethnographic landscapes, and their overall lack of understanding by many western experts. Social simulation using agent‐based models is one approach for furthering our understanding of distant societies and their security implications, and for preparing leaders to engage these populations if and when the need arises. Over the past ten years, the field of social simulation has become decidedly cross‐discipline, including academics and practitioners from the fields of sociology, anthropology, psychology, artificial intelligence and engineering. This has led to an unparalleled advancement in social simulation theory and practice, and as new threats evolve to operate within dense but expansive urban environments, social simulation has a unique opportunity to shape our perspectives and develop knowledge that may otherwise be difficult to obtain. This article presents a social simulation‐based training application (UrbanSim) developed by the University of Southern California’s Institute for Creative Technologies (USC‐ICT) in partnership with the US Army’s School for Command Preparation (SCP). UrbanSim has been in‐use since 2009 to help Army commanders understand and train for missions in complex, uncertain environments. The discussion describes how the social simulation‐based training application was designed to develop and hone commanders' skills for conducting missions in environs with multifaceted social, ethnic and political fabrics. We present a few considerations when attempting to recreate dense, rapidly growing population centers, and how the integration of real‐world data into social simulation frameworks can add a level of realism and understanding not possible even a few years ago.},
keywords = {Social Simulation, STG, UARC},
pubstate = {published},
tppubtype = {incollection}
}
As the United States reorients itself towards to a period of reduced military capacity and away from large‐footprint military engagements, there is an imperative to keep commanders and decision‐makers mentally sharp and prepared for the next ‘hot spot.’ One potential hot spot, megacities, presents a unique set of challenges due to their expansive, often interwoven ethnographic landscapes, and their overall lack of understanding by many western experts. Social simulation using agent‐based models is one approach for furthering our understanding of distant societies and their security implications, and for preparing leaders to engage these populations if and when the need arises. Over the past ten years, the field of social simulation has become decidedly cross‐discipline, including academics and practitioners from the fields of sociology, anthropology, psychology, artificial intelligence and engineering. This has led to an unparalleled advancement in social simulation theory and practice, and as new threats evolve to operate within dense but expansive urban environments, social simulation has a unique opportunity to shape our perspectives and develop knowledge that may otherwise be difficult to obtain. This article presents a social simulation‐based training application (UrbanSim) developed by the University of Southern California’s Institute for Creative Technologies (USC‐ICT) in partnership with the US Army’s School for Command Preparation (SCP). UrbanSim has been in‐use since 2009 to help Army commanders understand and train for missions in complex, uncertain environments. The discussion describes how the social simulation‐based training application was designed to develop and hone commanders' skills for conducting missions in environs with multifaceted social, ethnic and political fabrics. We present a few considerations when attempting to recreate dense, rapidly growing population centers, and how the integration of real‐world data into social simulation frameworks can add a level of realism and understanding not possible even a few years ago.
de Melo, Celso M.; Paiva, Ana; Gratch, Jonathan
Emotion in Games Book Section
In: Agius, Harry; Angelides, Marios (Ed.): Handbook of Digital Games, Wiley-IEEE Press, New Jersey, 2014, ISBN: 978-1-118-32803-3.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@incollection{melo_emotion_2014,
title = {Emotion in Games},
author = {Celso M. de Melo and Ana Paiva and Jonathan Gratch},
editor = {Harry Agius and Marios Angelides},
url = {http://www.amazon.com/Handbook-Digital-Games-Marios-Angelides/dp/1118328035},
isbn = {978-1-118-32803-3},
year = {2014},
date = {2014-03-01},
booktitle = {Handbook of Digital Games},
publisher = {Wiley-IEEE Press},
address = {New Jersey},
abstract = {Growing interest on the study of emotion in the behavioral sciences has led to the development of several psychological theories of human emotion. These theories, in turn, inspired computer scientists to propose computational models that synthesize, express, recognize and interpret emotion. This cross-disciplinary research on emotion introduces new possibilities for digital games. Complementing techniques from the arts for drama and storytelling, these models can be used to drive believable non-player characters that experience properly-motivated emotions and express them appropriately at the right time; these theories can also help interpret the emotions the human player is experiencing and suggest adequate reactions in the game. This chapter reviews relevant psychological theories of emotion as well as computational models of emotion and discusses implications for games. We give special emphasis to appraisal theories of emotion, undeniably one of the most influential theoretical perspectives within computational research. In appraisal theories, emotions arise from cognitive appraisal of events (e.g., is this event conducive to my goals? Who is responsible for this event? Can I cope with this event?). According to the pattern of appraisals that occur, different emotions are experienced and expressed. Appraisal theories can, therefore, be used to synthesize emotions in games, which are then expressed in different ways. Complementary, reverse appraisal has been recently proposed as a theory for the interpretation of emotion. Accordingly, people are argued to retrieve, from emotion displays, information about how others’ are appraising the ongoing interaction, which then leads to inferences about the others’ intentions. Reverse appraisal can, thus, be used to infer how human players, from their emotion displays, are appraising the game experience and, from this information, what their intentions in the game are. This information can then be used to adjust game parameters or have non-player characters react to the player’s intentions and, thus, contribute to improve the player’s overall experience.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {incollection}
}
Growing interest on the study of emotion in the behavioral sciences has led to the development of several psychological theories of human emotion. These theories, in turn, inspired computer scientists to propose computational models that synthesize, express, recognize and interpret emotion. This cross-disciplinary research on emotion introduces new possibilities for digital games. Complementing techniques from the arts for drama and storytelling, these models can be used to drive believable non-player characters that experience properly-motivated emotions and express them appropriately at the right time; these theories can also help interpret the emotions the human player is experiencing and suggest adequate reactions in the game. This chapter reviews relevant psychological theories of emotion as well as computational models of emotion and discusses implications for games. We give special emphasis to appraisal theories of emotion, undeniably one of the most influential theoretical perspectives within computational research. In appraisal theories, emotions arise from cognitive appraisal of events (e.g., is this event conducive to my goals? Who is responsible for this event? Can I cope with this event?). According to the pattern of appraisals that occur, different emotions are experienced and expressed. Appraisal theories can, therefore, be used to synthesize emotions in games, which are then expressed in different ways. Complementary, reverse appraisal has been recently proposed as a theory for the interpretation of emotion. Accordingly, people are argued to retrieve, from emotion displays, information about how others’ are appraising the ongoing interaction, which then leads to inferences about the others’ intentions. Reverse appraisal can, thus, be used to infer how human players, from their emotion displays, are appraising the game experience and, from this information, what their intentions in the game are. This information can then be used to adjust game parameters or have non-player characters react to the player’s intentions and, thus, contribute to improve the player’s overall experience.
Azmandian, Mahdi; Yahata, Rhys; Bolas, Mark; Suma, Evan
An Enhanced Steering Algorithm for Redirected Walking in Virtual Environments Proceedings Article
In: IEEE Virtual Reality 2014, pp. 65–66, 2014.
Abstract | Links | BibTeX | Tags: MxR, UARC
@inproceedings{azmandian_enhanced_2014,
title = {An Enhanced Steering Algorithm for Redirected Walking in Virtual Environments},
author = {Mahdi Azmandian and Rhys Yahata and Mark Bolas and Evan Suma},
url = {http://ict.usc.edu/pubs/An%20Enhanced%20Steering%20Algorithm%20for%20Redirected%20Walking%20in%20Virtual%20Environments.pdf},
year = {2014},
date = {2014-03-01},
booktitle = {IEEE Virtual Reality 2014},
pages = {65–66},
abstract = {Redirected walking techniques enable natural locomotion through immersive virtual environments that are considerably larger than the available real world walking space. However, the most effective strategy for steering the user remains an open question, as most previously presented algorithms simply redirect toward the center of the physical space. In this work, we present a theoretical framework that plans a walking path through a virtual environment and calculates the parameters for combining translation, rotation, and curvature gains such that the user can traverse a series of defined waypoints efficiently based on a utility function. This function minimizes the number of overt reorientations to avoid introducing potential breaks in presence. A notable advantage of this approach is that it leverages knowledge of the layout of both the physical and virtual environments to enhance the steering strategy.},
keywords = {MxR, UARC},
pubstate = {published},
tppubtype = {inproceedings}
}
Redirected walking techniques enable natural locomotion through immersive virtual environments that are considerably larger than the available real world walking space. However, the most effective strategy for steering the user remains an open question, as most previously presented algorithms simply redirect toward the center of the physical space. In this work, we present a theoretical framework that plans a walking path through a virtual environment and calculates the parameters for combining translation, rotation, and curvature gains such that the user can traverse a series of defined waypoints efficiently based on a utility function. This function minimizes the number of overt reorientations to avoid introducing potential breaks in presence. A notable advantage of this approach is that it leverages knowledge of the layout of both the physical and virtual environments to enhance the steering strategy.
Sukthankar, Gita; Goldman, Robert P.; Geib, Christopher; Pynadath, David V.; Bui, Hung
Plan, Activity, and Intent Recognition: Theory and Practice Book
Morgan Kaufmann, 2014, ISBN: 0-12-398532-3.
Links | BibTeX | Tags: Social Simulation, UARC
@book{sukthankar_plan_2014,
title = {Plan, Activity, and Intent Recognition: Theory and Practice},
author = {Gita Sukthankar and Robert P. Goldman and Christopher Geib and David V. Pynadath and Hung Bui},
url = {http://www.amazon.com/Plan-Activity-Intent-Recognition-Practice/dp/0123985323/ref=sr_1_1?s=books&ie=UTF8&qid=1408747877&sr=1-1&keywords=Plan%2C+Activity%2C+and+Intent+Recognition%3A+Theory+and+Practice},
isbn = {0-12-398532-3},
year = {2014},
date = {2014-03-01},
publisher = {Morgan Kaufmann},
keywords = {Social Simulation, UARC},
pubstate = {published},
tppubtype = {book}
}
Visser, Thomas; Traum, David; DeVault, David; Akker, Rieks
A model for incremental grounding in spoken dialogue systems Journal Article
In: Journal on Multimodal User Interfaces, vol. 8, no. 1, pp. 61–73, 2014, ISSN: 1783-7677, 1783-8738.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@article{visser_model_2014,
title = {A model for incremental grounding in spoken dialogue systems},
author = {Thomas Visser and David Traum and David DeVault and Rieks Akker},
url = {http://ict.usc.edu/pubs/A%20Model%20for%20Incremental%20Grounding%20in%20Spoken%20Dialogue%20Systems.pdf},
doi = {10.1007/s12193-013-0147-7},
issn = {1783-7677, 1783-8738},
year = {2014},
date = {2014-03-01},
journal = {Journal on Multimodal User Interfaces},
volume = {8},
number = {1},
pages = {61–73},
abstract = {We present a computational model of incremental grounding, including state updates and action selection. The model is inspired by corpus-based examples of overlapping utterances of several sorts, including backchannels and completions. The model has also been partially implemented within a virtual human system that includes incremental understanding, and can be used to track grounding and provide overlapping verbal and non-verbal behaviors from a listener, before a speaker has completed her utterance.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {article}
}
We present a computational model of incremental grounding, including state updates and action selection. The model is inspired by corpus-based examples of overlapping utterances of several sorts, including backchannels and completions. The model has also been partially implemented within a virtual human system that includes incremental understanding, and can be used to track grounding and provide overlapping verbal and non-verbal behaviors from a listener, before a speaker has completed her utterance.
Koh, Sukjin; Gordon, Andrew S; Wienberg, Christopher; Sood, Sara O; Morley, Stephanie; Burke, Deborah M
Stroke Experiences in Weblogs: A Feasibility Study of Sex Differences Journal Article
In: Journal of Medical Internet Research, vol. 16, no. 3, pp. e84, 2014, ISSN: 14388871.
Abstract | Links | BibTeX | Tags: The Narrative Group, UARC
@article{koh_stroke_2014,
title = {Stroke Experiences in Weblogs: A Feasibility Study of Sex Differences},
author = {Sukjin Koh and Andrew S Gordon and Christopher Wienberg and Sara O Sood and Stephanie Morley and Deborah M Burke},
url = {http://ict.usc.edu/pubs/Stroke%20Experiences%20in%20Weblogs%20-%20A%20Feasibility%20Study%20of%20Sex%20Differences.pdf},
doi = {10.2196/jmir.2838},
issn = {14388871},
year = {2014},
date = {2014-03-01},
journal = {Journal of Medical Internet Research},
volume = {16},
number = {3},
pages = {e84},
abstract = {Research on cerebral stroke symptoms using hospital records has reported that women experience more nontraditional symptoms of stroke (eg, mental status change, pain) than men do. This is an important issue because nontraditional symptoms may delay the decision to get medical assistance and increase the difficulty of correct diagnosis. In the present study, we investigate sex differences in the stroke experience as described in stories on weblogs.},
keywords = {The Narrative Group, UARC},
pubstate = {published},
tppubtype = {article}
}
Research on cerebral stroke symptoms using hospital records has reported that women experience more nontraditional symptoms of stroke (eg, mental status change, pain) than men do. This is an important issue because nontraditional symptoms may delay the decision to get medical assistance and increase the difficulty of correct diagnosis. In the present study, we investigate sex differences in the stroke experience as described in stories on weblogs.
Khademi, Mahmoud; Morency, Louis-Philippe
Relative Facial Action Unit Detection Proceedings Article
In: Proceedings of the Winter conference on Applications in Computer Vision, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{khademi_relative_2014,
title = {Relative Facial Action Unit Detection},
author = {Mahmoud Khademi and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Relative%20Facial%20Action%20Unit%20Detection.pdf},
year = {2014},
date = {2014-03-01},
booktitle = {Proceedings of the Winter conference on Applications in Computer Vision},
abstract = {This paper presents a subject-independent facial action unit (AU) detection method by introducing the concept of relative AU detection, for scenarios where the neutral face is not provided. We propose a new classification objective function which analyzes the temporal neighborhood of the current frame to decide if the expression recently increased, decreased or showed no change. This approach is a significant change from the conventional absolute method which decides about AU classification using the current frame, without an explicit comparison with its neighboring frames. Our proposed method improves robustness to individual differences such as face scale and shape, age-related wrinkles, and transitions among expressions (e.g., lower intensity of expressions). Our experiments on three publicly available datasets (Extended Cohn-Kanade (CK+), Bosphorus, and DISFA databases) show significant improvement of our approach over conventional absolute techniques.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
This paper presents a subject-independent facial action unit (AU) detection method by introducing the concept of relative AU detection, for scenarios where the neutral face is not provided. We propose a new classification objective function which analyzes the temporal neighborhood of the current frame to decide if the expression recently increased, decreased or showed no change. This approach is a significant change from the conventional absolute method which decides about AU classification using the current frame, without an explicit comparison with its neighboring frames. Our proposed method improves robustness to individual differences such as face scale and shape, age-related wrinkles, and transitions among expressions (e.g., lower intensity of expressions). Our experiments on three publicly available datasets (Extended Cohn-Kanade (CK+), Bosphorus, and DISFA databases) show significant improvement of our approach over conventional absolute techniques.
Jones, Andrew; Nagano, Koki; Liu, Jing; Busch, Jay; Yu, Xueming; Bolas, Mark; Debevec, Paul
Interpolating vertical parallax for an autostereoscopic three-dimensional projector array Journal Article
In: Journal of Electronic Imaging, vol. 23, no. 1, 2014, ISSN: 1017-9909.
Abstract | Links | BibTeX | Tags: Graphics, MxR, UARC
@article{jones_interpolating_2014,
title = {Interpolating vertical parallax for an autostereoscopic three-dimensional projector array},
author = {Andrew Jones and Koki Nagano and Jing Liu and Jay Busch and Xueming Yu and Mark Bolas and Paul Debevec},
url = {http://electronicimaging.spiedigitallibrary.org/article.aspx?doi=10.1117/1.JEI.23.1.011005},
doi = {10.1117/1.JEI.23.1.011005},
issn = {1017-9909},
year = {2014},
date = {2014-03-01},
journal = {Journal of Electronic Imaging},
volume = {23},
number = {1},
abstract = {We present a technique for achieving tracked vertical parallax for multiple users using a variety of autostereoscopic projector array setups, including front- and rear-projection and curved display surfaces. This hybrid parallax approach allows for immediate horizontal parallax as viewers move left and right and tracked parallax as they move up and down, allowing cues such as three-dimensional (3-D) perspective and eye contact to be conveyed faithfully. We use a low-cost RGB-depth sensor to simultaneously track multiple viewer head positions in 3-D space, and we interactively update the imagery sent to the array so that imagery directed to each viewer appears from a consistent and correct vertical perspective. Unlike previous work, we do not assume that the imagery sent to each projector in the array is rendered from a single vertical perspective. This lets us apply hybrid parallax to displays where a single projector forms parts of multiple viewers’ imagery. Thus, each individual projected image is rendered with multiple centers of projection, and might show an object from above on the left and from below on the right. We demonstrate this technique using a dense horizontal array of pico-projectors aimed into an anisotropic vertical diffusion screen, yielding 1.5 deg angular resolution over 110 deg field of view. To create a seamless viewing experience for multiple viewers, we smoothly interpolate the set of viewer heights and distances on a per-vertex basis across the array’s field of view, reducing image distortion, cross talk, and artifacts from tracking errors.},
keywords = {Graphics, MxR, UARC},
pubstate = {published},
tppubtype = {article}
}
We present a technique for achieving tracked vertical parallax for multiple users using a variety of autostereoscopic projector array setups, including front- and rear-projection and curved display surfaces. This hybrid parallax approach allows for immediate horizontal parallax as viewers move left and right and tracked parallax as they move up and down, allowing cues such as three-dimensional (3-D) perspective and eye contact to be conveyed faithfully. We use a low-cost RGB-depth sensor to simultaneously track multiple viewer head positions in 3-D space, and we interactively update the imagery sent to the array so that imagery directed to each viewer appears from a consistent and correct vertical perspective. Unlike previous work, we do not assume that the imagery sent to each projector in the array is rendered from a single vertical perspective. This lets us apply hybrid parallax to displays where a single projector forms parts of multiple viewers’ imagery. Thus, each individual projected image is rendered with multiple centers of projection, and might show an object from above on the left and from below on the right. We demonstrate this technique using a dense horizontal array of pico-projectors aimed into an anisotropic vertical diffusion screen, yielding 1.5 deg angular resolution over 110 deg field of view. To create a seamless viewing experience for multiple viewers, we smoothly interpolate the set of viewer heights and distances on a per-vertex basis across the array’s field of view, reducing image distortion, cross talk, and artifacts from tracking errors.
Hill, Randall W.
Virtual Reality and Leadership Development Book Section
In: Using Experience to Develop Leadership Talent: How Organizations Leverage On-The-Job Development, pp. 286–312, John Wiley & Sons, Inc., 2014, ISBN: 978-1-118-76783-2.
Links | BibTeX | Tags: Learning Sciences, Social Simulation, UARC, Virtual Humans, Virtual Worlds
@incollection{hill_virtual_2014,
title = {Virtual Reality and Leadership Development},
author = {Randall W. Hill},
url = {http://www.amazon.com/dp/1118767837/ref=cm_sw_su_dp},
isbn = {978-1-118-76783-2},
year = {2014},
date = {2014-03-01},
booktitle = {Using Experience to Develop Leadership Talent: How Organizations Leverage On-The-Job Development},
pages = {286–312},
publisher = {John Wiley & Sons, Inc.},
series = {J-B SIOP Professional Practice Series (Book 1)},
keywords = {Learning Sciences, Social Simulation, UARC, Virtual Humans, Virtual Worlds},
pubstate = {published},
tppubtype = {incollection}
}
Shapiro, Ari; Feng, Andrew; Wang, Ruizhe; Medioni, Gerard; Bolas, Mark; Suma, Evan A.
Automatic Acquisition and Animation of Virtual Avatars Proceedings Article
In: Virtual Reality (VR), 2014 iEEE, pp. 185–186, IEEE, Minneapolis, Minnesota, 2014, ISBN: 978-1-4799-2871-2.
Abstract | Links | BibTeX | Tags: MxR, UARC, Virtual Humans
@inproceedings{shapiro_automatic_2014,
title = {Automatic Acquisition and Animation of Virtual Avatars},
author = {Ari Shapiro and Andrew Feng and Ruizhe Wang and Gerard Medioni and Mark Bolas and Evan A. Suma},
url = {http://ict.usc.edu/pubs/Automatic%20acquisition%20and%20animation%20of%20virtual%20avatars.pdf},
doi = {10.1109/VR.2014.6802113},
isbn = {978-1-4799-2871-2},
year = {2014},
date = {2014-03-01},
booktitle = {Virtual Reality (VR), 2014 iEEE},
pages = {185–186},
publisher = {IEEE},
address = {Minneapolis, Minnesota},
abstract = {The USC Institute for Creative Technologies will demonstrate a pipline for automatic reconstruction and animation of lifelike 3D avatars acquired by rotating the user's body in front of a single Microsoft Kinect sensor. Based on a fusion of state-of-the-art techniques in computer vision, graphics, and animation, this approach can produce a fully rigged character model suitable for real-time virtual environments in less than four minutes.},
keywords = {MxR, UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
The USC Institute for Creative Technologies will demonstrate a pipline for automatic reconstruction and animation of lifelike 3D avatars acquired by rotating the user's body in front of a single Microsoft Kinect sensor. Based on a fusion of state-of-the-art techniques in computer vision, graphics, and animation, this approach can produce a fully rigged character model suitable for real-time virtual environments in less than four minutes.
Park, Sunghyun; Shoemark, Philippa; Morency, Louis-Philippe
Toward Crowdsourcing Micro-Level Behavior Annotations - The Challenges of Interface, Training, and Generalization Proceedings Article
In: Proceedings of the 19th international conference on Intelligent User Interfaces, ACM, Haifa, Israel, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{park_toward_2014,
title = {Toward Crowdsourcing Micro-Level Behavior Annotations - The Challenges of Interface, Training, and Generalization},
author = {Sunghyun Park and Philippa Shoemark and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Toward%20Crowdsourcing%20Micro-Level%20Behavior%20Annotations%20-%20The%20Challenges%20of%20Interface,%20Training,%20and%20Generalization.pdf},
year = {2014},
date = {2014-02-01},
booktitle = {Proceedings of the 19th international conference on Intelligent User Interfaces},
publisher = {ACM},
address = {Haifa, Israel},
abstract = {Research that involves human behavior analysis usually requires laborious and costly efforts for obtaining micro-level behavior annotations on a large video corpus. With the emerging paradigm of crowdsourcing however, these efforts can be considerably reduced. We first present OCTAB (Online Crowdsourcing Tool for Annotations of Behaviors), a web-based annotation tool that allows precise and convenient behavior annotations in videos, directly portable to popular crowdsourcing platforms. As part of OCTAB, we introduce a training module with specialized visualizations. The training module’s design was inspired by an observational study of local experienced coders, and it enables an iterative procedure for effectively training crowd workers online. Finally, we present an extensive set of experiments that evaluates the feasibility of our crowdsourcing approach for obtaining micro-level behavior annotations in videos, showing the reliability improvement in annotation accuracy when properly training online crowd workers. We also show the generalization of our training approach to a new independent video corpus.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Research that involves human behavior analysis usually requires laborious and costly efforts for obtaining micro-level behavior annotations on a large video corpus. With the emerging paradigm of crowdsourcing however, these efforts can be considerably reduced. We first present OCTAB (Online Crowdsourcing Tool for Annotations of Behaviors), a web-based annotation tool that allows precise and convenient behavior annotations in videos, directly portable to popular crowdsourcing platforms. As part of OCTAB, we introduce a training module with specialized visualizations. The training module’s design was inspired by an observational study of local experienced coders, and it enables an iterative procedure for effectively training crowd workers online. Finally, we present an extensive set of experiments that evaluates the feasibility of our crowdsourcing approach for obtaining micro-level behavior annotations in videos, showing the reliability improvement in annotation accuracy when properly training online crowd workers. We also show the generalization of our training approach to a new independent video corpus.
Leuski, Anton; Gowrisankar, Rasiga; Richmond, Todd; Shapiro, Ari; Xu, Yuyu; Feng, Andrew
Mobile Personal Healthcare Mediated by Virtual Humans Proceedings Article
In: Proceedings of the companion publication of the 19th international conference on Intelligent User Interfaces, pp. 21–24, ACM Press, Haifa, Israel, 2014, ISBN: 978-1-4503-2729-9.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{leuski_mobile_2014,
title = {Mobile Personal Healthcare Mediated by Virtual Humans},
author = {Anton Leuski and Rasiga Gowrisankar and Todd Richmond and Ari Shapiro and Yuyu Xu and Andrew Feng},
url = {http://dl.acm.org/citation.cfm?doid=2559184.2559200},
doi = {10.1145/2559184.2559200},
isbn = {978-1-4503-2729-9},
year = {2014},
date = {2014-02-01},
booktitle = {Proceedings of the companion publication of the 19th international conference on Intelligent User Interfaces},
pages = {21–24},
publisher = {ACM Press},
address = {Haifa, Israel},
abstract = {We demonstrate Ally—-a prototype interface for a consumer–level medical diagnostic device. It is an interactive virtual character—-Virtual Human (VH)—-that listens to user's concern, collects and processes sensor data, offers advice, guides the user through a self-administered medical tests, and answers the user's questions. The primary focus of this demo is on the VH, we describe and demonstrate the technologies for language analysis, dialogue management, response generation and presentation. The sensing and medical decision making components are simulated in the current system, but possible applications and extensions are discussed.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
We demonstrate Ally—-a prototype interface for a consumer–level medical diagnostic device. It is an interactive virtual character—-Virtual Human (VH)—-that listens to user's concern, collects and processes sensor data, offers advice, guides the user through a self-administered medical tests, and answers the user's questions. The primary focus of this demo is on the VH, we describe and demonstrate the technologies for language analysis, dialogue management, response generation and presentation. The sensing and medical decision making components are simulated in the current system, but possible applications and extensions are discussed.
Gandhe, Sudeep; Traum, David
A semi-automated evaluation metric for dialogue model coherence Proceedings Article
In: Fifth International Workshop on Spoken Dialogue systems, pp. 141–150, 2014.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{gandhe_semi-automated_2014,
title = {A semi-automated evaluation metric for dialogue model coherence},
author = {Sudeep Gandhe and David Traum},
url = {http://ict.usc.edu/pubs/A%20semi-automated%20evaluation%20metric%20for%20dialogue%20model%20coherence.pdf},
year = {2014},
date = {2014-01-01},
booktitle = {Fifth International Workshop on Spoken Dialogue systems},
pages = {141–150},
abstract = {We propose a new metric, Voted Appropriateness, which can be used to automatically evaluate dialogue policy decisions, once some wizard data has been collected. We show that this metric outperforms a previously proposed metric Weak agreement.We also present a taxonomy for dialogue model evaluation schemas, and orient our new metric within this taxonomy.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
We propose a new metric, Voted Appropriateness, which can be used to automatically evaluate dialogue policy decisions, once some wizard data has been collected. We show that this metric outperforms a previously proposed metric Weak agreement.We also present a taxonomy for dialogue model evaluation schemas, and orient our new metric within this taxonomy.
Feng, Andrew; Huang, Yazhou; Xu, Yuyu; Shapiro, Ari
Fast, automatic character animation pipelines Journal Article
In: Computer Animation and Virtual Worlds, vol. 25, no. 1, pp. 3–16, 2014, ISSN: 15464261.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@article{feng_fast_2014,
title = {Fast, automatic character animation pipelines},
author = {Andrew Feng and Yazhou Huang and Yuyu Xu and Ari Shapiro},
url = {http://ict.usc.edu/pubs/Fast,%20automatic%20character%20animation%20pipelines.pdf},
doi = {10.1002/cav.1560},
issn = {15464261},
year = {2014},
date = {2014-01-01},
journal = {Computer Animation and Virtual Worlds},
volume = {25},
number = {1},
pages = {3–16},
abstract = {Humanoid three-dimensional (3D) models can be easily acquired through various sources, including through online marketplaces. The use of such models within a game or simulation environment requires human input and intervention in order to associate such a model with a relevant set of motions and control mechanisms. In this paper, we demonstrate a pipeline where humanoid 3D models can be incorporated within seconds into an animation system and infused with a wide range of capabilities, such as locomotion, object manipulation, gazing, speech synthesis and lip syncing. We offer a set of heuristics that can associate arbitrary joint names with canonical ones and describe a fast retargeting algorithm that enables us to instill a set of behaviors onto an arbitrary humanoid skeleton on-the-fly. We believe that such a system will vastly increase the use of 3D interactive characters due to the ease that new models can be animated. Copyright © 2013 John Wiley & Sons, Ltd.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {article}
}
Humanoid three-dimensional (3D) models can be easily acquired through various sources, including through online marketplaces. The use of such models within a game or simulation environment requires human input and intervention in order to associate such a model with a relevant set of motions and control mechanisms. In this paper, we demonstrate a pipeline where humanoid 3D models can be incorporated within seconds into an animation system and infused with a wide range of capabilities, such as locomotion, object manipulation, gazing, speech synthesis and lip syncing. We offer a set of heuristics that can associate arbitrary joint names with canonical ones and describe a fast retargeting algorithm that enables us to instill a set of behaviors onto an arbitrary humanoid skeleton on-the-fly. We believe that such a system will vastly increase the use of 3D interactive characters due to the ease that new models can be animated. Copyright © 2013 John Wiley & Sons, Ltd.
Gratch, Jonathan; Kang, Sin-Hwa; Wang, Ning
Using Social Agents to Explore Theories of Rapport and Emotional Resonance Book Section
In: Social Emotions in Nature and Artifact, pp. 181 –195, 2014.
Abstract | Links | BibTeX | Tags: MxR, UARC, Virtual Humans
@incollection{gratch_using_2014,
title = {Using Social Agents to Explore Theories of Rapport and Emotional Resonance},
author = {Jonathan Gratch and Sin-Hwa Kang and Ning Wang},
url = {http://www.oxfordscholarship.com/view/10.1093/acprof:oso/9780195387643.001.0001/acprof-9780195387643-chapter-12},
year = {2014},
date = {2014-01-01},
booktitle = {Social Emotions in Nature and Artifact},
pages = {181 –195},
abstract = {We discuss several technical challenges must be overcome before realizing this vision. More importantly, success depends not on simply overcoming these challenges, but demonstrating that such interactivity has measurable and desirable consequences for human-computer interaction. In this chapter, we describe the Rapport Agent, an interactive agent and methodological tool designed to Emotions unfold in with bewildering complexity in face-to-face social interactions. Building computer programs that can engage people in this unfolding emotional dance is a fascinating prospect with potentially profound practical and scientific consequences. Computer agents that engage people in this manner could enhance our understanding of this fundamental social process and, more practically, have dramatic implications investigate the role of nonverbal patterning in human-computer and computer-mediated interaction. We outline a series of laboratory studies and resulting findings that give insight into how nonverbal patterns of behavior can influence both subjective perceptions (such as feelings of rapport or embarrassment) and ehavioural outcomes (such as speech fluency or intimate self-disclosure).},
keywords = {MxR, UARC, Virtual Humans},
pubstate = {published},
tppubtype = {incollection}
}
We discuss several technical challenges must be overcome before realizing this vision. More importantly, success depends not on simply overcoming these challenges, but demonstrating that such interactivity has measurable and desirable consequences for human-computer interaction. In this chapter, we describe the Rapport Agent, an interactive agent and methodological tool designed to Emotions unfold in with bewildering complexity in face-to-face social interactions. Building computer programs that can engage people in this unfolding emotional dance is a fascinating prospect with potentially profound practical and scientific consequences. Computer agents that engage people in this manner could enhance our understanding of this fundamental social process and, more practically, have dramatic implications investigate the role of nonverbal patterning in human-computer and computer-mediated interaction. We outline a series of laboratory studies and resulting findings that give insight into how nonverbal patterns of behavior can influence both subjective perceptions (such as feelings of rapport or embarrassment) and ehavioural outcomes (such as speech fluency or intimate self-disclosure).
2013
Faust, Lauren; Artstein, Ron
People hesitate more, talk less to virtual interviewers than to human interviewers Proceedings Article
In: Proceedings of the 17th Workshop on the Semantics and Pragmatics of Dialogue, Amsterdam, 2013.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{faust_people_2013,
title = {People hesitate more, talk less to virtual interviewers than to human interviewers},
author = {Lauren Faust and Ron Artstein},
url = {http://ict.usc.edu/pubs/People%20hesitate%20more,%20talk%20less%20to%20virtual%20interviewers%20than%20to%20human%20interviewers.pdf},
year = {2013},
date = {2013-12-01},
booktitle = {Proceedings of the 17th Workshop on the Semantics and Pragmatics of Dialogue},
address = {Amsterdam},
abstract = {In a series of screening interviews for psychological distress, conducted separately by a human interviewer and by an animated virtual character controlled by a human, participants talked substantially less and produced twice as many filled pauses when talking to the virtual character. This contrasts with earlier findings, where people were less disfluent when talking to a computer dialogue system. The results suggest that the characteristics of computer-directed speech vary depending on the type of dialogue system used.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
In a series of screening interviews for psychological distress, conducted separately by a human interviewer and by an animated virtual character controlled by a human, participants talked substantially less and produced twice as many filled pauses when talking to the virtual character. This contrasts with earlier findings, where people were less disfluent when talking to a computer dialogue system. The results suggest that the characteristics of computer-directed speech vary depending on the type of dialogue system used.
Yu, Zhou; Scherer, Stefen; Devault, David; Gratch, Jonathan; Stratou, Giota; Morency, Louis-Philippe; Cassell, Justine
Multimodal Prediction of Psychological Disorders: Learning Verbal and Nonverbal Commonalities in Adjacency Pairs Proceedings Article
In: Semdial 2013 DialDam: Proceedings of the 17th Workshop on the Semantics and Pragmatics of Dialogue, pp. 160–169, Amsterdam, The Netherlands, 2013.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{yu_multimodal_2013,
title = {Multimodal Prediction of Psychological Disorders: Learning Verbal and Nonverbal Commonalities in Adjacency Pairs},
author = {Zhou Yu and Stefen Scherer and David Devault and Jonathan Gratch and Giota Stratou and Louis-Philippe Morency and Justine Cassell},
url = {http://www.cs.cmu.edu/afs/cs/user/zhouyu/www/semdial_2013_zhou.pdf},
year = {2013},
date = {2013-12-01},
booktitle = {Semdial 2013 DialDam: Proceedings of the 17th Workshop on the Semantics and Pragmatics of Dialogue},
pages = {160–169},
address = {Amsterdam, The Netherlands},
abstract = {Semi-structured interviews are widely used in medical settings to gather information from individuals about psychological disorders, such as depression or anxiety. These interviews typically consist of a series of question and response pairs, which we refer to as adjacency pairs. We pro-pose a computational model, the Multi-modal HCRF, that considers the commonalities among adjacency pairs and information from multiple modalities to infer the psychological states of the interviewees. We collect data and perform experiments on a human to virtual human interaction data set. Our multimodal approach gives a significant advantage over conventional holistic approaches which ignore the adjacency pair context in predicting depression from semi-structured inter- views.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Semi-structured interviews are widely used in medical settings to gather information from individuals about psychological disorders, such as depression or anxiety. These interviews typically consist of a series of question and response pairs, which we refer to as adjacency pairs. We pro-pose a computational model, the Multi-modal HCRF, that considers the commonalities among adjacency pairs and information from multiple modalities to infer the psychological states of the interviewees. We collect data and perform experiments on a human to virtual human interaction data set. Our multimodal approach gives a significant advantage over conventional holistic approaches which ignore the adjacency pair context in predicting depression from semi-structured inter- views.
Baltrušaitis, Tadas; Robinson, Peter; Morency, Louis-Philippe
Constrained local neural fields for robust facial landmark detection in the wild Proceedings Article
In: Computer Vision Workshops (ICCVW), 2013 IEEE International Conference on, pp. 354–361, IEEE, Sydney, Australia, 2013.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{baltrusaitis_constrained_2013,
title = {Constrained local neural fields for robust facial landmark detection in the wild},
author = {Tadas Baltrušaitis and Peter Robinson and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Constrained%20local%20neural%20fields%20for%20robust%20facial%20landmark%20detection%20in%20the%20wild.pdf},
year = {2013},
date = {2013-12-01},
booktitle = {Computer Vision Workshops (ICCVW), 2013 IEEE International Conference on},
pages = {354–361},
publisher = {IEEE},
address = {Sydney, Australia},
abstract = {Facial feature detection algorithms have seen great progress over the recent years. However, they still struggle in poor lighting conditions and in the presence of extreme pose or occlusions. We present the Constrained Local Neural Field model for facial landmark detection. Our model includes two main novelties. First, we introduce a probabilistic patch expert (landmark detector) that can learn non-linear and spatial relationships between the input pixels and the probability of a landmark being aligned. Secondly, our model is optimised using a novel Non-uniform Regularised Landmark Mean-Shift optimisation technique, which takes into account the reliabilities of each patch expert. We demonstrate the benefit of our approach on a number of publicly available datasets over other state-of-the-art approaches when performing landmark detection in unseen lighting conditions and in the wild.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Facial feature detection algorithms have seen great progress over the recent years. However, they still struggle in poor lighting conditions and in the presence of extreme pose or occlusions. We present the Constrained Local Neural Field model for facial landmark detection. Our model includes two main novelties. First, we introduce a probabilistic patch expert (landmark detector) that can learn non-linear and spatial relationships between the input pixels and the probability of a landmark being aligned. Secondly, our model is optimised using a novel Non-uniform Regularised Landmark Mean-Shift optimisation technique, which takes into account the reliabilities of each patch expert. We demonstrate the benefit of our approach on a number of publicly available datasets over other state-of-the-art approaches when performing landmark detection in unseen lighting conditions and in the wild.
Kok, Iwan; Heylen, Dirk; Morency, Louis-Philippe
Speaker-Adaptive Multimodal Prediction Model for Listener Responses Proceedings Article
In: pp. 51–58, ACM Press, Sydney, Australia, 2013, ISBN: 978-1-4503-2129-7.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{de_kok_speaker-adaptive_2013,
title = {Speaker-Adaptive Multimodal Prediction Model for Listener Responses},
author = {Iwan Kok and Dirk Heylen and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Speaker-adaptive%20multimodal%20prediction%20model%20for%20listener%20responses.pdf},
doi = {10.1145/2522848.2522866},
isbn = {978-1-4503-2129-7},
year = {2013},
date = {2013-12-01},
pages = {51–58},
publisher = {ACM Press},
address = {Sydney, Australia},
abstract = {The goal of this paper is to acknowledge and model the variability in speaking styles in dyadic interactions and build a predictive algorithm for listener responses that is able to adapt to these different styles. The end result of this research will be a virtual human able to automatically respond to a human speaker with proper listener responses (e.g., head nods). Our novel speaker-adaptive prediction model is created from a corpus of dyadic interactions where speaker variability is analyzed to identify a subset of prototypical speaker styles. During a live interaction our prediction model automatically identifies the closest prototypical speaker style and predicts listener responses based on this communicative style. Central to our approach is the idea of "speaker profile" which uniquely identify each speaker and enables the matching between prototypical speakers and new speakers. The paper shows the merits of our speaker-adaptive listener response prediction model by showing improvement over a state-of-the-art approach which does not adapt to the speaker. Besides the merits of speaker-adaptation, our experiments highlights the importance of using multimodal features when comparing speakers to select the closest prototypical speaker style.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
The goal of this paper is to acknowledge and model the variability in speaking styles in dyadic interactions and build a predictive algorithm for listener responses that is able to adapt to these different styles. The end result of this research will be a virtual human able to automatically respond to a human speaker with proper listener responses (e.g., head nods). Our novel speaker-adaptive prediction model is created from a corpus of dyadic interactions where speaker variability is analyzed to identify a subset of prototypical speaker styles. During a live interaction our prediction model automatically identifies the closest prototypical speaker style and predicts listener responses based on this communicative style. Central to our approach is the idea of "speaker profile" which uniquely identify each speaker and enables the matching between prototypical speakers and new speakers. The paper shows the merits of our speaker-adaptive listener response prediction model by showing improvement over a state-of-the-art approach which does not adapt to the speaker. Besides the merits of speaker-adaptation, our experiments highlights the importance of using multimodal features when comparing speakers to select the closest prototypical speaker style.
Mahmoud, Marwa; Morency, Louis-Philippe; Robinson, Peter
Automatic Multimodal Descriptors of Rhythmic Body Movement Proceedings Article
In: Proceedings of the 15th ACM on International conference on multimodal interaction, pp. 429–436, ACM, 2013.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{mahmoud_automatic_2013,
title = {Automatic Multimodal Descriptors of Rhythmic Body Movement},
author = {Marwa Mahmoud and Louis-Philippe Morency and Peter Robinson},
url = {http://ict.usc.edu/pubs/Automatic%20multimodal%20descriptors%20of%20rhythmic%20body%20movement.pdf},
year = {2013},
date = {2013-12-01},
booktitle = {Proceedings of the 15th ACM on International conference on multimodal interaction},
pages = {429–436},
publisher = {ACM},
abstract = {Prolonged durations of rhythmic body gestures were proved to be correlated with different types of psychological disorders. To-date, there is no automatic descriptor that can robustly detect those behaviours. In this paper, we propose a cyclic gestures descriptor that can detect and localise rhythmic body movements by taking advantage of both colour and depth modalities. We show experimentally how our rhythmic descriptor can successfully localise the rhythmic gestures as: hands fidgeting, legs fidgeting or rocking, significantly higher than the majority vote classification baseline. Our experiments also demonstrate the importance of fusing both modalities, with a significant increase in performance when compared to individual modalities.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Prolonged durations of rhythmic body gestures were proved to be correlated with different types of psychological disorders. To-date, there is no automatic descriptor that can robustly detect those behaviours. In this paper, we propose a cyclic gestures descriptor that can detect and localise rhythmic body movements by taking advantage of both colour and depth modalities. We show experimentally how our rhythmic descriptor can successfully localise the rhythmic gestures as: hands fidgeting, legs fidgeting or rocking, significantly higher than the majority vote classification baseline. Our experiments also demonstrate the importance of fusing both modalities, with a significant increase in performance when compared to individual modalities.
Mohammadi, Gelareh; Park, Sunghyun; Sagae, Kenji; Vinciarelli, Alessandro; Morency, Louis-Philippe
Who Is Persuasive? The Role of Perceived Personality and Communication Modality in Social Multimedia Proceedings Article
In: Proceedings of the 15th ACM on International conference on multimodal interaction, pp. 19–26, ACM Press, New York, NY, 2013, ISBN: 978-1-4503-2129-7.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{mohammadi_who_2013,
title = {Who Is Persuasive? The Role of Perceived Personality and Communication Modality in Social Multimedia},
author = {Gelareh Mohammadi and Sunghyun Park and Kenji Sagae and Alessandro Vinciarelli and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Who%20is%20persuasive%20-%20the%20role%20of%20perceived%20personality%20and%20communication%20modality%20in%20social%20multimedia.pdf},
doi = {10.1145/2522848.2522857},
isbn = {978-1-4503-2129-7},
year = {2013},
date = {2013-12-01},
booktitle = {Proceedings of the 15th ACM on International conference on multimodal interaction},
pages = {19–26},
publisher = {ACM Press},
address = {New York, NY},
abstract = {Persuasive communication is part of everyone's daily life. With the emergence of social websites like YouTube, Facebook and Twitter, persuasive communication is now seen online on a daily basis. This paper explores the effect of multi-modality and perceived personality on persuasiveness of social multimedia content. The experiments are performed over a large corpus of movie review clips from Youtube which is presented to online annotators in three different modalities: only text, only audio and video. The annotators evaluated the persuasiveness of each review across different modalities and judged the personality of the speaker. Our detailed analysis confirmed several research hypotheses designed to study the relationships between persuasion, perceived personality and communicative channel, namely modality. Three hypotheses are designed: the first hypothesis studies the effect of communication modality on persuasion, the second hypothesis examines the correlation between persuasion and personality perception and finally the third hypothesis, derived from the first two hypotheses explores how communication modality influence the personality perception.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Persuasive communication is part of everyone's daily life. With the emergence of social websites like YouTube, Facebook and Twitter, persuasive communication is now seen online on a daily basis. This paper explores the effect of multi-modality and perceived personality on persuasiveness of social multimedia content. The experiments are performed over a large corpus of movie review clips from Youtube which is presented to online annotators in three different modalities: only text, only audio and video. The annotators evaluated the persuasiveness of each review across different modalities and judged the personality of the speaker. Our detailed analysis confirmed several research hypotheses designed to study the relationships between persuasion, perceived personality and communicative channel, namely modality. Three hypotheses are designed: the first hypothesis studies the effect of communication modality on persuasion, the second hypothesis examines the correlation between persuasion and personality perception and finally the third hypothesis, derived from the first two hypotheses explores how communication modality influence the personality perception.
Miller, Chreston; Quek, Francis; Morency, Louis-Philippe
Interactive Relevance Search and Modeling: Support for Expert-Driven Analysis of Multimodal Data Proceedings Article
In: Proceedings of the 15th ACM on International conference on multimodal interaction, pp. 149–156, ACM, Sydney, Australia, 2013.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{miller_interactive_2013,
title = {Interactive Relevance Search and Modeling: Support for Expert-Driven Analysis of Multimodal Data},
author = {Chreston Miller and Francis Quek and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Interactive%20relevance%20search%20and%20modeling%20-%20Support%20for%20expert-driven%20analysis%20of%20multimodal%20data.pdf},
year = {2013},
date = {2013-12-01},
booktitle = {Proceedings of the 15th ACM on International conference on multimodal interaction},
pages = {149–156},
publisher = {ACM},
address = {Sydney, Australia},
abstract = {In this paper we present the findings of three longitudinal case studies in which a new method for conducting multimodal analysis of human behavior is tested. The focus of this new method is to engage a researcher integrally in the analysis process and allow them to guide the identification and discovery of relevant behavior instances within multimodal data. The case studies resulted in the creation of two analysis strategies: Single-Focus Hypothesis Testing and Multi-Focus Hypothesis Testing. Each were shown to be beneficial to multimodal analysis through supporting either a single focused deep analysis or analysis across multiple angles in unison. These strategies exemplified how challenging questions can be answered for multimodal datasets. The new method is described and the case studies’ findings are presented detailing how the new method supports multimodal analysis and opens the door for a new breed of analysis methods. Two of the three case studies resulted in publishable results for the respective participants.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
In this paper we present the findings of three longitudinal case studies in which a new method for conducting multimodal analysis of human behavior is tested. The focus of this new method is to engage a researcher integrally in the analysis process and allow them to guide the identification and discovery of relevant behavior instances within multimodal data. The case studies resulted in the creation of two analysis strategies: Single-Focus Hypothesis Testing and Multi-Focus Hypothesis Testing. Each were shown to be beneficial to multimodal analysis through supporting either a single focused deep analysis or analysis across multiple angles in unison. These strategies exemplified how challenging questions can be answered for multimodal datasets. The new method is described and the case studies’ findings are presented detailing how the new method supports multimodal analysis and opens the door for a new breed of analysis methods. Two of the three case studies resulted in publishable results for the respective participants.
Scherer, Stefan; Stratou, Giota; Morency, Louis-Philippe
Audiovisual Behavior Descriptors for Depression Assessment Proceedings Article
In: Proceedings of ICMI'13, pp. 135–140, ACM Press, Sydney, Australia, 2013, ISBN: 978-1-4503-2129-7.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{scherer_audiovisual_2013,
title = {Audiovisual Behavior Descriptors for Depression Assessment},
author = {Stefan Scherer and Giota Stratou and Louis-Philippe Morency},
url = {http://ict.usc.edu/pubs/Audiovisual%20behavior%20descriptors%20for%20depression%20assessment.pdf},
doi = {10.1145/2522848.2522886},
isbn = {978-1-4503-2129-7},
year = {2013},
date = {2013-12-01},
booktitle = {Proceedings of ICMI'13},
pages = {135–140},
publisher = {ACM Press},
address = {Sydney, Australia},
abstract = {We investigate audiovisual indicators, in particular measures of reduced emotional expressivity and psycho-motor retardation, for depression within semi-structured virtual human interviews. Based on a standard self-assessment depression scale we investigate the statistical discriminative strength of the audiovisual features on a depression/no-depression basis. Within subject-independent unimodal and multimodal classification experiments we find that early feature-level fusion yields promising results and confirms the statistical findings. We further correlate the behavior descriptors with the assessed depression severity and find considerable correlation. Lastly, a joint multimodal factor analysis reveals two prominent factors within the data that show both statistical discriminative power as well as strong linear correlation with the depression severity score. These preliminary results based on a standard factor analysis are promising and motivate us to investigate this approach further in the future, while incorporating additional modalities.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
We investigate audiovisual indicators, in particular measures of reduced emotional expressivity and psycho-motor retardation, for depression within semi-structured virtual human interviews. Based on a standard self-assessment depression scale we investigate the statistical discriminative strength of the audiovisual features on a depression/no-depression basis. Within subject-independent unimodal and multimodal classification experiments we find that early feature-level fusion yields promising results and confirms the statistical findings. We further correlate the behavior descriptors with the assessed depression severity and find considerable correlation. Lastly, a joint multimodal factor analysis reveals two prominent factors within the data that show both statistical discriminative power as well as strong linear correlation with the depression severity score. These preliminary results based on a standard factor analysis are promising and motivate us to investigate this approach further in the future, while incorporating additional modalities.
Song, Yale; Morency, Louis-Philippe; Davis, Randall
Learning a Sparse Codebook of Facial and Body Microexpressions for Emotion Recognition Proceedings Article
In: Proceedings of the 15th ACM on International conference on multimodal interaction, pp. 237–244, ACM Press, 2013, ISBN: 978-1-4503-2129-7.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{song_learning_2013,
title = {Learning a Sparse Codebook of Facial and Body Microexpressions for Emotion Recognition},
author = {Yale Song and Louis-Philippe Morency and Randall Davis},
url = {http://ict.usc.edu/pubs/Learning%20a%20sparse%20codebook%20of%20facial%20and%20body%20microexpressions%20for%20emotion%20recognition.pdf},
doi = {10.1145/2522848.2522851},
isbn = {978-1-4503-2129-7},
year = {2013},
date = {2013-12-01},
booktitle = {Proceedings of the 15th ACM on International conference on multimodal interaction},
pages = {237–244},
publisher = {ACM Press},
abstract = {Obtaining a compact and discriminative representation of facial and body expressions is a difficult problem in emotion recognition. Part of the difficulty is capturing microexpressions, i.e., short, involuntary expressions that last for only a fraction of a second: at a micro-temporal scale, there are so many other subtle face and body movements that do not convey semantically meaningful information. We present a novel approach to this problem by exploiting the sparsity of the frequent micro-temporal motion patterns. Local space-time features are extracted over the face and body region for a very short time period, e.g., few milliseconds. A codebook of microexpressions is learned from the data and used to encode the features in a sparse manner. This allows us to obtain a representation that captures the most salient motion patterns of the face and body at a micro-temporal scale. Experiments performed on the AVEC 2012 dataset show our approach achieving the best published performance on the expectation dimension based solely on visual features. We also report experimental results on audio-visual emotion recognition, comparing early and late data fusion techniques.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Obtaining a compact and discriminative representation of facial and body expressions is a difficult problem in emotion recognition. Part of the difficulty is capturing microexpressions, i.e., short, involuntary expressions that last for only a fraction of a second: at a micro-temporal scale, there are so many other subtle face and body movements that do not convey semantically meaningful information. We present a novel approach to this problem by exploiting the sparsity of the frequent micro-temporal motion patterns. Local space-time features are extracted over the face and body region for a very short time period, e.g., few milliseconds. A codebook of microexpressions is learned from the data and used to encode the features in a sparse manner. This allows us to obtain a representation that captures the most salient motion patterns of the face and body at a micro-temporal scale. Experiments performed on the AVEC 2012 dataset show our approach achieving the best published performance on the expectation dimension based solely on visual features. We also report experimental results on audio-visual emotion recognition, comparing early and late data fusion techniques.
Xu, Yuyu; Feng, Andrew W.; Marsella, Stacy C.; Shapiro, Ari
A Practical and Configurable Lip Sync Method for Games Proceedings Article
In: ACM SIGGRAPH Motion in Games, Dublin, Ireland, 2013.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{xu_practical_2013,
title = {A Practical and Configurable Lip Sync Method for Games},
author = {Yuyu Xu and Andrew W. Feng and Stacy C. Marsella and Ari Shapiro},
url = {http://ict.usc.edu/pubs/A%20Practical%20and%20Configurable%20Lip%20Sync%20Method%20for%20Games.pdf},
year = {2013},
date = {2013-11-01},
booktitle = {ACM SIGGRAPH Motion in Games},
address = {Dublin, Ireland},
abstract = {We demonstrate a lip animation (lip sync) algorithm for real-time applications that can be used to generate synchronized facial movements with audio generated from natural speech or a text-to-speech engine. Our method requires an animator to construct animations using a canonical set of visemes for all pairwise combinations of a reduced phoneme set (phone bigrams). These animations are then stitched together to construct the final animation, adding velocity and lip-pose constraints. This method can be applied to any character that uses the same, small set of visemes. Our method can operate efficiently in multiple languages by reusing phone bigram animations that are shared among languages, and specific word sounds can be identified and changed on a per-character basis. Our method uses no machine learning, which offers two advantages over techniques that do: 1) data can be generated for non-human characters whose faces can not be easily retargeted from a human speaker’s face, and 2) the specific facial poses or shapes used for animation can be specified during the setup and rigging stage, and before the lip animation stage, thus making it suitable for game pipelines or circumstances where the speech targets poses are predetermined, such as after acquisition from an online 3D marketplace.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
We demonstrate a lip animation (lip sync) algorithm for real-time applications that can be used to generate synchronized facial movements with audio generated from natural speech or a text-to-speech engine. Our method requires an animator to construct animations using a canonical set of visemes for all pairwise combinations of a reduced phoneme set (phone bigrams). These animations are then stitched together to construct the final animation, adding velocity and lip-pose constraints. This method can be applied to any character that uses the same, small set of visemes. Our method can operate efficiently in multiple languages by reusing phone bigram animations that are shared among languages, and specific word sounds can be identified and changed on a per-character basis. Our method uses no machine learning, which offers two advantages over techniques that do: 1) data can be generated for non-human characters whose faces can not be easily retargeted from a human speaker’s face, and 2) the specific facial poses or shapes used for animation can be specified during the setup and rigging stage, and before the lip animation stage, thus making it suitable for game pipelines or circumstances where the speech targets poses are predetermined, such as after acquisition from an online 3D marketplace.
Gratch, Jonathan; Marsella, Stacy C. (Ed.)
Social Emotions in Nature and Artifact Book
Oxford University Press, 2013, ISBN: 0-19-538764-3.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@book{gratch_social_2013,
title = {Social Emotions in Nature and Artifact},
editor = {Jonathan Gratch and Stacy C. Marsella},
url = {http://www.amazon.com/Social-Emotions-Artifact-Cognitive-Architectures/dp/0195387643},
isbn = {0-19-538764-3},
year = {2013},
date = {2013-11-01},
publisher = {Oxford University Press},
series = {Oxford Series on Cognitive Models and Architectures},
abstract = {Recent years have seen the rise of a remarkable partnership between the social and computational sciences on the phenomena of emotions. Rallying around the term Affective Computing, this research can be seen as revival of the cognitive science revolution, albeit garbed in the cloak of affect, rather than cognition. Traditional cognitive science research, to the extent it considered emotion at all, cases it as at best a heuristic but more commonly a harmful bias to cognition. More recent scholarship in the social sciences has upended this view. Increasingly, emotions are viewed as a form of information processing that serves a functional role in human cognition and social interactions. Emotions shape social motives and communicate important information to social partners. When communicating face-to-face, people can rapidly detect nonverbal affective cues, make inferences about the other party's mental state, and respond in ways that co-construct an emotional trajectory between participants. Recent advances in biometrics and artificial intelligence are allowing computer systems to engage in this nonverbal dance, on the one hand opening a wealth of possibilities for human-machine systems, and on the other, creating powerful new tools for behavioral science research. Social Emotions in Nature and Artifact reports on the state-of-the-art in both social science theory and computational methods, and illustrates how these two fields, together, can both facilitate practical computer/robotic applications and illuminate human social processes.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {book}
}
Recent years have seen the rise of a remarkable partnership between the social and computational sciences on the phenomena of emotions. Rallying around the term Affective Computing, this research can be seen as revival of the cognitive science revolution, albeit garbed in the cloak of affect, rather than cognition. Traditional cognitive science research, to the extent it considered emotion at all, cases it as at best a heuristic but more commonly a harmful bias to cognition. More recent scholarship in the social sciences has upended this view. Increasingly, emotions are viewed as a form of information processing that serves a functional role in human cognition and social interactions. Emotions shape social motives and communicate important information to social partners. When communicating face-to-face, people can rapidly detect nonverbal affective cues, make inferences about the other party's mental state, and respond in ways that co-construct an emotional trajectory between participants. Recent advances in biometrics and artificial intelligence are allowing computer systems to engage in this nonverbal dance, on the one hand opening a wealth of possibilities for human-machine systems, and on the other, creating powerful new tools for behavioral science research. Social Emotions in Nature and Artifact reports on the state-of-the-art in both social science theory and computational methods, and illustrates how these two fields, together, can both facilitate practical computer/robotic applications and illuminate human social processes.
Lane, H. Chad; Hays, Matthew Jensen; Core, Mark G.; Auerbach, Daniel
Learning intercultural communication skills with virtual humans: Feedback and fidelity. Journal Article
In: Journal of Educational Psychology, vol. 105, no. 4, pp. 1026–1035, 2013, ISSN: 1939-2176, 0022-0663.
Abstract | Links | BibTeX | Tags: Learning Sciences, UARC
@article{lane_learning_2013,
title = {Learning intercultural communication skills with virtual humans: Feedback and fidelity.},
author = {H. Chad Lane and Matthew Jensen Hays and Mark G. Core and Daniel Auerbach},
url = {http://doi.apa.org/getdoi.cfm?doi=10.1037/a0031506},
doi = {10.1037/a0031506},
issn = {1939-2176, 0022-0663},
year = {2013},
date = {2013-11-01},
journal = {Journal of Educational Psychology},
volume = {105},
number = {4},
pages = {1026–1035},
abstract = {In the context of practicing intercultural communication skills, we investigated the role of fidelity in a game-based, virtual learning environment as well as the role of feedback delivered by an intelligent tutoring system. In 2 experiments, we compared variations on the game interface, use of the tutoring system, and the form of the feedback. Our findings suggest that for learning basic intercultural communicative skills, a 3-dimensional (3-D) interface with animation and sound produced equivalent learning to a more static 2-D interface. However, learners took significantly longer to analyze and respond to the actions of animated virtual humans, suggesting a deeper engagement. We found large gains in learning across conditions. There was no differential effect with the tutor engaged, but it was found to have a positive impact on learner success in a transfer task. This difference was most pronounced when the feedback was delivered in a more general form versus a concrete style.},
keywords = {Learning Sciences, UARC},
pubstate = {published},
tppubtype = {article}
}
In the context of practicing intercultural communication skills, we investigated the role of fidelity in a game-based, virtual learning environment as well as the role of feedback delivered by an intelligent tutoring system. In 2 experiments, we compared variations on the game interface, use of the tutoring system, and the form of the feedback. Our findings suggest that for learning basic intercultural communicative skills, a 3-dimensional (3-D) interface with animation and sound produced equivalent learning to a more static 2-D interface. However, learners took significantly longer to analyze and respond to the actions of animated virtual humans, suggesting a deeper engagement. We found large gains in learning across conditions. There was no differential effect with the tutor engaged, but it was found to have a positive impact on learner success in a transfer task. This difference was most pronounced when the feedback was delivered in a more general form versus a concrete style.
Nouri, Elnaz
Does History Help? An Experiment on How Context Affects Crowdsourcing Dialogue Annotation Proceedings Article
In: Proceedings of the Human Computation Workshop on Scaling Speech, Language Understanding and Dialogue through Crowdsourcing, Palm Springs, CA, 2013.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{nouri_does_2013,
title = {Does History Help? An Experiment on How Context Affects Crowdsourcing Dialogue Annotation},
author = {Elnaz Nouri},
url = {http://ict.usc.edu/pubs/Does%20History%20Help%20-%20An%20Experiment%20on%20How%20Context%20Affects%20Crowdsourcing%20Dialogue%20Annotation.pdf},
year = {2013},
date = {2013-11-01},
booktitle = {Proceedings of the Human Computation Workshop on Scaling Speech, Language Understanding and Dialogue through Crowdsourcing},
address = {Palm Springs, CA},
abstract = {Crowds of people can potentially solve some problems faster than individuals. Crowd sourced data can be leveraged to benefit the crowd by providing information or solutions faster than traditional means. Many tasks needed for developing dialogue systems such as annotation can benefit from crowdsourcing as well. We investigate how to outsource dialogue data annotation through Amazon Mechanical Turk. We are in particular interested in empirically analyzing how much context from previous parts of the dialogue (e.g. previous dialogue turns) is needed to be provided before the target part (dialogue turn) is presented to the annotator. The answer to this question is essentially important for leveraging crowd sourced data for appropriate and efficient response and coordination. We study the effect of presenting different numbers of previous data (turns) to the Turkers in annotating sentiments of dyadic negotiation dialogs on the inter annotator reliability and comparison to the gold standard.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
Crowds of people can potentially solve some problems faster than individuals. Crowd sourced data can be leveraged to benefit the crowd by providing information or solutions faster than traditional means. Many tasks needed for developing dialogue systems such as annotation can benefit from crowdsourcing as well. We investigate how to outsource dialogue data annotation through Amazon Mechanical Turk. We are in particular interested in empirically analyzing how much context from previous parts of the dialogue (e.g. previous dialogue turns) is needed to be provided before the target part (dialogue turn) is presented to the annotator. The answer to this question is essentially important for leveraging crowd sourced data for appropriate and efficient response and coordination. We study the effect of presenting different numbers of previous data (turns) to the Turkers in annotating sentiments of dyadic negotiation dialogs on the inter annotator reliability and comparison to the gold standard.
Morbini, Fabrizio; DeVault, David; Sagae, Kenji; Gerten, Jillian; Nazarian, Angela; Traum, David
FLoReS: A Forward Looking, Reward Seeking, Dialogue Manager Book Section
In: Natural Interaction with Robots, Knowbots and Smartphones - Putting Spoken Dialog Systems into Practice, pp. 313–325, Springer New York, 2013.
Links | BibTeX | Tags: UARC, Virtual Humans
@incollection{morbini_flores_2013,
title = {FLoReS: A Forward Looking, Reward Seeking, Dialogue Manager},
author = {Fabrizio Morbini and David DeVault and Kenji Sagae and Jillian Gerten and Angela Nazarian and David Traum},
url = {http://www.amazon.com/Natural-Interaction-Robots-Knowbots-Smartphones/dp/1461482798/ref=sr_1_1?s=books&ie=UTF8&qid=1409179426&sr=1-1&keywords=Natural+Interaction+with+Robots%2C+Knowbots+and+Smartphones},
year = {2013},
date = {2013-11-01},
booktitle = {Natural Interaction with Robots, Knowbots and Smartphones - Putting Spoken Dialog Systems into Practice},
pages = {313–325},
publisher = {Springer New York},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {incollection}
}
Ito, Jonathan Y.; Marsella, Stacy C.
Modeling Framing Effects Comparing an Appraisal-Based Model with Existing Models Proceedings Article
In: ACII 2013, pp. 381–386, IEEE Computer Society, 2013.
Abstract | Links | BibTeX | Tags: Social Simulation, UARC
@inproceedings{ito_modeling_2013,
title = {Modeling Framing Effects Comparing an Appraisal-Based Model with Existing Models},
author = {Jonathan Y. Ito and Stacy C. Marsella},
url = {http://ict.usc.edu/pubs/Modeling%20Framing%20Effects%20Comparing%20an%20Appraisal-Based%20Model%20with%20Existing%20Models.pdf},
year = {2013},
date = {2013-09-01},
booktitle = {ACII 2013},
pages = {381–386},
publisher = {IEEE Computer Society},
abstract = {One significant challenge in creating accurate models of human decision behavior is accounting for the effects of context. Research shows that seemingly minor changes in the presentation of a decision can lead to shifts in behavior; phenomena collectively referred to as framing effects. This work presents a computational modeling analysis comparing the effectiveness of Context Dependent Utility, an appraisal-based approach to modeling the multi-dimensional effects of context on decision behavior, against Cumulative Prospect Theory, Security-Potential/Aspiration Theory, the Transfer of Attention Exchange model, and a power-based utility function. To contrast model performance, a non-linear least-squares analysis and subsequent calculation of Akaike Information Criterion scores, which take into account goodness of fit while penalizing for model complexity, are employed. Results suggest that multi-dimensional models of context and framing, such as Context Dependent Utility, can be much more accurate in modeling decisions which similarly involve multi-dimensional considerations of context. Furthermore, this work demonstrates the effectiveness of employing affective constructs, such as appraisal, for encoding and evaluation of context within decision-theoretic frameworks to better model and predict human decision behavior.},
keywords = {Social Simulation, UARC},
pubstate = {published},
tppubtype = {inproceedings}
}
One significant challenge in creating accurate models of human decision behavior is accounting for the effects of context. Research shows that seemingly minor changes in the presentation of a decision can lead to shifts in behavior; phenomena collectively referred to as framing effects. This work presents a computational modeling analysis comparing the effectiveness of Context Dependent Utility, an appraisal-based approach to modeling the multi-dimensional effects of context on decision behavior, against Cumulative Prospect Theory, Security-Potential/Aspiration Theory, the Transfer of Attention Exchange model, and a power-based utility function. To contrast model performance, a non-linear least-squares analysis and subsequent calculation of Akaike Information Criterion scores, which take into account goodness of fit while penalizing for model complexity, are employed. Results suggest that multi-dimensional models of context and framing, such as Context Dependent Utility, can be much more accurate in modeling decisions which similarly involve multi-dimensional considerations of context. Furthermore, this work demonstrates the effectiveness of employing affective constructs, such as appraisal, for encoding and evaluation of context within decision-theoretic frameworks to better model and predict human decision behavior.
Khooshabeh, Peter; Dehghani, Morteza; Nazarian, Angela; Gratch, Jonathan
The Cultural Influence Model: When Accented Natural Language Spoken by Virtual Characters Matters Journal Article
In: Journal of Artificial Intelligence and Society, vol. 29, 2013.
Abstract | Links | BibTeX | Tags: ARL, DoD, UARC, Virtual Humans
@article{khooshabeh_cultural_2013,
title = {The Cultural Influence Model: When Accented Natural Language Spoken by Virtual Characters Matters},
author = {Peter Khooshabeh and Morteza Dehghani and Angela Nazarian and Jonathan Gratch},
url = {http://ict.usc.edu/pubs/The%20Cultural%20Influence%20Model-%20When%20Accented%20Natural%20Language%20Spoken%20by%20Virtual%20Characters%20Matters.pdf},
year = {2013},
date = {2013-09-01},
journal = {Journal of Artificial Intelligence and Society},
volume = {29},
abstract = {Advances in Artificial Intelligence (AI) and computer graphics digital technologies have contributed to a relative increase of realism in virtual characters. Preserving virtual characters’ communicative realism, in particular, joined the ranks of the improvements in natural language technology and animation algorithms. This paper focuses on culturally relevant paralinguistic cues in nonverbal communication. We model the effects of an English speaking digital character with different accents on human interactants (i.e., users). Our cultural influence model proposes that paralinguistic realism, in the form of accented speech, is effective in promoting culturally congruent cognition only when it is self-relevant to users. For example, a Chinese or Middle Eastern English accent may be perceived as foreign to individuals who do not share the same ethnic cultural background with members of those cultures. However, for individuals who are familiar and affiliate with those cultures (i.e., in-group members who are bicultural), accent not only serves as a motif of shared social identity, it also primes them to adopt culturally appropriate interpretive frames that influence their decision making.},
keywords = {ARL, DoD, UARC, Virtual Humans},
pubstate = {published},
tppubtype = {article}
}
Advances in Artificial Intelligence (AI) and computer graphics digital technologies have contributed to a relative increase of realism in virtual characters. Preserving virtual characters’ communicative realism, in particular, joined the ranks of the improvements in natural language technology and animation algorithms. This paper focuses on culturally relevant paralinguistic cues in nonverbal communication. We model the effects of an English speaking digital character with different accents on human interactants (i.e., users). Our cultural influence model proposes that paralinguistic realism, in the form of accented speech, is effective in promoting culturally congruent cognition only when it is self-relevant to users. For example, a Chinese or Middle Eastern English accent may be perceived as foreign to individuals who do not share the same ethnic cultural background with members of those cultures. However, for individuals who are familiar and affiliate with those cultures (i.e., in-group members who are bicultural), accent not only serves as a motif of shared social identity, it also primes them to adopt culturally appropriate interpretive frames that influence their decision making.
Park, Sunghyun; Scherer, Stefan; Gratch, Jonathan; Carnevale, Peter; Morency, Louis-Philippe
Mutual Behaviors during Dyadic Negotiation: Automatic Prediction of Respondent Reactions Proceedings Article
In: Affective Computing and Intelligent Interaction, pp. 423–428, Geneva, Switzerland, 2013, ISBN: 978-0-7695-5048-0.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{park_mutual_2013,
title = {Mutual Behaviors during Dyadic Negotiation: Automatic Prediction of Respondent Reactions},
author = {Sunghyun Park and Stefan Scherer and Jonathan Gratch and Peter Carnevale and Louis-Philippe Morency},
url = {http://ieeexplore.ieee.org/lpdocs/epic03/wrapper.htm?arnumber=6681467},
doi = {10.1109/ACII.2013.76},
isbn = {978-0-7695-5048-0},
year = {2013},
date = {2013-09-01},
booktitle = {Affective Computing and Intelligent Interaction},
pages = {423–428},
address = {Geneva, Switzerland},
abstract = {In this paper, we analyze face-to-face negotiation interactions with the goal of predicting the respondent’s immediate reaction (i.e., accept or reject) to a negotiation offer. Supported by the theory of social rapport, we focus on mutual behaviors which are defined as nonverbal characteristics that occur due to interactional influence. These patterns include behavioral symmetry (e.g., synchronized smiles) as well as asymmetry (e.g., opposite postures) between the two negotiators. In addition, we put emphasis on finding audio- visual mutual behaviors that can be extracted automatically, with the vision of a real-time decision support tool. We introduce a dyadic negotiation dataset consisting of 42 face-to- face interactions and show experiments confirming the importance of multimodal and mutual behaviors.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
In this paper, we analyze face-to-face negotiation interactions with the goal of predicting the respondent’s immediate reaction (i.e., accept or reject) to a negotiation offer. Supported by the theory of social rapport, we focus on mutual behaviors which are defined as nonverbal characteristics that occur due to interactional influence. These patterns include behavioral symmetry (e.g., synchronized smiles) as well as asymmetry (e.g., opposite postures) between the two negotiators. In addition, we put emphasis on finding audio- visual mutual behaviors that can be extracted automatically, with the vision of a real-time decision support tool. We introduce a dyadic negotiation dataset consisting of 42 face-to- face interactions and show experiments confirming the importance of multimodal and mutual behaviors.
Stratou, Giota; Scherer, Stefan; Gratch, Jonathan; Morency, Louis-Philippe
Automatic Nonverbal Behavior Indicators of Depression and PTSD: Exploring Gender Differences Proceedings Article
In: Affective Computing and Intelligent Interaction, pp. 147–152, IEEE, Geneva, Switzerland, 2013, ISBN: 978-0-7695-5048-0.
Abstract | Links | BibTeX | Tags: UARC, Virtual Humans
@inproceedings{stratou_automatic_2013,
title = {Automatic Nonverbal Behavior Indicators of Depression and PTSD: Exploring Gender Differences},
author = {Giota Stratou and Stefan Scherer and Jonathan Gratch and Louis-Philippe Morency},
url = {http://ieeexplore.ieee.org/lpdocs/epic03/wrapper.htm?arnumber=6681422},
doi = {10.1109/ACII.2013.31},
isbn = {978-0-7695-5048-0},
year = {2013},
date = {2013-09-01},
booktitle = {Affective Computing and Intelligent Interaction},
pages = {147–152},
publisher = {IEEE},
address = {Geneva, Switzerland},
abstract = {In this paper, we show that gender plays an important role in the automatic assessment of psychological conditions such as depression and post-traumatic stress disorder (PTSD). We identify a directly interpretable and intuitive set of predictive indicators, selected from three general categories of nonverbal behaviors: affect, expression variability and motor variability. For the analysis, we introduce a semi-structured virtual human interview dataset which includes 53 video recorded interactions. Our experiments on automatic classification of psychological conditions show that a gender-dependent approach significantly improves the performance over a gender agnostic one.},
keywords = {UARC, Virtual Humans},
pubstate = {published},
tppubtype = {inproceedings}
}
In this paper, we show that gender plays an important role in the automatic assessment of psychological conditions such as depression and post-traumatic stress disorder (PTSD). We identify a directly interpretable and intuitive set of predictive indicators, selected from three general categories of nonverbal behaviors: affect, expression variability and motor variability. For the analysis, we introduce a semi-structured virtual human interview dataset which includes 53 video recorded interactions. Our experiments on automatic classification of psychological conditions show that a gender-dependent approach significantly improves the performance over a gender agnostic one.