Publications

@inproceedings{dininni_sensing_2022,

title = {Sensing Human Signals of Motivation Processes During STEM Tasks},

author = {Richard DiNinni and Albert Rizzo},

editor = {Maria Mercedes Rodrigo and Noburu Matsuda and Alexandra I. Cristea and Vania Dimitrova},

doi = {10.1007/978-3-031-11647-6_28},

isbn = {978-3-031-11647-6},

year  = {2022},

date = {2022-01-01},

booktitle = {Artificial Intelligence in Education. Posters and Late Breaking Results, Workshops and Tutorials, Industry and Innovation Tracks, Practitioners’ and Doctoral Consortium},

pages = {163–167},

publisher = {Springer International Publishing},

address = {Cham},

series = {Lecture Notes in Computer Science},

abstract = {This paper outlines the linking of a multi-modal sensing platform with an Intelligent Tutoring System to perceive the motivational state of the learner during STEM tasks. Motivation is a critical element to learning but receives little attention in comparison to strategies related to cognitive processes. The EMPOWER project has developed a novel platform that offers researchers an opportunity to capture a learner’s multi-modal behavioral signals to develop models of motivation problems that can be used to develop best practice strategies for instructional systems.},

keywords = {DTIC, Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{liu_graph_2021,

title = {Graph Neural Network Based Behavior Prediction to Support Multi-Agent Reinforcement Learning in Military Training Simulations},

author = {Lixing Liu and Nikolos Gurney and Kyle McCullough and Volkan Ustun},

url = {https://ieeexplore.ieee.org/document/9715433/},

doi = {10.1109/WSC52266.2021.9715433},

isbn = {978-1-66543-311-2},

year  = {2021},

date = {2021-12-01},

urldate = {2022-09-21},

booktitle = {2021 Winter Simulation Conference (WSC)},

pages = {1–12},

publisher = {IEEE},

address = {Phoenix, AZ, USA},

keywords = {DTIC, Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{adami_immersive_2020,

title = {An Immersive Virtual Learning Environment for Worker-Robot Collaboration on Construction Sites},

author = {Pooya Adami and Burcin Becerik-Gerber and Lucio Soibelman and Tenzin Doleck and Yasemin Copur-Gencturk and Gale Lucas},

url = {https://ieeexplore.ieee.org/document/9383944/},

doi = {10.1109/WSC48552.2020.9383944},

isbn = {978-1-72819-499-8},

year  = {2020},

date = {2020-12-01},

urldate = {2022-10-24},

booktitle = {2020 Winter Simulation Conference (WSC)},

pages = {2400–2411},

publisher = {IEEE},

address = {Orlando, FL, USA},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{georgila_using_2019,

title = {Using Reinforcement Learning to Optimize the Policies of an Intelligent Tutoring System for Interpersonal Skills Training},

author = {Kallirroi Georgila and Mark G Core and Benjamin D Nye and Shamya Karumbaiah and Daniel Auerbach and Maya Ram},

url = {http://www.ifaamas.org/Proceedings/aamas2019/pdfs/p737.pdf},

year  = {2019},

date = {2019-05-01},

booktitle = {Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems},

pages = {9},

publisher = {IFAAMAS},

address = {Montreal, Canada},

abstract = {Reinforcement Learning (RL) has been applied successfully to Intelligent Tutoring Systems (ITSs) in a limited set of well-defined domains such as mathematics and physics. This work is unique in using a large state space and for applying RL to tutoring interpersonal skills. Interpersonal skills are increasingly recognized as critical to both social and economic development. In particular, this work enhances an ITS designed to teach basic counseling skills that can be applied to challenging issues such as sexual harassment and workplace conflict. An initial data collection was used to train RL policies for the ITS, and an evaluation with human participants compared a hand-crafted ITS which had been used for years with students (control) versus the new ITS guided by RL policies. The RL condition differed from the control condition most notably in the strikingly large quantity of guidance it provided to learners. Both systems were effective and there was an overall significant increase from pre- to post-test scores. Although learning gains did not differ significantly between conditions, learners had a significantly higher self-rating of confidence in the RL condition. Confidence and learning gains were both part of the reward function used to train the RL policies, and it could be the case that there was the most room for improvement in confidence, an important learner emotion. Thus, RL was successful in improving an ITS for teaching interpersonal skills without the need to prune the state space (as previously done).},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{davis_pedagogical_2019,

title = {Pedagogical Tools to Enhance Analytic Skills: Interactive Virtual Tutorial Environments},

author = {Dan M Davis and Christi L Phelps and Frederica J Stassi},

url = {http://www.modsimworld.org/conference-papers/2019},

year  = {2019},

date = {2019-04-01},

booktitle = {Proceedings of MODSIM World 2019},

pages = {12},

address = {Norfolk, VA},

abstract = {This paper examines the use of literature studies to enhance communication and critical thinking skills in technical students through the application of emerging Virtual Reality (VR) technologies to enable that pedagogical approach. The current state of analytic skills among students in Science, Technology, Engineering and Mathematics (STEM) tracks are outlined, focusing on the critical years in secondary schools. Their prospective needs as they advance into tertiary education and the needs of the technical community for improvement are presented. The requirements flowing from that analysis will be discussed in the light of programs implemented at the Sato Academy, with reports of both successes and missteps. In some detail, the use of the study of literature is described and discussed. The authors present their case for constructivist and Socratic approaches to fully engage and effectively inculcate communication proficiency, including conformance with standards, e.g. Next Generation Science Standards (NGSS). These results are then compared to the demands of college and professional leaders who are currently being burdened with having to provide disruptive remedial efforts. The methods found to be successful are considered, both in terms of their application and their extensibility to other fields. Also highlighted will be areas in which time and personnel constraints hindered achievement. A number of possible responses to these impediments will be presented, evaluating the feasibility of each. The paper will then focus on the advances in virtual humans and conversational avatars. Recent research into using large libraries of video-clips to create engaging on-line virtual tutorial conversations will be presented. Data as to the receptivity of students to conversing with computer-generated interlocutors is presented, along with a discussion as to how this technology is applicable to teaching the analysis of literature. The benefits of and the barriers to virtual tutorial environments are outlined and analyzed.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{davis_enhancements_2019,

title = {Enhancements for Homeschooling and ADL: Virtual Humans, Technologies and Insights},

author = {Dan M Davis and Nancy L H Young and Mark C Davis and North Carolina},

url = {http://www.modsimworld.org/conference-papers/2019},

year  = {2019},

date = {2019-04-01},

booktitle = {Proceedings of MODSIM World 2019},

pages = {12},

address = {Norfolk, VA},

abstract = {Homeschooling and DoD Advanced Distributed Learning (ADL) have many goals in common, so increasing the collaborative research and collegial information exchange between their respective communities would be mutually advantageous. The emerging capabilities of virtual humans provide a useful prototype of how both homeschooling and ADL can benefit from emerging technological advances. This paper begins with an examination of the home schooling movement in the United States, including a review of its foundations, demographics, results and trends. In examining the goals of homeschooling parents, the four major reasons cited by at least half of those parents are considered and explicated: desire to find environment most compatible to users, provision of ethics foundations, inclusion of accountability instruction and dissatisfaction with other pedagogical approaches. Also meriting review are the hurdles faced by homeschool teachers and students, followed by an item-by-item comparison with analogous challenges for ADL provisioners and learners. A short analysis of the constraints on the two communities focuses on similarities and differences between family limitations and defense organization restrictions. The authors then present data on the current scope, instantiations, and achievements of the two efforts. Many of the technologies currently in use are reviewed and discussed, concentrating on computer-aided education and distributed learning. Emerging technologies based on artificial intelligence, natural language processing, and virtual humans are described and considered. Their uses in various contexts provide sufficient data to quantify the impact on subjects and the authors adduce findings from research to support their thesis that increased use of these technologies would be beneficial both to homeschooled students and to DoD Learners. The paper closes with an evaluation of the arc of current research, the recognition of prenascent capabilities (e.g. quantum computing), the burgeoning needs of both communities, and the need to nurture a synergistic exchange between homeschool advocates and ADL architects.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_engaging_2018,

title = {Engaging with the Scenario: Affect and Facial Patterns from a Scenario-Based Intelligent Tutoring System},

author = {Benjamin D. Nye and Shamya Karumbaiah and S. Tugba Tokel and Mark G. Core and Giota Stratou and Daniel Auerbach and Kallirroi Georgila},

url = {http://link.springer.com/10.1007/978-3-319-93843-1_26},

doi = {10.1007/978-3-319-93843-1_26},

isbn = {978-3-319-93842-4 978-3-319-93843-1},

year  = {2018},

date = {2018-06-01},

booktitle = {Proceeding of the International Conference on Artificial Intelligence in Education},

volume = {10947},

pages = {352–366},

publisher = {Springer International Publishing},

address = {London, UK},

abstract = {Facial expression trackers output measures for facial action units (AUs), and are increasingly being used in learning technologies. In this paper, we compile patterns of AUs seen in related work as well as use factor analysis to search for categories implicit in our corpus. Although there was some overlap between the factors in our data and previous work, we also identified factors seen in the broader literature but not previously reported in the context of learning environments. In a correlational analysis, we found evidence for relationships between factors and self-reported traits such as academic effort, study habits, and interest in the subject. In addition, we saw differences in average levels of factors between a video watching activity, and a decision making activity. However, in this analysis, we were not able to isolate any facial expressions having a significant positive or negative relationship with either learning gain, or performance once question difficulty and related factors were also considered. Given the overall low levels of facial affect in the corpus, further research will explore different populations and learning tasks to test the possible hypothesis that learners may have been in a pattern of “Over-Flow” in which they were engaged with the system, but not deeply thinking about the content or their errors.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hampton_mitigating_2018,

title = {Mitigating Knowledge Decay from Instruction with Voluntary Use of an Adaptive Learning System},

author = {Andrew J. Hampton and Benjamin D. Nye and Philip I. Pavlik and William R. Swartout and Arthur C. Graesser and Joseph Gunderson},

url = {http://link.springer.com/10.1007/978-3-319-93846-2_23},

doi = {10.1007/978-3-319-93846-2_23},

isbn = {978-3-319-93845-5 978-3-319-93846-2},

year  = {2018},

date = {2018-06-01},

booktitle = {Proceedings of the International Conference on Artificial Intelligence in Education},

volume = {10948},

pages = {119–133},

publisher = {Springer International Publishing},

address = {London, UK},

abstract = {Knowledge decays across breaks in instruction. Learners lack the metacognition to self-assess their knowledge decay and effectively self-direct review, as well as lacking interactive exercises appropriate to their individual knowledge level. Adaptive learning systems offer the potential to mitigate these issues, by providing open learner models to facilitate learner’s understanding of their knowledge levels and by presenting personalized practice exercises. The current study analyzes differences in knowledge decay between learners randomly assigned to an intervention where they could use an adaptive system during a long gap between courses, compared with a control condition. The experimental condition used the Personal Assistant for Life-Long Learning (PAL3), a tablet-based adaptive learning system integrating multiple intelligent tutoring systems and conventional learning resources. It contained electronics content relevant to the experiment participants, Navy sailors who graduated from apprentice electronics courses (A-School) awaiting assignment to their next training (C-School). The study was conducted over one month, collecting performance data with a counterbalanced pre-, mid-, and post-test. The control condition exhibited the expected decay. The PAL3 condition showed a significant difference from the control, with no significant knowledge decay in their overall knowledge, despite substantial variance in usage for PAL3 (e.g., most of overall use in the first week, with fewer participants engaging as time went on). Interestingly, while overall decay was mitigated in PAL3, this result was primarily through gains in some knowledge offsetting losses in other knowledge. Overall, these results indicate that adaptive study tools can help prevent knowledge decay, even with voluntary usage.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_mentorpal_2017,

title = {MentorPal: Interactive Virtual Mentors Based on Real-Life STEM Professionals},

author = {Benjamin D Nye and Nicholas J Kaimakis and Madhusudhan Krishnamachari and William Swartout and Julia Campbell and Clinton Anderson and Dan M Davis},

url = {http://www.iitsecdocs.com/search},

year  = {2017},

date = {2017-11-01},

booktitle = {Proceedings of the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2017},

publisher = {a2z, Inc.},

address = {Orlando, Florida},

abstract = {In an ideal world, all students could meet STEM role models as they explore different careers. However, events such as career fairs do not scale well: professionals have limited time and effective mentors are not readily available in all fields. The result is that students’ understanding is minimal about what professionals in STEM fields do every day, what education is needed, and even what STEM fields exist. Moreover, since in-person interactions rely on finding people engaged in current STEM careers, students may form career goals for stagnant fields rather than growing fields (e.g., projected workforce needs). To address this problem, we are designing a scalable tablet-based app that gives students the opportunity to converse with interactive recordings of real-life STEM professionals. These conversational virtual agents will emulate a question-and-answer session with STEM professionals who have Navy ties and who are engaging, enthusiastic, and effective mentors. These interactions will allow students to have a lifelike informational interview with a virtual agent whose responses are directly drawn from a specific real professional’s video-recorded interview. This work differs from prior research on career guides by capturing the experiences of a collection of unique mentors, which should be more authentic and engaging than a generic agent or resource which speaks only about the average experience. This paper will discuss the process of creating the first such virtual STEM mentor prototype, including the development of an extensive mentoring question bank (approximately 500 questions); key mentoring topics that intersect STEM, DoD, and civilian life; techniques for cost-effective recording of remote mentors; and the process of training and verifying a natural language dialogue model for answering and suggesting career questions. Finally, we conclude with implications, strengths, and drawbacks of virtualizing the experience of talking with specific mentors, from the perspectives of efficacy, scalability, and maintainability.},

keywords = {Learning Sciences, MxR, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{fang_online_2017,

title = {Online Learning Persistence and Academic Achievement},

author = {Ying Fang and Yonghong Jade Xu and Benjamin Nye and Arthur Graesser and Philip Pavlik and Xiangen Hu},

url = {http://educationaldatamining.org/EDM2017/proc_files/papers/paper_114.pdf},

year  = {2017},

date = {2017-06-01},

booktitle = {Proceedings of Educational Data Mining (EDM) 2017},

pages = {312 – 317},

publisher = {EDM 2017},

address = {Wuhan, China},

abstract = {Student persistence in online learning environments has typically been studied at the macro-level (e.g., completion of an online course, number of academic terms completed, etc.). The current examines student persistence in an adaptive learning environment, ALEKS (Assessment and LEarning in Knowledge Spaces). Specifically, the study explores the relationship between students' academic achievement and their persistence during learning. By using archived data that included their math learning log data and performance on two standardized tests, we first explored student learning behavior patterns with regard to their persistence during learning. Clustering analysis identified three distinctive patterns of persistence-related learning behaviors: (1) High persistence and rare topic shifting; (2) Low persistence and frequent topic shifting; and (3) Moderate persistence and moderate topic shifting. We further explored the association between persistence and academic achievement. No significant differences were observed between academic achievement and the different learning patterns. We interpret this result in addition to a preliminary exploration of topic mastery trends, to suggest that wheel-spinning" behaviors coexist with persistence, and is ultimately not beneficial to learning.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_analyzing_2017,

title = {Analyzing Learner Affect in a Scenario-Based Intelligent Tutoring System},

author = {Benjamin Nye and Shamya Karumbaiah and S. Tugba Tokel and Mark G. Core and Giota Stratou and Daniel Auerbach and Kallirroi Georgila},

url = {https://link.springer.com/chapter/10.1007/978-3-319-61425-0_60},

doi = {https://doi.org/10.1007/978-3-319-61425-0_60},

isbn = {978-3-319-61425-0},

year  = {2017},

date = {2017-06-01},

booktitle = {Proceedings of the International Conference on Artificial Intelligence in Education},

pages = {544–547},

publisher = {Springer},

address = {Wuhan, China},

abstract = {Scenario-based tutoring systems influence affective states due to two distinct mechanisms during learning: 1) reactions to performance feedback and 2) responses to the scenario context or events. To explore the role of affect and engagement, a scenario-based ITS was instrumented to support unobtrusive facial affect detection. Results from a sample of university students showed relatively few traditional academic affective states such as confusion or frustration, even at decision points and after poor performance (e.g., incorrect responses). This may show evidence of "over-flow," with a high level of engagement and interest but insufficient confusion/disequilibrium for optimal learning.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_building_2017,

title = {Building a Backbone for Multi-Agent Tutoring in GIFT (Work in Progress)},

author = {Benjamin D. Nye and Daniel Auerbach and Tirth R. Mehta and Arno Hartholt},

url = {https://books.google.com/books?id=PwMtDwAAQBAJ&printsec=copyright&source=gbs_pub_info_r#v=onepage&q&f=false},

year  = {2017},

date = {2017-05-01},

booktitle = {Proceedings of the GIFTSym5},

pages = {23–35},

publisher = {ARL},

address = {Orlando, Florida},

abstract = {As intelligent tutoring systems (ITS) increasingly need to interoperate and co-exist, emerging systems have transitioned toward service-oriented designs to enable modularity and composability of tutoring components made and/or maintained by different research and development groups. However, as a research community, we have still not reached a point where it is trivial for a new service to be added into a system like the Generalized Intelligent Framework for Tutoring (GIFT; Sottilare, Goldberg, Brawner, & Holden, 2012). In an early paper considering this issue with respect to the GIFT architecture (Nye & Morrison, 2013), we proposed addressing this issue by building toward a lightweight multi-agent archi-tecture where certain services act as autonomous agents: “a system situated within and a part of an environment that senses that environment and acts on it, over time, in pursuit of its own agenda and so as to affect what it senses in the future” (Franklin & Graesser, 1997; p. 25). In our work in progress described here, we discuss how we are approaching the opportunity to build such capabilities into GIFT. The high level goals of our work are targeting two core goals for GIFT: A) to be a lightweight framework that will expand access to and use of ITS and B) to help GIFT to increase the intelligence and effectiveness of its services based on data over time. We are currently targeting the first goal, which will underpin the second goal. However, what does it mean to be a lightweight framework? In this context, a “lightweight framework” is framed as minimizing the following criteria: (1) hardware requirements, (2) software expertise to design services, (3) software expertise to use existing services, (4) software expertise to stand up the message-passing layer between agents, and (5) a minimal working message ontology (Nye & Morrison, 2013). Since our original paper four years ago, GIFT has made significant strides in reducing barriers related to hardware by building a cloud-based version and software expertise to use GIFT services through authoring tools. It has also developed a growing ontology of messages (e.g., https://gifttutoring.org/projects/gift/wiki/Interface_Control_Document_2016-1). With that said, despite now-extensive documentation, designing new services for GIFT is still not trivial and strong expertise is required to pass messages between GIFT modules and agents (either internal or external). To address these issues, the Building a Backbone project is working toward agent-oriented designs that build on GIFT's existing service-oriented framework. By moving from services toward agents, modules will be able to act more autonomously, enabling capabilities such as plug-and-play, hotswapping, and selecting between multiple services providing the same capabilities. These new capabilities are intended to reduce barriers to building new GIFT-compatible services and also to integrating GIFT with other service-oriented ecosystems. The first steps toward these capabilities are an ontology mapping service and an initial integration that combines GIFT, the Virtual Human Toolkit core framework for agents, and the SuperGLU framework for adding agent-oriented capabilities for coordinating services. This paper reports on work to date, with an emphasis on target capabilities, design decisions, challenges, and open research questions for this work.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{core_learning_2016,

title = {Learning, Adaptive Support, Student Traits, and Engagement in Scenario-Based Learning},

author = {Mark G. Core and Kallirroi Georgila and Benjamin D. Nye and Daniel Auerbach and Zhi Fei Liu and Richard DiNinni},

url = {http://www.iitsecdocs.com/search},

year  = {2016},

date = {2016-11-01},

booktitle = {Proceedings from the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2016},

publisher = {National Training and Simulation Association},

address = {Orlando, FL},

abstract = {Scenario-based training systems pose an especially difficult challenge for an intelligent tutoring system (ITS). In addition to the basic problems of deciding when to intervene and what guidance to provide, the ITS must decide whether to give guidance directly (e.g., a hint message), indirectly through positive/negative results in the scenario, or to delay guidance until a post-scenario review session. There are a number of factors that an adaptive ITS should consider and we use self-report survey instruments to investigate the relationship between traits, learning strategies, expectations, learner behaviors derived from log files, post-use perceptions of the system, and pre-test and post-test results. We use the ELITE Lite Counseling training system as a testbed for our experiments. This system uses virtual role players to allow learners to practice leadership counseling skills, and is in use at the United States Military Academy (USMA). This paper analyzes two data sets. We collected data from local university students, a non-military population of roughly the same age as USMA Cadets using the system. For these local participants, we could administer surveys and pre-tests and post-tests, and collect log files recording clicks made while using ELITE Lite. The second data set comes from USMA itself but is limited to log files. In both populations, the ITS’s hints are effective at boosting scenario performance, and for the university students, the overall experience promoted learning, and survey results suggest that higher levels of organization in study habits may lead to greater learning with ELITE Lite. For the USMA Cadets, ELITE Lite is part of their Military Leadership course rather than an experiment, which could explain why we found higher scenario performance on average than the non-military population, and more use of the post-scenario review feature.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mcalinden_cost-effective_2016,

title = {Cost-Effective Strategies for Producing Engaging Online Courseware},

author = {Ryan McAlinden and Sin-Hwa Kang and Benjamin Nye and Artemisa Phillips and Julia Campbell and Stephan L. Goldberg},

url = {http://www.iitsecdocs.com/search},

year  = {2016},

date = {2016-11-01},

booktitle = {Proceedings from the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2016},

publisher = {National Training and Simulation Association},

address = {Orlando, FL},

abstract = {As distributed learning (dL) and computer-based training (CBT) continue to proliferate, the methods of delivery often remain unengaging and bland for participants. Though many of the leaders in commercial online learning have improved their delivery style and quality in recent years, they continue to fall short in terms of user engagement and satisfaction. PowerPoint regurgitation and video lectures are commonplace and leave end users uninspired and wanting more. This paper discusses results from an ongoing research project, Captivating Virtual Instruction for Training (CVIT), which is aimed at understanding and improving dL through a series of recommendations and best practices for promoting and enhancing student engagement online. Though the central focus is on engagement, and how that translates to learning potential, a third variable (cost) has been examined to understand the financial and resource impacts on making content more interesting (i.e. the return on investment, or ROI). The paper presents findings from a 3-year long experiment comparing existing dL methods and techniques both within and outside of the Army. The project developed two dL versions of an existing Army course (Advanced Situational Awareness-Basic (ASA-B)) – the first was designed around producing material that was as engaging and as immersive as possible within a target budget; the second was a scaled-down version using more traditional, yet contemporary dL techniques (PowerPoint recital, video lectures). The two were then compared along three dimensions– engagement, learning and cost. The findings show that improved engagement in distributed courseware is possible without breaking the bank, though the returns on learning with these progressive approaches remain inconclusive. More importantly, it was determined that the quality and experience of the designers, production staff, writers, animators, programmers, and others cannot be underestimated, and that the familiar phrase – ‘you get what you pay for’ is as true with online learning as it is with other areas of content design and software development.},

keywords = {ARL, DoD, Learning Sciences, MedVR, MxR, STG, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{swartout_designing_2016,

title = {Designing a Personal Assistant for Life-Long Learning (PAL3)},

author = {William Swartout and Benjamin D. Nye and Arno Hartholt and Adam Reilly and Arthur C. Graesser and Kurt VanLehn and Jon Wetzel and Matt Liewer and Fabrizio Morbini and Brent Morgan and Lijia Wang and Grace Benn and Milton Rosenberg},

url = {http://www.aaai.org/ocs/index.php/FLAIRS/FLAIRS16/paper/view/12793},

isbn = {978-1-57735-756-8},

year  = {2016},

date = {2016-05-01},

booktitle = {Proceedings of The Twenty-Ninth International Flairs Conference},

pages = {491–496},

publisher = {AAAI Press},

address = {Key Largo, FL},

abstract = {Learners’ skills decay during gaps in instruction, since they lack the structure and motivation to continue studying. To meet this challenge, the PAL3 system was designed to accompany a learner throughout their career and mentor them to build and maintain skills through: 1) the use of an embodied pedagogical agent (Pal), 2) a persistent learning record that drives a student model which estimates forgetting, 3) an adaptive recommendation engine linking to both intelligent tutors and traditional learning resources, and 4) game-like mechanisms to promote engagement (e.g., leaderboards, effort-based point rewards, unlocking customizations). The design process for PAL3 is discussed, from the perspective of insights and revisions based on a series of formative feedback and evaluation sessions.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{chad_lane_situated_2015,

title = {Situated Pedagogical Authoring: Authoring Intelligent Tutors from a Student’s Perspective},

author = {H. Chad Lane and Mark G. Core and Matthew J. Hays and Daniel Auerbach and Milton Rosenberg},

url = {http://ict.usc.edu/pubs/Situated%20Pedagogical%20Authoring-Authoring%20Intelligent.pdf},

isbn = {978-3-319-19772-2 978-3-319-19773-9},

year  = {2015},

date = {2015-06-01},

booktitle = {Artificial Intelligence in Education},

volume = {9112},

pages = {195–204},

publisher = {Springer International Publishing},

address = {Madrid, Spain},

abstract = {We describe the Situated Pedagogical Authoring (SitPed) system that seeks to allow non-technical authors to create ITS content for soft-skills training, such as counseling skills. SitPed is built on the assertion that authoring tools should use the learner’s perspective to the greatest extent possible. SitPed provides tools for creating tasks lists, authoring assessment knowledge, and creating tutor messages. We present preliminary findings of a two-phase study comparing authoring in SitPed to an ablated version of the same system and a spreadsheet-based control. Findings suggest modest advantages for SitPed in terms of the quality of the authored content and student learning.},

keywords = {Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_automated_2015,

title = {Automated Session-Quality Assessment for Human Tutoring Based on Expert Ratings of Tutoring Success},

author = {Benjamin D. Nye and Donald M. Morrison and Borhan Samei},

url = {http://ict.usc.edu/pubs/Automated%20Session-Quality%20Assessment%20for%20Human%20Tutoring%20Based%20on%20Expert%20Ratings%20of%20Tutoring%20Success.pdf},

year  = {2015},

date = {2015-06-01},

booktitle = {Proceedings of Educational Data Mining (EDM) 2015},

pages = {195–202},

publisher = {Springer},

address = {Madrid, Spain},

abstract = {Archived transcripts from tens of millions of online human tutoring sessions potentially contain important knowledge about how online tutors help, or fail to help, students learn. However, without ways of automatically analyzing these large corpora, any knowledge in this data will remain buried. One way to approach this issue is to train an estimator for the learning e⬚ectiveness of an online tutoring interaction. While significant work has been done on automated assessment of student responses and artifacts (e.g., essays), automated assessment has not traditionally automated assessments of human-to-human tutoring sessions. In this work, we trained a model for estimating tutoring session quality based on a corpus of 1438 online tutoring sessions rated by expert tutors. Each session was rated for evidence of learning (outcomes) and educational soundness (process). Session features for this model included dialog act classifcations, mode classifcations (e.g., Scaffolding), statistically distinctive subsequences of such classifcations, dialog initiative (e.g., statements by tutor vs. student), and session length. The model correlated more highly with evidence of learning than educational soundness ratings, in part due to the greater difficulty of classifying tutoring modes. This model was then applied to a corpus of 242k online tutoring sessions, to examine the relationships between automated assessments and other available metadata (e.g., the tutor's self-assessment). On this large corpus, the automated assessments followed similar patterns as the expert rater's assessments, but with lower overall correlation strength. Based on the analyses presented, the assessment model for online tutoring sessions emulates the ratings of expert human tutors for session quality ratings with a reasonable degree of accuracy.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hays_must_2013,

title = {Must Feedback Disrupt Presence in Serious Games?},

author = {Matthew Jensen Hays and H. Chad Lane and Daniel Auerbach},

url = {http://ict.usc.edu/pubs/Must%20Feedback%20Disrupt%20Presence%20in%20Serious%20Games.pdf},

year  = {2013},

date = {2013-07-01},

booktitle = {Workshop on Formative Feedback in Interactive Learning Environments at the International Conference on Artificial Intelligence in Education},

address = {Memphis, TN},

abstract = {Serious games are generally designed with two goals in mind: promoting learning and creating compelling and engaging experiences (sometimes termed a sense of presence). Presence itself is believed to promote learning, but serious games often attempt to further increase pedagogical value. One way to do so is to use an intelligent tutoring system (ITS) to provide feedback during gameplay. Some researchers have expressed concern that, because feedback from an ITS is often extrinsic (i.e., it operates outside of the primary game mechanic), attending to it disrupts players’ sense of presence. As a result, learning may be unintentionally hindered by an ITS. However, the most beneficial conditions of instruction are often counterintuitive; in this paper, we challenge the assumption that feedback during learning hinders sense of presence. Across three experiments, we examined how an ITS that provided extrinsic feedback during a serious game affected presence. Across different modalities and conditions, we found that feedback and other ITS features do not always affect presence. Our results suggest that it is possible to provide extrinsic feedback in a serious game without detracting from the immersive power of the game itself.},

keywords = {Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{lane_effects_2013,

title = {The Effects of a Pedagogical Agent for Informal Science Education on Learner Behaviors and Self-efficacy},

author = {H. Chad Lane and Clara Cahill and Susan Foutz and Daniel Auerbach and Dan Noren and Catherine Lussenhop and William Swartout},

url = {http://ict.usc.edu/pubs/The%20Effects%20of%20a%20Pedagogical%20Agent%20for%20Informal%20Science%20Education%20on%20Learner%20Behaviors%20and%20Self-efficacy.pdf},

doi = {10.1007/978-3-642-39112-5_32},

isbn = {978-3-642-39111-8},

year  = {2013},

date = {2013-07-01},

booktitle = {Artificial Intelligence in Education},

volume = {7926},

pages = {309–318},

address = {Memphis, TN},

series = {Lecture Notes on Computer Science},

abstract = {We describe Coach Mike, an animated pedagogical agent for informal computer science education, and report findings from two experiments that provide initial evidence for the efficacy of the system. In the first study, we found that Coach Mike’s presence led to 20% longer holding times, increased acceptance of programming challenges, and reduced misuse of the exhibit, but had limited cumulative impact on attitudes, awareness, and knowledge beyond what the host exhibit already achieved. In the second study, we compared two different versions of Coach Mike and found that the use of enthusiasm and selfregulatory feedback led to greater self-efficacy for programming.⬚},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hays_can_2012,

title = {Can Role-Play with Virtual Humans Teach Interpersonal Skills?},

author = {Matthew Hays and Julia Campbell and Matthew Trimmer and Joshua Poore and Andrea Webb and Charles Stark and Teresa King},

url = {http://ict.usc.edu/pubs/Can%20Role-Play%20with%20Virtual%20Humans%20Teach%20Interpersonal%20Skills.pdf},

year  = {2012},

date = {2012-12-01},

booktitle = {Interservice/Industry Training, Simulation and Education Conference (I/ITSEC)},

address = {Orlando, FL},

abstract = {Interpersonal and counseling skills are essential to Officers’ ability to lead (Headquarters, Department of the Army, 2006, 2008, 2011). We developed a cognitive framework and an immersive training experience—the Immersive Naval Officer Training System (INOTS)—to help Officers learn and practice these skills (Campbell et al., 2011). INOTS includes up-front instruction about the framework, vignette-based demonstrations of its application, a roleplay session with a virtual human to practice the skills, and a guided after-action review (AAR). A critical component of any training effort is the assessment process; we conducted both formative and summative assessments of INOTS. Our formative assessments comprised surveys as well as physiological sensor equipment. Data from these instruments were used to evaluate how engaging the virtual-human based practice session was. We compared these data to a gold standard: a practice session with a live human role-player. We found that the trainees took the virtual-human practice session seriously—and that interacting with the virtual human was just as engaging as was interacting with the live human role-player. Our summative assessments comprised surveys as well as behavioral measures. We used these data to evaluate learning produced by the INOTS experience. In a pretestposttest design, we found reliable gains in the participants' understanding of and ability to apply interpersonal skills, although the limited practice with the virtual human did not provide additional immediate benefits. This paper details the development of our assessment approaches, the experimental procedures that yielded the data, and our results. We also discuss the implications of our efforts for the future design of assessments and training systems.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{roque_using_2011,

title = {Using virtual tour behavior to build dialogue models for training review},

author = {Antonio Roque and Dusan Jan and Mark Core and David Traum},

url = {http://ict.usc.edu/pubs/Using%20virtual%20tour%20behavior%20to%20build%20dialogue%20models%20for%20training%20review.pdf},

year  = {2011},

date = {2011-09-01},

booktitle = {Proceedings of the 11th International conference on Intelligent Virtual Agents (IVA 2011)},

address = {Reykjavík, Iceland},

abstract = {We develop an intelligent agent that builds a user model of a learner during a tour of a virtual world. The user model is based on the learner's answers to questions during the tour. A dialogue model for a simulated instructor is tailored to the individual learner based upon this user model. We describe an evaluation to track system accuracy and user perceptions.},

keywords = {Learning Sciences, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{campbell_developing_2011,

title = {Developing INOTS to Support Interpersonal Skills Practice},

author = {Julia Campbell and Mark Core and Ron Artstein and Lindsay Armstrong and Arno Hartholt and Cyrus A. Wilson and Kallirroi Georgila and Fabrizio Morbini and Kip Haynes and Dave Gomboc and Mike Birch and Jonathan Bobrow and H. Chad Lane and Jillian Gerten and Anton Leuski and David Traum and Matthew Trimmer and Rich DiNinni and Matthew Bosack and Timothy Jones and Richard E. Clark and Kenneth A. Yates},

url = {http://ict.usc.edu/pubs/Developing%20INOTS%20to%20Support%20Interpersonal%20Skills%20Practice.pdf},

year  = {2011},

date = {2011-03-01},

booktitle = {IEEE Aerospace Conference},

abstract = {The Immersive Naval Officer Training System (INOTS) is a blended learning environment that merges traditional classroom instruction with a mixed reality training setting. INOTS supports the instruction, practice and assessment of interpersonal communication skills. The goal of INOTS is to provide a consistent training experience to supplement interpersonal skills instruction for Naval officer candidates without sacrificing trainee throughput and instructor control. We developed an instructional design from cognitive task analysis interviews with experts to serve as a framework for system development. We also leveraged commercial student response technology and research technologies including natural language recognition, virtual humans, realistic graphics, intelligent tutoring and automated instructor support tools. In this paper, we describe our methodologies for developing a blended learning environment, and our challenges adding mixed reality and virtual human technologies to a traditional classroom to support interpersonal skills training.},

keywords = {Learning Sciences, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{buckwalter_analyzing_2011,

title = {Analyzing the Impact of Stress: A Comparison Between a Factor Analytic and a Composite Measurement of Allostatic Load},

author = {John Galen Buckwalter and Albert Rizzo and Bruce Sheffield John and Lisa Finlay and Andrew Wong and Ester Chin and John Wellman and Stephanie Smolinski},

url = {http://ict.usc.edu/pubs/Analyzing%20the%20Impact%20of%20Stress-%20A%20Comparison%20Between%20a%20Factor%20Analytic%20and%20a%20Composite%20Measurement%20of%20Allostatic%20Load.pdf},

year  = {2011},

date = {2011-01-01},

booktitle = {Interservice/Industry Training, Simulation and Education Conference (I/ITSEC)},

address = {Orlando, FL},

abstract = {Stress is possibly the hallmark characteristic of the current conflicts confronting the United States. Extended and repeated deployments require the ability on the part of war-fighters to effectively process stress in ways never before routinely encountered. Stress is well defined as a series of psychological and physiological processes that occur in response to a stressor, or the perception of stress. The physiological response to stress follows an identified path, a robust neuroendocrine response leads to responses in the cardiovascular, metabolic, renal, inflammatory and immune systems. After a stress response, the body's natural tendency is to return to a steady state, a process called allostasis. If the body is not effective in returning to homeostasis, or if the environment is such that stress is repeated, markers of dysfunction may be apparent in the physiological systems that respond to stress. A method of measuring multiple biomarkers of stress responsive systems and determining who shows consistent evidence of dysfunction was developed by Bruce McEwen and labeled allostatic load (AL). AL is most frequently measured by developing a level of risk for each biomarker and obtaining an AL score for the number of biomarkers the criterion for risk is met. This provides a single, equal-weighted measure of AL and does not allow for the identification of multi-systems. We employed a principal component factor analysis on a set of biomarkers and scored each factor using unit weighting. We compared the predictive power of 7 obliquely rotated factors to that of a composite AL marker. The set of factors predicted more of the variance in measures of depression, anxiety, and medical outcomes, it also provided evidence of the systems most involved in the development of pathology. The results confirm that AL is best analyzed as a multi-system construct. Not only does this predict more variance, it also provides suggestions as to the mechanisms underlying stress related disorders.},

keywords = {Learning Sciences, MedVR},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{swartout_virtual_2010,

title = {Virtual Museum Guides Demonstration},

author = {William Swartout and David Traum and Ron Artstein and Dan Noren and Paul Debevec and Kerry Bronnenkant and Josh Williams and Anton Leuski and Shrikanth Narayanan and Diane Piepol and H. Chad Lane and Jacquelyn Morie and Priti Aggarwal and Matt Liewer and Jen-Yuan Chiang and Jillian Gerten and Selina Chu and Kyle White},

url = {http://ict.usc.edu/pubs/Virtual%20Museum%20Guides%20Demonstration.pdf},

year  = {2010},

date = {2010-12-01},

booktitle = {IEEE Workshop on Spoken Language Technology},

address = {Berkeley, CA},

keywords = {Graphics, Learning Sciences, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{swartout_ada_2010,

title = {Ada and Grace: Toward Realistic and Engaging Virtual Museum Guides},

author = {William Swartout and David Traum and Ron Artstein and Dan Noren and Paul Debevec and Kerry Bronnenkant and Josh Williams and Anton Leuski and Shrikanth Narayanan and Diane Piepol and H. Chad Lane and Jacquelyn Morie and Priti Aggarwal and Matt Liewer and Jen-Yuan Chiang and Jillian Gerten and Selina Chu and Kyle White},

url = {http://ict.usc.edu/pubs/ada%20and%20grace.pdf},

year  = {2010},

date = {2010-09-01},

booktitle = {Proceedings of the 10th International Conference on Intelligent Virtual Agents (IVA 2010)},

address = {Philadelphia, PA},

abstract = {To increase the interest and engagement of middle school students in science and technology, the InterFaces project has created virtual museum guides that are in use at the Museum of Science, Boston. The characters use natural language interaction and have near photoreal appearance to increase and presents reports from museum staff on visitor reaction},

keywords = {Graphics, Learning Sciences, Virtual Humans, Virtual Worlds},

pubstate = {published},

tppubtype = {inproceedings}

}