Publications

@inproceedings{adami_immersive_2020,

title = {An Immersive Virtual Learning Environment for Worker-Robot Collaboration on Construction Sites},

author = {Pooya Adami and Burcin Becerik-Gerber and Lucio Soibelman and Tenzin Doleck and Yasemin Copur-Gencturk and Gale Lucas},

url = {https://ieeexplore.ieee.org/document/9383944/},

doi = {10.1109/WSC48552.2020.9383944},

isbn = {978-1-7281-9499-8},

year  = {2020},

date = {2020-12-01},

urldate = {2022-10-24},

booktitle = {2020 Winter Simulation Conference (WSC)},

pages = {2400–2411},

publisher = {IEEE},

address = {Orlando, FL, USA},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{bell_adapting_2020,

title = {Adapting Instruction by Measuring Engagement with Machine Learning in Virtual Reality Training},

author = {Benjamin Bell and Elaine Kelsey and Benjamin Nye and Winston (“Wink”) Bennett},

editor = {Robert A. Sottilare and Jessica Schwarz},

url = {https://link.springer.com/chapter/10.1007/978-3-030-50788-6_20},

doi = {10.1007/978-3-030-50788-6_20},

isbn = {978-3-030-50788-6},

year  = {2020},

date = {2020-01-01},

booktitle = {Adaptive Instructional Systems},

pages = {271–282},

publisher = {Springer International Publishing},

address = {Cham},

series = {Lecture Notes in Computer Science},

abstract = {The USAF has established a new approach to Specialized Undergraduate Pilot Training (SUPT) called Pilot Training Next (PTN) that integrates traditional flying sorties with VR-enabled ground-based training devices and data-driven proficiency tracking to achieve training efficiencies, improve readiness, and increase throughput. Eduworks and USC’s Institute for Creative Technologies are developing machine learning (ML) models that can measure user engagement during any computer-mediated training (simulation, courseware) and offer recommendations for restoring lapses in engagement. We are currently developing and testing this approach, called the Observational Motivation and Engagement Generalized Appliance (OMEGA) in a PTN context. Two factors motivate this work. First, one goal of PTN is for an instructor pilot (IP) to simultaneously monitor multiple simulator rides. Being alerted to distraction, attention and engagement can help an IP manage multiple students at the same time, with recommendations for restoring engagement providing further instructional support. Second, the virtual environment provides a rich source of raw data that machine learning models can use to associate user activity with user engagement. We have created a testbed for data capture in order to construct the ML models, based on theoretical foundations we developed previously. We are running pilots through multiple PTN scenarios and collecting formative data from instructors to evaluate the utility of the recommendations OMEGA generates regarding how lapsed engagement can be restored. We anticipate findings that validate the use of ML models for learning to detect engagement from the rich data sources characteristic of virtual environments. These findings will be applicable across a broad range of conventional and VR training applications.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{georgila_using_2019,

title = {Using Reinforcement Learning to Optimize the Policies of an Intelligent Tutoring System for Interpersonal Skills Training},

author = {Kallirroi Georgila and Mark G Core and Benjamin D Nye and Shamya Karumbaiah and Daniel Auerbach and Maya Ram},

url = {http://www.ifaamas.org/Proceedings/aamas2019/pdfs/p737.pdf},

year  = {2019},

date = {2019-05-01},

booktitle = {Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems},

pages = {9},

publisher = {IFAAMAS},

address = {Montreal, Canada},

abstract = {Reinforcement Learning (RL) has been applied successfully to Intelligent Tutoring Systems (ITSs) in a limited set of well-defined domains such as mathematics and physics. This work is unique in using a large state space and for applying RL to tutoring interpersonal skills. Interpersonal skills are increasingly recognized as critical to both social and economic development. In particular, this work enhances an ITS designed to teach basic counseling skills that can be applied to challenging issues such as sexual harassment and workplace conflict. An initial data collection was used to train RL policies for the ITS, and an evaluation with human participants compared a hand-crafted ITS which had been used for years with students (control) versus the new ITS guided by RL policies. The RL condition differed from the control condition most notably in the strikingly large quantity of guidance it provided to learners. Both systems were effective and there was an overall significant increase from pre- to post-test scores. Although learning gains did not differ significantly between conditions, learners had a significantly higher self-rating of confidence in the RL condition. Confidence and learning gains were both part of the reward function used to train the RL policies, and it could be the case that there was the most room for improvement in confidence, an important learner emotion. Thus, RL was successful in improving an ITS for teaching interpersonal skills without the need to prune the state space (as previously done).},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{davis_enhancements_2019,

title = {Enhancements for Homeschooling and ADL: Virtual Humans, Technologies and Insights},

author = {Dan M Davis and Nancy L H Young and Mark C Davis and North Carolina},

url = {http://www.modsimworld.org/conference-papers/2019},

year  = {2019},

date = {2019-04-01},

booktitle = {Proceedings of MODSIM World 2019},

pages = {12},

address = {Norfolk, VA},

abstract = {Homeschooling and DoD Advanced Distributed Learning (ADL) have many goals in common, so increasing the collaborative research and collegial information exchange between their respective communities would be mutually advantageous. The emerging capabilities of virtual humans provide a useful prototype of how both homeschooling and ADL can benefit from emerging technological advances. This paper begins with an examination of the home schooling movement in the United States, including a review of its foundations, demographics, results and trends. In examining the goals of homeschooling parents, the four major reasons cited by at least half of those parents are considered and explicated: desire to find environment most compatible to users, provision of ethics foundations, inclusion of accountability instruction and dissatisfaction with other pedagogical approaches. Also meriting review are the hurdles faced by homeschool teachers and students, followed by an item-by-item comparison with analogous challenges for ADL provisioners and learners. A short analysis of the constraints on the two communities focuses on similarities and differences between family limitations and defense organization restrictions. The authors then present data on the current scope, instantiations, and achievements of the two efforts. Many of the technologies currently in use are reviewed and discussed, concentrating on computer-aided education and distributed learning. Emerging technologies based on artificial intelligence, natural language processing, and virtual humans are described and considered. Their uses in various contexts provide sufficient data to quantify the impact on subjects and the authors adduce findings from research to support their thesis that increased use of these technologies would be beneficial both to homeschooled students and to DoD Learners. The paper closes with an evaluation of the arc of current research, the recognition of prenascent capabilities (e.g. quantum computing), the burgeoning needs of both communities, and the need to nurture a synergistic exchange between homeschool advocates and ADL architects.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{davis_pedagogical_2019,

title = {Pedagogical Tools to Enhance Analytic Skills: Interactive Virtual Tutorial Environments},

author = {Dan M Davis and Christi L Phelps and Frederica J Stassi},

url = {http://www.modsimworld.org/conference-papers/2019},

year  = {2019},

date = {2019-04-01},

booktitle = {Proceedings of MODSIM World 2019},

pages = {12},

address = {Norfolk, VA},

abstract = {This paper examines the use of literature studies to enhance communication and critical thinking skills in technical students through the application of emerging Virtual Reality (VR) technologies to enable that pedagogical approach. The current state of analytic skills among students in Science, Technology, Engineering and Mathematics (STEM) tracks are outlined, focusing on the critical years in secondary schools. Their prospective needs as they advance into tertiary education and the needs of the technical community for improvement are presented. The requirements flowing from that analysis will be discussed in the light of programs implemented at the Sato Academy, with reports of both successes and missteps. In some detail, the use of the study of literature is described and discussed. The authors present their case for constructivist and Socratic approaches to fully engage and effectively inculcate communication proficiency, including conformance with standards, e.g. Next Generation Science Standards (NGSS). These results are then compared to the demands of college and professional leaders who are currently being burdened with having to provide disruptive remedial efforts. The methods found to be successful are considered, both in terms of their application and their extensibility to other fields. Also highlighted will be areas in which time and personnel constraints hindered achievement. A number of possible responses to these impediments will be presented, evaluating the feasibility of each. The paper will then focus on the advances in virtual humans and conversational avatars. Recent research into using large libraries of video-clips to create engaging on-line virtual tutorial conversations will be presented. Data as to the receptivity of students to conversing with computer-generated interlocutors is presented, along with a discussion as to how this technology is applicable to teaching the analysis of literature. The benefits of and the barriers to virtual tutorial environments are outlined and analyzed.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hampton_mitigating_2018,

title = {Mitigating Knowledge Decay from Instruction with Voluntary Use of an Adaptive Learning System},

author = {Andrew J. Hampton and Benjamin D. Nye and Philip I. Pavlik and William R. Swartout and Arthur C. Graesser and Joseph Gunderson},

url = {http://link.springer.com/10.1007/978-3-319-93846-2_23},

doi = {10.1007/978-3-319-93846-2_23},

isbn = {978-3-319-93845-5 978-3-319-93846-2},

year  = {2018},

date = {2018-06-01},

booktitle = {Proceedings of the International Conference on Artificial Intelligence in Education},

volume = {10948},

pages = {119–133},

publisher = {Springer International Publishing},

address = {London, UK},

abstract = {Knowledge decays across breaks in instruction. Learners lack the metacognition to self-assess their knowledge decay and effectively self-direct review, as well as lacking interactive exercises appropriate to their individual knowledge level. Adaptive learning systems offer the potential to mitigate these issues, by providing open learner models to facilitate learner’s understanding of their knowledge levels and by presenting personalized practice exercises. The current study analyzes differences in knowledge decay between learners randomly assigned to an intervention where they could use an adaptive system during a long gap between courses, compared with a control condition. The experimental condition used the Personal Assistant for Life-Long Learning (PAL3), a tablet-based adaptive learning system integrating multiple intelligent tutoring systems and conventional learning resources. It contained electronics content relevant to the experiment participants, Navy sailors who graduated from apprentice electronics courses (A-School) awaiting assignment to their next training (C-School). The study was conducted over one month, collecting performance data with a counterbalanced pre-, mid-, and post-test. The control condition exhibited the expected decay. The PAL3 condition showed a significant difference from the control, with no significant knowledge decay in their overall knowledge, despite substantial variance in usage for PAL3 (e.g., most of overall use in the first week, with fewer participants engaging as time went on). Interestingly, while overall decay was mitigated in PAL3, this result was primarily through gains in some knowledge offsetting losses in other knowledge. Overall, these results indicate that adaptive study tools can help prevent knowledge decay, even with voluntary usage.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_engaging_2018,

title = {Engaging with the Scenario: Affect and Facial Patterns from a Scenario-Based Intelligent Tutoring System},

author = {Benjamin D. Nye and Shamya Karumbaiah and S. Tugba Tokel and Mark G. Core and Giota Stratou and Daniel Auerbach and Kallirroi Georgila},

url = {http://link.springer.com/10.1007/978-3-319-93843-1_26},

doi = {10.1007/978-3-319-93843-1_26},

isbn = {978-3-319-93842-4 978-3-319-93843-1},

year  = {2018},

date = {2018-06-01},

booktitle = {Proceeding of the International Conference on Artificial Intelligence in Education},

volume = {10947},

pages = {352–366},

publisher = {Springer International Publishing},

address = {London, UK},

abstract = {Facial expression trackers output measures for facial action units (AUs), and are increasingly being used in learning technologies. In this paper, we compile patterns of AUs seen in related work as well as use factor analysis to search for categories implicit in our corpus. Although there was some overlap between the factors in our data and previous work, we also identified factors seen in the broader literature but not previously reported in the context of learning environments. In a correlational analysis, we found evidence for relationships between factors and self-reported traits such as academic effort, study habits, and interest in the subject. In addition, we saw differences in average levels of factors between a video watching activity, and a decision making activity. However, in this analysis, we were not able to isolate any facial expressions having a significant positive or negative relationship with either learning gain, or performance once question difficulty and related factors were also considered. Given the overall low levels of facial affect in the corpus, further research will explore different populations and learning tasks to test the possible hypothesis that learners may have been in a pattern of “Over-Flow” in which they were engaged with the system, but not deeply thinking about the content or their errors.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_mentorpal_2017,

title = {MentorPal: Interactive Virtual Mentors Based on Real-Life STEM Professionals},

author = {Benjamin D Nye and Nicholas J Kaimakis and Madhusudhan Krishnamachari and William Swartout and Julia Campbell and Clinton Anderson and Dan M Davis},

url = {http://www.iitsecdocs.com/search},

year  = {2017},

date = {2017-11-01},

booktitle = {Proceedings of the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2017},

publisher = {a2z, Inc.},

address = {Orlando, Florida},

abstract = {In an ideal world, all students could meet STEM role models as they explore different careers. However, events such as career fairs do not scale well: professionals have limited time and effective mentors are not readily available in all fields. The result is that students’ understanding is minimal about what professionals in STEM fields do every day, what education is needed, and even what STEM fields exist. Moreover, since in-person interactions rely on finding people engaged in current STEM careers, students may form career goals for stagnant fields rather than growing fields (e.g., projected workforce needs). To address this problem, we are designing a scalable tablet-based app that gives students the opportunity to converse with interactive recordings of real-life STEM professionals. These conversational virtual agents will emulate a question-and-answer session with STEM professionals who have Navy ties and who are engaging, enthusiastic, and effective mentors. These interactions will allow students to have a lifelike informational interview with a virtual agent whose responses are directly drawn from a specific real professional’s video-recorded interview. This work differs from prior research on career guides by capturing the experiences of a collection of unique mentors, which should be more authentic and engaging than a generic agent or resource which speaks only about the average experience. This paper will discuss the process of creating the first such virtual STEM mentor prototype, including the development of an extensive mentoring question bank (approximately 500 questions); key mentoring topics that intersect STEM, DoD, and civilian life; techniques for cost-effective recording of remote mentors; and the process of training and verifying a natural language dialogue model for answering and suggesting career questions. Finally, we conclude with implications, strengths, and drawbacks of virtualizing the experience of talking with specific mentors, from the perspectives of efficacy, scalability, and maintainability.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_analyzing_2017,

title = {Analyzing Learner Affect in a Scenario-Based Intelligent Tutoring System},

author = {Benjamin Nye and Shamya Karumbaiah and S. Tugba Tokel and Mark G. Core and Giota Stratou and Daniel Auerbach and Kallirroi Georgila},

url = {https://link.springer.com/chapter/10.1007/978-3-319-61425-0_60},

doi = {https://doi.org/10.1007/978-3-319-61425-0_60},

isbn = {978-3-319-61425-0},

year  = {2017},

date = {2017-06-01},

booktitle = {Proceedings of the International Conference on Artificial Intelligence in Education},

pages = {544–547},

publisher = {Springer},

address = {Wuhan, China},

abstract = {Scenario-based tutoring systems influence affective states due to two distinct mechanisms during learning: 1) reactions to performance feedback and 2) responses to the scenario context or events. To explore the role of affect and engagement, a scenario-based ITS was instrumented to support unobtrusive facial affect detection. Results from a sample of university students showed relatively few traditional academic affective states such as confusion or frustration, even at decision points and after poor performance (e.g., incorrect responses). This may show evidence of "over-flow," with a high level of engagement and interest but insufficient confusion/disequilibrium for optimal learning.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{fang_online_2017,

title = {Online Learning Persistence and Academic Achievement},

author = {Ying Fang and Yonghong Jade Xu and Benjamin Nye and Arthur Graesser and Philip Pavlik and Xiangen Hu},

url = {http://educationaldatamining.org/EDM2017/proc_files/papers/paper_114.pdf},

year  = {2017},

date = {2017-06-01},

booktitle = {Proceedings of Educational Data Mining (EDM) 2017},

pages = {312 – 317},

publisher = {EDM 2017},

address = {Wuhan, China},

abstract = {Student persistence in online learning environments has typically been studied at the macro-level (e.g., completion of an online course, number of academic terms completed, etc.). The current examines student persistence in an adaptive learning environment, ALEKS (Assessment and LEarning in Knowledge Spaces). Specifically, the study explores the relationship between students' academic achievement and their persistence during learning. By using archived data that included their math learning log data and performance on two standardized tests, we first explored student learning behavior patterns with regard to their persistence during learning. Clustering analysis identified three distinctive patterns of persistence-related learning behaviors: (1) High persistence and rare topic shifting; (2) Low persistence and frequent topic shifting; and (3) Moderate persistence and moderate topic shifting. We further explored the association between persistence and academic achievement. No significant differences were observed between academic achievement and the different learning patterns. We interpret this result in addition to a preliminary exploration of topic mastery trends, to suggest that wheel-spinning" behaviors coexist with persistence, and is ultimately not beneficial to learning.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_building_2017,

title = {Building a Backbone for Multi-Agent Tutoring in GIFT (Work in Progress)},

author = {Benjamin D. Nye and Daniel Auerbach and Tirth R. Mehta and Arno Hartholt},

url = {https://books.google.com/books?id=PwMtDwAAQBAJ&printsec=copyright&source=gbs_pub_info_r#v=onepage&q&f=false},

year  = {2017},

date = {2017-05-01},

booktitle = {Proceedings of the GIFTSym5},

pages = {23–35},

publisher = {ARL},

address = {Orlando, Florida},

abstract = {As intelligent tutoring systems (ITS) increasingly need to interoperate and co-exist, emerging systems have transitioned toward service-oriented designs to enable modularity and composability of tutoring components made and/or maintained by different research and development groups. However, as a research community, we have still not reached a point where it is trivial for a new service to be added into a system like the Generalized Intelligent Framework for Tutoring (GIFT; Sottilare, Goldberg, Brawner, & Holden, 2012). In an early paper considering this issue with respect to the GIFT architecture (Nye & Morrison, 2013), we proposed addressing this issue by building toward a lightweight multi-agent archi-tecture where certain services act as autonomous agents: “a system situated within and a part of an environment that senses that environment and acts on it, over time, in pursuit of its own agenda and so as to affect what it senses in the future” (Franklin & Graesser, 1997; p. 25). In our work in progress described here, we discuss how we are approaching the opportunity to build such capabilities into GIFT. The high level goals of our work are targeting two core goals for GIFT: A) to be a lightweight framework that will expand access to and use of ITS and B) to help GIFT to increase the intelligence and effectiveness of its services based on data over time. We are currently targeting the first goal, which will underpin the second goal. However, what does it mean to be a lightweight framework? In this context, a “lightweight framework” is framed as minimizing the following criteria: (1) hardware requirements, (2) software expertise to design services, (3) software expertise to use existing services, (4) software expertise to stand up the message-passing layer between agents, and (5) a minimal working message ontology (Nye & Morrison, 2013). Since our original paper four years ago, GIFT has made significant strides in reducing barriers related to hardware by building a cloud-based version and software expertise to use GIFT services through authoring tools. It has also developed a growing ontology of messages (e.g., https://gifttutoring.org/projects/gift/wiki/Interface_Control_Document_2016-1). With that said, despite now-extensive documentation, designing new services for GIFT is still not trivial and strong expertise is required to pass messages between GIFT modules and agents (either internal or external). To address these issues, the Building a Backbone project is working toward agent-oriented designs that build on GIFT's existing service-oriented framework. By moving from services toward agents, modules will be able to act more autonomously, enabling capabilities such as plug-and-play, hotswapping, and selecting between multiple services providing the same capabilities. These new capabilities are intended to reduce barriers to building new GIFT-compatible services and also to integrating GIFT with other service-oriented ecosystems. The first steps toward these capabilities are an ontology mapping service and an initial integration that combines GIFT, the Virtual Human Toolkit core framework for agents, and the SuperGLU framework for adding agent-oriented capabilities for coordinating services. This paper reports on work to date, with an emphasis on target capabilities, design decisions, challenges, and open research questions for this work.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{core_learning_2016,

title = {Learning, Adaptive Support, Student Traits, and Engagement in Scenario-Based Learning},

author = {Mark G. Core and Kallirroi Georgila and Benjamin D. Nye and Daniel Auerbach and Zhi Fei Liu and Richard DiNinni},

url = {http://www.iitsecdocs.com/search},

year  = {2016},

date = {2016-11-01},

booktitle = {Proceedings from the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2016},

publisher = {National Training and Simulation Association},

address = {Orlando, FL},

abstract = {Scenario-based training systems pose an especially difficult challenge for an intelligent tutoring system (ITS). In addition to the basic problems of deciding when to intervene and what guidance to provide, the ITS must decide whether to give guidance directly (e.g., a hint message), indirectly through positive/negative results in the scenario, or to delay guidance until a post-scenario review session. There are a number of factors that an adaptive ITS should consider and we use self-report survey instruments to investigate the relationship between traits, learning strategies, expectations, learner behaviors derived from log files, post-use perceptions of the system, and pre-test and post-test results. We use the ELITE Lite Counseling training system as a testbed for our experiments. This system uses virtual role players to allow learners to practice leadership counseling skills, and is in use at the United States Military Academy (USMA). This paper analyzes two data sets. We collected data from local university students, a non-military population of roughly the same age as USMA Cadets using the system. For these local participants, we could administer surveys and pre-tests and post-tests, and collect log files recording clicks made while using ELITE Lite. The second data set comes from USMA itself but is limited to log files. In both populations, the ITS’s hints are effective at boosting scenario performance, and for the university students, the overall experience promoted learning, and survey results suggest that higher levels of organization in study habits may lead to greater learning with ELITE Lite. For the USMA Cadets, ELITE Lite is part of their Military Leadership course rather than an experiment, which could explain why we found higher scenario performance on average than the non-military population, and more use of the post-scenario review feature.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mcalinden_cost-effective_2016,

title = {Cost-Effective Strategies for Producing Engaging Online Courseware},

author = {Ryan McAlinden and Sin-Hwa Kang and Benjamin Nye and Artemisa Phillips and Julia Campbell and Stephan L. Goldberg},

url = {http://www.iitsecdocs.com/search},

year  = {2016},

date = {2016-11-01},

booktitle = {Proceedings from the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2016},

publisher = {National Training and Simulation Association},

address = {Orlando, FL},

abstract = {As distributed learning (dL) and computer-based training (CBT) continue to proliferate, the methods of delivery often remain unengaging and bland for participants. Though many of the leaders in commercial online learning have improved their delivery style and quality in recent years, they continue to fall short in terms of user engagement and satisfaction. PowerPoint regurgitation and video lectures are commonplace and leave end users uninspired and wanting more. This paper discusses results from an ongoing research project, Captivating Virtual Instruction for Training (CVIT), which is aimed at understanding and improving dL through a series of recommendations and best practices for promoting and enhancing student engagement online. Though the central focus is on engagement, and how that translates to learning potential, a third variable (cost) has been examined to understand the financial and resource impacts on making content more interesting (i.e. the return on investment, or ROI). The paper presents findings from a 3-year long experiment comparing existing dL methods and techniques both within and outside of the Army. The project developed two dL versions of an existing Army course (Advanced Situational Awareness-Basic (ASA-B)) – the first was designed around producing material that was as engaging and as immersive as possible within a target budget; the second was a scaled-down version using more traditional, yet contemporary dL techniques (PowerPoint recital, video lectures). The two were then compared along three dimensions– engagement, learning and cost. The findings show that improved engagement in distributed courseware is possible without breaking the bank, though the returns on learning with these progressive approaches remain inconclusive. More importantly, it was determined that the quality and experience of the designers, production staff, writers, animators, programmers, and others cannot be underestimated, and that the familiar phrase – ‘you get what you pay for’ is as true with online learning as it is with other areas of content design and software development.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}