Publications

@inproceedings{roemmele_test-taking_2024,

title = {From Test-Taking to Test-Making: Examining LLM Authoring of Commonsense Assessment Items},

author = {Melissa Roemmele and Andrew Gordon},

url = {https://aclanthology.org/2024.findings-emnlp.299},

doi = {10.18653/v1/2024.findings-emnlp.299},

year  = {2024},

date = {2024-11-01},

urldate = {2024-12-05},

booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2024},

pages = {5193–5203},

publisher = {Association for Computational Linguistics},

address = {Miami, Florida, USA},

keywords = {DTIC, Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_generative_2023,

title = {Generative Large Language Models for Dialog-Based Tutoring: An Early Consideration of Opportunities and Concerns},

author = {Benjamin D Nye and Dillon Mee and Mark G Core},

url = {https://ceur-ws.org/Vol-3487/paper4.pdf},

year  = {2023},

date = {2023-07-01},

abstract = {After many years of relatively limited capabilities for generative language models, recent large language models (LLM’s) have demonstrated qualitatively better capabilities for understanding, synthesis, and inference on text. Due to the prominence of ChatGPT’s chat system, both the media and many educational developers have suggested using generative AI to directly tutor students. However, despite surface-level similarity between ChatGPT interactions and tutoring dialogs, generative AI has other strengths which may be substantially more relevant for intelligent tutoring (e.g., detecting misconceptions, improved language translation, content generation) and weaknesses that make it problematic for on-the-fly tutoring (e.g., hallucinations, lack of pedagogical training data). In this paper, we discuss how we are approaching generative LLM’s for tutoring dialogs, for problems such as multi- concept short answer grading and semi-supervised interactive content generation. This work shows interesting opportunities for prompt engineering approaches for short-answer classification, despite sometimes quirky behavior. The time savings for high-quality content generation for tutoring is not yet clear and further research is needed. The paper concludes with a consideration of longer-term equity and access in a world where essential capabilities require low-latency real-time connections to large, pay-peruse models. Risks and mitigating technologies for this kind of “AI digital divide” are discussed, including optimized / edge-computing LLM’s and using generative AI models as simulated students to train specialized tutoring models.},

keywords = {DTIC, Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_mentorstudio_2023,

title = {MentorStudio: Amplifying diverse voices through rapid, self-authorable virtual mentors},

author = {Benjamin D. Nye and Yuko Okado and Aaron Shiel and Kayla Carr and Milton Rosenberg and Enora Rice and Luke Ostrander and Megan Ju and Cassandra Gutierrez and Dilan Ramirez and Daniel Auerbach and Angelica Aguirre and William Swartout},

url = {https://zenodo.org/record/8226275},

doi = {10.5281/ZENODO.8226275},

year  = {2023},

date = {2023-07-01},

urldate = {2024-01-11},

publisher = {Zenodo},

abstract = {Mentoring promotes underserved students' STEM persistence but it is difficult to scale up. Virtual agents can amplify mentors' experiences to larger audiences, which is particularly important for mentors from under-represented backgrounds and for underserved students with less access to mentors. This paper introduces MentorStudio, an online platform that allows real-life mentors to self-record and publish video-based conversational virtual agents. MentorStudio's goals are to increase speed, scheduling flexibility, and autonomy in creating intelligent virtual mentors. MentorStudio platform components are introduced, along with initial feedback regarding usability and acceptance collected from 20 STEM mentors who recorded virtual mentors. Overall, the MentorStudio platform has good ease-of-use and acceptance among mentors and offers a platform capable of recording large number of mentors to expand their reach to an unlimited number of students.},

note = {Publisher: Zenodo},

keywords = {DTIC, Learning Sciences, UARC, Virtual Agents},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{deneve_weeks_tar_2022,

title = {TAR AR: Researching How Augmented Reality Activities Can Facilitate Visitor Learning at La Brea Tar Pits},

author = {Danaan DeNeve Weeks and Emily Lindsey and Matt Davis and Alana Kennedy and Benjamin Nye and David Nelson and Molly Porter and William Swartout and Gale Sinatra},

url = {https://gsa.confex.com/gsa/2022CD/webprogram/Paper373373.html},

year  = {2022},

date = {2022-03-01},

urldate = {2023-03-31},

publisher = {Geological Society of America},

keywords = {Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{dininni_sensing_2022,

title = {Sensing Human Signals of Motivation Processes During STEM Tasks},

author = {Richard DiNinni and Albert Rizzo},

editor = {Maria Mercedes Rodrigo and Noburu Matsuda and Alexandra I. Cristea and Vania Dimitrova},

doi = {10.1007/978-3-031-11647-6_28},

isbn = {978-3-031-11647-6},

year  = {2022},

date = {2022-01-01},

booktitle = {Artificial Intelligence in Education. Posters and Late Breaking Results, Workshops and Tutorials, Industry and Innovation Tracks, Practitioners’ and Doctoral Consortium},

pages = {163–167},

publisher = {Springer International Publishing},

address = {Cham},

series = {Lecture Notes in Computer Science},

abstract = {This paper outlines the linking of a multi-modal sensing platform with an Intelligent Tutoring System to perceive the motivational state of the learner during STEM tasks. Motivation is a critical element to learning but receives little attention in comparison to strategies related to cognitive processes. The EMPOWER project has developed a novel platform that offers researchers an opportunity to capture a learner’s multi-modal behavioral signals to develop models of motivation problems that can be used to develop best practice strategies for instructional systems.},

keywords = {DTIC, Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{herrick_using_2022,

title = {Using Augmented Reality (AR) to Bring the Past to Life in Informal Science Learning},

author = {Imogen Herrick and Gale Sinatra and Alana Kennedy and Benjamin Nye and William Swartout and Emily Lindsey},

url = {https://par.nsf.gov/biblio/10344989},

year  = {2022},

date = {2022-01-01},

booktitle = {National Association for Research in Science Teaching (NARST) International Conference},

abstract = {A key mission for museums is to engage a large and diverse public audience in science learning (Macdonald, 1997). To that end, science museums attempt to use immersive technologies in entertaining, socially oriented, and innovative ways. An example is the use of augmented reality (AR) to overlay virtual objects onto the real-world (Azuma, Baillot, Behringer, Feiner, Julier, & MacIntyre, 2001).We used a Design Based Research (DBR) approach to develop and test four features of an AR experience to promote place-based science learning in an museum setting. While quantitative differences were not found among conditions in knowledge gained, significant learning gains were seen from pre to post, illustrating the potential for place-based informal science learning. Incorporating AR technology into museum exhibits can update them with 21st tools to support visitor engagement in the learning experience. This research contributes to understanding of usability and logistical issues for different AR designs for a public, outdoor informal settings.},

keywords = {Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{liu_graph_2021,

title = {Graph Neural Network Based Behavior Prediction to Support Multi-Agent Reinforcement Learning in Military Training Simulations},

author = {Lixing Liu and Nikolos Gurney and Kyle McCullough and Volkan Ustun},

url = {https://ieeexplore.ieee.org/document/9715433/},

doi = {10.1109/WSC52266.2021.9715433},

isbn = {978-1-6654-3311-2},

year  = {2021},

date = {2021-12-01},

urldate = {2022-09-21},

booktitle = {2021 Winter Simulation Conference (WSC)},

pages = {1–12},

publisher = {IEEE},

address = {Phoenix, AZ, USA},

keywords = {DTIC, Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{davis_augment_2021,

title = {Augment Reality In Natural Hostory Museums: Impact on Visitor Engagement and Science Learning},

author = {Matt Davis},

url = {https://gsa.confex.com/gsa/2021AM/webprogram/Paper371425.html},

year  = {2021},

date = {2021-10-01},

urldate = {2023-03-31},

publisher = {GSA},

keywords = {AR, Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_acting_2021,

title = {Acting Engaged: Leveraging Play Persona Archetypes for Semi-Supervised Classification of Engagement},

author = {Benjamin D. Nye and Mark G. Core and Shikhar Jaiswa and Aviroop Ghosal and Daniel Auerbach},

url = {https://eric.ed.gov/?id=ED615498},

year  = {2021},

date = {2021-01-01},

urldate = {2023-03-31},

publisher = {International Educational Data Mining Society},

institution = {International Educational Data Mining Society},

abstract = {Engaged and disengaged behaviors have been studied across a variety of educational contexts. However, tools to analyze engagement typically require custom-coding and calibration for a system. This limits engagement detection to systems where experts are available to study patterns and build detectors. This work studies a new approach to classify engagement patterns without expert input, by using a play persona methodology where labeled archetype data is generated by novice testers acting out different engagement patterns in a system. Domain-agnostic task features (e.g., response time to an activity, scores/correctness, task difficulty) are extracted from standardized data logs for both archetype and authentic user sessions. A semi-supervised methodology was used to label engagement; bottom-up clusters were combined with archetype data to build a classifier. This approach was analyzed with a focus on cold-start performance on small samples, using two metrics: consistency with larger full-sample cluster assignments and stability of points staying in the same cluster once assigned. These were compared against a baseline of clustering without an incrementally trained classifier. Findings on a data set from a branching multiple-choice scenario-based tutoring system indicated that approximately 52 unlabeled samples and 51 play-test labeled samples were sufficient to classify holdout sessions at 85% consistency with a full set of 145 unsupervised samples. Additionally, alignment to play persona samples for the full set matched expert labels for clusters. Use-cases and limitations of this approach are discussed. [For the full proceedings, see ED615472.]},

note = {Publication Title: International Educational Data Mining Society 

ERIC Number: ED615498},

keywords = {Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{adami_immersive_2020,

title = {An Immersive Virtual Learning Environment for Worker-Robot Collaboration on Construction Sites},

author = {Pooya Adami and Burcin Becerik-Gerber and Lucio Soibelman and Tenzin Doleck and Yasemin Copur-Gencturk and Gale Lucas},

url = {https://ieeexplore.ieee.org/document/9383944/},

doi = {10.1109/WSC48552.2020.9383944},

isbn = {978-1-7281-9499-8},

year  = {2020},

date = {2020-12-01},

urldate = {2022-10-24},

booktitle = {2020 Winter Simulation Conference (WSC)},

pages = {2400–2411},

publisher = {IEEE},

address = {Orlando, FL, USA},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{bell_adapting_2020,

title = {Adapting Instruction by Measuring Engagement with Machine Learning in Virtual Reality Training},

author = {Benjamin Bell and Elaine Kelsey and Benjamin Nye and Winston (“Wink”) Bennett},

editor = {Robert A. Sottilare and Jessica Schwarz},

url = {https://link.springer.com/chapter/10.1007/978-3-030-50788-6_20},

doi = {10.1007/978-3-030-50788-6_20},

isbn = {978-3-030-50788-6},

year  = {2020},

date = {2020-01-01},

booktitle = {Adaptive Instructional Systems},

pages = {271–282},

publisher = {Springer International Publishing},

address = {Cham},

series = {Lecture Notes in Computer Science},

abstract = {The USAF has established a new approach to Specialized Undergraduate Pilot Training (SUPT) called Pilot Training Next (PTN) that integrates traditional flying sorties with VR-enabled ground-based training devices and data-driven proficiency tracking to achieve training efficiencies, improve readiness, and increase throughput. Eduworks and USC’s Institute for Creative Technologies are developing machine learning (ML) models that can measure user engagement during any computer-mediated training (simulation, courseware) and offer recommendations for restoring lapses in engagement. We are currently developing and testing this approach, called the Observational Motivation and Engagement Generalized Appliance (OMEGA) in a PTN context. Two factors motivate this work. First, one goal of PTN is for an instructor pilot (IP) to simultaneously monitor multiple simulator rides. Being alerted to distraction, attention and engagement can help an IP manage multiple students at the same time, with recommendations for restoring engagement providing further instructional support. Second, the virtual environment provides a rich source of raw data that machine learning models can use to associate user activity with user engagement. We have created a testbed for data capture in order to construct the ML models, based on theoretical foundations we developed previously. We are running pilots through multiple PTN scenarios and collecting formative data from instructors to evaluate the utility of the recommendations OMEGA generates regarding how lapsed engagement can be restored. We anticipate findings that validate the use of ML models for learning to detect engagement from the rich data sources characteristic of virtual environments. These findings will be applicable across a broad range of conventional and VR training applications.},

keywords = {Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{georgila_using_2019,

title = {Using Reinforcement Learning to Optimize the Policies of an Intelligent Tutoring System for Interpersonal Skills Training},

author = {Kallirroi Georgila and Mark G Core and Benjamin D Nye and Shamya Karumbaiah and Daniel Auerbach and Maya Ram},

url = {http://www.ifaamas.org/Proceedings/aamas2019/pdfs/p737.pdf},

year  = {2019},

date = {2019-05-01},

booktitle = {Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems},

pages = {9},

publisher = {IFAAMAS},

address = {Montreal, Canada},

abstract = {Reinforcement Learning (RL) has been applied successfully to Intelligent Tutoring Systems (ITSs) in a limited set of well-defined domains such as mathematics and physics. This work is unique in using a large state space and for applying RL to tutoring interpersonal skills. Interpersonal skills are increasingly recognized as critical to both social and economic development. In particular, this work enhances an ITS designed to teach basic counseling skills that can be applied to challenging issues such as sexual harassment and workplace conflict. An initial data collection was used to train RL policies for the ITS, and an evaluation with human participants compared a hand-crafted ITS which had been used for years with students (control) versus the new ITS guided by RL policies. The RL condition differed from the control condition most notably in the strikingly large quantity of guidance it provided to learners. Both systems were effective and there was an overall significant increase from pre- to post-test scores. Although learning gains did not differ significantly between conditions, learners had a significantly higher self-rating of confidence in the RL condition. Confidence and learning gains were both part of the reward function used to train the RL policies, and it could be the case that there was the most room for improvement in confidence, an important learner emotion. Thus, RL was successful in improving an ITS for teaching interpersonal skills without the need to prune the state space (as previously done).},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{davis_enhancements_2019,

title = {Enhancements for Homeschooling and ADL: Virtual Humans, Technologies and Insights},

author = {Dan M Davis and Nancy L H Young and Mark C Davis and North Carolina},

url = {http://www.modsimworld.org/conference-papers/2019},

year  = {2019},

date = {2019-04-01},

booktitle = {Proceedings of MODSIM World 2019},

pages = {12},

address = {Norfolk, VA},

abstract = {Homeschooling and DoD Advanced Distributed Learning (ADL) have many goals in common, so increasing the collaborative research and collegial information exchange between their respective communities would be mutually advantageous. The emerging capabilities of virtual humans provide a useful prototype of how both homeschooling and ADL can benefit from emerging technological advances. This paper begins with an examination of the home schooling movement in the United States, including a review of its foundations, demographics, results and trends. In examining the goals of homeschooling parents, the four major reasons cited by at least half of those parents are considered and explicated: desire to find environment most compatible to users, provision of ethics foundations, inclusion of accountability instruction and dissatisfaction with other pedagogical approaches. Also meriting review are the hurdles faced by homeschool teachers and students, followed by an item-by-item comparison with analogous challenges for ADL provisioners and learners. A short analysis of the constraints on the two communities focuses on similarities and differences between family limitations and defense organization restrictions. The authors then present data on the current scope, instantiations, and achievements of the two efforts. Many of the technologies currently in use are reviewed and discussed, concentrating on computer-aided education and distributed learning. Emerging technologies based on artificial intelligence, natural language processing, and virtual humans are described and considered. Their uses in various contexts provide sufficient data to quantify the impact on subjects and the authors adduce findings from research to support their thesis that increased use of these technologies would be beneficial both to homeschooled students and to DoD Learners. The paper closes with an evaluation of the arc of current research, the recognition of prenascent capabilities (e.g. quantum computing), the burgeoning needs of both communities, and the need to nurture a synergistic exchange between homeschool advocates and ADL architects.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{davis_pedagogical_2019,

title = {Pedagogical Tools to Enhance Analytic Skills: Interactive Virtual Tutorial Environments},

author = {Dan M Davis and Christi L Phelps and Frederica J Stassi},

url = {http://www.modsimworld.org/conference-papers/2019},

year  = {2019},

date = {2019-04-01},

booktitle = {Proceedings of MODSIM World 2019},

pages = {12},

address = {Norfolk, VA},

abstract = {This paper examines the use of literature studies to enhance communication and critical thinking skills in technical students through the application of emerging Virtual Reality (VR) technologies to enable that pedagogical approach. The current state of analytic skills among students in Science, Technology, Engineering and Mathematics (STEM) tracks are outlined, focusing on the critical years in secondary schools. Their prospective needs as they advance into tertiary education and the needs of the technical community for improvement are presented. The requirements flowing from that analysis will be discussed in the light of programs implemented at the Sato Academy, with reports of both successes and missteps. In some detail, the use of the study of literature is described and discussed. The authors present their case for constructivist and Socratic approaches to fully engage and effectively inculcate communication proficiency, including conformance with standards, e.g. Next Generation Science Standards (NGSS). These results are then compared to the demands of college and professional leaders who are currently being burdened with having to provide disruptive remedial efforts. The methods found to be successful are considered, both in terms of their application and their extensibility to other fields. Also highlighted will be areas in which time and personnel constraints hindered achievement. A number of possible responses to these impediments will be presented, evaluating the feasibility of each. The paper will then focus on the advances in virtual humans and conversational avatars. Recent research into using large libraries of video-clips to create engaging on-line virtual tutorial conversations will be presented. Data as to the receptivity of students to conversing with computer-generated interlocutors is presented, along with a discussion as to how this technology is applicable to teaching the analysis of literature. The benefits of and the barriers to virtual tutorial environments are outlined and analyzed.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hampton_mitigating_2018,

title = {Mitigating Knowledge Decay from Instruction with Voluntary Use of an Adaptive Learning System},

author = {Andrew J. Hampton and Benjamin D. Nye and Philip I. Pavlik and William R. Swartout and Arthur C. Graesser and Joseph Gunderson},

url = {http://link.springer.com/10.1007/978-3-319-93846-2_23},

doi = {10.1007/978-3-319-93846-2_23},

isbn = {978-3-319-93845-5 978-3-319-93846-2},

year  = {2018},

date = {2018-06-01},

booktitle = {Proceedings of the International Conference on Artificial Intelligence in Education},

volume = {10948},

pages = {119–133},

publisher = {Springer International Publishing},

address = {London, UK},

abstract = {Knowledge decays across breaks in instruction. Learners lack the metacognition to self-assess their knowledge decay and effectively self-direct review, as well as lacking interactive exercises appropriate to their individual knowledge level. Adaptive learning systems offer the potential to mitigate these issues, by providing open learner models to facilitate learner’s understanding of their knowledge levels and by presenting personalized practice exercises. The current study analyzes differences in knowledge decay between learners randomly assigned to an intervention where they could use an adaptive system during a long gap between courses, compared with a control condition. The experimental condition used the Personal Assistant for Life-Long Learning (PAL3), a tablet-based adaptive learning system integrating multiple intelligent tutoring systems and conventional learning resources. It contained electronics content relevant to the experiment participants, Navy sailors who graduated from apprentice electronics courses (A-School) awaiting assignment to their next training (C-School). The study was conducted over one month, collecting performance data with a counterbalanced pre-, mid-, and post-test. The control condition exhibited the expected decay. The PAL3 condition showed a significant difference from the control, with no significant knowledge decay in their overall knowledge, despite substantial variance in usage for PAL3 (e.g., most of overall use in the first week, with fewer participants engaging as time went on). Interestingly, while overall decay was mitigated in PAL3, this result was primarily through gains in some knowledge offsetting losses in other knowledge. Overall, these results indicate that adaptive study tools can help prevent knowledge decay, even with voluntary usage.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_engaging_2018,

title = {Engaging with the Scenario: Affect and Facial Patterns from a Scenario-Based Intelligent Tutoring System},

author = {Benjamin D. Nye and Shamya Karumbaiah and S. Tugba Tokel and Mark G. Core and Giota Stratou and Daniel Auerbach and Kallirroi Georgila},

url = {http://link.springer.com/10.1007/978-3-319-93843-1_26},

doi = {10.1007/978-3-319-93843-1_26},

isbn = {978-3-319-93842-4 978-3-319-93843-1},

year  = {2018},

date = {2018-06-01},

booktitle = {Proceeding of the International Conference on Artificial Intelligence in Education},

volume = {10947},

pages = {352–366},

publisher = {Springer International Publishing},

address = {London, UK},

abstract = {Facial expression trackers output measures for facial action units (AUs), and are increasingly being used in learning technologies. In this paper, we compile patterns of AUs seen in related work as well as use factor analysis to search for categories implicit in our corpus. Although there was some overlap between the factors in our data and previous work, we also identified factors seen in the broader literature but not previously reported in the context of learning environments. In a correlational analysis, we found evidence for relationships between factors and self-reported traits such as academic effort, study habits, and interest in the subject. In addition, we saw differences in average levels of factors between a video watching activity, and a decision making activity. However, in this analysis, we were not able to isolate any facial expressions having a significant positive or negative relationship with either learning gain, or performance once question difficulty and related factors were also considered. Given the overall low levels of facial affect in the corpus, further research will explore different populations and learning tasks to test the possible hypothesis that learners may have been in a pattern of “Over-Flow” in which they were engaged with the system, but not deeply thinking about the content or their errors.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_mentorpal_2017,

title = {MentorPal: Interactive Virtual Mentors Based on Real-Life STEM Professionals},

author = {Benjamin D Nye and Nicholas J Kaimakis and Madhusudhan Krishnamachari and William Swartout and Julia Campbell and Clinton Anderson and Dan M Davis},

url = {http://www.iitsecdocs.com/search},

year  = {2017},

date = {2017-11-01},

booktitle = {Proceedings of the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2017},

publisher = {a2z, Inc.},

address = {Orlando, Florida},

abstract = {In an ideal world, all students could meet STEM role models as they explore different careers. However, events such as career fairs do not scale well: professionals have limited time and effective mentors are not readily available in all fields. The result is that students’ understanding is minimal about what professionals in STEM fields do every day, what education is needed, and even what STEM fields exist. Moreover, since in-person interactions rely on finding people engaged in current STEM careers, students may form career goals for stagnant fields rather than growing fields (e.g., projected workforce needs). To address this problem, we are designing a scalable tablet-based app that gives students the opportunity to converse with interactive recordings of real-life STEM professionals. These conversational virtual agents will emulate a question-and-answer session with STEM professionals who have Navy ties and who are engaging, enthusiastic, and effective mentors. These interactions will allow students to have a lifelike informational interview with a virtual agent whose responses are directly drawn from a specific real professional’s video-recorded interview. This work differs from prior research on career guides by capturing the experiences of a collection of unique mentors, which should be more authentic and engaging than a generic agent or resource which speaks only about the average experience. This paper will discuss the process of creating the first such virtual STEM mentor prototype, including the development of an extensive mentoring question bank (approximately 500 questions); key mentoring topics that intersect STEM, DoD, and civilian life; techniques for cost-effective recording of remote mentors; and the process of training and verifying a natural language dialogue model for answering and suggesting career questions. Finally, we conclude with implications, strengths, and drawbacks of virtualizing the experience of talking with specific mentors, from the perspectives of efficacy, scalability, and maintainability.},

keywords = {Learning Sciences, MxR, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_analyzing_2017,

title = {Analyzing Learner Affect in a Scenario-Based Intelligent Tutoring System},

author = {Benjamin Nye and Shamya Karumbaiah and S. Tugba Tokel and Mark G. Core and Giota Stratou and Daniel Auerbach and Kallirroi Georgila},

url = {https://link.springer.com/chapter/10.1007/978-3-319-61425-0_60},

doi = {https://doi.org/10.1007/978-3-319-61425-0_60},

isbn = {978-3-319-61425-0},

year  = {2017},

date = {2017-06-01},

booktitle = {Proceedings of the International Conference on Artificial Intelligence in Education},

pages = {544–547},

publisher = {Springer},

address = {Wuhan, China},

abstract = {Scenario-based tutoring systems influence affective states due to two distinct mechanisms during learning: 1) reactions to performance feedback and 2) responses to the scenario context or events. To explore the role of affect and engagement, a scenario-based ITS was instrumented to support unobtrusive facial affect detection. Results from a sample of university students showed relatively few traditional academic affective states such as confusion or frustration, even at decision points and after poor performance (e.g., incorrect responses). This may show evidence of "over-flow," with a high level of engagement and interest but insufficient confusion/disequilibrium for optimal learning.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{fang_online_2017,

title = {Online Learning Persistence and Academic Achievement},

author = {Ying Fang and Yonghong Jade Xu and Benjamin Nye and Arthur Graesser and Philip Pavlik and Xiangen Hu},

url = {http://educationaldatamining.org/EDM2017/proc_files/papers/paper_114.pdf},

year  = {2017},

date = {2017-06-01},

booktitle = {Proceedings of Educational Data Mining (EDM) 2017},

pages = {312 – 317},

publisher = {EDM 2017},

address = {Wuhan, China},

abstract = {Student persistence in online learning environments has typically been studied at the macro-level (e.g., completion of an online course, number of academic terms completed, etc.). The current examines student persistence in an adaptive learning environment, ALEKS (Assessment and LEarning in Knowledge Spaces). Specifically, the study explores the relationship between students' academic achievement and their persistence during learning. By using archived data that included their math learning log data and performance on two standardized tests, we first explored student learning behavior patterns with regard to their persistence during learning. Clustering analysis identified three distinctive patterns of persistence-related learning behaviors: (1) High persistence and rare topic shifting; (2) Low persistence and frequent topic shifting; and (3) Moderate persistence and moderate topic shifting. We further explored the association between persistence and academic achievement. No significant differences were observed between academic achievement and the different learning patterns. We interpret this result in addition to a preliminary exploration of topic mastery trends, to suggest that wheel-spinning" behaviors coexist with persistence, and is ultimately not beneficial to learning.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_building_2017,

title = {Building a Backbone for Multi-Agent Tutoring in GIFT (Work in Progress)},

author = {Benjamin D. Nye and Daniel Auerbach and Tirth R. Mehta and Arno Hartholt},

url = {https://books.google.com/books?id=PwMtDwAAQBAJ&printsec=copyright&source=gbs_pub_info_r#v=onepage&q&f=false},

year  = {2017},

date = {2017-05-01},

booktitle = {Proceedings of the GIFTSym5},

pages = {23–35},

publisher = {ARL},

address = {Orlando, Florida},

abstract = {As intelligent tutoring systems (ITS) increasingly need to interoperate and co-exist, emerging systems have transitioned toward service-oriented designs to enable modularity and composability of tutoring components made and/or maintained by different research and development groups. However, as a research community, we have still not reached a point where it is trivial for a new service to be added into a system like the Generalized Intelligent Framework for Tutoring (GIFT; Sottilare, Goldberg, Brawner, & Holden, 2012). In an early paper considering this issue with respect to the GIFT architecture (Nye & Morrison, 2013), we proposed addressing this issue by building toward a lightweight multi-agent archi-tecture where certain services act as autonomous agents: “a system situated within and a part of an environment that senses that environment and acts on it, over time, in pursuit of its own agenda and so as to affect what it senses in the future” (Franklin & Graesser, 1997; p. 25). In our work in progress described here, we discuss how we are approaching the opportunity to build such capabilities into GIFT. The high level goals of our work are targeting two core goals for GIFT: A) to be a lightweight framework that will expand access to and use of ITS and B) to help GIFT to increase the intelligence and effectiveness of its services based on data over time. We are currently targeting the first goal, which will underpin the second goal. However, what does it mean to be a lightweight framework? In this context, a “lightweight framework” is framed as minimizing the following criteria: (1) hardware requirements, (2) software expertise to design services, (3) software expertise to use existing services, (4) software expertise to stand up the message-passing layer between agents, and (5) a minimal working message ontology (Nye & Morrison, 2013). Since our original paper four years ago, GIFT has made significant strides in reducing barriers related to hardware by building a cloud-based version and software expertise to use GIFT services through authoring tools. It has also developed a growing ontology of messages (e.g., https://gifttutoring.org/projects/gift/wiki/Interface_Control_Document_2016-1). With that said, despite now-extensive documentation, designing new services for GIFT is still not trivial and strong expertise is required to pass messages between GIFT modules and agents (either internal or external). To address these issues, the Building a Backbone project is working toward agent-oriented designs that build on GIFT's existing service-oriented framework. By moving from services toward agents, modules will be able to act more autonomously, enabling capabilities such as plug-and-play, hotswapping, and selecting between multiple services providing the same capabilities. These new capabilities are intended to reduce barriers to building new GIFT-compatible services and also to integrating GIFT with other service-oriented ecosystems. The first steps toward these capabilities are an ontology mapping service and an initial integration that combines GIFT, the Virtual Human Toolkit core framework for agents, and the SuperGLU framework for adding agent-oriented capabilities for coordinating services. This paper reports on work to date, with an emphasis on target capabilities, design decisions, challenges, and open research questions for this work.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{core_learning_2016,

title = {Learning, Adaptive Support, Student Traits, and Engagement in Scenario-Based Learning},

author = {Mark G. Core and Kallirroi Georgila and Benjamin D. Nye and Daniel Auerbach and Zhi Fei Liu and Richard DiNinni},

url = {http://www.iitsecdocs.com/search},

year  = {2016},

date = {2016-11-01},

booktitle = {Proceedings from the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2016},

publisher = {National Training and Simulation Association},

address = {Orlando, FL},

abstract = {Scenario-based training systems pose an especially difficult challenge for an intelligent tutoring system (ITS). In addition to the basic problems of deciding when to intervene and what guidance to provide, the ITS must decide whether to give guidance directly (e.g., a hint message), indirectly through positive/negative results in the scenario, or to delay guidance until a post-scenario review session. There are a number of factors that an adaptive ITS should consider and we use self-report survey instruments to investigate the relationship between traits, learning strategies, expectations, learner behaviors derived from log files, post-use perceptions of the system, and pre-test and post-test results. We use the ELITE Lite Counseling training system as a testbed for our experiments. This system uses virtual role players to allow learners to practice leadership counseling skills, and is in use at the United States Military Academy (USMA). This paper analyzes two data sets. We collected data from local university students, a non-military population of roughly the same age as USMA Cadets using the system. For these local participants, we could administer surveys and pre-tests and post-tests, and collect log files recording clicks made while using ELITE Lite. The second data set comes from USMA itself but is limited to log files. In both populations, the ITS’s hints are effective at boosting scenario performance, and for the university students, the overall experience promoted learning, and survey results suggest that higher levels of organization in study habits may lead to greater learning with ELITE Lite. For the USMA Cadets, ELITE Lite is part of their Military Leadership course rather than an experiment, which could explain why we found higher scenario performance on average than the non-military population, and more use of the post-scenario review feature.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mcalinden_cost-effective_2016,

title = {Cost-Effective Strategies for Producing Engaging Online Courseware},

author = {Ryan McAlinden and Sin-Hwa Kang and Benjamin Nye and Artemisa Phillips and Julia Campbell and Stephan L. Goldberg},

url = {http://www.iitsecdocs.com/search},

year  = {2016},

date = {2016-11-01},

booktitle = {Proceedings from the Interservice/Industry Training, Simulation and Education Conference (I/ITSEC) 2016},

publisher = {National Training and Simulation Association},

address = {Orlando, FL},

abstract = {As distributed learning (dL) and computer-based training (CBT) continue to proliferate, the methods of delivery often remain unengaging and bland for participants. Though many of the leaders in commercial online learning have improved their delivery style and quality in recent years, they continue to fall short in terms of user engagement and satisfaction. PowerPoint regurgitation and video lectures are commonplace and leave end users uninspired and wanting more. This paper discusses results from an ongoing research project, Captivating Virtual Instruction for Training (CVIT), which is aimed at understanding and improving dL through a series of recommendations and best practices for promoting and enhancing student engagement online. Though the central focus is on engagement, and how that translates to learning potential, a third variable (cost) has been examined to understand the financial and resource impacts on making content more interesting (i.e. the return on investment, or ROI). The paper presents findings from a 3-year long experiment comparing existing dL methods and techniques both within and outside of the Army. The project developed two dL versions of an existing Army course (Advanced Situational Awareness-Basic (ASA-B)) – the first was designed around producing material that was as engaging and as immersive as possible within a target budget; the second was a scaled-down version using more traditional, yet contemporary dL techniques (PowerPoint recital, video lectures). The two were then compared along three dimensions– engagement, learning and cost. The findings show that improved engagement in distributed courseware is possible without breaking the bank, though the returns on learning with these progressive approaches remain inconclusive. More importantly, it was determined that the quality and experience of the designers, production staff, writers, animators, programmers, and others cannot be underestimated, and that the familiar phrase – ‘you get what you pay for’ is as true with online learning as it is with other areas of content design and software development.},

keywords = {ARL, DoD, Learning Sciences, MedVR, MxR, STG, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{swartout_designing_2016,

title = {Designing a Personal Assistant for Life-Long Learning (PAL3)},

author = {William Swartout and Benjamin D. Nye and Arno Hartholt and Adam Reilly and Arthur C. Graesser and Kurt VanLehn and Jon Wetzel and Matt Liewer and Fabrizio Morbini and Brent Morgan and Lijia Wang and Grace Benn and Milton Rosenberg},

url = {http://www.aaai.org/ocs/index.php/FLAIRS/FLAIRS16/paper/view/12793},

isbn = {978-1-57735-756-8},

year  = {2016},

date = {2016-05-01},

booktitle = {Proceedings of The Twenty-Ninth International Flairs Conference},

pages = {491–496},

publisher = {AAAI Press},

address = {Key Largo, FL},

abstract = {Learners’ skills decay during gaps in instruction, since they lack the structure and motivation to continue studying. To meet this challenge, the PAL3 system was designed to accompany a learner throughout their career and mentor them to build and maintain skills through: 1) the use of an embodied pedagogical agent (Pal), 2) a persistent learning record that drives a student model which estimates forgetting, 3) an adaptive recommendation engine linking to both intelligent tutors and traditional learning resources, and 4) game-like mechanisms to promote engagement (e.g., leaderboards, effort-based point rewards, unlocking customizations). The design process for PAL3 is discussed, from the perspective of insights and revisions based on a series of formative feedback and evaluation sessions.},

keywords = {Learning Sciences, UARC, Virtual Humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nye_automated_2015,

title = {Automated Session-Quality Assessment for Human Tutoring Based on Expert Ratings of Tutoring Success},

author = {Benjamin D. Nye and Donald M. Morrison and Borhan Samei},

url = {http://ict.usc.edu/pubs/Automated%20Session-Quality%20Assessment%20for%20Human%20Tutoring%20Based%20on%20Expert%20Ratings%20of%20Tutoring%20Success.pdf},

year  = {2015},

date = {2015-06-01},

booktitle = {Proceedings of Educational Data Mining (EDM) 2015},

pages = {195–202},

publisher = {Springer},

address = {Madrid, Spain},

abstract = {Archived transcripts from tens of millions of online human tutoring sessions potentially contain important knowledge about how online tutors help, or fail to help, students learn. However, without ways of automatically analyzing these large corpora, any knowledge in this data will remain buried. One way to approach this issue is to train an estimator for the learning e⬚ectiveness of an online tutoring interaction. While significant work has been done on automated assessment of student responses and artifacts (e.g., essays), automated assessment has not traditionally automated assessments of human-to-human tutoring sessions. In this work, we trained a model for estimating tutoring session quality based on a corpus of 1438 online tutoring sessions rated by expert tutors. Each session was rated for evidence of learning (outcomes) and educational soundness (process). Session features for this model included dialog act classifcations, mode classifcations (e.g., Scaffolding), statistically distinctive subsequences of such classifcations, dialog initiative (e.g., statements by tutor vs. student), and session length. The model correlated more highly with evidence of learning than educational soundness ratings, in part due to the greater difficulty of classifying tutoring modes. This model was then applied to a corpus of 242k online tutoring sessions, to examine the relationships between automated assessments and other available metadata (e.g., the tutor's self-assessment). On this large corpus, the automated assessments followed similar patterns as the expert rater's assessments, but with lower overall correlation strength. Based on the analyses presented, the assessment model for online tutoring sessions emulates the ratings of expert human tutors for session quality ratings with a reasonable degree of accuracy.},

keywords = {Learning Sciences},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{chad_lane_situated_2015,

title = {Situated Pedagogical Authoring: Authoring Intelligent Tutors from a Student’s Perspective},

author = {H. Chad Lane and Mark G. Core and Matthew J. Hays and Daniel Auerbach and Milton Rosenberg},

url = {http://ict.usc.edu/pubs/Situated%20Pedagogical%20Authoring-Authoring%20Intelligent.pdf},

isbn = {978-3-319-19772-2 978-3-319-19773-9},

year  = {2015},

date = {2015-06-01},

booktitle = {Artificial Intelligence in Education},

volume = {9112},

pages = {195–204},

publisher = {Springer International Publishing},

address = {Madrid, Spain},

abstract = {We describe the Situated Pedagogical Authoring (SitPed) system that seeks to allow non-technical authors to create ITS content for soft-skills training, such as counseling skills. SitPed is built on the assertion that authoring tools should use the learner’s perspective to the greatest extent possible. SitPed provides tools for creating tasks lists, authoring assessment knowledge, and creating tutor messages. We present preliminary findings of a two-phase study comparing authoring in SitPed to an ablated version of the same system and a spreadsheet-based control. Findings suggest modest advantages for SitPed in terms of the quality of the authored content and student learning.},

keywords = {Learning Sciences, UARC},

pubstate = {published},

tppubtype = {inproceedings}

}