Publications

@article{pynadath_disaster_2022,

title = {Disaster world},

author = {David V. Pynadath and Bistra Dilkina and David C. Jeong and Richard S. John and Stacy C. Marsella and Chirag Merchant and Lynn C. Miller and Stephen J. Read},

url = {https://doi.org/10.1007/s10588-022-09359-y},

doi = {10.1007/s10588-022-09359-y},

issn = {1572-9346},

year  = {2022},

date = {2022-05-01},

urldate = {2022-09-28},

journal = {Comput Math Organ Theory},

abstract = {Artificial intelligence (AI) research provides a rich source of modeling languages capable of generating socially plausible simulations of human behavior, while also providing a transparent ground truth that can support validation of social-science methods applied to that simulation. In this work, we leverage two established AI representations: decision-theoretic planning and recursive modeling. Decision-theoretic planning (specifically Partially Observable Markov Decision Processes) provides agents with quantitative models of their corresponding real-world entities’ subjective (and possibly incorrect) perspectives of ground truth in the form of probabilistic beliefs and utility functions. Recursive modeling gives an agent a theory of mind, which is necessary when a person’s (again, possibly incorrect) subjective perspectives are of another person, rather than of just his/her environment. We used PsychSim, a multiagent social-simulation framework combining these two AI frameworks, to build a general parameterized model of human behavior during disaster response, grounding the model in social-psychological theories to ensure social plausibility. We then instantiated that model into alternate ground truths for simulating population response to a series of natural disasters, namely, hurricanes. The simulations generate data in response to socially plausible instruments (e.g., surveys) that serve as input to the Ground Truth program’s designated research teams for them to conduct simulated social science. The simulation also provides a graphical ground truth and a set of outcomes to be used as the gold standard in evaluating the research teams’ inferences.},

keywords = {DTIC, Social Simulation},

pubstate = {published},

tppubtype = {article}

}

@article{schweitzer_virtual_2022,

title = {Virtual Reality and ADHD: Clinical Assessment and Treatment in the Metaverse},

author = {Julie B. Schweitzer and Albert “Skip” Rizzo},

url = {https://guilfordjournals.com/doi/abs/10.1521/adhd.2022.30.3.1},

doi = {10.1521/adhd.2022.30.3.1},

issn = {1065-8025},

year  = {2022},

date = {2022-05-01},

urldate = {2022-09-13},

journal = {The ADHD Report},

volume = {30},

number = {3},

pages = {1--9},

keywords = {MedVR, VR},

pubstate = {published},

tppubtype = {article}

}

@article{aris_learning_2022,

title = {Learning to Take Cover on Geo-Specific Terrains via Reinforcement Learning},

author = {Timothy Aris and Volkan Ustun and Rajay Kumar},

url = {https://journals.flvc.org/FLAIRS/article/view/130871},

doi = {10.32473/flairs.v35i.130871},

issn = {2334-0762},

year  = {2022},

date = {2022-05-01},

urldate = {2022-09-15},

journal = {FLAIRS},

volume = {35},

abstract = {This paper presents a reinforcement learning model designed to learn how to take cover on geo-specific terrains, an essential behavior component for military training simulations. Training of the models is performed on the Rapid Integration and Development Environment (RIDE) leveraging the Unity ML-Agents framework. We show that increasing the number of novel situations the agent is exposed to increases the performance on the test set. In addition, the trained models possess some ability to generalize across terrains, and it can also take less time to retrain an agent to a new terrain, if that terrain has a level of complexity less than or equal to the terrain it was previously trained on.},

keywords = {DTIC, Integration Technology},

pubstate = {published},

tppubtype = {article}

}

@article{rodrigues_ergonomic_2022,

title = {Ergonomic assessment of office worker postures using 3D automated joint angle assessment},

author = {Patrick B. Rodrigues and Yijing Xiao and Yoko E. Fukumura and Mohamad Awada and Ashrant Aryal and Burcin Becerik-Gerber and Gale Lucas and Shawn C. Roll},

url = {https://linkinghub.elsevier.com/retrieve/pii/S1474034622000672},

doi = {10.1016/j.aei.2022.101596},

issn = {14740346},

year  = {2022},

date = {2022-04-01},

urldate = {2022-09-26},

journal = {Advanced Engineering Informatics},

volume = {52},

pages = {101596},

keywords = {DTIC, Machine Learning, UARC},

pubstate = {published},

tppubtype = {article}

}

@article{fujiwara_synchrony_2022,

title = {Synchrony facilitates altruistic decision making for non-human avatars},

author = {Ken Fujiwara and Rens Hoegen and Jonathan Gratch and Norah E. Dunbar},

url = {https://linkinghub.elsevier.com/retrieve/pii/S0747563221004027},

doi = {10.1016/j.chb.2021.107079},

issn = {07475632},

year  = {2022},

date = {2022-03-01},

urldate = {2022-09-28},

journal = {Computers in Human Behavior},

volume = {128},

pages = {107079},

keywords = {DTIC, Virtual Humans},

pubstate = {published},

tppubtype = {article}

}

@article{de_visser_designing_2022,

title = {Designing Man’s New Best Friend: Enhancing Human-Robot Dog Interaction through Dog-Like Framing and Appearance},

author = {Ewart J. Visser and Yigit Topoglu and Shawn Joshi and Frank Krueger and Elizabeth Phillips and Jonathan Gratch and Chad C. Tossell and Hasan Ayaz},

url = {https://www.mdpi.com/1424-8220/22/3/1287},

doi = {10.3390/s22031287},

issn = {1424-8220},

year  = {2022},

date = {2022-02-01},

urldate = {2022-09-28},

journal = {Sensors},

volume = {22},

number = {3},

pages = {1287},

abstract = {To understand how to improve interactions with dog-like robots, we evaluated the importance of “dog-like” framing and physical appearance on interaction, hypothesizing multiple interactive benefits of each. We assessed whether framing Aibo as a puppy (i.e., in need of development) versus simply a robot would result in more positive responses and interactions. We also predicted that adding fur to Aibo would make it appear more dog-like, likable, and interactive. Twenty-nine participants engaged with Aibo in a 2 × 2 (framing × appearance) design by issuing commands to the robot. Aibo and participant behaviors were monitored per second, and evaluated via an analysis of commands issued, an analysis of command blocks (i.e., chains of commands), and using a T-pattern analysis of participant behavior. Participants were more likely to issue the “Come Here” command than other types of commands. When framed as a puppy, participants used Aibo’s dog name more often, praised it more, and exhibited more unique, interactive, and complex behavior with Aibo. Participants exhibited the most smiling and laughing behaviors with Aibo framed as a puppy without fur. Across conditions, after interacting with Aibo, participants felt Aibo was more trustworthy, intelligent, warm, and connected than at their initial meeting. This study shows the benefits of introducing a socially robotic agent with a particular frame and importance on realism (i.e., introducing the robot dog as a puppy) for more interactive engagement.},

keywords = {DTIC, Virtual Humans},

pubstate = {published},

tppubtype = {article}

}

@article{saxon_novel_2022,

title = {A Novel Digital Research Methodology for Continuous Health Assessment of the Special Operations Warfighter: The Digital cORA Study},

author = {Leslie Saxon and Robert T Faulk and Travor Barrett and Steve McLelland and Jill Boberg},

url = {https://www.jsomonline.org/Citations/4SSJ-AHIB.php},

doi = {10.55460/4SSJ-AHIB},

issn = {1553-9768},

year  = {2022},

date = {2022-01-01},

urldate = {2023-03-31},

journal = {J. Spec. Oper. Med.},

volume = {22},

number = {4},

pages = {78},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{hoegen_exploring_2022,

title = {Exploring the Function of Expressions in Negotiation: the DyNego-WOZ Corpus},

author = {Jessie Hoegen and David DeVault and Jonathan Gratch},

doi = {10.1109/TAFFC.2022.3223030},

issn = {1949-3045},

year  = {2022},

date = {2022-01-01},

journal = {IEEE Transactions on Affective Computing},

pages = {1--12},

abstract = {For affective computing to have an impact outside the laboratory, facial expressions must be studied in rich naturalistic situations. We argue negotiations are one such situation as they are ubiquitous in daily life, often evoke strong emotions, and perceived emotion shapes decisions and outcomes. Negotiations are a growing focus in AI research and applications, including agents that negotiate directly with people and attempt to use affective information. We introduce the DyNego-WOZ Corpus, which includes dyadic negotiation between participants and wizard-controlled virtual humans. We demonstrate the value of this corpus to the affective computing community by examining participants' facial expressions in response to a virtual human negotiation partner. We show that people's facial expressions typically co-occur with the end of their partner's speech (suggesting they reflect a reaction to the content of this speech), that these reactions do not correspond to prototypical emotional expressions, and that these reactions can help predict the expresser's subsequent action. We highlight challenges in working with such naturalistic data, including difficulties of expression recognition during speech, and the extreme variability of expressions, both across participants and within a negotiation. Our findings reinforce arguments that facial expressions convey more than emotional state but serve important communicative functions.},

note = {Conference Name: IEEE Transactions on Affective Computing},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{liu_general_2022,

title = {A General Differentiable Mesh Renderer for Image-Based 3D Reasoning},

author = {Shichen Liu and Tianye Li and Weikai Chen and Hao Li},

doi = {10.1109/TPAMI.2020.3007759},

issn = {1939-3539},

year  = {2022},

date = {2022-01-01},

journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},

volume = {44},

number = {1},

pages = {50--62},

abstract = {Rendering bridges the gap between 2D vision and 3D scenes by simulating the physical process of image formation. By inverting such renderer, one can think of a learning approach to infer 3D information from 2D images. However, standard graphics renderers involve a fundamental step called rasterization, which prevents rendering to be differentiable. Unlike the state-of-the-art differentiable renderers (Kato et al. 2018 and Loper 2018), which only approximate the rendering gradient in the backpropagation, we propose a natually differentiable rendering framework that is able to (1) directly render colorized mesh using differentiable functions and (2) back-propagate efficient supervisions to mesh vertices and their attributes from various forms of image representations. The key to our framework is a novel formulation that views rendering as an aggregation function that fuses the probabilistic contributions of all mesh triangles with respect to the rendered pixels. Such formulation enables our framework to flow gradients to the occluded and distant vertices, which cannot be achieved by the previous state-of-the-arts. We show that by using the proposed renderer, one can achieve significant improvement in 3D unsupervised single-view reconstruction both qualitatively and quantitatively. Experiments also demonstrate that our approach can handle the challenging tasks in image-based shape fitting, which remain nontrivial to existing differentiable renders.},

note = {Conference Name: IEEE Transactions on Pattern Analysis and Machine Intelligence},

keywords = {VGL},

pubstate = {published},

tppubtype = {article}

}

@article{marge_spoken_2022,

title = {Spoken language interaction with robots: Recommendations for future research},

author = {Matthew Marge and Carol Espy-Wilson and Nigel G. Ward and Abeer Alwan and Yoav Artzi and Mohit Bansal and Gil Blankenship and Joyce Chai and Hal Daumé and Debadeepta Dey and Mary Harper and Thomas Howard and Casey Kennington and Ivana Kruijff-Korbayová and Dinesh Manocha and Cynthia Matuszek and Ross Mead and Raymond Mooney and Roger K. Moore and Mari Ostendorf and Heather Pon-Barry and Alexander I. Rudnicky and Matthias Scheutz and Robert St. Amant and Tong Sun and Stefanie Tellex and David Traum and Zhou Yu},

url = {https://linkinghub.elsevier.com/retrieve/pii/S0885230821000620},

doi = {10.1016/j.csl.2021.101255},

issn = {08852308},

year  = {2022},

date = {2022-01-01},

urldate = {2022-09-23},

journal = {Computer Speech & Language},

volume = {71},

pages = {101255},

keywords = {ARL, Dialogue},

pubstate = {published},

tppubtype = {article}

}

@article{hou_investigation_2022,

title = {Investigation on performance of RGB point cloud and thermal information data fusion for 3D building thermal map modeling using aerial images under different experimental conditions},

author = {Yu Hou and Meida Chen and Rebekka Volk and Lucio Soibelman},

url = {https://linkinghub.elsevier.com/retrieve/pii/S2352710221012389},

doi = {10.1016/j.jobe.2021.103380},

issn = {23527102},

year  = {2022},

date = {2022-01-01},

urldate = {2022-09-28},

journal = {Journal of Building Engineering},

volume = {45},

pages = {103380},

keywords = {Graphics},

pubstate = {published},

tppubtype = {article}

}

@article{aster_responsivity_2022,

title = {Responsivity of the Striatal Dopamine System to Methylphenidate—A Within-Subject I-123-β-CIT-SPECT Study in Male Children and Adolescents With Attention-Deficit/Hyperactivity Disorder},

author = {Hans-Christoph Aster and Marcel Romanos and Susanne Walitza and Manfred Gerlach and Andreas Mühlberger and Albert Rizzo and Marta Andreatta and Natalie Hasenauer and Philipp E. Hartrampf and Kai Nerlich and Christoph Reiners and Reinhard Lorenz and Andreas K. Buck and Lorenz Deserno},

url = {https://www.frontiersin.org/articles/10.3389/fpsyt.2022.804730},

issn = {1664-0640},

year  = {2022},

date = {2022-01-01},

urldate = {2022-09-13},

journal = {Frontiers in Psychiatry},

volume = {13},

abstract = {Background:Methylphenidate (MPH) is the first-line pharmacological treatment of attention-deficit/hyperactivity disorder (ADHD). MPH binds to the dopamine (DA) transporter (DAT), which has high density in the striatum. Assessments of the striatal dopamine transporter by single positron emission computed tomography (SPECT) in childhood and adolescent patients are rare but can provide insight on how the effects of MPH affect DAT availability. The aim of our within-subject study was to investigate the effect of MPH on DAT availability and how responsivity to MPH in DAT availability is linked to clinical symptoms and cognitive functioning.MethodsThirteen adolescent male patients (9–16 years) with a diagnosis of ADHD according to the DSM-IV and long-term stimulant medication (for at least 6 months) with MPH were assessed twice within 7 days using SPECT after application of I-123-β-CIT to examine DAT binding potential (DAT BP). SPECT measures took place in an on- and off-MPH status balanced for order across participants. A virtual reality continuous performance test was performed at each time point. Further clinical symptoms were assessed for baseline off-MPH.ResultsOn-MPH status was associated with a highly significant change (−29.9%) of striatal DAT BP as compared to off-MPH (t = −4.12},

keywords = {MedVR},

pubstate = {published},

tppubtype = {article}

}

@article{stokes_measuring_2022,

title = {Measuring Attentional Distraction in Children With ADHD Using Virtual Reality Technology With Eye-Tracking},

author = {Jared D. Stokes and Albert Rizzo and Joy J. Geng and Julie B. Schweitzer},

url = {https://www.frontiersin.org/articles/10.3389/frvir.2022.855895},

issn = {2673-4192},

year  = {2022},

date = {2022-01-01},

urldate = {2022-09-13},

journal = {Frontiers in Virtual Reality},

volume = {3},

abstract = {Objective: Distractions inordinately impair attention in children with Attention-Deficit Hyperactivity Disorder (ADHD) but examining this behavior under real-life conditions poses a challenge for researchers and clinicians. Virtual reality (VR) technologies may mitigate the limitations of traditional laboratory methods by providing a more ecologically relevant experience. The use of eye-tracking measures to assess attentional functioning in a VR context in ADHD is novel. In this proof of principle project, we evaluate the temporal dynamics of distraction via eye-tracking measures in a VR classroom setting with 20 children diagnosed with ADHD between 8 and 12 years of age.Method: We recorded continuous eye movements while participants performed math, Stroop, and continuous performance test (CPT) tasks with a series of “real-world” classroom distractors presented. We analyzed the impact of the distractors on rates of on-task performance and on-task, eye-gaze (i.e., looking at a classroom whiteboard) versus off-task eye-gaze (i.e., looking away from the whiteboard).Results: We found that while children did not always look at distractors themselves for long periods of time, the presence of a distractor disrupted on-task gaze at task-relevant whiteboard stimuli and lowered rates of task performance. This suggests that children with attention deficits may have a hard time returning to tasks once those tasks are interrupted, even if the distractor itself does not hold attention. Eye-tracking measures within the VR context can reveal rich information about attentional disruption.Conclusions: Leveraging virtual reality technology in combination with eye-tracking measures is well-suited to advance the understanding of mechanisms underlying attentional impairment in naturalistic settings. Assessment within these immersive and well-controlled simulated environments provides new options for increasing our understanding of distractibility and its potential impact on the development of interventions for children with ADHD.},

keywords = {MedVR, VR},

pubstate = {published},

tppubtype = {article}

}

@article{davis_designing_2022,

title = {Designing scientifically-grounded paleoart for augmented reality at La Brea Tar Pits},

author = {Matt Davis and Benjamin Nye and Gale Sinatra and William Swartout and Molly Sjӧberg and Molly Porter and David Nelson and Alana Kennedy and Imogen Herrick and Danaan DeNeve Weeks and Emily Lindsey},

url = {https://palaeo-electronica.org/content/2022/3524-la-brea-tar-pits-paleoart},

doi = {10.26879/1191},

issn = {19353952, 10948074},

year  = {2022},

date = {2022-01-01},

urldate = {2022-09-15},

journal = {Palaeontol Electron},

keywords = {AR, MxR, VR},

pubstate = {published},

tppubtype = {article}

}

@article{ionescu_360_2021,

title = {360° Videos for Immersive Mental Health Interventions: a Systematic Review},

author = {Alina Ionescu and Tom Van Daele and Albert Rizzo and Carolyn Blair and Paul Best},

url = {https://doi.org/10.1007/s41347-021-00221-7},

doi = {10.1007/s41347-021-00221-7},

issn = {2366-5963},

year  = {2021},

date = {2021-12-01},

urldate = {2022-09-13},

journal = {J. technol. behav. sci.},

volume = {6},

number = {4},

pages = {631--651},

abstract = {Virtual reality is increasingly recognized as a powerful method for clinical interventions in the mental health field, but has yet to achieve mainstream adoption in routine mental healthcare settings. A similar, yet slightly different technology, immersive 360° videos might have the potential to cover this gap, by requiring both lower costs and less technical skills to construct and operate such virtual environments. This systematic review therefore aims to identify, evaluate, and summarize mental health interventions using immersive 360° videos to support an understanding of their implementation in daily clinical practice. The quality of the 14 selected studies was evaluated using a critical appraisal tool, addressing populations with clinical levels of psychopathological symptoms, somatic conditions associated with psychological implications, and other at-risk groups. Immersive 360° videos successfully increased users’ feelings of presence, given their realistic features, and therefore yielded positive outcomes in clinical interventions where presence is considered as an essential precondition. Because the technical skills required to create immersive 360° video footage are fairly limited, most of the interventions using this approach have been created by mental health researchers or clinicians themselves. Immersive 360° videos are still in an early phase of implementation as a tool for clinical interventions for mental health, resulting in high heterogeneity in focus, procedures, and research designs. An important next step for making use of this technology may therefore involve the creation of standardized procedures, as a means to increase the quality of research and evidence-based interventions.},

keywords = {MedVR, VR},

pubstate = {published},

tppubtype = {article}

}

@article{talbot_advances_2021,

title = {Advances in Affective Computing for Psychological Applications: From the Fundamentals to the Future of Emotional Cognizant Artificial Intelligence Entities},

author = {Thomas B. Talbot and Albert S. Rizzo and Mohammed Soleymani},

url = {https://tmb.apaopen.org/pub/zm0la9di/release/1},

doi = {10.1037/tms0000011},

year  = {2021},

date = {2021-11-01},

urldate = {2023-03-31},

journal = {Technology, Mind, and Behavior},

abstract = {Keywords: Affective Computing, Emotion Perception, Virtual Humans, Conversational Simulations},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{awada_associations_2021,

title = {Associations Among Home Indoor Environmental Quality Factors and Worker Health While Working From Home During COVID-19 Pandemic},

author = {Mohamad Awada and Burcin Becerik-Gerber and Gale Lucas and Shawn C. Roll},

url = {https://asmedigitalcollection.asme.org/sustainablebuildings/article/2/4/041001/1122847/Associations-Among-Home-Indoor-Environmental},

doi = {10.1115/1.4052822},

issn = {2642-6641, 2642-6625},

year  = {2021},

date = {2021-11-01},

urldate = {2022-09-26},

journal = {ASME Journal of Engineering for Sustainable Buildings and Cities},

volume = {2},

number = {4},

pages = {041001},

abstract = {Abstract The outbreak of SARS-CoV-2 virus forced office workers to conduct their daily work activities from home over an extended period. Given this unique situation, an opportunity emerged to study the satisfaction of office workers with indoor environmental quality (IEQ) factors of their houses where work activities took place and associate these factors with mental and physical health. We designed and administered a questionnaire that was open for 45 days during the COVID-19 pandemic and received valid data from 988 respondents. The results show that low satisfaction with natural lighting, glare, and humidity predicted eye-related symptoms, while low satisfaction with noise was a strong predictor of fatigue or tiredness, headaches or migraines, anxiety, and depression or sadness. Nose- and throat-related symptoms and skin-related symptoms were only uniquely predicted by low satisfaction with humidity. Low satisfaction with glare uniquely predicted an increase in musculoskeletal discomfort. Symptoms related to mental stress, rumination, or worry were predicted by low satisfaction with air quality and noise. Finally, low satisfaction with noise and indoor temperature predicted the prevalence of symptoms related to trouble concentrating, maintaining attention, or focus. Workers with higher income were more satisfied with humidity, air quality, and indoor temperature and had better overall mental health. Older individuals had increased satisfaction with natural lighting, humidity, air quality, noise, and indoor temperature. Findings from this study can inform future design practices that focus on hybrid home-work environments by highlighting the impact of IEQ factors on occupant well-being.},

keywords = {DTIC, UARC},

pubstate = {published},

tppubtype = {article}

}

@article{noauthor_weve_2021,

title = {We've Entered a New Era of Streaming Health Care. Now What?},

url = {https://spectrum.ieee.org/digital-health},

year  = {2021},

date = {2021-11-01},

urldate = {2022-09-29},

journal = {IEEE Spectrum},

abstract = {COVID-19 forced the transition to digital medicine, but there's much still to do.},

keywords = {CBC},

pubstate = {published},

tppubtype = {article}

}

@article{schuller_intelligent_2021,

title = {Intelligent Signal Processing for Affective Computing [From the Guest Editors]},

author = {Bjorn W. Schuller and Rosalind Picard and Elisabeth Andre and Jonathan Gratch and Jianhua Tao},

url = {https://ieeexplore.ieee.org/document/9591500/},

doi = {10.1109/MSP.2021.3096415},

issn = {1053-5888, 1558-0792},

year  = {2021},

date = {2021-11-01},

urldate = {2022-09-29},

journal = {IEEE Signal Process. Mag.},

volume = {38},

number = {6},

pages = {9--11},

keywords = {Emotions, Virtual Humans},

pubstate = {published},

tppubtype = {article}

}

@article{adami_effectiveness_2021,

title = {Effectiveness of VR-based training on improving construction workers’ knowledge, skills, and safety behavior in robotic teleoperation},

author = {Pooya Adami and Patrick B. Rodrigues and Peter J. Woods and Burcin Becerik-Gerber and Lucio Soibelman and Yasemin Copur-Gencturk and Gale Lucas},

url = {https://linkinghub.elsevier.com/retrieve/pii/S147403462100183X},

doi = {10.1016/j.aei.2021.101431},

issn = {14740346},

year  = {2021},

date = {2021-10-01},

urldate = {2022-09-26},

journal = {Advanced Engineering Informatics},

volume = {50},

pages = {101431},

keywords = {DTIC, Learning Sciences, UARC, VR},

pubstate = {published},

tppubtype = {article}

}

@article{hou_approach_2021,

title = {An Approach to Semantically Segmenting Building Components and Outdoor Scenes Based on Multichannel Aerial Imagery Datasets},

author = {Yu Hou and Meida Chen and Rebekka Volk and Lucio Soibelman},

url = {https://www.mdpi.com/2072-4292/13/21/4357},

doi = {10.3390/rs13214357},

issn = {2072-4292},

year  = {2021},

date = {2021-10-01},

urldate = {2022-09-28},

journal = {Remote Sensing},

volume = {13},

number = {21},

pages = {4357},

abstract = {As-is building modeling plays an important role in energy audits and retrofits. However, in order to understand the source(s) of energy loss, researchers must know the semantic information of the buildings and outdoor scenes. Thermal information can potentially be used to distinguish objects that have similar surface colors but are composed of different materials. To utilize both the red–green–blue (RGB) color model and thermal information for the semantic segmentation of buildings and outdoor scenes, we deployed and adapted various pioneering deep convolutional neural network (DCNN) tools that combine RGB information with thermal information to improve the semantic and instance segmentation processes. When both types of information are available, the resulting DCNN models allow us to achieve better segmentation performance. By deploying three case studies, we experimented with our proposed DCNN framework, deploying datasets of building components and outdoor scenes, and testing the models to determine whether the segmentation performance had improved or not. In our observation, the fusion of RGB and thermal information can help the segmentation task in specific cases, but it might also make the neural networks hard to train or deteriorate their prediction performance in some cases. Additionally, different algorithms perform differently in semantic and instance segmentation.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}