Publications
Search
1.
Oh, Jinwoo; Chen, Po-Yu; Hsing, Hsiang-Wen; Lau, Nathan; Wu, Peggy; Srivastava, Kunal; Gurney, Nikolos; Molinaro, Kylie; Trent, Stoney
Understanding Cybersecurity Skill Levels Through Psychological Measures: Clustering Hackers with Traits Questionnaires Journal Article
In: Proceedings of the Human Factors and Ergonomics Society Annual Meeting, pp. 10711813251371034, 2025, ISSN: 1071-1813, 2169-5067.
@article{oh_understanding_2025,
title = {Understanding Cybersecurity Skill Levels Through Psychological Measures: Clustering Hackers with Traits Questionnaires},
author = {Jinwoo Oh and Po-Yu Chen and Hsiang-Wen Hsing and Nathan Lau and Peggy Wu and Kunal Srivastava and Nikolos Gurney and Kylie Molinaro and Stoney Trent},
url = {https://journals.sagepub.com/doi/10.1177/10711813251371034},
doi = {10.1177/10711813251371034},
issn = {1071-1813, 2169-5067},
year = {2025},
date = {2025-09-01},
urldate = {2025-09-18},
journal = {Proceedings of the Human Factors and Ergonomics Society Annual Meeting},
pages = {10711813251371034},
abstract = {In cybersecurity, performance in offensive tasks such as penetration testing or red-team exercises can be influenced by both technical skill and psychological traits. This exploratory study examines how specific psychometric characteristics relate to hacking performance in a controlled environment. Sixty-one participants who passed a cybersecurity skills test completed a two-day simulated hacking exercise and responded to psychometric questionnaires. A Random Forest analysis identified five questionnaire items—drawn from decision-making and personality measures—as the most predictive of cybersecurity skills test scores. The responses to these items were used in a k-means clustering analysis (
k = 3), which revealed significant differences in skills test scores and response patterns across clusters. The findings suggest that certain psychological traits may serve as auxiliary indicators of cybersecurity skill. Further research could explore this relationship using aggregated trait-level metrics and broader participant samples, including professional red-teamers, to examine the robustness of these preliminary findings in more ecologically valid settings.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
In cybersecurity, performance in offensive tasks such as penetration testing or red-team exercises can be influenced by both technical skill and psychological traits. This exploratory study examines how specific psychometric characteristics relate to hacking performance in a controlled environment. Sixty-one participants who passed a cybersecurity skills test completed a two-day simulated hacking exercise and responded to psychometric questionnaires. A Random Forest analysis identified five questionnaire items—drawn from decision-making and personality measures—as the most predictive of cybersecurity skills test scores. The responses to these items were used in a k-means clustering analysis (
k = 3), which revealed significant differences in skills test scores and response patterns across clusters. The findings suggest that certain psychological traits may serve as auxiliary indicators of cybersecurity skill. Further research could explore this relationship using aggregated trait-level metrics and broader participant samples, including professional red-teamers, to examine the robustness of these preliminary findings in more ecologically valid settings.
k = 3), which revealed significant differences in skills test scores and response patterns across clusters. The findings suggest that certain psychological traits may serve as auxiliary indicators of cybersecurity skill. Further research could explore this relationship using aggregated trait-level metrics and broader participant samples, including professional red-teamers, to examine the robustness of these preliminary findings in more ecologically valid settings.
2.
Beltz, Brandon; Doty, Jim; Fonken, Yvonne; Gurney, Nikolos; Israelsen, Brett; Lau, Nathan; Marsella, Stacy; Thomas, Rachelle; Trent, Stoney; Wu, Peggy; Yang, Ya-Ting; Zhu, Quanyan
2025, (arXiv:2508.20963 [cs]).
@misc{beltz_guarding_2025,
title = {Guarding Against Malicious Biased Threats (GAMBiT) Experiments: Revealing Cognitive Bias in Human-Subjects Red-Team Cyber Range Operations},
author = {Brandon Beltz and Jim Doty and Yvonne Fonken and Nikolos Gurney and Brett Israelsen and Nathan Lau and Stacy Marsella and Rachelle Thomas and Stoney Trent and Peggy Wu and Ya-Ting Yang and Quanyan Zhu},
url = {http://arxiv.org/abs/2508.20963},
doi = {10.48550/arXiv.2508.20963},
year = {2025},
date = {2025-08-01},
urldate = {2025-09-18},
publisher = {arXiv},
abstract = {We present three large-scale human-subjects red-team cyber range datasets from the Guarding Against Malicious Biased Threats (GAMBiT) project. Across Experiments 1-3 (July 2024-March 2025), 19-20 skilled attackers per experiment conducted two 8-hour days of self-paced operations in a simulated enterprise network (SimSpace Cyber Force Platform) while we captured multi-modal data: self-reports (background, demographics, psychometrics), operational notes, terminal histories, keylogs, network packet captures (PCAP), and NIDS alerts (Suricata). Each participant began from a standardized Kali Linux VM and pursued realistic objectives (e.g., target discovery and data exfiltration) under controlled constraints. Derivative curated logs and labels are included. The combined release supports research on attacker behavior modeling, bias-aware analytics, and method benchmarking. Data are available via IEEE Dataport entries for Experiments 1-3.},
note = {arXiv:2508.20963 [cs]},
keywords = {},
pubstate = {published},
tppubtype = {misc}
}
We present three large-scale human-subjects red-team cyber range datasets from the Guarding Against Malicious Biased Threats (GAMBiT) project. Across Experiments 1-3 (July 2024-March 2025), 19-20 skilled attackers per experiment conducted two 8-hour days of self-paced operations in a simulated enterprise network (SimSpace Cyber Force Platform) while we captured multi-modal data: self-reports (background, demographics, psychometrics), operational notes, terminal histories, keylogs, network packet captures (PCAP), and NIDS alerts (Suricata). Each participant began from a standardized Kali Linux VM and pursued realistic objectives (e.g., target discovery and data exfiltration) under controlled constraints. Derivative curated logs and labels are included. The combined release supports research on attacker behavior modeling, bias-aware analytics, and method benchmarking. Data are available via IEEE Dataport entries for Experiments 1-3.
Filter
2025
Oh, Jinwoo; Chen, Po-Yu; Hsing, Hsiang-Wen; Lau, Nathan; Wu, Peggy; Srivastava, Kunal; Gurney, Nikolos; Molinaro, Kylie; Trent, Stoney
Understanding Cybersecurity Skill Levels Through Psychological Measures: Clustering Hackers with Traits Questionnaires Journal Article
In: Proceedings of the Human Factors and Ergonomics Society Annual Meeting, pp. 10711813251371034, 2025, ISSN: 1071-1813, 2169-5067.
Abstract | Links | BibTeX | Tags: DTIC, Security
@article{oh_understanding_2025,
title = {Understanding Cybersecurity Skill Levels Through Psychological Measures: Clustering Hackers with Traits Questionnaires},
author = {Jinwoo Oh and Po-Yu Chen and Hsiang-Wen Hsing and Nathan Lau and Peggy Wu and Kunal Srivastava and Nikolos Gurney and Kylie Molinaro and Stoney Trent},
url = {https://journals.sagepub.com/doi/10.1177/10711813251371034},
doi = {10.1177/10711813251371034},
issn = {1071-1813, 2169-5067},
year = {2025},
date = {2025-09-01},
urldate = {2025-09-18},
journal = {Proceedings of the Human Factors and Ergonomics Society Annual Meeting},
pages = {10711813251371034},
abstract = {In cybersecurity, performance in offensive tasks such as penetration testing or red-team exercises can be influenced by both technical skill and psychological traits. This exploratory study examines how specific psychometric characteristics relate to hacking performance in a controlled environment. Sixty-one participants who passed a cybersecurity skills test completed a two-day simulated hacking exercise and responded to psychometric questionnaires. A Random Forest analysis identified five questionnaire items—drawn from decision-making and personality measures—as the most predictive of cybersecurity skills test scores. The responses to these items were used in a k-means clustering analysis (
k = 3), which revealed significant differences in skills test scores and response patterns across clusters. The findings suggest that certain psychological traits may serve as auxiliary indicators of cybersecurity skill. Further research could explore this relationship using aggregated trait-level metrics and broader participant samples, including professional red-teamers, to examine the robustness of these preliminary findings in more ecologically valid settings.},
keywords = {DTIC, Security},
pubstate = {published},
tppubtype = {article}
}
In cybersecurity, performance in offensive tasks such as penetration testing or red-team exercises can be influenced by both technical skill and psychological traits. This exploratory study examines how specific psychometric characteristics relate to hacking performance in a controlled environment. Sixty-one participants who passed a cybersecurity skills test completed a two-day simulated hacking exercise and responded to psychometric questionnaires. A Random Forest analysis identified five questionnaire items—drawn from decision-making and personality measures—as the most predictive of cybersecurity skills test scores. The responses to these items were used in a k-means clustering analysis (
k = 3), which revealed significant differences in skills test scores and response patterns across clusters. The findings suggest that certain psychological traits may serve as auxiliary indicators of cybersecurity skill. Further research could explore this relationship using aggregated trait-level metrics and broader participant samples, including professional red-teamers, to examine the robustness of these preliminary findings in more ecologically valid settings.
k = 3), which revealed significant differences in skills test scores and response patterns across clusters. The findings suggest that certain psychological traits may serve as auxiliary indicators of cybersecurity skill. Further research could explore this relationship using aggregated trait-level metrics and broader participant samples, including professional red-teamers, to examine the robustness of these preliminary findings in more ecologically valid settings.
Beltz, Brandon; Doty, Jim; Fonken, Yvonne; Gurney, Nikolos; Israelsen, Brett; Lau, Nathan; Marsella, Stacy; Thomas, Rachelle; Trent, Stoney; Wu, Peggy; Yang, Ya-Ting; Zhu, Quanyan
2025, (arXiv:2508.20963 [cs]).
Abstract | Links | BibTeX | Tags: DTIC, Security
@misc{beltz_guarding_2025,
title = {Guarding Against Malicious Biased Threats (GAMBiT) Experiments: Revealing Cognitive Bias in Human-Subjects Red-Team Cyber Range Operations},
author = {Brandon Beltz and Jim Doty and Yvonne Fonken and Nikolos Gurney and Brett Israelsen and Nathan Lau and Stacy Marsella and Rachelle Thomas and Stoney Trent and Peggy Wu and Ya-Ting Yang and Quanyan Zhu},
url = {http://arxiv.org/abs/2508.20963},
doi = {10.48550/arXiv.2508.20963},
year = {2025},
date = {2025-08-01},
urldate = {2025-09-18},
publisher = {arXiv},
abstract = {We present three large-scale human-subjects red-team cyber range datasets from the Guarding Against Malicious Biased Threats (GAMBiT) project. Across Experiments 1-3 (July 2024-March 2025), 19-20 skilled attackers per experiment conducted two 8-hour days of self-paced operations in a simulated enterprise network (SimSpace Cyber Force Platform) while we captured multi-modal data: self-reports (background, demographics, psychometrics), operational notes, terminal histories, keylogs, network packet captures (PCAP), and NIDS alerts (Suricata). Each participant began from a standardized Kali Linux VM and pursued realistic objectives (e.g., target discovery and data exfiltration) under controlled constraints. Derivative curated logs and labels are included. The combined release supports research on attacker behavior modeling, bias-aware analytics, and method benchmarking. Data are available via IEEE Dataport entries for Experiments 1-3.},
note = {arXiv:2508.20963 [cs]},
keywords = {DTIC, Security},
pubstate = {published},
tppubtype = {misc}
}
We present three large-scale human-subjects red-team cyber range datasets from the Guarding Against Malicious Biased Threats (GAMBiT) project. Across Experiments 1-3 (July 2024-March 2025), 19-20 skilled attackers per experiment conducted two 8-hour days of self-paced operations in a simulated enterprise network (SimSpace Cyber Force Platform) while we captured multi-modal data: self-reports (background, demographics, psychometrics), operational notes, terminal histories, keylogs, network packet captures (PCAP), and NIDS alerts (Suricata). Each participant began from a standardized Kali Linux VM and pursued realistic objectives (e.g., target discovery and data exfiltration) under controlled constraints. Derivative curated logs and labels are included. The combined release supports research on attacker behavior modeling, bias-aware analytics, and method benchmarking. Data are available via IEEE Dataport entries for Experiments 1-3.