Publications

@misc{chaubey_face-llava_2025,

title = {Face-LLaVA: Facial Expression and Attribute Understanding through Instruction Tuning},

author = {Ashutosh Chaubey and Xulang Guan and Mohammad Soleymani},

url = {https://arxiv.org/abs/2504.07198},

doi = {10.48550/ARXIV.2504.07198},

year  = {2025},

date = {2025-04-01},

urldate = {2025-04-15},

publisher = {arXiv},

abstract = {The human face plays a central role in social communication, necessitating the use of performant computer vision tools for human-centered applications. We propose Face-LLaVA, a multimodal large language model for face-centered, in-context learning, including facial expression and attribute recognition. Additionally, Face-LLaVA is able to generate natural language descriptions that can be used for reasoning. Leveraging existing visual databases, we first developed FaceInstruct-1M, a face-centered database for instruction tuning MLLMs for face processing. We then developed a novel face-specific visual encoder powered by Face-Region Guided Cross-Attention that integrates face geometry with local visual features. We evaluated the proposed method across nine different datasets and five different face processing tasks, including facial expression recognition, action unit detection, facial attribute detection, age estimation and deepfake detection. Face-LLaVA achieves superior results compared to existing open-source MLLMs and competitive performance compared to commercial solutions. Our model output also receives a higher reasoning rating by GPT under a zero-shot setting across all the tasks. Both our dataset and model wil be released at https://face-llava.github.io to support future advancements in social AI and foundational vision-language research.},

note = {Version Number: 1},

keywords = {DTIC, LLM},

pubstate = {published},

tppubtype = {misc}

}

@misc{brun_exploring_2025,

title = {Exploring Emotion-Sensitive LLM-Based Conversational AI},

author = {Antonin Brun and Ruying Liu and Aryan Shukla and Frances Watson and Jonathan Gratch},

url = {http://arxiv.org/abs/2502.08920},

doi = {10.48550/arXiv.2502.08920},

year  = {2025},

date = {2025-02-01},

urldate = {2025-02-20},

publisher = {arXiv},

abstract = {Conversational AI chatbots have become increasingly common within the customer service industry. Despite improvements in their emotional development, they often lack the authenticity of real customer service interactions or the competence of service providers. By comparing emotion-sensitive and emotion-insensitive LLM-based chatbots across 30 participants, we aim to explore how emotional sensitivity in chatbots influences perceived competence and overall customer satisfaction in service interactions. Additionally, we employ sentiment analysis techniques to analyze and interpret the emotional content of user inputs. We highlight that perceptions of chatbot trustworthiness and competence were higher in the case of the emotion-sensitive chatbot, even if issue resolution rates were not affected. We discuss implications of improved user satisfaction from emotion-sensitive chatbots and potential applications in support services.},

note = {arXiv:2502.08920 [cs]},

keywords = {AI, LLM},

pubstate = {published},

tppubtype = {misc}

}

@misc{tak_mechanistic_2025,

title = {Mechanistic Interpretability of Emotion Inference in Large Language Models},

author = {Ala N. Tak and Amin Banayeeanzade and Anahita Bolourani and Mina Kian and Robin Jia and Jonathan Gratch},

url = {http://arxiv.org/abs/2502.05489},

doi = {10.48550/arXiv.2502.05489},

year  = {2025},

date = {2025-02-01},

urldate = {2025-02-20},

publisher = {arXiv},

abstract = {Large language models (LLMs) show promising capabilities in predicting human emotions from text. However, the mechanisms through which these models process emotional stimuli remain largely unexplored. Our study addresses this gap by investigating how autoregressive LLMs infer emotions, showing that emotion representations are functionally localized to specific regions in the model. Our evaluation includes diverse model families and sizes and is supported by robustness checks. We then show that the identified representations are psychologically plausible by drawing on cognitive appraisal theory, a well-established psychological framework positing that emotions emerge from evaluations (appraisals) of environmental stimuli. By causally intervening on construed appraisal concepts, we steer the generation and show that the outputs align with theoretical and intuitive expectations. This work highlights a novel way to causally intervene and precisely shape emotional text generation, potentially benefiting safety and alignment in sensitive affective domains.},

note = {arXiv:2502.05489 [cs]},

keywords = {LLM},

pubstate = {published},

tppubtype = {misc}

}

@misc{addison_c-fedrag_2024,

title = {C-FedRAG: A Confidential Federated Retrieval-Augmented Generation System},

author = {Parker Addison and Minh-Tuan H. Nguyen and Tomislav Medan and Jinali Shah and Mohammad T. Manzari and Brendan McElrone and Laksh Lalwani and Aboli More and Smita Sharma and Holger R. Roth and Isaac Yang and Chester Chen and Daguang Xu and Yan Cheng and Andrew Feng and Ziyue Xu},

url = {http://arxiv.org/abs/2412.13163},

doi = {10.48550/arXiv.2412.13163},

year  = {2024},

date = {2024-12-01},

urldate = {2025-03-20},

publisher = {arXiv},

abstract = {Organizations seeking to utilize Large Language Models (LLMs) for knowledge querying and analysis often encounter challenges in maintaining an LLM fine-tuned on targeted, up-to-date information that keeps answers relevant and grounded. Retrieval Augmented Generation (RAG) has quickly become a feasible solution for organizations looking to overcome the challenges of maintaining proprietary models and to help reduce LLM hallucinations in their query responses. However, RAG comes with its own issues regarding scaling data pipelines across tiered-access and disparate data sources. In many scenarios, it is necessary to query beyond a single data silo to provide richer and more relevant context for an LLM. Analyzing data sources within and across organizational trust boundaries is often limited by complex data-sharing policies that prohibit centralized data storage, therefore, inhibit the fast and effective setup and scaling of RAG solutions. In this paper, we introduce Confidential Computing (CC) techniques as a solution for secure Federated Retrieval Augmented Generation (FedRAG). Our proposed Confidential FedRAG system (C-FedRAG) enables secure connection and scaling of a RAG workflows across a decentralized network of data providers by ensuring context confidentiality. We also demonstrate how to implement a C-FedRAG system using the NVIDIA FLARE SDK and assess its performance using the MedRAG toolkit and MIRAGE benchmarking dataset.},

note = {arXiv:2412.13163 [cs]},

keywords = {LLM},

pubstate = {published},

tppubtype = {misc}

}