@inproceedings{8398b557f9e4483c88ad67d7bde697d3,
title = "Integration of LLMs for Multitasking Workload Prediction in Mixed Reality Environments",
abstract = "Multitasking in mixed reality (MR) environments introduces unique cognitive demands, particularly in workload management. Accurate workload prediction is critical for optimizing user experience, safety, and performance in such settings. This study proposes a novel framework that integrates large language models (LLMs) with traditional workload assessment tools to enhance prediction accuracy in MR multitasking scenarios. A multitasking experiment involving 36 participants was conducted, combining realworld and virtual tasks, with workload evaluated using NASA-TLX. To address limited sample sizes, synthetic data was generated using generative adversarial networks (GANs), enabling robust model training. We employed a hybrid deep learning model that integrates LLM-generated text embeddings with numerical features in a feedforward neural network (FNN). Results show that integrating LLMs, specifically BERT and GPT-2, significantly improves workload prediction accuracy, with a root mean square error (RMSE) reduction from 6.82 (FNN-only) to 0.95 (BERTintegrated model). The findings underscore the potential of LLMs to augment cognitive workload assessment, supporting more adaptive and scalable human-machine collaboration in MR environments.",
keywords = "LLMs, Mixed environment, NASA-TLX, Workload prediction",
author = "Safanah Abbas and Heejin Jeong and David He",
note = "Publisher Copyright: {\textcopyright} 2025, Prognostics and Health Management Society. All rights reserved.; 17th Annual Conference of the Prognostics and Health Management Society, PHM 2025 ; Conference date: 25-10-2025 Through 30-10-2025",
year = "2025",
doi = "10.36001/phmconf.2025.v17i1.4408",
language = "English (US)",
isbn = "9781936263295",
series = "Proceedings of the Annual Conference of the Prognostics and Health Management Society, PHM",
publisher = "Prognostics and Health Management Society",
number = "1",
editor = "Kulkarni, \{Chetan S.\} and Orchard, \{Marcos E.\}",
booktitle = "Proceedings of the Annual Conference of the Prognostics and Health Management Society, PHM",
edition = "1",
}