@Article{cmc.2026.085839,
AUTHOR = {Junjie Wu, Yang Liu, Danyi Sheng, Shiwei Cheng},
TITLE = {A Multi-Modal Approach to Emotion Recognition Fusing EEG and Eye Movement in Virtual Reality},
JOURNAL = {Computers, Materials \& Continua},
VOLUME = {},
YEAR = {},
NUMBER = {},
PAGES = {{pages}},
URL = {http://www.techscience.com/cmc/online/detail/28079},
ISSN = {1546-2226},
ABSTRACT = {With the development of brain-computer interfaces (BCI), more and more studies are using electroencephalography (EEG) for emotion recognition. Traditional emotion recognition often uses 2D videos and pictures to stimulate emotions, which do not provide an immersive feeling. Virtual reality (VR) can provide a more immersive and realistic experience, and recent studies are beginning to utilize EEG for emotion recognition in VR. However, due to the limited information on single-modal features, it is not possible to fully recognize individual emotions. To address this problem, we proposed a multi-modal approach in VR, which utilized a VR scene featuring videos to stimulate participants’ emotions and simultaneously extracted and complementarily fused the features of EEG and eye movement. Then, we introduced a cross-attention mechanism to recognize multiple emotional states. The experimental results showed that our approach achieved 90.04% classification accuracy (for four emotional states) and 97.17% (for three emotional states) on the public dataset SEED-IV and the self-collected dataset VR-EED, respectively. This results not only outperformed the single-modal emotion recognition approach (pure EEG or pure eye movement-based) but also surpassed the traditional multi-modal emotion recognition approach (feature layer fusion or decision layer fusion-based). This verifies the superiority of the proposed fusion strategy and is valuable for multi-modal emotion recognition, promoting the development of affective computing applications in VR. This work demonstrates how emotion-aware VR systems can enable adaptive and context-aware interactions in immersive environments.},
DOI = {10.32604/cmc.2026.085839}
}