@Article{jcs.2026.086419,
AUTHOR = {John Ojo Ajayi, Grace Egenti},
TITLE = {Bias and False Positive Challenges in AI-Based Intrusion Detection Systems under Extreme Class Imbalance},
JOURNAL = {Journal of Cyber Security},
VOLUME = {8},
YEAR = {2026},
NUMBER = {1},
PAGES = {641--666},
URL = {http://www.techscience.com/JCS/v8n1/68752},
ISSN = {2579-0064},
ABSTRACT = {Artificial intelligence (AI)-based intrusion detection systems (IDS) have evolved to be fundamental in detecting cyberattacks in contemporary networks. Unfortunately, the significant class imbalance present in cybersecurity datasets may introduce unfair biases during model learning, producing unpredictable predictions and inflating false alarm alerts, which eventually hampers practical deployment. This research examines the influence of class imbalance mitigation on model performance, operational fairness, explainability, and the operational integrity of an AI-based IDS subject to extreme class imbalance conditions. Three learning strategies based on Random Forest (i.e., baseline, Synthetic Minority Over-sampling Technique (SMOTE), and cost-sensitive learning) are evaluated on the CIC-BCCC-NRC TabularIoTAttack-2024 dataset and integrated into a systematic evaluation process encompassing SHAP explainability, statistical validation, external validation, and deployment analysis. The evaluation outcomes establish that the Baseline Random Forest, SMOTE Random Forest, and Cost-Sensitive Random Forest achieved classification accuracies of 99.9278%, 99.8866%, and 99.9246%, respectively, with all three models achieving superior classification accuracy of over 99.88% but displaying variations in their operational behaviour. The Baseline Random Forest demonstrated overall strong predictive capability, while the SMOTE Random Forest had the highest balanced accuracy and specificity and the lowest false-positive rate. Although SMOTE achieved the best internal false-positive performance, it is presented as a comparative imbalance-mitigation strategy rather than the recommended deployment model. The Cost-Sensitive Random Forest showed the capacity to remain competent while sustaining a pragmatic equilibrium between threat identification performance and deployment utility. SHAP analysis further revealed distinct feature importance patterns across the three models. Temporal inter-arrival features were the dominant predictors for the Baseline Random Forest, whereas the SMOTE Random Forest emphasized packet-size related features and the Cost-Sensitive Random Forest was primarily influenced by ACK Flag Count, Src Port, and Flow Bytes/s, thereby improving model interpretability. Moreover, an important increase in false positive behaviours was also found when externally validated using 2250 previously unseen legitimate network flows from the BCCC-MalNetMem-2025 dataset, where the external False Positive Rate increased to 23.51% (95% Wilson confidence interval: 21.80%–25.31%) compared with the internal benchmark evaluation, revealing an important gap between benchmark evaluation and real-world deployment. The results herein prove that combining imbalance-aware learning, explainable AI, statistical validation, and external validation lead to a more dependable and believable framework for assessing and designing AI-based IDSs for real-world cybersecurity deployments.},
DOI = {10.32604/jcs.2026.086419}
}