From e4b8f63f906990f73d8af20be386d5478b4f56ab Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 17 Jun 2026 22:23:45 +0000 Subject: [PATCH 01/11] Add compile definition MPAS_OPENMP is build option to use OpenMP is on --- cmake/Functions/MPAS_Functions.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/Functions/MPAS_Functions.cmake b/cmake/Functions/MPAS_Functions.cmake index 15d9f63fc1..6ba923273e 100644 --- a/cmake/Functions/MPAS_Functions.cmake +++ b/cmake/Functions/MPAS_Functions.cmake @@ -92,6 +92,7 @@ function(mpas_fortran_target target) list(APPEND MPAS_FORTRAN_TARGET_COMPILE_DEFINITIONS _MPI=1) # Enable OpenMP support if(MPAS_OPENMP) + list(APPEND MPAS_FORTRAN_TARGET_COMPILE_DEFINITIONS MPAS_OPENMP=1) target_link_libraries(${target} PUBLIC OpenMP::OpenMP_Fortran) endif() From 36f11f3d5f6a68b7fb9aaa46173c55276793985c Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Sat, 20 Jun 2026 20:18:26 +0000 Subject: [PATCH 02/11] Make aer_opt and ysu_pblmix in mpas_atmphys_vars thread private ThreadSanitizer warns about a potential data race. Two variables (aer_opt and ysu_pblmix) are defined in mpas_atmphys_vars module and are global (a single copy shared by all threads). In this specific case all threads assign 0 to both variables, which means even if there's a data race the result will be the same regardless of the thread count. But I think this should be fixed if nothing else then to have clean(er) sanitizer report, which will make it easier to spot the actual data races later on. 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=3577273) 0: Write of size 4 at 0x000005375c28 by thread T2: 0: #0 mpas_atmphys_driver_radiation_sw::radiation_sw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:471:2 (mpas_atmosphere+0xc805f7) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #1 mpas_atmphys_driver_radiation_sw::driver_radiation_sw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:877:7 (mpas_atmosphere+0xca19f1) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.102086.split2131 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:231:16 (mpas_atmosphere+0xa99c86) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:229:7 (mpas_atmosphere+0xa8fdfd) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: 0: Previous write of size 4 at 0x000005375c28 by main thread: 0: #0 mpas_atmphys_driver_radiation_sw::radiation_sw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:471:2 (mpas_atmosphere+0xc805f7) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #1 mpas_atmphys_driver_radiation_sw::driver_radiation_sw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:877:7 (mpas_atmosphere+0xca19f1) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.102086.split2131 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:231:16 (mpas_atmosphere+0xa99c86) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:229:7 (mpas_atmosphere+0xa8fdfd) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #5 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1036:15 (mpas_atmosphere+0xa10c41) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #6 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0xa0b767) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #7 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f92b3) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #8 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3169) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f3160) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #10 main (mpas_atmosphere+0x43aa4c) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: 0: Location is global 'mpas_atmphys_vars::aer_opt_' of size 4 at 0x000005375c28 (mpas_atmosphere+0x5375c28) 0: 0: Thread T2 'openmp_worker' (tid=3577284, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d4e) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x9fda57) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f8e34) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:471:2 in mpas_atmphys_driver_radiation_sw::radiation_sw_from_mpas_ 0: ================== 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=3577273) 0: Write of size 4 at 0x000005375c48 by thread T2: 0: #0 mpas_atmphys_driver_pbl::pbl_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_pbl.F:392:8 (mpas_atmosphere+0xbcbf08) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #1 mpas_atmphys_driver_pbl::driver_pbl_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_pbl.F:818:7 (mpas_atmosphere+0xc06258) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.38.split2117 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:324:16 (mpas_atmosphere+0xa9601a) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:322:7 (mpas_atmosphere+0xa92256) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: 0: Previous write of size 4 at 0x000005375c48 by main thread: 0: #0 mpas_atmphys_driver_pbl::pbl_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_pbl.F:392:8 (mpas_atmosphere+0xbcbf08) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #1 mpas_atmphys_driver_pbl::driver_pbl_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_pbl.F:818:7 (mpas_atmosphere+0xc06258) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.38.split2117 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:324:16 (mpas_atmosphere+0xa9601a) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:322:7 (mpas_atmosphere+0xa92256) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #5 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1036:15 (mpas_atmosphere+0xa10c41) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #6 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0xa0b767) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #7 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f92b3) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #8 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3169) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f3160) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #10 main (mpas_atmosphere+0x43aa4c) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: 0: Location is global 'mpas_atmphys_vars::ysu_pblmix_' of size 4 at 0x000005375c48 (mpas_atmosphere+0x5375c48) 0: 0: Thread T2 'openmp_worker' (tid=3577284, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d4e) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x9fda57) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f8e34) (BuildId: 1db7011d81c236acf1d054158b592f58bdcdcc55) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_pbl.F:392:8 in mpas_atmphys_driver_pbl::pbl_from_mpas_ 0: ================== --- src/core_atmosphere/physics/mpas_atmphys_vars.F | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core_atmosphere/physics/mpas_atmphys_vars.F b/src/core_atmosphere/physics/mpas_atmphys_vars.F index 134009f537..caee0479ef 100644 --- a/src/core_atmosphere/physics/mpas_atmphys_vars.F +++ b/src/core_atmosphere/physics/mpas_atmphys_vars.F @@ -382,6 +382,7 @@ module mpas_atmphys_vars !do not run urban physics, idiff is set to zero. integer:: ysu_pblmix +!$omp threadprivate(ysu_pblmix) integer,dimension(:,:),allocatable:: & kpbl_p !index of PBL top [-] @@ -670,6 +671,8 @@ module mpas_atmphys_vars integer,parameter:: taer_asy_opt = 3!input option for nwfa, nifa aerosol asymmetry factor. integer:: aer_opt !=[0,3] : 0 for no aerosols, 3 for "water-" and "ice-friendly" aerosols. +!$omp threadprivate(aer_opt) + integer,dimension(:,:),allocatable:: & taer_type_p !=[1,2,3]: 1 for rural, 2 is urban and 3 is maritime in WRF. In MPAS, !aer_type is initialized as a function of landmask (=1 over land; =2 over From 61b804b3091b5873c53f9a33aae49671f536a38e Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Sat, 20 Jun 2026 21:17:13 +0000 Subject: [PATCH 03/11] Assign value to dzs_p an pin_p on single thread only Two arrays (dzs_p and pin_p) are declared as one-dimensional arrays in mpas_atmphys_vars. Both are alllocated in driver_lsm and driver_radiation_lw/sw allocate routines, which are called outside parallel regions. They are global/shared arrays but access to their elements from different threads does not depend on horizontal indexes, which means every assignment must be done on a single thread only. Here are thread sanitizer reports about potential data races 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=448722) 0: Write of size 4 at 0x7f64b1c0df88 by thread T2: 0: #0 mpas_atmphys_driver_lsm::lsm_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F:361:5 (mpas_atmosphere+0xb6cf8b) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #1 mpas_atmphys_driver_lsm::driver_lsm_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F:697:7 (mpas_atmosphere+0xb9cf5e) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.262099.split2123 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:288:19 (mpas_atmosphere+0xa978bb) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:286:7 (mpas_atmosphere+0xa91023) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: 0: Previous write of size 4 at 0x7f64b1c0df88 by main thread: 0: #0 mpas_atmphys_driver_lsm::lsm_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F:361:5 (mpas_atmosphere+0xb6cf8b) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #1 mpas_atmphys_driver_lsm::driver_lsm_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F:697:7 (mpas_atmosphere+0xb9cf5e) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.262099.split2123 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:288:19 (mpas_atmosphere+0xa978bb) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:286:7 (mpas_atmosphere+0xa91023) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #5 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1036:15 (mpas_atmosphere+0xa10c41) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #6 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0xa0b767) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #7 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f92b3) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #8 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3169) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f3160) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #10 main (mpas_atmosphere+0x43aa4c) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: 0: Thread T2 'openmp_worker' (tid=448734, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d4e) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x9fda57) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f8e34) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F:361:5 in mpas_atmphys_driver_lsm::lsm_from_mpas_ 0: ================== 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=447023) 0: Write of size 4 at 0x7f310610d900 by thread T2: 0: #0 mpas_atmphys_driver_radiation_sw::radiation_sw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:602:14 (mpas_atmosphere+0xc8e4bf) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #1 mpas_atmphys_driver_radiation_sw::driver_radiation_sw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:877:7 (mpas_atmosphere+0xca309b) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.102086.split2131 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:231:16 (mpas_atmosphere+0xa99c86) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:229:7 (mpas_atmosphere+0xa8fdfd) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: 0: Previous write of size 4 at 0x7f310610d900 by main thread: 0: #0 mpas_atmphys_driver_radiation_sw::radiation_sw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:602:14 (mpas_atmosphere+0xc8e4bf) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #1 mpas_atmphys_driver_radiation_sw::driver_radiation_sw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:877:7 (mpas_atmosphere+0xca309b) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.102086.split2131 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:231:16 (mpas_atmosphere+0xa99c86) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:229:7 (mpas_atmosphere+0xa8fdfd) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #5 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1036:15 (mpas_atmosphere+0xa10c41) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #6 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0xa0b767) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #7 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f92b3) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #8 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3169) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f3160) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #10 main (mpas_atmosphere+0x43aa4c) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: 0: Thread T2 'openmp_worker' (tid=447034, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d4e) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x9fda57) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f8e34) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F:602:14 in mpas_atmphys_driver_radiation_sw::radiation_sw_from_mpas_ 0: ================== 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=447023) 0: Write of size 4 at 0x7f310610d900 by thread T2: 0: #0 mpas_atmphys_driver_radiation_lw::radiation_lw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:473:14 (mpas_atmosphere+0xc35420) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #1 mpas_atmphys_driver_radiation_lw::driver_radiation_lw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:842:7 (mpas_atmosphere+0xc52b75) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.142089.split2129 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:244:16 (mpas_atmosphere+0xa9925f) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:242:7 (mpas_atmosphere+0xa900b2) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: 0: Previous write of size 4 at 0x7f310610d900 by main thread: 0: #0 mpas_atmphys_driver_radiation_lw::radiation_lw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:473:14 (mpas_atmosphere+0xc35420) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #1 mpas_atmphys_driver_radiation_lw::driver_radiation_lw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:842:7 (mpas_atmosphere+0xc52b75) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.142089.split2129 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:244:16 (mpas_atmosphere+0xa9925f) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:242:7 (mpas_atmosphere+0xa900b2) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #5 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1036:15 (mpas_atmosphere+0xa10c41) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #6 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0xa0b767) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #7 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f92b3) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #8 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3169) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f3160) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #10 main (mpas_atmosphere+0x43aa4c) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: 0: Thread T2 'openmp_worker' (tid=447034, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d4e) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x9fda57) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f8e34) (BuildId: ebe0b2300b84755b756c0b6ea7e42da9068a3aa3) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:473:14 in mpas_atmphys_driver_radiation_lw::radiation_lw_from_mpas_ 0: ================== --- src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F | 2 ++ .../physics/mpas_atmphys_driver_radiation_lw.F | 6 ++++++ .../physics/mpas_atmphys_driver_radiation_sw.F | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F b/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F index 0116dcf561..d26ad7c714 100644 --- a/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F +++ b/src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F @@ -357,9 +357,11 @@ subroutine lsm_from_MPAS(configs,mesh,diag_physics,sfc_input,its,ite) !In Registry.xml, dzs is a function of nCells. In the Noah lsm scheme, dzs is independent !of cell locations: +!$omp single do n = 1,num_soils dzs_p(n) = dzs(n,its) enddo +!$omp end single do j = jts,jte do n = 1,num_soils diff --git a/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F b/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F index dfa327fea8..cc7742e725 100644 --- a/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F +++ b/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F @@ -469,9 +469,11 @@ subroutine radiation_lw_from_MPAS(xtime_s,configs,mesh,state,time_lev,diag_physi if(config_o3climatology) then !ozone mixing ratio: +!$omp single do k = 1, num_oznLevels pin_p(k) = pin(k) enddo +!$omp end single do j = jts,jte do k = 1, num_oznLevels do i = its,ite @@ -501,9 +503,11 @@ subroutine radiation_lw_from_MPAS(xtime_s,configs,mesh,state,time_lev,diag_physi if(allocated(p2d)) deallocate(p2d) if(allocated(o32d)) deallocate(o32d) else +!$omp single do k = 1, num_oznLevels pin_p(k) = 0.0_RKIND enddo +!$omp end single do j = jts,jte do k = 1, num_oznLevels do i = its,ite @@ -571,9 +575,11 @@ subroutine radiation_lw_from_MPAS(xtime_s,configs,mesh,state,time_lev,diag_physi endif !ozone mixing ratio: +!$omp single do k = 1, num_oznlevels pin_p(k) = pin(k) enddo +!$omp end single do n = 1, num_months do j = jts,jte do k = 1, num_oznlevels diff --git a/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F b/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F index 3a14275826..0498d15651 100644 --- a/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F +++ b/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F @@ -598,9 +598,11 @@ subroutine radiation_sw_from_MPAS(configs,mesh,state,time_lev,diag_physics,atm_i !ozone volume mixing ratio: if(config_o3climatology) then +!$omp single do k = 1, num_oznLevels pin_p(k) = pin(k) enddo +!$omp end single do j = jts,jte do k = 1, num_oznLevels do i = its,ite @@ -609,9 +611,11 @@ subroutine radiation_sw_from_MPAS(configs,mesh,state,time_lev,diag_physics,atm_i enddo enddo else +!$omp single do k = 1, num_oznLevels pin_p(k) = 0.0_RKIND enddo +!$omp end single do j = jts,jte do k = 1, num_oznLevels do i = its,ite @@ -672,9 +676,11 @@ subroutine radiation_sw_from_MPAS(configs,mesh,state,time_lev,diag_physics,atm_i enddo endif !ozone mixing ratio: +!$omp single do k = 1, num_oznlevels pin_p(k) = pin(k) enddo +!$omp end single do n = 1, num_months do j = jts,jte do k = 1, num_oznlevels From 142248c542e34758cdef81140cf6edab608a56c3 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Sat, 20 Jun 2026 21:37:02 +0000 Subject: [PATCH 04/11] Horizontal loop bounds in atmphys_sfc_diagnostics should be restricted to 'thread owned elements' These data race warnings are because two horizontal do loops in atmphys_sfc_diagnostics go from 1 to nCellsSolve These loops should actually iterate from its to ite. Threads must not write into horizontal array elements 'owned by other threads' 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=448722) 0: Write of size 4 at 0x7f649850fb84 by main thread: 0: #0 mpas_atmphys_sfc_diagnostics::atmphys_sfc_diagnostics_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F:85:14 (mpas_atmosphere+0x2686c1d) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #1 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.34.split2121 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:313:16 (mpas_atmosphere+0xa97098) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #2 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #3 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:311:7 (mpas_atmosphere+0xa91edb) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #4 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1036:15 (mpas_atmosphere+0xa10c41) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #5 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0xa0b767) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #6 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f92b3) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #7 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3169) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #8 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f3160) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #9 main (mpas_atmosphere+0x43aa4c) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: 0: Previous write of size 4 at 0x7f649850fb84 by thread T2: 0: #0 mpas_atmphys_sfc_diagnostics::atmphys_sfc_diagnostics_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F:85:14 (mpas_atmosphere+0x2686c1d) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #1 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.34.split2121 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:313:16 (mpas_atmosphere+0xa97098) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #2 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #3 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:311:7 (mpas_atmosphere+0xa91edb) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: 0: Thread T2 'openmp_worker' (tid=448734, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d4e) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x9fda57) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f8e34) (BuildId: 6accb5e96f6fa6dfe5760f5ea3bf82167d3b3be9) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F:85:14 in mpas_atmphys_sfc_diagnostics::atmphys_sfc_diagnostics_ 0: ================== --- src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F b/src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F index 6e3a84c394..0360157390 100644 --- a/src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F +++ b/src/core_atmosphere/physics/mpas_atmphys_sfc_diagnostics.F @@ -41,7 +41,6 @@ subroutine atmphys_sfc_diagnostics(configs,mesh,diag,diag_physics,sfc_input,outp !local variables and pointers: character(len=StrKIND),pointer:: lsm_scheme - integer,pointer:: nCellsSolve integer:: i real(kind=RKIND),dimension(:),pointer:: psfc @@ -58,8 +57,6 @@ subroutine atmphys_sfc_diagnostics(configs,mesh,diag,diag_physics,sfc_input,outp call mpas_pool_get_config(configs,'config_lsm_scheme',lsm_scheme) - call mpas_pool_get_dimension(mesh,'nCellsSolve',nCellsSolve) - call mpas_pool_get_array(diag,'surface_pressure',psfc) call mpas_pool_get_array(diag_physics,'chs2',chs2) @@ -77,7 +74,7 @@ subroutine atmphys_sfc_diagnostics(configs,mesh,diag,diag_physics,sfc_input,outp sf_select: select case(trim(lsm_scheme)) case("sf_noah") - do i = 1,nCellsSolve + do i = its,ite rho = psfc(i)/(R_d*tsk(i)) if(cqs2(i) .lt. 1.e-5) then q2(i) = qsfc(i) @@ -95,7 +92,7 @@ subroutine atmphys_sfc_diagnostics(configs,mesh,diag,diag_physics,sfc_input,outp case("sf_noahmp") call mpas_pool_get_array(output_noahmp,'q2mxy',q2mxy) call mpas_pool_get_array(output_noahmp,'t2mxy',t2mxy) - do i = 1,nCellsSolve + do i = its,ite rho = psfc(i)/(R_d*tsk(i)) if((xland(i)-1.5 .gt. 0._RKIND) .or. (xland(i)-1.5.le.0._RKIND .and. xice(i).ge.xice_threshold)) then if(cqs2(i) .lt. 1.e-5) then From 9303cecf34de147632f990cae2d8223825d21a58 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Sun, 21 Jun 2026 00:32:04 +0000 Subject: [PATCH 05/11] Fix passing of o3clim_p subarray to vinterp_ozn subroutine The shared array o3clim_p is allocated in allocate_radiation_lw with first dimension bounds 'ims:ime'. In radiation_lw_from_MPAS which runs on multiple threads, values are assigned only to 'thraad owned' array subsection 'its:ite'. When it's passed as an actual argument to vinterp_ozn, it is passed as call vinterp_ozn(1,ncols,ncols,nlevs,p2d,pin_p,num_oznlevels,o3clim_p(1,1,j),o32d) and the corresponding dummy argument in vinterp_ozn is ozmix_in dimensioned as real(kind=RKIND), intent(in) :: ozmix_in(pcols,levsiz) where pcols is equal to (ite-its+1). Passing o3clim_p with a lower bound of 1 for the first dimension is incorrect on all but the first thread. Actual argument must be an array subsection valid for a given thread, which is its:ite for the first dimension. 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=381796) 0: Read of size 4 at 0x7f5d70932064 by main thread: 0: #0 module_ra_rrtmg_vinterp::vinterp_ozn_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/physics_wrf/module_ra_rrtmg_vinterp.F:92:10 (mpas_atmosphere+0x9ee6af) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #1 mpas_atmphys_driver_radiation_lw::radiation_lw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:496:19 (mpas_atmosphere+0x6dc097) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #2 mpas_atmphys_driver_radiation_lw::driver_radiation_lw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:849:7 (mpas_atmosphere+0x6e57ca) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #3 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.16.split1594 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:244:16 (mpas_atmosphere+0x64571c) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #4 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #5 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:242:7 (mpas_atmosphere+0x64289d) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #6 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1036:15 (mpas_atmosphere+0x616631) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #7 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0x6142dc) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #8 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f5811) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3195) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #10 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f318b) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #11 main (mpas_atmosphere+0x43aa9c) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: 0: Previous write of size 4 at 0x7f5d70932064 by thread T2: 0: #0 mpas_atmphys_driver_radiation_lw::radiation_lw_from_mpas_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:480:17 (mpas_atmosphere+0x6db879) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #1 mpas_atmphys_driver_radiation_lw::driver_radiation_lw_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F:849:7 (mpas_atmosphere+0x6e57ca) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #2 mpas_atmphys_driver::physics_driver_.DIR.OMP.PARALLEL.LOOP.16.split1594 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:244:16 (mpas_atmosphere+0x64571c) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 mpas_atmphys_driver::physics_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver.F:242:7 (mpas_atmosphere+0x64289d) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: 0: Thread T2 'openmp_worker' (tid=381849, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d9e) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x60e765) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f511c) (BuildId: 661a0696f86a62568e541fd06a7b066a026681b3) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/physics_wrf/module_ra_rrtmg_vinterp.F:92:10 in module_ra_rrtmg_vinterp::vinterp_ozn_ 0: ================== --- src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F b/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F index cc7742e725..9091105a87 100644 --- a/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F +++ b/src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F @@ -493,7 +493,7 @@ subroutine radiation_lw_from_MPAS(xtime_s,configs,mesh,state,time_lev,diag_physi p2d(i,k) = pres_hyd_p(i,k,j) / 100._RKIND enddo enddo - call vinterp_ozn(1,ncols,ncols,nlevs,p2d,pin_p,num_oznlevels,o3clim_p(1,1,j),o32d) + call vinterp_ozn(1,ncols,ncols,nlevs,p2d,pin_p,num_oznlevels,o3clim_p(its:ite,1:num_oznLevels,j),o32d) do i = its,ite do k = kts,kte o3vmr(k,i) = o32d(i,k) From 52e357b476255c3efe6a642d3797a889bd4fb901 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Sun, 21 Jun 2026 20:53:29 +0000 Subject: [PATCH 06/11] Initialize the last horizontal element of a shared array (rho_zz) on a single thread This thread santitizer data race warning is about a shared array (rho_zz) being asssigened value at horizontal index nCell+1 in a routine (atm_recover_large_step_variables_work) that runs on all threads, so each thread is writing to the same memory location. Few other arrays that also set the value at index nCell+1 are now restricted to a single thread 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=3327569) 0: Write of size 8 at 0x7f4fc3698458 by main thread: 0: #0 atm_time_integration::atm_recover_large_step_variables_work_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:4390:10 (mpas_atmosphere+0xbd0728) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #1 atm_time_integration::atm_recover_large_step_variables_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:4294:12 (mpas_atmosphere+0xbcffe2) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #2 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.51.split10812 /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2427:21 (mpas_atmosphere+0xc1ad64) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2425:7 (mpas_atmosphere+0xba9d47) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #5 atm_time_integration::atm_timestep_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:1829:15 (mpas_atmosphere+0xba377f) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #6 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1044:12 (mpas_atmosphere+0x616655) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #7 atm_core::atm_core_run_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0x6142dc) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #8 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f5811) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #9 MAIN /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3195) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #10 MAIN /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f318b) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #11 main (mpas_atmosphere+0x43aa9c) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: 0: Previous write of size 8 at 0x7f4fc3698458 by thread T2: 0: #0 atm_time_integration::atm_recover_large_step_variables_work_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:4390:10 (mpas_atmosphere+0xbd0728) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #1 atm_time_integration::atm_recover_large_step_variables_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:4294:12 (mpas_atmosphere+0xbcffe2) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #2 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.51.split10812 /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2427:21 (mpas_atmosphere+0xc1ad64) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2425:7 (mpas_atmosphere+0xba9d47) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: 0: Thread T2 'openmp_worker' (tid=3327621, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d9e) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x60e765) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f511c) (BuildId: 1106bfe5a75efd6bf52d3fef2ee2ba7468c1b91f) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/fv3-cam/Dusan.Jovic/dev/mpas/simple-mpas/src/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:4390:10 in atm_time_integration::atm_recover_large_step_variables_work_ 0: ================== --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 227fbde862..b5b45d6a22 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4384,12 +4384,14 @@ subroutine atm_recover_large_step_variables_work(nCells, nEdges, nCellsSolve, nE ! Avoid FP errors caused by a potential division by zero below by ! initializing the "garbage cell" of rho_zz to a non-zero value +!$omp single !$acc parallel default(present) !$acc loop gang vector do k=1,nVertLevels rho_zz(k,nCells+1) = 1.0 end do !$acc end parallel +!$omp end single ! compute new density everywhere so we can compute u from ru. ! we will also need it to compute theta_m below @@ -5294,10 +5296,12 @@ subroutine atm_advance_scalars_mono_work(field_name, block, state, nCells, nEdge end do #ifndef MPAS_OPENACC +!$omp single do k=1,nVertLevels scalar_old(k,nCells+1) = 0.0_RKIND scalar_new(k,nCells+1) = 0.0_RKIND end do +!$omp end single #endif !$acc end parallel From affb8da9914b89c49c84248d93a6379014f1ad69 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Mon, 22 Jun 2026 21:08:23 +0000 Subject: [PATCH 07/11] Assign value to cellEnd+1 index of theta_m_1/2 only on last thread At two places in mpas_atm_time_integration.F, in atm_rk_integration_setup (theta_m_2) and in atm_rk_dynamics_substep_finish (theta_m_1) zero is assigned to the cellEnd+1 element of those arrays on all threads. But cellEnd+1 is the same memory location as cellStart on the following thread. To avoid data races only the last thread should be allowed to write to cellEnd+1. 4: ================== 4: WARNING: ThreadSanitizer: data race (pid=3062683) 4: Write of size 4 at 0x7f0f6ac5dbc0 by thread T3: 4: #0 atm_time_integration::atm_rk_dynamics_substep_finish_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:7757:16 (mpas_atmosphere+0xbfe47f) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #1 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.71.split10818 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2645:18 (mpas_atmosphere+0xc1c942) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #2 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 4: #3 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2643:7 (mpas_atmosphere+0xba65cd) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: 4: Previous write of size 4 at 0x7f0f6ac5dbc0 by thread T2: 4: #0 atm_time_integration::atm_rk_dynamics_substep_finish_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:7734:7 (mpas_atmosphere+0xbfcbf1) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #1 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.71.split10818 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2645:18 (mpas_atmosphere+0xc1c942) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #2 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 4: #3 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2643:7 (mpas_atmosphere+0xba65cd) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: 4: Thread T3 'openmp_worker' (tid=3062733, running) created by main thread at: 4: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d9e) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 4: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x60e765) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f511c) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: 4: Thread T2 'openmp_worker' (tid=3062727, running) created by main thread at: 4: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d9e) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 4: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x60e765) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f511c) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 4: 4: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:7757:16 in atm_time_integration::atm_rk_dynamics_substep_finish_ 4: ================== 1: ================== 1: WARNING: ThreadSanitizer: data race (pid=3062680) 1: Write of size 4 at 0x7f5feac7ab28 by thread T7: 1: #0 atm_time_integration::atm_rk_integration_setup_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3158:13 (mpas_atmosphere+0xbb45bb) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #1 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.210748.split10795 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2113:15 (mpas_atmosphere+0xc17879) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #2 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 1: #3 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2111:7 (mpas_atmosphere+0xba60c3) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: 1: Previous write of size 4 at 0x7f5feac7ab28 by thread T6: 1: #0 atm_time_integration::atm_rk_integration_setup_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3139:7 (mpas_atmosphere+0xbb2c99) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #1 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.210748.split10795 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2113:15 (mpas_atmosphere+0xc17879) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #2 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 1: #3 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2111:7 (mpas_atmosphere+0xba60c3) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: 1: Thread T7 'openmp_worker' (tid=3062773, running) created by main thread at: 1: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d9e) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 1: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x60e765) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f511c) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: 1: Thread T6 'openmp_worker' (tid=3062764, running) created by main thread at: 1: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d9e) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 1: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x60e765) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f511c) (BuildId: 1beca6f25de1e63cd10f0c4f537870fb672cadc3) 1: 1: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3158:13 in atm_time_integration::atm_rk_integration_setup_ 1: ================== --- .../dynamics/mpas_atm_time_integration.F | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index b5b45d6a22..b71336ad99 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -16,6 +16,10 @@ module atm_time_integration +#ifdef MPAS_OPENMP + use omp_lib +#endif + use mpas_pool_routines use mpas_kind_types use mpas_constants @@ -3135,9 +3139,18 @@ subroutine atm_rk_integration_setup( state, diag, nVertLevels, num_scalars, & call mpas_pool_get_array(state, 'scalars', scalars_2, 2) - !$acc kernels - theta_m_2(:,cellEnd+1) = 0.0_RKIND - !$acc end kernels +#ifdef MPAS_OPENMP + ! Assign 0 to cellEnd+1 element only on the last thread, because on + ! all other threads cellEnd+1 is the same memory location + ! as cellStart on the following threads + if (omp_get_thread_num() == omp_get_num_threads() - 1) then +#endif + !$acc kernels + theta_m_2(:,cellEnd+1) = 0.0_RKIND + !$acc end kernels +#ifdef MPAS_OPENMP + end if +#endif !$acc parallel default(present) !$acc loop gang worker @@ -7728,11 +7741,20 @@ subroutine atm_rk_dynamics_substep_finish( state, diag, nVertLevels, dynamics_su call mpas_pool_get_array(state, 'rho_zz', rho_zz_1, 1) call mpas_pool_get_array(state, 'rho_zz', rho_zz_2, 2) - ! Interim fix for the atm_compute_dyn_tend_work subroutine accessing uninitialized values - ! in garbage cells of theta_m - !$acc kernels - theta_m_1(:,cellEnd+1) = 0.0_RKIND - !$acc end kernels +#ifdef MPAS_OPENMP + ! Assign 0 to cellEnd+1 element only on the last thread, because on + ! all other threads cellEnd+1 is the same memory location + ! as cellStart on the following threads + if (omp_get_thread_num() == omp_get_num_threads() - 1) then +#endif + ! Interim fix for the atm_compute_dyn_tend_work subroutine accessing uninitialized values + ! in garbage cells of theta_m + !$acc kernels + theta_m_1(:,cellEnd+1) = 0.0_RKIND + !$acc end kernels +#ifdef MPAS_OPENMP + end if +#endif inv_dynamics_split = 1.0_RKIND / real(dynamics_split) From 8076726f3328d166eae27a26b1d49991bc0e8517 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Mon, 22 Jun 2026 22:54:56 +0000 Subject: [PATCH 08/11] Fix data race in mpas_atm_time_integration.F (cofrz array) Vertical array cofrz is shared and only single thread should assign the values 1: ================== 1: WARNING: ThreadSanitizer: data race (pid=3066462) 1: Write of size 4 at 0x7f29eed0de00 by thread T2: 1: #0 atm_time_integration::atm_compute_vert_imp_coefs_work_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3442:10 (mpas_atmosphere+0x21190fd) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #1 atm_time_integration::atm_compute_vert_imp_coefs_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3368:12 (mpas_atmosphere+0x21184e6) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #2 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.1013079.split13105 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2191:18 (mpas_atmosphere+0x225cd69) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 1: #4 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2189:7 (mpas_atmosphere+0x20ecc60) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: 1: Previous write of size 4 at 0x7f29eed0de00 by main thread: 1: #0 atm_time_integration::atm_compute_vert_imp_coefs_work_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3442:10 (mpas_atmosphere+0x21190fd) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #1 atm_time_integration::atm_compute_vert_imp_coefs_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3368:12 (mpas_atmosphere+0x21184e6) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #2 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.1013079.split13105 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2191:18 (mpas_atmosphere+0x225cd69) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 1: #4 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2189:7 (mpas_atmosphere+0x20ecc60) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #5 atm_time_integration::atm_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:1833:15 (mpas_atmosphere+0x20e5e57) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #6 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1044:12 (mpas_atmosphere+0xa0e42b) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #7 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0xa09076) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #8 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f9170) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f316c) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #10 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f315d) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #11 main (mpas_atmosphere+0x43aa4c) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: 1: Thread T2 'openmp_worker' (tid=3066514, running) created by main thread at: 1: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d4e) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 1: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x9fb576) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f8d18) (BuildId: f8ba73f4bd7c32ffd1a891f5c1fd8932ce0ca4cc) 1: 1: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:3442:10 in atm_time_integration::atm_compute_vert_imp_coefs_work_ 1: ================== --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index b71336ad99..4d3bd972d9 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -3435,6 +3435,7 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, rcv = rgas/(cp-rgas) c2 = cp*rcv +!$omp single !$acc parallel default(present) !$acc loop gang worker ! MGD bad to have all threads setting this variable? @@ -3442,6 +3443,7 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, cofrz(k) = rdzw(k) end do !$acc end parallel +!$omp end single !$acc parallel default(present) !$acc loop gang worker private(b_tri,c_tri) From 75ab74988377b9c37333bfa7f1aafb8a9a6187bd Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 23 Jun 2026 13:31:00 +0000 Subject: [PATCH 09/11] Make two module variables (Nt_c and mu_c) thread private in module_mp_thompson Two variables declared in MODULE module_mp_thompson are shared across all threads, and are assigned values from different threads in mp_gt_driver. Declare them as thread private. There are many other variables (scalars or arrays) declared in module_mp_thompson that are also shared, but thread sanitizer did not report potential data races. This could be for several reasons. Either they are assigned values only in init routine which is executed only on the main thread (fe. look up tables), or they are multidimensional arrays that have one 'tile' dimension and different tile elements are accessed on different threads, or maybe in my configuration that I'm currently running a certain code path is never executed. 0: ================== 0: WARNING: ThreadSanitizer: data race (pid=3071161) 0: Write of size 4 at 0x00000299bb78 by thread T2: 0: #0 module::thompson::mp_gt_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F:1145:10 (mpas_atmosphere+0x852a06) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #1 mpas_atmphys_driver_microphysics::driver_microphysics_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_microphysics.F:391:13 (mpas_atmosphere+0xd321cd) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #2 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.83.split10822 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2815:17 (mpas_atmosphere+0xc1d45c) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2813:7 (mpas_atmosphere+0xbaf36a) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: 0: Previous write of size 4 at 0x00000299bb78 by main thread: 0: #0 module::thompson::mp_gt_driver_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F:1145:10 (mpas_atmosphere+0x852a06) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #1 mpas_atmphys_driver_microphysics::driver_microphysics_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/mpas_atmphys_driver_microphysics.F:391:13 (mpas_atmosphere+0xd321cd) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #2 atm_time_integration::atm_srk3_.DIR.OMP.PARALLEL.LOOP.83.split10822 /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2815:17 (mpas_atmosphere+0xc1d45c) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #3 __kmp_invoke_microtask (libiomp5.so+0x14fdc8) 0: #4 atm_time_integration::atm_srk3_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:2813:7 (mpas_atmosphere+0xbaf36a) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #5 atm_time_integration::atm_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/dynamics/mpas_atm_time_integration.F:1833:15 (mpas_atmosphere+0xba3b1f) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #6 atm_core::atm_do_timestep_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:1044:12 (mpas_atmosphere+0x616655) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #7 atm_core::atm_core_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:816:15 (mpas_atmosphere+0x6142dc) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #8 mpas_subdriver::mpas_run_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:416:14 (mpas_atmosphere+0x4f5811) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #9 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:20:9 (mpas_atmosphere+0x4f3195) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #10 MAIN /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas.F:18:9 (mpas_atmosphere+0x4f318b) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #11 main (mpas_atmosphere+0x43aa9c) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: 0: Location is global 'module::thompson::nt_c_' of size 4 at 0x00000299bb78 (mpas_atmosphere+0x299bb78) 0: 0: Thread T2 'openmp_worker' (tid=3071208, running) created by main thread at: 0: #0 pthread_create /netbatch/donb58527_00/runDir/dir/workspace/NIT/xmain-rel/LX/xmainefi2linux_release/ws/icsws/llvm/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1090:3 (mpas_atmosphere+0x442d9e) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #1 __kmp_create_worker /localdisk/promo-build/20250910/tmp/lin_32e-rtl_int_5_nor_dyn.rel.c0.s0.tcm1.t1..h1.u1-anompclxlin01/../../src/z_Linux_util.cpp:1015:7 (libiomp5.so+0x1508e5) 0: #2 atm_core::atm_core_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/mpas_atm_core.F:78:12 (mpas_atmosphere+0x60e765) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: #3 mpas_subdriver::mpas_init_ /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/driver/mpas_subdriver.F:394:14 (mpas_atmosphere+0x4f511c) (BuildId: 99aefae946b3e065f9008f4ae5a97210f82d7b01) 0: 0: SUMMARY: ThreadSanitizer: data race /scratch3/NCEPDEV/stmp/Dusan.Jovic/MPAS-Model-ncar-dev/src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F:1145:10 in module::thompson::mp_gt_driver_ 0: ================== --- src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F b/src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F index 8e24340501..2e569ea114 100644 --- a/src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F +++ b/src/core_atmosphere/physics/physics_wrf/module_mp_thompson.F @@ -110,6 +110,7 @@ MODULE module_mp_thompson ! REAL, PARAMETER, PRIVATE:: Nt_c = 100.E6 REAL, PARAMETER, PRIVATE:: Nt_c_max = 1999.E6 REAL, PRIVATE:: Nt_c + !$omp threadprivate(Nt_c) !..Declaration of constants for assumed CCN/IN aerosols when none in !.. the input data. Look inside the init routine for modifications @@ -127,6 +128,7 @@ MODULE module_mp_thompson REAL, PARAMETER, PRIVATE:: mu_g = 0.0 REAL, PARAMETER, PRIVATE:: mu_i = 0.0 REAL, PRIVATE:: mu_c + !$omp threadprivate(mu_c) !..Sum of two gamma distrib for snow (Field et al. 2005). !.. N(D) = M2**4/M3**3 * [Kap0*exp(-M2*Lam0*D/M3) From 567f6433bc4c8d7853016daa6356b6e843393f8e Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 30 Jun 2026 13:33:53 +0000 Subject: [PATCH 10/11] Fix argument passed from sfclay_mynn to sf_mynn_pre_run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit fixes a bug in 'sfclay_mynn routine' (in module_sf_mynn.F) where 3d arrays dimensioned with lower:upper bounds of the first dimension (ims:ime, ...)  are passed as an actual arguments to 'sf_mynn_pre_run' where dummy arguments are dimensioned with first dimension bounds its:ite. Inside the 'sf_mynn_pre_run' routine the values are assigned to those dummy arguments in loops: do  i = its,ite which means the first (ite-its+1) values, corresponding to (ims:ims+ite-its) values of the actual argument are modified, which is incorrect. In addition to being incorrect it is also producing diffetent results between runs running on different number of threads. There are three ways to fix this. Pass only the (its:ite, ...) subsection of those 3d arrays as actual arguments from 'sfclay_mynn' matching the dummy arguments' shape,or changing the lower:upper bounds of dummy arguments in sf_mynn_pre_run to (ims:ime, ...), or maybe simply assign the values to the resulting column arrays directly in 'sfclay_mynn' skipping the call to 'sf_mynn_pre_run' altogether. This commit implements the first option and also provides an alternative in #ifdef-ed code without calling 'sf_mynn_pre_run' at all. I'm not sure which option is preferable. In my opinion, unless I'm missingi some other reason, I do not see the need for a subroutine call here. --- .../physics/physics_wrf/module_sf_mynn.F | 59 ++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/src/core_atmosphere/physics/physics_wrf/module_sf_mynn.F b/src/core_atmosphere/physics/physics_wrf/module_sf_mynn.F index ed41320ac7..950746df1d 100644 --- a/src/core_atmosphere/physics/physics_wrf/module_sf_mynn.F +++ b/src/core_atmosphere/physics/physics_wrf/module_sf_mynn.F @@ -226,6 +226,8 @@ subroutine sfclay_mynn( & real(kind=RKIND),dimension(its:ite):: u10_hv,v10_hv,th2_hv,t2_hv,q2_hv,wstar_hv,qstar_hv real(kind=RKIND),dimension(its:ite):: cd_hv,cda_hv,ck_hv,cka_hv,ustm_hv + real(kind=RKIND),dimension(its:ite,1:kte):: dz8w_hv,u3d_hv,v3d_hv,p3d_hv,t3d_hv,rho3d_hv,qv3d_hv,qc3d_hv,pattern_spp_pbl_hv + !----------------------------------------------------------------------------------------------------------------- f_spp = .false. @@ -244,9 +246,60 @@ subroutine sfclay_mynn( & qstar_hv(i) = qstar(i,j) enddo - call sf_mynn_pre_run(its,ite,kte,itimestep,dz8w,u3d,v3d,p3d,t3d,rho3d,qv3d,qc3d,f_spp, & - pattern_spp_pbl,ust_hv,mol_hv,qsfc_hv,qstar_hv,dz8w1d,u1d,v1d,p1d,t1d,rho1d, & - qv1d,qc1d,rstoch1d,dz2w1d,u1d2,v1d2,errmsg,errflg) +#if 1 + dz8w_hv = dz8w(its:ite,1:kte,j) + u3d_hv = u3d(its:ite,1:kte,j) + v3d_hv = v3d(its:ite,1:kte,j) + p3d_hv = p3d(its:ite,1:kte,j) + t3d_hv = t3d(its:ite,1:kte,j) + rho3d_hv = rho3d(its:ite,1:kte,j) + qv3d_hv = qv3d(its:ite,1:kte,j) + qc3d_hv = qc3d(its:ite,1:kte,j) + if(f_spp) then + pattern_spp_pbl_hv = pattern_spp_pbl(its:ite,1:kte,j) + end if + + call sf_mynn_pre_run(its,ite,kte,itimestep, & + dz8w_hv,u3d_hv,v3d_hv,p3d_hv,t3d_hv,rho3d_hv,qv3d_hv,qc3d_hv, & + f_spp,pattern_spp_pbl_hv, & + ust_hv,mol_hv,qsfc_hv,qstar_hv, & + dz8w1d,u1d,v1d,p1d,t1d,rho1d,qv1d,qc1d,rstoch1d, & + dz2w1d,u1d2,v1d2,errmsg,errflg) +#else + do i = its,ite + dz8w1d(i) = dz8w(i,1,j) + u1d(i) = u3d(i,1,j) + v1d(i) = v3d(i,1,j) + qv1d(i) = qv3d(i,1,j) + qc1d(i) = qc3d(i,1,j) + p1d(i) = p3d(i,1,j) + t1d(i) = t3d(i,1,j) + rho1d(i) = rho3d(i,1,j) + !--- 2nd model level winds - for diags with high-resolution grids: + dz2w1d(i) = dz8w(i,2,j) + u1d2(i) = u3d(i,2,j) + v1d2(i) = v3d(i,2,j) + enddo + + if(f_spp) then + do i = its,ite + rstoch1d(i) = pattern_spp_pbl(i,1,j) + enddo + else + do i = its,ite + rstoch1d(i)=0._RKIND + enddo + endif + + if(itimestep == 1) then + do i = its,ite + ust_hv(i) = max(0.04*sqrt(u1d(i)*u1d(i) + v1d(i)*v1d(i)),0.001) + mol_hv(i) = 0._RKIND + qsfc_hv(i) = qv1d(i)/(1.+qv1d(i)) + qstar_hv(i) = 0._RKIND + enddo + endif +#endif !input arguments: do i = its,ite From 3234513815e38221ef0d956bf0f1dfcff41d0c50 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 30 Jun 2026 17:37:58 +0000 Subject: [PATCH 11/11] =?UTF-8?q?Fix=20bug=20(typo)=20in=C2=A0calc=5Fgradx?= =?UTF-8?q?u=5Fcell=20in=C2=A0module=20mpas=5Fpv=5Fdiagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When floating point trap flags are enabled with the thread sanitizer the following error stops the model execution: 0: forrtl: error (182): floating invalid - possible uninitialized real/complex variable. 0: Image              PC                Routine            Line        Source 0: mpas_atmosphere    000000000044B086  Unknown               Unknown  Unknown 0: mpas_atmosphere    000000000044B616  Unknown               Unknown  Unknown 0: libc.so.6          00007F886D086BF0  Unknown               Unknown  Unknown 0: mpas_atmosphere    000000000206F625  dotproduct                168  mpas_pv_diagnostics.F 0: mpas_atmosphere    000000000209D21D  calc_gradxu_cell         1032  mpas_pv_diagnostics.F 0: mpas_atmosphere    00000000020A6F72  calc_epv                 1232  mpas_pv_diagnostics.F 0: mpas_atmosphere    00000000020ABCB7  atm_compute_pv_di        1311  mpas_pv_diagnostics.F 0: mpas_atmosphere    000000000206EE6A  pv_diagnostics_co         144  mpas_pv_diagnostics.F 0: mpas_atmosphere    0000000001FB9A8A  mpas_atm_diag_com         116  mpas_atm_diagnostics_manager.F 0: mpas_atmosphere    0000000000A06BAB  atm_core_run              707  mpas_atm_core.F 0: mpas_atmosphere    00000000004F9171  mpas_run                  416  mpas_subdriver.F 0: mpas_atmosphere    00000000004F316D  mpas                       20  mpas.F 0: mpas_atmosphere    000000000043AA4D  Unknown               Unknown  Unknown 0: libc.so.6          00007F886D0715D0  Unknown               Unknown  Unknown 0: libc.so.6          00007F886D071680  __libc_start_main     Unknown  Unknown 0: mpas_atmosphere    000000000043A965  Unknown               Unknown  Unknown This is due to a typo in calc_gradxu_cell. It seems this bug was causing the differences between threaded and non-threaded runs --- src/core_atmosphere/diagnostics/mpas_pv_diagnostics.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core_atmosphere/diagnostics/mpas_pv_diagnostics.F b/src/core_atmosphere/diagnostics/mpas_pv_diagnostics.F index d21061b0fb..4a1f2283f3 100644 --- a/src/core_atmosphere/diagnostics/mpas_pv_diagnostics.F +++ b/src/core_atmosphere/diagnostics/mpas_pv_diagnostics.F @@ -1029,7 +1029,7 @@ subroutine calc_gradxu_cell(gradxu, addEarthVort, & du_dz = calc_vertDeriv_one(val0, valm, z0-zm) !v val0 = dotProduct(velCell0, xyzLocal(:,2),3) - valm = dotProduct(velCellp, xyzLocal(:,2),3) + valm = dotProduct(velCellm, xyzLocal(:,2),3) dv_dz = calc_vertDeriv_one(val0, valm, z0-zm) else !u