Weak scaling performance, ReaxFF HNS benchmark, Haswell, 128K atoms/node
Performance in millions of atom-timesteps / second / node

Nodes CPU (mpi,hyper) OMP (mpi,hyper,thread) Kokkos/OMP (mpi,hyper,thread) Kokkos/serial (mpi,hyper)
1 0.2557 (64,2) 0.2822 (16,2,4) 0.2406 (32,2,2) 0.3096 (64,2)
2 0.2494 (64,2) 0.2829 (16,2,4) 0.2385 (32,2,2) 0.3017 (64,2)
4 0.2506 (64,2) 0.2797 (16,2,4) 0.2305 (32,2,2) 0.2977 (64,2)
8 0.2486 (64,2) 0.2776 (16,2,4) 0.2285 (32,2,2) 0.2925 (64,2)
16 0.2463 (64,2) 0.2755 (16,2,4) 0.2234 (32,2,2) 0.2846 (64,2)
32 0.2467 (64,2) 0.2733 (16,2,4) 0.2252 (32,2,2) 0.2855 (64,2)
64 0.2447 (64,2) 0.2733 (16,2,4) 0.2237 (32,2,2) 0.2844 (64,2)

Run commands and logfile links for column CPU

1 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=weak.size=128K.node=1.mpi=64.hyper=2
2 srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=weak.size=128K.node=2.mpi=64.hyper=2
4 srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=weak.size=128K.node=4.mpi=64.hyper=2
8 srun -n 512 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=weak.size=128K.node=8.mpi=64.hyper=2
16 srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 16 -v y 16 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=weak.size=128K.node=16.mpi=64.hyper=2
32 srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 16 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=weak.size=128K.node=32.mpi=64.hyper=2
64 srun -n 4096 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 32 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=weak.size=128K.node=64.mpi=64.hyper=2

Run commands and logfile links for column OMP

1 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=weak.size=128K.node=1.mpi=16.thread=4.hyper=2
2 setenv OMP_NUM_THREADS 4; srun -n 32 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=weak.size=128K.node=2.mpi=16.thread=4.hyper=2
4 setenv OMP_NUM_THREADS 4; srun -n 64 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=weak.size=128K.node=4.mpi=16.thread=4.hyper=2
8 setenv OMP_NUM_THREADS 4; srun -n 128 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=weak.size=128K.node=8.mpi=16.thread=4.hyper=2
16 setenv OMP_NUM_THREADS 4; srun -n 256 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 16 -v y 16 -v z 24 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=weak.size=128K.node=16.mpi=16.thread=4.hyper=2
32 setenv OMP_NUM_THREADS 4; srun -n 512 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 16 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=weak.size=128K.node=32.mpi=16.thread=4.hyper=2
64 setenv OMP_NUM_THREADS 4; srun -n 1024 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 32 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=weak.size=128K.node=64.mpi=16.thread=4.hyper=2

Run commands and logfile links for column Kokkos/OMP

1 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=weak.size=128K.node=1.mpi=32.thread=2.hyper=2
2 setenv OMP_NUM_THREADS 2; srun -n 64 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=weak.size=128K.node=2.mpi=32.thread=2.hyper=2
4 setenv OMP_NUM_THREADS 2; srun -n 128 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=weak.size=128K.node=4.mpi=32.thread=2.hyper=2
8 setenv OMP_NUM_THREADS 2; srun -n 256 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=weak.size=128K.node=8.mpi=32.thread=2.hyper=2
16 setenv OMP_NUM_THREADS 2; srun -n 512 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 16 -v y 16 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=weak.size=128K.node=16.mpi=32.thread=2.hyper=2
32 setenv OMP_NUM_THREADS 2; srun -n 1024 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 16 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=weak.size=128K.node=32.mpi=32.thread=2.hyper=2
64 setenv OMP_NUM_THREADS 2; srun -n 2048 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 32 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=weak.size=128K.node=64.mpi=32.thread=2.hyper=2

Run commands and logfile links for column Kokkos/serial

1 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=1.mpi=64.hyper=2
2 srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=2.mpi=64.hyper=2
4 srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=4.mpi=64.hyper=2
8 srun -n 512 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=8.mpi=64.hyper=2
16 srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 16 -v y 16 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=16.mpi=64.hyper=2
32 srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 16 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=32.mpi=64.hyper=2
64 srun -n 4096 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 32 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=64.mpi=64.hyper=2