Single node performance, ReaxFF HNS benchmark, Haswell
Performance in millions of atom-timesteps / second

Natoms CPU (mpi,hyper) OMP (mpi,hyper,thread) Kokkos/OMP (mpi,hyper,thread) Kokkos/serial (mpi,hyper)
3648 0.08565 (32,1) 0.1299 (16,2,4) 0.1017 (16,2,4) 0.1011 (32,1)
7296 0.1174 (32,1) 0.1643 (16,2,4) 0.1283 (16,2,4) 0.1404 (32,1)
14592 0.1466 (32,1) 0.2093 (16,2,4) 0.1646 (32,2,2) 0.1873 (32,1)
29184 0.1844 (64,2) 0.2276 (16,2,4) 0.1923 (32,2,2) 0.2316 (32,1)
58368 0.2254 (64,2) 0.2523 (16,2,4) 0.2151 (32,2,2) 0.268 (64,2)
116736 0.2547 (64,2) 0.2821 (16,2,4) 0.2399 (32,2,2) 0.3054 (64,2)
233472 0.2851 (64,2) 0.302 (16,2,4) 0.2643 (32,2,2) 0.3382 (64,2)
466944 0.3173 (64,2) 0.3209 (16,2,4) 0.278 (32,2,2) 0.3733 (64,2)
933888 0.3396 (64,2) 0.3396 (16,2,4) 0.291 (32,2,2) 0.3907 (64,2)

Run commands and logfile links for column CPU

3648 srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./lmp_mutrino_cpu -v x 2 -v y 2 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=4K.node=1.mpi=32.hyper=1
7296 srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./lmp_mutrino_cpu -v x 2 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=8K.node=1.mpi=32.hyper=1
14592 srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./lmp_mutrino_cpu -v x 4 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=16K.node=1.mpi=32.hyper=1
29184 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 4 -v y 4 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=32K.node=1.mpi=64.hyper=2
58368 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 4 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=64K.node=1.mpi=64.hyper=2
116736 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=128K.node=1.mpi=64.hyper=2
233472 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=256K.node=1.mpi=64.hyper=2
466944 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=512K.node=1.mpi=64.hyper=2
933888 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_cpu -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=cpu.kind=node.size=1M.node=1.mpi=64.hyper=2

Run commands and logfile links for column OMP

3648 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 2 -v y 2 -v z 3 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=4K.node=1.mpi=16.thread=4.hyper=2
7296 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 2 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=8K.node=1.mpi=16.thread=4.hyper=2
14592 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 4 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=16K.node=1.mpi=16.thread=4.hyper=2
29184 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 4 -v y 4 -v z 6 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=32K.node=1.mpi=16.thread=4.hyper=2
58368 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 4 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=64K.node=1.mpi=16.thread=4.hyper=2
116736 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=128K.node=1.mpi=16.thread=4.hyper=2
233472 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=256K.node=1.mpi=16.thread=4.hyper=2
466944 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=512K.node=1.mpi=16.thread=4.hyper=2
933888 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_cpu -sf omp -pk omp 4 -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp.kind=node.size=1M.node=1.mpi=16.thread=4.hyper=2

Run commands and logfile links for column Kokkos/OMP

3648 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_kokkos_omp -sf kk -k on t 4 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 2 -v y 2 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=4K.node=1.mpi=16.thread=4.hyper=2
7296 setenv OMP_NUM_THREADS 4; srun -n 16 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./lmp_mutrino_kokkos_omp -sf kk -k on t 4 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 2 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=8K.node=1.mpi=16.thread=4.hyper=2
14592 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 4 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=16K.node=1.mpi=32.thread=2.hyper=2
29184 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 4 -v y 4 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=32K.node=1.mpi=32.thread=2.hyper=2
58368 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 4 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=64K.node=1.mpi=32.thread=2.hyper=2
116736 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=128K.node=1.mpi=32.thread=2.hyper=2
233472 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=256K.node=1.mpi=32.thread=2.hyper=2
466944 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=512K.node=1.mpi=32.thread=2.hyper=2
933888 setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./lmp_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos neigh half neigh/qeq full newton on comm no -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_omp.kind=node.size=1M.node=1.mpi=32.thread=2.hyper=2

Run commands and logfile links for column Kokkos/serial

3648 srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 2 -v y 2 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=4K.node=1.mpi=32.hyper=1
7296 srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 2 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=8K.node=1.mpi=32.hyper=1
14592 srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 4 -v y 4 -v z 3 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=16K.node=1.mpi=32.hyper=1
29184 srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 4 -v y 4 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=32K.node=1.mpi=32.hyper=1
58368 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 4 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=64K.node=1.mpi=64.hyper=2
116736 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=128K.node=1.mpi=64.hyper=2
233472 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=256K.node=1.mpi=64.hyper=2
466944 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=512K.node=1.mpi=64.hyper=2
933888 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=node.size=1M.node=1.mpi=64.hyper=2