Weak scaling performance, ReaxFF HNS benchmark, 128K atoms/node
Performance in millions of atom-timesteps / second / node

Nodes Haswell KNL K80-1 P100-1
1 0.3096 (Kokkos/serial,mpi=64,hyper=2) 0.2608 (OMP/KNL,mpi=32,thread=8,hyper=4) 0.3454 (Kokkos/Cuda,mpi=2) 0.6664 (Kokkos/Cuda,mpi=1)
2 0.3017 (Kokkos/serial,mpi=64,hyper=2) 0.2572 (OMP/KNL,mpi=32,thread=8,hyper=4) 0.2674 (Kokkos/Cuda,mpi=2) 0.635 (Kokkos/Cuda,mpi=1)
4 0.2977 (Kokkos/serial,mpi=64,hyper=2) 0.2576 (OMP/KNL,mpi=32,thread=8,hyper=4) 0.281 (Kokkos/Cuda,mpi=2) 0.6079 (Kokkos/Cuda,mpi=1)
8 0.2925 (Kokkos/serial,mpi=64,hyper=2) 0.2571 (OMP/KNL,mpi=32,thread=8,hyper=4) None 0.6046 (Kokkos/Cuda,mpi=1)
16 0.2846 (Kokkos/serial,mpi=64,hyper=2) 0.2522 (OMP/KNL,mpi=32,thread=8,hyper=4) None None
32 0.2855 (Kokkos/serial,mpi=64,hyper=2) 0.2516 (OMP/KNL,mpi=32,thread=8,hyper=4) None None
64 0.2844 (Kokkos/serial,mpi=64,hyper=2) 0.2504 (OMP/KNL,mpi=32,thread=8,hyper=4) None None

Run commands and logfile links for column Haswell

1 srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=1.mpi=64.hyper=2
2 srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=2.mpi=64.hyper=2
4 srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=4.mpi=64.hyper=2
8 srun -n 512 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=8.mpi=64.hyper=2
16 srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 16 -v y 16 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=16.mpi=64.hyper=2
32 srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 16 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=32.mpi=64.hyper=2
64 srun -n 4096 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./lmp_mutrino_kokkos_serial -sf kk -k on -pk kokkos neigh half neigh/qeq half newton on comm no -v x 32 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=kokkos_serial.kind=weak.size=128K.node=64.mpi=64.hyper=2

Run commands and logfile links for column KNL

1 setenv OMP_NUM_THREADS 8; srun -n 32 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./lmp_mutrino_knl -sf omp -pk omp 8 -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp_knl.kind=weak.size=128K.node=1.mpi=32.thread=8.hyper=4
2 setenv OMP_NUM_THREADS 8; srun -n 64 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./lmp_mutrino_knl -sf omp -pk omp 8 -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp_knl.kind=weak.size=128K.node=2.mpi=32.thread=8.hyper=4
4 setenv OMP_NUM_THREADS 8; srun -n 128 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./lmp_mutrino_knl -sf omp -pk omp 8 -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp_knl.kind=weak.size=128K.node=4.mpi=32.thread=8.hyper=4
8 setenv OMP_NUM_THREADS 8; srun -n 256 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./lmp_mutrino_knl -sf omp -pk omp 8 -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp_knl.kind=weak.size=128K.node=8.mpi=32.thread=8.hyper=4
16 setenv OMP_NUM_THREADS 8; srun -n 512 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./lmp_mutrino_knl -sf omp -pk omp 8 -v x 16 -v y 16 -v z 24 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp_knl.kind=weak.size=128K.node=16.mpi=32.thread=8.hyper=4
32 setenv OMP_NUM_THREADS 8; srun -n 1024 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./lmp_mutrino_knl -sf omp -pk omp 8 -v x 16 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp_knl.kind=weak.size=128K.node=32.mpi=32.thread=8.hyper=4
64 setenv OMP_NUM_THREADS 8; srun -n 2048 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./lmp_mutrino_knl -sf omp -pk omp 8 -v x 32 -v y 32 -v z 24 -v t 100 -in in.reaxc.hns.omp.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=mutrino.pkg=omp_knl.kind=weak.size=128K.node=64.mpi=32.thread=8.hyper=4

Run commands and logfile links for column K80-1

1 mpirun -np 2 --npersocket 1 --bind-to core lmp_ride80_kokkos_cuda -sf kk -k on g 2 -pk kokkos neigh half neigh/qeq full newton on comm device binsize 11.0 -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.kokkos_cuda.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=ride80.pkg=kokkos_cuda.kind=weak.size=128K.node=1.mpi=2.gpu=2
2 mpirun -np 4 --npersocket 1 --bind-to core lmp_ride80_kokkos_cuda -sf kk -k on g 2 -pk kokkos neigh half neigh/qeq full newton on comm device binsize 11.0 -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.kokkos_cuda.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=ride80.pkg=kokkos_cuda.kind=weak.size=128K.node=2.mpi=2.gpu=2
4 mpirun -np 8 --npersocket 1 --bind-to core lmp_ride80_kokkos_cuda -sf kk -k on g 2 -pk kokkos neigh half neigh/qeq full newton on comm device binsize 11.0 -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.kokkos_cuda.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=ride80.pkg=kokkos_cuda.kind=weak.size=128K.node=4.mpi=2.gpu=2
8 None
16 None
32 None
64 None

Run commands and logfile links for column P100-1

1 mpirun -np 1 --npernode 1 --bind-to core lmp_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos neigh half neigh/qeq full newton on comm device binsize 11.0 -v x 8 -v y 8 -v z 6 -v t 100 -in in.reaxc.hns.kokkos_cuda.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=ride100.pkg=kokkos_cuda.kind=weak.size=128K.node=1.mpi=1.gpu=1
2 mpirun -np 2 --npernode 1 --bind-to core lmp_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos neigh half neigh/qeq full newton on comm device binsize 11.0 -v x 8 -v y 8 -v z 12 -v t 100 -in in.reaxc.hns.kokkos_cuda.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=ride100.pkg=kokkos_cuda.kind=weak.size=128K.node=2.mpi=1.gpu=1
4 mpirun -np 4 --npernode 1 --bind-to core lmp_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos neigh half neigh/qeq full newton on comm device binsize 11.0 -v x 8 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.kokkos_cuda.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=ride100.pkg=kokkos_cuda.kind=weak.size=128K.node=4.mpi=1.gpu=1
8 mpirun -np 8 --npernode 1 --bind-to core lmp_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos neigh half neigh/qeq full newton on comm device binsize 11.0 -v x 16 -v y 16 -v z 12 -v t 100 -in in.reaxc.hns.kokkos_cuda.steps -nocite -log log.lammps.date=17Jan18.model=hns.machine=ride100.pkg=kokkos_cuda.kind=weak.size=128K.node=8.mpi=1.gpu=1
16 None
32 None
64 None