From 80c49985659c5a580705a46ffba9cbf462c13e15 Mon Sep 17 00:00:00 2001 From: Carlo Camilloni Date: Sun, 15 Nov 2026 21:44:16 +0000 Subject: [PATCH 01/26] openaccPTM: split the tasks of the GPU actions between the MPI processes With MPI each process computes a contiguous block of the active tasks on its own GPU and the value stashes and the forces are summed, as in the CPU task manager; with SERIAL (or one process) the first process computes everything and broadcasts the results, as before. Only the part of the value stash touched by the block is cleared, uploaded and downloaded. - The low memory force gathering (AVOID_THREAD_GATHER, the default on the GPU) works with MPI: in the second pass only the first process keeps the forces of the first one; a process without tasks in the first pass uploads the positions in the second. - GPU choice: without an MD code and with several GPUs on the node (e.g. mpirun -np 4 plumed driver), process i of the node uses GPU i % nGPUs (local rank from the launcher environment), unless ACC_DEVICE_NUM is set. - Shortcuts: DISTANCES, Q1-Q6, LOCAL_Q*, LOCAL_AVERAGE, TETRAHEDRAL-like shortcuts accept SERIAL and pass it with USEGPU; COORDINATIONNUMBER and COORDINATION_MOMENTS accept USEGPU; Q* and LOCAL_AVERAGE put the contact matrix on the GPU too (as LOCAL_Q* already did). - Regtests rt-GPU-mpi-nogather (4 processes, a process without rows), rt-GPU-mpi-serial and rt-GPU-mpi-shortcuts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016vwxtUCrVzSZSsREbiW4jL --- CHANGES/v2.11.md | 2 + plugins/openaccPTM/README.md | 17 +- .../targetGPU/rt-GPU-mpi-nogather/Makefile | 1 + .../rt-GPU-mpi-nogather/colvar.reference | 6 + .../targetGPU/rt-GPU-mpi-nogather/config | 6 + .../rt-GPU-mpi-nogather/forces.0.reference | 550 ++++++++++++++++++ .../rt-GPU-mpi-nogather/forces.1.reference | 550 ++++++++++++++++++ .../rt-GPU-mpi-nogather/forces.2.reference | 550 ++++++++++++++++++ .../rt-GPU-mpi-nogather/forces.3.reference | 550 ++++++++++++++++++ .../targetGPU/rt-GPU-mpi-nogather/plumed.dat | 19 + .../targetGPU/rt-GPU-mpi-serial/Makefile | 1 + .../rt-GPU-mpi-serial/colvar.reference | 6 + .../targetGPU/rt-GPU-mpi-serial/config | 6 + .../rt-GPU-mpi-serial/forces.0.reference | 550 ++++++++++++++++++ .../rt-GPU-mpi-serial/forces.1.reference | 550 ++++++++++++++++++ .../targetGPU/rt-GPU-mpi-serial/plumed.dat | 17 + .../targetGPU/rt-GPU-mpi-shortcuts/Makefile | 1 + .../rt-GPU-mpi-shortcuts/colvar.reference | 2 + .../targetGPU/rt-GPU-mpi-shortcuts/config | 6 + .../rt-GPU-mpi-shortcuts/forces.0.reference | 66 +++ .../rt-GPU-mpi-shortcuts/forces.1.reference | 66 +++ .../rt-GPU-mpi-shortcuts/forces.2.reference | 66 +++ .../targetGPU/rt-GPU-mpi-shortcuts/plumed.dat | 9 + plugins/openaccPTM/src/ACCDevice.cpp | 37 ++ plugins/openaccPTM/src/ACCDevice.h | 7 +- .../openaccPTM/src/ACCParallelTaskManager.h | 169 ++++-- src/core/ActionWithMatrix.cpp | 6 +- src/core/ActionWithMatrix.h | 3 +- src/multicolvar/Distances.cpp | 6 + src/symfunc/CoordShellVectorFunction.cpp | 8 +- src/symfunc/CoordinationNumbers.cpp | 17 +- src/symfunc/CoordinationNumbers.h | 4 +- src/symfunc/LocalAverage.cpp | 9 +- src/symfunc/LocalSteinhardt.cpp | 6 +- src/symfunc/Steinhardt.cpp | 21 +- 35 files changed, 3814 insertions(+), 76 deletions(-) create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/Makefile create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/colvar.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/config create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.0.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.1.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.2.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.3.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/plumed.dat create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/Makefile create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/colvar.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/config create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.0.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.1.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/plumed.dat create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/Makefile create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/colvar.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/config create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.0.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.1.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.2.reference create mode 100644 plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/plumed.dat diff --git a/CHANGES/v2.11.md b/CHANGES/v2.11.md index 7c80256e5f..ac60d6d926 100644 --- a/CHANGES/v2.11.md +++ b/CHANGES/v2.11.md @@ -81,6 +81,8 @@ The entries below give the details; the main cases are: - [READ](READ.md) has an `IGNORE_COMPONENT` flag, to read a component (e.g. `d.x`) as a plain value. - The experimental openaccPTM plugin (`plugins/openaccPTM`, compiled separately with the NVIDIA HPC SDK) adds the `USEGPU` flag to some actions, e.g. [DISTANCE](DISTANCE.md), [KDE](KDE.md), [SAXS](SAXS.md) and [SANS](SANS.md), computed on the GPU with OpenACC. + With MPI the tasks of these actions are split between the ranks, each using its own GPU (with `SERIAL` only the first rank uses its GPU), + and with several GPUs on a node and no MD code (e.g. `mpirun -np 4 plumed driver`) the ranks of the node use the GPUs in turn. #### Restart and checkpoints diff --git a/plugins/openaccPTM/README.md b/plugins/openaccPTM/README.md index 9020019105..d504786090 100644 --- a/plugins/openaccPTM/README.md +++ b/plugins/openaccPTM/README.md @@ -67,8 +67,11 @@ LOAD FILE=/path/to/plumedOpenACC.so CONTACT_MATRIX GROUP=1-1000 SWITCH={EXP D_0=0.2 R_0=0.1 D_MAX=0.66} USEGPU ``` -When PLUMED runs with MPI (e.g. an MD code with several MPI ranks), the actions with `USEGPU` are computed by rank 0 only, -on its GPU, and the results are sent to the other ranks. The other ranks do not use the GPU and wait for rank 0. +When PLUMED runs with MPI (e.g. an MD code with several MPI ranks, or `mpirun -np 4 plumed driver`), the tasks of an action +with `USEGPU` are split in blocks between the ranks, each rank computes its block on its GPU (see below), and the results are +summed, as for the CPU version. With `SERIAL` only the first rank computes, on its GPU, and the results are sent to the other +ranks, which do not use their GPU. Several ranks pay off when the action has many tasks; for a small action (a few thousands +of tasks or less) each call is dominated by fixed costs, and `SERIAL` (one GPU) can be faster. ### Which GPU is used @@ -77,8 +80,12 @@ several GPUs and do not interfere with an MD code that keeps its data on the GPU 1. the GPU assigned to PLUMED by the MD code (with the `setGpuDeviceId` command; the GROMACS 2024 patch does it), otherwise 2. the GPU the MD code is using (the one whose CUDA context is current when PLUMED is called, as for GROMACS 2025 and 2026), otherwise -3. the default OpenACC device: the first GPU, or the one set with `ACC_DEVICE_NUM` (e.g. for [driver](driver.md), where there is no MD code). - With `CUDA_VISIBLE_DEVICES` each rank can be given its own GPU. +3. with several GPUs on the node, and if `ACC_DEVICE_NUM` is not set, GPU number (local rank % number of GPUs), where the local rank + is the rank of the process within its node, as set by the MPI launcher (`OMPI_COMM_WORLD_LOCAL_RANK`, `MPI_LOCALRANKID`, `SLURM_LOCALID`, ...). + This is what happens with [driver](driver.md), where there is no MD code: e.g. `mpirun -np 4 plumed driver` on a node with 4 GPUs + uses one GPU per rank, otherwise +4. the default OpenACC device: the first GPU, or the one set with `ACC_DEVICE_NUM`. + With `CUDA_VISIBLE_DEVICES` (or `srun --gpus-per-task=1`) each rank can be given its own GPU. The GPU of an action is chosen the first time it is computed, and it does not change afterwards. It is written in the log, in the same way for all the actions, e.g. `d runs on GPU 1` for an action labelled `d` (or `d runs on the host` when OpenACC @@ -172,7 +179,7 @@ SAXS ... (This example is not highlighted only because the manual is built without the plugin.) - The GPU is chosen as for the other actions (see above): there is no `DEVICEID` keyword. -- Differently from the other actions, with MPI the rows of the Debye sum are split over the ranks, each one using its own +- As for the other actions, with MPI the rows of the Debye sum are split over the ranks, each one using its own GPU (or sharing one), and the results are summed. With `SERIAL` only the first rank computes on its GPU and the results are shared with the others. - With `DOSCORE` the derivatives of the intensities stay on the GPU and only the forces are copied back. Without it the diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/Makefile b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/Makefile new file mode 100644 index 0000000000..3703b27cea --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/Makefile @@ -0,0 +1 @@ +include ../../scripts/test.make diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/colvar.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/colvar.reference new file mode 100644 index 0000000000..4b6b6fab04 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/colvar.reference @@ -0,0 +1,6 @@ +#! FIELDS time s diag.vals-1 + 0.000000 0.13443 42.22566 + 1.000000 0.13067 41.67056 + 2.000000 0.10292 40.98616 + 3.000000 0.10205 40.05492 + 4.000000 0.09599 40.13433 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/config b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/config new file mode 100644 index 0000000000..7c673c7af0 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/config @@ -0,0 +1,6 @@ +type=driver +mpiprocs=4 +arg="--plumed plumed.dat --ixyz trajectory.xyz --dump-forces forces --dump-forces-fmt=%10.5f" +extra_files="../../trajectories/trajectory.xyz" +# some contributions are computed in single precision on the GPU: the last printed digit may differ from the CPU +plumed_tolerance=(forces.0:1e-4 forces.1:1e-4 forces.2:1e-4 forces.3:1e-4 colvar:1e-4) diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.0.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.0.reference new file mode 100644 index 0000000000..88519b9f4b --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.0.reference @@ -0,0 +1,550 @@ +108 + -30.07783 -61.87946 -85.31495 +X -2.99958 -4.41155 -1.40689 +X 3.66746 -5.18612 -0.38351 +X 4.41451 -0.40098 -3.31410 +X -5.41424 -1.26872 -1.80431 +X -3.58153 -5.68061 0.95972 +X 2.79921 -4.19021 1.35698 +X 6.05708 -2.33856 2.35001 +X -4.19779 -0.85594 2.86511 +X -2.39362 -2.56428 0.36621 +X 1.93567 -3.06353 -1.01896 +X 3.20164 0.69639 1.18694 +X -3.21626 0.79417 -1.80440 +X -2.83854 3.48827 -3.30190 +X 4.01562 4.51654 -2.81866 +X 1.26921 2.58287 -1.53033 +X -1.93124 4.52659 -1.31977 +X -5.35213 4.17413 3.16661 +X 2.37661 3.41411 3.67707 +X 2.16620 4.34272 1.43869 +X -0.85840 2.31272 1.49981 +X 0.00000 0.00000 0.00000 +X 0.01301 0.13167 -0.13480 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00118 -0.01732 -0.00012 +X -0.00174 -0.03906 -0.00057 +X 0.07042 -0.26603 -0.18082 +X -0.22145 -0.29256 0.12129 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.01801 -0.23817 0.24226 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00167 -0.07943 -0.08325 +X 0.39187 -0.18704 -0.30656 +X 0.05833 -0.00179 0.00382 +X 0.02080 -0.00018 -0.00023 +X 0.36558 0.19822 0.12250 +X 0.18307 0.00602 0.20187 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.13548 0.01132 -0.14473 +X 0.13624 0.13569 -0.00865 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.22858 -0.25083 -0.00953 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01120 -0.00623 0.00585 +X 0.00522 -0.00561 0.01164 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02664 -0.00055 -0.00099 +X -0.14515 -0.00889 0.11546 +X -0.11918 0.08906 -0.00900 +X -0.00584 0.00313 0.00288 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.11028 -0.00427 -0.10656 +X -0.00262 0.00303 -0.00549 +X -0.00547 0.00315 -0.00294 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00831 0.00445 -0.00424 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07783 -0.07581 0.00658 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -30.96535 -60.92028 -85.02883 +X -3.09317 -4.62648 -1.49722 +X 4.05363 -4.97291 -0.31789 +X 4.57259 -0.29362 -3.17490 +X -5.34072 -0.96747 -2.14880 +X -3.88826 -5.77992 0.97985 +X 2.93845 -4.13990 1.48873 +X 6.49283 -2.46677 2.58708 +X -4.25359 -1.10369 2.44960 +X -2.58938 -2.47609 0.19873 +X 1.67134 -3.31648 -1.06329 +X 3.51684 0.83486 1.16210 +X -3.26438 0.97605 -1.90707 +X -2.50787 3.43257 -2.77732 +X 3.30258 4.76753 -3.20819 +X 1.34062 2.69142 -1.42676 +X -1.68937 4.48869 -1.27415 +X -5.52988 4.16635 3.68079 +X 1.94590 3.39048 3.54908 +X 2.11936 4.15692 1.55456 +X -0.67460 2.17957 1.26752 +X 0.00000 0.00000 0.00000 +X 0.01460 0.10155 -0.10476 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00149 -0.01137 -0.00015 +X -0.00281 -0.03808 -0.00103 +X 0.03524 -0.21375 -0.15997 +X -0.25911 -0.28383 0.11590 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.04430 -0.27757 0.29517 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00228 -0.05012 -0.05411 +X 0.36818 -0.19776 -0.40432 +X 0.08148 -0.00670 0.01066 +X 0.01479 -0.00037 -0.00038 +X 0.36931 0.21175 0.08529 +X 0.19162 0.00961 0.22922 +X 0.00000 0.00000 0.00000 +X 0.01282 0.00691 0.00625 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.10378 0.01511 -0.12315 +X 0.11276 0.11291 -0.01346 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00401 0.00808 0.00319 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00449 -0.01221 -0.00673 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.25464 -0.30934 -0.01158 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01483 -0.00926 0.00826 +X 0.00599 -0.00694 0.01458 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02469 -0.00096 -0.00167 +X -0.12282 -0.01384 0.09785 +X -0.08853 0.06565 -0.00988 +X -0.00590 0.00337 0.00292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.09870 -0.00812 -0.09345 +X -0.00229 0.00299 -0.00503 +X -0.00437 0.00283 -0.00251 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00966 0.00882 -0.00595 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05205 -0.05047 0.00640 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -32.98101 -59.03102 -83.58927 +X -3.09839 -4.66881 -1.66771 +X 4.46365 -4.95427 -0.41530 +X 4.82154 -0.33738 -3.22017 +X -6.07511 -1.69468 -1.78388 +X -4.05694 -5.68465 1.00361 +X 2.98507 -4.06140 1.57964 +X 6.70585 -2.74477 2.60852 +X -4.61588 -1.34503 2.44520 +X -2.57514 -2.39762 0.09213 +X 1.26319 -3.38373 -1.04851 +X 3.68454 0.83542 0.97590 +X -2.78209 1.11013 -1.99050 +X -2.15131 3.58366 -2.95760 +X 3.37153 5.24185 -3.02446 +X 1.41455 2.77689 -1.37217 +X -1.53215 4.53255 -1.11675 +X -6.00713 4.24043 3.40831 +X 2.32968 3.30202 3.62311 +X 2.10078 4.22772 1.59236 +X -0.74279 2.28416 1.22154 +X 0.00000 0.00000 0.00000 +X 0.00513 0.04428 -0.04434 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00191 -0.01093 -0.00028 +X -0.00246 -0.02811 -0.00150 +X 0.03369 -0.15656 -0.10834 +X -0.21100 -0.26140 0.12161 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05908 -0.24945 0.26292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00282 -0.04906 -0.05325 +X 0.25012 -0.12705 -0.34889 +X 0.09679 -0.01266 0.01307 +X 0.00796 -0.00044 -0.00018 +X 0.28173 0.18779 0.02795 +X 0.19716 0.01681 0.25966 +X 0.00000 0.00000 0.00000 +X 0.01373 0.00806 0.00681 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.05344 0.00956 -0.06670 +X 0.05757 0.05757 -0.00762 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00221 0.00456 0.00171 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.20136 -0.27733 -0.02619 +X 0.00323 -0.01168 0.00549 +X 0.00000 0.00000 0.00000 +X 0.01418 -0.01049 0.00863 +X 0.00520 -0.00674 0.01344 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02909 -0.00193 -0.00193 +X -0.12864 -0.02355 0.11648 +X -0.10765 0.09722 -0.00864 +X -0.00782 0.00461 0.00405 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.12363 -0.02061 -0.12378 +X -0.00242 0.00340 -0.00559 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00633 0.00816 -0.00430 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05354 -0.05653 0.00643 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -36.06068 -57.41089 -81.58879 +X -3.06608 -4.45659 -1.53716 +X 4.59498 -4.67231 -0.49859 +X 5.07670 -0.51077 -3.48976 +X -6.90312 -2.42141 -1.40220 +X -4.23532 -5.48777 1.06201 +X 2.99397 -3.78407 1.80396 +X 6.81081 -3.03005 2.34615 +X -4.90791 -1.34655 2.35739 +X -2.46536 -2.19423 0.08654 +X 0.80286 -3.34341 -0.95315 +X 3.73035 0.50785 0.87413 +X -2.23332 0.98051 -1.93905 +X -1.75184 3.70192 -3.13347 +X 3.72999 5.69240 -2.74530 +X 1.39698 2.76181 -1.39661 +X -1.51326 4.48424 -1.02419 +X -6.63427 4.21372 2.85228 +X 2.96687 3.09033 3.68966 +X 2.11737 4.31377 1.59487 +X -0.86490 2.30670 1.31673 +X 0.00000 0.00000 0.00000 +X 0.00120 0.02372 -0.02333 +X 0.00000 0.00000 0.00000 +X -0.00128 0.00246 -0.00139 +X 0.00228 -0.01228 -0.00062 +X -0.00207 -0.02464 -0.00209 +X 0.05238 -0.15265 -0.08858 +X -0.17021 -0.27037 0.12999 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06975 -0.23296 0.23756 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 -0.05275 -0.05764 +X 0.18395 -0.09289 -0.29559 +X 0.13663 -0.02476 0.01187 +X 0.01909 0.00519 -0.00701 +X 0.28612 0.20026 -0.00192 +X 0.21313 0.02216 0.32092 +X 0.00000 0.00000 0.00000 +X 0.01580 0.00995 0.00842 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.03893 0.00647 -0.04894 +X 0.03845 0.03897 -0.00452 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00124 0.00276 0.00106 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00215 0.00131 -0.00132 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.15538 -0.24702 -0.04168 +X 0.00315 -0.01323 0.00582 +X 0.00000 0.00000 0.00000 +X 0.01430 -0.01311 0.00960 +X 0.00519 -0.00761 0.01384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.03784 -0.00343 -0.00129 +X -0.15347 -0.03143 0.14969 +X -0.14720 0.15527 -0.00863 +X -0.01033 0.00621 0.00579 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.15542 -0.04041 -0.16998 +X -0.00263 0.00384 -0.00633 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00525 0.00909 -0.00390 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06301 -0.07423 0.00597 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -38.65934 -56.74544 -80.55182 +X -3.12953 -4.39135 -1.44388 +X 4.61881 -4.76109 -0.34105 +X 5.27923 -0.75337 -3.72766 +X -7.64412 -2.43833 -1.46420 +X -4.29289 -5.64014 1.32867 +X 2.99425 -3.47009 1.98388 +X 7.00602 -3.06000 2.23022 +X -4.93958 -1.16005 2.42709 +X -2.25754 -2.04854 0.10271 +X 0.39218 -3.22692 -1.00018 +X 3.57293 0.14087 0.62450 +X -1.82959 0.60450 -2.25084 +X -1.38239 3.87147 -3.54073 +X 4.44766 5.69624 -2.15455 +X 1.37954 2.89725 -1.66507 +X -1.58369 4.82583 -1.02111 +X -7.48941 3.84037 2.52487 +X 3.53781 2.62869 4.12855 +X 2.15533 4.66012 1.72352 +X -1.02592 2.39802 1.50804 +X 0.00000 0.00000 0.00000 +X -0.00006 0.03340 -0.03445 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00230 -0.01443 -0.00111 +X -0.00143 -0.01593 -0.00158 +X 0.08964 -0.16338 -0.07190 +X -0.07253 -0.25738 0.15712 +X -0.00338 -0.00671 0.00257 +X 0.00000 0.00000 0.00000 +X -0.03833 -0.12442 0.13556 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00385 -0.05373 -0.05928 +X 0.15383 -0.10836 -0.18107 +X 0.11043 -0.02403 0.00628 +X 0.01976 0.00295 -0.00484 +X 0.23424 0.13411 0.02126 +X 0.13077 0.00997 0.21001 +X 0.00000 0.00000 0.00000 +X 0.01050 0.00687 0.00618 +X 0.00000 0.00000 0.00000 +X 0.00208 -0.00168 -0.00384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.07772 0.01034 -0.09412 +X 0.08310 0.09050 -0.01176 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00194 0.00461 0.00191 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 0.00242 -0.00219 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00652 -0.00365 -0.00371 +X 0.08418 -0.14892 -0.02839 +X 0.00190 -0.00912 0.00374 +X 0.00000 0.00000 0.00000 +X 0.00899 -0.00930 0.00637 +X 0.00292 -0.00455 0.00807 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05439 -0.00563 0.00009 +X -0.18431 -0.03676 0.18473 +X -0.20546 0.22657 -0.01410 +X -0.01238 0.00749 0.00711 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.18806 -0.06796 -0.20969 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00394 0.00928 -0.00356 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07310 -0.09603 0.00180 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.1.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.1.reference new file mode 100644 index 0000000000..88519b9f4b --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.1.reference @@ -0,0 +1,550 @@ +108 + -30.07783 -61.87946 -85.31495 +X -2.99958 -4.41155 -1.40689 +X 3.66746 -5.18612 -0.38351 +X 4.41451 -0.40098 -3.31410 +X -5.41424 -1.26872 -1.80431 +X -3.58153 -5.68061 0.95972 +X 2.79921 -4.19021 1.35698 +X 6.05708 -2.33856 2.35001 +X -4.19779 -0.85594 2.86511 +X -2.39362 -2.56428 0.36621 +X 1.93567 -3.06353 -1.01896 +X 3.20164 0.69639 1.18694 +X -3.21626 0.79417 -1.80440 +X -2.83854 3.48827 -3.30190 +X 4.01562 4.51654 -2.81866 +X 1.26921 2.58287 -1.53033 +X -1.93124 4.52659 -1.31977 +X -5.35213 4.17413 3.16661 +X 2.37661 3.41411 3.67707 +X 2.16620 4.34272 1.43869 +X -0.85840 2.31272 1.49981 +X 0.00000 0.00000 0.00000 +X 0.01301 0.13167 -0.13480 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00118 -0.01732 -0.00012 +X -0.00174 -0.03906 -0.00057 +X 0.07042 -0.26603 -0.18082 +X -0.22145 -0.29256 0.12129 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.01801 -0.23817 0.24226 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00167 -0.07943 -0.08325 +X 0.39187 -0.18704 -0.30656 +X 0.05833 -0.00179 0.00382 +X 0.02080 -0.00018 -0.00023 +X 0.36558 0.19822 0.12250 +X 0.18307 0.00602 0.20187 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.13548 0.01132 -0.14473 +X 0.13624 0.13569 -0.00865 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.22858 -0.25083 -0.00953 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01120 -0.00623 0.00585 +X 0.00522 -0.00561 0.01164 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02664 -0.00055 -0.00099 +X -0.14515 -0.00889 0.11546 +X -0.11918 0.08906 -0.00900 +X -0.00584 0.00313 0.00288 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.11028 -0.00427 -0.10656 +X -0.00262 0.00303 -0.00549 +X -0.00547 0.00315 -0.00294 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00831 0.00445 -0.00424 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07783 -0.07581 0.00658 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -30.96535 -60.92028 -85.02883 +X -3.09317 -4.62648 -1.49722 +X 4.05363 -4.97291 -0.31789 +X 4.57259 -0.29362 -3.17490 +X -5.34072 -0.96747 -2.14880 +X -3.88826 -5.77992 0.97985 +X 2.93845 -4.13990 1.48873 +X 6.49283 -2.46677 2.58708 +X -4.25359 -1.10369 2.44960 +X -2.58938 -2.47609 0.19873 +X 1.67134 -3.31648 -1.06329 +X 3.51684 0.83486 1.16210 +X -3.26438 0.97605 -1.90707 +X -2.50787 3.43257 -2.77732 +X 3.30258 4.76753 -3.20819 +X 1.34062 2.69142 -1.42676 +X -1.68937 4.48869 -1.27415 +X -5.52988 4.16635 3.68079 +X 1.94590 3.39048 3.54908 +X 2.11936 4.15692 1.55456 +X -0.67460 2.17957 1.26752 +X 0.00000 0.00000 0.00000 +X 0.01460 0.10155 -0.10476 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00149 -0.01137 -0.00015 +X -0.00281 -0.03808 -0.00103 +X 0.03524 -0.21375 -0.15997 +X -0.25911 -0.28383 0.11590 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.04430 -0.27757 0.29517 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00228 -0.05012 -0.05411 +X 0.36818 -0.19776 -0.40432 +X 0.08148 -0.00670 0.01066 +X 0.01479 -0.00037 -0.00038 +X 0.36931 0.21175 0.08529 +X 0.19162 0.00961 0.22922 +X 0.00000 0.00000 0.00000 +X 0.01282 0.00691 0.00625 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.10378 0.01511 -0.12315 +X 0.11276 0.11291 -0.01346 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00401 0.00808 0.00319 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00449 -0.01221 -0.00673 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.25464 -0.30934 -0.01158 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01483 -0.00926 0.00826 +X 0.00599 -0.00694 0.01458 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02469 -0.00096 -0.00167 +X -0.12282 -0.01384 0.09785 +X -0.08853 0.06565 -0.00988 +X -0.00590 0.00337 0.00292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.09870 -0.00812 -0.09345 +X -0.00229 0.00299 -0.00503 +X -0.00437 0.00283 -0.00251 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00966 0.00882 -0.00595 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05205 -0.05047 0.00640 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -32.98101 -59.03102 -83.58927 +X -3.09839 -4.66881 -1.66771 +X 4.46365 -4.95427 -0.41530 +X 4.82154 -0.33738 -3.22017 +X -6.07511 -1.69468 -1.78388 +X -4.05694 -5.68465 1.00361 +X 2.98507 -4.06140 1.57964 +X 6.70585 -2.74477 2.60852 +X -4.61588 -1.34503 2.44520 +X -2.57514 -2.39762 0.09213 +X 1.26319 -3.38373 -1.04851 +X 3.68454 0.83542 0.97590 +X -2.78209 1.11013 -1.99050 +X -2.15131 3.58366 -2.95760 +X 3.37153 5.24185 -3.02446 +X 1.41455 2.77689 -1.37217 +X -1.53215 4.53255 -1.11675 +X -6.00713 4.24043 3.40831 +X 2.32968 3.30202 3.62311 +X 2.10078 4.22772 1.59236 +X -0.74279 2.28416 1.22154 +X 0.00000 0.00000 0.00000 +X 0.00513 0.04428 -0.04434 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00191 -0.01093 -0.00028 +X -0.00246 -0.02811 -0.00150 +X 0.03369 -0.15656 -0.10834 +X -0.21100 -0.26140 0.12161 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05908 -0.24945 0.26292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00282 -0.04906 -0.05325 +X 0.25012 -0.12705 -0.34889 +X 0.09679 -0.01266 0.01307 +X 0.00796 -0.00044 -0.00018 +X 0.28173 0.18779 0.02795 +X 0.19716 0.01681 0.25966 +X 0.00000 0.00000 0.00000 +X 0.01373 0.00806 0.00681 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.05344 0.00956 -0.06670 +X 0.05757 0.05757 -0.00762 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00221 0.00456 0.00171 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.20136 -0.27733 -0.02619 +X 0.00323 -0.01168 0.00549 +X 0.00000 0.00000 0.00000 +X 0.01418 -0.01049 0.00863 +X 0.00520 -0.00674 0.01344 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02909 -0.00193 -0.00193 +X -0.12864 -0.02355 0.11648 +X -0.10765 0.09722 -0.00864 +X -0.00782 0.00461 0.00405 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.12363 -0.02061 -0.12378 +X -0.00242 0.00340 -0.00559 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00633 0.00816 -0.00430 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05354 -0.05653 0.00643 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -36.06068 -57.41089 -81.58879 +X -3.06608 -4.45659 -1.53716 +X 4.59498 -4.67231 -0.49859 +X 5.07670 -0.51077 -3.48976 +X -6.90312 -2.42141 -1.40220 +X -4.23532 -5.48777 1.06201 +X 2.99397 -3.78407 1.80396 +X 6.81081 -3.03005 2.34615 +X -4.90791 -1.34655 2.35739 +X -2.46536 -2.19423 0.08654 +X 0.80286 -3.34341 -0.95315 +X 3.73035 0.50785 0.87413 +X -2.23332 0.98051 -1.93905 +X -1.75184 3.70192 -3.13347 +X 3.72999 5.69240 -2.74530 +X 1.39698 2.76181 -1.39661 +X -1.51326 4.48424 -1.02419 +X -6.63427 4.21372 2.85228 +X 2.96687 3.09033 3.68966 +X 2.11737 4.31377 1.59487 +X -0.86490 2.30670 1.31673 +X 0.00000 0.00000 0.00000 +X 0.00120 0.02372 -0.02333 +X 0.00000 0.00000 0.00000 +X -0.00128 0.00246 -0.00139 +X 0.00228 -0.01228 -0.00062 +X -0.00207 -0.02464 -0.00209 +X 0.05238 -0.15265 -0.08858 +X -0.17021 -0.27037 0.12999 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06975 -0.23296 0.23756 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 -0.05275 -0.05764 +X 0.18395 -0.09289 -0.29559 +X 0.13663 -0.02476 0.01187 +X 0.01909 0.00519 -0.00701 +X 0.28612 0.20026 -0.00192 +X 0.21313 0.02216 0.32092 +X 0.00000 0.00000 0.00000 +X 0.01580 0.00995 0.00842 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.03893 0.00647 -0.04894 +X 0.03845 0.03897 -0.00452 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00124 0.00276 0.00106 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00215 0.00131 -0.00132 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.15538 -0.24702 -0.04168 +X 0.00315 -0.01323 0.00582 +X 0.00000 0.00000 0.00000 +X 0.01430 -0.01311 0.00960 +X 0.00519 -0.00761 0.01384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.03784 -0.00343 -0.00129 +X -0.15347 -0.03143 0.14969 +X -0.14720 0.15527 -0.00863 +X -0.01033 0.00621 0.00579 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.15542 -0.04041 -0.16998 +X -0.00263 0.00384 -0.00633 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00525 0.00909 -0.00390 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06301 -0.07423 0.00597 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -38.65934 -56.74544 -80.55182 +X -3.12953 -4.39135 -1.44388 +X 4.61881 -4.76109 -0.34105 +X 5.27923 -0.75337 -3.72766 +X -7.64412 -2.43833 -1.46420 +X -4.29289 -5.64014 1.32867 +X 2.99425 -3.47009 1.98388 +X 7.00602 -3.06000 2.23022 +X -4.93958 -1.16005 2.42709 +X -2.25754 -2.04854 0.10271 +X 0.39218 -3.22692 -1.00018 +X 3.57293 0.14087 0.62450 +X -1.82959 0.60450 -2.25084 +X -1.38239 3.87147 -3.54073 +X 4.44766 5.69624 -2.15455 +X 1.37954 2.89725 -1.66507 +X -1.58369 4.82583 -1.02111 +X -7.48941 3.84037 2.52487 +X 3.53781 2.62869 4.12855 +X 2.15533 4.66012 1.72352 +X -1.02592 2.39802 1.50804 +X 0.00000 0.00000 0.00000 +X -0.00006 0.03340 -0.03445 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00230 -0.01443 -0.00111 +X -0.00143 -0.01593 -0.00158 +X 0.08964 -0.16338 -0.07190 +X -0.07253 -0.25738 0.15712 +X -0.00338 -0.00671 0.00257 +X 0.00000 0.00000 0.00000 +X -0.03833 -0.12442 0.13556 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00385 -0.05373 -0.05928 +X 0.15383 -0.10836 -0.18107 +X 0.11043 -0.02403 0.00628 +X 0.01976 0.00295 -0.00484 +X 0.23424 0.13411 0.02126 +X 0.13077 0.00997 0.21001 +X 0.00000 0.00000 0.00000 +X 0.01050 0.00687 0.00618 +X 0.00000 0.00000 0.00000 +X 0.00208 -0.00168 -0.00384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.07772 0.01034 -0.09412 +X 0.08310 0.09050 -0.01176 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00194 0.00461 0.00191 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 0.00242 -0.00219 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00652 -0.00365 -0.00371 +X 0.08418 -0.14892 -0.02839 +X 0.00190 -0.00912 0.00374 +X 0.00000 0.00000 0.00000 +X 0.00899 -0.00930 0.00637 +X 0.00292 -0.00455 0.00807 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05439 -0.00563 0.00009 +X -0.18431 -0.03676 0.18473 +X -0.20546 0.22657 -0.01410 +X -0.01238 0.00749 0.00711 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.18806 -0.06796 -0.20969 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00394 0.00928 -0.00356 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07310 -0.09603 0.00180 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.2.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.2.reference new file mode 100644 index 0000000000..88519b9f4b --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.2.reference @@ -0,0 +1,550 @@ +108 + -30.07783 -61.87946 -85.31495 +X -2.99958 -4.41155 -1.40689 +X 3.66746 -5.18612 -0.38351 +X 4.41451 -0.40098 -3.31410 +X -5.41424 -1.26872 -1.80431 +X -3.58153 -5.68061 0.95972 +X 2.79921 -4.19021 1.35698 +X 6.05708 -2.33856 2.35001 +X -4.19779 -0.85594 2.86511 +X -2.39362 -2.56428 0.36621 +X 1.93567 -3.06353 -1.01896 +X 3.20164 0.69639 1.18694 +X -3.21626 0.79417 -1.80440 +X -2.83854 3.48827 -3.30190 +X 4.01562 4.51654 -2.81866 +X 1.26921 2.58287 -1.53033 +X -1.93124 4.52659 -1.31977 +X -5.35213 4.17413 3.16661 +X 2.37661 3.41411 3.67707 +X 2.16620 4.34272 1.43869 +X -0.85840 2.31272 1.49981 +X 0.00000 0.00000 0.00000 +X 0.01301 0.13167 -0.13480 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00118 -0.01732 -0.00012 +X -0.00174 -0.03906 -0.00057 +X 0.07042 -0.26603 -0.18082 +X -0.22145 -0.29256 0.12129 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.01801 -0.23817 0.24226 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00167 -0.07943 -0.08325 +X 0.39187 -0.18704 -0.30656 +X 0.05833 -0.00179 0.00382 +X 0.02080 -0.00018 -0.00023 +X 0.36558 0.19822 0.12250 +X 0.18307 0.00602 0.20187 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.13548 0.01132 -0.14473 +X 0.13624 0.13569 -0.00865 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.22858 -0.25083 -0.00953 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01120 -0.00623 0.00585 +X 0.00522 -0.00561 0.01164 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02664 -0.00055 -0.00099 +X -0.14515 -0.00889 0.11546 +X -0.11918 0.08906 -0.00900 +X -0.00584 0.00313 0.00288 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.11028 -0.00427 -0.10656 +X -0.00262 0.00303 -0.00549 +X -0.00547 0.00315 -0.00294 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00831 0.00445 -0.00424 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07783 -0.07581 0.00658 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -30.96535 -60.92028 -85.02883 +X -3.09317 -4.62648 -1.49722 +X 4.05363 -4.97291 -0.31789 +X 4.57259 -0.29362 -3.17490 +X -5.34072 -0.96747 -2.14880 +X -3.88826 -5.77992 0.97985 +X 2.93845 -4.13990 1.48873 +X 6.49283 -2.46677 2.58708 +X -4.25359 -1.10369 2.44960 +X -2.58938 -2.47609 0.19873 +X 1.67134 -3.31648 -1.06329 +X 3.51684 0.83486 1.16210 +X -3.26438 0.97605 -1.90707 +X -2.50787 3.43257 -2.77732 +X 3.30258 4.76753 -3.20819 +X 1.34062 2.69142 -1.42676 +X -1.68937 4.48869 -1.27415 +X -5.52988 4.16635 3.68079 +X 1.94590 3.39048 3.54908 +X 2.11936 4.15692 1.55456 +X -0.67460 2.17957 1.26752 +X 0.00000 0.00000 0.00000 +X 0.01460 0.10155 -0.10476 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00149 -0.01137 -0.00015 +X -0.00281 -0.03808 -0.00103 +X 0.03524 -0.21375 -0.15997 +X -0.25911 -0.28383 0.11590 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.04430 -0.27757 0.29517 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00228 -0.05012 -0.05411 +X 0.36818 -0.19776 -0.40432 +X 0.08148 -0.00670 0.01066 +X 0.01479 -0.00037 -0.00038 +X 0.36931 0.21175 0.08529 +X 0.19162 0.00961 0.22922 +X 0.00000 0.00000 0.00000 +X 0.01282 0.00691 0.00625 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.10378 0.01511 -0.12315 +X 0.11276 0.11291 -0.01346 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00401 0.00808 0.00319 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00449 -0.01221 -0.00673 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.25464 -0.30934 -0.01158 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01483 -0.00926 0.00826 +X 0.00599 -0.00694 0.01458 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02469 -0.00096 -0.00167 +X -0.12282 -0.01384 0.09785 +X -0.08853 0.06565 -0.00988 +X -0.00590 0.00337 0.00292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.09870 -0.00812 -0.09345 +X -0.00229 0.00299 -0.00503 +X -0.00437 0.00283 -0.00251 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00966 0.00882 -0.00595 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05205 -0.05047 0.00640 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -32.98101 -59.03102 -83.58927 +X -3.09839 -4.66881 -1.66771 +X 4.46365 -4.95427 -0.41530 +X 4.82154 -0.33738 -3.22017 +X -6.07511 -1.69468 -1.78388 +X -4.05694 -5.68465 1.00361 +X 2.98507 -4.06140 1.57964 +X 6.70585 -2.74477 2.60852 +X -4.61588 -1.34503 2.44520 +X -2.57514 -2.39762 0.09213 +X 1.26319 -3.38373 -1.04851 +X 3.68454 0.83542 0.97590 +X -2.78209 1.11013 -1.99050 +X -2.15131 3.58366 -2.95760 +X 3.37153 5.24185 -3.02446 +X 1.41455 2.77689 -1.37217 +X -1.53215 4.53255 -1.11675 +X -6.00713 4.24043 3.40831 +X 2.32968 3.30202 3.62311 +X 2.10078 4.22772 1.59236 +X -0.74279 2.28416 1.22154 +X 0.00000 0.00000 0.00000 +X 0.00513 0.04428 -0.04434 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00191 -0.01093 -0.00028 +X -0.00246 -0.02811 -0.00150 +X 0.03369 -0.15656 -0.10834 +X -0.21100 -0.26140 0.12161 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05908 -0.24945 0.26292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00282 -0.04906 -0.05325 +X 0.25012 -0.12705 -0.34889 +X 0.09679 -0.01266 0.01307 +X 0.00796 -0.00044 -0.00018 +X 0.28173 0.18779 0.02795 +X 0.19716 0.01681 0.25966 +X 0.00000 0.00000 0.00000 +X 0.01373 0.00806 0.00681 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.05344 0.00956 -0.06670 +X 0.05757 0.05757 -0.00762 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00221 0.00456 0.00171 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.20136 -0.27733 -0.02619 +X 0.00323 -0.01168 0.00549 +X 0.00000 0.00000 0.00000 +X 0.01418 -0.01049 0.00863 +X 0.00520 -0.00674 0.01344 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02909 -0.00193 -0.00193 +X -0.12864 -0.02355 0.11648 +X -0.10765 0.09722 -0.00864 +X -0.00782 0.00461 0.00405 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.12363 -0.02061 -0.12378 +X -0.00242 0.00340 -0.00559 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00633 0.00816 -0.00430 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05354 -0.05653 0.00643 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -36.06068 -57.41089 -81.58879 +X -3.06608 -4.45659 -1.53716 +X 4.59498 -4.67231 -0.49859 +X 5.07670 -0.51077 -3.48976 +X -6.90312 -2.42141 -1.40220 +X -4.23532 -5.48777 1.06201 +X 2.99397 -3.78407 1.80396 +X 6.81081 -3.03005 2.34615 +X -4.90791 -1.34655 2.35739 +X -2.46536 -2.19423 0.08654 +X 0.80286 -3.34341 -0.95315 +X 3.73035 0.50785 0.87413 +X -2.23332 0.98051 -1.93905 +X -1.75184 3.70192 -3.13347 +X 3.72999 5.69240 -2.74530 +X 1.39698 2.76181 -1.39661 +X -1.51326 4.48424 -1.02419 +X -6.63427 4.21372 2.85228 +X 2.96687 3.09033 3.68966 +X 2.11737 4.31377 1.59487 +X -0.86490 2.30670 1.31673 +X 0.00000 0.00000 0.00000 +X 0.00120 0.02372 -0.02333 +X 0.00000 0.00000 0.00000 +X -0.00128 0.00246 -0.00139 +X 0.00228 -0.01228 -0.00062 +X -0.00207 -0.02464 -0.00209 +X 0.05238 -0.15265 -0.08858 +X -0.17021 -0.27037 0.12999 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06975 -0.23296 0.23756 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 -0.05275 -0.05764 +X 0.18395 -0.09289 -0.29559 +X 0.13663 -0.02476 0.01187 +X 0.01909 0.00519 -0.00701 +X 0.28612 0.20026 -0.00192 +X 0.21313 0.02216 0.32092 +X 0.00000 0.00000 0.00000 +X 0.01580 0.00995 0.00842 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.03893 0.00647 -0.04894 +X 0.03845 0.03897 -0.00452 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00124 0.00276 0.00106 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00215 0.00131 -0.00132 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.15538 -0.24702 -0.04168 +X 0.00315 -0.01323 0.00582 +X 0.00000 0.00000 0.00000 +X 0.01430 -0.01311 0.00960 +X 0.00519 -0.00761 0.01384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.03784 -0.00343 -0.00129 +X -0.15347 -0.03143 0.14969 +X -0.14720 0.15527 -0.00863 +X -0.01033 0.00621 0.00579 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.15542 -0.04041 -0.16998 +X -0.00263 0.00384 -0.00633 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00525 0.00909 -0.00390 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06301 -0.07423 0.00597 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -38.65934 -56.74544 -80.55182 +X -3.12953 -4.39135 -1.44388 +X 4.61881 -4.76109 -0.34105 +X 5.27923 -0.75337 -3.72766 +X -7.64412 -2.43833 -1.46420 +X -4.29289 -5.64014 1.32867 +X 2.99425 -3.47009 1.98388 +X 7.00602 -3.06000 2.23022 +X -4.93958 -1.16005 2.42709 +X -2.25754 -2.04854 0.10271 +X 0.39218 -3.22692 -1.00018 +X 3.57293 0.14087 0.62450 +X -1.82959 0.60450 -2.25084 +X -1.38239 3.87147 -3.54073 +X 4.44766 5.69624 -2.15455 +X 1.37954 2.89725 -1.66507 +X -1.58369 4.82583 -1.02111 +X -7.48941 3.84037 2.52487 +X 3.53781 2.62869 4.12855 +X 2.15533 4.66012 1.72352 +X -1.02592 2.39802 1.50804 +X 0.00000 0.00000 0.00000 +X -0.00006 0.03340 -0.03445 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00230 -0.01443 -0.00111 +X -0.00143 -0.01593 -0.00158 +X 0.08964 -0.16338 -0.07190 +X -0.07253 -0.25738 0.15712 +X -0.00338 -0.00671 0.00257 +X 0.00000 0.00000 0.00000 +X -0.03833 -0.12442 0.13556 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00385 -0.05373 -0.05928 +X 0.15383 -0.10836 -0.18107 +X 0.11043 -0.02403 0.00628 +X 0.01976 0.00295 -0.00484 +X 0.23424 0.13411 0.02126 +X 0.13077 0.00997 0.21001 +X 0.00000 0.00000 0.00000 +X 0.01050 0.00687 0.00618 +X 0.00000 0.00000 0.00000 +X 0.00208 -0.00168 -0.00384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.07772 0.01034 -0.09412 +X 0.08310 0.09050 -0.01176 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00194 0.00461 0.00191 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 0.00242 -0.00219 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00652 -0.00365 -0.00371 +X 0.08418 -0.14892 -0.02839 +X 0.00190 -0.00912 0.00374 +X 0.00000 0.00000 0.00000 +X 0.00899 -0.00930 0.00637 +X 0.00292 -0.00455 0.00807 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05439 -0.00563 0.00009 +X -0.18431 -0.03676 0.18473 +X -0.20546 0.22657 -0.01410 +X -0.01238 0.00749 0.00711 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.18806 -0.06796 -0.20969 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00394 0.00928 -0.00356 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07310 -0.09603 0.00180 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.3.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.3.reference new file mode 100644 index 0000000000..88519b9f4b --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/forces.3.reference @@ -0,0 +1,550 @@ +108 + -30.07783 -61.87946 -85.31495 +X -2.99958 -4.41155 -1.40689 +X 3.66746 -5.18612 -0.38351 +X 4.41451 -0.40098 -3.31410 +X -5.41424 -1.26872 -1.80431 +X -3.58153 -5.68061 0.95972 +X 2.79921 -4.19021 1.35698 +X 6.05708 -2.33856 2.35001 +X -4.19779 -0.85594 2.86511 +X -2.39362 -2.56428 0.36621 +X 1.93567 -3.06353 -1.01896 +X 3.20164 0.69639 1.18694 +X -3.21626 0.79417 -1.80440 +X -2.83854 3.48827 -3.30190 +X 4.01562 4.51654 -2.81866 +X 1.26921 2.58287 -1.53033 +X -1.93124 4.52659 -1.31977 +X -5.35213 4.17413 3.16661 +X 2.37661 3.41411 3.67707 +X 2.16620 4.34272 1.43869 +X -0.85840 2.31272 1.49981 +X 0.00000 0.00000 0.00000 +X 0.01301 0.13167 -0.13480 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00118 -0.01732 -0.00012 +X -0.00174 -0.03906 -0.00057 +X 0.07042 -0.26603 -0.18082 +X -0.22145 -0.29256 0.12129 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.01801 -0.23817 0.24226 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00167 -0.07943 -0.08325 +X 0.39187 -0.18704 -0.30656 +X 0.05833 -0.00179 0.00382 +X 0.02080 -0.00018 -0.00023 +X 0.36558 0.19822 0.12250 +X 0.18307 0.00602 0.20187 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.13548 0.01132 -0.14473 +X 0.13624 0.13569 -0.00865 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.22858 -0.25083 -0.00953 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01120 -0.00623 0.00585 +X 0.00522 -0.00561 0.01164 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02664 -0.00055 -0.00099 +X -0.14515 -0.00889 0.11546 +X -0.11918 0.08906 -0.00900 +X -0.00584 0.00313 0.00288 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.11028 -0.00427 -0.10656 +X -0.00262 0.00303 -0.00549 +X -0.00547 0.00315 -0.00294 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00831 0.00445 -0.00424 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07783 -0.07581 0.00658 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -30.96535 -60.92028 -85.02883 +X -3.09317 -4.62648 -1.49722 +X 4.05363 -4.97291 -0.31789 +X 4.57259 -0.29362 -3.17490 +X -5.34072 -0.96747 -2.14880 +X -3.88826 -5.77992 0.97985 +X 2.93845 -4.13990 1.48873 +X 6.49283 -2.46677 2.58708 +X -4.25359 -1.10369 2.44960 +X -2.58938 -2.47609 0.19873 +X 1.67134 -3.31648 -1.06329 +X 3.51684 0.83486 1.16210 +X -3.26438 0.97605 -1.90707 +X -2.50787 3.43257 -2.77732 +X 3.30258 4.76753 -3.20819 +X 1.34062 2.69142 -1.42676 +X -1.68937 4.48869 -1.27415 +X -5.52988 4.16635 3.68079 +X 1.94590 3.39048 3.54908 +X 2.11936 4.15692 1.55456 +X -0.67460 2.17957 1.26752 +X 0.00000 0.00000 0.00000 +X 0.01460 0.10155 -0.10476 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00149 -0.01137 -0.00015 +X -0.00281 -0.03808 -0.00103 +X 0.03524 -0.21375 -0.15997 +X -0.25911 -0.28383 0.11590 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.04430 -0.27757 0.29517 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00228 -0.05012 -0.05411 +X 0.36818 -0.19776 -0.40432 +X 0.08148 -0.00670 0.01066 +X 0.01479 -0.00037 -0.00038 +X 0.36931 0.21175 0.08529 +X 0.19162 0.00961 0.22922 +X 0.00000 0.00000 0.00000 +X 0.01282 0.00691 0.00625 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.10378 0.01511 -0.12315 +X 0.11276 0.11291 -0.01346 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00401 0.00808 0.00319 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00449 -0.01221 -0.00673 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.25464 -0.30934 -0.01158 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01483 -0.00926 0.00826 +X 0.00599 -0.00694 0.01458 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02469 -0.00096 -0.00167 +X -0.12282 -0.01384 0.09785 +X -0.08853 0.06565 -0.00988 +X -0.00590 0.00337 0.00292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.09870 -0.00812 -0.09345 +X -0.00229 0.00299 -0.00503 +X -0.00437 0.00283 -0.00251 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00966 0.00882 -0.00595 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05205 -0.05047 0.00640 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -32.98101 -59.03102 -83.58927 +X -3.09839 -4.66881 -1.66771 +X 4.46365 -4.95427 -0.41530 +X 4.82154 -0.33738 -3.22017 +X -6.07511 -1.69468 -1.78388 +X -4.05694 -5.68465 1.00361 +X 2.98507 -4.06140 1.57964 +X 6.70585 -2.74477 2.60852 +X -4.61588 -1.34503 2.44520 +X -2.57514 -2.39762 0.09213 +X 1.26319 -3.38373 -1.04851 +X 3.68454 0.83542 0.97590 +X -2.78209 1.11013 -1.99050 +X -2.15131 3.58366 -2.95760 +X 3.37153 5.24185 -3.02446 +X 1.41455 2.77689 -1.37217 +X -1.53215 4.53255 -1.11675 +X -6.00713 4.24043 3.40831 +X 2.32968 3.30202 3.62311 +X 2.10078 4.22772 1.59236 +X -0.74279 2.28416 1.22154 +X 0.00000 0.00000 0.00000 +X 0.00513 0.04428 -0.04434 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00191 -0.01093 -0.00028 +X -0.00246 -0.02811 -0.00150 +X 0.03369 -0.15656 -0.10834 +X -0.21100 -0.26140 0.12161 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05908 -0.24945 0.26292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00282 -0.04906 -0.05325 +X 0.25012 -0.12705 -0.34889 +X 0.09679 -0.01266 0.01307 +X 0.00796 -0.00044 -0.00018 +X 0.28173 0.18779 0.02795 +X 0.19716 0.01681 0.25966 +X 0.00000 0.00000 0.00000 +X 0.01373 0.00806 0.00681 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.05344 0.00956 -0.06670 +X 0.05757 0.05757 -0.00762 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00221 0.00456 0.00171 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.20136 -0.27733 -0.02619 +X 0.00323 -0.01168 0.00549 +X 0.00000 0.00000 0.00000 +X 0.01418 -0.01049 0.00863 +X 0.00520 -0.00674 0.01344 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02909 -0.00193 -0.00193 +X -0.12864 -0.02355 0.11648 +X -0.10765 0.09722 -0.00864 +X -0.00782 0.00461 0.00405 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.12363 -0.02061 -0.12378 +X -0.00242 0.00340 -0.00559 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00633 0.00816 -0.00430 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05354 -0.05653 0.00643 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -36.06068 -57.41089 -81.58879 +X -3.06608 -4.45659 -1.53716 +X 4.59498 -4.67231 -0.49859 +X 5.07670 -0.51077 -3.48976 +X -6.90312 -2.42141 -1.40220 +X -4.23532 -5.48777 1.06201 +X 2.99397 -3.78407 1.80396 +X 6.81081 -3.03005 2.34615 +X -4.90791 -1.34655 2.35739 +X -2.46536 -2.19423 0.08654 +X 0.80286 -3.34341 -0.95315 +X 3.73035 0.50785 0.87413 +X -2.23332 0.98051 -1.93905 +X -1.75184 3.70192 -3.13347 +X 3.72999 5.69240 -2.74530 +X 1.39698 2.76181 -1.39661 +X -1.51326 4.48424 -1.02419 +X -6.63427 4.21372 2.85228 +X 2.96687 3.09033 3.68966 +X 2.11737 4.31377 1.59487 +X -0.86490 2.30670 1.31673 +X 0.00000 0.00000 0.00000 +X 0.00120 0.02372 -0.02333 +X 0.00000 0.00000 0.00000 +X -0.00128 0.00246 -0.00139 +X 0.00228 -0.01228 -0.00062 +X -0.00207 -0.02464 -0.00209 +X 0.05238 -0.15265 -0.08858 +X -0.17021 -0.27037 0.12999 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06975 -0.23296 0.23756 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 -0.05275 -0.05764 +X 0.18395 -0.09289 -0.29559 +X 0.13663 -0.02476 0.01187 +X 0.01909 0.00519 -0.00701 +X 0.28612 0.20026 -0.00192 +X 0.21313 0.02216 0.32092 +X 0.00000 0.00000 0.00000 +X 0.01580 0.00995 0.00842 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.03893 0.00647 -0.04894 +X 0.03845 0.03897 -0.00452 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00124 0.00276 0.00106 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00215 0.00131 -0.00132 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.15538 -0.24702 -0.04168 +X 0.00315 -0.01323 0.00582 +X 0.00000 0.00000 0.00000 +X 0.01430 -0.01311 0.00960 +X 0.00519 -0.00761 0.01384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.03784 -0.00343 -0.00129 +X -0.15347 -0.03143 0.14969 +X -0.14720 0.15527 -0.00863 +X -0.01033 0.00621 0.00579 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.15542 -0.04041 -0.16998 +X -0.00263 0.00384 -0.00633 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00525 0.00909 -0.00390 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06301 -0.07423 0.00597 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -38.65934 -56.74544 -80.55182 +X -3.12953 -4.39135 -1.44388 +X 4.61881 -4.76109 -0.34105 +X 5.27923 -0.75337 -3.72766 +X -7.64412 -2.43833 -1.46420 +X -4.29289 -5.64014 1.32867 +X 2.99425 -3.47009 1.98388 +X 7.00602 -3.06000 2.23022 +X -4.93958 -1.16005 2.42709 +X -2.25754 -2.04854 0.10271 +X 0.39218 -3.22692 -1.00018 +X 3.57293 0.14087 0.62450 +X -1.82959 0.60450 -2.25084 +X -1.38239 3.87147 -3.54073 +X 4.44766 5.69624 -2.15455 +X 1.37954 2.89725 -1.66507 +X -1.58369 4.82583 -1.02111 +X -7.48941 3.84037 2.52487 +X 3.53781 2.62869 4.12855 +X 2.15533 4.66012 1.72352 +X -1.02592 2.39802 1.50804 +X 0.00000 0.00000 0.00000 +X -0.00006 0.03340 -0.03445 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00230 -0.01443 -0.00111 +X -0.00143 -0.01593 -0.00158 +X 0.08964 -0.16338 -0.07190 +X -0.07253 -0.25738 0.15712 +X -0.00338 -0.00671 0.00257 +X 0.00000 0.00000 0.00000 +X -0.03833 -0.12442 0.13556 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00385 -0.05373 -0.05928 +X 0.15383 -0.10836 -0.18107 +X 0.11043 -0.02403 0.00628 +X 0.01976 0.00295 -0.00484 +X 0.23424 0.13411 0.02126 +X 0.13077 0.00997 0.21001 +X 0.00000 0.00000 0.00000 +X 0.01050 0.00687 0.00618 +X 0.00000 0.00000 0.00000 +X 0.00208 -0.00168 -0.00384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.07772 0.01034 -0.09412 +X 0.08310 0.09050 -0.01176 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00194 0.00461 0.00191 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 0.00242 -0.00219 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00652 -0.00365 -0.00371 +X 0.08418 -0.14892 -0.02839 +X 0.00190 -0.00912 0.00374 +X 0.00000 0.00000 0.00000 +X 0.00899 -0.00930 0.00637 +X 0.00292 -0.00455 0.00807 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05439 -0.00563 0.00009 +X -0.18431 -0.03676 0.18473 +X -0.20546 0.22657 -0.01410 +X -0.01238 0.00749 0.00711 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.18806 -0.06796 -0.20969 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00394 0.00928 -0.00356 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07310 -0.09603 0.00180 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/plumed.dat b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/plumed.dat new file mode 100644 index 0000000000..67e500e5c2 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-nogather/plumed.dat @@ -0,0 +1,19 @@ +LOAD FILE=../../../../plumedOpenACC.so +# With MPI the tasks of the GPU actions are split in blocks between the 4 processes, and +# the results summed. The contact matrix c1 has 3 rows and 108 columns: the last process +# computes no row, but some of the columns in the second pass of the low memory force +# gathering (used by default on the GPU), so it must upload the positions there. +# The CPU references come from regtest/adjmat/rt-nogather-serial-mpi +c1: CONTACT_MATRIX GROUPA=1-3 GROUPB=1-108 SWITCH={RATIONAL R_0=1 NN=6 MM=12 D_MAX=2} USEGPU +ones: ONES SIZE=108 +cc: MATRIX_VECTOR_PRODUCT ARG=c1,ones USEGPU +mtc: MORE_THAN ARG=cc SWITCH={RATIONAL D_0=2 R_0=2} USEGPU +s: SUM ARG=mtc PERIODIC=NO +BIASVALUE ARG=s + +c2: CONTACT_MATRIX GROUP=1-20 SWITCH={RATIONAL R_0=1.5 NN=6 MM=12} USEGPU +c22: MATRIX_PRODUCT ARG=c2,c2 USEGPU +diag: DIAGONALIZE ARG=c22 VECTORS=1 +BIASVALUE ARG=diag.vals-1 + +PRINT ARG=s,diag.vals-1 FILE=colvar FMT=%10.5f diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/Makefile b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/Makefile new file mode 100644 index 0000000000..3703b27cea --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/Makefile @@ -0,0 +1 @@ +include ../../scripts/test.make diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/colvar.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/colvar.reference new file mode 100644 index 0000000000..4b6b6fab04 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/colvar.reference @@ -0,0 +1,6 @@ +#! FIELDS time s diag.vals-1 + 0.000000 0.13443 42.22566 + 1.000000 0.13067 41.67056 + 2.000000 0.10292 40.98616 + 3.000000 0.10205 40.05492 + 4.000000 0.09599 40.13433 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/config b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/config new file mode 100644 index 0000000000..f673f4f757 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/config @@ -0,0 +1,6 @@ +type=driver +mpiprocs=2 +arg="--plumed plumed.dat --ixyz trajectory.xyz --dump-forces forces --dump-forces-fmt=%10.5f" +extra_files="../../trajectories/trajectory.xyz" +# some contributions are computed in single precision on the GPU: the last printed digit may differ from the CPU +plumed_tolerance=(forces.0:1e-4 forces.1:1e-4 colvar:1e-4) diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.0.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.0.reference new file mode 100644 index 0000000000..88519b9f4b --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.0.reference @@ -0,0 +1,550 @@ +108 + -30.07783 -61.87946 -85.31495 +X -2.99958 -4.41155 -1.40689 +X 3.66746 -5.18612 -0.38351 +X 4.41451 -0.40098 -3.31410 +X -5.41424 -1.26872 -1.80431 +X -3.58153 -5.68061 0.95972 +X 2.79921 -4.19021 1.35698 +X 6.05708 -2.33856 2.35001 +X -4.19779 -0.85594 2.86511 +X -2.39362 -2.56428 0.36621 +X 1.93567 -3.06353 -1.01896 +X 3.20164 0.69639 1.18694 +X -3.21626 0.79417 -1.80440 +X -2.83854 3.48827 -3.30190 +X 4.01562 4.51654 -2.81866 +X 1.26921 2.58287 -1.53033 +X -1.93124 4.52659 -1.31977 +X -5.35213 4.17413 3.16661 +X 2.37661 3.41411 3.67707 +X 2.16620 4.34272 1.43869 +X -0.85840 2.31272 1.49981 +X 0.00000 0.00000 0.00000 +X 0.01301 0.13167 -0.13480 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00118 -0.01732 -0.00012 +X -0.00174 -0.03906 -0.00057 +X 0.07042 -0.26603 -0.18082 +X -0.22145 -0.29256 0.12129 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.01801 -0.23817 0.24226 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00167 -0.07943 -0.08325 +X 0.39187 -0.18704 -0.30656 +X 0.05833 -0.00179 0.00382 +X 0.02080 -0.00018 -0.00023 +X 0.36558 0.19822 0.12250 +X 0.18307 0.00602 0.20187 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.13548 0.01132 -0.14473 +X 0.13624 0.13569 -0.00865 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.22858 -0.25083 -0.00953 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01120 -0.00623 0.00585 +X 0.00522 -0.00561 0.01164 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02664 -0.00055 -0.00099 +X -0.14515 -0.00889 0.11546 +X -0.11918 0.08906 -0.00900 +X -0.00584 0.00313 0.00288 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.11028 -0.00427 -0.10656 +X -0.00262 0.00303 -0.00549 +X -0.00547 0.00315 -0.00294 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00831 0.00445 -0.00424 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07783 -0.07581 0.00658 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -30.96535 -60.92028 -85.02883 +X -3.09317 -4.62648 -1.49722 +X 4.05363 -4.97291 -0.31789 +X 4.57259 -0.29362 -3.17490 +X -5.34072 -0.96747 -2.14880 +X -3.88826 -5.77992 0.97985 +X 2.93845 -4.13990 1.48873 +X 6.49283 -2.46677 2.58708 +X -4.25359 -1.10369 2.44960 +X -2.58938 -2.47609 0.19873 +X 1.67134 -3.31648 -1.06329 +X 3.51684 0.83486 1.16210 +X -3.26438 0.97605 -1.90707 +X -2.50787 3.43257 -2.77732 +X 3.30258 4.76753 -3.20819 +X 1.34062 2.69142 -1.42676 +X -1.68937 4.48869 -1.27415 +X -5.52988 4.16635 3.68079 +X 1.94590 3.39048 3.54908 +X 2.11936 4.15692 1.55456 +X -0.67460 2.17957 1.26752 +X 0.00000 0.00000 0.00000 +X 0.01460 0.10155 -0.10476 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00149 -0.01137 -0.00015 +X -0.00281 -0.03808 -0.00103 +X 0.03524 -0.21375 -0.15997 +X -0.25911 -0.28383 0.11590 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.04430 -0.27757 0.29517 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00228 -0.05012 -0.05411 +X 0.36818 -0.19776 -0.40432 +X 0.08148 -0.00670 0.01066 +X 0.01479 -0.00037 -0.00038 +X 0.36931 0.21175 0.08529 +X 0.19162 0.00961 0.22922 +X 0.00000 0.00000 0.00000 +X 0.01282 0.00691 0.00625 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.10378 0.01511 -0.12315 +X 0.11276 0.11291 -0.01346 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00401 0.00808 0.00319 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00449 -0.01221 -0.00673 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.25464 -0.30934 -0.01158 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01483 -0.00926 0.00826 +X 0.00599 -0.00694 0.01458 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02469 -0.00096 -0.00167 +X -0.12282 -0.01384 0.09785 +X -0.08853 0.06565 -0.00988 +X -0.00590 0.00337 0.00292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.09870 -0.00812 -0.09345 +X -0.00229 0.00299 -0.00503 +X -0.00437 0.00283 -0.00251 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00966 0.00882 -0.00595 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05205 -0.05047 0.00640 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -32.98101 -59.03102 -83.58927 +X -3.09839 -4.66881 -1.66771 +X 4.46365 -4.95427 -0.41530 +X 4.82154 -0.33738 -3.22017 +X -6.07511 -1.69468 -1.78388 +X -4.05694 -5.68465 1.00361 +X 2.98507 -4.06140 1.57964 +X 6.70585 -2.74477 2.60852 +X -4.61588 -1.34503 2.44520 +X -2.57514 -2.39762 0.09213 +X 1.26319 -3.38373 -1.04851 +X 3.68454 0.83542 0.97590 +X -2.78209 1.11013 -1.99050 +X -2.15131 3.58366 -2.95760 +X 3.37153 5.24185 -3.02446 +X 1.41455 2.77689 -1.37217 +X -1.53215 4.53255 -1.11675 +X -6.00713 4.24043 3.40831 +X 2.32968 3.30202 3.62311 +X 2.10078 4.22772 1.59236 +X -0.74279 2.28416 1.22154 +X 0.00000 0.00000 0.00000 +X 0.00513 0.04428 -0.04434 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00191 -0.01093 -0.00028 +X -0.00246 -0.02811 -0.00150 +X 0.03369 -0.15656 -0.10834 +X -0.21100 -0.26140 0.12161 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05908 -0.24945 0.26292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00282 -0.04906 -0.05325 +X 0.25012 -0.12705 -0.34889 +X 0.09679 -0.01266 0.01307 +X 0.00796 -0.00044 -0.00018 +X 0.28173 0.18779 0.02795 +X 0.19716 0.01681 0.25966 +X 0.00000 0.00000 0.00000 +X 0.01373 0.00806 0.00681 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.05344 0.00956 -0.06670 +X 0.05757 0.05757 -0.00762 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00221 0.00456 0.00171 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.20136 -0.27733 -0.02619 +X 0.00323 -0.01168 0.00549 +X 0.00000 0.00000 0.00000 +X 0.01418 -0.01049 0.00863 +X 0.00520 -0.00674 0.01344 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02909 -0.00193 -0.00193 +X -0.12864 -0.02355 0.11648 +X -0.10765 0.09722 -0.00864 +X -0.00782 0.00461 0.00405 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.12363 -0.02061 -0.12378 +X -0.00242 0.00340 -0.00559 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00633 0.00816 -0.00430 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05354 -0.05653 0.00643 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -36.06068 -57.41089 -81.58879 +X -3.06608 -4.45659 -1.53716 +X 4.59498 -4.67231 -0.49859 +X 5.07670 -0.51077 -3.48976 +X -6.90312 -2.42141 -1.40220 +X -4.23532 -5.48777 1.06201 +X 2.99397 -3.78407 1.80396 +X 6.81081 -3.03005 2.34615 +X -4.90791 -1.34655 2.35739 +X -2.46536 -2.19423 0.08654 +X 0.80286 -3.34341 -0.95315 +X 3.73035 0.50785 0.87413 +X -2.23332 0.98051 -1.93905 +X -1.75184 3.70192 -3.13347 +X 3.72999 5.69240 -2.74530 +X 1.39698 2.76181 -1.39661 +X -1.51326 4.48424 -1.02419 +X -6.63427 4.21372 2.85228 +X 2.96687 3.09033 3.68966 +X 2.11737 4.31377 1.59487 +X -0.86490 2.30670 1.31673 +X 0.00000 0.00000 0.00000 +X 0.00120 0.02372 -0.02333 +X 0.00000 0.00000 0.00000 +X -0.00128 0.00246 -0.00139 +X 0.00228 -0.01228 -0.00062 +X -0.00207 -0.02464 -0.00209 +X 0.05238 -0.15265 -0.08858 +X -0.17021 -0.27037 0.12999 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06975 -0.23296 0.23756 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 -0.05275 -0.05764 +X 0.18395 -0.09289 -0.29559 +X 0.13663 -0.02476 0.01187 +X 0.01909 0.00519 -0.00701 +X 0.28612 0.20026 -0.00192 +X 0.21313 0.02216 0.32092 +X 0.00000 0.00000 0.00000 +X 0.01580 0.00995 0.00842 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.03893 0.00647 -0.04894 +X 0.03845 0.03897 -0.00452 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00124 0.00276 0.00106 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00215 0.00131 -0.00132 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.15538 -0.24702 -0.04168 +X 0.00315 -0.01323 0.00582 +X 0.00000 0.00000 0.00000 +X 0.01430 -0.01311 0.00960 +X 0.00519 -0.00761 0.01384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.03784 -0.00343 -0.00129 +X -0.15347 -0.03143 0.14969 +X -0.14720 0.15527 -0.00863 +X -0.01033 0.00621 0.00579 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.15542 -0.04041 -0.16998 +X -0.00263 0.00384 -0.00633 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00525 0.00909 -0.00390 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06301 -0.07423 0.00597 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -38.65934 -56.74544 -80.55182 +X -3.12953 -4.39135 -1.44388 +X 4.61881 -4.76109 -0.34105 +X 5.27923 -0.75337 -3.72766 +X -7.64412 -2.43833 -1.46420 +X -4.29289 -5.64014 1.32867 +X 2.99425 -3.47009 1.98388 +X 7.00602 -3.06000 2.23022 +X -4.93958 -1.16005 2.42709 +X -2.25754 -2.04854 0.10271 +X 0.39218 -3.22692 -1.00018 +X 3.57293 0.14087 0.62450 +X -1.82959 0.60450 -2.25084 +X -1.38239 3.87147 -3.54073 +X 4.44766 5.69624 -2.15455 +X 1.37954 2.89725 -1.66507 +X -1.58369 4.82583 -1.02111 +X -7.48941 3.84037 2.52487 +X 3.53781 2.62869 4.12855 +X 2.15533 4.66012 1.72352 +X -1.02592 2.39802 1.50804 +X 0.00000 0.00000 0.00000 +X -0.00006 0.03340 -0.03445 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00230 -0.01443 -0.00111 +X -0.00143 -0.01593 -0.00158 +X 0.08964 -0.16338 -0.07190 +X -0.07253 -0.25738 0.15712 +X -0.00338 -0.00671 0.00257 +X 0.00000 0.00000 0.00000 +X -0.03833 -0.12442 0.13556 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00385 -0.05373 -0.05928 +X 0.15383 -0.10836 -0.18107 +X 0.11043 -0.02403 0.00628 +X 0.01976 0.00295 -0.00484 +X 0.23424 0.13411 0.02126 +X 0.13077 0.00997 0.21001 +X 0.00000 0.00000 0.00000 +X 0.01050 0.00687 0.00618 +X 0.00000 0.00000 0.00000 +X 0.00208 -0.00168 -0.00384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.07772 0.01034 -0.09412 +X 0.08310 0.09050 -0.01176 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00194 0.00461 0.00191 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 0.00242 -0.00219 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00652 -0.00365 -0.00371 +X 0.08418 -0.14892 -0.02839 +X 0.00190 -0.00912 0.00374 +X 0.00000 0.00000 0.00000 +X 0.00899 -0.00930 0.00637 +X 0.00292 -0.00455 0.00807 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05439 -0.00563 0.00009 +X -0.18431 -0.03676 0.18473 +X -0.20546 0.22657 -0.01410 +X -0.01238 0.00749 0.00711 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.18806 -0.06796 -0.20969 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00394 0.00928 -0.00356 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07310 -0.09603 0.00180 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.1.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.1.reference new file mode 100644 index 0000000000..88519b9f4b --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/forces.1.reference @@ -0,0 +1,550 @@ +108 + -30.07783 -61.87946 -85.31495 +X -2.99958 -4.41155 -1.40689 +X 3.66746 -5.18612 -0.38351 +X 4.41451 -0.40098 -3.31410 +X -5.41424 -1.26872 -1.80431 +X -3.58153 -5.68061 0.95972 +X 2.79921 -4.19021 1.35698 +X 6.05708 -2.33856 2.35001 +X -4.19779 -0.85594 2.86511 +X -2.39362 -2.56428 0.36621 +X 1.93567 -3.06353 -1.01896 +X 3.20164 0.69639 1.18694 +X -3.21626 0.79417 -1.80440 +X -2.83854 3.48827 -3.30190 +X 4.01562 4.51654 -2.81866 +X 1.26921 2.58287 -1.53033 +X -1.93124 4.52659 -1.31977 +X -5.35213 4.17413 3.16661 +X 2.37661 3.41411 3.67707 +X 2.16620 4.34272 1.43869 +X -0.85840 2.31272 1.49981 +X 0.00000 0.00000 0.00000 +X 0.01301 0.13167 -0.13480 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00118 -0.01732 -0.00012 +X -0.00174 -0.03906 -0.00057 +X 0.07042 -0.26603 -0.18082 +X -0.22145 -0.29256 0.12129 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.01801 -0.23817 0.24226 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00167 -0.07943 -0.08325 +X 0.39187 -0.18704 -0.30656 +X 0.05833 -0.00179 0.00382 +X 0.02080 -0.00018 -0.00023 +X 0.36558 0.19822 0.12250 +X 0.18307 0.00602 0.20187 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.13548 0.01132 -0.14473 +X 0.13624 0.13569 -0.00865 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.22858 -0.25083 -0.00953 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01120 -0.00623 0.00585 +X 0.00522 -0.00561 0.01164 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02664 -0.00055 -0.00099 +X -0.14515 -0.00889 0.11546 +X -0.11918 0.08906 -0.00900 +X -0.00584 0.00313 0.00288 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.11028 -0.00427 -0.10656 +X -0.00262 0.00303 -0.00549 +X -0.00547 0.00315 -0.00294 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00831 0.00445 -0.00424 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07783 -0.07581 0.00658 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -30.96535 -60.92028 -85.02883 +X -3.09317 -4.62648 -1.49722 +X 4.05363 -4.97291 -0.31789 +X 4.57259 -0.29362 -3.17490 +X -5.34072 -0.96747 -2.14880 +X -3.88826 -5.77992 0.97985 +X 2.93845 -4.13990 1.48873 +X 6.49283 -2.46677 2.58708 +X -4.25359 -1.10369 2.44960 +X -2.58938 -2.47609 0.19873 +X 1.67134 -3.31648 -1.06329 +X 3.51684 0.83486 1.16210 +X -3.26438 0.97605 -1.90707 +X -2.50787 3.43257 -2.77732 +X 3.30258 4.76753 -3.20819 +X 1.34062 2.69142 -1.42676 +X -1.68937 4.48869 -1.27415 +X -5.52988 4.16635 3.68079 +X 1.94590 3.39048 3.54908 +X 2.11936 4.15692 1.55456 +X -0.67460 2.17957 1.26752 +X 0.00000 0.00000 0.00000 +X 0.01460 0.10155 -0.10476 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00149 -0.01137 -0.00015 +X -0.00281 -0.03808 -0.00103 +X 0.03524 -0.21375 -0.15997 +X -0.25911 -0.28383 0.11590 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.04430 -0.27757 0.29517 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00228 -0.05012 -0.05411 +X 0.36818 -0.19776 -0.40432 +X 0.08148 -0.00670 0.01066 +X 0.01479 -0.00037 -0.00038 +X 0.36931 0.21175 0.08529 +X 0.19162 0.00961 0.22922 +X 0.00000 0.00000 0.00000 +X 0.01282 0.00691 0.00625 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.10378 0.01511 -0.12315 +X 0.11276 0.11291 -0.01346 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00401 0.00808 0.00319 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00449 -0.01221 -0.00673 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.25464 -0.30934 -0.01158 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.01483 -0.00926 0.00826 +X 0.00599 -0.00694 0.01458 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02469 -0.00096 -0.00167 +X -0.12282 -0.01384 0.09785 +X -0.08853 0.06565 -0.00988 +X -0.00590 0.00337 0.00292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.09870 -0.00812 -0.09345 +X -0.00229 0.00299 -0.00503 +X -0.00437 0.00283 -0.00251 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00966 0.00882 -0.00595 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05205 -0.05047 0.00640 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -32.98101 -59.03102 -83.58927 +X -3.09839 -4.66881 -1.66771 +X 4.46365 -4.95427 -0.41530 +X 4.82154 -0.33738 -3.22017 +X -6.07511 -1.69468 -1.78388 +X -4.05694 -5.68465 1.00361 +X 2.98507 -4.06140 1.57964 +X 6.70585 -2.74477 2.60852 +X -4.61588 -1.34503 2.44520 +X -2.57514 -2.39762 0.09213 +X 1.26319 -3.38373 -1.04851 +X 3.68454 0.83542 0.97590 +X -2.78209 1.11013 -1.99050 +X -2.15131 3.58366 -2.95760 +X 3.37153 5.24185 -3.02446 +X 1.41455 2.77689 -1.37217 +X -1.53215 4.53255 -1.11675 +X -6.00713 4.24043 3.40831 +X 2.32968 3.30202 3.62311 +X 2.10078 4.22772 1.59236 +X -0.74279 2.28416 1.22154 +X 0.00000 0.00000 0.00000 +X 0.00513 0.04428 -0.04434 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00191 -0.01093 -0.00028 +X -0.00246 -0.02811 -0.00150 +X 0.03369 -0.15656 -0.10834 +X -0.21100 -0.26140 0.12161 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05908 -0.24945 0.26292 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00282 -0.04906 -0.05325 +X 0.25012 -0.12705 -0.34889 +X 0.09679 -0.01266 0.01307 +X 0.00796 -0.00044 -0.00018 +X 0.28173 0.18779 0.02795 +X 0.19716 0.01681 0.25966 +X 0.00000 0.00000 0.00000 +X 0.01373 0.00806 0.00681 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.05344 0.00956 -0.06670 +X 0.05757 0.05757 -0.00762 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00221 0.00456 0.00171 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.20136 -0.27733 -0.02619 +X 0.00323 -0.01168 0.00549 +X 0.00000 0.00000 0.00000 +X 0.01418 -0.01049 0.00863 +X 0.00520 -0.00674 0.01344 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.02909 -0.00193 -0.00193 +X -0.12864 -0.02355 0.11648 +X -0.10765 0.09722 -0.00864 +X -0.00782 0.00461 0.00405 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.12363 -0.02061 -0.12378 +X -0.00242 0.00340 -0.00559 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00633 0.00816 -0.00430 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05354 -0.05653 0.00643 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -36.06068 -57.41089 -81.58879 +X -3.06608 -4.45659 -1.53716 +X 4.59498 -4.67231 -0.49859 +X 5.07670 -0.51077 -3.48976 +X -6.90312 -2.42141 -1.40220 +X -4.23532 -5.48777 1.06201 +X 2.99397 -3.78407 1.80396 +X 6.81081 -3.03005 2.34615 +X -4.90791 -1.34655 2.35739 +X -2.46536 -2.19423 0.08654 +X 0.80286 -3.34341 -0.95315 +X 3.73035 0.50785 0.87413 +X -2.23332 0.98051 -1.93905 +X -1.75184 3.70192 -3.13347 +X 3.72999 5.69240 -2.74530 +X 1.39698 2.76181 -1.39661 +X -1.51326 4.48424 -1.02419 +X -6.63427 4.21372 2.85228 +X 2.96687 3.09033 3.68966 +X 2.11737 4.31377 1.59487 +X -0.86490 2.30670 1.31673 +X 0.00000 0.00000 0.00000 +X 0.00120 0.02372 -0.02333 +X 0.00000 0.00000 0.00000 +X -0.00128 0.00246 -0.00139 +X 0.00228 -0.01228 -0.00062 +X -0.00207 -0.02464 -0.00209 +X 0.05238 -0.15265 -0.08858 +X -0.17021 -0.27037 0.12999 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06975 -0.23296 0.23756 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 -0.05275 -0.05764 +X 0.18395 -0.09289 -0.29559 +X 0.13663 -0.02476 0.01187 +X 0.01909 0.00519 -0.00701 +X 0.28612 0.20026 -0.00192 +X 0.21313 0.02216 0.32092 +X 0.00000 0.00000 0.00000 +X 0.01580 0.00995 0.00842 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.03893 0.00647 -0.04894 +X 0.03845 0.03897 -0.00452 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00124 0.00276 0.00106 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00215 0.00131 -0.00132 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.15538 -0.24702 -0.04168 +X 0.00315 -0.01323 0.00582 +X 0.00000 0.00000 0.00000 +X 0.01430 -0.01311 0.00960 +X 0.00519 -0.00761 0.01384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.03784 -0.00343 -0.00129 +X -0.15347 -0.03143 0.14969 +X -0.14720 0.15527 -0.00863 +X -0.01033 0.00621 0.00579 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.15542 -0.04041 -0.16998 +X -0.00263 0.00384 -0.00633 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00525 0.00909 -0.00390 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.06301 -0.07423 0.00597 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +108 + -38.65934 -56.74544 -80.55182 +X -3.12953 -4.39135 -1.44388 +X 4.61881 -4.76109 -0.34105 +X 5.27923 -0.75337 -3.72766 +X -7.64412 -2.43833 -1.46420 +X -4.29289 -5.64014 1.32867 +X 2.99425 -3.47009 1.98388 +X 7.00602 -3.06000 2.23022 +X -4.93958 -1.16005 2.42709 +X -2.25754 -2.04854 0.10271 +X 0.39218 -3.22692 -1.00018 +X 3.57293 0.14087 0.62450 +X -1.82959 0.60450 -2.25084 +X -1.38239 3.87147 -3.54073 +X 4.44766 5.69624 -2.15455 +X 1.37954 2.89725 -1.66507 +X -1.58369 4.82583 -1.02111 +X -7.48941 3.84037 2.52487 +X 3.53781 2.62869 4.12855 +X 2.15533 4.66012 1.72352 +X -1.02592 2.39802 1.50804 +X 0.00000 0.00000 0.00000 +X -0.00006 0.03340 -0.03445 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00230 -0.01443 -0.00111 +X -0.00143 -0.01593 -0.00158 +X 0.08964 -0.16338 -0.07190 +X -0.07253 -0.25738 0.15712 +X -0.00338 -0.00671 0.00257 +X 0.00000 0.00000 0.00000 +X -0.03833 -0.12442 0.13556 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00385 -0.05373 -0.05928 +X 0.15383 -0.10836 -0.18107 +X 0.11043 -0.02403 0.00628 +X 0.01976 0.00295 -0.00484 +X 0.23424 0.13411 0.02126 +X 0.13077 0.00997 0.21001 +X 0.00000 0.00000 0.00000 +X 0.01050 0.00687 0.00618 +X 0.00000 0.00000 0.00000 +X 0.00208 -0.00168 -0.00384 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.07772 0.01034 -0.09412 +X 0.08310 0.09050 -0.01176 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00194 0.00461 0.00191 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00359 0.00242 -0.00219 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00652 -0.00365 -0.00371 +X 0.08418 -0.14892 -0.02839 +X 0.00190 -0.00912 0.00374 +X 0.00000 0.00000 0.00000 +X 0.00899 -0.00930 0.00637 +X 0.00292 -0.00455 0.00807 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.05439 -0.00563 0.00009 +X -0.18431 -0.03676 0.18473 +X -0.20546 0.22657 -0.01410 +X -0.01238 0.00749 0.00711 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.18806 -0.06796 -0.20969 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.00394 0.00928 -0.00356 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X -0.07310 -0.09603 0.00180 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 +X 0.00000 0.00000 0.00000 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/plumed.dat b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/plumed.dat new file mode 100644 index 0000000000..fabb5f2b74 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-serial/plumed.dat @@ -0,0 +1,17 @@ +LOAD FILE=../../../../plumedOpenACC.so +# With SERIAL only the first of the 2 processes computes the GPU actions, on its GPU, and +# sends the results to the other one. +# The CPU references come from regtest/adjmat/rt-nogather-serial-mpi +c1: CONTACT_MATRIX GROUPA=1-3 GROUPB=1-108 SWITCH={RATIONAL R_0=1 NN=6 MM=12 D_MAX=2} USEGPU SERIAL +ones: ONES SIZE=108 +cc: MATRIX_VECTOR_PRODUCT ARG=c1,ones USEGPU SERIAL +mtc: MORE_THAN ARG=cc SWITCH={RATIONAL D_0=2 R_0=2} USEGPU SERIAL +s: SUM ARG=mtc PERIODIC=NO +BIASVALUE ARG=s + +c2: CONTACT_MATRIX GROUP=1-20 SWITCH={RATIONAL R_0=1.5 NN=6 MM=12} USEGPU SERIAL +c22: MATRIX_PRODUCT ARG=c2,c2 USEGPU SERIAL +diag: DIAGONALIZE ARG=c22 VECTORS=1 +BIASVALUE ARG=diag.vals-1 + +PRINT ARG=s,diag.vals-1 FILE=colvar FMT=%10.5f diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/Makefile b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/Makefile new file mode 100644 index 0000000000..3703b27cea --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/Makefile @@ -0,0 +1 @@ +include ../../scripts/test.make diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/colvar.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/colvar.reference new file mode 100644 index 0000000000..b1f15576b4 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/colvar.reference @@ -0,0 +1,2 @@ +#! FIELDS time q6_mean w6_mean cn_mean + 0.000000 0.2451 0.0658 13.5703 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/config b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/config new file mode 100644 index 0000000000..324aa5d02d --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/config @@ -0,0 +1,6 @@ +type=driver +mpiprocs=3 +arg="--plumed plumed.dat --ixyz 64.xyz --dump-forces forces --dump-forces-fmt=%8.4f" +extra_files="../../trajectories/64.xyz" +# some contributions are computed in single precision on the GPU: the last printed digit may differ from the CPU +plumed_tolerance=(forces.0:2e-4 forces.1:2e-4 forces.2:2e-4 colvar:2e-4) diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.0.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.0.reference new file mode 100644 index 0000000000..24ec1d5038 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.0.reference @@ -0,0 +1,66 @@ +64 + -5.7254 -5.5713 -5.7397 +X 0.0013 0.0031 -0.0021 +X 0.0124 -0.0032 -0.0191 +X -0.0064 -0.0036 -0.0102 +X 0.0121 0.0188 0.0081 +X 0.0093 0.0070 0.0017 +X -0.0037 -0.0260 -0.0005 +X -0.0029 0.0024 -0.0081 +X -0.0041 0.0029 -0.0124 +X 0.0091 -0.0179 -0.0156 +X -0.0069 -0.0068 0.0136 +X -0.0061 0.0072 0.0006 +X 0.0041 0.0029 0.0114 +X -0.0025 -0.0007 -0.0070 +X -0.0067 0.0240 0.0046 +X 0.0006 -0.0076 -0.0015 +X 0.0010 -0.0053 -0.0037 +X -0.0030 0.0094 0.0113 +X 0.0081 -0.0147 0.0076 +X -0.0204 0.0191 0.0065 +X -0.0093 0.0039 -0.0102 +X 0.0085 -0.0037 -0.0105 +X -0.0099 0.0056 0.0084 +X -0.0100 -0.0067 0.0066 +X -0.0034 -0.0142 -0.0123 +X 0.0104 0.0058 0.0185 +X -0.0008 -0.0230 -0.0242 +X -0.0087 0.0063 0.0000 +X -0.0028 -0.0163 0.0002 +X 0.0214 -0.0144 0.0036 +X 0.0146 0.0038 0.0259 +X 0.0072 0.0112 0.0011 +X 0.0221 -0.0063 -0.0092 +X -0.0084 -0.0118 0.0016 +X -0.0159 -0.0105 0.0087 +X -0.0074 0.0063 0.0001 +X -0.0124 0.0054 0.0025 +X 0.0152 0.0278 0.0099 +X -0.0041 0.0123 0.0014 +X -0.0026 0.0041 -0.0013 +X 0.0189 0.0069 -0.0069 +X -0.0063 -0.0093 0.0075 +X 0.0205 -0.0210 -0.0057 +X -0.0028 0.0064 -0.0002 +X 0.0195 -0.0153 0.0036 +X -0.0057 0.0095 0.0046 +X 0.0038 0.0111 0.0043 +X 0.0152 -0.0087 -0.0036 +X -0.0126 0.0138 0.0023 +X -0.0029 -0.0054 -0.0120 +X -0.0100 0.0032 0.0028 +X -0.0005 0.0125 -0.0083 +X 0.0005 0.0047 0.0006 +X -0.0000 0.0020 -0.0247 +X -0.0021 -0.0054 -0.0051 +X -0.0192 -0.0051 0.0076 +X 0.0052 0.0028 0.0092 +X 0.0024 -0.0015 -0.0017 +X -0.0016 0.0072 0.0027 +X 0.0089 -0.0150 0.0046 +X -0.0076 0.0042 0.0066 +X 0.0062 0.0055 -0.0096 +X -0.0015 -0.0085 0.0012 +X -0.0168 -0.0069 0.0039 +X -0.0106 0.0157 0.0103 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.1.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.1.reference new file mode 100644 index 0000000000..24ec1d5038 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.1.reference @@ -0,0 +1,66 @@ +64 + -5.7254 -5.5713 -5.7397 +X 0.0013 0.0031 -0.0021 +X 0.0124 -0.0032 -0.0191 +X -0.0064 -0.0036 -0.0102 +X 0.0121 0.0188 0.0081 +X 0.0093 0.0070 0.0017 +X -0.0037 -0.0260 -0.0005 +X -0.0029 0.0024 -0.0081 +X -0.0041 0.0029 -0.0124 +X 0.0091 -0.0179 -0.0156 +X -0.0069 -0.0068 0.0136 +X -0.0061 0.0072 0.0006 +X 0.0041 0.0029 0.0114 +X -0.0025 -0.0007 -0.0070 +X -0.0067 0.0240 0.0046 +X 0.0006 -0.0076 -0.0015 +X 0.0010 -0.0053 -0.0037 +X -0.0030 0.0094 0.0113 +X 0.0081 -0.0147 0.0076 +X -0.0204 0.0191 0.0065 +X -0.0093 0.0039 -0.0102 +X 0.0085 -0.0037 -0.0105 +X -0.0099 0.0056 0.0084 +X -0.0100 -0.0067 0.0066 +X -0.0034 -0.0142 -0.0123 +X 0.0104 0.0058 0.0185 +X -0.0008 -0.0230 -0.0242 +X -0.0087 0.0063 0.0000 +X -0.0028 -0.0163 0.0002 +X 0.0214 -0.0144 0.0036 +X 0.0146 0.0038 0.0259 +X 0.0072 0.0112 0.0011 +X 0.0221 -0.0063 -0.0092 +X -0.0084 -0.0118 0.0016 +X -0.0159 -0.0105 0.0087 +X -0.0074 0.0063 0.0001 +X -0.0124 0.0054 0.0025 +X 0.0152 0.0278 0.0099 +X -0.0041 0.0123 0.0014 +X -0.0026 0.0041 -0.0013 +X 0.0189 0.0069 -0.0069 +X -0.0063 -0.0093 0.0075 +X 0.0205 -0.0210 -0.0057 +X -0.0028 0.0064 -0.0002 +X 0.0195 -0.0153 0.0036 +X -0.0057 0.0095 0.0046 +X 0.0038 0.0111 0.0043 +X 0.0152 -0.0087 -0.0036 +X -0.0126 0.0138 0.0023 +X -0.0029 -0.0054 -0.0120 +X -0.0100 0.0032 0.0028 +X -0.0005 0.0125 -0.0083 +X 0.0005 0.0047 0.0006 +X -0.0000 0.0020 -0.0247 +X -0.0021 -0.0054 -0.0051 +X -0.0192 -0.0051 0.0076 +X 0.0052 0.0028 0.0092 +X 0.0024 -0.0015 -0.0017 +X -0.0016 0.0072 0.0027 +X 0.0089 -0.0150 0.0046 +X -0.0076 0.0042 0.0066 +X 0.0062 0.0055 -0.0096 +X -0.0015 -0.0085 0.0012 +X -0.0168 -0.0069 0.0039 +X -0.0106 0.0157 0.0103 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.2.reference b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.2.reference new file mode 100644 index 0000000000..24ec1d5038 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/forces.2.reference @@ -0,0 +1,66 @@ +64 + -5.7254 -5.5713 -5.7397 +X 0.0013 0.0031 -0.0021 +X 0.0124 -0.0032 -0.0191 +X -0.0064 -0.0036 -0.0102 +X 0.0121 0.0188 0.0081 +X 0.0093 0.0070 0.0017 +X -0.0037 -0.0260 -0.0005 +X -0.0029 0.0024 -0.0081 +X -0.0041 0.0029 -0.0124 +X 0.0091 -0.0179 -0.0156 +X -0.0069 -0.0068 0.0136 +X -0.0061 0.0072 0.0006 +X 0.0041 0.0029 0.0114 +X -0.0025 -0.0007 -0.0070 +X -0.0067 0.0240 0.0046 +X 0.0006 -0.0076 -0.0015 +X 0.0010 -0.0053 -0.0037 +X -0.0030 0.0094 0.0113 +X 0.0081 -0.0147 0.0076 +X -0.0204 0.0191 0.0065 +X -0.0093 0.0039 -0.0102 +X 0.0085 -0.0037 -0.0105 +X -0.0099 0.0056 0.0084 +X -0.0100 -0.0067 0.0066 +X -0.0034 -0.0142 -0.0123 +X 0.0104 0.0058 0.0185 +X -0.0008 -0.0230 -0.0242 +X -0.0087 0.0063 0.0000 +X -0.0028 -0.0163 0.0002 +X 0.0214 -0.0144 0.0036 +X 0.0146 0.0038 0.0259 +X 0.0072 0.0112 0.0011 +X 0.0221 -0.0063 -0.0092 +X -0.0084 -0.0118 0.0016 +X -0.0159 -0.0105 0.0087 +X -0.0074 0.0063 0.0001 +X -0.0124 0.0054 0.0025 +X 0.0152 0.0278 0.0099 +X -0.0041 0.0123 0.0014 +X -0.0026 0.0041 -0.0013 +X 0.0189 0.0069 -0.0069 +X -0.0063 -0.0093 0.0075 +X 0.0205 -0.0210 -0.0057 +X -0.0028 0.0064 -0.0002 +X 0.0195 -0.0153 0.0036 +X -0.0057 0.0095 0.0046 +X 0.0038 0.0111 0.0043 +X 0.0152 -0.0087 -0.0036 +X -0.0126 0.0138 0.0023 +X -0.0029 -0.0054 -0.0120 +X -0.0100 0.0032 0.0028 +X -0.0005 0.0125 -0.0083 +X 0.0005 0.0047 0.0006 +X -0.0000 0.0020 -0.0247 +X -0.0021 -0.0054 -0.0051 +X -0.0192 -0.0051 0.0076 +X 0.0052 0.0028 0.0092 +X 0.0024 -0.0015 -0.0017 +X -0.0016 0.0072 0.0027 +X 0.0089 -0.0150 0.0046 +X -0.0076 0.0042 0.0066 +X 0.0062 0.0055 -0.0096 +X -0.0015 -0.0085 0.0012 +X -0.0168 -0.0069 0.0039 +X -0.0106 0.0157 0.0103 diff --git a/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/plumed.dat b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/plumed.dat new file mode 100644 index 0000000000..4176feb300 --- /dev/null +++ b/plugins/openaccPTM/regtest/targetGPU/rt-GPU-mpi-shortcuts/plumed.dat @@ -0,0 +1,9 @@ +LOAD FILE=../../../../plumedOpenACC.so +# The shortcuts pass USEGPU and SERIAL to the actions they create that run on the GPU. With 3 MPI +# processes, the GPU actions of q6 and w6 split their tasks between the processes, and those of cn +# are computed by the first process only (SERIAL). The references are the results of the CPU versions +q6: Q6 SPECIES=1-64 D_0=3.0 R_0=1.5 NN=12 MM=24 MEAN USEGPU +w6: LOCAL_AVERAGE SPECIES=q6 SWITCH={RATIONAL D_0=3.0 R_0=1.5} MEAN USEGPU +cn: COORDINATIONNUMBER SPECIES=1-64 SWITCH={RATIONAL D_0=3.0 R_0=1.5} MEAN USEGPU SERIAL +r: RESTRAINT ARG=q6.mean,w6_mean,cn.mean AT=0.3,0.3,10 KAPPA=10,10,0.1 +PRINT ARG=q6.mean,w6_mean,cn.mean FILE=colvar FMT=%8.4f diff --git a/plugins/openaccPTM/src/ACCDevice.cpp b/plugins/openaccPTM/src/ACCDevice.cpp index 5fd1a34173..a6c99b6e82 100644 --- a/plugins/openaccPTM/src/ACCDevice.cpp +++ b/plugins/openaccPTM/src/ACCDevice.cpp @@ -27,6 +27,8 @@ #include +#include + #if defined(PLUMED_OPENACC_GPU) && defined(__linux__) #include #endif @@ -99,6 +101,32 @@ class NoCurrentContext { NoCurrentContext& operator=(const NoCurrentContext&)=delete; }; +/// The rank of this process among the processes of its node, as set by the MPI launcher +/// in the environment (-1 if not set); var is the variable it was read from +int localRank(const char*& var) { + // the variables of the launchers first: SLURM_LOCALID is also set (to 0) in the batch + // step of a job, where a launcher like OpenMPI's mpirun may be used + static const char* const vars[]= {"OMPI_COMM_WORLD_LOCAL_RANK","MV2_COMM_WORLD_LOCAL_RANK","MPI_LOCALRANKID", + "PMI_LOCAL_RANK","PALS_LOCAL_RANKID","FLUX_TASK_LOCAL_ID","SLURM_LOCALID" + }; + for(const char* v : vars) { + if(const char* e=std::getenv(v)) { + char* end=nullptr; + const long r=std::strtol(e,&end,10); + if(end!=e && *end=='\0' && r>=0) { + var=v; + return static_cast(r); + } + } + } + return -1; +} + +/// True if the user chose the OpenACC device with the environment +bool deviceSetInEnvironment() { + return std::getenv("ACC_DEVICE_NUM") || std::getenv("NVCOMPILER_ACC_DEVICE_NUM") || std::getenv("PGI_ACC_DEVICE_NUM"); +} + } int currentCudaDevice() { @@ -125,6 +153,15 @@ int Device::resolve(const Action& action) { } else if(caller>=0 && caller1 && !deviceSetInEnvironment()) ? localRank(var) : -1; + if(lr>=0) { + id_=lr%ndevices; + action.log.printf(" %s runs on GPU %d (process %d of the node, from %s, with %d GPUs)\n", + action.getLabel().c_str(),id_,lr,var,ndevices); + return id_; + } id_=acc_get_device_num(acc_device_not_host); } action.log.printf(" %s runs on GPU %d\n",action.getLabel().c_str(),id_); diff --git a/plugins/openaccPTM/src/ACCDevice.h b/plugins/openaccPTM/src/ACCDevice.h index e05a32d254..b77660c2b0 100644 --- a/plugins/openaccPTM/src/ACCDevice.h +++ b/plugins/openaccPTM/src/ACCDevice.h @@ -33,8 +33,11 @@ and every part of the plugin follows them in the same way: time it is needed with Device::resolve(action), which also writes it in the log of the action ("