merge changes or Kripke

LLNL · Sep 2, 2024 · eb7b77b · eb7b77b
2 parents f8782a2 + e5ed7d2
commit eb7b77b
Show file tree

Hide file tree

Showing 18 changed files with 248 additions and 35 deletions.
diff --git a/.github/workflows/requirements/docs.txt b/.github/workflows/requirements/docs.txt
@@ -3,7 +3,7 @@ sphinx==7.4.7
 sphinx-rtd-theme==2.0.0
 codespell==2.3.0
 pandas==2.2.2
-pyyaml==6.0.1
+pyyaml==6.0.2
 sphinxcontrib-programoutput==0.17
 # The remaining requirements are from Ramble
 pytest

diff --git a/.github/workflows/requirements/style.txt b/.github/workflows/requirements/style.txt
@@ -1,4 +1,4 @@
-black==24.4.2
-flake8==7.1.0
+black==24.8.0
+flake8==7.1.1
 isort==5.13.2
 codespell==2.3.0
diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
@@ -70,7 +70,7 @@ jobs:
 
       - name: Upload Workspace Archive as CI Artifact
         if: always()
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
         with:
           name: workspace-archive
           path: './workspace/saxpy/openmp/nosite-x86_64/workspace/archive/**'
@@ -199,3 +199,13 @@ jobs:
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
+
+      - name: Dry run genesis/openmp with allocation modifier on Fugaku
+        run: |
+          ./bin/benchpark setup genesis/openmp RCCS-Fugaku-Fujitsu-A64FX-TofuD workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir workspace/genesis/openmp/RCCS-Fugaku-Fujitsu-A64FX-TofuD/workspace \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml
@@ -64,6 +64,8 @@ packages:
         prefix: /usr
   cmake:
     externals:
+      - spec: "[email protected] arch=linux-rhel8-a64fx"
+        prefix: /vol0004/apps/oss/spack/opt/spack/linux-rhel8-a64fx/fj-4.10.0/cmake-3.27.7-ussgjuqkqbxi5dcv7kbp6bugdcjc5ph6
       - spec: "[email protected] arch=linux-rhel8-a64fx"
         prefix: /usr
   curl:

diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/software.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/software.yaml
@@ -6,15 +6,19 @@
 software:
   packages:
     default-compiler:
-      pkg_spec: "{default_comp} {sys_arch}"
+      pkg_spec: clang@{default_llvm_version}
     default-mpi:
-      pkg_spec: fujitsu-mpi@{fj_comp_version}%{default_comp} {sys_arch}
+      pkg_spec: fujitsu-mpi@{default_fj_version}
+      compiler: default-compiler
+    compiler-clang:
+      pkg_spec: clang@{default_llvm_version}
+    compiler-fj:
+      pkg_spec: fj@{default_fj_version}
+    compiler-gcc:
+      pkg_spec: gcc@{default_gnu_version}
     blas:
-      pkg_spec: fujitsu-ssl2@{fj_comp_version}%{default_comp} {sys_arch}
+      pkg_spec: fujitsu-ssl2@{default_fj_version}
+      compiler: default-compiler
     lapack:
-      pkg_spec: fujitsu-ssl2@{fj_comp_version}%{default_comp} {sys_arch}
-    gmake:
-      pkg_spec: [email protected] {sys_arch}
-    cmake:
-      pkg_spec: [email protected] {sys_arch}
-
+      pkg_spec: fujitsu-ssl2@{default_fj_version}
+      compiler: default-compiler
diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml
@@ -22,4 +22,3 @@ variables:
   n_nodes: '1000001'  # placeholder value
   batch_submit: "placeholder"
   mpi_command: "placeholder"
-  #sys_arch: 'arch=linux-rhel8-a64fx'
diff --git a/docs/system/fugaku.rst b/docs/system/fugaku.rst
@@ -48,12 +48,10 @@ Patch some files in various repos:
 
 .. code:: bash
 
-    sed -i -e "s@1280000000@160000000@g" -e 's@cflags=".*"@@g' experiments/streamc/openmp/ramble.yaml
+    sed -i -e "s@256000000@134217728@g" experiments/stream/openmp/ramble.yaml
     wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku5/lib/spack/spack/util/libc.py -O workspace/spack/lib/spack/spack/util/libc.py
-    wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku6/var/spack/repos/builtin/packages/hpcg/package.py -O workspace/spack/var/spack/repos/builtin/packages/hpcg/package.py
     wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku10/var/spack/repos/builtin/packages/hpl/package.py -O workspace/spack/var/spack/repos/builtin/packages/hpl/package.py
     wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku11/var/spack/repos/builtin/packages/fujitsu-ssl2/package.py -O workspace/spack/var/spack/repos/builtin/packages/fujitsu-ssl2/package.py
-    wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku12/var/spack/repos/builtin/packages/fujitsu-mpi/package.py -O workspace/spack/var/spack/repos/builtin/packages/fujitsu-mpi/package.py
     #ONLY FOR CLANG BUILDS: sed -i -e 's@SYSTEM_PATHS = \[\(.*\)\]@SYSTEM_PATHS = [\1, "/opt/FJSVxtclanga/tcsds-mpi-1.2.38", "/opt/FJSVxtclanga/tcsds-ssl2-1.2.38"]@g' workspace/spack/lib/spack/spack/util/environment.py
     #ONLY FOR CLANG BUILDS: sed -i -e 's@%fj"):@%fj") or (spec.target == "a64fx" and spec.satisfies("%clang\@11:")):@g' workspace/spack/var/spack/repos/builtin/packages/cmake/package.py
 

diff --git a/experiments/amg2023/openmp/ramble.yaml b/experiments/amg2023/openmp/ramble.yaml
@@ -47,9 +47,7 @@ ramble:
   software:
     packages:
       hypre-omp:
-        # spack_spec: [email protected] +mpi+openmp+mixedint~fortran{modifier_spack_variant}
-        spack_spec: hypre@comm_cali +mpi+openmp+mixedint~fortran{modifier_spack_variant}
-        # pkg_spec: [email protected] +mpi+openmp+mixedint~fortran{modifier_spack_variant}
+        pkg_spec: [email protected] +mpi+openmp+mixedint~fortran{modifier_spack_variant}
         compiler: default-compiler
       amg2023-omp:
         pkg_spec: amg2023@develop +mpi+openmp{modifier_spack_variant}

diff --git a/experiments/genesis/openmp/ramble.yaml b/experiments/genesis/openmp/ramble.yaml
@@ -0,0 +1,51 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/software.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  modifiers:
+  - name: allocation
+
+  applications:
+    genesis:
+      workloads:
+        DHFR:
+          env_vars:
+            set:
+              OMP_NUM_THREADS: '{omp_num_threads}'
+          variants:
+            package_manager: spack
+          variables:
+            n_ranks: '{processes_per_node} * {n_nodes}'
+            experiment_setup: ''
+            processes_per_node: ['int({sys_cores_per_node} / 6)']
+            n_nodes: ['1']
+            omp_num_threads: ['{processes_per_node} / 2']
+            arch: 'OpenMP'
+          experiments:
+            genesis_omp_{n_nodes}_{processes_per_node}_{omp_num_threads}:
+              variables: {}
+  software:
+    packages:
+      genesis:
+        pkg_spec: [email protected] +mpi+openmp{modifier_spack_variant}
+        compiler: default-compiler
+    environments:
+      genesis:
+        packages:
+        - lapack
+        - default-mpi
+        - genesis
+        - '{modifier_package_name}'
diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml
@@ -64,7 +64,6 @@ ramble:
         pkg_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
         compiler: default-compiler
       kripke:
-        # spack_spec: kripke@develop +mpi+cuda{modifier_spack_variant} ^[email protected]
         spack_spec: kripke@comm_cali +mpi+cuda{modifier_spack_variant} ^[email protected]
         compiler: default-compiler
     environments:

diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml
@@ -63,7 +63,7 @@ ramble:
   software:
     packages:
       kripke:
-# spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^[email protected]
+        # spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^[email protected]
         pkg_spec: kripke@comm_cali +mpi+openmp{modifier_spack_variant} ^[email protected]
         # spack_spec: [email protected] +mpi+openmp{modifier_spack_variant} ^[email protected]
         # # 3918228 (edit spack spec version)

diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml
@@ -62,7 +62,6 @@ ramble:
     packages:
       kripke:
         spack_spec: kripke@comm_cali +mpi+rocm{modifier_spack_variant} ^[email protected]
-        # spack_spec: kripke@develop +mpi+rocm{modifier_spack_variant} ^[email protected]
         compiler: default-compiler
     environments:
       kripke:

diff --git a/repo/amg2023/package.py b/repo/amg2023/package.py
@@ -34,7 +34,7 @@ class Amg2023(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("adiak", when="+caliper")
     depends_on("hypre+caliper", when="+caliper")
     # depends_on("[email protected]:")
-    depends_on("hypre@comm_cali")
+    depends_on("hypre@3.1_comm_cali")
     depends_on("hypre+cuda", when="+cuda")
     requires("+cuda", when="^hypre+cuda")
     depends_on("hypre+rocm", when="+rocm")

diff --git a/repo/genesis/application.py b/repo/genesis/application.py
@@ -0,0 +1,51 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from ramble.appkit import *
+
+import sys
+
+class Genesis(SpackApplication):
+    """GENESIS package contains two MD programs (atdyn and spdyn), trajectory
+    analysis programs, and other useful tools. GENESIS (GENeralized-Ensemble
+    SImulation System) has been developed mainly by Sugita group in RIKEN-CCS.
+    """
+    name = "GENESIS"
+
+    tags = ['molecular-dynamics','mpi', 'openmp', 'cuda']
+
+    executable('chdir', 'cd $(dirname {input})', use_mpi=False)
+    executable('genesis', 'spdyn {input}', use_mpi=True)
+
+    input_file('benchmark-2020',
+               url='https://www.r-ccs.riken.jp/labs/cbrt/wp-content/uploads/2020/12/benchmark_mkl_ver4_nocrowding.tar.gz',
+               sha256='2ca8b2d4974dc0be0a42064392f1d5c603c64ffa9adc1f3bcf7c146a3bbf5bdb',
+               description='Benchmark set for GENESIS 2.0 beta / 1.6 on FUGAKU')
+    input_file('tests-2.1.1',
+               url='https://www.r-ccs.riken.jp/labs/cbrt/wp-content/uploads/2023/09/tests-2.1.1.tar.bz2',
+               sha256='f24d872beae5e38baa6a591906f78e3438186973c88e4879e4b04a0cca74f83e',
+               description='Regression tests are prepared for ATDYN, SPDYN, prst_setup (parallel I/O), and analysis tools to check if these programs work correctly.')
+
+    workload('DHFR', executables=['chdir', 'genesis'], input='benchmark-2020')
+    workload('ApoA1', executables=['chdir', 'genesis'], input='benchmark-2020')
+    workload('UUN', executables=['chdir', 'genesis'], input='benchmark-2020')
+    workload('cryoEM', executables=['chdir', 'genesis'], input='tests-2.1.1')
+
+    workload_variable('input', default='{benchmark-2020}/npt/genesis2.0beta/jac_amber/p{n_ranks}.inp',
+                      description='jac_amber/ : DHFR (27,346 atoms), AMBER format, soluble system',
+                      workloads=['DHFR'])
+    workload_variable('input', default='{benchmark-2020}/npt/genesis2.0beta/apoa1/p{n_ranks}.inp',
+                      description='apoa1/ : apoa1 (92,224 atoms), CHARMM format, soluble system',
+                      workloads=['ApoA1'])
+    workload_variable('input', default='{benchmark-2020}/npt/genesis2.0beta/uun/p{n_ranks}.inp',
+                      description='uun/ : uun (216,726 atoms), CHARMM format, membrane+solvent system',
+                      workloads=['UUN'])
+    workload_variable('input', default='{tests-2.1.1}/regression_test/test_spdyn/cryoEM/All_atom/inp',
+                      description='cryoEM/All_atom/ : cryoEM (? atoms), CHARMM format',
+                      workloads=['cryoEM'])
+
+    figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'^\s+dynamics\s+=\s+(?P<fom>[-+]?([0-9]*[.])?[0-9]+([eED][-+]?[0-9]+)?)', group_name='fom', units='')
+
+    success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out')
diff --git a/repo/genesis/package.py b/repo/genesis/package.py
@@ -0,0 +1,102 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from spack.package import *
+
+
+class Genesis(AutotoolsPackage):
+    """GENESIS package contains two MD programs (atdyn and spdyn), trajectory
+    analysis programs, and other useful tools. GENESIS (GENeralized-Ensemble
+    SImulation System) has been developed mainly by Sugita group in RIKEN-CCS.
+    """
+
+    tags = ['genesis','benchmark']
+
+    homepage = "https://www.r-ccs.riken.jp/labs/cbrt/"
+    git = "https://github.com/genesis-release-r-ccs/genesis"
+
+    version("master", branch="master", submodules=False)
+    version(
+        "2.1.3", submodules=False, tag="v2.1.3", commit="835ef1538f9350cfa7e9489f340837d0908afbd2"
+    )
+    version(
+        "2.1.1", submodules=False, tag="v2.1.1", commit="38a54fe1c749f4d87bff591e65c61b23a7396f9d"
+    )
+    version(
+        "2.0.3", submodules=False, tag="v2.0.3", commit="6989e0b24470e374ea343b2b7b685aca87909571"
+    )
+
+    variant("mpi", default=True, description="Build with MPI.")
+    variant("openmp", default=True, description="Build with OpenMP enabled.")
+    variant("lapack", default=True, description="Build with LAPACK enabled.")
+    variant("gpu", default=False, description="Build with GPGPU enabled.")
+    variant("precision", description="Build with selected precision.", default="double", values=("double", "mixed", "single", "large_int"), multi=False)
+    variant("simd", description="Build with SIMD width.", default="auto", values=("auto", "MIC-AVX512", "CORE-AVX512", "CORE-AVX2"), multi=False)
+    variant("debug", description="Set Debug level", default="0", values=("0", "1", "2", "3", "4"), multi=False)
+
+    depends_on("mpi", when="+mpi")
+    depends_on("lapack", when="+lapack")
+    depends_on("cuda", when="+gpu")
+
+    def autoreconf(self, spec, prefix):
+        bash = which("bash")
+        bash("./bootstrap")
+
+    @run_before("configure")
+    def fix_programming_error(self):
+        spec = self.spec
+        if self.version < Version("2.1.3"):
+            filter_file(r"atomcls1\(1:3\)", "atomcls1(1:6)", join_path(self.stage.source_path, "src/analysis/sp_analysis/hbond_analysis/hbond_analyze.fpp"))
+            filter_file(r"atomcls2\(1:3\)", "atomcls2(1:6)", join_path(self.stage.source_path, "src/analysis/sp_analysis/hbond_analysis/hbond_analyze.fpp"))
+
+    def configure_args(self):
+        spec = self.spec
+        config_args = [f"--enable-{spec.variants['precision'].value}",
+                       f"--with-simd={spec.variants['simd'].value}"]
+
+        if int(spec.variants['debug'].value) > 0:
+            config_args.extend(f"--enable-debug={spec.variants['debug'].value}")
+
+        config_args.extend(self.enable_or_disable("mpi"))
+        config_args.extend(self.enable_or_disable("openmp"))
+        config_args.extend(self.with_or_without("lapack"))
+        config_args.extend(self.enable_or_disable("gpu"))
+
+        if "+mpi" in spec:
+            env["CC"] = spec["mpi"].mpicc
+            env["CXX"] = spec["mpi"].mpicxx
+            env["FC"] = spec["mpi"].mpifc
+            env["F77"] = spec["mpi"].mpif77
+
+        if "+openmp" in spec and spec.satisfies("%clang"):
+            env["OPT_OPENMP"] = "-fopenmp"
+
+        if spec.satisfies("%clang"):
+            opt_flags = "-Ofast -ffast-math"
+            env["CFLAGS"] = f"{opt_flags}"
+            env["CXXFLAGS"] = f"{opt_flags}"
+            env["FCFLAGS"] = f"{opt_flags} -Mbackslash"
+            env["F77FLAGS"] = f"{opt_flags} -Mbackslash"
+            # cpp workaround; other systems and OS likely need different pre-processor fix
+            if spec.target == "a64fx":
+                env["FPP"] = "/opt/FJSVxtclanga/tcsds-1.2.38/bin/../lib/fpp"
+                env["PPFLAGS"] = "-traditional-cpp -traditional"
+        elif spec.satisfies("%fj"):
+            opt_flags = "-Kfast"
+            env["CFLAGS"] = f"{opt_flags}"
+            env["CXXFLAGS"] = f"{opt_flags}"
+            env["FCFLAGS"] = f"{opt_flags}"
+            env["F77FLAGS"] = f"{opt_flags}"
+        elif spec.satisfies("%gcc"):
+            opt_flags = "-O3 -ffast-math"
+            env["CFLAGS"] = f"{opt_flags}"
+            env["CXXFLAGS"] = f"{opt_flags}"
+            env["FCFLAGS"] = f"{opt_flags} -ffree-line-length-none"
+            env["F77FLAGS"] = f"{opt_flags} -ffree-line-length-none"
+
+        if "+gpu" in spec:
+            config_args.extend(self.with_or_without("cuda", activation_value=spec["cuda"].prefix))
+
+        return config_args
diff --git a/repo/hpl/package.py b/repo/hpl/package.py
@@ -116,25 +116,24 @@ def configure(self, spec, prefix):
     def configure_args(self):
         filter_file(r"^libs10=.*", "libs10=%s" % self.spec["blas"].libs.ld_flags, "configure")
 
+        cc, cflags, ldflags = self.spec["mpi"].mpicc, ["-O3"], []
         if "+openmp" in self.spec:
-            config = ["CFLAGS=-O3 " + self.compiler.openmp_flag]
-        else:
-            config = ["CFLAGS=-O3"]
+            cflags.append(self.compiler.openmp_flag)
 
         if (
             self.spec.satisfies("^intel-mkl")
             or self.spec.satisfies("^intel-oneapi-mkl")
             or self.spec.satisfies("^intel-parallel-studio+mkl")
         ):
-            config.append("LDFLAGS={0}".format(self.spec["blas"].libs.ld_flags))
+            ldflags.append(self.spec["blas"].libs.ld_flags)
 
         if "%aocc" in self.spec:
-            amd_ldflags = " "
             if "%aocc@3:" in self.spec:
-                amd_ldflags += "-lamdlibm -lm "
+                ldflags.extend(["-lamdlibm", "-lm"])
             if "%aocc@4:" in self.spec:
-                amd_ldflags += "-lamdalloc "
-            config.append("LDFLAGS=" + amd_ldflags)
+                ldflags.append("-lamdalloc")
+
+        config = [f"CC={cc}", f"CFLAGS={' '.join(cflags)}", f"LDFLAGS={' '.join(ldflags)}"]
 
         if "+caliper" in self.spec:
             config.append("--with-adiak={0}".format(self.spec["adiak"].prefix))

diff --git a/repo/hypre/package.py b/repo/hypre/package.py
@@ -18,6 +18,8 @@ class Hypre(BuiltinHypre):
         "gcc": "gnu",
     }
 
+    version("3.1_comm_cali", branch="comm_cali", submodules=False, git="https://github.com/gracenansamba/hypre.git")
+
     def configure_args(self):
         configure_args = super().configure_args()
 

diff --git a/repo/qws/application.py b/repo/qws/application.py
@@ -60,7 +60,6 @@ class Qws(ExecutableApplication):
                       description='Maximum iterations of inner DD solver. (Default: 50)',
                       workloads=['qws'])
 
-    #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P<fom>[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='')
+    figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'etime for so.ler =\s+(?P<fom>[-+]?([0-9]*[.])?[0-9]+([eED][-+]?[0-9]+)?)', group_name='fom', units='')
 
-    #TODO: Fix the FOM success_criteria(...)
-    #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out')
+    success_criteria('pass', mode='string', match=r'print timing', file='{experiment_run_dir}/{experiment_name}.out')