diff --git a/.github/workflows/requirements/docs.txt b/.github/workflows/requirements/docs.txt index dd0e07106..9969fb9dc 100644 --- a/.github/workflows/requirements/docs.txt +++ b/.github/workflows/requirements/docs.txt @@ -3,7 +3,7 @@ sphinx==7.4.7 sphinx-rtd-theme==2.0.0 codespell==2.3.0 pandas==2.2.2 -pyyaml==6.0.1 +pyyaml==6.0.2 sphinxcontrib-programoutput==0.17 # The remaining requirements are from Ramble pytest diff --git a/.github/workflows/requirements/style.txt b/.github/workflows/requirements/style.txt index e866ca92c..63d6dd7d3 100644 --- a/.github/workflows/requirements/style.txt +++ b/.github/workflows/requirements/style.txt @@ -1,4 +1,4 @@ -black==24.4.2 -flake8==7.1.0 +black==24.8.0 +flake8==7.1.1 isort==5.13.2 codespell==2.3.0 diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index c88275b61..638121283 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -70,7 +70,7 @@ jobs: - name: Upload Workspace Archive as CI Artifact if: always() - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a with: name: workspace-archive path: './workspace/saxpy/openmp/nosite-x86_64/workspace/archive/**' @@ -199,3 +199,13 @@ jobs: --disable-progress-bar \ --disable-logger \ workspace setup --dry-run + + - name: Dry run genesis/openmp with allocation modifier on Fugaku + run: | + ./bin/benchpark setup genesis/openmp RCCS-Fugaku-Fujitsu-A64FX-TofuD workspace/ + . workspace/setup.sh + ramble \ + --workspace-dir workspace/genesis/openmp/RCCS-Fugaku-Fujitsu-A64FX-TofuD/workspace \ + --disable-progress-bar \ + --disable-logger \ + workspace setup --dry-run diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml index 4fea04d63..8e96ba0ce 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml @@ -64,6 +64,8 @@ packages: prefix: /usr cmake: externals: + - spec: "cmake@3.27.7 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/spack/opt/spack/linux-rhel8-a64fx/fj-4.10.0/cmake-3.27.7-ussgjuqkqbxi5dcv7kbp6bugdcjc5ph6 - spec: "cmake@3.20.2 arch=linux-rhel8-a64fx" prefix: /usr curl: diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/software.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/software.yaml index ab7f49c24..c87e49162 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/software.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/software.yaml @@ -6,15 +6,19 @@ software: packages: default-compiler: - pkg_spec: "{default_comp} {sys_arch}" + pkg_spec: clang@{default_llvm_version} default-mpi: - pkg_spec: fujitsu-mpi@{fj_comp_version}%{default_comp} {sys_arch} + pkg_spec: fujitsu-mpi@{default_fj_version} + compiler: default-compiler + compiler-clang: + pkg_spec: clang@{default_llvm_version} + compiler-fj: + pkg_spec: fj@{default_fj_version} + compiler-gcc: + pkg_spec: gcc@{default_gnu_version} blas: - pkg_spec: fujitsu-ssl2@{fj_comp_version}%{default_comp} {sys_arch} + pkg_spec: fujitsu-ssl2@{default_fj_version} + compiler: default-compiler lapack: - pkg_spec: fujitsu-ssl2@{fj_comp_version}%{default_comp} {sys_arch} - gmake: - pkg_spec: gmake@4.2.1 {sys_arch} - cmake: - pkg_spec: cmake@3.20.2 {sys_arch} - + pkg_spec: fujitsu-ssl2@{default_fj_version} + compiler: default-compiler diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml index 7dd239943..5e29684d6 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -22,4 +22,3 @@ variables: n_nodes: '1000001' # placeholder value batch_submit: "placeholder" mpi_command: "placeholder" - #sys_arch: 'arch=linux-rhel8-a64fx' diff --git a/docs/system/fugaku.rst b/docs/system/fugaku.rst index c97290fd1..7549b9a24 100644 --- a/docs/system/fugaku.rst +++ b/docs/system/fugaku.rst @@ -48,12 +48,10 @@ Patch some files in various repos: .. code:: bash - sed -i -e "s@1280000000@160000000@g" -e 's@cflags=".*"@@g' experiments/streamc/openmp/ramble.yaml + sed -i -e "s@256000000@134217728@g" experiments/stream/openmp/ramble.yaml wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku5/lib/spack/spack/util/libc.py -O workspace/spack/lib/spack/spack/util/libc.py - wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku6/var/spack/repos/builtin/packages/hpcg/package.py -O workspace/spack/var/spack/repos/builtin/packages/hpcg/package.py wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku10/var/spack/repos/builtin/packages/hpl/package.py -O workspace/spack/var/spack/repos/builtin/packages/hpl/package.py wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku11/var/spack/repos/builtin/packages/fujitsu-ssl2/package.py -O workspace/spack/var/spack/repos/builtin/packages/fujitsu-ssl2/package.py - wget https://raw.githubusercontent.com/jdomke/spack/RIKEN_CCS_fugaku12/var/spack/repos/builtin/packages/fujitsu-mpi/package.py -O workspace/spack/var/spack/repos/builtin/packages/fujitsu-mpi/package.py #ONLY FOR CLANG BUILDS: sed -i -e 's@SYSTEM_PATHS = \[\(.*\)\]@SYSTEM_PATHS = [\1, "/opt/FJSVxtclanga/tcsds-mpi-1.2.38", "/opt/FJSVxtclanga/tcsds-ssl2-1.2.38"]@g' workspace/spack/lib/spack/spack/util/environment.py #ONLY FOR CLANG BUILDS: sed -i -e 's@%fj"):@%fj") or (spec.target == "a64fx" and spec.satisfies("%clang\@11:")):@g' workspace/spack/var/spack/repos/builtin/packages/cmake/package.py diff --git a/experiments/amg2023/openmp/ramble.yaml b/experiments/amg2023/openmp/ramble.yaml index 135cda993..38c44bb0d 100644 --- a/experiments/amg2023/openmp/ramble.yaml +++ b/experiments/amg2023/openmp/ramble.yaml @@ -47,9 +47,7 @@ ramble: software: packages: hypre-omp: - # spack_spec: hypre@2.31.0 +mpi+openmp+mixedint~fortran{modifier_spack_variant} - spack_spec: hypre@comm_cali +mpi+openmp+mixedint~fortran{modifier_spack_variant} - # pkg_spec: hypre@2.31.0 +mpi+openmp+mixedint~fortran{modifier_spack_variant} + pkg_spec: hypre@2.31.0 +mpi+openmp+mixedint~fortran{modifier_spack_variant} compiler: default-compiler amg2023-omp: pkg_spec: amg2023@develop +mpi+openmp{modifier_spack_variant} diff --git a/experiments/genesis/openmp/ramble.yaml b/experiments/genesis/openmp/ramble.yaml new file mode 100644 index 000000000..471e53035 --- /dev/null +++ b/experiments/genesis/openmp/ramble.yaml @@ -0,0 +1,51 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + include: + - ./configs/software.yaml + - ./configs/variables.yaml + - ./configs/modifier.yaml + + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + modifiers: + - name: allocation + + applications: + genesis: + workloads: + DHFR: + env_vars: + set: + OMP_NUM_THREADS: '{omp_num_threads}' + variants: + package_manager: spack + variables: + n_ranks: '{processes_per_node} * {n_nodes}' + experiment_setup: '' + processes_per_node: ['int({sys_cores_per_node} / 6)'] + n_nodes: ['1'] + omp_num_threads: ['{processes_per_node} / 2'] + arch: 'OpenMP' + experiments: + genesis_omp_{n_nodes}_{processes_per_node}_{omp_num_threads}: + variables: {} + software: + packages: + genesis: + pkg_spec: genesis@2.1.3 +mpi+openmp{modifier_spack_variant} + compiler: default-compiler + environments: + genesis: + packages: + - lapack + - default-mpi + - genesis + - '{modifier_package_name}' diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml index 58cc9135c..1a53697c2 100644 --- a/experiments/kripke/cuda/ramble.yaml +++ b/experiments/kripke/cuda/ramble.yaml @@ -64,7 +64,6 @@ ramble: pkg_spec: cuda@{default_cuda_version}+allow-unsupported-compilers compiler: default-compiler kripke: - # spack_spec: kripke@develop +mpi+cuda{modifier_spack_variant} ^chai@2024.02 spack_spec: kripke@comm_cali +mpi+cuda{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml index 7dde93ae2..ca17f89be 100644 --- a/experiments/kripke/openmp/ramble.yaml +++ b/experiments/kripke/openmp/ramble.yaml @@ -63,7 +63,7 @@ ramble: software: packages: kripke: -# spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^chai@2024.02 + # spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^chai@2024.02 pkg_spec: kripke@comm_cali +mpi+openmp{modifier_spack_variant} ^chai@2024.02 # spack_spec: kripke@1.2.7.0 +mpi+openmp{modifier_spack_variant} ^chai@2024.02 # # 3918228 (edit spack spec version) diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml index 1bf07a995..29892ce21 100644 --- a/experiments/kripke/rocm/ramble.yaml +++ b/experiments/kripke/rocm/ramble.yaml @@ -62,7 +62,6 @@ ramble: packages: kripke: spack_spec: kripke@comm_cali +mpi+rocm{modifier_spack_variant} ^chai@2024.02 - # spack_spec: kripke@develop +mpi+rocm{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: kripke: diff --git a/repo/amg2023/package.py b/repo/amg2023/package.py index 65ec932cb..6b81bb7fb 100644 --- a/repo/amg2023/package.py +++ b/repo/amg2023/package.py @@ -34,7 +34,7 @@ class Amg2023(CMakePackage, CudaPackage, ROCmPackage): depends_on("adiak", when="+caliper") depends_on("hypre+caliper", when="+caliper") # depends_on("hypre@2.31.0:") - depends_on("hypre@comm_cali") + depends_on("hypre@3.1_comm_cali") depends_on("hypre+cuda", when="+cuda") requires("+cuda", when="^hypre+cuda") depends_on("hypre+rocm", when="+rocm") diff --git a/repo/genesis/application.py b/repo/genesis/application.py new file mode 100644 index 000000000..b00d41b3f --- /dev/null +++ b/repo/genesis/application.py @@ -0,0 +1,51 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from ramble.appkit import * + +import sys + +class Genesis(SpackApplication): + """GENESIS package contains two MD programs (atdyn and spdyn), trajectory + analysis programs, and other useful tools. GENESIS (GENeralized-Ensemble + SImulation System) has been developed mainly by Sugita group in RIKEN-CCS. + """ + name = "GENESIS" + + tags = ['molecular-dynamics','mpi', 'openmp', 'cuda'] + + executable('chdir', 'cd $(dirname {input})', use_mpi=False) + executable('genesis', 'spdyn {input}', use_mpi=True) + + input_file('benchmark-2020', + url='https://www.r-ccs.riken.jp/labs/cbrt/wp-content/uploads/2020/12/benchmark_mkl_ver4_nocrowding.tar.gz', + sha256='2ca8b2d4974dc0be0a42064392f1d5c603c64ffa9adc1f3bcf7c146a3bbf5bdb', + description='Benchmark set for GENESIS 2.0 beta / 1.6 on FUGAKU') + input_file('tests-2.1.1', + url='https://www.r-ccs.riken.jp/labs/cbrt/wp-content/uploads/2023/09/tests-2.1.1.tar.bz2', + sha256='f24d872beae5e38baa6a591906f78e3438186973c88e4879e4b04a0cca74f83e', + description='Regression tests are prepared for ATDYN, SPDYN, prst_setup (parallel I/O), and analysis tools to check if these programs work correctly.') + + workload('DHFR', executables=['chdir', 'genesis'], input='benchmark-2020') + workload('ApoA1', executables=['chdir', 'genesis'], input='benchmark-2020') + workload('UUN', executables=['chdir', 'genesis'], input='benchmark-2020') + workload('cryoEM', executables=['chdir', 'genesis'], input='tests-2.1.1') + + workload_variable('input', default='{benchmark-2020}/npt/genesis2.0beta/jac_amber/p{n_ranks}.inp', + description='jac_amber/ : DHFR (27,346 atoms), AMBER format, soluble system', + workloads=['DHFR']) + workload_variable('input', default='{benchmark-2020}/npt/genesis2.0beta/apoa1/p{n_ranks}.inp', + description='apoa1/ : apoa1 (92,224 atoms), CHARMM format, soluble system', + workloads=['ApoA1']) + workload_variable('input', default='{benchmark-2020}/npt/genesis2.0beta/uun/p{n_ranks}.inp', + description='uun/ : uun (216,726 atoms), CHARMM format, membrane+solvent system', + workloads=['UUN']) + workload_variable('input', default='{tests-2.1.1}/regression_test/test_spdyn/cryoEM/All_atom/inp', + description='cryoEM/All_atom/ : cryoEM (? atoms), CHARMM format', + workloads=['cryoEM']) + + figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'^\s+dynamics\s+=\s+(?P[-+]?([0-9]*[.])?[0-9]+([eED][-+]?[0-9]+)?)', group_name='fom', units='') + + success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') diff --git a/repo/genesis/package.py b/repo/genesis/package.py new file mode 100644 index 000000000..91258679d --- /dev/null +++ b/repo/genesis/package.py @@ -0,0 +1,102 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * + + +class Genesis(AutotoolsPackage): + """GENESIS package contains two MD programs (atdyn and spdyn), trajectory + analysis programs, and other useful tools. GENESIS (GENeralized-Ensemble + SImulation System) has been developed mainly by Sugita group in RIKEN-CCS. + """ + + tags = ['genesis','benchmark'] + + homepage = "https://www.r-ccs.riken.jp/labs/cbrt/" + git = "https://github.com/genesis-release-r-ccs/genesis" + + version("master", branch="master", submodules=False) + version( + "2.1.3", submodules=False, tag="v2.1.3", commit="835ef1538f9350cfa7e9489f340837d0908afbd2" + ) + version( + "2.1.1", submodules=False, tag="v2.1.1", commit="38a54fe1c749f4d87bff591e65c61b23a7396f9d" + ) + version( + "2.0.3", submodules=False, tag="v2.0.3", commit="6989e0b24470e374ea343b2b7b685aca87909571" + ) + + variant("mpi", default=True, description="Build with MPI.") + variant("openmp", default=True, description="Build with OpenMP enabled.") + variant("lapack", default=True, description="Build with LAPACK enabled.") + variant("gpu", default=False, description="Build with GPGPU enabled.") + variant("precision", description="Build with selected precision.", default="double", values=("double", "mixed", "single", "large_int"), multi=False) + variant("simd", description="Build with SIMD width.", default="auto", values=("auto", "MIC-AVX512", "CORE-AVX512", "CORE-AVX2"), multi=False) + variant("debug", description="Set Debug level", default="0", values=("0", "1", "2", "3", "4"), multi=False) + + depends_on("mpi", when="+mpi") + depends_on("lapack", when="+lapack") + depends_on("cuda", when="+gpu") + + def autoreconf(self, spec, prefix): + bash = which("bash") + bash("./bootstrap") + + @run_before("configure") + def fix_programming_error(self): + spec = self.spec + if self.version < Version("2.1.3"): + filter_file(r"atomcls1\(1:3\)", "atomcls1(1:6)", join_path(self.stage.source_path, "src/analysis/sp_analysis/hbond_analysis/hbond_analyze.fpp")) + filter_file(r"atomcls2\(1:3\)", "atomcls2(1:6)", join_path(self.stage.source_path, "src/analysis/sp_analysis/hbond_analysis/hbond_analyze.fpp")) + + def configure_args(self): + spec = self.spec + config_args = [f"--enable-{spec.variants['precision'].value}", + f"--with-simd={spec.variants['simd'].value}"] + + if int(spec.variants['debug'].value) > 0: + config_args.extend(f"--enable-debug={spec.variants['debug'].value}") + + config_args.extend(self.enable_or_disable("mpi")) + config_args.extend(self.enable_or_disable("openmp")) + config_args.extend(self.with_or_without("lapack")) + config_args.extend(self.enable_or_disable("gpu")) + + if "+mpi" in spec: + env["CC"] = spec["mpi"].mpicc + env["CXX"] = spec["mpi"].mpicxx + env["FC"] = spec["mpi"].mpifc + env["F77"] = spec["mpi"].mpif77 + + if "+openmp" in spec and spec.satisfies("%clang"): + env["OPT_OPENMP"] = "-fopenmp" + + if spec.satisfies("%clang"): + opt_flags = "-Ofast -ffast-math" + env["CFLAGS"] = f"{opt_flags}" + env["CXXFLAGS"] = f"{opt_flags}" + env["FCFLAGS"] = f"{opt_flags} -Mbackslash" + env["F77FLAGS"] = f"{opt_flags} -Mbackslash" + # cpp workaround; other systems and OS likely need different pre-processor fix + if spec.target == "a64fx": + env["FPP"] = "/opt/FJSVxtclanga/tcsds-1.2.38/bin/../lib/fpp" + env["PPFLAGS"] = "-traditional-cpp -traditional" + elif spec.satisfies("%fj"): + opt_flags = "-Kfast" + env["CFLAGS"] = f"{opt_flags}" + env["CXXFLAGS"] = f"{opt_flags}" + env["FCFLAGS"] = f"{opt_flags}" + env["F77FLAGS"] = f"{opt_flags}" + elif spec.satisfies("%gcc"): + opt_flags = "-O3 -ffast-math" + env["CFLAGS"] = f"{opt_flags}" + env["CXXFLAGS"] = f"{opt_flags}" + env["FCFLAGS"] = f"{opt_flags} -ffree-line-length-none" + env["F77FLAGS"] = f"{opt_flags} -ffree-line-length-none" + + if "+gpu" in spec: + config_args.extend(self.with_or_without("cuda", activation_value=spec["cuda"].prefix)) + + return config_args diff --git a/repo/hpl/package.py b/repo/hpl/package.py index 549fbcccc..d6ea7c3c6 100644 --- a/repo/hpl/package.py +++ b/repo/hpl/package.py @@ -116,25 +116,24 @@ def configure(self, spec, prefix): def configure_args(self): filter_file(r"^libs10=.*", "libs10=%s" % self.spec["blas"].libs.ld_flags, "configure") + cc, cflags, ldflags = self.spec["mpi"].mpicc, ["-O3"], [] if "+openmp" in self.spec: - config = ["CFLAGS=-O3 " + self.compiler.openmp_flag] - else: - config = ["CFLAGS=-O3"] + cflags.append(self.compiler.openmp_flag) if ( self.spec.satisfies("^intel-mkl") or self.spec.satisfies("^intel-oneapi-mkl") or self.spec.satisfies("^intel-parallel-studio+mkl") ): - config.append("LDFLAGS={0}".format(self.spec["blas"].libs.ld_flags)) + ldflags.append(self.spec["blas"].libs.ld_flags) if "%aocc" in self.spec: - amd_ldflags = " " if "%aocc@3:" in self.spec: - amd_ldflags += "-lamdlibm -lm " + ldflags.extend(["-lamdlibm", "-lm"]) if "%aocc@4:" in self.spec: - amd_ldflags += "-lamdalloc " - config.append("LDFLAGS=" + amd_ldflags) + ldflags.append("-lamdalloc") + + config = [f"CC={cc}", f"CFLAGS={' '.join(cflags)}", f"LDFLAGS={' '.join(ldflags)}"] if "+caliper" in self.spec: config.append("--with-adiak={0}".format(self.spec["adiak"].prefix)) diff --git a/repo/hypre/package.py b/repo/hypre/package.py index 31613bcd3..7252caa2d 100644 --- a/repo/hypre/package.py +++ b/repo/hypre/package.py @@ -18,6 +18,8 @@ class Hypre(BuiltinHypre): "gcc": "gnu", } + version("3.1_comm_cali", branch="comm_cali", submodules=False, git="https://github.com/gracenansamba/hypre.git") + def configure_args(self): configure_args = super().configure_args() diff --git a/repo/qws/application.py b/repo/qws/application.py index cef80fe55..eaec856df 100644 --- a/repo/qws/application.py +++ b/repo/qws/application.py @@ -60,7 +60,6 @@ class Qws(ExecutableApplication): description='Maximum iterations of inner DD solver. (Default: 50)', workloads=['qws']) - #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'etime for so.ler =\s+(?P[-+]?([0-9]*[.])?[0-9]+([eED][-+]?[0-9]+)?)', group_name='fom', units='') - #TODO: Fix the FOM success_criteria(...) - #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') + success_criteria('pass', mode='string', match=r'print timing', file='{experiment_run_dir}/{experiment_name}.out')