Merge pull request #47 from lanl/brryan/databox_template_arg

Add template args to `DataBox`

.github/workflows/tests.yml

@@ -13,13 +13,15 @@ jobs:
 
       steps:
         - name: Checkout code
-          uses: actions/checkout@v2
+          uses: actions/checkout@v3
           with:
-            submodules: true
+            submodules: recursive
         - name: Set system to non-interactive mode
           run: export DEBIAN_FRONTEND=noninteractive
         - name: install dependencies
-          run: sudo apt-get install -y --force-yes -qq build-essential libhdf5-serial-dev
+          run: |
+            sudo apt-get update -y -qq
+            sudo apt-get install -y --force-yes -qq build-essential libhdf5-serial-dev
         - name: build and run tests
           run: |
             mkdir -p bin

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "utils/kokkos"]
 	path = utils/kokkos
 	url = https://github.com/kokkos/kokkos.git
+[submodule "utils/ports-of-call"]
+	path = utils/ports-of-call
+	url = [email protected]:lanl/ports-of-call.git

CMakeLists.txt

@@ -1,4 +1,4 @@
-# © 2021. Triad National Security, LLC. All rights reserved.  This
+# © 2021-2023. Triad National Security, LLC. All rights reserved.  This
 # program was produced under U.S. Government contract
 # 89233218CNA000001 for Los Alamos National Laboratory (LANL), which
 # is operated by Triad National Security, LLC for the U.S.  Department
@@ -53,11 +53,21 @@ target_include_directories(singularity-opac::flags
                            INTERFACE
                            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>)
 
-include (SetupDeps)
 include (SetupOptions)
+include (SetupDeps)
 include (SetupCompilers)
 include (SetupFlags)
 
+if (SINGULARITY_USE_HDF5)
+  message(STATUS "HDF5 enabled. Requesting it in spiner")
+  set(SPINER_USE_HDF ON CACHE BOOL "")
+else()
+  message(STATUS "HDF5 not enabled. Turning it off in spiner")
+  set(SPINER_USE_HDF OFF CACHE BOOL "")
+endif()
+add_subdirectory(utils/spiner)
+target_link_libraries(singularity-opac::flags INTERFACE spiner)
+
 include(GNUInstallDirs)
 include(CTest)
 

cmake/SetupCompilers.cmake

@@ -6,7 +6,7 @@
 enable_language(CXX)
 include(CMakeDetermineCXXCompiler)
 
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 

cmake/SetupDeps.cmake

@@ -33,7 +33,9 @@ endif()
 #=======================================
 # Can't play the target games above unless using
 # cmake 3.20+, so leave this one for now.
-find_package(HDF5 COMPONENTS C HL QUIET)
+# JMM: DO NOT SEARCH FOR HDF5 IF WE DON'T NEED IT
+if (SINGULARITY_USE_HDF5)
+find_package(HDF5 COMPONENTS C HL)
 
 # findpackage doesnt export an interface for HDF5,
 # so create one
@@ -64,9 +66,22 @@ if (HDF5_FOUND)
       else()
 	message("MPI::MPI_CXX provided by parent package")
       endif()
+    else()
+      message(status "HDF5 is not parallel")
     endif()
+else()
+  message(STATUS "No HDF5")
+endif()
 endif()
 
+# JMM: I'm putting this here, as it depends on what happens with HDF5,
+# and I want that to be set AFTER we set all our options, so we don't
+# set HDF5 unnecessarily.
+cmake_dependent_option(SINGULARITY_USE_MPI
+        "Link to MPI"
+        ON "${SINGULARITY_USE_MPI};${HDF5_IS_PARALLEL}"
+        OFF)
+
 #=======================================
 # Setup Catch2
 # - provides Catch2::Catch2

singularity-opac/base/indexers.hpp

@@ -46,6 +46,7 @@ struct Identity {
 
 class Linear {
  public:
+  using DataBox = Spiner::DataBox<Real>;
   Linear() = default;
 
   PORTABLE_INLINE_FUNCTION
@@ -58,7 +59,7 @@ class Linear {
     SetRange_(numin, numax, N);
   }
 
-  PORTABLE_INLINE_FUNCTION Linear(const Spiner::DataBox &data, Real numin,
+  PORTABLE_INLINE_FUNCTION Linear(const DataBox &data, Real numin,
                                   Real numax, int N)
       : data_(data) {
     SetRange_(numin, numax, N);
@@ -79,11 +80,12 @@ class Linear {
   void SetRange_(Real numin, Real numax, int N) {
     data_.setRange(0, numin, numax, N);
   }
-  Spiner::DataBox data_;
+  DataBox data_;
 };
 
 class LogLinear {
  public:
+  using DataBox = Spiner::DataBox<Real>;
   LogLinear() = default;
 
   PORTABLE_INLINE_FUNCTION
@@ -97,7 +99,7 @@ class LogLinear {
   }
 
   PORTABLE_INLINE_FUNCTION
-  LogLinear(const Spiner::DataBox &data, Real numin, Real numax, int N)
+  LogLinear(const DataBox &data, Real numin, Real numax, int N)
       : data_(data) {
     SetRange_(numin, numax, N);
   }
@@ -119,7 +121,7 @@ class LogLinear {
   void SetRange_(Real numin, Real numax, int N) {
     data_.setRange(0, BDMath::log10(numin), BDMath::log10(numax), N);
   }
-  Spiner::DataBox data_;
+  DataBox data_;
 };
 
 template <int N, typename Data_t>

singularity-opac/neutrinos/mean_opacity_neutrinos.hpp

@@ -210,8 +210,8 @@ class MeanOpacity {
   PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const {
     return std::pow(10., lx);
   }
-  Spiner::DataBox lkappaPlanck_;
-  Spiner::DataBox lkappaRosseland_;
+  Spiner::DataBox<Real> lkappaPlanck_;
+  Spiner::DataBox<Real> lkappaRosseland_;
   const char *filename_;
 };
 

singularity-opac/neutrinos/mean_s_opacity_neutrinos.hpp

@@ -41,6 +41,7 @@ namespace impl {
 template <typename ThermalDistribution, typename pc = PhysicalConstantsCGS>
 class MeanSOpacity {
  public:
+  using DataBox = Spiner::DataBox<Real>;
   MeanSOpacity() = default;
   template <typename SOpacity>
   MeanSOpacity(const SOpacity &s_opac, const Real lRhoMin, const Real lRhoMax,
@@ -211,8 +212,8 @@ class MeanSOpacity {
   PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const {
     return std::pow(10., lx);
   }
-  Spiner::DataBox lkappaPlanck_;
-  Spiner::DataBox lkappaRosseland_;
+  DataBox lkappaPlanck_;
+  DataBox lkappaRosseland_;
   const char *filename_;
 };
 

singularity-opac/neutrinos/spiner_opac_neutrinos.hpp

@@ -61,6 +61,7 @@ enum class DataStatus { Deallocated, OnDevice, OnHost };
 template <typename ThermalDistribution, typename pc = PhysicalConstantsCGS>
 class SpinerOpacity {
  public:
+  using DataBox = Spiner::DataBox<Real>;
   static constexpr Real EPS = 10.0 * std::numeric_limits<Real>::min();
   static constexpr Real Hz2MeV = pc::h / (1e6 * pc::eV);
   static constexpr Real MeV2Hz = 1 / Hz2MeV;
@@ -130,8 +131,8 @@ class SpinerOpacity {
 
   // DataBox constructor. Note that this constructor *shallow* copies
   // the databoxes, so they must be managed externally.
-  SpinerOpacity(const Spiner::DataBox &lalphanu, const Spiner::DataBox ljnu,
-                const Spiner::DataBox lJ, const Spiner::DataBox lJYe)
+  SpinerOpacity(const DataBox &lalphanu, const DataBox ljnu,
+                const DataBox lJ, const DataBox lJYe)
       : memoryStatus_(impl::DataStatus::OnHost), lalphanu_(lalphanu),
         ljnu_(ljnu), lJ_(lJ), lJYe_(lJYe) {}
 
@@ -391,7 +392,7 @@ class SpinerOpacity {
   impl::DataStatus memoryStatus_ = impl::DataStatus::Deallocated;
   // TODO(JMM): Integrating J and JYe seems wise.
   // We can add more things here as needed.
-  Spiner::DataBox lalphanu_, ljnu_, lJ_, lJYe_;
+  DataBox lalphanu_, ljnu_, lJ_, lJYe_;
   // TODO(JMM): Should we add table bounds? Given they're recorded in
   // each spiner table, I lean towards no, but could be convinced
   // otherwise if we need to do extrapolation, etc.

singularity-opac/photons/mean_opacity_photons.hpp

@@ -41,6 +41,7 @@ template <typename pc = PhysicalConstantsCGS>
 class MeanOpacity {
 
  public:
+  using DataBox = Spiner::DataBox<Real>;
   MeanOpacity() = default;
   template <typename Opacity>
   MeanOpacity(const Opacity &opac, const Real lRhoMin, const Real lRhoMax,
@@ -186,8 +187,8 @@ class MeanOpacity {
   PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const {
     return std::pow(10., lx);
   }
-  Spiner::DataBox lkappaPlanck_;
-  Spiner::DataBox lkappaRosseland_;
+  DataBox lkappaPlanck_;
+  DataBox lkappaRosseland_;
   const char *filename_;
 };
 

singularity-opac/photons/mean_s_opacity_photons.hpp

@@ -42,6 +42,8 @@ template <typename ThermalDistribution, typename pc = PhysicalConstantsCGS>
 class MeanSOpacity {
 
  public:
+  using DataBox = Spiner::DataBox<Real>;
+
   MeanSOpacity() = default;
   template <typename SOpacity>
   MeanSOpacity(const SOpacity &s_opac, const Real lRhoMin, const Real lRhoMax,
@@ -192,8 +194,8 @@ class MeanSOpacity {
   PORTABLE_INLINE_FUNCTION Real fromLog_(const Real lx) const {
     return std::pow(10., lx);
   }
-  Spiner::DataBox lkappaPlanck_;
-  Spiner::DataBox lkappaRosseland_;
+  DataBox lkappaPlanck_;
+  DataBox lkappaRosseland_;
   const char *filename_;
 };
 

test/CMakeLists.txt

@@ -29,6 +29,8 @@ message(STATUS "Configuring unit tests")
 add_library(catch2_define)
 target_sources(catch2_define PUBLIC catch2_define.cpp)
 target_link_libraries(catch2_define PUBLIC
+                      ${PROJECT_NAME}
+                      singularity-opac::flags
                       Catch2::Catch2)
 
 add_executable(${PROJECT_NAME}_unit_tests)
@@ -50,7 +52,8 @@ PRIVATE
 target_link_libraries(${PROJECT_NAME}_unit_tests
 PRIVATE
   catch2_define
-  ${PROJECT_NAME})
+  ${PROJECT_NAME}
+  singularity-opac::flags)
 
 # Ensure code works with C++11 and earlier
 # TODO(MM): Remove this later when it's not needed.

test/catch2_define.cpp

@@ -13,7 +13,7 @@
 // publicly, and to permit others to do so.
 // ======================================================================
 
-#include "../utils/ports-of-call/portability.hpp"
+#include <ports-of-call/portability.hpp>
 
 #define CATCH_CONFIG_RUNNER
 #include <catch2/catch.hpp>

test/test_brt_opacities.cpp

@@ -32,6 +32,7 @@
 using namespace singularity;
 
 using pc = PhysicalConstantsCGS;
+using DataBox = Spiner::DataBox<Real>;
 
 #ifdef PORTABILITY_STRATEGY_KOKKOS
 using atomic_view = Kokkos::MemoryTraits<Kokkos::Atomic>;
@@ -169,7 +170,7 @@ TEST_CASE("BRT neutrino opacities", "[BRTNeutrinos]") {
                                                     nu_coeffs, nu_min, nu_max);
             opac.EmissivityPerNu(rho, temp, Ye, type, nu_bins, J_cheb, nbins);
             Real Jtrue = opac.EmissivityPerNu(rho, temp, Ye, type, nu);
-            J_cheb.SetInterpCoeffs(Spiner::DataBox(vm9, 9, 9));
+            J_cheb.SetInterpCoeffs(DataBox(vm9, 9, 9));
             if (std::isnan(J_cheb(nu)) ||
                 ((std::abs(Jtrue) >= 1e-14 || J_cheb(nu) >= 1e-14) &&
                  FractionalDifference(J_cheb(nu), Jtrue) > EPS_TEST)) {

test/test_chebyshev.cpp

@@ -23,6 +23,8 @@
 #include <singularity-opac/chebyshev/chebyshev.hpp>
 using namespace singularity::chebyshev;
 
+using DataBox = Spiner::DataBox<Real>;
+
 #ifdef PORTABILITY_STRATEGY_KOKKOS
 using atomic_view = Kokkos::MemoryTraits<Kokkos::Atomic>;
 #endif
@@ -62,7 +64,7 @@ TEST_CASE("Chebyshev Polynomials", "[Chebyshev]") {
         Real *ycoeffs = (Real *)PORTABLE_MALLOC(sizeof(Real) * N);
         portableFor(
             "Compute Cheb polynomial", 0, 1, PORTABLE_LAMBDA(const int &i) {
-              MatMultiply(Spiner::DataBox(vm9, 9, 9), y, ycoeffs, N);
+              MatMultiply(DataBox(vm9, 9, 9), y, ycoeffs, N);
             });
         AND_THEN("The chebyshev polynomials fit") {
           int n_wrong_h = 0;

test/test_epbremsstrahlung_opacities.cpp

@@ -31,6 +31,7 @@
 using namespace singularity;
 
 using pc = PhysicalConstantsCGS;
+using DataBox = Spiner::DataBox<Real>;
 
 #ifdef PORTABILITY_STRATEGY_KOKKOS
 using atomic_view = Kokkos::MemoryTraits<Kokkos::Atomic>;
@@ -132,7 +133,7 @@ TEST_CASE("E-P bremsstrahlung photon opacities", "[EPBremsstrahlungPhotons]") {
 
       Real *nu_bins = (Real *)PORTABLE_MALLOC(nbins * sizeof(Real));
       Real *temp_bins = (Real *)PORTABLE_MALLOC(ntemps * sizeof(Real));
-      Spiner::DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps,
+      DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps,
                                   nbins);
 
       portableFor(

test/test_gray_opacities.cpp

@@ -32,6 +32,7 @@
 using namespace singularity;
 
 using pc = PhysicalConstantsCGS;
+using DataBox = Spiner::DataBox<Real>;
 
 #ifdef PORTABILITY_STRATEGY_KOKKOS
 using atomic_view = Kokkos::MemoryTraits<Kokkos::Atomic>;
@@ -167,7 +168,7 @@ TEST_CASE("Gray neutrino opacities", "[GrayNeutrinos]") {
                                                     nu_coeffs, nu_min, nu_max);
             opac.EmissivityPerNu(rho, temp, Ye, type, nu_bins, J_cheb, nbins);
             Real Jtrue = opac.EmissivityPerNu(rho, temp, Ye, type, nu);
-            J_cheb.SetInterpCoeffs(Spiner::DataBox(vm9, 9, 9));
+            J_cheb.SetInterpCoeffs(DataBox(vm9, 9, 9));
             if (std::isnan(J_cheb(nu)) ||
                 ((std::abs(Jtrue) >= 1e-14 || J_cheb(nu) >= 1e-14) &&
                  FractionalDifference(J_cheb(nu), Jtrue) > EPS_TEST)) {
@@ -282,7 +283,7 @@ TEST_CASE("Gray photon opacities", "[GrayPhotons]") {
 
       Real *nu_bins = (Real *)PORTABLE_MALLOC(nbins * sizeof(Real));
       Real *temp_bins = (Real *)PORTABLE_MALLOC(ntemps * sizeof(Real));
-      Spiner::DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps,
+      DataBox loglin_bins(Spiner::AllocationTarget::Device, ntemps,
                                   nbins);
 
       portableFor(

test/test_spiner_opac_neutrinos.cpp

@@ -73,7 +73,7 @@ TEST_CASE("Spiner opacities, filled with gray data",
   const std::string grayname = "gray.sp5";
 
   WHEN("We initialize a gray neutrino opacity and tabulate it") {
-    using Grid_t = Spiner::RegularGrid1D;
+    using Grid_t = Spiner::RegularGrid1D<Real>;
 
     // Offset the frequencies to hopefully get over the point
     // where the power laws transition