diff --git a/platforms/cori-gcc-haswell.sh b/platforms/cori-gcc-haswell.sh deleted file mode 100755 index f62ef690f..000000000 --- a/platforms/cori-gcc-haswell.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. -# To install to my scratch directory, I might run this script with: -# -# $> ./platforms/cori-gcc_mkl.sh \ -# --prefix=$SCRATCH/software/toast-gcc -# - -OPTS="$@" - -export PYTHON=python3 -export CC=cc -export CXX=CC -export MPICC=cc -export MPICXX=CC -export CFLAGS="-O3 -g -fPIC -march=haswell" -export CXXFLAGS="-O3 -g -fPIC -march=haswell" -export OPENMP_CFLAGS="-fopenmp" -export OPENMP_CXXFLAGS="-fopenmp" - -./configure ${OPTS} \ - --without-mkl \ - --with-blas="-lopenblas -fopenmp -lpthread -lgfortran -lm" diff --git a/platforms/cori-gcc-knl.sh b/platforms/cori-gcc-knl.sh deleted file mode 100755 index e5a8f8abb..000000000 --- a/platforms/cori-gcc-knl.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. -# To install to my scratch directory, I might run this script with: -# -# $> ./platforms/cori-gcc_mkl.sh \ -# --prefix=$SCRATCH/software/toast-gcc -# - -OPTS="$@" - -export PYTHON=python3 -export CC=cc -export CXX=CC -export MPICC=cc -export MPICXX=CC -export CFLAGS="-O3 -g -march=knl -fPIC -pthread -malign-data=cacheline -finline-functions" -export CXXFLAGS="-O3 -g -march=knl -fPIC -pthread -malign-data=cacheline -finline-functions" -export OPENMP_CFLAGS="-fopenmp" -export OPENMP_CXXFLAGS="-fopenmp" - -./configure ${OPTS} \ - --without-mkl \ - --with-blas="-lopenblas -fopenmp -lpthread -lgfortran -lm" diff --git a/platforms/cori-intel-haswell.sh b/platforms/cori-intel-haswell.sh deleted file mode 100755 index e1152572f..000000000 --- a/platforms/cori-intel-haswell.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. -# To install to my scratch directory, I might run this script with: -# -# $> ./platforms/cori-haswell_intel_mkl.sh \ -# --prefix=$SCRATCH/software/toast-haswell -# - -OPTS="$@" - -export PYTHON=python3 -export CC=cc -export CXX=CC -export MPICC=cc -export MPICXX=CC -export CFLAGS="-O3 -g -fPIC -xcore-avx2 -pthread" -export CXXFLAGS="-O3 -g -fPIC -xcore-avx2 -pthread" -export OPENMP_CFLAGS="-qopenmp" -export OPENMP_CXXFLAGS="-qopenmp" - -./configure ${OPTS} \ - --with-math="-limf -lsvml" \ - --with-mkl="${INTEL_PATH}/linux/mkl/lib/intel64" diff --git a/platforms/cori-intel-knl.sh b/platforms/cori-intel-knl.sh deleted file mode 100755 index 295a9e382..000000000 --- a/platforms/cori-intel-knl.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. -# To install to my scratch directory, I might run this script with: -# -# $> ./platforms/cori-knl_intel_mkl.sh \ -# --prefix=$SCRATCH/software/toast-knl -# - -OPTS="$@" - -export PYTHON=python3 -export CC=cc -export CXX=CC -export MPICC=cc -export MPICXX=CC -export CFLAGS="-O3 -g -fPIC -xmic-avx512 -pthread" -export CXXFLAGS="-O3 -g -fPIC -xmic-avx512 -pthread" -export OPENMP_CFLAGS="-qopenmp" -export OPENMP_CXXFLAGS="-qopenmp" - -./configure ${OPTS} \ - --build x86_64-pc-linux-gnu --host x86_64-unknown-linux-gnu \ - --with-math="-limf -lsvml" \ - --with-mkl="${INTEL_PATH}/linux/mkl/lib/intel64" diff --git a/platforms/edison-gcc.sh b/platforms/edison-gcc.sh deleted file mode 100755 index 647eeea78..000000000 --- a/platforms/edison-gcc.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. -# To install to my scratch directory, I might run this script with: -# -# $> ./platforms/cori-gcc_mkl.sh \ -# --prefix=$SCRATCH/software/toast-gcc -# - -OPTS="$@" - -export PYTHON=python3 -export CC=cc -export CXX=CC -export MPICC=cc -export MPICXX=CC -export CFLAGS="-O3 -g -fPIC" -export CXXFLAGS="-O3 -g -fPIC" -export OPENMP_CFLAGS="-fopenmp" -export OPENMP_CXXFLAGS="-fopenmp" - -./configure ${OPTS} \ - --without-mkl \ - --with-blas="-lopenblas" diff --git a/platforms/edison-intel.sh b/platforms/edison-intel.sh deleted file mode 100755 index c9f440a8f..000000000 --- a/platforms/edison-intel.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. -# To install to my scratch directory, I might run this script with: -# -# $> ./platforms/cori-haswell_intel_mkl.sh \ -# --prefix=$SCRATCH/software/toast-haswell -# - -OPTS="$@" - -export PYTHON=python3 -export CC=cc -export CXX=CC -export MPICC=cc -export MPICXX=CC -export CFLAGS="-O3 -g -fPIC -xavx -pthread" -export CXXFLAGS="-O3 -g -fPIC -xavx -pthread" -export OPENMP_CFLAGS="-qopenmp" -export OPENMP_CXXFLAGS="-qopenmp" - -./configure ${OPTS} \ - --with-math="-limf -lsvml" \ - --with-mkl="${INTEL_PATH}/linux/mkl/lib/intel64" diff --git a/platforms/linux-gcc-dbg.sh b/platforms/linux-gcc-dbg.sh deleted file mode 100755 index a37344125..000000000 --- a/platforms/linux-gcc-dbg.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Pass extra configure options to this script, including -# things like --prefix, --with-elemental, etc. - -OPTS="$@" - -export PYTHON=python3 -export CC=mpicc -export CXX=mpicxx -export MPICC=mpicc -export MPICXX=mpicxx -export CFLAGS="-O0 -g -fPIC -pthread -std=c99" -export CXXFLAGS="-O0 -g -fPIC -pthread -std=c++11 -fext-numeric-literals" -export OPENMP_CFLAGS="-fopenmp" -export OPENMP_CXXFLAGS="-fopenmp" -export LDFLAGS="-lpthread" - -./configure ${OPTS} diff --git a/platforms/linux-gcc.sh b/platforms/linux-gcc.sh index 74f580144..b408aa54d 100755 --- a/platforms/linux-gcc.sh +++ b/platforms/linux-gcc.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Pass extra configure options to this script, including -# things like --prefix, --with-elemental, etc. +# Pass extra cmake options to this script, including +# things like -DCMAKE_INSTALL_PREFIX=/path/to/install, etc. opts="$@" diff --git a/platforms/linux-intel-dbg.sh b/platforms/linux-intel-dbg.sh deleted file mode 100755 index 1bc72bbf8..000000000 --- a/platforms/linux-intel-dbg.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Pass extra configure options to this script, including -# things like --prefix, --with-elemental, etc. - -OPTS="$@" - -export PYTHON=python3 -export CC=mpiicc -export CXX=mpiicpc -export MPICC=mpiicc -export MPICXX=mpiicpc -export CFLAGS="-O0 -g -fPIC -pthread" -export CXXFLAGS="-O0 -g -fPIC -pthread" -export OPENMP_CFLAGS="-qopenmp" -export OPENMP_CXXFLAGS="-qopenmp" - -./configure ${OPTS} \ - --with-math="-limf -lsvml" \ - --with-mkl="${MKLROOT}/lib/intel64" \ diff --git a/platforms/linux-intel.sh b/platforms/linux-intel.sh index 3e3712944..02328fdac 100755 --- a/platforms/linux-intel.sh +++ b/platforms/linux-intel.sh @@ -1,21 +1,20 @@ #!/bin/bash -# Pass extra configure options to this script, including -# things like --prefix, --with-elemental, etc. +# Pass extra cmake options to this script, including +# things like -DCMAKE_INSTALL_PREFIX=/path/to/install, etc. -OPTS="$@" +opts="$@" -export PYTHON=python3 -export CC=mpiicc -export CXX=mpiicpc -export MPICC=mpiicc -export MPICXX=mpiicpc -export CFLAGS="-O3 -g -fPIC -pthread" -export CXXFLAGS="-O3 -g -fPIC -pthread" -export OPENMP_CFLAGS="-qopenmp" -export OPENMP_CXXFLAGS="-qopenmp" - -./configure ${OPTS} \ - --with-math="-limf -lsvml" \ - --with-mkl="${MKLROOT}/lib/intel64" \ - --with-tbb="${TBBROOT}/lib/intel64/gcc4.7" +cmake \ + -DCMAKE_C_COMPILER="icc" \ + -DCMAKE_CXX_COMPILER="icpc" \ + -DMPI_C_COMPILER="mpiicc" \ + -DMPI_CXX_COMPILER="mpiicpc" \ + -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \ + -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \ + -DBLAS_LIBRARIES=${MKLROOT}/lib/intel64/libmkl_rt.so \ + -DLAPACK_LIBRARIES=${MKLROOT}/lib/intel64/libmkl_rt.so \ + -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ + ${opts} \ + .. diff --git a/platforms/nersc-intel.sh b/platforms/nersc-intel.sh deleted file mode 100755 index 1a1949b67..000000000 --- a/platforms/nersc-intel.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. -# To install to my scratch directory, I might run this script with: -# -# $> ./platforms/nersc-intel.sh \ -# --prefix=$SCRATCH/software/toast-intel -# -# This compiles "universal" or "fat" binaries with multiple optimized -# code paths. It builds one with AVX optimizations (highest that -# edison supports), and another for KNL. -# - -OPTS="$@" - -export PYTHON=python3 -export CC=cc -export CXX=CC -export MPICC=cc -export MPICXX=CC -export CFLAGS="-O3 -g -fPIC -axmic-avx512,avx -pthread" -export CXXFLAGS="-O3 -g -fPIC -axmic-avx512,avx -pthread" -export OPENMP_CFLAGS="-qopenmp" -export OPENMP_CXXFLAGS="-qopenmp" - -./configure ${OPTS} \ - --build x86_64-pc-linux-gnu --host x86_64-unknown-linux-gnu \ - --with-math="-limf" \ - --with-mkl="${INTEL_PATH}/linux/mkl/lib/intel64" diff --git a/platforms/osx-clang-condaforge.sh b/platforms/osx-clang-condaforge.sh deleted file mode 100755 index 26bd9747a..000000000 --- a/platforms/osx-clang-condaforge.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# This config assumes use of clang and Anaconda python, -# using mpich, mpi4py, cfitsio, and fftw from the -# conda-forge channel. -# -# 1. Install miniconda / anaconda -# -# 2. Edit ~/.condarc to look like this: -# -# channels: -# - conda-forge -# - defaults -# -# 3. Install numpy, mpich, mpi4py, cfitsio, fftw -# -# 4. In order to link against the relocatable shared -# libraries installed by conda packages (which have -# relative @rpaths), you must add the conda -# environment lib directory to $DYLD_LIBRARY_PATH. -# This is needed (for example) to use FFTW and CFITSIO -# libraries installed by conda packages. -# - -OPTS="$@" - -condabin=$(dirname $(which python)) - -export PYTHON=python3 -export CC=mpicc -export CXX=mpicxx -export MPICC=mpicc -export MPICXX=mpicxx -export CFLAGS="-O3 -g" -export CXXFLAGS="-O3 -g" - -./configure ${OPTS} \ ---without-mkl diff --git a/platforms/osx_macports.sh b/platforms/osx_macports.sh deleted file mode 100755 index 9b3caa0bc..000000000 --- a/platforms/osx_macports.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# This script assumes that you have installed fftw, python3, gcc and MPI -# wrappers from macports. You may also have installed Elemental -# separately as there is no macports package yet. -# -# Note that on OSX, Elemental uses relative paths to its dependencies. -# You can only run the toast_test executable after make_install: -# "make check" will fail. -# -# You can supply extra arguments to the configure script from the -# command line: -# -# $> ./platforms/osx_macports.sh --with_elemental=$HOME/software \ -# --prefix=$HOME/software -# - -OPTS="$@" - -export PYTHON=python3 -export CC=mpicc -export CXX=mpicxx -export MPICC=mpicc -export MPICXX=mpicxx -export CFLAGS="-O3 -std=c99" -export CXXFLAGS="-O3" -export LDFLAGS="-Wl,-twolevel_namespace" - -./configure ${OPTS} \ - --with-fftw=/opt/local \ - LIBS="-L/opt/local/lib" diff --git a/platforms/tacc-intel-knl.sh b/platforms/tacc-intel-knl.sh deleted file mode 100755 index 09ce6d3f7..000000000 --- a/platforms/tacc-intel-knl.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# This script assumes that the "toast-deps" module has been loaded. - -OPTS="$@" - -export PYTHON=python3 -export MPICC=mpiicc -export MPICXX=mpiicpc -export CFLAGS="-O3 -g -fPIC -xMIC-AVX512 -pthread" -export CXXFLAGS="-O3 -g -fPIC -xMIC-AVX512 -pthread" -export OPENMP_CFLAGS="-qopenmp" -export OPENMP_CXXFLAGS="-qopenmp" - -./configure ${OPTS} \ - --build x86_64-pc-linux-gnu --host x86_64-unknown-linux-gnu \ - --with-math="-limf" \ - --with-mkl="${TACC_INTEL_DIR}/mkl/lib/intel64" diff --git a/src/toast/_libtoast_mpi.cpp b/src/toast/_libtoast_mpi.cpp index 7b9792f8e..a25f9aa96 100644 --- a/src/toast/_libtoast_mpi.cpp +++ b/src/toast/_libtoast_mpi.cpp @@ -5,9 +5,28 @@ #include <_libtoast_mpi.hpp> -// This fixes compilation errors with OpenMPI 4 -// (see https://github.com/hpc4cmb/toast/issues/298) -struct ompi_communicator_t {}; +#include + + +MPI_Comm toast_mpi_extract_comm(py::object & pycomm) { + PyObject * pp = pycomm.ptr(); + + // For some reason this official way segfaults. So we access the + // structure directly. Although an internal detail, this has been + // consistent in mpi4py since version 1.2.2. + // MPI_Comm * comm = PyMPIComm_Get(pp); + MPI_Comm * comm = &(((struct PyMPICommObject *)pp)->ob_mpi); + + if (import_mpi4py() < 0) { + throw std::runtime_error("could not import mpi4py\n"); + } + if (comm == nullptr) { + throw std::runtime_error("mpi4py Comm pointer is null\n"); + } + MPI_Comm newcomm; + int ret = MPI_Comm_dup((*comm), &newcomm); + return newcomm; +} // Currently the only compiled code that uses MPI and needs to be bound to python is // the atmosphere simulation code. If the number of things increases, we should split @@ -66,23 +85,36 @@ void init_mpi_atm(py::module & m) { rmax (float): Maximum line of sight observing distance. )") - .def(py::init (), py::arg("azmin"), py::arg( - "azmax"), py::arg("elmin"), py::arg("elmax"), py::arg("tmin"), py::arg( - "tmax"), py::arg("lmin_center"), py::arg("lmin_sigma"), py::arg( - "lmax_center"), py::arg("lmax_sigma"), py::arg("w_center"), py::arg( - "w_sigma"), py::arg("wdir_center"), py::arg("wdir_sigma"), py::arg( - "z0_center"), py::arg("z0_sigma"), py::arg("T0_center"), py::arg( - "T0_sigma"), py::arg("zatm"), py::arg("zmax"), py::arg("xstep"), py::arg( - "ystep"), py::arg("zstep"), py::arg("nelem_sim_max"), py::arg( - "verbosity"), py::arg("comm"), py::arg("key1"), py::arg("key2"), - py::arg("counterval1"), py::arg("counterval2"), py::arg("cachedir"), py::arg( - "rmin"), py::arg("rmax") - ) + .def( + py::init( + [](double azmin, double azmax, double elmin, double elmax, + double tmin, double tmax, double lmin_center, double lmin_sigma, + double lmax_center, double lmax_sigma, double w_center, double w_sigma, + double wdir_center, double wdir_sigma, double z0_center, double z0_sigma, + double T0_center, double T0_sigma, double zatm, double zmax, + double xstep, double ystep, double zstep, long nelem_sim_max, + int verbosity, py::object & pycomm, uint64_t key1, uint64_t key2, + uint64_t counterval1, uint64_t counterval2, std::string cachedir, + double rmin, double rmax) { + MPI_Comm comm = toast_mpi_extract_comm(pycomm); + return new toast::mpi_atm_sim( + azmin, azmax, elmin, elmax, tmin, tmax, lmin_center, lmin_sigma, + lmax_center, lmax_sigma, w_center, w_sigma, wdir_center, + wdir_sigma, z0_center, z0_sigma, T0_center, T0_sigma, zatm, + zmax, xstep, ystep, zstep, nelem_sim_max, verbosity, comm, key1, + key2, counterval1, counterval2, cachedir, rmin, rmax); + }), py::arg("azmin"), py::arg("azmax"), py::arg("elmin"), py::arg("elmax"), + py::arg("tmin"), py::arg("tmax"), py::arg("lmin_center"), + py::arg("lmin_sigma"), py::arg("lmax_center"), py::arg("lmax_sigma"), + py::arg("w_center"), py::arg("w_sigma"), py::arg("wdir_center"), + py::arg("wdir_sigma"), py::arg("z0_center"), py::arg("z0_sigma"), + py::arg("T0_center"), py::arg("T0_sigma"), py::arg("zatm"), + py::arg("zmax"), py::arg("xstep"), py::arg("ystep"), py::arg("zstep"), + py::arg("nelem_sim_max"), py::arg("verbosity"), py::arg("comm"), + py::arg("key1"), py::arg("key2"), py::arg("counterval1"), + py::arg("counterval2"), py::arg("cachedir"), py::arg("rmin"), + py::arg("rmax") + ) .def("simulate", &toast::mpi_atm_sim::simulate, py::arg( "use_cache"), R"( Perform the simulation. diff --git a/src/toast/mpi.py b/src/toast/mpi.py index 903d106a7..761a3b793 100644 --- a/src/toast/mpi.py +++ b/src/toast/mpi.py @@ -4,7 +4,6 @@ import sys import itertools -import ctypes as ct import numpy as np @@ -26,16 +25,6 @@ "not found at run time. Is mpi4py currently in " "your python search path?" ) - try: - if MPI._sizeof(MPI.Comm) == ct.sizeof(ct.c_int): - MPI_Comm = ct.c_int - else: - MPI_Comm = ct.c_void_p - except Exception as e: - raise Exception( - "Failed to set the portable MPI communicator datatype. MPI4py is " - "probably too old. You need to have at least version 2.0. ({})".format(e) - ) # We set the numba threading here, **after** importing MPI. The reasons are: # @@ -52,15 +41,6 @@ set_numba_threading() _have_set_numba_threading = True - -def comm_py2c(comm): - if MPI_Comm is None: - return None - else: - comm_ptr = MPI._addressof(comm) - return MPI_Comm.from_address(comm_ptr) - - def get_world(): """Retrieve the default world communicator and its properties. diff --git a/src/toast/todmap/sim_det_atm.py b/src/toast/todmap/sim_det_atm.py index 76bdcbdfd..f2d497b41 100644 --- a/src/toast/todmap/sim_det_atm.py +++ b/src/toast/todmap/sim_det_atm.py @@ -28,7 +28,7 @@ if available_mpi: from .atm import AtmSimMPI -from toast.mpi import MPI, comm_py2c +from toast.mpi import MPI import toast.qarray as qa @@ -760,7 +760,7 @@ def _simulate_atmosphere( self._zstep, self._nelem_sim_max, self._verbosity, - comm_py2c(comm).value, + comm, key1, key2, counter1,