Skip to content

Commit

Permalink
Update NVSHMEM building block
Browse files Browse the repository at this point in the history 
Bump default version to 2.9.0-2
Use CMake to build
  • Loading branch information
@samcmill
samcmill committed Sep 1, 2023
1 parent 04bcb83 commit f7e3663
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 199 deletions.
23 changes: 6 additions & 17 deletions docs/building_blocks.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3246,14 +3246,14 @@ Stage1 += n.runtime()
nvshmem(self, **kwargs)
```
The `nvshmem` building block builds and installs the
[NVSHMEM](https://developer.nvidia.com/nvshmem) component.
[NVSHMEM](https://developer.nvidia.com/nvshmem) component. CMake
version 3.19 or later is required and must be installed separately.

__Parameters__


- __binary_tarball__: Path to NVSHMEM binary tarball relative to the
build context. The default value is empty. Either this parameter
or `package` must be specified.
- __cmake_opts__: List of additional options to pass to `cmake`. The
default value is an empty list.

- __cuda__: Flag to specify the path to the CUDA installation. The
default is `/usr/local/cuda`.
Expand All @@ -3265,42 +3265,31 @@ include NVSHMEM. The default is True.
- __gdrcopy__: Flag to specify the path to the GDRCOPY installation.
The default is empty.

- __hydra__: Boolean flag to specify whether the Hydra process launcher
should be installed. If True, adds `automake` to the list of OS
packages. The default is False.

- __ldconfig__: Boolean flag to specify whether the NVSHMEM library
directory should be added dynamic linker cache. If False, then
`LD_LIBRARY_PATH` is modified to include the NVSHMEM library
directory. The default value is False.

- __make_variables__: Dictionary of environment variables and values to
set when building NVSHMEM. The default is an empty dictionary.

- __mpi__: Flag to specify the path to the MPI installation. The
default is empty, i.e., do not build NVSHMEM with MPI support.

- __ospackages__: List of OS packages to install prior to building. The
default values are `make` and `wget`.

- __package__: Path to the NVSHMEM source package relative to the build
context. The default value is empty. Either this parameter or
`binary_tarball` must be specified.

- __prefix__: The top level install location. The default value is
`/usr/local/nvshmem`.

- __shmem__: Flag to specify the path to the SHMEM installation. The
default is empty, i.e., do not build NVSHMEM with SHMEM support.

- __version__: The version of NVSHMEM source to download. The default
value is `2.2.1`.
value is `2.9.0-2`.

__Examples__


```python
nvshmem(mpi='/usr/local/openmpi', version='2.1.2')
nvshmem(mpi='/usr/local/nvshmem', version='2.9.0-2')
```


Expand Down
129 changes: 30 additions & 99 deletions hpccm/building_blocks/nvshmem.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import hpccm.templates.tar

from hpccm.building_blocks.base import bb_base
from hpccm.building_blocks.generic_build import generic_build
from hpccm.building_blocks.generic_cmake import generic_cmake
from hpccm.building_blocks.packages import packages
from hpccm.primitives.comment import comment
from hpccm.primitives.copy import copy
Expand All @@ -42,13 +42,13 @@ class nvshmem(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
hpccm.templates.ldconfig, hpccm.templates.rm,
hpccm.templates.tar):
"""The `nvshmem` building block builds and installs the
[NVSHMEM](https://developer.nvidia.com/nvshmem) component.
[NVSHMEM](https://developer.nvidia.com/nvshmem) component. CMake
version 3.19 or later is required and must be installed separately.
# Parameters
binary_tarball: Path to NVSHMEM binary tarball relative to the
build context. The default value is empty. Either this parameter
or `package` must be specified.
cmake_opts: List of additional options to pass to `cmake`. The
default value is an empty list.
cuda: Flag to specify the path to the CUDA installation. The
default is `/usr/local/cuda`.
Expand All @@ -60,41 +60,30 @@ class nvshmem(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
gdrcopy: Flag to specify the path to the GDRCOPY installation.
The default is empty.
hydra: Boolean flag to specify whether the Hydra process launcher
should be installed. If True, adds `automake` to the list of OS
packages. The default is False.
ldconfig: Boolean flag to specify whether the NVSHMEM library
directory should be added dynamic linker cache. If False, then
`LD_LIBRARY_PATH` is modified to include the NVSHMEM library
directory. The default value is False.
make_variables: Dictionary of environment variables and values to
set when building NVSHMEM. The default is an empty dictionary.
mpi: Flag to specify the path to the MPI installation. The
default is empty, i.e., do not build NVSHMEM with MPI support.
ospackages: List of OS packages to install prior to building. The
default values are `make` and `wget`.
package: Path to the NVSHMEM source package relative to the build
context. The default value is empty. Either this parameter or
`binary_tarball` must be specified.
prefix: The top level install location. The default value is
`/usr/local/nvshmem`.
shmem: Flag to specify the path to the SHMEM installation. The
default is empty, i.e., do not build NVSHMEM with SHMEM support.
version: The version of NVSHMEM source to download. The default
value is `2.2.1`.
value is `2.9.0-2`.
# Examples
```python
nvshmem(mpi='/usr/local/openmpi', version='2.1.2')
nvshmem(mpi='/usr/local/nvshmem', version='2.9.0-2')
```
"""
Expand All @@ -104,25 +93,19 @@ def __init__(self, **kwargs):

super(nvshmem, self).__init__(**kwargs)

self.__binary_tarball = kwargs.pop('binary_tarball', None)
self.__cmake_opts = kwargs.pop('cmake_opts', [])
self.__cuda = kwargs.pop('cuda', '/usr/local/cuda')
self.__gdrcopy = kwargs.pop('gdrcopy', None)
self.__hydra = kwargs.pop('hydra', False)
self.__make_variables = kwargs.pop('make_variables', {})
self.__mpi = kwargs.pop('mpi', None)
self.__ospackages = kwargs.pop('ospackages', ['make', 'wget'])
self.__prefix = kwargs.pop('prefix', '/usr/local/nvshmem')
self.__release = kwargs.pop('release', '0')
self.__shmem = kwargs.pop('shmem', None)
self.__src_directory = kwargs.pop('src_directory', None)
self.__version = kwargs.pop('version', '2.2.1')
self.__version = kwargs.pop('version', '2.9.0-2')
self.__wd = kwargs.get('wd', hpccm.config.g_wd) # working directory

# Set the download specific parameters
self.__download()
kwargs['url'] = self.url
if self.__src_directory:
kwargs['directory'] = self.__src_directory

# Setup the environment variables
self.environment_variables['CPATH'] = '{}:$CPATH'.format(
Expand All @@ -134,96 +117,48 @@ def __init__(self, **kwargs):
if not self.ldconfig:
self.environment_variables['LD_LIBRARY_PATH'] = '{}:$LD_LIBRARY_PATH'.format(posixpath.join(self.__prefix, 'lib'))

# Add packages
if self.__hydra:
self.__ospackages.append('automake')

if self.__version and not self.__binary_tarball and not self.package:
if self.__version and not self.package:
self += comment('NVSHMEM {}'.format(self.__version))
else:
self += comment('NVSHMEM')
self += packages(ospackages=self.__ospackages)

if self.__binary_tarball:
# Shorthand for the tarball file inside the container
tarball = posixpath.join(self.__wd,
os.path.basename(self.__binary_tarball))

self += copy(src=self.__binary_tarball, dest=tarball)
self += shell(commands=[
# Untar binary package
self.untar_step(
tarball=tarball,
# remove the leading directory, e.g., install in
# /usr/local/nvshmem not
# /usr/local/nvshmem/nvshmem_<version>_<arch>.
args=['--strip-components=1'],
directory=self.__prefix),
# Install Hydra process launcher
'{0}/scripts/install_hydra.sh {1} {0}'.format(
self.__prefix, self.__wd) if self.__hydra else None,
# Remove temporary files and cleanup
self.cleanup_step(items=[tarball])])
self += environment(variables=self.environment_variables)
# Set the build options
self.__configure()

else:
# Build from source

# Set the build options
self.__configure()

self.__bb = generic_build(
build = [
'{} make -j$(nproc) install'.format(
self.__build_environment),
'./scripts/install_hydra.sh {1} {0}'.format(
self.__prefix, self.__wd) if self.__hydra else None],
comment=False,
devel_environment=self.environment_variables,
prefix=self.__prefix,
runtime_environment=self.environment_variables,
**kwargs)
self += self.__bb
self.__bb = generic_cmake(
cmake_opts=self.__cmake_opts,
comment=False,
devel_environment=self.environment_variables,
prefix=self.__prefix,
runtime_environment=self.environment_variables,
**kwargs)
self += self.__bb

def __configure(self):
"""Setup build options based on user parameters"""

e = {}

e['NVSHMEM_PREFIX'] = self.__prefix

# Default to 0 unless MPI/SHMEM is requested
e['NVSHMEM_MPI_SUPPORT'] = 0

if self.__cuda:
e['CUDA_HOME'] = self.__cuda
self.__cmake_opts.append('-DCUDA_HOME={}'.format(self.__cuda))

if self.__gdrcopy:
e['GDRCOPY_HOME'] = self.__gdrcopy
self.__cmake_opts.append('-DGDRCOPY_HOME={}'.format(self.__gdrcopy))

if self.__mpi:
e['NVSHMEM_MPI_SUPPORT'] = 1
e['MPI_HOME'] = self.__mpi
self.__cmake_opts.append('-DNVSHMEM_MPI_SUPPORT= 1')
self.__cmake_opts.append('-DMPI_HOME={}'.format(self.__mpi))
#else:
# self.__cmake_opts.append('-DNVSHMEM_MPI_SUPPORT=0')

if self.__shmem:
e['NVSHMEM_SHMEM_SUPPORT'] = 1
e['SHMEM_HOME'] = self.__shmem

if self.__make_variables:
e.update(self.__make_variables)

l = []
if e:
for key, val in sorted(e.items()):
l.append('{0}={1}'.format(key, val))

self.__build_environment = ' '.join(l)
self.__cmake_opts.append('-DNVSHMEM_SHMEM_SUPPORT=1')
self.__cmake_opts.append('-DSHMEM_HOME={}'.format(self.__shmem))

def __download(self):
"""Set download source based on user parameters"""

if not self.package and not self.repository and not self.url:
self.url = 'https://developer.download.nvidia.com/compute/redist/nvshmem/{0}/source/nvshmem_src_{0}-{1}.txz'.format(self.__version, self.__release)
self.url = 'https://developer.download.nvidia.com/compute/redist/nvshmem/{0}/source/nvshmem_src_{1}.txz'.format(self.__version.split('-')[0], self.__version)

def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
Expand All @@ -238,9 +173,5 @@ def runtime(self, _from='0'):
```
"""
self.rt += comment('NVSHMEM')
if self.__binary_tarball:
self.rt += copy(_from=_from, src=self.__prefix, dest=self.__prefix)
self.rt += environment(variables=self.environment_step())
else:
self.rt += self.__bb.runtime(_from=_from)
self.rt += self.__bb.runtime(_from=_from)
return str(self.rt)
Loading

0 comments on commit f7e3663

Please sign in to comment.