From 44953194f87b4b75c1d514f0ac2b70c0c3f886ba Mon Sep 17 00:00:00 2001 From: "Samuel K. Gutierrez" Date: Tue, 9 Apr 2024 16:49:13 -0600 Subject: [PATCH] Lift [0, npieces) color space restriction in split. Allow for the use of an arbitrary positive color space in splits. This lifts the requirement that colors be in [0, npieces). Also include work on QV_SCOPE_SPLIT_UNDEFINED, but this is incomplete. Signed-off-by: Samuel K. Gutierrez --- include/quo-vadis.h | 8 ++++- src/fortran/quo-vadisf.f90 | 6 ++-- src/qvi-scope.cc | 67 +++++++++++++++++++++++++++++--------- 3 files changed, 63 insertions(+), 18 deletions(-) diff --git a/include/quo-vadis.h b/include/quo-vadis.h index f50456fe..3ab553b5 100644 --- a/include/quo-vadis.h +++ b/include/quo-vadis.h @@ -121,11 +121,17 @@ typedef enum qv_bind_string_format_e { * accomplished. */ enum { + /** + * Constant used to indicate undefined or unknown integer value. This means + * that the caller will not be considered in the split, and therefore + * receive an empty scope. + */ + QV_SCOPE_SPLIT_UNDEFINED = -1, /** * Split the provided group by attempting to preserve tasks' current * affinities (at time of the split call) as much as possible. */ - QV_SCOPE_SPLIT_AFFINITY_PRESERVING = -1 + QV_SCOPE_SPLIT_AFFINITY_PRESERVING = -2 }; /** diff --git a/src/fortran/quo-vadisf.f90 b/src/fortran/quo-vadisf.f90 index 48a42baf..c7733b9a 100644 --- a/src/fortran/quo-vadisf.f90 +++ b/src/fortran/quo-vadisf.f90 @@ -1,5 +1,5 @@ ! -! Copyright (c) 2013-2023 Triad National Security, LLC +! Copyright (c) 2013-2024 Triad National Security, LLC ! All rights reserved. ! ! This file is part of the quo-vadis project. See the LICENSE file at the @@ -111,9 +111,11 @@ module quo_vadisf !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Automatic grouping options for qv_scope_split(). !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + integer(c_int) QV_SCOPE_SPLIT_UNDEFINED integer(c_int) QV_SCOPE_SPLIT_AFFINITY_PRESERVING - parameter (QV_SCOPE_SPLIT_AFFINITY_PRESERVING = -1) + parameter (QV_SCOPE_SPLIT_UNDEFINED = -1) + parameter (QV_SCOPE_SPLIT_AFFINITY_PRESERVING = -2) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Device ID types diff --git a/src/qvi-scope.cc b/src/qvi-scope.cc index 405b5494..56fdee7f 100644 --- a/src/qvi-scope.cc +++ b/src/qvi-scope.cc @@ -410,11 +410,8 @@ qvi_global_split_get_cpuset( // This shouldn't happen. assert(gsplit.hwpools.size() != 0); - int rc = qvi_hwloc_bitmap_calloc(result); - if (rc != QV_SUCCESS) return rc; - - return qvi_hwloc_bitmap_copy( - qvi_hwpool_cpuset_get(gsplit.hwpools[0]), *result + return qvi_hwloc_bitmap_dup( + qvi_hwpool_cpuset_get(gsplit.hwpools[0]), result ); } @@ -624,10 +621,7 @@ global_split_devices_user_defined( // Determine mapping of colors to task IDs. The array index i of colors is // the color requested by task i. Also determine the number of distinct // colors provided in the colors array. - std::set color_set; - for (const auto &color : gsplit.colors) { - color_set.insert(color); - } + std::set color_set(gsplit.colors.begin(), gsplit.colors.end()); // Adjust the color set so that the distinct colors provided // fall within the range of the number of splits requested. std::set color_setp; @@ -931,6 +925,33 @@ global_split_affinity_preserving( return qvi_global_split_devices_affinity_preserving(gsplit); } +/** + * Takes a vector of colors and clamps their values to [0, max) in place. If + * this function finds more colors than slots available from [0, max), then it + * returns an error code. + */ +static int +clamp_colors( + std::vector &values, + uint_t max +) { + // Recall: sets are ordered. + std::set valset(values.begin(), values.end()); + // Too many colors for the given space. + if (valset.size() > max) return QV_ERR_INVLD_ARG; + // Maps the input vector colors to their clamped values. + std::map colors2clamped; + // color': the clamped color. + int colorp = 0; + for (auto val : valset) { + colors2clamped.insert({val, colorp++}); + } + for (uint_t i = 0; i < values.size(); ++i) { + values[i] = colors2clamped[values[i]]; + } + return QV_SUCCESS; +} + /** * Splits global scope data. */ @@ -942,14 +963,30 @@ qvi_global_split( bool auto_split = false; // Make sure that the supplied colors are consistent and determine the type // of coloring we are using. Positive values denote an explicit coloring - // provided by the caller. Negative values are reserved for automatic - // coloring algorithms and should be constants defined in quo-vadis.h. + // provided by the caller. Negative values are reserved for internal + // use and shall be constants defined in quo-vadis.h. Note we don't sort the + // gsplit's colors directly because they are ordered by task ID. std::vector tcolors(gsplit.colors); std::sort(tcolors.begin(), tcolors.end()); - // If all the values are negative and equal, then auto split. If not, then - // we were called with an invalid request. Else the values are all positive - // and we are going to split based on the input from the caller. - if (tcolors.front() < 0) { + // We have a few possibilities here: + // * The values are all positive: user-defined split, but we have to clamp + // their values to a usable range for internal consumption. + // * The values are negative and equal: + // - All the same, valid auto split constant: auto split + // - All the same, undefined constant: user-defined split, but this is a + // strange case since all participants will get empty sets. + // * A mix if positive and negative values: + // - A strict subset is QV_SCOPE_SPLIT_UNDEFINED: user-defined split + // - A strict subset is not QV_SCOPE_SPLIT_UNDEFINED: return error code. + + // All colors are positive. + if (tcolors.front() >= 0) { + rc = clamp_colors(gsplit.colors, gsplit.size); + if (rc != QV_SUCCESS) return rc; + } + // Some values are negative. + else { + // TODO(skg) Implement the rest. if (tcolors.front() != tcolors.back()) { return QV_ERR_INVLD_ARG; }