diff --git a/kagen/generators/barabassi/barabassi.cpp b/kagen/generators/barabassi/barabassi.cpp index 065cd4c..4f8f33d 100644 --- a/kagen/generators/barabassi/barabassi.cpp +++ b/kagen/generators/barabassi/barabassi.cpp @@ -54,7 +54,7 @@ Barabassi::Barabassi(const PGeneratorConfig& config, const PEID rank, const PEID void Barabassi::FinalizeEdgeList(MPI_Comm comm) { if (!config_.directed) { - AddNonlocalReverseEdges(edges_, vertex_range_, comm); + AddNonlocalReverseEdges(graph_.edges, graph_.vertex_range, comm); } } diff --git a/kagen/generators/file/file_graph.cpp b/kagen/generators/file/file_graph.cpp index d7fa0b9..d861263 100644 --- a/kagen/generators/file/file_graph.cpp +++ b/kagen/generators/file/file_graph.cpp @@ -30,106 +30,38 @@ bool FileGraphGenerator::CheckDeficit(const ReaderDeficits deficit) const { void FileGraphGenerator::GenerateImpl(const GraphRepresentation representation) { auto reader = CreateGraphReader(config_.input_graph.format, config_.input_graph, rank_, size_); deficits_ = reader->Deficits(); - - const auto [n, m] = [&] { - if (CheckDeficit(ReaderDeficits::UNKNOWN_NUM_VERTICES) && CheckDeficit(ReaderDeficits::UNKNOWN_NUM_EDGES)) { - return std::pair{size_, size_}; - } - - auto [n, m] = reader->ReadSize(); - if (CheckDeficit(ReaderDeficits::UNKNOWN_NUM_VERTICES)) { - n = size_; - } - if (CheckDeficit(ReaderDeficits::UNKNOWN_NUM_EDGES)) { - m = size_; - } - return std::pair{n, m}; - }(); - - if (CheckDeficit(ReaderDeficits::REQUIRES_REDISTRIBUTION) - && config_.input_graph.distribution == GraphDistribution::BALANCE_EDGES) { - throw std::invalid_argument("not implemented"); - } - - // If we need postprocessing, always generate an edge list because postprocessing is not implemented for CSR - actual_representation_ = - CheckDeficit(ReaderDeficits::REQUIRES_REDISTRIBUTION) ? GraphRepresentation::EDGE_LIST : representation; - - SInt from = 0; - SInt to_node = std::numeric_limits::max(); - SInt to_edge = std::numeric_limits::max(); - - switch (config_.input_graph.distribution) { - case GraphDistribution::BALANCE_VERTICES: - std::tie(from, to_node) = ComputeRange(n, size_, rank_); - break; - - case GraphDistribution::BALANCE_EDGES: { - const auto edge_range = ComputeRange(m, size_, rank_); - from = reader->FindNodeByEdge(edge_range.first); - to_edge = edge_range.second; - break; - } - } - - auto graph = reader->Read(from, to_node, to_edge, actual_representation_); - - vertex_range_ = graph.vertex_range; - xadj_ = std::move(graph.xadj); - adjncy_ = std::move(graph.adjncy); - edges_ = std::move(graph.edges); - edge_weights_ = std::move(graph.edge_weights); - vertex_weights_ = std::move(graph.vertex_weights); - coordinates_ = std::move(graph.coordinates); + graph_ = ReadGraph(*reader, representation, config_.input_graph, rank_, size_); } void FileGraphGenerator::FinalizeEdgeList(MPI_Comm comm) { - if (actual_representation_ == GraphRepresentation::CSR) { + if (graph_.representation == GraphRepresentation::CSR) { FinalizeCSR(comm); if (Output()) { std::cout << "converting to edge list ... " << std::flush; } - edges_ = BuildEdgeListFromCSR(vertex_range_, xadj_, adjncy_); - { - [[maybe_unused]] auto free_xadj = std::move(xadj_); - [[maybe_unused]] auto free_adjncy = std::move(adjncy_); - } + graph_.edges = BuildEdgeListFromCSR(graph_.vertex_range, graph_.xadj, graph_.adjncy); + graph_.FreeCSR(); } - if (CheckDeficit(ReaderDeficits::REQUIRES_REDISTRIBUTION)) { - if (Output()) { - std::cout << "redistributing edges ... " << std::flush; - } - - const SInt n = [&] { - SInt n = 0; - if (CheckDeficit(ReaderDeficits::UNKNOWN_NUM_VERTICES)) { - n = FindNumberOfVerticesInEdgelist(edges_, comm); - } else { - n = vertex_range_.second; - MPI_Bcast(&n, 1, KAGEN_MPI_SINT, size_ - 1, comm); - } - return n; - }(); - - std::tie(vertex_range_.first, vertex_range_.second) = ComputeRange(n, size_, rank_); - RedistributeEdgesByVertexRange(edges_, vertex_range_, comm); - } + graph_ = FinalizeReadGraph(deficits_, std::move(graph_), Output(), comm); } void FileGraphGenerator::FinalizeCSR(MPI_Comm comm) { - if (actual_representation_ == GraphRepresentation::EDGE_LIST) { + if (graph_.representation == GraphRepresentation::EDGE_LIST) { FinalizeEdgeList(comm); if (Output()) { std::cout << "converting to CSR ... " << std::flush; } - std::tie(xadj_, adjncy_) = BuildCSRFromEdgeList(vertex_range_, edges_, edge_weights_); - { [[maybe_unused]] auto free_edges = std::move(edges_); } + std::tie(graph_.xadj, graph_.adjncy) = + BuildCSRFromEdgeList(graph_.vertex_range, graph_.edges, graph_.edge_weights); + graph_.FreeEdgelist(); } + + graph_ = FinalizeReadGraph(deficits_, std::move(graph_), Output(), comm); } bool FileGraphGenerator::Output() const { diff --git a/kagen/generators/file/file_graph.h b/kagen/generators/file/file_graph.h index a153448..d7103e1 100644 --- a/kagen/generators/file/file_graph.h +++ b/kagen/generators/file/file_graph.h @@ -34,7 +34,6 @@ class FileGraphGenerator : public Generator { PEID rank_; PEID size_; - int deficits_; - GraphRepresentation actual_representation_; + int deficits_; }; } // namespace kagen diff --git a/kagen/generators/generator.cpp b/kagen/generators/generator.cpp index 421af90..2e68dfd 100644 --- a/kagen/generators/generator.cpp +++ b/kagen/generators/generator.cpp @@ -15,9 +15,9 @@ Generator::~Generator() = default; void Generator::Generate(const GraphRepresentation representation) { Reset(); - representation_ = representation; + graph_.representation = representation; - switch (representation_) { + switch (graph_.representation) { case GraphRepresentation::EDGE_LIST: GenerateEdgeList(); break; @@ -29,7 +29,7 @@ void Generator::Generate(const GraphRepresentation representation) { } void Generator::Finalize(MPI_Comm comm) { - switch (representation_) { + switch (graph_.representation) { case GraphRepresentation::EDGE_LIST: FinalizeEdgeList(comm); break; @@ -49,21 +49,21 @@ void CSROnlyGenerator::GenerateEdgeList() { } void CSROnlyGenerator::FinalizeEdgeList(MPI_Comm comm) { - if (xadj_.empty()) { + if (graph_.xadj.empty()) { return; } // Otherwise, we have generated the graph in CSR representation, but // actually want edge list representation -> transform graph FinalizeCSR(comm); - edges_ = BuildEdgeListFromCSR(vertex_range_, xadj_, adjncy_); + graph_.edges = BuildEdgeListFromCSR(graph_.vertex_range, graph_.xadj, graph_.adjncy); { XadjArray tmp; - std::swap(xadj_, tmp); + std::swap(graph_.xadj, tmp); } { AdjncyArray tmp; - std::swap(adjncy_, tmp); + std::swap(graph_.adjncy, tmp); } } @@ -72,54 +72,40 @@ void EdgeListOnlyGenerator::GenerateCSR() { } void EdgeListOnlyGenerator::FinalizeCSR(MPI_Comm comm) { - if (!xadj_.empty()) { + if (!graph_.xadj.empty()) { return; } // Otherwise, we have generated the graph in edge list representation, but // actually want CSR format --> transform graph FinalizeEdgeList(comm); - std::tie(xadj_, adjncy_) = BuildCSRFromEdgeList(vertex_range_, edges_, edge_weights_); + std::tie(graph_.xadj, graph_.adjncy) = BuildCSRFromEdgeList(graph_.vertex_range, graph_.edges, graph_.edge_weights); { Edgelist tmp; - std::swap(edges_, tmp); + std::swap(graph_.edges, tmp); } } SInt Generator::GetNumberOfEdges() const { - return std::max(adjncy_.size(), edges_.size()); + return std::max(graph_.adjncy.size(), graph_.edges.size()); } Graph Generator::Take() { - return { - vertex_range_, - representation_, - std::move(edges_), - std::move(xadj_), - std::move(adjncy_), - std::move(vertex_weights_), - std::move(edge_weights_), - std::move(coordinates_)}; + return std::move(graph_); } void Generator::SetVertexRange(const VertexRange vertex_range) { - vertex_range_ = vertex_range; + graph_.vertex_range = vertex_range; } void Generator::FilterDuplicateEdges() { - std::sort(edges_.begin(), edges_.end()); - auto it = std::unique(edges_.begin(), edges_.end()); - edges_.erase(it, edges_.end()); + std::sort(graph_.edges.begin(), graph_.edges.end()); + auto it = std::unique(graph_.edges.begin(), graph_.edges.end()); + graph_.edges.erase(it, graph_.edges.end()); } void Generator::Reset() { - edges_.clear(); - xadj_.clear(); - adjncy_.clear(); - vertex_weights_.clear(); - edge_weights_.clear(); - coordinates_.first.clear(); - coordinates_.second.clear(); + graph_.Clear(); } GeneratorFactory::~GeneratorFactory() = default; diff --git a/kagen/generators/generator.h b/kagen/generators/generator.h index 98b7587..482a8fd 100644 --- a/kagen/generators/generator.h +++ b/kagen/generators/generator.h @@ -34,43 +34,35 @@ class Generator { void SetVertexRange(VertexRange vetrex_range); inline void PushCoordinate(const HPFloat x, const HPFloat y) { - coordinates_.first.emplace_back(x, y); + graph_.coordinates.first.emplace_back(x, y); } inline void PushCoordinate(const HPFloat x, const HPFloat y, const HPFloat z) { - coordinates_.second.emplace_back(x, y, z); + graph_.coordinates.second.emplace_back(x, y, z); } inline void PushVertexWeight(const SSInt weight) { - vertex_weights_.push_back(weight); + graph_.vertex_weights.push_back(weight); } inline void PushEdge(const SInt from, const SInt to) { - edges_.emplace_back(from, to); + graph_.edges.emplace_back(from, to); } inline void PushEdgeWeight(const SSInt weight) { - edge_weights_.push_back(weight); + graph_.edge_weights.push_back(weight); } inline void SetVertexRange(const SInt first_vertex, const SInt first_invalid_vertex) { - vertex_range_ = std::make_pair(first_vertex, first_invalid_vertex); + graph_.vertex_range = std::make_pair(first_vertex, first_invalid_vertex); } void FilterDuplicateEdges(); - VertexRange vertex_range_; - Edgelist edges_; - XadjArray xadj_; - AdjncyArray adjncy_; - Coordinates coordinates_; - VertexWeights vertex_weights_; - EdgeWeights edge_weights_; + Graph graph_; private: void Reset(); - - GraphRepresentation representation_; }; class ConfigurationError : public std::exception { diff --git a/kagen/generators/graph500_generator.cpp b/kagen/generators/graph500_generator.cpp index 1996897..fac859e 100644 --- a/kagen/generators/graph500_generator.cpp +++ b/kagen/generators/graph500_generator.cpp @@ -13,8 +13,8 @@ namespace kagen { Graph500Generator::Graph500Generator(const PGeneratorConfig& config) : config_(config) {} void Graph500Generator::FinalizeEdgeList(MPI_Comm comm) { - const SInt log_n = std::log2(config_.n); - const SInt n = 1ull << log_n; - vertex_range_ = RedistributeEdgesRoundRobin(local_edges_, edges_, n, comm); + const SInt log_n = std::log2(config_.n); + const SInt n = 1ull << log_n; + graph_.vertex_range = RedistributeEdgesRoundRobin(local_edges_, graph_.edges, n, comm); } } // namespace kagen diff --git a/kagen/generators/hyperbolic/hyperbolic.cpp b/kagen/generators/hyperbolic/hyperbolic.cpp index a6093a4..489f901 100644 --- a/kagen/generators/hyperbolic/hyperbolic.cpp +++ b/kagen/generators/hyperbolic/hyperbolic.cpp @@ -118,7 +118,7 @@ Hyperbolic::Hyperbolic(const PGeneratorConfig& config, const PEID rank, template void Hyperbolic::FinalizeEdgeList(MPI_Comm comm) { - AddNonlocalReverseEdges(edges_, vertex_range_, comm); + AddNonlocalReverseEdges(graph_.edges, graph_.vertex_range, comm); } template diff --git a/kagen/io.cpp b/kagen/io.cpp index 2fc7f5c..b903a3e 100644 --- a/kagen/io.cpp +++ b/kagen/io.cpp @@ -9,6 +9,7 @@ #include "kagen/io/hmetis.h" #include "kagen/io/metis.h" #include "kagen/io/parhip.h" +#include "kagen/tools/postprocessor.h" #include "kagen/tools/statistics.h" #include "kagen/tools/utils.h" @@ -116,6 +117,85 @@ CreateGraphReader(const FileFormat format, const InputGraphConfig& config, const throw IOError(error_msg.str()); } +Graph ReadGraph( + GraphReader& reader, const GraphRepresentation representation, const InputGraphConfig& config, const PEID rank, + const PEID size) { + const auto [n, m] = [&] { + if (reader.HasDeficit(ReaderDeficits::UNKNOWN_NUM_VERTICES) + && reader.HasDeficit(ReaderDeficits::UNKNOWN_NUM_EDGES)) { + return std::pair{size, size}; + } + + auto [n, m] = reader.ReadSize(); + if (reader.HasDeficit(ReaderDeficits::UNKNOWN_NUM_VERTICES)) { + n = size; + } + if (reader.HasDeficit(ReaderDeficits::UNKNOWN_NUM_EDGES)) { + m = size; + } + return std::pair{n, m}; + }(); + + if (reader.HasDeficit(ReaderDeficits::REQUIRES_REDISTRIBUTION) + && config.distribution == GraphDistribution::BALANCE_EDGES) { + throw std::invalid_argument("not implemented"); + } + + // If we need postprocessing, always generate an edge list because postprocessing is not implemented for CSR + GraphRepresentation actual_representation = + reader.HasDeficit(ReaderDeficits::REQUIRES_REDISTRIBUTION) ? GraphRepresentation::EDGE_LIST : representation; + + SInt from = 0; + SInt to_node = std::numeric_limits::max(); + SInt to_edge = std::numeric_limits::max(); + + switch (config.distribution) { + case GraphDistribution::BALANCE_VERTICES: + std::tie(from, to_node) = ComputeRange(n, size, rank); + break; + + case GraphDistribution::BALANCE_EDGES: { + const auto edge_range = ComputeRange(m, size, rank); + from = reader.FindNodeByEdge(edge_range.first); + to_edge = edge_range.second; + break; + } + } + + return reader.Read(from, to_node, to_edge, actual_representation); +} + +Graph FinalizeReadGraph(const int deficits, Graph graph, const bool output, MPI_Comm comm) { + if (deficits & ReaderDeficits::REQUIRES_REDISTRIBUTION) { + if (graph.representation == GraphRepresentation::CSR) { + throw std::invalid_argument("not implemented"); + } + + const PEID size = GetCommSize(comm); + const PEID rank = GetCommRank(comm); + + if (output) { + std::cout << "redistributing edges ... " << std::flush; + } + + const SInt n = [&] { + SInt n = 0; + if (deficits & ReaderDeficits::UNKNOWN_NUM_VERTICES) { + n = FindNumberOfVerticesInEdgelist(graph.edges, comm); + } else { + n = graph.vertex_range.second; + MPI_Bcast(&n, 1, KAGEN_MPI_SINT, size - 1, comm); + } + return n; + }(); + + std::tie(graph.vertex_range.first, graph.vertex_range.second) = ComputeRange(n, size, rank); + RedistributeEdgesByVertexRange(graph.edges, graph.vertex_range, comm); + } + + return graph; +} + void WriteGraph(GraphWriter& writer, const OutputGraphConfig& config, const bool output, MPI_Comm comm) { const PEID size = GetCommSize(comm); const PEID rank = GetCommRank(comm); diff --git a/kagen/io.h b/kagen/io.h index 5634111..aedf168 100644 --- a/kagen/io.h +++ b/kagen/io.h @@ -32,5 +32,10 @@ CreateGraphReader(const std::string& filename, const InputGraphConfig& config, P std::unique_ptr CreateGraphReader(const FileFormat format, const InputGraphConfig& config, PEID rank, PEID size); +Graph ReadGraph( + GraphReader& reader, GraphRepresentation representation, const InputGraphConfig& config, PEID rank, PEID size); + +Graph FinalizeReadGraph(int deficits, Graph graph, bool output, MPI_Comm comm); + void WriteGraph(GraphWriter& writer, const OutputGraphConfig& config, bool output, MPI_Comm comm); } // namespace kagen diff --git a/kagen/io/graph_format.h b/kagen/io/graph_format.h index f1feb99..1975a3e 100644 --- a/kagen/io/graph_format.h +++ b/kagen/io/graph_format.h @@ -74,6 +74,10 @@ class GraphReader { virtual int Deficits() const { return ReaderDeficits::NONE; } + + bool HasDeficit(const ReaderDeficits deficit) const { + return Deficits() & deficit; + } }; struct GraphInfo { diff --git a/kagen/kagen.cpp b/kagen/kagen.cpp index 820ef07..a362d4d 100644 --- a/kagen/kagen.cpp +++ b/kagen/kagen.cpp @@ -56,6 +56,25 @@ SInt Graph::NumberOfLocalEdges() const { __builtin_unreachable(); } +void Graph::Clear() { + edges.clear(); + xadj.clear(); + adjncy.clear(); + vertex_weights.clear(); + edge_weights.clear(); + coordinates.first.clear(); + coordinates.second.clear(); +} + +void Graph::FreeEdgelist() { + [[maybe_unused]] auto free_edgelist = std::move(edges); +} + +void Graph::FreeCSR() { + [[maybe_unused]] auto free_xadj = std::move(xadj); + [[maybe_unused]] auto free_adjncy = std::move(adjncy); +} + void Graph::SortEdgelist() { auto cmp_from = [](const auto& lhs, const auto& rhs) { return std::get<0>(lhs) < std::get<0>(rhs); diff --git a/kagen/kagen.h b/kagen/kagen.h index 4860798..cf92d9d 100644 --- a/kagen/kagen.h +++ b/kagen/kagen.h @@ -160,6 +160,11 @@ struct Graph { void SortEdgelist(); + void FreeEdgelist(); + void FreeCSR(); + + void Clear(); + template std::vector> TakeEdges() { return TakeVector>(edges);