Merge branch 'oneapi-src:SYCLomatic' into cuErr_AnT

.git-blame-ignore-revs

@@ -64,3 +64,9 @@ f6d557ee34b6bbdb1dc32f29e34b4a4a8ad35e81
 
 # [libc++][NFC] clang-format <shared_mutex>
 2d7eb9c9ea1a146412a83603d5c0c6339a5d8284
+
+# [libc++] Rename _LIBCPP_INLINE_VISIBILITY to _LIBCPP_HIDE_FROM_ABI
+4c198542226223f6a5c5511a1f89b37d15ee10b9
+
+# [libc++] Replace uses of _VSTD:: by std:: (#74331)
+77a00c0d546cd4aa8311b5b9031ae9ea8cdb050c

.mailmap

@@ -24,6 +24,7 @@
 
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
+<[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
@@ -32,6 +33,8 @@
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
+Jianjian GUAN <[email protected]>
+Jianjian GUAN <[email protected]> <[email protected]>
 Jon Roelofs <[email protected]> <[email protected]>
 Jon Roelofs <[email protected]> <[email protected]>
 Jonathan Thackray <[email protected]> <[email protected]>
@@ -42,3 +45,6 @@ Med Ismail Bennani <[email protected]> <[email protected]>
 Ramkumar Ramachandra <[email protected]> <[email protected]>
 Saleem Abdulrasool <[email protected]>
 Tommy Chiang <[email protected]> <[email protected]>
+Wang Pengcheng <[email protected]>
+Wang Pengcheng <[email protected]> <[email protected]>
+Wang Pengcheng <[email protected]> <[email protected]>

bolt/include/bolt/Core/BinaryContext.h

@@ -614,6 +614,11 @@ class BinaryContext {
   /// Indicates if the function ordering of the binary is finalized.
   bool HasFinalizedFunctionOrder{false};
 
+  /// Indicates if a separate .text.warm section is needed that contains
+  /// function fragments with
+  /// FunctionFragment::getFragmentNum() == FragmentNum::warm()
+  bool HasWarmSection{false};
+
   /// Is the binary always loaded at a fixed address. Shared objects and
   /// position-independent executables (PIEs) are examples of binaries that
   /// will have HasFixedLoadAddress set to false.
@@ -930,6 +935,8 @@ class BinaryContext {
 
   const char *getMainCodeSectionName() const { return ".text"; }
 
+  const char *getWarmCodeSectionName() const { return ".text.warm"; }
+
   const char *getColdCodeSectionName() const { return ".text.cold"; }
 
   const char *getHotTextMoverSectionName() const { return ".text.mover"; }

bolt/include/bolt/Core/BinaryFunction.h

@@ -75,6 +75,14 @@ enum IndirectCallPromotionType : char {
   ICP_ALL          /// Perform ICP on calls and jump tables.
 };
 
+/// Hash functions supported for BF/BB hashing.
+enum class HashFunction : char {
+  StdHash, /// std::hash, implementation is platform-dependent. Provided for
+           /// backwards compatibility.
+  XXH3,    /// llvm::xxh3_64bits, the default.
+  Default = XXH3,
+};
+
 /// Information on a single indirect call to a particular callee.
 struct IndirectCallProfile {
   MCSymbol *Symbol;
@@ -1236,6 +1244,8 @@ class BinaryFunction {
       return SmallString<32>(CodeSectionName);
     if (Fragment == FragmentNum::cold())
       return SmallString<32>(ColdCodeSectionName);
+    if (BC.HasWarmSection && Fragment == FragmentNum::warm())
+      return SmallString<32>(BC.getWarmCodeSectionName());
     return formatv("{0}.{1}", ColdCodeSectionName, Fragment.get() - 1);
   }
 
@@ -2232,18 +2242,21 @@ class BinaryFunction {
   ///
   /// If \p UseDFS is set, process basic blocks in DFS order. Otherwise, use
   /// the existing layout order.
+  /// \p HashFunction specifies which function is used for BF hashing.
   ///
   /// By default, instruction operands are ignored while calculating the hash.
   /// The caller can change this via passing \p OperandHashFunc function.
   /// The return result of this function will be mixed with internal hash.
   size_t computeHash(
-      bool UseDFS = false,
+      bool UseDFS = false, HashFunction HashFunction = HashFunction::Default,
       OperandHashFuncTy OperandHashFunc = [](const MCOperand &) {
         return std::string();
       }) const;
 
   /// Compute hash values for each block of the function.
-  void computeBlockHashes() const;
+  /// \p HashFunction specifies which function is used for BB hashing.
+  void
+  computeBlockHashes(HashFunction HashFunction = HashFunction::Default) const;
 
   void setDWARFUnit(DWARFUnit *Unit) { DwarfUnit = Unit; }
 

bolt/include/bolt/Core/DebugData.h

@@ -459,8 +459,6 @@ class DebugStrOffsetsWriter {
   std::unique_ptr<raw_svector_ostream> StrOffsetsStream;
   std::map<uint32_t, uint32_t> IndexToAddressMap;
   std::unordered_map<uint64_t, uint64_t> ProcessedBaseOffsets;
-  // Section size not including header.
-  uint32_t CurrentSectionSize{0};
   bool StrOffsetSectionWasModified = false;
 };
 

bolt/include/bolt/Core/FunctionLayout.h

@@ -63,6 +63,7 @@ class FragmentNum {
 
   static constexpr FragmentNum main() { return FragmentNum(0); }
   static constexpr FragmentNum cold() { return FragmentNum(1); }
+  static constexpr FragmentNum warm() { return FragmentNum(2); }
 };
 
 /// A freestanding subset of contiguous blocks of a function.

bolt/include/bolt/Profile/ProfileYAMLMapping.h

@@ -178,6 +178,14 @@ template <> struct ScalarBitSetTraits<PROFILE_PF> {
   }
 };
 
+template <> struct ScalarEnumerationTraits<llvm::bolt::HashFunction> {
+  using HashFunction = llvm::bolt::HashFunction;
+  static void enumeration(IO &io, HashFunction &value) {
+    io.enumCase(value, "std-hash", HashFunction::StdHash);
+    io.enumCase(value, "xxh3", HashFunction::XXH3);
+  }
+};
+
 namespace bolt {
 struct BinaryProfileHeader {
   uint32_t Version{1};
@@ -188,6 +196,7 @@ struct BinaryProfileHeader {
   std::string Origin;     // How the profile was obtained.
   std::string EventNames; // Events used for sample profile.
   bool IsDFSOrder{true};  // Whether using DFS block order in function profile
+  llvm::bolt::HashFunction HashFunction; // Hash used for BB/BF hashing
 };
 } // end namespace bolt
 
@@ -200,6 +209,8 @@ template <> struct MappingTraits<bolt::BinaryProfileHeader> {
     YamlIO.mapOptional("profile-origin", Header.Origin);
     YamlIO.mapOptional("profile-events", Header.EventNames);
     YamlIO.mapOptional("dfs-order", Header.IsDFSOrder);
+    YamlIO.mapOptional("hash-func", Header.HashFunction,
+                       llvm::bolt::HashFunction::StdHash);
   }
 };
 

bolt/lib/Core/BinaryEmitter.cpp

@@ -287,7 +287,10 @@ void BinaryEmitter::emitFunctions() {
 
   // Mark the end of hot text.
   if (opts::HotText) {
-    Streamer.switchSection(BC.getTextSection());
+    if (BC.HasWarmSection)
+      Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName()));
+    else
+      Streamer.switchSection(BC.getTextSection());
     Streamer.emitLabel(BC.getHotTextEndSymbol());
   }
 }

bolt/lib/Core/BinaryFunction.cpp

@@ -3352,9 +3352,9 @@ void BinaryFunction::fixBranches() {
         // We are going to generate two branches. Check if their targets are in
         // the same fragment as this block. If only one target is in the same
         // fragment, make it the destination of the conditional branch. There
-        // is a chance it will be a short branch which takes 5 bytes fewer than
+        // is a chance it will be a short branch which takes 4 bytes fewer than
         // a long conditional branch. For unconditional branch, the difference
-        // is 4 bytes.
+        // is 3 bytes.
         if (BB->getFragmentNum() != TSuccessor->getFragmentNum() &&
             BB->getFragmentNum() == FSuccessor->getFragmentNum())
           swapSuccessors();
@@ -3633,7 +3633,7 @@ BinaryFunction::BasicBlockListType BinaryFunction::dfs() const {
   return DFS;
 }
 
-size_t BinaryFunction::computeHash(bool UseDFS,
+size_t BinaryFunction::computeHash(bool UseDFS, HashFunction HashFunction,
                                    OperandHashFuncTy OperandHashFunc) const {
   if (size() == 0)
     return 0;
@@ -3652,7 +3652,13 @@ size_t BinaryFunction::computeHash(bool UseDFS,
   for (const BinaryBasicBlock *BB : Order)
     HashString.append(hashBlock(BC, *BB, OperandHashFunc));
 
-  return Hash = llvm::xxh3_64bits(HashString);
+  switch (HashFunction) {
+  case HashFunction::StdHash:
+    return Hash = std::hash<std::string>{}(HashString);
+  case HashFunction::XXH3:
+    return Hash = llvm::xxh3_64bits(HashString);
+  }
+  llvm_unreachable("Unhandled HashFunction");
 }
 
 void BinaryFunction::insertBasicBlocks(

bolt/lib/Core/BinaryFunctionProfile.cpp

@@ -225,6 +225,7 @@ void BinaryFunction::mergeProfileDataInto(BinaryFunction &BF) const {
     for (const BinaryBasicBlock *BBSucc : BB->successors()) {
       (void)BBSucc;
       assert(getIndex(BBSucc) == BF.getIndex(*BBMergeSI));
+      (void)BBMergeSI;
 
       // At this point no branch count should be set to COUNT_NO_PROFILE.
       assert(BII->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&

bolt/lib/Core/DebugData.cpp

@@ -889,8 +889,10 @@ void DebugStrOffsetsWriter::finalizeSection(DWARFUnit &Unit,
   // Handling re-use of str-offsets section.
   if (RetVal == ProcessedBaseOffsets.end() || StrOffsetSectionWasModified) {
     // Writing out the header for each section.
-    support::endian::write(*StrOffsetsStream, CurrentSectionSize + 4,
-                           llvm::endianness::little);
+    support::endian::write(
+        *StrOffsetsStream,
+        static_cast<uint32_t>(IndexToAddressMap.size() * 4 + 4),
+        llvm::endianness::little);
     support::endian::write(*StrOffsetsStream, static_cast<uint16_t>(5),
                            llvm::endianness::little);
     support::endian::write(*StrOffsetsStream, static_cast<uint16_t>(0),

bolt/lib/Core/Exceptions.cpp

@@ -108,7 +108,8 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
   DWARFDataExtractor Data(
       StringRef(reinterpret_cast<const char *>(LSDASectionData.data()),
                 LSDASectionData.size()),
-      BC.DwCtx->getDWARFObj().isLittleEndian(), 8);
+      BC.DwCtx->getDWARFObj().isLittleEndian(),
+      BC.DwCtx->getDWARFObj().getAddressSize());
   uint64_t Offset = getLSDAAddress() - LSDASectionAddress;
   assert(Data.isValidOffset(Offset) && "wrong LSDA address");
 

bolt/lib/Core/FunctionLayout.cpp

@@ -188,10 +188,6 @@ bool FunctionLayout::update(const ArrayRef<BinaryBasicBlock *> NewLayout) {
   for (BinaryBasicBlock *const BB : NewLayout) {
     FragmentNum Num = BB->getFragmentNum();
 
-    assert(Num >= Fragments.back()->getFragmentNum() &&
-           "Blocks must be arranged such that fragments are monotonically "
-           "increasing.");
-
     // Add empty fragments if necessary
     while (Fragments.back()->getFragmentNum() < Num)
       addFragment();

bolt/lib/Passes/IdenticalCodeFolding.cpp

@@ -360,9 +360,9 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
 
       // Pre-compute hash before pushing into hashtable.
       // Hash instruction operands to minimize hash collisions.
-      BF.computeHash(opts::ICFUseDFS, [&BC](const MCOperand &Op) {
-        return hashInstOperand(BC, Op);
-      });
+      BF.computeHash(
+          opts::ICFUseDFS, HashFunction::Default,
+          [&BC](const MCOperand &Op) { return hashInstOperand(BC, Op); });
     };
 
     ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) {

bolt/lib/Passes/IndirectCallPromotion.cpp

@@ -158,6 +158,7 @@ static cl::opt<bool> ICPPeelForInline(
 
 } // namespace opts
 
+#ifndef NDEBUG
 static bool verifyProfile(std::map<uint64_t, BinaryFunction> &BFs) {
   bool IsValid = true;
   for (auto &BFI : BFs) {
@@ -182,6 +183,7 @@ static bool verifyProfile(std::map<uint64_t, BinaryFunction> &BFs) {
   }
   return IsValid;
 }
+#endif
 
 namespace llvm {
 namespace bolt {
@@ -1467,7 +1469,6 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
                                std::max<uint64_t>(TotalIndexBasedCandidates, 1))
          << "%\n";
 
-  (void)verifyProfile;
 #ifndef NDEBUG
   verifyProfile(BFs);
 #endif

bolt/lib/Passes/LongJmp.cpp

@@ -202,10 +202,23 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst,
     }
   } else if (LocalStubsIter != Stubs.end() &&
              LocalStubsIter->second.count(TgtBB)) {
-    // If we are replacing a local stub (because it is now out of range),
-    // use its target instead of creating a stub to jump to another stub
+    // The TgtBB and TgtSym now are the local out-of-range stub and its label.
+    // So, we are attempting to restore BB to its previous state without using
+    // this stub.
     TgtSym = BC.MIB->getTargetSymbol(*TgtBB->begin());
-    TgtBB = BB.getSuccessor(TgtSym, BI);
+    assert(TgtSym &&
+           "First instruction is expected to contain a target symbol.");
+    BinaryBasicBlock *TgtBBSucc = TgtBB->getSuccessor(TgtSym, BI);
+
+    // TgtBB might have no successor. e.g. a stub for a function call.
+    if (TgtBBSucc) {
+      BB.replaceSuccessor(TgtBB, TgtBBSucc, BI.Count, BI.MispredictedCount);
+      assert(TgtBB->getExecutionCount() >= BI.Count &&
+             "At least equal or greater than the branch count.");
+      TgtBB->setExecutionCount(TgtBB->getExecutionCount() - BI.Count);
+    }
+
+    TgtBB = TgtBBSucc;
   }
 
   BinaryBasicBlock *StubBB = lookupLocalStub(BB, Inst, TgtSym, DotAddress);

bolt/lib/Passes/ReorderAlgorithm.cpp

@@ -20,7 +20,7 @@
 #include <random>
 #include <stack>
 
-#undef  DEBUG_TYPE
+#undef DEBUG_TYPE
 #define DEBUG_TYPE "bolt"
 
 using namespace llvm;
@@ -425,7 +425,7 @@ void TSPReorderAlgorithm::reorderBasicBlocks(BinaryFunction &BF,
   }
 
   std::vector<std::vector<int64_t>> DP;
-  DP.resize(1 << N);
+  DP.resize(static_cast<size_t>(1) << N);
   for (std::vector<int64_t> &Elmt : DP)
     Elmt.resize(N, -1);