Merge pull request #4286 from randombit/jack/new-cpuid-bits

Add CPUID enablement for x86 VAES, GFNI, SHA-512, SM3, and SM4

src/build-data/arch/x86_64.txt

@@ -15,6 +15,7 @@ x86_amd64
 aesni
 avx2
 bmi2
+gfni
 rdrand
 rdseed
 sha
@@ -23,4 +24,8 @@ sse41
 sse42
 ssse3
 avx512
+vaes
+sha512
+sm3
+sm4
 </isa_extensions>

src/build-data/cc/clang.txt

@@ -66,6 +66,12 @@ sse42  -> "-msse4.2"
 avx2   -> "-mavx2"
 avx512  -> "-mavx512f -mavx512bw -mavx512dq -mavx512vbmi -mavx512vbmi2 -mavx512bitalg -mavx512vl -mavx512ifma"
 
+vaes    -> "-mvaes -mavx2"
+sha512  -> "-msha512 -mavx2"
+sm3     -> "-msm3"
+sm4     -> "-msm4 -mavx2"
+
+gfni   -> "-mgfni -mavx2"
 bmi2   -> "-mbmi -mbmi2"
 aesni  -> "-maes -mpclmul"
 rdrand -> "-mrdrnd"

src/build-data/cc/gcc.txt

@@ -71,6 +71,12 @@ sse42   -> "-msse4.2"
 avx2    -> "-mavx2"
 avx512  -> "-mavx512f -mavx512bw -mavx512dq -mavx512vbmi -mavx512vbmi2 -mavx512bitalg -mavx512vl -mavx512ifma"
 
+vaes    -> "-mvaes -mavx2"
+sha512  -> "-msha512 -mavx2"
+sm3     -> "-msm3"
+sm4     -> "-msm4 -mavx2"
+
+gfni    -> "-mgfni -mavx2"
 bmi2    -> "-mbmi -mbmi2"
 aesni   -> "-maes -mpclmul"
 rdrand  -> "-mrdrnd"

src/lib/utils/cpuid/cpuid.cpp

@@ -49,16 +49,25 @@ std::string CPUID::to_string() {
 
    CPUID_PRINT(bmi2);
    CPUID_PRINT(adx);
+   CPUID_PRINT(gfni);
 
    CPUID_PRINT(aes_ni);
    CPUID_PRINT(clmul);
    CPUID_PRINT(rdrand);
    CPUID_PRINT(rdseed);
    CPUID_PRINT(intel_sha);
+   CPUID_PRINT(intel_sha512);
+
+   CPUID_PRINT(avx2_vaes);
+   CPUID_PRINT(avx2_clmul);
 
    CPUID_PRINT(avx512);
    CPUID_PRINT(avx512_aes);
    CPUID_PRINT(avx512_clmul);
+
+   CPUID_PRINT(intel_sm3);
+   CPUID_PRINT(intel_sm4);
+
 #elif defined(BOTAN_TARGET_CPU_IS_PPC_FAMILY)
    CPUID_PRINT(altivec);
    CPUID_PRINT(power_crypto);
@@ -150,78 +159,78 @@ std::vector<CPUID::CPUID_bits> CPUID::bit_from_string(std::string_view tok) {
 #if defined(BOTAN_TARGET_CPU_IS_X86_FAMILY)
    if(tok == "sse2" || tok == "simd") {
       return {CPUID::CPUID_SSE2_BIT};
-   }
-   if(tok == "ssse3") {
+   } else if(tok == "ssse3") {
       return {CPUID::CPUID_SSSE3_BIT};
-   }
-   // aes_ni is the string printed on the console when running "botan cpuid"
-   if(tok == "aesni" || tok == "aes_ni") {
+   } else if(tok == "aesni" || tok == "aes_ni") {
+      // aes_ni is the string printed on the console when running "botan cpuid"
       return {CPUID::CPUID_AESNI_BIT};
-   }
-   if(tok == "clmul") {
+   } else if(tok == "clmul") {
       return {CPUID::CPUID_CLMUL_BIT};
-   }
-   if(tok == "avx2") {
+   } else if(tok == "avx2") {
       return {CPUID::CPUID_AVX2_BIT};
-   }
-   if(tok == "avx512") {
+   } else if(tok == "avx512") {
       return {CPUID::CPUID_AVX512_BIT};
    }
    // there were two if statements testing "sha" and "intel_sha" separately; combined
-   if(tok == "sha" || tok == "intel_sha") {
+   else if(tok == "sha" || tok == "intel_sha") {
       return {CPUID::CPUID_SHA_BIT};
-   }
-   if(tok == "rdtsc") {
+   } else if(tok == "rdtsc") {
       return {CPUID::CPUID_RDTSC_BIT};
-   }
-   if(tok == "bmi2") {
+   } else if(tok == "bmi2") {
       return {CPUID::CPUID_BMI_BIT};
-   }
-   if(tok == "adx") {
+   } else if(tok == "adx") {
       return {CPUID::CPUID_ADX_BIT};
-   }
-   if(tok == "rdrand") {
+   } else if(tok == "gfni") {
+      return {CPUID::CPUID_GFNI_BIT};
+   } else if(tok == "rdrand") {
       return {CPUID::CPUID_RDRAND_BIT};
-   }
-   if(tok == "rdseed") {
+   } else if(tok == "rdseed") {
       return {CPUID::CPUID_RDSEED_BIT};
-   }
-   if(tok == "avx512_aes") {
+   } else if(tok == "avx512_aes") {
       return {CPUID::CPUID_AVX512_AES_BIT};
-   }
-   if(tok == "avx512_clmul") {
+   } else if(tok == "avx512_clmul") {
       return {CPUID::CPUID_AVX512_CLMUL_BIT};
+   } else if(tok == "avx2_vaes") {
+      return {CPUID::CPUID_AVX2_AES_BIT};
+   } else if(tok == "avx2_clmul") {
+      return {CPUID::CPUID_AVX2_CLMUL_BIT};
+   } else if(tok == "intel_sm3") {
+      return {CPUID::CPUID_SM3_BIT};
+   } else if(tok == "intel_sm4") {
+      return {CPUID::CPUID_SM4_BIT};
    }
 
 #elif defined(BOTAN_TARGET_CPU_IS_PPC_FAMILY)
-   if(tok == "altivec" || tok == "simd")
+   if(tok == "altivec" || tok == "simd") {
       return {CPUID::CPUID_ALTIVEC_BIT};
-   if(tok == "power_crypto")
+   } else if(tok == "power_crypto") {
       return {CPUID::CPUID_POWER_CRYPTO_BIT};
-   if(tok == "darn_rng")
+   } else if(tok == "darn_rng") {
       return {CPUID::CPUID_DARN_BIT};
+   }
 
 #elif defined(BOTAN_TARGET_CPU_IS_ARM_FAMILY)
-   if(tok == "neon" || tok == "simd")
+   if(tok == "neon" || tok == "simd") {
       return {CPUID::CPUID_ARM_NEON_BIT};
-   if(tok == "arm_sve")
+   } else if(tok == "arm_sve") {
       return {CPUID::CPUID_ARM_SVE_BIT};
-   if(tok == "armv8sha1" || tok == "arm_sha1")
+   } else if(tok == "armv8sha1" || tok == "arm_sha1") {
       return {CPUID::CPUID_ARM_SHA1_BIT};
-   if(tok == "armv8sha2" || tok == "arm_sha2")
+   } else if(tok == "armv8sha2" || tok == "arm_sha2") {
       return {CPUID::CPUID_ARM_SHA2_BIT};
-   if(tok == "armv8aes" || tok == "arm_aes")
+   } else if(tok == "armv8aes" || tok == "arm_aes") {
       return {CPUID::CPUID_ARM_AES_BIT};
-   if(tok == "armv8pmull" || tok == "arm_pmull")
+   } else if(tok == "armv8pmull" || tok == "arm_pmull") {
       return {CPUID::CPUID_ARM_PMULL_BIT};
-   if(tok == "armv8sha3" || tok == "arm_sha3")
+   } else if(tok == "armv8sha3" || tok == "arm_sha3") {
       return {CPUID::CPUID_ARM_SHA3_BIT};
-   if(tok == "armv8sha2_512" || tok == "arm_sha2_512")
+   } else if(tok == "armv8sha2_512" || tok == "arm_sha2_512") {
       return {CPUID::CPUID_ARM_SHA2_512_BIT};
-   if(tok == "armv8sm3" || tok == "arm_sm3")
+   } else if(tok == "armv8sm3" || tok == "arm_sm3") {
       return {CPUID::CPUID_ARM_SM3_BIT};
-   if(tok == "armv8sm4" || tok == "arm_sm4")
+   } else if(tok == "armv8sm4" || tok == "arm_sm4") {
       return {CPUID::CPUID_ARM_SM4_BIT};
+   }
 
 #else
    BOTAN_UNUSED(tok);

src/lib/utils/cpuid/cpuid.h

@@ -90,6 +90,7 @@ class BOTAN_TEST_API CPUID final {
          CPUID_RDTSC_BIT = (1U << 10),
          CPUID_ADX_BIT = (1U << 11),
          CPUID_BMI_BIT = (1U << 12),
+         CPUID_GFNI_BIT = (1U << 13),
 
          // Crypto-specific ISAs
          CPUID_AESNI_BIT = (1U << 16),
@@ -99,6 +100,11 @@ class BOTAN_TEST_API CPUID final {
          CPUID_SHA_BIT = (1U << 20),
          CPUID_AVX512_AES_BIT = (1U << 21),
          CPUID_AVX512_CLMUL_BIT = (1U << 22),
+         CPUID_AVX2_AES_BIT = (1U << 23),
+         CPUID_AVX2_CLMUL_BIT = (1U << 24),
+         CPUID_SHA512_BIT = (1U << 25),
+         CPUID_SM3_BIT = (1U << 26),
+         CPUID_SM4_BIT = (1U << 27),
 #endif
 
 #if defined(BOTAN_TARGET_CPU_IS_PPC_FAMILY)
@@ -226,8 +232,20 @@ class BOTAN_TEST_API CPUID final {
 
       /**
       * Check if the processor supports AVX-512 AES (VAES)
+      *
+      * Only set if the baseline AVX-512 profile is also satisfied
+      */
+      static bool has_avx512_aes() { return has_cpuid_bit(CPUID_AVX512_AES_BIT); }
+
+      /**
+      * Check if the processor supports AVX2 AES (VAES)
+      */
+      static bool has_avx2_vaes() { return has_cpuid_bit(CPUID_AVX2_AES_BIT); }
+
+      /**
+      * Check if the processor supports AVX2 CLMUL
       */
-      static bool has_avx512_aes() { return has_avx512() && has_cpuid_bit(CPUID_AVX512_AES_BIT); }
+      static bool has_avx2_clmul() { return has_cpuid_bit(CPUID_AVX2_CLMUL_BIT); }
 
       /**
       * Check if the processor supports AVX-512 VPCLMULQDQ
@@ -239,6 +257,14 @@ class BOTAN_TEST_API CPUID final {
       */
       static bool has_bmi2() { return has_cpuid_bit(CPUID_BMI_BIT); }
 
+      /**
+      * Check if the processor supports GFNI
+      *
+      * A few Atom processors supported GFNI only for SSE; we gate this bit
+      * on the processor also supporting GFNI-AVX2
+      */
+      static bool has_gfni() { return has_cpuid_bit(CPUID_GFNI_BIT); }
+
       /**
       * Check if the processor supports AES-NI
       */
@@ -254,6 +280,21 @@ class BOTAN_TEST_API CPUID final {
       */
       static bool has_intel_sha() { return has_sse2() && has_cpuid_bit(CPUID_SHA_BIT); }
 
+      /**
+      * Check if the processor supports Intel SHA-512 extension
+      */
+      static bool has_intel_sha512() { return has_cpuid_bit(CPUID_SHA512_BIT); }
+
+      /**
+      * Check if the processor supports Intel SM3
+      */
+      static bool has_intel_sm3() { return has_cpuid_bit(CPUID_SM3_BIT); }
+
+      /**
+      * Check if the processor supports Intel SM4
+      */
+      static bool has_intel_sm4() { return has_cpuid_bit(CPUID_SM4_BIT); }
+
       /**
       * Check if the processor supports ADX extension
       */

src/lib/utils/cpuid/cpuid_aarch64.cpp

@@ -51,24 +51,33 @@ uint32_t CPUID::CPUID_Data::detect_cpu_features() {
    const unsigned long hwcap = OS::get_auxval(ARM_hwcap_bit::ARCH_hwcap);
    if(hwcap & ARM_hwcap_bit::NEON_bit) {
       detected_features |= CPUID::CPUID_ARM_NEON_BIT;
-      if(hwcap & ARM_hwcap_bit::AES_bit)
+      if(hwcap & ARM_hwcap_bit::AES_bit) {
          detected_features |= CPUID::CPUID_ARM_AES_BIT;
-      if(hwcap & ARM_hwcap_bit::PMULL_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::PMULL_bit) {
          detected_features |= CPUID::CPUID_ARM_PMULL_BIT;
-      if(hwcap & ARM_hwcap_bit::SHA1_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::SHA1_bit) {
          detected_features |= CPUID::CPUID_ARM_SHA1_BIT;
-      if(hwcap & ARM_hwcap_bit::SHA2_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::SHA2_bit) {
          detected_features |= CPUID::CPUID_ARM_SHA2_BIT;
-      if(hwcap & ARM_hwcap_bit::SHA3_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::SHA3_bit) {
          detected_features |= CPUID::CPUID_ARM_SHA3_BIT;
-      if(hwcap & ARM_hwcap_bit::SM3_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::SM3_bit) {
          detected_features |= CPUID::CPUID_ARM_SM3_BIT;
-      if(hwcap & ARM_hwcap_bit::SM4_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::SM4_bit) {
          detected_features |= CPUID::CPUID_ARM_SM4_BIT;
-      if(hwcap & ARM_hwcap_bit::SHA2_512_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::SHA2_512_bit) {
          detected_features |= CPUID::CPUID_ARM_SHA2_512_BIT;
-      if(hwcap & ARM_hwcap_bit::SVE_bit)
+      }
+      if(hwcap & ARM_hwcap_bit::SVE_bit) {
          detected_features |= CPUID::CPUID_ARM_SVE_BIT;
+      }
    }
 
    #elif defined(BOTAN_TARGET_OS_IS_IOS) || defined(BOTAN_TARGET_OS_IS_MACOS)
@@ -87,10 +96,12 @@ uint32_t CPUID::CPUID_Data::detect_cpu_features() {
       return (feature == 1);
    };
 
-   if(sysctlbyname_has_feature("hw.optional.armv8_2_sha3"))
+   if(sysctlbyname_has_feature("hw.optional.armv8_2_sha3")) {
       detected_features |= CPUID::CPUID_ARM_SHA3_BIT;
-   if(sysctlbyname_has_feature("hw.optional.armv8_2_sha512"))
+   }
+   if(sysctlbyname_has_feature("hw.optional.armv8_2_sha512")) {
       detected_features |= CPUID::CPUID_ARM_SHA2_512_BIT;
+   }
 
    #elif defined(BOTAN_USE_GCC_INLINE_ASM)
 
@@ -133,16 +144,21 @@ uint32_t CPUID::CPUID_Data::detect_cpu_features() {
    if(OS::run_cpu_instruction_probe(neon_probe) == 1) {
       detected_features |= CPUID::CPUID_ARM_NEON_BIT;
 
-      if(OS::run_cpu_instruction_probe(aes_probe) == 1)
+      if(OS::run_cpu_instruction_probe(aes_probe) == 1) {
          detected_features |= CPUID::CPUID_ARM_AES_BIT;
-      if(OS::run_cpu_instruction_probe(pmull_probe) == 1)
+      }
+      if(OS::run_cpu_instruction_probe(pmull_probe) == 1) {
          detected_features |= CPUID::CPUID_ARM_PMULL_BIT;
-      if(OS::run_cpu_instruction_probe(sha1_probe) == 1)
+      }
+      if(OS::run_cpu_instruction_probe(sha1_probe) == 1) {
          detected_features |= CPUID::CPUID_ARM_SHA1_BIT;
-      if(OS::run_cpu_instruction_probe(sha2_probe) == 1)
+      }
+      if(OS::run_cpu_instruction_probe(sha2_probe) == 1) {
          detected_features |= CPUID::CPUID_ARM_SHA2_BIT;
-      if(OS::run_cpu_instruction_probe(sha512_probe) == 1)
+      }
+      if(OS::run_cpu_instruction_probe(sha512_probe) == 1) {
          detected_features |= CPUID::CPUID_ARM_SHA2_512_BIT;
+      }
    }
 
    #endif

src/lib/utils/cpuid/cpuid_arm32.cpp

@@ -40,14 +40,18 @@ uint32_t CPUID::CPUID_Data::detect_cpu_features() {
       detected_features |= CPUID::CPUID_ARM_NEON_BIT;
 
       const unsigned long hwcap_crypto = OS::get_auxval(ARM_hwcap_bit::ARCH_hwcap_crypto);
-      if(hwcap_crypto & ARM_hwcap_bit::AES_bit)
+      if(hwcap_crypto & ARM_hwcap_bit::AES_bit) {
          detected_features |= CPUID::CPUID_ARM_AES_BIT;
-      if(hwcap_crypto & ARM_hwcap_bit::PMULL_bit)
+      }
+      if(hwcap_crypto & ARM_hwcap_bit::PMULL_bit) {
          detected_features |= CPUID::CPUID_ARM_PMULL_BIT;
-      if(hwcap_crypto & ARM_hwcap_bit::SHA1_bit)
+      }
+      if(hwcap_crypto & ARM_hwcap_bit::SHA1_bit) {
          detected_features |= CPUID::CPUID_ARM_SHA1_BIT;
-      if(hwcap_crypto & ARM_hwcap_bit::SHA2_bit)
+      }
+      if(hwcap_crypto & ARM_hwcap_bit::SHA2_bit) {
          detected_features |= CPUID::CPUID_ARM_SHA2_BIT;
+      }
    }
    #endif