From 04cd254d208a62be94224872fcca0850bb13f6f2 Mon Sep 17 00:00:00 2001
From: Fangjun Kuang <csukuangfj@gmail.com>
Date: Sun, 14 Apr 2024 21:37:27 +0800
Subject: [PATCH] Add include-what-you-use (#44)

---
 .github/scripts/filter.py                     |  85 +++++++++
 .github/scripts/filter.yaml                   |  23 +++
 .github/workflows/iwyu.yaml                   | 171 ++++++++++++++++++
 CMakeLists.txt                                |   2 +-
 kaldi-native-fbank/csrc/feature-fbank.cc      |   2 +
 kaldi-native-fbank/csrc/feature-fbank.h       |   2 +
 kaldi-native-fbank/csrc/feature-window.cc     |   1 +
 kaldi-native-fbank/csrc/feature-window.h      |   1 +
 kaldi-native-fbank/csrc/kaldi-math.cc         |   2 +
 kaldi-native-fbank/csrc/mel-computations.cc   |   2 +
 kaldi-native-fbank/csrc/mel-computations.h    |   3 +
 kaldi-native-fbank/csrc/online-feature.cc     |   1 +
 kaldi-native-fbank/csrc/online-feature.h      |   1 +
 kaldi-native-fbank/csrc/rfft.h                |   2 +
 kaldi-native-fbank/csrc/test-log.cc           |   6 +-
 kaldi-native-fbank/csrc/test-online-fbank.cc  |   6 +
 kaldi-native-fbank/csrc/test-rfft.cc          |   3 +
 kaldi-native-fbank/csrc/whisper-feature.cc    |   2 +
 kaldi-native-fbank/csrc/whisper-feature.h     |   1 +
 .../python/csrc/feature-fbank.cc              |   3 +-
 .../python/csrc/online-feature.cc             |   8 +
 kaldi-native-fbank/python/csrc/rfft.cc        |   2 +-
 22 files changed, 322 insertions(+), 7 deletions(-)
 create mode 100644 .github/scripts/filter.py
 create mode 100644 .github/scripts/filter.yaml
 create mode 100644 .github/workflows/iwyu.yaml

diff --git a/.github/scripts/filter.py b/.github/scripts/filter.py
new file mode 100644
index 0000000..7c36e64
--- /dev/null
+++ b/.github/scripts/filter.py
@@ -0,0 +1,85 @@
+# This file is from
+# https://github.com/acts-project/acts/blob/main/CI/iwyu/filter.py
+import sys
+import yaml
+import re
+import argparse
+from collections import namedtuple
+
+
+Config = namedtuple(
+    "Config", ["remove_lines", "replace_lines", "ignore_files"], defaults=[[], [], []]
+)
+
+
+class State:
+    skip_file: bool = False
+
+
+def parse_config(config: Config):
+    remove_lines = []
+    for s in config["remove_lines"]:
+        remove_lines.append(re.compile(s))
+
+    replace_lines = []
+    for s in config["replace_lines"]:
+        s, r = list(s.items())[0]
+        replace_lines.append((re.compile(s), r))
+
+    ignore_files = []
+    for s in config["ignore_files"]:
+        ignore_files.append(re.compile(s))
+
+    return Config(
+        remove_lines=remove_lines,
+        replace_lines=replace_lines,
+        ignore_files=ignore_files,
+    )
+
+
+def filter(line: str, config: Config, state: State):
+    if state.skip_file:
+        if line.endswith("---\n"):
+            state.skip_file = False
+        return None
+
+    if line.endswith(" should add these lines:\n"):
+        for s in config.ignore_files:
+            if s.search(line):
+                state.skip_file = True
+                return None
+
+    for s in config.remove_lines:
+        if s.search(line):
+            return None
+
+    for s, r in config.replace_lines:
+        if s.search(line):
+            return s.sub(r, line)
+
+    return line
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("config")
+parser.add_argument("input")
+parser.add_argument("output")
+args = parser.parse_args()
+
+with open(args.config, "r") as config_file:
+    try:
+        config = yaml.safe_load(config_file)
+        config = parse_config(config)
+    except yaml.YAMLError as exc:
+        print(exc)
+        sys.exit(1)
+
+with open(args.input, "r") as input_file, open(args.output, "w") as output_file:
+    state = State()
+
+    for line in input_file:
+        filtered_line = filter(line, config, state)
+        if filtered_line is not None:
+            output_file.write(filtered_line)
+
+sys.exit(0)
diff --git a/.github/scripts/filter.yaml b/.github/scripts/filter.yaml
new file mode 100644
index 0000000..23578cb
--- /dev/null
+++ b/.github/scripts/filter.yaml
@@ -0,0 +1,23 @@
+# This file is from
+# https://github.com/acts-project/acts/blob/main/CI/iwyu/filter.yaml
+remove_lines:
+  # ignore pybind11
+  - "^(- )?#include <pybind11/"
+  - "^(- )?#include [<\"]pybind11/"
+  # ignore python
+  - "^#include <abstract.h>"
+  - "^#include <floatobject.h>"
+  - "^#include <longobject.h>"
+  - "^#include <pyerrors.h>"
+
+replace_lines:
+  - "^#include <assert\\.h>": "#include <cassert>"
+  - "^#include <stddef\\.h>": "#include <cstddef>"
+  - "^#include <math\\.h>": "#include <cmath>"
+  - "^#include <limits\\.h>": "#include <climits>"
+  - "^#include <unistd\\.h>": "#include <cunistd>"
+  - "^#include <stdint\\.h>": "#include <cstdint>"
+  - "^#include <stdlib.h>": "#include <cstdlib>"
+
+ignore_files:
+  - ".*_deps/"
diff --git a/.github/workflows/iwyu.yaml b/.github/workflows/iwyu.yaml
new file mode 100644
index 0000000..6cc3d4c
--- /dev/null
+++ b/.github/workflows/iwyu.yaml
@@ -0,0 +1,171 @@
+name: iwyu
+
+on:
+  push:
+    branches:
+      - master
+      - iwyu-2
+  pull_request:
+    branches:
+      - master
+
+  workflow_dispatch:
+
+concurrency:
+  group: iwyu-${{ github.ref }}
+  cancel-in-progress: true
+
+# References
+# https://github.com/official-stockfish/Stockfish/blob/master/.github/workflows/iwyu.yml
+# https://github.com/acts-project/acts/actions/runs/8588671882/workflow
+# https://github.com/acts-project/acts/tree/main/CI/iwyu
+
+jobs:
+  iwyu:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    permissions:
+      contents: write  # for stefanzweifel/git-auto-commit-action to push code in repo
+    name: Include what you use
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Download required linux packages
+        shell: bash
+        run: |
+          echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE"
+          sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main'
+          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+          sudo apt update
+          sudo apt install -y libclang-17-dev clang-17 libc++-17-dev
+
+      - name: Display llvm
+        shell: bash
+        run: |
+          ls -lh /usr/lib/*llvm*
+
+          ls -lh /usr/bin/*clang*
+
+      - name: cache-iwyu
+        id: cache-iwyu
+        uses: actions/cache@v4
+        with:
+          path: /tmp/iwyu-install
+          key: iwyu-install-2024-04-13
+
+      - name: Get IWYU source
+        if: steps.cache-iwyu.outputs.cache-hit != 'true'
+        uses: actions/checkout@v4
+        with:
+          repository: include-what-you-use/include-what-you-use
+          ref: clang_17
+          path: iwyu
+
+      - name: Build IWYU
+        if: steps.cache-iwyu.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          mkdir iwyu-build iwyu-install
+          cmake -B iwyu-build -S iwyu -DCMAKE_PREFIX_PATH=/usr/lib/llvm-17 -DCMAKE_INSTALL_PREFIX=/tmp/iwyu-install
+          cmake --build iwyu-build --target install
+
+          ls -lh /tmp/iwyu-install
+          tree /tmp/iwyu-install
+
+          /tmp/iwyu-install/bin/include-what-you-use --version
+
+      - name: setup path
+        shell: bash
+        run: |
+          echo "/tmp/iwyu-install/bin"  >> "$GITHUB_PATH"
+
+      - name: display include what you use version
+        shell: bash
+        run: |
+          include-what-you-use --version
+          which include-what-you-use
+
+          include-what-you-use --help
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: iwyu-install
+          path: /tmpiwyu-install/*
+
+      - name: Check
+        shell: bash
+        run: |
+          mkdir build
+          cd build
+
+          cmake \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+            ..
+
+      - name: Run iwyu_tool.py
+        shell: bash
+        run: |
+          python3 /tmp/iwyu-install/bin/iwyu_tool.py -p build/ -j2 | tee iwyu-output.txt
+
+      - name: Filter IWYU output
+        shell: bash
+        run: |
+          python3 .github/scripts/filter.py .github/scripts/filter.yaml iwyu-output.txt iwyu-filtered.txt
+
+      - name: Show IWYU output
+        shell: bash
+        run: |
+          cat iwyu-output.txt
+
+      - name: Show filtered IWYU output
+        shell: bash
+        run: |
+          cat iwyu-filtered.txt
+
+      - name: Upload IWYU output
+        uses: actions/upload-artifact@v4
+        with:
+          name: iwyu
+          path: |
+            iwyu-output.txt
+            iwyu-filtered.txt
+
+      - name: Apply iwyu
+        shell: bash
+        run: |
+          git status
+          python3 /tmp/iwyu-install/bin/fix_includes.py < iwyu-filtered.txt
+
+          rm -rf iwyu*
+
+      - name: Show diff
+        shell: bash
+        run: |
+          git status
+
+          git diff
+          git diff > a.diff
+          # Download a.diff
+          # Run
+          #   git apply a.diff
+
+      - name: Upload IWYU output
+        uses: actions/upload-artifact@v4
+        with:
+          name: diff
+          path: |
+            a.diff
+
+      # https://github.com/stefanzweifel/git-auto-commit-action
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        if: false
+        with:
+          file_pattern: '*.h *.cc'
+          commit_message: "apply iwyu changes"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d994b1f..858b831 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,7 +25,7 @@ cmake_minimum_required(VERSION 3.3 FATAL_ERROR)
 
 project(kaldi-native-fbank CXX C)
 
-set(KALDI_NATIVE_FBANK_VERSION "1.19.1")
+set(KALDI_NATIVE_FBANK_VERSION "1.19.2")
 
 # Disable warning about
 #
diff --git a/kaldi-native-fbank/csrc/feature-fbank.cc b/kaldi-native-fbank/csrc/feature-fbank.cc
index 068f495..284cfb7 100644
--- a/kaldi-native-fbank/csrc/feature-fbank.cc
+++ b/kaldi-native-fbank/csrc/feature-fbank.cc
@@ -24,8 +24,10 @@
 #include <cmath>
 #include <limits>
 #include <vector>
+#include <utility>
 
 #include "kaldi-native-fbank/csrc/feature-functions.h"
+#include "kaldi-native-fbank/csrc/log.h"
 
 namespace knf {
 
diff --git a/kaldi-native-fbank/csrc/feature-fbank.h b/kaldi-native-fbank/csrc/feature-fbank.h
index 0786aad..84005e0 100644
--- a/kaldi-native-fbank/csrc/feature-fbank.h
+++ b/kaldi-native-fbank/csrc/feature-fbank.h
@@ -24,6 +24,8 @@
 #include <map>
 #include <string>
 #include <vector>
+#include <cstdint>
+#include <sstream>
 
 #include "kaldi-native-fbank/csrc/feature-window.h"
 #include "kaldi-native-fbank/csrc/mel-computations.h"
diff --git a/kaldi-native-fbank/csrc/feature-window.cc b/kaldi-native-fbank/csrc/feature-window.cc
index 4f695c9..c5d606f 100644
--- a/kaldi-native-fbank/csrc/feature-window.cc
+++ b/kaldi-native-fbank/csrc/feature-window.cc
@@ -10,6 +10,7 @@
 #include <cmath>
 #include <limits>
 #include <vector>
+#include <cstddef>
 
 #include "kaldi-native-fbank/csrc/kaldi-math.h"
 
diff --git a/kaldi-native-fbank/csrc/feature-window.h b/kaldi-native-fbank/csrc/feature-window.h
index 0a71bb5..88b509e 100644
--- a/kaldi-native-fbank/csrc/feature-window.h
+++ b/kaldi-native-fbank/csrc/feature-window.h
@@ -10,6 +10,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
+#include <cstdint>
 
 #include "kaldi-native-fbank/csrc/log.h"
 
diff --git a/kaldi-native-fbank/csrc/kaldi-math.cc b/kaldi-native-fbank/csrc/kaldi-math.cc
index 7e0cdc2..4b966a1 100644
--- a/kaldi-native-fbank/csrc/kaldi-math.cc
+++ b/kaldi-native-fbank/csrc/kaldi-math.cc
@@ -6,7 +6,9 @@
 
 #include "kaldi-native-fbank/csrc/kaldi-math.h"
 
+#if defined(_POSIX_THREAD_SAFE_FUNCTIONS)
 #include <mutex>  // NOLINT
+#endif
 
 namespace knf {
 
diff --git a/kaldi-native-fbank/csrc/mel-computations.cc b/kaldi-native-fbank/csrc/mel-computations.cc
index a0bbac9..af4c9f1 100644
--- a/kaldi-native-fbank/csrc/mel-computations.cc
+++ b/kaldi-native-fbank/csrc/mel-computations.cc
@@ -20,11 +20,13 @@
 
 #include "kaldi-native-fbank/csrc/mel-computations.h"
 
+#include <stdio.h>
 #include <algorithm>
 #include <sstream>
 #include <vector>
 
 #include "kaldi-native-fbank/csrc/feature-window.h"
+#include "kaldi-native-fbank/csrc/log.h"
 
 namespace knf {
 
diff --git a/kaldi-native-fbank/csrc/mel-computations.h b/kaldi-native-fbank/csrc/mel-computations.h
index 46b5fb5..4c229e6 100644
--- a/kaldi-native-fbank/csrc/mel-computations.h
+++ b/kaldi-native-fbank/csrc/mel-computations.h
@@ -23,10 +23,13 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <cstdint>
+#include <sstream>
 
 #include "kaldi-native-fbank/csrc/feature-window.h"
 
 namespace knf {
+struct FrameExtractionOptions;
 
 struct MelBanksOptions {
   int32_t num_bins = 25;  // e.g. 25; number of triangular bins
diff --git a/kaldi-native-fbank/csrc/online-feature.cc b/kaldi-native-fbank/csrc/online-feature.cc
index 0001abc..57737a3 100644
--- a/kaldi-native-fbank/csrc/online-feature.cc
+++ b/kaldi-native-fbank/csrc/online-feature.cc
@@ -24,6 +24,7 @@
 #include <algorithm>
 #include <utility>
 #include <vector>
+#include <cstddef>
 
 #include "kaldi-native-fbank/csrc/feature-window.h"
 #include "kaldi-native-fbank/csrc/log.h"
diff --git a/kaldi-native-fbank/csrc/online-feature.h b/kaldi-native-fbank/csrc/online-feature.h
index 99bfc97..d9fea75 100644
--- a/kaldi-native-fbank/csrc/online-feature.h
+++ b/kaldi-native-fbank/csrc/online-feature.h
@@ -27,6 +27,7 @@
 
 #include "kaldi-native-fbank/csrc/feature-fbank.h"
 #include "kaldi-native-fbank/csrc/whisper-feature.h"
+#include "kaldi-native-fbank/csrc/feature-window.h"
 
 namespace knf {
 
diff --git a/kaldi-native-fbank/csrc/rfft.h b/kaldi-native-fbank/csrc/rfft.h
index c8cb9f8..d4652f8 100644
--- a/kaldi-native-fbank/csrc/rfft.h
+++ b/kaldi-native-fbank/csrc/rfft.h
@@ -20,6 +20,7 @@
 #define KALDI_NATIVE_FBANK_CSRC_RFFT_H_
 
 #include <memory>
+#include <cstdint>
 
 namespace knf {
 
@@ -48,6 +49,7 @@ class Rfft {
 
  private:
   class RfftImpl;
+
   std::unique_ptr<RfftImpl> impl_;
 };
 
diff --git a/kaldi-native-fbank/csrc/test-log.cc b/kaldi-native-fbank/csrc/test-log.cc
index 6379633..1e6ef73 100644
--- a/kaldi-native-fbank/csrc/test-log.cc
+++ b/kaldi-native-fbank/csrc/test-log.cc
@@ -16,13 +16,12 @@
  * limitations under the License.
  */
 
+#if KNF_ENABLE_CHECK
 #include "gtest/gtest.h"
 #include "kaldi-native-fbank/csrc/log.h"
 
 namespace knf {
 
-#if KNF_ENABLE_CHECK
-
 TEST(Log, TestLog) {
   KNF_LOG(TRACE) << "this is a trace message";
   KNF_LOG(DEBUG) << "this is a debug message";
@@ -68,6 +67,5 @@ TEST(Log, TestCheck) {
 #endif
 }
 
-#endif
-
 }  // namespace knf
+#endif
diff --git a/kaldi-native-fbank/csrc/test-online-fbank.cc b/kaldi-native-fbank/csrc/test-online-fbank.cc
index 9f595cf..c2d437a 100644
--- a/kaldi-native-fbank/csrc/test-online-fbank.cc
+++ b/kaldi-native-fbank/csrc/test-online-fbank.cc
@@ -17,8 +17,14 @@
  */
 
 #include <iostream>
+#include <cstdint>
+#include <sstream>
+#include <string>
 
 #include "kaldi-native-fbank/csrc/online-feature.h"
+#include "kaldi-native-fbank/csrc/feature-fbank.h"
+#include "kaldi-native-fbank/csrc/feature-window.h"
+#include "kaldi-native-fbank/csrc/mel-computations.h"
 
 int main() {
   knf::FbankOptions opts;
diff --git a/kaldi-native-fbank/csrc/test-rfft.cc b/kaldi-native-fbank/csrc/test-rfft.cc
index 47f9904..e93adac 100644
--- a/kaldi-native-fbank/csrc/test-rfft.cc
+++ b/kaldi-native-fbank/csrc/test-rfft.cc
@@ -16,6 +16,9 @@
  * limitations under the License.
  */
 
+#include <cstdint>
+#include <vector>
+
 #include "gtest/gtest.h"
 #include "kaldi-native-fbank/csrc/rfft.h"
 
diff --git a/kaldi-native-fbank/csrc/whisper-feature.cc b/kaldi-native-fbank/csrc/whisper-feature.cc
index b887a04..e8ae5a2 100644
--- a/kaldi-native-fbank/csrc/whisper-feature.cc
+++ b/kaldi-native-fbank/csrc/whisper-feature.cc
@@ -20,8 +20,10 @@
 
 #include <cmath>
 #include <vector>
+#include <string>
 
 #include "kaldi-native-fbank/csrc/mel-computations.h"
+#include "kaldi-native-fbank/csrc/log.h"
 
 #ifndef M_2PI
 #define M_2PI 6.283185307179586476925286766559005
diff --git a/kaldi-native-fbank/csrc/whisper-feature.h b/kaldi-native-fbank/csrc/whisper-feature.h
index 18ce636..033bd7c 100644
--- a/kaldi-native-fbank/csrc/whisper-feature.h
+++ b/kaldi-native-fbank/csrc/whisper-feature.h
@@ -21,6 +21,7 @@
 
 #include <memory>
 #include <vector>
+#include <cstdint>
 
 #include "kaldi-native-fbank/csrc/feature-window.h"
 #include "kaldi-native-fbank/csrc/mel-computations.h"
diff --git a/kaldi-native-fbank/python/csrc/feature-fbank.cc b/kaldi-native-fbank/python/csrc/feature-fbank.cc
index 4f32895..870dc86 100644
--- a/kaldi-native-fbank/python/csrc/feature-fbank.cc
+++ b/kaldi-native-fbank/python/csrc/feature-fbank.cc
@@ -18,11 +18,12 @@
 
 #include "kaldi-native-fbank/python/csrc/feature-fbank.h"
 
-#include <memory>
 #include <string>
 
 #include "kaldi-native-fbank/csrc/feature-fbank.h"
 #include "kaldi-native-fbank/python/csrc/utils.h"
+#include "kaldi-native-fbank/csrc/feature-window.h"
+#include "kaldi-native-fbank/csrc/mel-computations.h"
 
 namespace knf {
 
diff --git a/kaldi-native-fbank/python/csrc/online-feature.cc b/kaldi-native-fbank/python/csrc/online-feature.cc
index ac16044..aebf33b 100644
--- a/kaldi-native-fbank/python/csrc/online-feature.cc
+++ b/kaldi-native-fbank/python/csrc/online-feature.cc
@@ -20,8 +20,16 @@
 
 #include <string>
 #include <vector>
+#include <cstdint>
 
 #include "kaldi-native-fbank/csrc/online-feature.h"
+#include "kaldi-native-fbank/csrc/feature-fbank.h"
+#include "kaldi-native-fbank/csrc/whisper-feature.h"
+
+namespace pybind11 {
+class gil_scoped_release;
+}  // namespace pybind11
+
 namespace knf {
 
 template <typename C>
diff --git a/kaldi-native-fbank/python/csrc/rfft.cc b/kaldi-native-fbank/python/csrc/rfft.cc
index b73384f..90d2ea3 100644
--- a/kaldi-native-fbank/python/csrc/rfft.cc
+++ b/kaldi-native-fbank/python/csrc/rfft.cc
@@ -18,8 +18,8 @@
 
 #include "kaldi-native-fbank/python/csrc/rfft.h"
 
-#include <string>
 #include <vector>
+#include <cstdint>
 
 #include "kaldi-native-fbank/csrc/rfft.h"