From f5838a696e491b72f9e30422e61fba753998f337 Mon Sep 17 00:00:00 2001
From: Mason Tran <mason.tran@silabs.com>
Date: Tue, 22 Aug 2023 18:54:46 -0400
Subject: [PATCH] [script] refactor `bootstrap` and `make-raspbian.bash` (#67)

---
 requirements.txt                   |   1 +
 script/bootstrap.bash              | 181 +++++++++++++++--------------
 script/make-raspbian.bash          | 165 ++++++++++++++++----------
 script/make-reference-release.bash |   1 +
 script/otbr-setup.bash             |   2 +-
 5 files changed, 202 insertions(+), 148 deletions(-)
 create mode 100644 requirements.txt
 mode change 100644 => 100755 script/otbr-setup.bash

diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..1e305cd
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+git_archive_all
\ No newline at end of file
diff --git a/script/bootstrap.bash b/script/bootstrap.bash
index af4644d..4fc01d5 100755
--- a/script/bootstrap.bash
+++ b/script/bootstrap.bash
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-#  Copyright (c) 2021, The OpenThread Authors.
+#  Copyright (c) 2023, The OpenThread Authors.
 #  All rights reserved.
 #
 #  Redistribution and use in source and binary forms, with or without
@@ -29,113 +29,120 @@
 
 set -euxo pipefail
 
-TOOLS_HOME="$HOME"/.cache/tools
-[[ -d $TOOLS_HOME ]] || mkdir -p "$TOOLS_HOME"
+# ==============================================================================
+# Bash definitions
 
-disable_install_recommends()
-{
-    OTBR_APT_CONF_FILE=/etc/apt/apt.conf
+if [[ -n ${BASH_SOURCE[0]} ]]; then
+    script_path="${BASH_SOURCE[0]}"
+else
+    script_path="$0"
+fi
+script_dir="$(realpath "$(dirname "${script_path}")")"
+repo_dir="$(dirname "${script_dir}")"
 
-    if [[ -f ${OTBR_APT_CONF_FILE} ]] && grep Install-Recommends "${OTBR_APT_CONF_FILE}"; then
-        return 0
-    fi
+# ==============================================================================
 
-    sudo tee -a /etc/apt/apt.conf <<EOF
-APT::Get::Install-Recommends "false";
-APT::Get::Install-Suggests "false";
-EOF
+install_packages_apt()
+{
+    echo 'Installing apt dependencies...'
+
+    # apt-get update and install dependencies
+    sudo apt-get update
+    sudo apt-get --no-install-recommends install -y \
+        parted \
+        fdisk \
+        git \
+        wget \
+        file \
+        xz-utils \
+        zip \
+        python3-pip \
+        dcfldd \
+        lsof
 }
 
-install_common_dependencies()
+install_packages_opkg()
 {
-    # Common dependencies
-    sudo apt-get install --no-install-recommends -y \
-        libdbus-1-dev \
-        ninja-build \
-        expect \
-        net-tools \
-        libboost-dev \
-        libboost-filesystem-dev \
-        libboost-system-dev \
-        libavahi-common-dev \
-        libavahi-client-dev \
-        libreadline-dev \
-        libncurses-dev \
-        libjsoncpp-dev \
-        coreutils
+    echo 'opkg not supported currently' && false
 }
 
-install_openthread_binaries()
+install_packages_rpm()
 {
-    pip3 install -U cmake
-    cd third_party/openthread/repo
-    mkdir -p build && cd build
-
-    cmake .. -GNinja -DOT_PLATFORM=simulation -DOT_FULL_LOGS=1 -DOT_COMMISSIONER=ON -DOT_JOINER=ON
-    ninja
-    sudo ninja install
-
-    sudo apt-get install --no-install-recommends -y socat
+    echo 'rpm not supported currently' && false
 }
 
-install_qemu()
+install_packages_brew()
 {
-    # Different versions of Linux may support qemu or qemu-system-arm.
-    # Install qemu first. If the installation of qemu fails, try install qemu-system-arm.
-
-    set +e
-    QEMU=qemu
-    sudo apt-get install --no-install-recommends --allow-unauthenticated -y $QEMU qemu-user-static binfmt-support parted dcfldd
-    ret=$?
-    set -e
-
-    if [[ ${ret} != 0 ]]; then
-        QEMU=qemu-system-arm
-        sudo apt-get install --no-install-recommends --allow-unauthenticated -y $QEMU qemu-user-static binfmt-support parted dcfldd
-    fi
+    echo 'brew not supported currently' && false
 }
 
-configure_network()
+install_packages_source()
 {
-    echo 0 | sudo tee /proc/sys/net/ipv6/conf/all/disable_ipv6
-    echo 1 | sudo tee /proc/sys/net/ipv6/conf/all/forwarding
-    echo 1 | sudo tee /proc/sys/net/ipv4/conf/all/forwarding
+    echo 'source not supported currently' && false
 }
 
-disable_install_recommends
-sudo apt-get update
-install_common_dependencies
-
-if [ "${OTBR_MDNS-}" == 'mDNSResponder' ]; then
-    SOURCE_NAME=mDNSResponder-878.30.4
-    wget https://github.com/apple-oss-distributions/mDNSResponder/archive/refs/tags/$SOURCE_NAME.tar.gz \
-        && tar xvf $SOURCE_NAME.tar.gz \
-        && cd $SOURCE_NAME/mDNSPosix \
-        && make os=linux && sudo make install os=linux
-fi
-
-install_qemu
-
-pip3 install git-archive-all
-
-# Prepare Raspbian image
+install_packages_pip3()
+{
+    echo 'Installing python3 dependencies...'
+    pip3 install --upgrade -r "${repo_dir}/requirements.txt"
+}
 
-IMAGE_NAME=$(basename "${IMAGE_URL}" .zip)
-IMAGE_FILE="$IMAGE_NAME".img
-[ -f "$TOOLS_HOME"/images/"$IMAGE_FILE" ] || {
-    # unit MB
-    EXPAND_SIZE=6144
+install_packages()
+{
+    PM=source
+    if command -v apt-get; then
+        PM=apt
+    elif command -v rpm; then
+        PM=rpm
+    elif command -v opkg; then
+        PM=opkg
+    elif command -v brew; then
+        PM=brew
+    fi
+    install_packages_$PM
 
-    [ -d "$TOOLS_HOME"/images ] || mkdir -p "$TOOLS_HOME"/images
+    if command -v pip3; then
+        install_packages_pip3
+    fi
+}
 
-    [[ -f "$IMAGE_NAME".zip ]] || curl -LO "$IMAGE_URL"
+install_qemu()
+{
+    # Different versions of Linux may support qemu or qemu-system-arm.
+    # Search for the correct one to install
+    if apt-cache search '^qemu$' | grep -q 'qemu'; then
+        QEMU=qemu
+    elif apt-cache search '^qemu-system-arm$' | grep -q 'qemu-system-arm'; then
+        QEMU=qemu-system-arm
+    else
+        echo "ERROR: Could not find 'qemu' or 'qemu-system-arm'"
+        exit 1
+    fi
 
-    unzip "$IMAGE_NAME".zip -d /tmp
+    sudo apt-get install --no-install-recommends --allow-unauthenticated -y \
+        $QEMU \
+        qemu-user-static \
+        binfmt-support
+}
 
-    (cd /tmp \
-        && dd if=/dev/zero bs=1048576 count="$EXPAND_SIZE" >>"$IMAGE_FILE" \
-        && mv "$IMAGE_FILE" "$TOOLS_HOME"/images/"$IMAGE_FILE")
+main()
+{
+    if [ $# == 0 ]; then
+        install_packages
+        install_qemu
+    elif [ "$1" == 'packages' ]; then
+        install_packages
+    elif [ "$1" == 'python' ]; then
+        install_packages_pip3
+    elif [ "$1" == 'qemu' ]; then
+        install_qemu
+    else
+        echo >&2 "Unsupported action: $1. Supported: packages, python, qemu"
+        # 128 for Invalid arguments
+        exit 128
+    fi
 
-    (cd docker-rpi-emu/scripts \
-        && sudo ./expand.sh "$TOOLS_HOME"/images/"$IMAGE_FILE" "$EXPAND_SIZE")
+    echo "Bootstrap completed successfully."
 }
+
+main "$@"
diff --git a/script/make-raspbian.bash b/script/make-raspbian.bash
index 8654d45..575de69 100755
--- a/script/make-raspbian.bash
+++ b/script/make-raspbian.bash
@@ -36,9 +36,8 @@ else
 fi
 
 script_dir="$(dirname "$(realpath "$script_path")")"
-repo_dir="$(dirname "$script_dir")"
+OT_REFERENCE_RELEASE="$(dirname "$script_dir")"
 
-IMAGE_URL=https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-05-28/2021-05-07-raspios-buster-armhf-lite.zip
 echo "REFERENCE_RELEASE_TYPE=${REFERENCE_RELEASE_TYPE?}"
 echo "IN_CHINA=${IN_CHINA:=0}"
 echo "OUTPUT_ROOT=${OUTPUT_ROOT?}"
@@ -51,23 +50,22 @@ if [ "$REFERENCE_RELEASE_TYPE" != "1.2" ] && [ "$REFERENCE_RELEASE_TYPE" != "1.3
     exit 1
 fi
 
-BUILD_TARGET=raspbian-gcc
 STAGE_DIR=/tmp/raspbian
-IMAGE_DIR=${repo_dir}/mnt-rpi
-TOOLS_HOME=$HOME/.cache/tools
+QEMU_ROOT=${OT_REFERENCE_RELEASE}/mnt-rpi
+IMAGES_DIR="${IMAGES_DIR-"${HOME}/.cache/tools/images"}"
 
 cleanup()
 {
     set +e
-
-    # Unmount and detach any loop devices
-    loop_names=$(losetup -j $STAGE_DIR/raspbian.img --output NAME -n)
-    for loop in ${loop_names}; do
-        sudo umount -lf "${loop}p1"
-        sudo umount -lf "${loop}p2"
-        sudo losetup -d "${loop}"
+    for pid in $(sudo lsof -t "$QEMU_ROOT"); do
+        sudo kill -9 "$pid"
     done
 
+    # Teardown QEMU machine
+    sudo "${OT_REFERENCE_RELEASE}"/docker-rpi-emu/scripts/qemu-cleanup.sh "$QEMU_ROOT" || true
+
+    # Unmount
+    sudo umount -f -R "$QEMU_ROOT" || true
     set -e
 }
 
@@ -75,59 +73,106 @@ trap cleanup EXIT
 
 main()
 {
-    BUILD_TARGET=$BUILD_TARGET IMAGE_URL=$IMAGE_URL ./script/bootstrap.bash
+    OPENTHREAD_COMMIT_HASH=$(git -C "${OT_REFERENCE_RELEASE}"/openthread rev-parse --short HEAD)
+    OT_BR_POSIX_COMMIT_HASH=$(git -C "${OT_REFERENCE_RELEASE}"/ot-br-posix rev-parse --short HEAD)
+
+    # Ensure qemu is installed
+    if ! command -v /usr/bin/qemu-arm-static; then
+        "${OT_REFERENCE_RELEASE}"/script/bootstrap.bash qemu
+    fi
+
+    # Ensure OUTPUT_ROOT exists
+    mkdir -p "$OUTPUT_ROOT"
+
+    # Ensure IMAGES_DIR exists
+    [ -d "$IMAGES_DIR" ] || mkdir -p "$IMAGES_DIR"
+
+    # Download raspios image
+    RASPIOS_URL=https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-05-28/2021-05-07-raspios-buster-armhf-lite.zip
+    IMAGE_ARCHIVE=$(basename "${RASPIOS_URL}")
+    IMAGE_FILE=$(basename "${IMAGE_ARCHIVE}" .zip).img
+    wget -q -O "$IMAGES_DIR/$IMAGE_ARCHIVE" -c "$RASPIOS_URL"
+
+    # Extract the downloaded archive
+    mime_type=$(file "$IMAGES_DIR/$IMAGE_ARCHIVE" --mime-type)
+    if [[ $mime_type == *"application/zip"* ]]; then
+        unzip -o "$IMAGES_DIR/$IMAGE_ARCHIVE" -d $IMAGES_DIR
+    elif [[ $mime_type == *"application/"* ]]; then
+        xz -f -k -d "$IMAGES_DIR/$IMAGE_ARCHIVE"
+    else
+        echo "ERROR: Unrecognized archive type\n${mime_type}"
+        exit 3
+    fi
+    ls -alh $IMAGES_DIR/$IMAGE_FILE
+
+    # Ensure STAGE_DIR exists. Create a copy of IMAGE_FILE in STAGE_DIR
+    [ -d "$STAGE_DIR" ] || mkdir -p "$STAGE_DIR"
 
-    IMAGE_NAME=$(basename "${IMAGE_URL}" .zip)
-    IMAGE_FILE="$TOOLS_HOME"/images/"$IMAGE_NAME".img
+    export STAGING_IMAGE_FILE="$STAGE_DIR/otbr.${REFERENCE_RELEASE_TYPE?}-$(date +%Y%m%d).ot_${OPENTHREAD_COMMIT_HASH}.ot-br_${OT_BR_POSIX_COMMIT_HASH}.img"
+    cp "$IMAGES_DIR/$IMAGE_FILE" "$STAGING_IMAGE_FILE"
 
-    [ -d "$STAGE_DIR" ] || mkdir -p "$STAGE_DIR"
-    cp -v "$IMAGE_FILE" "$STAGE_DIR"/raspbian.img
+    # Expand IMAGE_FILE by EXPAND_SIZE
+    # unit MB
+    EXPAND_SIZE=6144
+    dd if=/dev/zero bs=1M count=$EXPAND_SIZE >>"$STAGING_IMAGE_FILE"
+    ls -alh "$STAGING_IMAGE_FILE"
+    sudo "${OT_REFERENCE_RELEASE}"/docker-rpi-emu/scripts/expand.sh "$STAGING_IMAGE_FILE" "$EXPAND_SIZE"
+
+    # Create mount dir
+    mkdir -p "$QEMU_ROOT"
+
+    # Mount .img
+    sudo "${OT_REFERENCE_RELEASE}"/docker-rpi-emu/scripts/mount.sh "$STAGING_IMAGE_FILE" $QEMU_ROOT
 
+    # Mount /etc/resolv.conf
+    if [ -f "/etc/resolv.conf" ]; then
+        sudo mount -o ro,bind /etc/resolv.conf "$QEMU_ROOT"/etc/resolv.conf
+    fi
+
+    # Start RPi QEMU machine
+    sudo "${OT_REFERENCE_RELEASE}"/docker-rpi-emu/scripts/qemu-setup.sh "$QEMU_ROOT"
+
+    # Ensure git_archive_all is installed
+    if ! python3 -m pip show git_archive_all; then
+        "${OT_REFERENCE_RELEASE}"/script/bootstrap.bash python
+    fi
+
+    # Copy ot-reference-release repo into QEMU_ROOT
     python3 -m git_archive_all "$STAGE_DIR"/repo.tar.gz
+    sudo mkdir -p "$QEMU_ROOT"/home/pi/repo
+    sudo tar xzf "$STAGE_DIR"/repo.tar.gz --absolute-names --strip-components 1 -C "$QEMU_ROOT"/home/pi/repo
+
+    # Run OTBR install
+    sudo chroot "$QEMU_ROOT" /bin/bash -c "export DOCKER=${DOCKER-0}; /home/pi/repo/script/otbr-setup.bash ${REFERENCE_RELEASE_TYPE?} $IN_CHINA ${REFERENCE_PLATFORM?} ${OPENTHREAD_COMMIT_HASH} ${OT_BR_POSIX_COMMIT_HASH} ${OTBR_RCP_BUS} ${OTBR_RADIO_URL}"
+    sudo chroot "$QEMU_ROOT" /bin/bash /home/pi/repo/script/otbr-cleanup.bash
+    echo "enable_uart=1" | sudo tee -a "$QEMU_ROOT"/boot/config.txt
+    echo "dtoverlay=disable-bt" | sudo tee -a "$QEMU_ROOT"/boot/config.txt
+    if [[ ${OTBR_RCP_BUS} == "SPI" ]]; then
+        echo "dtparam=spi=on" | sudo tee -a "$QEMU_ROOT"/boot/config.txt
+    fi
+    sudo touch "$QEMU_ROOT"/boot/ssh && sync && sleep 1
+
+    # Shrink .img
+    if [[ ! -f /usr/bin/pishrink.sh ]]; then
+        sudo wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh -O /usr/bin/pishrink.sh && sudo chmod a+x /usr/bin/pishrink.sh
+    fi
+    set +e
+    sudo /usr/bin/pishrink.sh "$STAGING_IMAGE_FILE"
+    ret_val=$?
+    # Ignore error when pishrink can't shrink the image any further
+    if [[ $ret_val -ne 11 ]] && [[ $ret_val -ne 0 ]]; then
+        exit $ret_val
+    fi
+    set -e
+
+    # Write .img to SD Card
+    if [[ -n ${SD_CARD:=} ]]; then
+        sudo sh -c "dcfldd if="$STAGING_IMAGE_FILE" of=$SD_CARD bs=1m && sync"
+    fi
 
-    mkdir -p "$IMAGE_DIR"
-    chown -R "$USER": "$IMAGE_DIR"
-    ls -alh "$IMAGE_DIR"
-    script/mount.bash "$STAGE_DIR"/raspbian.img "$IMAGE_DIR"
-
-    (
-        OPENTHREAD_COMMIT_HASH=$(cd "${repo_dir}"/openthread && git rev-parse --short HEAD)
-        OT_BR_POSIX_COMMIT_HASH=$(cd "${repo_dir}"/ot-br-posix && git rev-parse --short HEAD)
-        cd docker-rpi-emu/scripts
-        sudo mount --bind /dev/pts "$IMAGE_DIR"/dev/pts
-        sudo mkdir -p "$IMAGE_DIR"/home/pi/repo
-        sudo tar xzf "$STAGE_DIR"/repo.tar.gz --absolute-names --strip-components 1 -C "$IMAGE_DIR"/home/pi/repo
-        sudo ./qemu-setup.sh "$IMAGE_DIR"
-        sudo chroot "$IMAGE_DIR" /bin/bash /home/pi/repo/script/otbr-setup.bash "${REFERENCE_RELEASE_TYPE?}" "$IN_CHINA" "${REFERENCE_PLATFORM?}" "${OPENTHREAD_COMMIT_HASH}" "${OT_BR_POSIX_COMMIT_HASH}" "${OTBR_RCP_BUS}" "${OTBR_RADIO_URL}"
-        sudo chroot "$IMAGE_DIR" /bin/bash /home/pi/repo/script/otbr-cleanup.bash
-        echo "enable_uart=1" | sudo tee -a "$IMAGE_DIR"/boot/config.txt
-        echo "dtoverlay=disable-bt" | sudo tee -a "$IMAGE_DIR"/boot/config.txt
-        if [[ ${OTBR_RCP_BUS} == "SPI" ]]; then
-            echo "dtparam=spi=on" | sudo tee -a "$IMAGE_DIR"/boot/config.txt
-        fi
-        sudo touch "$IMAGE_DIR"/boot/ssh && sync && sleep 1
-        sudo ./qemu-cleanup.sh "$IMAGE_DIR"
-        LOOP_NAME=$(losetup -j $STAGE_DIR/raspbian.img --output NAME -n)
-        sudo sh -c "dcfldd of=$STAGE_DIR/otbr.img if=$LOOP_NAME bs=1m && sync"
-        sudo cp $STAGE_DIR/otbr.img $STAGE_DIR/otbr_original.img
-        if [[ ! -f /usr/bin/pishrink.sh ]]; then
-            sudo wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh -O /usr/bin/pishrink.sh && sudo chmod a+x /usr/bin/pishrink.sh
-        fi
-        set +e
-        sudo /usr/bin/pishrink.sh $STAGE_DIR/otbr.img
-        ret_val=$?
-        # Ignore error when pishrink can't shrink the image any further
-        if [[ $ret_val -ne 11 ]] && [[ $ret_val -ne 0 ]]; then
-            exit $ret_val
-        fi
-        set -e
-        if [[ -n ${SD_CARD:=} ]]; then
-            sudo sh -c "dcfldd if=$STAGE_DIR/otbr.img of=$SD_CARD bs=1m && sync"
-        fi
-        IMG_ZIP_FILE="otbr.${REFERENCE_RELEASE_TYPE?}-$(date +%Y%m%d).ot_${OPENTHREAD_COMMIT_HASH}.ot-br_${OT_BR_POSIX_COMMIT_HASH}.img.zip"
-        (cd $STAGE_DIR && zip "$IMG_ZIP_FILE" otbr.img && mv "$IMG_ZIP_FILE" "$OUTPUT_ROOT")
-
-    )
+    # Compress .img and move archive to OUTPUT_ROOT
+    IMG_ZIP_FILE="$(basename "$STAGING_IMAGE_FILE").zip"
+    zip -j "$OUTPUT_ROOT/$IMG_ZIP_FILE" "$STAGING_IMAGE_FILE"
 }
 
 main "$@"
diff --git a/script/make-reference-release.bash b/script/make-reference-release.bash
index 17bc8e1..e3e0df6 100755
--- a/script/make-reference-release.bash
+++ b/script/make-reference-release.bash
@@ -38,6 +38,7 @@ main()
     mkdir -p build
     OUTPUT_ROOT=$(realpath build/ot-"${REFERENCE_RELEASE_TYPE?}-$(date +%Y%m%d)-$(cd openthread && git rev-parse --short HEAD)")
     mkdir -p "$OUTPUT_ROOT"
+    ./script/bootstrap.bash
 
     # ==========================================================================
     # Build firmware
diff --git a/script/otbr-setup.bash b/script/otbr-setup.bash
old mode 100644
new mode 100755
index 97aef36..7f72247
--- a/script/otbr-setup.bash
+++ b/script/otbr-setup.bash
@@ -189,7 +189,7 @@ cmake --version
 
 pip3 install zeroconf
 
-su -c "${build_options[*]} script/setup" pi || true
+su -c "${build_options[*]} script/setup" pi
 
 if [ "$REFERENCE_RELEASE_TYPE" = "1.2" ]; then
     cd /home/pi/repo/