Skip to content

Commit

Permalink
Make regular kernel build artifacts available for download
Browse files Browse the repository at this point in the history 
* artifact zip file contains:
  * zstd compressed build log
  * zstd compressed tarball of kernel images and modules
  * `SHA256` with hash values of the files above
* retention time is 2 days

> **NOTICE:** Unfortunately only registered users can download artifacts
> (GH limitation). Linux 4.19 and 5.4 do not support the `dir-pkg` make
> target, so no artifacts for those, yet.
  • Loading branch information
@johnny-mnemonic
johnny-mnemonic committed Aug 2, 2024
1 parent e281c97 commit 3bc6243
Show file tree
Hide file tree
Showing 2 changed files with 217 additions and 92 deletions.
41 changes: 37 additions & 4 deletions .github/workflows/mirror.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,9 +167,26 @@ jobs:
fi
# build ##############################################################
cd "${TOOLS_DIR}"
echo "| STATUS | BRANCH | VERSION | TYPE | USED PATCHES | COMPILER |" >> ${SUMMARY_FILE}
echo "| ------ | ------ | ------- | ---- | ------------ | -------- |" >> ${SUMMARY_FILE}
if time ./tools/make-kernel.bash ./config/linux-for-hp-integrities-v2 ia64 ${BUILD_DIR}; then
echo "| STATUS | BRANCH | VERSION | TYPE | USED PATCHES | COMPILER | DOWNLOAD |" >> ${SUMMARY_FILE}
echo "| ------ | ------ | ------- | ---- | ------------ | -------- | -------- |" >> ${SUMMARY_FILE}
if [[ ${{ matrix.branch }} == "linux-4.19.y" || ${{ matrix.branch }} == "linux-5.4.y" ]]; then
# 4.19 and 5.4 don't support dir-pkg!
BUILD_COMMAND="time ./tools/make-kernel.bash ./config/linux-for-hp-integrities-v3 ia64 ${BUILD_DIR} gha"
else
BUILD_COMMAND="time ./tools/make-kernel.bash ./config/linux-for-hp-integrities-v3 ia64 ${BUILD_DIR} gha tarball"
fi
if ${BUILD_COMMAND}; then
if [[ ${{ matrix.branch }} != "linux-4.19.y" && ${{ matrix.branch }} != "linux-5.4.y" ]]; then
mkdir build-artifacts && mv $( cat ${BUILD_DIR}/LINUX_BUILD_LOG ) $( cat ${BUILD_DIR}/LINUX_TARBALL ) build-artifacts/
pushd build-artifacts/
sha256sum * > SHA256
popd
fi
# Still make the exports, so the the upload-artifact action succeeds
# exports ##########################################################
echo "BUILD_ARTIFACTS=${PWD}/build-artifacts" >> $GITHUB_ENV
echo "KERNEL_VERSION=${KERNEL_VERSION}" >> $GITHUB_ENV
####################################################################
echo "| :white_check_mark: | ${BRANCH} | ${KERNEL_VERSION} | regular | ${USED_PATCHES} | ${CROSS_COMPILER_AND_VERSION} |" >> ${SUMMARY_FILE}
echo "${KERNEL_VERSION}" > ${GITHUB_WORKSPACE}/${BRANCH}-last-kernel-version
true
Expand All @@ -178,6 +195,22 @@ jobs:
true
fi
- uses: actions/upload-artifact@v4
id: artifact-upload-step
with:
name: ${{ env.KERNEL_VERSION }}
path: ${{ env.BUILD_ARTIFACTS }}/
compression-level: 0
retention-days: 2

- name: Output artifact ID
run: |
if [[ ${{ matrix.branch }} != "linux-4.19.y" && ${{ matrix.branch }} != "linux-5.4.y" ]]; then
echo '|||||| [build artifacts](${{ steps.artifact-upload-step.outputs.artifact-url }}) |' >> ${SUMMARY_FILE}
else
true
fi
# - uses: actions/upload-artifact@v4
# with:
# name: ${{ matrix.branch }}-last-kernel-version
Expand Down Expand Up @@ -213,7 +246,7 @@ jobs:
echo "EXIT_EARLY=1" >> $GITHUB_ENV
true
fi
HP_SIM_KERNEL="${BUILD_DIR}/../vmlinux-ski"
HP_SIM_KERNEL="${BUILD_DIR}/vmlinux"
# exports ############################################################
echo "HP_SIM_KERNEL=${HP_SIM_KERNEL}" >> $GITHUB_ENV
Expand Down
268 changes: 180 additions & 88 deletions tools/make-kernel.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

# Frank Scheiner <[email protected]>

# 2024-08-02 - v0.9.0
# make kernel tarball if needed

# 2024-05-15 - v0.7.0
# Add env var EMPTY_LOCALVERSION to compile with empty localversion

Expand Down Expand Up @@ -36,7 +39,7 @@ _flavor=""

usageMsg()
{
echo "Usage: $0 <KERNEL_CONFIG> <ARCH> <KERNEL_TREE> [<FLAVOR>]"
echo "Usage: $0 <KERNEL_CONFIG> <ARCH> <KERNEL_TREE> [<FLAVOR> [<EXTRA_MAKE_OPTION>]]"

return
}
Expand Down Expand Up @@ -95,6 +98,12 @@ if [[ "$1" != "" ]]; then

_flavor="-${1}"
shift

if [[ "$1" != "" ]]; then

_extraMakeOption="$1"
shift
fi
fi

####
Expand Down Expand Up @@ -186,6 +195,11 @@ else
fi
) | tee -a "$_tempLogFile"

if [[ ! $? -eq 0 ]]; then

exit 1
fi

_oldPWD="$PWD"
cd ${_kernelTree} || ( echo "$0: cannot cd to \"${_kernelTree}\"."; exit 1 )

Expand Down Expand Up @@ -221,119 +235,197 @@ else
fi
) 2>&1 | tee -a "$_tempLogFile"

if [[ ! $? -eq 0 ]]; then

# make all #####################################################################
(
echo "Making kernel..."
debug "time make -j${_threads} LOCALVERSION=\"${_localversion}\" ARCH=${_arch} ${_crossCompileOption} all"
date
time make -j${_threads} LOCALVERSION="${_localversion}" ARCH=${_arch} ${_crossCompileOption} all
_makeReturned=$?
echo ${_makeReturned}
date

if [[ ${_makeReturned} -eq 0 ]]; then

echo "done"
else
echo "failed"
exit 1
fi
) 2>&1 | tee -a "$_tempLogFile"

# copy kernels to the expected place
cp vmlinux "../vmlinux${_flavor}"
cp vmlinux.gz "../vmlinux.gz${_flavor}"
_kernelRelease=$( make -s LOCALVERSION="${_localversion}" kernelrelease )

# exit early for CI run
exit
_tarball="linux-${_kernelRelease}.tar"
_finalLogFileName="build.log-${_kernelRelease}"

# make modules_install #########################################################
(
echo "Installing modules..."
debug "time make LOCALVERSION=\"${_localversion}\" ARCH=${_arch} ${_crossCompileOption} modules_install"
date
time make LOCALVERSION="${_localversion}" ARCH=${_arch} ${_crossCompileOption} modules_install
_makeReturned=$?
echo ${_makeReturned}
date
if [[ "$_extraMakeOption" != "" ]]; then

if [[ ${_makeReturned} -eq 0 ]]; then
# make tarball #####################################################################
(
echo "Making kernel tarball..."
debug "time make -j${_threads} LOCALVERSION=\"${_localversion}\" ARCH=${_arch} ${_crossCompileOption} dir-pkg"
date
time make -j${_threads} LOCALVERSION="${_localversion}" ARCH=${_arch} ${_crossCompileOption} dir-pkg
_makeReturned=$?
echo ${_makeReturned}
date

echo "done"
if [[ ${_makeReturned} -eq 0 ]]; then

echo "done"
else
echo "failed"
exit 1
fi
) 2>&1 | tee -a "$_tempLogFile"

if [[ ! $? -eq 0 ]]; then

exit 1
fi

# replace vmlinuz (which is just a copy of the unstripped and uncompressed vmlinux) with vmlinux.gz
_vmlinuz="tar-install/boot/vmlinuz-${_kernelRelease}"
echo "cp vmlinux.gz \"$_vmlinuz\"" | tee -a "$_tempLogFile"
cp vmlinux.gz "$_vmlinuz"

# make actual tarball
pushd tar-install &>/dev/null
(
echo "time tar -cf ../${_tarball} boot lib"
date
time tar -cf ../${_tarball} boot lib
date
) 2>&1 | tee -a "$_tempLogFile"
popd &>/dev/null

# compress tarball
(
echo "zstd -T0 -9 ${_tarball}"
date
time zstd -T0 -9 ${_tarball} &>/dev/null
date
) 2>&1 | tee -a "$_tempLogFile"

(
echo "Build artifacts:"
echo "$PWD/${_finalLogFileName}.zst"
echo "$PWD/${_tarball}.zst"
echo "END: $( date )"
) | tee -a "$_tempLogFile"

zstd -T0 -9 "$_tempLogFile" -o "${_finalLogFileName}.zst" &>/dev/null && rm "$_tempLogFile"

echo "$PWD/${_finalLogFileName}.zst" > LINUX_BUILD_LOG
echo "$PWD/${_tarball}.zst" > LINUX_TARBALL
else
echo "failed"
exit 1
fi
) 2>&1 | tee -a "$_tempLogFile"
# make all #####################################################################
(
echo "Making kernel..."
debug "time make -j${_threads} LOCALVERSION=\"${_localversion}\" ARCH=${_arch} ${_crossCompileOption} all"
date
time make -j${_threads} LOCALVERSION="${_localversion}" ARCH=${_arch} ${_crossCompileOption} all
_makeReturned=$?
echo ${_makeReturned}
date

if [[ ${_makeReturned} -eq 0 ]]; then

echo "done"
else
echo "failed"
exit 1
fi
) 2>&1 | tee -a "$_tempLogFile"

if [[ ! $? -eq 0 ]]; then

# save artifacts ###############################################################
_kernelRelease=$( make -s LOCALVERSION="${_localversion}" kernelrelease )
cp vmlinux /boot/vmlinux-${_kernelRelease}
exit 1
fi

_uncompressedKernel="/boot/vmlinux-${_kernelRelease}"
if [[ "$_flavor" == "ski" ]]; then

if [[ "${_arch}" == "alpha" ]]; then
# If building a kernel to run in ski exit here already
exit
fi

# Due to aboot being unable to handle arbitrary large kernel and initrd images (total max size roughly 13 MiB or so!)
alpha-linux-strip vmlinux
gzip -c -9 vmlinux > vmlinuz
_compressedKernel="/boot/vmlinuz-${_kernelRelease}-stripped"
cp vmlinuz "${_compressedKernel}"
# make modules_install #########################################################
(
echo "Installing modules..."
debug "time make LOCALVERSION=\"${_localversion}\" ARCH=${_arch} ${_crossCompileOption} modules_install"
date
time make LOCALVERSION="${_localversion}" ARCH=${_arch} ${_crossCompileOption} modules_install
_makeReturned=$?
echo ${_makeReturned}
date

elif [[ "${_arch}" == "x86_64" ]]; then
if [[ ${_makeReturned} -eq 0 ]]; then

_compressedKernel="/boot/vmlinuz-${_kernelRelease}"
cp arch/x86/boot/bzImage "${_compressedKernel}"
else
_compressedKernel="/boot/vmlinuz-${_kernelRelease}"
cp vmlinux.gz "${_compressedKernel}"
fi
echo "done"
else
echo "failed"
exit 1
fi
) 2>&1 | tee -a "$_tempLogFile"

_configuration="/boot/config-${_kernelRelease}"
cp .config ${_configuration}
if [[ ! $? -eq 0 ]]; then

_systemMap="/boot/System.map-${_kernelRelease}"
cp System.map ${_systemMap}
exit 1
fi
# save artifacts ###############################################################
cp vmlinux /boot/vmlinux-${_kernelRelease}

cd /lib/modules
_uncompressedKernel="/boot/vmlinux-${_kernelRelease}"

if [[ "${_arch}" == "alpha" ]]; then
if [[ "${_arch}" == "alpha" ]]; then

# same here (see above)
tar -cf ${_kernelRelease}-non-stripped.tar ${_kernelRelease}
cd ${_kernelRelease}
find . -name "*.ko" -exec alpha-linux-strip --strip-debug '{}' \;
cd ..
tar -cf ${_kernelRelease}-stripped.tar ${_kernelRelease}
# Due to aboot being unable to handle arbitrary large kernel and initrd images (total max size roughly 13 MiB or so!)
alpha-linux-strip vmlinux
gzip -c -9 vmlinux > vmlinuz
_compressedKernel="/boot/vmlinuz-${_kernelRelease}-stripped"
cp vmlinuz "${_compressedKernel}"

_kernelModules=( "/lib/modules/${_kernelRelease}-non-stripped.tar" "/lib/modules/${_kernelRelease}-stripped.tar" )
else
tar -cf ${_kernelRelease}.tar ${_kernelRelease}
elif [[ "${_arch}" == "x86_64" ]]; then

_kernelModules=( "/lib/modules/${_kernelRelease}.tar" )
fi
_compressedKernel="/boot/vmlinuz-${_kernelRelease}"
cp arch/x86/boot/bzImage "${_compressedKernel}"
else
_compressedKernel="/boot/vmlinuz-${_kernelRelease}"
cp vmlinux.gz "${_compressedKernel}"
fi

_finalLogFile="/boot/build.log-${_kernelRelease}"
_configuration="/boot/config-${_kernelRelease}"
cp .config ${_configuration}

cd ${_oldPWD}
(
echo "Build artifacts:"
echo "${_finalLogFile}.zst"
echo "${_configuration}"
echo "${_systemMap}"
echo "${_uncompressedKernel}"
echo "${_compressedKernel}"
for _tarball in "${_kernelModules[@]}"; do

echo "${_tarball}"
done
echo "END: $( date )"
) | tee -a "$_tempLogFile"
_systemMap="/boot/System.map-${_kernelRelease}"
cp System.map ${_systemMap}

cd /lib/modules

if [[ "${_arch}" == "alpha" ]]; then

# same here (see above)
tar -cf ${_kernelRelease}-non-stripped.tar ${_kernelRelease}
cd ${_kernelRelease}
find . -name "*.ko" -exec alpha-linux-strip --strip-debug '{}' \;
cd ..
tar -cf ${_kernelRelease}-stripped.tar ${_kernelRelease}

_kernelModules=( "/lib/modules/${_kernelRelease}-non-stripped.tar" "/lib/modules/${_kernelRelease}-stripped.tar" )
else
tar -cf ${_kernelRelease}.tar ${_kernelRelease}

_kernelModules=( "/lib/modules/${_kernelRelease}.tar" )
fi

_finalLogFile="/boot/build.log-${_kernelRelease}"

mv "$_tempLogFile" "$_finalLogFile"
cd ${_oldPWD}
(
echo "Build artifacts:"
echo "${_finalLogFile}.zst"
echo "${_configuration}"
echo "${_systemMap}"
echo "${_uncompressedKernel}"
echo "${_compressedKernel}"
for _tarball in "${_kernelModules[@]}"; do

compressLogFile "$_finalLogFile" &
echo "${_tarball}"
done
echo "END: $( date )"
) | tee -a "$_tempLogFile"

mv "$_tempLogFile" "$_finalLogFile"

compressLogFile "$_finalLogFile"

fi

exit

0 comments on commit 3bc6243

Please sign in to comment.