From 4ee3c5b14d479b1872296bf9fe68f233a21d3f05 Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Tue, 7 Nov 2023 18:14:20 +0700 Subject: [PATCH] LOCAL: add Github build jobs --- .github/workflows/build.yml | 238 ++++++++++++++++++ .../scripts/configure-cross-linux-arm64.sh | 29 +++ .github/workflows/scripts/configure-macos.sh | 31 +++ .github/workflows/scripts/configure-native.sh | 31 +++ .github/workflows/scripts/configure-win.sh | 32 +++ .github/workflows/scripts/install-native.sh | 16 ++ .../prerequisites-cross-linux-arm64.sh | 27 ++ .../workflows/scripts/prerequisites-macos.sh | 38 +++ .../workflows/scripts/prerequisites-native.sh | 23 ++ .../workflows/scripts/prerequisites-old.sh | 44 ++++ 10 files changed, 509 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100755 .github/workflows/scripts/configure-cross-linux-arm64.sh create mode 100755 .github/workflows/scripts/configure-macos.sh create mode 100755 .github/workflows/scripts/configure-native.sh create mode 100755 .github/workflows/scripts/configure-win.sh create mode 100755 .github/workflows/scripts/install-native.sh create mode 100755 .github/workflows/scripts/prerequisites-cross-linux-arm64.sh create mode 100755 .github/workflows/scripts/prerequisites-macos.sh create mode 100755 .github/workflows/scripts/prerequisites-native.sh create mode 100755 .github/workflows/scripts/prerequisites-old.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..26f9bb4086 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,238 @@ +name: 'build' + +on: [push, pull_request] + +jobs: + + Vars: + runs-on: ubuntu-latest + outputs: + VERSION: ${{ steps.Vars.outputs.VERSION }} + PROJECT_NAME: ${{ steps.Vars.outputs.PROJECT_NAME }} + REPOSITORY_URL: ${{ steps.Vars.outputs.REPOSITORY_URL }} + steps: + - id: Vars + run: | + echo "ref_type: ${{ github.ref_type }}" + echo "ref: ${{ github.ref }}" + echo "sha: ${{ github.sha }}" + echo "" + if [[ "${{ github.ref_type }}" == "tag" ]]; then + VERSION=${{ github.ref }} + VERSION=${VERSION#"refs/tags/"} + VERSION=${VERSION//"-"/"_"} # replace dashes with underscores in the version + else + VERSION=${{ github.sha }} + VERSION=g${VERSION:0:8} + fi + echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" + echo $VERSION + + OWNER=${{ github.repository_owner }} + FULLNAME=${{ github.repository }} + PROJECT_NAME=${FULLNAME#"$OWNER/"} + echo "PROJECT_NAME=$PROJECT_NAME" >> "$GITHUB_OUTPUT" + echo $PROJECT_NAME + + REPOSITORY_URL=${{ github.server_url }}/${{ github.repository }} + echo "REPOSITORY_URL=$REPOSITORY_URL" >> "$GITHUB_OUTPUT" + echo $REPOSITORY_URL + + Build: + needs: [ Vars ] + env: + # A simple way to spread several environments for Windows. But to be more precise, we need to somehow add different envs to matrix.include + + MSYSTEM: MINGW64 # Ref: https://www.msys2.org/docs/environments/ + CHERE_INVOKING: 'yes' + MSYS: 'winsymlinks:native' + strategy: + fail-fast: false + matrix: + # Naming MinGW platforms: https://sourceforge.net/p/mingw-w64/wiki2/TypeTriplets/ + platform: + - x86_64-linux-gnu + - aarch64-linux-gnu + - x86_64-w64-mingw32 + - x86_64-apple-darwin + - aarch64-apple-darwin + target: [xtensa-softmmu, riscv32-softmmu] + + include: + - platform: x86_64-linux-gnu + runs_on: ubuntu-22.04 + # Debian 11 provides the lowest possible Glibc version (2.31), given QEMU's glib2 => 2.66 requirement. + container: debian:11.11 + shell: bash -e {0} + run_prerequisites: "./.github/workflows/scripts/prerequisites-native.sh" + run_configure: "./.github/workflows/scripts/configure-native.sh" + + - platform: aarch64-linux-gnu + runs_on: ubuntu-22.04 + # Debian 11 provides the lowest possible Glibc version (2.31), given QEMU's glib2 => 2.66 requirement. + container: debian:11.11 + shell: bash -e {0} + run_prerequisites: "./.github/workflows/scripts/prerequisites-cross-linux-arm64.sh" + run_configure: "./.github/workflows/scripts/configure-cross-linux-arm64.sh" + + - platform: x86_64-w64-mingw32 + runs_on: windows-2022 + shell: msys2 {0} + run_configure: "./.github/workflows/scripts/configure-win.sh" + + - platform: x86_64-apple-darwin + runs_on: macos-13 + shell: bash -e {0} + run_prerequisites: "./.github/workflows/scripts/prerequisites-macos.sh" + run_configure: "./.github/workflows/scripts/configure-macos.sh" + + - platform: aarch64-apple-darwin + runs_on: macos-13-xlarge + shell: bash -e {0} + run_prerequisites: "./.github/workflows/scripts/prerequisites-macos.sh" + run_configure: "./.github/workflows/scripts/configure-macos.sh" + + runs-on: ${{ matrix.runs_on }} + container: + image: ${{ matrix.container }} + defaults: + run: + shell: ${{ matrix.shell }} + + steps: + - uses: actions/checkout@v4 + + - name: Install prerequisites (${{ matrix.target }} @ ${{ matrix.platform }}) + if : ${{ matrix.platform != 'x86_64-w64-mingw32' }} + run: | + ${{ matrix.run_prerequisites }} + + - name: Install MSYS2 prerequisites (${{ matrix.target }} @ ${{ matrix.platform }}) + if : ${{ matrix.platform == 'x86_64-w64-mingw32' }} + uses: msys2/setup-msys2@v2 + with: + cache: true + msystem: MINGW64 + install: >- + diffutils + git + mingw-w64-x86_64-gcc + mingw-w64-x86_64-glib2 + mingw-w64-x86_64-libgcrypt + mingw-w64-x86_64-libiconv + mingw-w64-x86_64-libslirp + mingw-w64-x86_64-ninja + mingw-w64-x86_64-pixman + mingw-w64-x86_64-pkg-config + mingw-w64-x86_64-python + mingw-w64-x86_64-SDL2 + + - name: Configure (${{ matrix.target }} @ ${{ matrix.platform }}) + env: + VERSION: ${{ needs.Vars.outputs.VERSION }} + TARGET: ${{ matrix.target }} + run: | + ${{ matrix.run_configure }} + + - name: Build (${{ matrix.target }} @ ${{ matrix.platform }}) + run: | + ninja -C build install + + - name: Package (${{ matrix.target }} @ ${{ matrix.platform }}) + env: + VERSION: ${{ needs.Vars.outputs.VERSION }} + PROJECT_NAME: ${{ needs.Vars.outputs.PROJECT_NAME }} + PLATFORM: ${{ matrix.platform }} + TARGET: ${{ matrix.target }} + run: | + find $PWD/install/qemu/share/qemu/ -maxdepth 1 -mindepth 1 -not -name 'esp*.bin' -exec rm -rfv {} \; + export DIST_DIR=${PWD}/dist + mkdir -p ${DIST_DIR} + cd $PWD/install + ARCHIVE_NAME=${PROJECT_NAME}-${TARGET}-${VERSION}-${PLATFORM}.tar.xz + tar -cJvf ${DIST_DIR}/${ARCHIVE_NAME} qemu > ${DIST_DIR}/dist-filelist-${TARGET}-${PLATFORM}.txt 2>&1 + echo "dist/dist-${PROJECT_NAME}-${TARGET}-${PLATFORM}/${ARCHIVE_NAME}" > ${DIST_DIR}/file_${PLATFORM}_${TARGET} + + - name: Save build/build.ninja for failed builds + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: build-ninja-${{ matrix.target }}-${{ matrix.platform }} + path: build/build.ninja + + - name: Dist filelist (dbg) (${{ matrix.target }} @ ${{ matrix.platform }}) + uses: actions/upload-artifact@v4 + with: + name: dist-filelist-${{ matrix.target }}-${{ matrix.platform }} + path: | + dist/dist-filelist-* + + - name: Dist artifacts (${{ matrix.target }} @ ${{ matrix.platform }}) + uses: actions/upload-artifact@v4 + with: + name: dist-qemu-${{ matrix.target }}-${{ matrix.platform }} + path: | + dist/ + + Upload: + # if: startsWith(github.ref, 'refs/tags/') + needs: [ Vars, Build ] + runs-on: ubuntu-latest + steps : + - name: Create a source archive + id: create-source + env: + VERSION: ${{ needs.Vars.outputs.VERSION }} + PROJECT_NAME: ${{ needs.Vars.outputs.PROJECT_NAME }} + REPOSITORY_URL: ${{ needs.Vars.outputs.REPOSITORY_URL }} + run: | + export RELEASE_CHECKSUM_FILE="${PROJECT_NAME}-${VERSION}-checksum.sha256" + echo "RELEASE_CHECKSUM_FILE=$RELEASE_CHECKSUM_FILE" >> "$GITHUB_OUTPUT" + RELEASE_SRC_NAME="${PROJECT_NAME}-${VERSION}-src" + export RELEASE_SRC_FILE="${PROJECT_NAME}-${VERSION}-src.tar.xz" + echo "RELEASE_SRC_FILE=$RELEASE_SRC_FILE" >> "$GITHUB_OUTPUT" + + rm -rf -- "${RELEASE_SRC_NAME}" "${RELEASE_SRC_FILE}" + git clone --quiet --depth 1 --recurse-submodules --shallow-submodules "${REPOSITORY_URL}" "${RELEASE_SRC_NAME}" + find "${RELEASE_SRC_NAME}" -name ".git" -type d -exec rm -rf -- "{}" + + find "${RELEASE_SRC_NAME}" -name .git\* -exec rm -rf -- {} + + tar -cJvf "${RELEASE_SRC_FILE}" "${RELEASE_SRC_NAME}" > src-filelist.txt 2>&1 + rm -rf -- "${RELEASE_SRC_NAME}" + ls -l + + - name: Source filelist (dbg) + uses: actions/upload-artifact@v4 + with: + name: src-filelist + path: | + src-filelist.txt + + - uses: actions/download-artifact@v4 + with: + pattern: dist-qemu-* + path: dist + + - name: Create a checksum file + env: + RELEASE_CHECKSUM_FILE: ${{ steps.create-source.outputs.RELEASE_CHECKSUM_FILE }} + RELEASE_SRC_FILE: ${{ steps.create-source.outputs.RELEASE_SRC_FILE }} + run: | + ls -l + DIST_FILE_LIST=$(find . -name file_\* -exec cat {} \+) + DIST_FILE_LIST="${DIST_FILE_LIST} ${RELEASE_SRC_FILE}" + for n in $DIST_FILE_LIST; do + sz=$(stat -c%s "${n}") + printf "# %s: %s bytes\n" "${n}" "${sz}" >> "${RELEASE_CHECKSUM_FILE}" + sha256sum -b "${n}" >> "${RELEASE_CHECKSUM_FILE}" + done + DIST_FILE_LIST="${DIST_FILE_LIST} ${RELEASE_CHECKSUM_FILE}" + # remove new lines from file list + DIST_FILE_LIST="${DIST_FILE_LIST//$'\n'/ }" + cat "${RELEASE_CHECKSUM_FILE}" + rm -rf -- upload && mkdir -p upload && mv -v ${DIST_FILE_LIST} upload/ + + - name: Release + uses: softprops/action-gh-release@v1 + with: + draft: true + files: upload/* diff --git a/.github/workflows/scripts/configure-cross-linux-arm64.sh b/.github/workflows/scripts/configure-cross-linux-arm64.sh new file mode 100755 index 0000000000..03cc996b5a --- /dev/null +++ b/.github/workflows/scripts/configure-cross-linux-arm64.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TARGET=${TARGET:-xtensa-softmmu} +VERSION=${VERSION:-dev} + +# Replacing libgcrypt method to 'pkg-config' for crossbuilding on Linux +sed -z -i "s/\(.*dependency('libgcrypt'.*method: '\)config-tool\('.*\)/\1pkg-config\2/g" -- meson.build + +echo DBG +./configure --help + +./configure \ + --bindir=bin \ + --cross-prefix=aarch64-linux-gnu- \ + --datadir=share/qemu \ + --enable-gcrypt \ + --enable-sdl \ + --enable-pixman \ + --enable-slirp \ + --enable-stack-protector \ + --extra-cflags=-Werror \ + --prefix=${PWD}/install/qemu \ + --target-list=${TARGET} \ + --with-pkgversion="${VERSION}" \ + --with-suffix="" \ + --without-default-features \ +|| { cat meson-logs/meson-log.txt && false; } diff --git a/.github/workflows/scripts/configure-macos.sh b/.github/workflows/scripts/configure-macos.sh new file mode 100755 index 0000000000..fc3fc67674 --- /dev/null +++ b/.github/workflows/scripts/configure-macos.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TARGET=${TARGET:-xtensa-softmmu} +VERSION=${VERSION:-dev} + +sed -i '' "s/project('qemu', \['c'\],/project('qemu', \['c', 'objc'\],/" meson.build + +# workaround for some headers that macOS couldn't find for some unknown reason +sed -i '' "s/common_user_inc = \[\]/common_user_inc = \['include', 'build'\]/" meson.build + +echo DBG +./configure --help + +./configure \ + --bindir=bin \ + --datadir=share/qemu \ + --enable-fdt=internal \ + --enable-gcrypt \ + --enable-sdl \ + --enable-pixman \ + --enable-slirp \ + --enable-stack-protector \ + --prefix=$PWD/install/qemu \ + --python=python3 \ + --target-list=${TARGET} \ + --with-pkgversion="${VERSION}" \ + --with-suffix="" \ + --without-default-features \ +|| { cat meson-logs/meson-log.txt && false; } diff --git a/.github/workflows/scripts/configure-native.sh b/.github/workflows/scripts/configure-native.sh new file mode 100755 index 0000000000..b68c7ad753 --- /dev/null +++ b/.github/workflows/scripts/configure-native.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TARGET=${TARGET:-xtensa-softmmu} +VERSION=${VERSION:-dev} + +echo DBG +./configure --help + +# Building with -Werror only on Linux as that breaks some features detection in meson on macOS. +# Defining --bindir, --datadir, etc - to have the same directory tree on Linux and Windows +# also adding --with-suffix="" to avoid doubled "qemu/qemu" path. +# +# MinGW build ref: https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-qemu/PKGBUILD + +./configure \ + --bindir=bin \ + --datadir=share/qemu \ + --enable-gcrypt \ + --enable-sdl \ + --enable-pixman \ + --enable-slirp \ + --enable-stack-protector \ + --extra-cflags=-Werror \ + --prefix=${PWD}/install/qemu \ + --target-list=${TARGET} \ + --with-pkgversion="${VERSION}" \ + --with-suffix="" \ + --without-default-features \ +|| { cat meson-logs/meson-log.txt && false; } diff --git a/.github/workflows/scripts/configure-win.sh b/.github/workflows/scripts/configure-win.sh new file mode 100755 index 0000000000..f5b62ecf33 --- /dev/null +++ b/.github/workflows/scripts/configure-win.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TARGET=${TARGET:-xtensa-softmmu} +VERSION=${VERSION:-dev} + +echo DBG +./configure --help + +./configure \ + --bindir=bin \ + --datadir=share/qemu \ + --enable-gcrypt \ + --enable-sdl \ + --enable-pixman \ + --enable-slirp \ + --enable-stack-protector \ + --extra-cflags=-Werror \ + --prefix=${PWD}/install/qemu \ + --static \ + --target-list=${TARGET} \ + --with-pkgversion="${VERSION}" \ + --with-suffix="" \ + --without-default-features \ +|| { cat meson-logs/meson-log.txt && false; } + + +# This fixes the issue that for some reason, meson is not able to determine correct +# paths for libiconv and libintl libraries from 'pkg-config --libs --static libgcrypt'. +MSYS_BASE=$(cygpath -w / | sed 's/\\/\//g') +sed -i "s|/mingw64/lib/libintl.dll.a|${MSYS_BASE}/mingw64/lib/libintl.dll.a|g; s|/mingw64/lib/libiconv.dll.a|${MSYS_BASE}/mingw64/lib/libiconv.dll.a|g" build/build.ninja diff --git a/.github/workflows/scripts/install-native.sh b/.github/workflows/scripts/install-native.sh new file mode 100755 index 0000000000..116d148d1c --- /dev/null +++ b/.github/workflows/scripts/install-native.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export DEBIAN_FRONTEND="noninteractive" + +# For libslirp0 on ubuntu-18.04: +# add-apt-repository ppa:openstack-ubuntu-testing/ussuri + +apt-get update -y -q +apt-get install -y -q --no-install-recommends \ + libglib2.0-0 \ + libpixman-1-0 \ + libsdl2-2.0-0 \ + libslirp0 \ +&& : diff --git a/.github/workflows/scripts/prerequisites-cross-linux-arm64.sh b/.github/workflows/scripts/prerequisites-cross-linux-arm64.sh new file mode 100755 index 0000000000..df12fbae49 --- /dev/null +++ b/.github/workflows/scripts/prerequisites-cross-linux-arm64.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export DEBIAN_FRONTEND="noninteractive" + +dpkg --add-architecture arm64 + +apt-get update -y -q +apt-get install -y -q --no-install-recommends \ + binutils-aarch64-linux-gnu \ + build-essential \ + crossbuild-essential-arm64 \ + gcc-aarch64-linux-gnu \ + git \ + libgcrypt20-dev:arm64 \ + libglib2.0-dev:arm64 \ + libgpg-error-dev:arm64 \ + libpixman-1-dev:arm64 \ + libsdl2-dev:arm64 \ + libslirp-dev:arm64 \ + ninja-build \ + python3-pip \ + zlib1g-dev:arm64 \ +&& : + +/usr/bin/pip3 install meson==1.7.0 tomli==2.2.1 diff --git a/.github/workflows/scripts/prerequisites-macos.sh b/.github/workflows/scripts/prerequisites-macos.sh new file mode 100755 index 0000000000..58b5b0b7a3 --- /dev/null +++ b/.github/workflows/scripts/prerequisites-macos.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -euo pipefail + +brew install \ + glib \ + libgcrypt \ + libslirp \ + ninja \ + pixman \ + pkg-config \ + sdl2 \ +&& : + +# workaround if deprecated module 'distutils.version' is missing +# https://peps.python.org/pep-0632/#migration-advice +# https://pypi.org/project/looseversion/ +# +# Used for building only, not for qemu run-time +# +# Also if something goes wrong, 'setup-python' action can be used +# as a general solution, https://github.com/actions/runner-images/issues/8932#issuecomment-1836013315) +PYFIX_FILE=/usr/local/Cellar/glib/2.78.1/share/glib-2.0/codegen/utils.py +if [ -f "${PYFIX_FILE}" ] ; then + echo "Fixing ${PYFIX_FILE}" + python3 -m pip install --upgrade pip + python3 -m pip install looseversion + + sed -i '' "s/distutils.version/looseversion/" "${PYFIX_FILE}" +fi + +echo "Installing meson and tomli" +python3 -m pip install --break-system-packages --user meson==1.7.0 tomli==2.2.1 + +# dbg +command -v python3 +python3 --version +python3 -m pip freeze diff --git a/.github/workflows/scripts/prerequisites-native.sh b/.github/workflows/scripts/prerequisites-native.sh new file mode 100755 index 0000000000..ba65cc267b --- /dev/null +++ b/.github/workflows/scripts/prerequisites-native.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export DEBIAN_FRONTEND="noninteractive" + +apt-get update -y -q +apt-get install -y -q --no-install-recommends \ + build-essential \ + git \ + libgcrypt-dev \ + libglib2.0-dev \ + libpixman-1-dev \ + libsdl2-dev \ + libslirp-dev \ + ninja-build \ + python3-pip \ + wget \ + zlib1g-dev \ +&& : + +# Even though ./configure installs meson, just specify a version +/usr/bin/pip3 install meson==1.7.0 tomli==2.2.1 diff --git a/.github/workflows/scripts/prerequisites-old.sh b/.github/workflows/scripts/prerequisites-old.sh new file mode 100755 index 0000000000..0d9dc55ca0 --- /dev/null +++ b/.github/workflows/scripts/prerequisites-old.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# ubuntu 18.04 for old glibc + +set -euo pipefail + +export DEBIAN_FRONTEND="noninteractive" + +apt-get update -y -q +apt-get install -y -q --no-install-recommends software-properties-common +add-apt-repository ppa:deadsnakes/ppa +apt-get update -y -q +apt-get install -y -q --fix-missing --no-install-recommends python3.8 python3.8-venv python3.8-distutils python3.8-lib2to3 +update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 100 +command -v python3 +python3 -V +cd /usr/local/src +python3 -m venv envdir +source envdir/bin/activate +python3 -m pip install meson tomli + +apt-get install -y -q --no-install-recommends \ + build-essential \ + git \ + libgcrypt-dev \ + libglib2.0-dev \ + libpixman-1-dev \ + libsdl2-dev \ + ninja-build \ + wget \ + zlib1g-dev \ +&& : + +# libslirp +libslirp_VER=v4.1.0 +libslirp_DIST=libslirp-${libslirp_VER}.tar.bz2 +libslirp_SHA256=f423c54c96eb3310bf9519abc8c9c11539801a14327b48b8f36acf407584bbd1 +wget --no-verbose https://gitlab.com/qemu-project/libslirp/-/archive/${libslirp_VER}/${libslirp_DIST} +echo "${libslirp_SHA256} *${libslirp_DIST}" | sha256sum --check --strict - +tar -xf ${libslirp_DIST} +cd libslirp-${libslirp_VER} +meson build +ninja -C build install +rm -rf ${libslirp_DIST} libslirp-${libslirp_VER}