From 81604b1764b8685d3dbb9f4a0a2d150d8de13ad1 Mon Sep 17 00:00:00 2001 From: Jake Wharton Date: Fri, 11 Feb 2022 14:41:37 -0500 Subject: [PATCH] Use Zig to cross-compile QuickJS for the JVM No more CMake. No more dealing with building on multiple OSs just to release. Everything can be built on any OS thanks to Zig's cross-compilation. We still test on Mac and Linux (x86 only for now). --- .github/workflows/build.yaml | 81 +++++++++-------- .github/workflows/release.yaml | 138 ----------------------------- .gitignore | 4 + zipline/build.zig | 102 +++++++++++++++++++++ zipline/src/jvmMain/CMakeLists.txt | 21 ----- 5 files changed, 146 insertions(+), 200 deletions(-) delete mode 100644 .github/workflows/release.yaml create mode 100644 zipline/build.zig delete mode 100644 zipline/src/jvmMain/CMakeLists.txt diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index df98c77c04..7c5c122930 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,28 +5,21 @@ on: push: branches: - '**' - tags-ignore: + tags: - '**' env: GRADLE_OPTS: "-Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dkotlin.incremental=false" jobs: - build: + jvm-test: strategy: fail-fast: false matrix: - os: [ubuntu-latest] - arch: [amd64] - cmake-arch: [x86_64] - include: - - os: macOS-latest - arch: x86_64 - cmake-arch: x86_64 - - os: macOS-latest - arch: aarch64 - cmake-arch: arm64 - # TODO: build on 'windows-latest' + os: + - ubuntu-latest + - macos-latest + # TODO windows runs-on: ${{ matrix.os }} @@ -38,32 +31,21 @@ jobs: distribution: 'zulu' java-version: 11 - # Clang is seemingly already installed on Ubuntu but fails to build the project with errors - # like "file not found" for `#include `. Explicitly (re?)installing fixes things. - - run: sudo apt-get install clang + - run: brew install zig + if: matrix.os == 'macos-latest' + - run: sudo snap install zig --classic --beta if: matrix.os == 'ubuntu-latest' - - name: Build native library + - name: Build native libraries run: | - mkdir -p build/jni/${{ matrix.arch }}/ - mkdir -p zipline/src/jvmMain/resources/jni/${{ matrix.arch }}/ - cmake -S zipline/src/jvmMain/ -B build/jni/${{ matrix.arch }}/ \ - -DQUICKJS_VERSION="$(cat zipline/native/quickjs/VERSION)" \ - -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.cmake-arch }} - cmake --build build/jni/${{ matrix.arch }}/ --verbose - cp -v build/jni/${{ matrix.arch }}/libquickjs.* zipline/src/jvmMain/resources/jni/${{ matrix.arch }}/ + cd zipline + mkdir -p src/jvmMain/resources/jni + zig build + cp -Rv zig-out/* src/jvmMain/resources/jni/ - run: ./gradlew build --stacktrace - # TODO: configure builds to run on the same architecture they produce so we can test what we've built. - if: matrix.arch == 'amd64' - - uses: actions/upload-artifact@v2 - with: - name: jvm-native-libraries - path: zipline/src/jvmMain/resources/* - if-no-files-found: error - - android-emulator: + android-test: # We build on a Mac to get hardware acceleration for the Android emulator. runs-on: macos-latest @@ -85,8 +67,8 @@ jobs: publish: runs-on: ubuntu-latest needs: - - build - - android-emulator + - jvm-test + - android-test steps: - uses: actions/checkout@v2 @@ -95,10 +77,17 @@ jobs: distribution: 'zulu' java-version: 11 - - uses: actions/download-artifact@v2 - - run: | - mkdir -p zipline/src/jvmMain/resources/ - cp -av jvm-native-libraries/* zipline/src/jvmMain/resources/ + - run: brew install zig + if: matrix.os == 'macos-latest' + - run: sudo snap install zig --classic --beta + if: matrix.os == 'ubuntu-latest' + + - name: Build native libraries + run: | + cd zipline + mkdir -p src/jvmMain/resources/jni + zig build + cp -Rv zig-out/* src/jvmMain/resources/jni/ - run: ./gradlew assemble dokkaHtml @@ -115,13 +104,13 @@ jobs: if-no-files-found: error - run: ./gradlew publish - if: ${{ github.ref == 'refs/heads/trunk' && github.repository == 'cashapp/zipline' }} + if: ${{ (github.ref == 'refs/heads/trunk' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cashapp/zipline' }} env: ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.ARTIFACT_SIGNING_PRIVATE_KEY }} - - name: Deploy docs to website + - name: Deploy snapshot docs to website if: ${{ github.ref == 'refs/heads/trunk' && github.repository == 'cashapp/zipline' }} uses: JamesIves/github-pages-deploy-action@releases/v3 with: @@ -130,3 +119,13 @@ jobs: FOLDER: zipline/build/dokka/html TARGET_FOLDER: docs/latest/ CLEAN: true + + - name: Deploy release docs to website + if: ${{ startsWith(github.ref, 'refs/tags/') && github.repository == 'cashapp/zipline' }} + uses: JamesIves/github-pages-deploy-action@releases/v3 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: site + FOLDER: zipline/build/dokka/html + TARGET_FOLDER: docs/0.x/ + CLEAN: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 7889157026..0000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,138 +0,0 @@ -name: release - -on: - push: - tags: - - '**' - -env: - GRADLE_OPTS: "-Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dkotlin.incremental=false" - -jobs: - build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - arch: [amd64] - cmake-arch: [x86_64] - include: - - os: macOS-latest - arch: x86_64 - cmake-arch: x86_64 - - os: macOS-latest - arch: aarch64 - cmake-arch: arm64 - # TODO: build on 'windows-latest' - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - - uses: gradle/wrapper-validation-action@v1 - - uses: actions/setup-java@v2 - with: - distribution: 'zulu' - java-version: 11 - - # Clang is seemingly already installed on Ubuntu but fails to build the project with errors - # like "file not found" for `#include `. Explicitly (re?)installing fixes things. - - run: sudo apt-get install clang - if: matrix.os == 'ubuntu-latest' - - - name: Build native library - run: | - mkdir -p build/jni/${{ matrix.arch }}/ - mkdir -p zipline/src/jvmMain/resources/jni/${{ matrix.arch }}/ - cmake -S zipline/src/jvmMain/ -B build/jni/${{ matrix.arch }}/ \ - -DQUICKJS_VERSION="$(cat zipline/native/quickjs/VERSION)" \ - -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.cmake-arch }} - cmake --build build/jni/${{ matrix.arch }}/ --verbose - cp -v build/jni/${{ matrix.arch }}/libquickjs.* zipline/src/jvmMain/resources/jni/${{ matrix.arch }}/ - - - run: ./gradlew build - # TODO: configure builds to run on the same architecture they produce so we can test what we've built. - if: matrix.arch == 'amd64' - - - uses: actions/upload-artifact@v2 - with: - name: jvm-native-libraries - path: zipline/src/jvmMain/resources/* - if-no-files-found: error - - android-emulator: - # We build on a Mac to get hardware acceleration for the Android emulator. - runs-on: macos-latest - - steps: - - uses: actions/checkout@v2 - - uses: gradle/wrapper-validation-action@v1 - - uses: actions/setup-java@v2 - with: - distribution: 'zulu' - java-version: 11 - - - run: ./gradlew assembleAndroidTest - - - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 29 - script: ./gradlew connectedCheck - - publish: - runs-on: ubuntu-latest - needs: - - build - - android-emulator - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v2 - with: - distribution: 'zulu' - java-version: 11 - - - uses: actions/download-artifact@v2 - - run: | - mkdir -p zipline/src/jvmMain/resources/ - cp -av jvm-native-libraries/* zipline/src/jvmMain/resources/ - - - run: ./gradlew assemble dokkaHtml - - - uses: actions/upload-artifact@v2 - with: - name: zipline-jvm.jar - path: zipline/build/libs/zipline-jvm-*.jar - if-no-files-found: error - - - uses: actions/upload-artifact@v2 - with: - name: zipline-android.aar - path: zipline/build/outputs/aar/*-release.aar - if-no-files-found: error - - - run: ./gradlew publish - if: ${{ github.ref == 'refs/heads/trunk' && github.repository == 'cashapp/zipline' }} - env: - ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} - ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} - ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.ARTIFACT_SIGNING_PRIVATE_KEY }} - - - uses: ffurrer2/extract-release-notes@v1 - id: release_notes - - - uses: softprops/action-gh-release@v1 - with: - body: ${{ steps.release_notes.outputs.release_notes }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Deploy docs to website - if: ${{ github.ref == 'refs/heads/trunk' && github.repository == 'cashapp/zipline' }} - uses: JamesIves/github-pages-deploy-action@releases/v3 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: site - FOLDER: zipline/build/dokka/html - TARGET_FOLDER: docs/0.x/ - CLEAN: true diff --git a/.gitignore b/.gitignore index 917504b923..f957b106fc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,9 @@ jacoco.exec .externalNativeBuild .cxx +# Zig +zig-cache +zig-out + # Release docs/0.x diff --git a/zipline/build.zig b/zipline/build.zig new file mode 100644 index 0000000000..b4922891f7 --- /dev/null +++ b/zipline/build.zig @@ -0,0 +1,102 @@ +const std = @import("std"); + +pub fn build(b: *std.build.Builder) !void { + const mode = b.standardReleaseOptions(); + + var version_buf: [10]u8 = undefined; + const version = try readVersionFile(&version_buf); + + const linux_x86 = b.addSharedLibrary("quickjs", null, .unversioned); + try commonQuickJsSetup(linux_x86, version); + linux_x86.setBuildMode(mode); + linux_x86.override_dest_dir = std.build.InstallDir { .custom = "amd64" }; + linux_x86.setTarget(std.zig.CrossTarget{ + .cpu_arch = .x86_64, + .os_tag = .linux, + .abi = .gnu, + }); + linux_x86.install(); + + const macos_arm = b.addSharedLibrary("quickjs", null, .unversioned); + try commonQuickJsSetup(macos_arm, version); + macos_arm.setBuildMode(mode); + macos_arm.override_dest_dir = std.build.InstallDir { .custom = "aarch64" }; + macos_arm.setTarget(std.zig.CrossTarget{ + .cpu_arch = .aarch64, + .os_tag = .macos, + .abi = .gnu, + }); + macos_arm.install(); + + const macos_x86 = b.addSharedLibrary("quickjs", null, .unversioned); + try commonQuickJsSetup(macos_x86, version); + macos_x86.setBuildMode(mode); + macos_x86.override_dest_dir = std.build.InstallDir { .custom = "x86_64" }; + macos_x86.setTarget(std.zig.CrossTarget{ + .cpu_arch = .x86_64, + .os_tag = .macos, + .abi = .gnu, + }); + macos_x86.install(); +} + +fn commonQuickJsSetup(quickjs: *std.build.LibExeObjStep, version: []const u8) !void { + var quoted_version_buf: [12]u8 = undefined; + const quoted_version = try std.fmt.bufPrint("ed_version_buf, "\"{s}\"", .{ version }); + quickjs.defineCMacro("CONFIG_VERSION", quoted_version); + + // Platform-independent code (i.e., relative jumps) to be safe. + quickjs.force_pic = true; + + // Add the JDK's include/ headers. + const java_home = std.os.getenv("JAVA_HOME").?; + const java_include = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ java_home, "include" }); + quickjs.addIncludeDir(java_include); + + // Walk the include/ directory for any child dirs (usually platform specific) and add them too. + const java_include_dir = try std.fs.cwd().openDir(java_include, .{ .iterate = true }); + var jdk_walker = try java_include_dir.walk(std.testing.allocator); + defer jdk_walker.deinit(); + while (try jdk_walker.next()) |entry| { + switch (entry.kind) { + .Directory => { + const include_subdir = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ java_include, entry.path }); + quickjs.addIncludeDir(include_subdir); + }, + else => {}, + } + } + + quickjs.linkLibC(); + quickjs.addCSourceFiles(&.{ + "native/quickjs/cutils.c", + "native/quickjs/libregexp.c", + "native/quickjs/libunicode.c", + "native/quickjs/quickjs.c", + }, &.{ + "-std=gnu99", + }); + + quickjs.linkLibCpp(); + quickjs.addCSourceFiles(&.{ + "native/Context.cpp", + "native/ExceptionThrowers.cpp", + "native/InboundCallChannel.cpp", + "native/OutboundCallChannel.cpp", + }, &.{ + "-std=c++11", + }); +} + +fn readVersionFile(version_buf: []u8) ![]u8 { + const version_file = try std.fs.cwd().openFile( + "native/quickjs/VERSION", + .{ .read = true }, + ); + defer version_file.close(); + + var version_file_reader = std.io.bufferedReader(version_file.reader()); + var version_file_stream = version_file_reader.reader(); + const version = try version_file_stream.readUntilDelimiterOrEof(version_buf, '\n'); + return version.?; +} diff --git a/zipline/src/jvmMain/CMakeLists.txt b/zipline/src/jvmMain/CMakeLists.txt deleted file mode 100644 index bb4eef8923..0000000000 --- a/zipline/src/jvmMain/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -cmake_minimum_required(VERSION 3.4.1) -project(quickjs) - -set(CMAKE_C_STANDARD 99) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_CXX_COMPILER "clang++") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - -set(JAVA_AWT_LIBRARY NotNeeded) -set(JAVA_AWT_INCLUDE_PATH NotNeeded) -find_package(JNI REQUIRED) -include_directories(${JNI_INCLUDE_DIRS}) - -file(GLOB_RECURSE sources "../../native/*.c" "../../native/*.cpp") - -add_library(quickjs SHARED ${sources}) -target_compile_definitions(quickjs PUBLIC CONFIG_VERSION="${QUICKJS_VERSION}") - -target_link_libraries(quickjs)