Use Zig to cross-compile QuickJS for the JVM

.github/workflows/build.yaml

@@ -5,28 +5,21 @@ on:
  push:
  branches:
  - '**'
- tags-ignore:
+ tags:
  - '**'
 
 env:
  GRADLE_OPTS: "-Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dkotlin.incremental=false"
 
 jobs:
- build:
+ jvm-test:
  strategy:
  fail-fast: false
  matrix:
- os: [ubuntu-latest]
- arch: [amd64]
- cmake-arch: [x86_64]
- include:
- - os: macOS-latest
- arch: x86_64
- cmake-arch: x86_64
- - os: macOS-latest
- arch: aarch64
- cmake-arch: arm64
- # TODO: build on 'windows-latest'
+ os:
+ - ubuntu-latest
+ - macos-latest
+ # TODO windows
 
  runs-on: ${{ matrix.os }}
 
@@ -38,32 +31,21 @@ jobs:
  distribution: 'zulu'
  java-version: 11
 
- # Clang is seemingly already installed on Ubuntu but fails to build the project with errors
- # like "file not found" for `#include <string>`. Explicitly (re?)installing fixes things.
- - run: sudo apt-get install clang
+ - run: brew install zig
+  if: matrix.os == 'macos-latest'
+ - run: sudo snap install zig --classic --beta
  if: matrix.os == 'ubuntu-latest'
 
- - name: Build native library
+ - name: Build native libraries
  run: |
- mkdir -p build/jni/${{ matrix.arch }}/
- mkdir -p zipline/src/jvmMain/resources/jni/${{ matrix.arch }}/
- cmake -S zipline/src/jvmMain/ -B build/jni/${{ matrix.arch }}/ \
- -DQUICKJS_VERSION="$(cat zipline/native/quickjs/VERSION)" \
- -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.cmake-arch }}
- cmake --build build/jni/${{ matrix.arch }}/ --verbose
- cp -v build/jni/${{ matrix.arch }}/libquickjs.* zipline/src/jvmMain/resources/jni/${{ matrix.arch }}/
+ cd zipline
+ mkdir -p src/jvmMain/resources/jni
+ zig build
+ cp -Rv zig-out/* src/jvmMain/resources/jni/
 
  - run: ./gradlew build --stacktrace
- # TODO: configure builds to run on the same architecture they produce so we can test what we've built.
- if: matrix.arch == 'amd64'
 
- - uses: actions/upload-artifact@v2
- with:
- name: jvm-native-libraries
- path: zipline/src/jvmMain/resources/*
- if-no-files-found: error
-
- android-emulator:
+ android-test:
  # We build on a Mac to get hardware acceleration for the Android emulator.
  runs-on: macos-latest
 
@@ -85,8 +67,8 @@ jobs:
  publish:
  runs-on: ubuntu-latest
  needs:
- - build
- - android-emulator
+ - jvm-test
+ - android-test
 
  steps:
  - uses: actions/checkout@v2
@@ -95,10 +77,17 @@ jobs:
  distribution: 'zulu'
  java-version: 11
 
- - uses: actions/download-artifact@v2
- - run: |
- mkdir -p zipline/src/jvmMain/resources/
- cp -av jvm-native-libraries/* zipline/src/jvmMain/resources/
+ - run: brew install zig
+ if: matrix.os == 'macos-latest'
+ - run: sudo snap install zig --classic --beta
+ if: matrix.os == 'ubuntu-latest'
+
+ - name: Build native libraries
+ run: |
+ cd zipline
+ mkdir -p src/jvmMain/resources/jni
+ zig build
+ cp -Rv zig-out/* src/jvmMain/resources/jni/
 
  - run: ./gradlew assemble dokkaHtml
 
@@ -115,13 +104,13 @@ jobs:
  if-no-files-found: error
 
  - run: ./gradlew publish
- if: ${{ github.ref == 'refs/heads/trunk' && github.repository == 'cashapp/zipline' }}
+ if: ${{ (github.ref == 'refs/heads/trunk' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cashapp/zipline' }}
  env:
  ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
  ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
  ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.ARTIFACT_SIGNING_PRIVATE_KEY }}
 
- - name: Deploy docs to website
+ - name: Deploy snapshot docs to website
  if: ${{ github.ref == 'refs/heads/trunk' && github.repository == 'cashapp/zipline' }}
  uses: JamesIves/github-pages-deploy-action@releases/v3
  with:
@@ -130,3 +119,13 @@ jobs:
  FOLDER: zipline/build/dokka/html
  TARGET_FOLDER: docs/latest/
  CLEAN: true
+
+ - name: Deploy release docs to website
+ if: ${{ startsWith(github.ref, 'refs/tags/') && github.repository == 'cashapp/zipline' }}
+ uses: JamesIves/github-pages-deploy-action@releases/v3
+ with:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ BRANCH: site
+ FOLDER: zipline/build/dokka/html
+ TARGET_FOLDER: docs/0.x/
+ CLEAN: true

.gitignore

@@ -11,5 +11,9 @@ jacoco.exec
 .externalNativeBuild
 .cxx
 
+# Zig
+zig-cache
+zig-out
+
 # Release
 docs/0.x

zipline/build.zig

@@ -0,0 +1,102 @@
+const std = @import("std");
+
+pub fn build(b: *std.build.Builder) !void {
+ const mode = b.standardReleaseOptions();
+
+ var version_buf: [10]u8 = undefined;
+ const version = try readVersionFile(&version_buf);
+
+ const linux_x86 = b.addSharedLibrary("quickjs", null, .unversioned);
+ try commonQuickJsSetup(linux_x86, version);
+ linux_x86.setBuildMode(mode);
+ linux_x86.override_dest_dir = std.build.InstallDir { .custom = "amd64" };
+ linux_x86.setTarget(std.zig.CrossTarget{
+ .cpu_arch = .x86_64,
+ .os_tag = .linux,
+ .abi = .gnu,
+ });
+ linux_x86.install();
+
+ const macos_arm = b.addSharedLibrary("quickjs", null, .unversioned);
+ try commonQuickJsSetup(macos_arm, version);
+ macos_arm.setBuildMode(mode);
+ macos_arm.override_dest_dir = std.build.InstallDir { .custom = "aarch64" };
+ macos_arm.setTarget(std.zig.CrossTarget{
+ .cpu_arch = .aarch64,
+ .os_tag = .macos,
+ .abi = .gnu,
+ });
+ macos_arm.install();
+
+ const macos_x86 = b.addSharedLibrary("quickjs", null, .unversioned);
+ try commonQuickJsSetup(macos_x86, version);
+ macos_x86.setBuildMode(mode);
+ macos_x86.override_dest_dir = std.build.InstallDir { .custom = "x86_64" };
+ macos_x86.setTarget(std.zig.CrossTarget{
+ .cpu_arch = .x86_64,
+ .os_tag = .macos,
+ .abi = .gnu,
+ });
+ macos_x86.install();
+}
+
+fn commonQuickJsSetup(quickjs: *std.build.LibExeObjStep, version: []const u8) !void {
+ var quoted_version_buf: [12]u8 = undefined;
+ const quoted_version = try std.fmt.bufPrint(&quoted_version_buf, "\"{s}\"", .{ version });
+ quickjs.defineCMacro("CONFIG_VERSION", quoted_version);
+
+ // Platform-independent code (i.e., relative jumps) to be safe.
+ quickjs.force_pic = true;
+
+ // Add the JDK's include/ headers.
+ const java_home = std.os.getenv("JAVA_HOME").?;
+ const java_include = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ java_home, "include" });
+ quickjs.addIncludeDir(java_include);
+
+ // Walk the include/ directory for any child dirs (usually platform specific) and add them too.
+ const java_include_dir = try std.fs.cwd().openDir(java_include, .{ .iterate = true });
+ var jdk_walker = try java_include_dir.walk(std.testing.allocator);
+ defer jdk_walker.deinit();
+ while (try jdk_walker.next()) |entry| {
+ switch (entry.kind) {
+ .Directory => {
+ const include_subdir = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ java_include, entry.path });
+ quickjs.addIncludeDir(include_subdir);
+ },
+ else => {},
+ }
+ }
+
+ quickjs.linkLibC();
+ quickjs.addCSourceFiles(&.{
+ "native/quickjs/cutils.c",
+ "native/quickjs/libregexp.c",
+ "native/quickjs/libunicode.c",
+ "native/quickjs/quickjs.c",
+ }, &.{
+ "-std=gnu99",
+ });
+
+ quickjs.linkLibCpp();
+ quickjs.addCSourceFiles(&.{
+ "native/Context.cpp",
+ "native/ExceptionThrowers.cpp",
+ "native/InboundCallChannel.cpp",
+ "native/OutboundCallChannel.cpp",
+ }, &.{
+ "-std=c++11",
+ });
+}
+
+fn readVersionFile(version_buf: []u8) ![]u8 {
+ const version_file = try std.fs.cwd().openFile(
+ "native/quickjs/VERSION",
+ .{ .read = true },
+ );
+ defer version_file.close();
+
+ var version_file_reader = std.io.bufferedReader(version_file.reader());
+ var version_file_stream = version_file_reader.reader();
+ const version = try version_file_stream.readUntilDelimiterOrEof(version_buf, '\n');
+ return version.?;
+}