Skip to content

Commit

Permalink
build: impl build args in bazelrc (#58)
Browse files Browse the repository at this point in the history
* build: impl build args in bazelrc

fix libstdc++ linking issue on centos7 with devtoolset

* build: with 'release-static' profile

update pack_zetasql_parser.sh

* ci: upgrade actions version for deprecation

* dbg

* fix pack_zetasql.sh
  • Loading branch information
aceforeverd authored Sep 24, 2024
1 parent 18146d1 commit 086aaf1
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 46 deletions.
12 changes: 11 additions & 1 deletion .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,14 @@
# c++1z was used prior to c++17 being standardized, and is therefore more
# widely accepted by compilers. This may lead to strange behavior or compiler
# errors in earlier compilers.
build --cxxopt="-std=c++1z"
build --cxxopt="-std=c++1z" --sandbox_debug --incompatible_linkopts_to_linklibs

# --config=release
build:release -c opt

# --config=static
build:static --dynamic_mode='off' --features=-supports_dynamic_linker

# --config=release-static
# fully static profile
build:release-static --config=release --config=static
2 changes: 1 addition & 1 deletion .bazelversion
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0.0
3.7.2
24 changes: 12 additions & 12 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ jobs:
runs-on: ubuntu-latest
container:
image: ghcr.io/aceforeverd/hybridsql-base:0.4.0
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v2

Expand All @@ -30,6 +32,9 @@ jobs:
- name: Install Java
run: |
sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
yum install -y java-1.8.0-openjdk-devel
- name: build zetasql parser
Expand All @@ -53,7 +58,7 @@ jobs:

- name: Upload Artifacts
if: ${{ github.event_name == 'push' }}
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
path: libzetasql-*.tar.gz
name: release-artifacts
Expand All @@ -69,7 +74,7 @@ jobs:
DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl build-essential unzip python3-dev git
update-alternatives --install /usr/bin/python python /usr/bin/python3 100
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- uses: actions/setup-java@v3
with:
Expand Down Expand Up @@ -101,18 +106,16 @@ jobs:

- name: Upload Artifacts
if: ${{ github.event_name == 'push' }}
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
path: libzetasql-*.tar.gz
name: release-artifacts

macos-build:
runs-on: macos-12
timeout-minutes: 120
env:
bazel_version: '3.7.2'
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

# xcode 14.0.1 comes with macOS SDK 12.3, ref https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md#installed-sdks
# so pre-compile zetasql requires macOS >= 12.3
Expand All @@ -130,9 +133,6 @@ jobs:
run: |
sdkmanager tools "platforms;android-27" "platforms;android-28" "platforms;android-29" "platforms;android-30"
- name: update bazel version
run: echo ${{ env.bazel_version }} > .bazelversion

- name: setup python3.9
uses: actions/setup-python@v4
with:
Expand Down Expand Up @@ -171,7 +171,7 @@ jobs:
- name: Upload Artifacts
if: ${{ github.event_name == 'push' }}
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
path: libzetasql-*.tar.gz
name: release-artifacts
Expand All @@ -183,7 +183,7 @@ jobs:
timeout-minutes: 120
steps:
- name: Download Release Artifacts
uses: actions/download-artifact@v2
uses: actions/download-artifact@v4
with:
name: release-artifacts

Expand All @@ -193,7 +193,7 @@ jobs:
- name: Release
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@v2
with:
files: |
libzetasql-*.tar.gz
Expand Down
61 changes: 50 additions & 11 deletions build_zetasql_parser.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,6 @@ set -eE
pushd "$(dirname "$0")"
pushd "$(git rev-parse --show-toplevel)"

if grep -q centos /etc/os-release ; then
# for thoese using rhel devtoolset
export BAZEL_LINKOPTS='-static-libstdc++:-lm'
export BAZEL_LINKLIBS='-l%:libstdc++.a'
fi

if [[ $(arch) = 'aarch64' ]]; then
git checkout .
# need upgrade abseil and bazel to compile on aarch64
Expand All @@ -35,21 +29,66 @@ fi

echo "build with python: $(python -V), python3: $(python3 -V)"

#=== FUNCTION ================================================================
# NAME: usage
# DESCRIPTION: Display usage information.
#===============================================================================
function usage ()
{
echo "Usage : $0 [options] [--]
Options:
-c Build profile. static|release|release-static, default static
-h|help Display this message"

} # ---------- end of function usage ----------

#-----------------------------------------------------------------------
# Handle command line arguments
#-----------------------------------------------------------------------

# NOTE: default to static profile: compile all as archive but not define `NDEBUG`
# since pre-compiled lib may used in Release/Debug profile in OpenMLDB
#
# acceptable values:
# - release: `-O2 -DNDEBUG`, shared libs
# - static: static libs
# - release-static: `-O2 -DNDEBUG`, static libs
PROFILE=static
ACCEPT_PROFILES=(static release release-static)

while getopts ":hc:" opt
do
case $opt in

c ) PROFILE=$OPTARG ;;

h|help ) usage; exit 0 ;;

* ) echo -e "\n Option does not exist : $OPTARG\n"
usage; exit 1 ;;

esac # --- end of case ---
done
shift $((OPTIND-1))

if [[ ! " ${ACCEPT_PROFILES[*]} " =~ " ${PROFILE} " ]] ; then
echo "profile named $PROFILE not found in accept profile list: ${ACCEPT_PROFILES[*]}"
exit 1
fi

TARGET='//zetasql/parser/...'
BUILD_ARGV=(--features=-supports_dynamic_linker --sandbox_debug)
BUILD_ARGV=(--config="$PROFILE")

bazel build "$TARGET" "${BUILD_ARGV[@]}"
bazel test "$TARGET" "${BUILD_ARGV[@]}"

# explicitly build dependencies into static library
bazel clean
# bazel clean
bazel query "deps(//zetasql/parser:parser)" | grep //zetasql | xargs bazel build "${BUILD_ARGV[@]}"
bazel build "@com_googleapis_googleapis//:all" "${BUILD_ARGV[@]}"
bazel query "@com_google_file_based_test_driver//..." | xargs bazel build "${BUILD_ARGV[@]}"
bazel build "@com_googlesource_code_re2//:re2" "${BUILD_ARGV[@]}"

unset BAZEL_LINKLIBS
unset BAZEL_LINKOPTS

popd
popd
52 changes: 31 additions & 21 deletions pack_zetasql.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ function usage ()
Options:
-h Display this message
-d Linux distribution name, e.g centos, ubuntu, default empty
-p Pick PIC libraries only, OFF by default
-i Request install to given directory after pack"

} # ---------- end of function usage ----------
Expand All @@ -26,15 +27,18 @@ function usage ()
INSTALL_DIR=
# linux distribution name
DISTRO=
PICK_PIC=OFF

while getopts ":hi:d:" opt
while getopts ":hi:d:p:" opt
do
case $opt in

h ) usage; exit 0 ;;

d ) DISTRO=$OPTARG ;;

p ) PICK_PIC=$OPTARG ;;

i )
INSTALL_DIR=$OPTARG
mkdir -p "$INSTALL_DIR"
Expand All @@ -48,6 +52,12 @@ do
done
shift $((OPTIND-1))

if [[ "$PICK_PIC" == 'ON' ]] ; then
LIB_PATTERN='*.pic.a'
else
LIB_PATTERN='*.a'
fi

pushd "$(dirname "$0")"
pushd "$(git rev-parse --show-toplevel)"

Expand All @@ -68,15 +78,18 @@ install_lib() {
local file
file=$1
local libname
libname=lib$(echo "$file" | tr '/' '_' | sed -e 's/lib//')
libname=lib$(echo "$file" | tr '/' '_' | sed -e 's/lib//' )
if [[ "$PICK_PIC" == 'ON' ]]; then
libname=$(echo "$libname" | sed -e 's/\.pic\.a$/.a/')
fi

if [[ "$OSTYPE" == "linux-gnu"* ]]
then
INSTALL_BIN="install"
else
INSTALL_BIN="ginstall"
fi
${INSTALL_BIN} -D "$file" "$ROOT/tmp-lib/$libname"
${INSTALL_BIN} -Dv "$file" "$ROOT/tmp-lib/$libname"
}

install_gen_include_file() {
Expand All @@ -85,27 +98,24 @@ install_gen_include_file() {
local outfile
outfile=$(echo "$file" | sed -e 's/^.*proto\///')

if [[ "$OSTYPE" == "linux-gnu"* ]]
then
INSTALL_BIN="install"
else
INSTALL_BIN="ginstall"
fi
${INSTALL_BIN} -D "$file" "$PREFIX/include/$outfile"
${INSTALL_BIN} -Dv "$file" "$PREFIX/include/$outfile"
}

install_external_lib() {
local file
file=$1
local libname
libname=$(basename "$file")
if [[ "$PICK_PIC" == 'ON' ]]; then
libname=$(echo "$libname" | sed -e 's/\.pic\.a$/.a/')
fi
if [[ "$OSTYPE" == "linux-gnu"* ]]
then
INSTALL_BIN="install"
else
INSTALL_BIN="ginstall"
fi
${INSTALL_BIN} -D "$file" "$PREFIX/lib/$libname"
${INSTALL_BIN} -Dv "$file" "$PREFIX/lib/$libname"
}

export -f install_gen_include_file
Expand All @@ -121,38 +131,38 @@ else
fi

pushd bazel-bin/
find zetasql -type f -iname '*.a' -exec bash -c 'install_lib $0' {} \;
find zetasql -type f -iname "$LIB_PATTERN" -exec bash -c 'install_lib $0' {} \;

# external lib headers
pushd "$(realpath .)/../../../../../external/com_googlesource_code_re2"
find re2 -iname "*.h" -exec ${INSTALL_BIN} -D {} "$PREFIX"/include/{} \;
find re2 -iname "*.h" -exec ${INSTALL_BIN} -Dv {} "$PREFIX"/include/{} \;
popd

pushd "$(realpath .)/../../../../../external/com_googleapis_googleapis"
find google -iname "*.h" -exec ${INSTALL_BIN} -D {} "$PREFIX"/include/{} \;
find google -iname "*.h" -exec ${INSTALL_BIN} -Dv {} "$PREFIX"/include/{} \;
popd

pushd "$(realpath .)/../../../../../external/com_google_file_based_test_driver"
find file_based_test_driver -iname "*.h" -exec ${INSTALL_BIN} -D {} "$PREFIX"/include/{} \;
find file_based_test_driver -iname "*.h" -exec ${INSTALL_BIN} -Dv {} "$PREFIX"/include/{} \;
popd

# external lib
pushd external

find icu -type f -iname '*.a' -exec bash -c 'install_external_lib $0' {} \;
find com_googlesource_code_re2 -type f -iname '*.a' -exec bash -c 'install_external_lib $0' {} \;
find com_googleapis_googleapis -type f -iname '*.a' -exec bash -c 'install_external_lib $0' {} \;
find com_google_file_based_test_driver -type f -iname '*.a' -exec bash -c 'install_external_lib $0' {} \;
find icu -type f -iname "$LIB_PATTERN" -exec bash -c 'install_external_lib $0' {} \;
find com_googlesource_code_re2 -type f -iname "$LIB_PATTERN" -exec bash -c 'install_external_lib $0' {} \;
find com_googleapis_googleapis -type f -iname "$LIB_PATTERN" -exec bash -c 'install_external_lib $0' {} \;
find com_google_file_based_test_driver -type f -iname "$LIB_PATTERN" -exec bash -c 'install_external_lib $0' {} \;

popd

# zetasql generated files: protobuf & template generated files
find zetasql -type f -iname "*.h" -exec ${INSTALL_BIN} -D {} "$PREFIX"/include/{} \;
find zetasql -type f -iname "*.h" -exec ${INSTALL_BIN} -Dv {} "$PREFIX"/include/{} \;
find zetasql -iregex ".*/_virtual_includes/.*\.h\$" -exec bash -c 'install_gen_include_file $0' {} \;
popd # bazel-bin

# header files from source
find zetasql -type f -iname "*.h" -exec ${INSTALL_BIN} -D {} "$PREFIX"/include/{} \;
find zetasql -type f -iname "*.h" -exec ${INSTALL_BIN} -Dv {} "$PREFIX"/include/{} \;


if [[ "$OSTYPE" == "linux-gnu"* ]]
Expand Down

0 comments on commit 086aaf1

Please sign in to comment.