Merge pull request #1033 from permaweb/feat/fix-image-pipelines

feat/fix-image-pipelines

.github/workflows/devnet-ecr-push.yml

@@ -0,0 +1,135 @@
+name: 🏗️ Build and Push Docker images to devnet ECR
+
+on:
+ pull_request:
+ branches:
+ - '*'
+
+jobs: 
+ build-and-push-cu:
+ runs-on: ubuntu-latest
+ env:
+ REPOSITORY: devnet-ecr
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+
+ - name: Extract shortened commit hash
+ id: vars
+ run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+ - name: Log in to Amazon ECR
+ id: login-devnet-ecr
+ uses: aws-actions/amazon-ecr-login@v1
+ with:
+ mask-password: true
+ skip-logout: true
+ registry-type: private
+ registries: ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_AO_DEVNET }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_AO_DEVNET }}
+ AWS_REGION: ${{ secrets.AWS_REGION_AO_DEVNET }}
+
+ - name: Build and push CU Docker image
+ run: |
+ docker build -t ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-cu-$COMMIT_HASH -f servers/cu/Dockerfile servers/cu/
+ docker push ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-cu-$COMMIT_HASH
+
+ build-and-push-mu:
+ env:
+ REPOSITORY: devnet-ecr
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+
+ - name: Extract shortened commit hash
+ id: vars
+ run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+ - name: Log in to Amazon ECR
+ id: login-devnet-ecr
+ uses: aws-actions/amazon-ecr-login@v1
+ with:
+ mask-password: true
+ skip-logout: true
+ registry-type: private
+ registries: ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_AO_DEVNET }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_AO_DEVNET }}
+ AWS_REGION: ${{ secrets.AWS_REGION_AO_DEVNET }}
+
+ - name: Build and push MU Docker image
+ run: |
+ docker build -t ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-mu-$COMMIT_HASH -f servers/mu/Dockerfile servers/mu/
+ docker push ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-mu-$COMMIT_HASH
+
+ build-and-push-su:
+ env:
+ REPOSITORY: devnet-ecr
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+
+ - name: Extract shortened commit hash
+ id: vars
+ run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+ - name: Log in to Amazon ECR
+ id: login-devnet-ecr
+ uses: aws-actions/amazon-ecr-login@v1
+ with:
+ mask-password: true
+ skip-logout: true
+ registry-type: private
+ registries: ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_AO_DEVNET }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_AO_DEVNET }}
+ AWS_REGION: ${{ secrets.AWS_REGION_AO_DEVNET }}
+
+ - name: Build and push SU Docker image
+ run: |
+ docker build -t ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-su-$COMMIT_HASH -f servers/su/Dockerfile.x86 servers/su/
+ docker push ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-su-$COMMIT_HASH
+
+ update-devnet-image-tags:
+ runs-on: ubuntu-latest
+ needs: 
+ - build-and-push-cu
+ - build-and-push-mu
+ - build-and-push-su
+
+ steps:
+ - name: Checkout Helmfile Repository
+ run: |
+ git clone https://$GITHUB_USERNAME:${{ secrets.AO_TO_CHARTS_PAT }}@github.com/permaweb/charts.git
+ env:
+ GITHUB_USERNAME: bredamatt
+
+ - name: Extract shortened commit hash
+ id: vars
+ run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+ - name: Update commit-sha.yaml
+ env:
+ TRUNCATED_SHA: ${{ env.COMMIT_HASH }}
+ run: |
+ cd charts
+ sed -i "s/commitSHA: \".*\"/commitSHA: \"${TRUNCATED_SHA}\"/" helmfile/devnet/commit-sha.yaml
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+ git commit -am "fix: from ao-repo - update commit-sha to ${TRUNCATED_SHA} for syncing latest image tags"
+ git push https://${{ secrets.AO_TO_CHARTS_PAT }}@github.com/permaweb/charts.git main

.github/workflows/su.yml

@@ -18,20 +18,20 @@ defaults:
 
 jobs:
 
- test:
- runs-on: ubuntu-latest
- steps:
- - name: ⬇️ Checkout repo
- uses: actions/checkout@v4
+ # test:
+ #  runs-on: ubuntu-latest
+ #  steps:
+ #  - name: ⬇️ Checkout repo
+ #  uses: actions/checkout@v4
 
- - name: ⎔ Setup Rust
- uses: actions-rs/toolchain@v1
- with:
- toolchain: stable
+ #  - name: ⎔ Setup Rust
+ #  uses: actions-rs/toolchain@v1
+ #  with:
+ #  toolchain: 1.75.0
 
- - name: Run Tests
- working-directory: servers/su
- run: cargo test
+ #  - name: Run Tests
+ #  working-directory: servers/su
+ #  run: cargo test
 
 
  publish:

servers/cu/Dockerfile

@@ -6,7 +6,9 @@ COPY ./package.json .
 COPY ./package-lock.json .
 COPY ./src ./src
 
-RUN npm install --omit=dev
+RUN npm install --omit=dev && \
+ mkdir /db/ && \
+ mkdir /file-checkpoints/
 
 ENV NODE_ENV=production
 

servers/mu/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:21
+FROM node:22
 
 RUN apt update && apt install -y bash git awscli
 
@@ -7,14 +7,11 @@ WORKDIR /usr/app
 COPY ./package.json .
 COPY ./package-lock.json .
 COPY ./src ./src
-COPY ./entrypoint.sh ./entrypoint.sh
 
-RUN chmod +x /usr/app/entrypoint.sh
+RUN mkdir /usr/app/mu-data/
 
 RUN npm install --ignore-engines
 
-ENV NODE_ENV=production
-
 EXPOSE 3005
 
-ENTRYPOINT ["/usr/app/entrypoint.sh"]
+CMD ["node", "src/app.js"]

servers/su/Dockerfile

@@ -49,4 +49,4 @@ LABEL build_instructions="To build just the binary, run the following command: d
 ENTRYPOINT [ "/app/su" ]
 
 # Run time command arguments, default is empty
-CMD []
+CMD []

servers/su/Dockerfile.arm64

@@ -0,0 +1,40 @@
+### NOTE: THIS WON'T WORK ON AN X86 DEVICE
+
+# Stage 1: Planner
+FROM --platform=linux/arm64 rust:1.75.0 AS planner
+WORKDIR /app
+RUN cargo install cargo-chef
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# Stage 2: Cache the build of the dependencies
+FROM --platform=linux/arm64 rust:1.75.0 AS cacher
+WORKDIR /app
+RUN cargo install cargo-chef
+RUN apt-get update && apt-get install -y \
+ llvm-dev \
+ libclang-dev \
+ clang \
+ librocksdb-dev \
+ libpq5 \
+ libssl-dev
+COPY --from=planner /app/recipe.json recipe.json
+RUN ls /lib/
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Stage 3: Build binary with pre-built and cached dependencies
+FROM --platform=linux/arm64 rust:1.75.0 AS builder
+COPY . /app
+WORKDIR /app
+COPY --from=cacher /app/target target
+COPY --from=cacher /usr/local/cargo /usr/local/cargo
+COPY --from=cacher /lib/aarch64-linux-gnu/* /lib/aarch64-linux-gnu/
+# Set the correct Rust target based on architecture
+RUN cargo build --release
+
+# Stage 4: Runner 
+FROM --platform=linux/arm64 gcr.io/distroless/cc-debian12 
+RUN apt-get update && apt install ca-certificates openssl -y
+COPY --from=builder /app/target/release/su /
+COPY --from=cacher /lib/aarch64-linux-gnu/* /lib/aarch64-linux-gnu/
+CMD ["sh", "-c", "sleep 10 && ./su su 9000"]

servers/su/Dockerfile.x86

@@ -0,0 +1,41 @@
+### NOTE: THIS WON'T WORK ON AN ARM64 DEVICE
+
+# Stage 1: Planner
+FROM --platform=linux/amd64 rust:1.75.0 AS planner
+WORKDIR /app
+RUN cargo install cargo-chef
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# Stage 2: Cache the build of the dependencies
+FROM --platform=linux/amd64 rust:1.75.0 AS cacher
+WORKDIR /app
+RUN cargo install cargo-chef
+RUN apt-get update && apt-get install -y \
+ llvm-dev \
+ libclang-dev \
+ clang \
+ librocksdb-dev \
+ libpq5 \
+ libssl-dev 
+
+COPY --from=planner /app/recipe.json recipe.json
+RUN ls /lib/
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Stage 3: Build binary with pre-built and cached dependencies
+FROM --platform=linux/amd64 rust:1.75.0 AS builder
+COPY . /app
+WORKDIR /app
+COPY --from=cacher /app/target target
+COPY --from=cacher /usr/local/cargo /usr/local/cargo
+COPY --from=cacher /lib/x86_64-linux-gnu/* /lib/x86_64-linux-gnu/
+# Set the correct Rust target based on architecture
+RUN cargo build --release
+
+# Stage 4: Runner 
+FROM --platform=linux/amd64 ubuntu:22.04
+RUN apt-get update && apt install ca-certificates openssl -y
+COPY --from=builder /app/target/release/su /
+COPY --from=cacher /lib/x86_64-linux-gnu/* /lib/x86_64-linux-gnu/
+CMD ["sh", "-c", "sleep 10 && ./su su 9000"]