Merge pull request #3 from dqops/develop

Release 0.2.0-beta2

.dockerignore

@@ -1,9 +1,14 @@
 userhome
 examples
-dqoai
-lib
 docs
+.git*
 .run
-.mvn
 .idea
-home
+.vscode
+dqoai/target
+distribution/target
+lib/target
+home/venv
+dqoai/src/main/frontend/build
+dqoai/src/main/frontend/node
+dqoai/src/main/frontend/node_modules

.gitattributes

@@ -1,8 +1,8 @@
-mvnw text eol=cr
-mvnw.sh text eol=cr
-dqo text eol=cr
-rebuild.sh text eol=cr
-make_docs text eol=cr
-find-dqo-home.sh text eol=cr
-set-dqo-envs.sh text eol=cr
-find-dqo-classpath.sh text eol=cr
+mvnw text eol=lf
+mvnw.sh text eol=lf
+dqo text eol=lf
+rebuild.sh text eol=lf
+make_docs text eol=lf
+find-dqo-home.sh text eol=lf
+set-dqo-envs.sh text eol=lf
+find-dqo-classpath.sh text eol=lf

.github/workflows/main-docker-image-amd64.yml

@@ -0,0 +1,57 @@
+name: Docker Image CI for linux/amd64
+
+on:
+ push:
+ branches:
+ - main
+
+jobs:
+ build-release:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up JDK 17 for x64
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.11
+
+ - name: Build with Maven
+ run: mvn package -DskipTests -Pbuild-with-jdk-11 -Prun-npm
+
+
+ docker-build-on-ubuntu:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - linux/amd64
+ steps:
+ - uses: actions/checkout@v3
+
+ - uses: juliangruber/read-file-action@v1
+ name: Read VERSION
+ id: read_version
+ with:
+ path: ./VERSION
+
+ - uses: docker-practice/actions-setup-docker@master
+
+ - uses: mr-smithers-excellent/docker-build-push@v6
+ name: Build and Push Docker Image
+ with:
+ image: dqops/dqo
+ tags: $(date +%s),${{ steps.read_version.outputs.content }},latest
+ enableBuildKit: true
+ multiPlatform: true
+ platform: ${{ matrix.platform }}
+ registry: docker.io
+ username: ${{ secrets.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/main-docker-image-arm64.yml

@@ -0,0 +1,37 @@
+name: Docker Image CI for linux/arm64
+on:
+ push:
+ branches:
+ - main
+
+jobs:
+
+ docker-build-on-ubuntu:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - linux/arm64
+ steps:
+ - uses: actions/checkout@v3
+
+ - uses: juliangruber/read-file-action@v1
+ name: Read VERSION
+ id: read_version
+ with:
+ path: ./VERSION
+
+ - uses: docker-practice/actions-setup-docker@master
+
+ - uses: mr-smithers-excellent/docker-build-push@v6
+ name: Build and Push Docker Image
+ with:
+ image: dqops/dqo
+ tags: $(date +%s),${{ steps.read_version.outputs.content }},latest
+ enableBuildKit: true
+ multiPlatform: true
+ platform: ${{ matrix.platform }}
+ registry: docker.io
+ username: ${{ secrets.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/main-publish-release.yml

@@ -0,0 +1,69 @@
+name: Build and publish the release to GitHub releases and PyPi
+
+on:
+ push:
+ branches:
+ - main
+
+jobs:
+ build-release:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.8
+
+ - name: Build with Maven
+ run: mvn package -DskipTests -Pbuild-with-jdk-11 -Prun-npm
+
+ - uses: juliangruber/read-file-action@v1
+ name: Read VERSION
+ id: read_version
+ with:
+ path: ./VERSION
+
+ - name: Create GitHub release
+ uses: actions/create-release@v1
+ id: create_release
+ with:
+ draft: false
+ prerelease: false
+ release_name: ${{ steps.read_version.outputs.content }}
+ tag_name: ${{ steps.read_version.outputs.content }}
+ body_path: CHANGELOG.md
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+
+ - name: Upload distribution binary file
+ uses: actions/upload-release-asset@v1
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ with:
+ upload_url: ${{ steps.create_release.outputs.upload_url }}
+ asset_path: ./distribution/target/dqo-distribution-${{steps.read_version.outputs.content}}-bin.zip
+ asset_name: dqo-distribution-${{steps.read_version.outputs.content}}-bin.zip
+ asset_content_type: application/zip
+
+ - name: Install python dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r ./distribution/python/requirements.txt
+
+ - name: Build PyPi package
+ run: |
+ python -m build ./distribution/python
+
+ - name: Publish PyPi package
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ user: __token__
+ password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/mvn-build.yml

@@ -0,0 +1,29 @@
+name: Build code with Maven
+
+on:
+ push:
+ branches:
+ - "*"
+ pull_request:
+ branches:
+ - "*"
+
+jobs:
+ build-release:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up JDK 17 for x64
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.11
+
+ - name: Build with Maven
+ run: mvn package -DskipTests -Pbuild-with-jdk-11 -Prun-npm

.gitignore

@@ -55,11 +55,19 @@ build/
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
 /userhome/
+/dqoai/
 /home/venv/
 venv/
 *.pyc
 
 .DS_Store
 
 # Documentation site
-site/
+/site/
+/.cache/
+
+# DQO Examples
+.localsettings.dqosettings.yaml
+.data/
+.logs/
+.index/

.mvn/jvm.config

@@ -0,0 +1 @@
+-Xmx1024m

.run/.gitattributes

@@ -1 +1 @@
-dqo run.run.xml text eol=cr
+dqo run.run.xml text eol=lf

.run/dqo run.run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
  <configuration default="false" name="dqo run" type="JarApplication">
- <option name="JAR_PATH" value="$PROJECT_DIR$/dqoai/target/dqo-dqoai-0.2.0-alpha0.jar" />
+ <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-0.2.0-beta2.jar" />
  <option name="VM_PARAMETERS" value="-Xmx1024m --add-opens java.base/java.nio=ALL-UNNAMED" />
  <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
  <option name="ALTERNATIVE_JRE_PATH" value="temurin-17" />
@@ -9,7 +9,6 @@
  <env name="DQO_USER_HOME" value="$PROJECT_DIR$/userhome" />
  <env name="HADOOP_HOME" value="$PROJECT_DIR$/home" />
  </envs>
- <module name="dqo-dqoai" />
  <method v="2" />
  </configuration>
-</component>
+</component>

CHANGELOG.md

@@ -0,0 +1,3 @@
+* Table comparison support
+* Anomaly detection checks renamed
+* Bug fixes

Dockerfile

@@ -1,26 +1,70 @@
-FROM alpine:3.17.0 AS dqo-fetcher
-RUN apk update && apk add unzip
+FROM openjdk:17-jdk-slim-buster AS dqo-libs
+WORKDIR /workspace/app
 
+COPY --chmod=755 mvnw.sh ./
+COPY .mvn .mvn
+
+COPY pom.xml .
+COPY dqops/pom.xml dqops/
+COPY distribution/pom.xml distribution/
+COPY lib lib
+
+# resolve dependencies
+ENV USER_HOME="/user/mvn"
+RUN mkdir -p "${USER_HOME}/.m2" && ./mvnw.sh dependency:resolve -pl !distribution && ./mvnw.sh dependency:resolve-plugins -pl !distribution
+RUN ./mvnw.sh frontend:install-node-and-npm -pl dqops
+
+# npm install
+COPY dqops/src/main/frontend/package.json dqops/src/main/frontend/
+ENV PATH="/workspace/app/dqops/src/main/frontend/node:${PATH}"
+WORKDIR /workspace/app/dqops/src/main/frontend
+RUN npm install --legacy-peer-deps
+
+WORKDIR /workspace/app
+
+# compile project
+COPY dqops/src dqops/src
+RUN ./mvnw.sh install -DskipTests -pl !distribution
+RUN mkdir -p dqops/target/dependency && (cd dqops/target/dependency; jar -xf ../*.jar)
+
+FROM python:3.11.3-slim-bullseye AS dqo-home
 WORKDIR /dqo
-COPY ./distribution/target/ /dqo/temp/
-COPY ./VERSION /dqo/temp/
-RUN unzip /dqo/temp/dqo-distribution-$(cat /dqo/temp/VERSION)-bin.zip -d /dqo/home
 
+# copy dqo home
+COPY home home
+WORKDIR /dqo/home
+RUN rm -rf venv/
+
+# recreate venv
+ENV VIRTUAL_ENV=/dqo/home/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cp lib/requirements.txt $VIRTUAL_ENV/home_requirements.txt
+RUN pip3 install -r $VIRTUAL_ENV/home_requirements.txt
+
+RUN python3 -m compileall ./
 
-FROM python:3.10.8-slim-bullseye AS dqo-main
+FROM python:3.11.3-slim-bullseye AS dqo-main
 EXPOSE 8888
 WORKDIR /dqo
 
 # install java
-RUN apt-get update && apt-get install -y openjdk-17-jre && apt-get clean
-ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
+RUN apt-get update && apt-get install -y ca-certificates-java openjdk-17-jre && apt-get clean
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-$TARGETARCH
 ENV PATH=$PATH:$JAVA_HOME/bin
 
-# dqo
-COPY --from=dqo-fetcher /dqo/home /dqo/home
-WORKDIR /dqo/userhome
+# dqo launch setup
 ENV DQO_HOME=/dqo/home
 ENV DQO_USER_HOME=/dqo/userhome
 ENV DQO_USER_INITIALIZE_USER_HOME=true
+RUN mkdir $DQO_USER_HOME
+RUN touch $DQO_USER_HOME/.DQO_USER_HOME_NOT_MOUNTED
+COPY --from=dqo-home /dqo/home home
 
-ENTRYPOINT ["/bin/sh", "/dqo/home/bin/dqo"]
+# copy spring dependencies
+ARG DEPENDENCY=/workspace/app/dqops/target/dependency
+COPY --from=dqo-libs ${DEPENDENCY}/BOOT-INF/lib /dqo/lib
+COPY --from=dqo-libs ${DEPENDENCY}/META-INF /dqo/META-INF
+COPY --from=dqo-libs ${DEPENDENCY}/BOOT-INF/classes /dqo
+ENV JAVA_TOOL_OPTIONS="-Xmx1024m"
+ENTRYPOINT ["java", "-cp", "/dqo:/dqo/lib/*", "com.dqops.cli.CliApplication"]

LICENSE

@@ -187,7 +187,7 @@
  same "printed page" as the copyright notice for easier
  identification within third-party archives.
 
- Copyright [2021] [DQO.ai (Documati sp. z o.o.)]
+ Copyright [2021] [DQOps (Documati sp. z o.o.)]
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

NOTICE

@@ -1,5 +1,5 @@
-DQO.ai Data Quality Observer
+DQOps Data Quality Operations Center
 Copyright 2021 and onwards Documati Sp. z o.o.
 
 This product includes software developed at
-Documati Sp. z o.o. (https://dqo.ai/).
+Documati Sp. z o.o. (https://dqops.com/).