build content docs - in gitlab

.circleci/config.yml

@@ -124,8 +124,8 @@ jobs:
  export PATH="$PATH:`pwd`/node_modules/.bin"
  export GOOGLE_APPLICATION_CREDENTIALS="$HOME"/gcloud.json
 
- if [ "$CIRCLE_BRANCH" = "master" ]; then
- echo "Deploying to prod as CIRCLE_BRANCH is equal to master!"
+ if [[ "${CIRCLE_BRANCH}" = "master" ]] || [[ "${CI_COMMIT_REF_NAME}" == "master" ]]; then
+ echo "Deploying to prod as CI_COMMIT_REF_NAME is equal to master!"
  firebase deploy --only hosting > deploy-info-firebase.txt
  else
  # Get the PR number from the CIRCLE_PULL_REQUEST env variable which is in the form of:

.gitignore

@@ -43,4 +43,7 @@ static/assets/deprecated*.json
 /src/pages/marketplace/details
 /index
 index.zip
-index.json
+index.json
+/.firebase/hosting.YnVpbGQ.cache
+/firebase-debug.log
+/firebase_json_key.json

.gitlab/ci/.gitlab-ci.yml

@@ -0,0 +1,8 @@
+variables:
+ CURRENT_BRANCH_NAME: infra-content-docs
+
+include:
+ - file: "/.gitlab/ci/content-docs/.gitlab-ci.yml"
+ ref: infra-content-docs
+ project: "${CI_PROJECT_NAMESPACE}/infra"
+

.gitlab/helper_functions.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+CYAN="\e[0;36m"
+CLEAR="\e[0m"
+SECTION_START="\e[0Ksection_start:the_time:section_id\r\e[0K${CYAN}section_header${CLEAR}"
+SECTION_END="\e[0Ksection_end:the_time:section_id\r\e[0K"
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+section_start() {
+ local section_header section_id start
+ start="$SECTION_START"
+ if [[ "$#" -eq 1 ]]; then
+ section_header="$1"
+ section_id="$(echo "$1" | tr -c '[:alnum:]\n\r' '_')"
+ elif [[ "$#" -eq 2 ]]; then
+ if [[ "$2" =~ -{0,2}collapsed ]]; then
+ start="${start/section_id/section_id[collapsed=true]}"
+ section_header="$1"
+ section_id="$(echo "$1" | tr -c '[:alnum:]\n\r' '_')"
+ else
+ section_header="$2"
+ section_id="$1"
+ fi
+ elif [[ "$#" -eq 3 && "$3" =~ /^-{0,2}collapsed$/ ]]; then
+ start="${start/section_id/section_id[collapsed=true]}"
+ section_header="$2"
+ section_id="$1"
+ else
+ echo "section_start should be called with 1-3 args but it was called with $#"
+ echo "acceptable usages:"
+ echo " 1. section_start \"<section-start-id>\" \"<section-header>\""
+ echo " 2. section_start \"<section-header>\""
+ echo " 3. section_start \"<section-start-id>\" \"<section-header>\" --collapse"
+ echo " 4. section_start \"<section-header>\" --collapse"
+ echo "where <section-start-id> is only alphanumeric characters and underscore and"
+ echo "--collapse indicates that you would like those log steps to be collapsed in the job log output by default"
+ exit 9
+ fi
+ start_time=$(date +%s)
+ start="$(echo "$start" | sed -e "s/the_time/$start_time/" -e "s/section_id/$section_id/" -e "s/section_header/$section_header/")"
+ echo -e "$start"
+ date +"[%Y-%m-%dT%H:%M:%S.%3N] section start"
+}
+
+section_end() {
+ local section_id end
+ date +"[%Y-%m-%dT%H:%M:%S.%3N] section end"
+ end="$SECTION_END"
+ if [[ "$#" -eq 1 ]]; then
+ section_id="$(echo "$1" | tr -c '[:alnum:]\n\r' '_')"
+ else
+ echo "section_end should be called with 1 arg but it was called with $#"
+ echo "acceptable usage:"
+ echo " 1. section_end \"<section-start-id>\""
+ echo " 2. section_start \"<section-header>\""
+ echo "where <section-start-id> or <section-header> is that of the section this marks the end of"
+ exit 9
+ fi
+ end_time=$(date +%s)
+ end="$(echo "$end" | sed -e "s/the_time/$end_time/" -e "s/section_id/$section_id/")"
+ echo -e "$end"
+}
+
+job-done() {
+ mkdir -p "${PIPELINE_JOBS_FOLDER}"
+ echo "creating file ${PIPELINE_JOBS_FOLDER}/${CI_JOB_NAME}.txt"
+ echo "done" > "${PIPELINE_JOBS_FOLDER}/${CI_JOB_NAME}.txt"
+ echo "finished writing to file ${PIPELINE_JOBS_FOLDER}/${CI_JOB_NAME}.txt"
+}
+
+sleep-with-progress() {
+ local sleep_time=${1:-10}
+ local sleep_interval=${2:-1}
+ local sleep_message=${3:-"Sleeping... "}
+ local columns=${4:-$(tput cols)}
+ local sleep_step=$((sleep_time / sleep_interval))
+ for ((i=0; i< sleep_step;i++)); do echo "${sleep_interval}";sleep "${sleep_interval}"; done | poetry run tqdm --total ${sleep_time} --unit seconds --leave --update --colour green -ncols ${columns} --desc "${sleep_message}" 1> /dev/null
+}

.hooks/bootstrap

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+
+#
+# Bootstraps a development environment.
+#
+# This includes:
+# * install pre-commit hooks
+# * setup infra dependencies with poetry
+
+function exit_on_error {
+ if [ "${1}" -ne 0 ]; then
+ echo "ERROR: ${2}, exiting with code ${1}" 1>&2
+ exit "${1}"
+ fi
+}
+
+# poetry is installed in ~/.local/bin
+PATH=~/.local/bin:$PATH
+
+if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then
+ cat << __EOF__
+Setup development environment (run with no arguments):
+* install pre-commit hooks (set NO_HOOKS=1 to skip)
+__EOF__
+ exit 0
+fi
+
+if [ ! "${PWD}" == "$(git rev-parse --show-toplevel)" ]; then
+ cat >&2 <<__EOF__
+ERROR: this script must be run at the root of the source tree
+__EOF__
+ exit 1
+fi
+
+echo "======================="
+if [ -z "${INSTALL_POETRY}" ]; then
+ if ! command -v poetry >/dev/null 2>&1; then
+ echo "ERROR: poetry is missing. Please run the following command to install poetry:
+ curl -sSL https://install.python-poetry.org | python3 -" 1>&2
+ exit 1
+ fi
+else
+ should_install_poetry="yes"
+ if command -v poetry >/dev/null 2>&1; then
+ if [[ "$(poetry --version)" == "Poetry (version ${POETRY_VERSION})" ]]; then
+ echo "Poetry already installed:$(poetry --version) with correct version"
+ should_install_poetry="no"
+ else
+ echo "Poetry installed with a different version:$(poetry --version) required:${POETRY_VERSION}"
+ fi
+ else
+ echo "Poetry isn't installed"
+ fi
+ if [[ "${should_install_poetry}" == "yes" ]]; then
+ echo "Installing Poetry version:${POETRY_VERSION}"
+ curl -sSL https://install.python-poetry.org | python3 - --version "${POETRY_VERSION}"
+ error_code=$?
+ if ! command -v poetry >/dev/null 2>&1; then
+ exit_on_error $? "Poetry isn't installed"
+ fi
+ if [[ "$(poetry --version)" == "Poetry (version ${POETRY_VERSION})" ]]; then
+ echo "Poetry version ${POETRY_VERSION} installed successfully"
+ else
+ exit_on_error 1 "Poetry version $(poetry --version) doesn't match the required version: ${POETRY_VERSION}"
+ fi
+ if [ -n "${ARTIFACTS_FOLDER}" ] && [ "${error_code}" -ne 0 ]; then
+ cp "${PWD}"/poetry-installer-error-*.log "${ARTIFACTS_FOLDER}"
+ fi
+ exit_on_error $error_code "Failed to install Poetry version:${POETRY_VERSION}"
+ fi
+fi
+
+GIT_HOOKS_DIR="${PWD}/.git/hooks"
+
+if [ -n "${NO_HOOKS}" ]; then
+ echo "Skipping hooks setup as environment variable NO_HOOKS is set"
+else
+ if [ ! -e "${GIT_HOOKS_DIR}/pre-commit" ]; then
+ echo "Installing 'pre-commit' hooks"
+ poetry run pre-commit install
+ exit_on_error $? "Failed to install pre-commit hook"
+ else
+ echo "Skipping install of pre-commit hook as it already exists."
+ echo "If you want to re-install: 'rm ${GIT_HOOKS_DIR}/pre-commit' and then run this script again."
+ exit 1
+ fi
+fi
+
+if [ -n "${CI}" ]; then
+ echo "Detected CI environment"
+ echo "Checking whether poetry files are valid"
+ poetry check --no-interaction
+ exit_on_error $? "Failed to check poetry files"
+ echo "Installing dependencies..."
+ poetry install --no-interaction
+ exit_on_error $? "Failed to install dependencies"
+else
+ echo "Detected local environment"
+ echo "Check if poetry files are valid"
+ poetry check
+ exit_on_error $? "Failed to check poetry files"
+ echo "Installing dependencies..."
+ poetry install
+ exit_on_error $? "Failed to install dependencies"
+fi
+
+echo "=========================="
+echo "Done setting up virtualenv with poetry"
+echo "Activate the venv by running: poetry shell"
+echo "Deactivate by running: deactivate"
+echo "======================="
+
+echo "Finished setting up the environment."
+exit 0

.pre-commit-config.yaml

@@ -0,0 +1,69 @@
+repos:
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+ rev: v0.5.4
+ hooks:
+ - id: ruff
+ args:
+ - --fix
+ exclude: CommonServerPython.py
+ - id: ruff-format
+- repo: https://github.com/pappasam/toml-sort
+ rev: v0.23.1
+ hooks:
+ - id: toml-sort-fix
+ exclude: poetry.lock
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.6.0
+ hooks:
+ - id: check-json
+ - id: check-yaml
+ exclude: .gitlab
+ - id: check-ast
+ - id: check-merge-conflict
+# - id: debug-statements
+# language_version: python3
+ - id: name-tests-test
+ files: .+_test.py$
+ - id: trailing-whitespace
+ - id: check-toml
+ - id: check-xml
+- repo: https://github.com/python-poetry/poetry
+ rev: 1.8.3
+ hooks:
+ - id: poetry-check
+ args:
+ - --lock
+ files: ^pyproject.toml$
+- repo: https://github.com/hadialqattan/pycln
+ rev: v2.4.0
+ hooks:
+ - id: pycln
+ args:
+ - --all
+- repo: https://github.com/pre-commit/mirrors-mypy
+ rev: v1.10.1
+ hooks:
+ - id: mypy
+ entry: mypy
+ args:
+ - --show-error-codes
+ additional_dependencies:
+ - types-requests
+ - types-paramiko
+ - types-PyYAML
+ - types-setuptools
+ - types-ujson
+ - types-decorator
+ - types-retry
+ - types-pytz
+ - types-python-dateutil
+ - types-tabulate
+ - types-dateparser
+ types: [python]
+ #duplicate here and in pyproject.toml because https://github.com/python/mypy/issues/13916
+ exclude: artifacts*|.*_test|test_.*|test_data|tests_data|.venv
+- repo: https://github.com/koalaman/shellcheck-precommit
+ rev: v0.10.0
+ hooks:
+ - id: shellcheck
+ args: ["--severity=error"]

content-repo/create-docs.sh

@@ -6,7 +6,7 @@ set -e
 # Script will check out the Demisto content repo and then generate documentation based upon the checkout
 
 SCRIPT_DIR=$(dirname ${BASH_SOURCE})
-CURRENT_DIR=`pwd`
+CURRENT_DIR=$(pwd)
 if [[ "${SCRIPT_DIR}" != /* ]]; then
  SCRIPT_DIR="${CURRENT_DIR}/${SCRIPT_DIR}"
 fi
@@ -20,8 +20,10 @@ if [[ -n "$CONTENT_REPO_DIR" ]]; then
  echo "================================="
 else
  CONTENT_GIT_DIR=${SCRIPT_DIR}/.content
- if [[ -n "${CIRCLE_BRANCH}" ]]; then
+ if [[ -n "${CIRCLE_BRANCH}" ]]; then # CircleCI - deprecated remove!
  CURRENT_BRANCH=${CIRCLE_BRANCH}
+ elif [[ -n "${CI_COMMIT_REF_NAME}" ]]; then
+ CURRENT_BRANCH=${CI_COMMIT_REF_NAME}
  else
  CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
  fi
@@ -48,17 +50,17 @@ else
  echo "==== content git url: ${CONTENT_GIT_URL} branch: ${CONTENT_BRANCH} ===="
 
  if [[ -d ${CONTENT_GIT_DIR} && $(cd ${CONTENT_GIT_DIR}; git remote get-url origin) != "${CONTENT_GIT_URL}" ]]; then
- echo "Deleting dir: ${CONTENT_GIT_DIR} as remote url dooesn't match ${CONTENT_GIT_URL} ..."
+ echo "Deleting dir: ${CONTENT_GIT_DIR} as remote url doesn't match ${CONTENT_GIT_URL} ..."
  rm -rf "${CONTENT_GIT_DIR}"
  fi
 
  if [ ! -d ${CONTENT_GIT_DIR} ]; then
  # Do not do "git clone --depth 1" as we need full history for the deprecated integrations data generation
  echo "Cloning content to dir: ${CONTENT_GIT_DIR} ..."
- git clone ${CONTENT_GIT_URL} ${CONTENT_GIT_DIR} 
+ git clone -q ${CONTENT_GIT_URL} ${CONTENT_GIT_DIR}
  else
  echo "Content dir: ${CONTENT_GIT_DIR} exists. Skipped clone."
- if [ -z "${CONTENT_REPO_SKIP_PULL}"]; then 
+ if [ -z "${CONTENT_REPO_SKIP_PULL}" ]; then
  echo "Doing pull..."
  (cd ${CONTENT_GIT_DIR}; git pull)
  fi
@@ -75,7 +77,7 @@ else
  echo "Using content master to generate build"
  CONTENT_BRANCH=master
  git checkout master
- # you can use an old hash to try to see if bulid passes when there is a failure.
+ # you can use an old hash to try to see if build passes when there is a failure.
  # git checkout b11f4cfe4a3bf567656ef021f3d8f1bf66bcb9f6
  fi
  echo "Git log:"
@@ -117,15 +119,16 @@ if [[ ( "$PULL_REQUEST" == "true" || -n "$CI_PULL_REQUEST" ) && "$CONTENT_BRANCH
  fi
 fi
 
-BUCKET_DIR="${SCRIPT_DIR}/.content-bucket"
-if [[ ! -d "$BUCKET_DIR" ]]; then
- echo "Copying bucket docs content to: $BUCKET_DIR"
- mkdir "${BUCKET_DIR}"
- gsutil -m cp -r gs://marketplace-dist/content/docs/Packs/ "${BUCKET_DIR}"
-else
- echo "Skipping copying bucket data as dir: $BUCKET_DIR already exists"
- echo "If you want to re-copy, delete the dir: $BUCKET_DIR"
-fi
+# FIXME! restore the bucket copy
+#BUCKET_DIR="${SCRIPT_DIR}/.content-bucket"
+#if [[ ! -d "$BUCKET_DIR" ]]; then
+# echo "Copying bucket docs content to: $BUCKET_DIR"
+# mkdir "${BUCKET_DIR}"
+# gsutil -q -m cp -r gs://marketplace-dist/content/docs/Packs/ "${BUCKET_DIR}"
+#else
+# echo "Skipping copying bucket data as dir: $BUCKET_DIR already exists"
+# echo "If you want to re-copy, delete the dir: $BUCKET_DIR"
+#fi
 
 TARGET_DIR=${SCRIPT_DIR}/../docs/reference
 CONTRIB_TARGET_DIR=${SCRIPT_DIR}/../src/pages/marketplace
@@ -152,16 +155,19 @@ sed -i -e '/from DemistoClassApiModule import */d' CommonServerPython.py
 # Removing the first lines from CommonServerPython.py which are a description of the script we don't need here
 echo "$(tail -n +6 CommonServerPython.py)" > CommonServerPython.py
 
-echo "Installing pipenv..."
-pipenv install
+#echo "Installing pipenv..."
+#pipenv install
 echo "Generating docs..."
-pipenv run ./gendocs.py -t "${TARGET_DIR}" -d "${CONTENT_GIT_DIR}" -b "${CURRENT_BRANCH}"
+#pipenv run ./gendocs.py -t "${TARGET_DIR}" -d "${CONTENT_GIT_DIR}" -b "${CURRENT_BRANCH}"
+poetry run ./gendocs.py -t "${TARGET_DIR}" -d "${CONTENT_GIT_DIR}" -b "${CURRENT_BRANCH}"
 echo "Generating Demisto class and CommonServerPython docs..."
-pipenv run ./gen_pydocs.py -t "${TARGET_DIR}"
+#pipenv run ./gen_pydocs.py -t "${TARGET_DIR}"
+poetry run ./gen_pydocs.py -t "${TARGET_DIR}"
 if [[ "$CURRENT_BRANCH" != "master" && "$CURRENT_BRANCH" != *"gen-top-contrib"* ]]; then
  echo "Skipping top contributors page generation, should run only on master or branch containing 'gen-top-contrib'."
  exit 0
 else
  echo "Generating top contributors page..."
- pipenv run python ./gen_top_contrib.py -t "${CONTRIB_TARGET_DIR}"
+ poetry run ./gen_top_contrib.py -t "${CONTRIB_TARGET_DIR}"
+# pipenv run python ./gen_top_contrib.py -t "${CONTRIB_TARGET_DIR}"
 fi