Bump pip from 21.3.1 to 23.3 in /Packs/AzureSentinel/Integrations/AzureSentinel

.circleci/config.yml

@@ -1,4 +1,8 @@
 version: 2.1
+orbs:
+ python: circleci/python@<< pipeline.parameters.python-orb-version >>
+ node: circleci/node@<< pipeline.parameters.node-orb-version >>
+
 commands:
  set-instance-role-env-variable:
  description: "Sets INSTANCE_ROLE env variable into $BASH_ENV file"
@@ -14,41 +18,6 @@ commands:
  command: |
  echo 'export INSTANCE_ROLE="<< parameters.instance_role >>"' >> $BASH_ENV
  echo 'export INSTANCE_WAS_CREATED="<< parameters.instance_created >>"' >> $BASH_ENV
- start-tunnel:
- description: "Opens an ssh tunnel to the demisto servers and wait until the tunnels are established"
- parameters:
- timeout:
- type: integer
- default: 10
- steps:
- - run:
- name: add ssh configurations
- shell: /bin/bash
- command: |
- if [ -z $INSTANCE_WAS_CREATED ];
- then
- echo "Skipping - instance was not created"
- exit 0
- fi
- # Modifying ssh config file
- echo "Host 10.0.*
- StrictHostKeyChecking no
- LogLevel ERROR
- ProxyJump [email protected] # disable-secrets-detection
- Host content-build-lb.demisto.works
- Port 43567
- UserKnownHostsFile /dev/null
- StrictHostKeyChecking no
- LogLevel ERROR" >> ~/.ssh/config
- - run:
- name: Open SSH Tunnel
- command: |
- if [ -z $INSTANCE_WAS_CREATED ];
- then
- echo "Skipping - instance was not created"
- exit 0
- fi
- ./Tests/scripts/open_ssh_tunnel.sh
 
 parameters:
  artifact_build_num:
@@ -75,7 +44,16 @@ parameters:
  gcs_market_bucket:
  type: string
  default: "marketplace-dist"
-
+ cache-version:
+ type: string
+ default: v1 # Change this parameter to clear cache.
+ python-orb-version:
+ type: string
+ default: "2.0.3" # disable-secrets-detection
+ node-orb-version:
+ type: string
+ default: "5.0.1" # disable-secrets-detection
+
 references:
  environment: &environment
  environment:
@@ -90,11 +68,48 @@ references:
  PULL_REQUEST_NUMBER: << pipeline.parameters.pr_number >>
  NIGHTLY_PARAMETER: << pipeline.parameters.nightly >>
  GCS_MARKET_BUCKET: << pipeline.parameters.gcs_market_bucket >>
-
+
+ install_build_dependencies: &install_build_dependencies
+ python/install-packages: 
+ pkg-manager: "poetry"
+ args: "--with ci"
+ cache-version: << pipeline.parameters.cache-version >>
+ pre-install-steps:
+ - run:
+ name: Check if pyproject.toml is consistent with poetry.lock
+ command: poetry lock --check
+
+ install_node_ci: &install_node_ci
+ node/install-packages:
+ cache-version: << pipeline.parameters.cache-version >>
+
+
+ install_neo4j: &install_neo4j
+ run:
+ name: Install Neo4j
+ command: |
+ wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
+ echo 'deb https://debian.neo4j.com stable 5' | sudo tee /etc/apt/sources.list.d/neo4j.list
+ sudo apt-get update
+ apt list -a neo4j
+ sudo apt-get install neo4j
+ sudo chown -R circleci /var/log/neo4j
+ sudo chown -R circleci /var/lib/neo4j
+ sudo chown -R circleci /etc/neo4j
+ mkdir -p /var/lib/neo4j/plugins
+ wget -O /var/lib/neo4j/plugins/apoc-5.5.0-core.jar https://github.com/neo4j/apoc/releases/download/5.5.0/apoc-5.5.0-core.jar
+ neo4j_conf_file="/etc/neo4j/neo4j.conf"
+ sudo echo "dbms.security.procedures.unrestricted=apoc.*" >> $neo4j_conf_file
+ sudo echo "dbms.security.procedures.allowlist=apoc.*" >> $neo4j_conf_file
+ apoc_conf_file="/etc/neo4j/apoc.conf"
+ sudo echo "apoc.export.file.enabled=true" > $apoc_conf_file
+ sudo echo "apoc.import.file.enabled=true" >> $apoc_conf_file
+ sudo echo "apoc.import.file.use_neo4j_config=true" >> $apoc_conf_file
+ neo4j-admin dbms set-initial-password contentgraph
 
  container_config: &container_config
  docker:
- - image: devdemisto/content-build:3.0.0.33772 # disable-secrets-detection
+ - image: devdemisto/content-build:3.0.0.49685 # disable-secrets-detection
  auth:
  username: $DOCKERHUB_USER
  password: $DOCKERHUB_PASSWORD
@@ -112,24 +127,20 @@ references:
  attach_workspace:
  at: *workspace_root
 
- add_ssh_keys: &add_ssh_keys
- add_ssh_keys:
- fingerprints:
- - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" # disable-secrets-detection
- - "f5:25:6a:e5:ac:4b:84:fb:60:54:14:82:f1:e9:6c:f9" # disable-secrets-detection
-
  prepare_environment: &prepare_environment
  run:
  name: Prepare Environment
  when: always
  command: |
  poetry --version
-
  # Check if CircleCI's config file and poetry files files are up to date
  # if poetry isn't up-to-date, checkout from origin/master.
  ./Tests/scripts/is_file_up_to_date.sh .circleci/config.yml $CIRCLE_BRANCH
  ./Tests/scripts/is_file_up_to_date.sh poetry.lock $CIRCLE_BRANCH true
  ./Tests/scripts/is_file_up_to_date.sh pyproject.toml $CIRCLE_BRANCH true
+ ./Tests/scripts/is_file_up_to_date.sh Tests/Marketplace/core_packs_list.json $CIRCLE_BRANCH true
+ ./Tests/scripts/is_file_up_to_date.sh Tests/Marketplace/core_packs_mpv2_list.json $CIRCLE_BRANCH true
+ ./Tests/scripts/is_file_up_to_date.sh Tests/Marketplace/core_packs_xpanse_list.json $CIRCLE_BRANCH true
 
  echo 'export CIRCLE_ARTIFACTS="/home/circleci/project/artifacts"' >> $BASH_ENV
  echo 'export PATH="/home/circleci/.local/bin:${PWD}/node_modules/.bin:${PATH}"' >> $BASH_ENV # disable-secrets-detection
@@ -156,12 +167,6 @@ references:
  chmod +x ./Tests/scripts/*
  chmod +x ./Tests/Marketplace/*
 
- echo "Checking if pyproject.toml is consistent with poetry.lock"
- poetry lock --check
-
- # we still need to install even if cached. if cached, `poetry` will handle it
- echo "installing venv"
- NO_HOOKS=1 .hooks/bootstrap
  source ./.venv/bin/activate
 
  # store in bash env so we load our venv in each step
@@ -176,13 +181,9 @@ references:
  npm --version
  demisto-sdk --version
 
- restore_cache: &restore_cache
- restore_cache:
- key: virtualenv-venv-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum "package-lock.json" }}
-
  remote_docker: &remote_docker
  setup_remote_docker:
- version: 20.10.6
+ version: 20.10.17
  docker_layer_caching: true
 
  persist_to_workspace: &persist_to_workspace
@@ -210,12 +211,19 @@ references:
  exit 0
  fi
 
+ neo4j start
+ # poll for neo4j status until available
+ while ! curl --fail http://127.0.0.1:7474 &> /dev/null; do sleep 1; done
+
  ./Tests/scripts/linters_runner.sh
  ./Tests/scripts/validate.sh
 
  run_unit_testing_and_lint: &run_unit_testing_and_lint
  run:
- name: Run Unit Testing and Lint
+ parameters:
+ dockerimageflag:
+ type: string
+ name: Run Unit Testing And Lint - Docker Image:<< parameters.dockerimageflag >>
  when: always
  no_output_timeout: 5h
  command: |
@@ -233,7 +241,12 @@ references:
  echo "vulture py2 version: $(python2 -m vulture --version 2>&1)"
  echo "vulture py3 version: $(python3 -m vulture --version 2>&1)"
  mkdir ./unit-tests
- demisto-sdk lint -p 8 -g -vvv --test-xml ./unit-tests --log-path ./artifacts --failure-report ./artifacts --coverage-report $ARTIFACTS_FOLDER/coverage_report
+
+ neo4j start
+ # poll for neo4j status until available
+ while ! curl --fail http://127.0.0.1:7474 &> /dev/null; do sleep 1; done
+
+ demisto-sdk lint -p 8 -g -vvv --test-xml ./unit-tests --log-path ./artifacts --failure-report ./artifacts --coverage-report $ARTIFACTS_FOLDER/coverage_report --docker-image << parameters.dockerimageflag >> --check-dependent-api-module
 
  generate_coverage_reports: &generate_coverage_reports
  run:
@@ -267,32 +280,6 @@ references:
  ./Tests/scripts/sdk_pylint_check.sh
  fi
 
- create_id_set: &create_id_set
- run:
- name: Create ID Set
- when: always
- command: |
- demisto-sdk create-id-set -o ./Tests/id_set.json --fail-duplicates
- cp ./Tests/id_set.json $CIRCLE_ARTIFACTS
-
- merge_public_and_private_id_sets: &merge_public_and_private_id_sets
- run:
- name: Merge public and private ID sets
- when: always
- command: |
- if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then
- echo "Skipping, Should not run on contributor's branch."
- exit 0
- fi
-
- # Download private ID set
- gsutil cp gs://marketplace-dist/content/private_id_set.json $CIRCLE_ARTIFACTS/unified_id_set.json
- echo "successfully downloaded private ID set"
-
- # Merge public and private ID sets
- demisto-sdk merge-id-sets -i1 ./Tests/id_set.json -i2 $CIRCLE_ARTIFACTS/unified_id_set.json -o $CIRCLE_ARTIFACTS/unified_id_set.json
- echo "successfully merged public and private ID sets"
-
  get_contribution_pack: &get_contribution_pack
  when:
  condition: << pipeline.parameters.contrib_branch >>
@@ -323,6 +310,10 @@ references:
  context: nightly_env
  requires:
  - Setup Environment
+ matrix:
+ parameters:
+ dockerimageflag: [ "native:ga", "native:maintenance", "native:dev", "from-yml" ]
+ name: Run Unit Testing And Lint - Docker Image:<< matrix.dockerimageflag >>
  - Run Validations:
  requires:
  - Setup Environment
@@ -335,26 +326,25 @@ jobs:
  <<: *environment
  steps:
  - checkout
- - *restore_cache
+ - *install_build_dependencies
+ - *install_node_ci
  - *prepare_environment
- - save_cache:
- paths:
- - .venv
- - node_modules
- key: virtualenv-venv-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum "package-lock.json" }}
  - *get_contribution_pack
- - *add_ssh_keys
  - *persist_to_workspace
 
  Run Unit Testing And Lint:
  <<: *container_config
- resource_class: medium
+ resource_class: large
  <<: *environment
+ parameters:
+ dockerimageflag:
+ type: string
  steps:
  - *attach_workspace
  - *remote_docker
- - *restore_cache
- - *add_ssh_keys
+ - *install_build_dependencies
+ - *install_node_ci
+ - *install_neo4j
  - *prepare_environment
  - *infrastructure_testing
  - *run_unit_testing_and_lint
@@ -365,16 +355,15 @@ jobs:
 
  Run Validations:
  <<: *container_config
- resource_class: medium
+ resource_class: large
  <<: *environment
  steps:
  - *attach_workspace
- - *restore_cache
- - *add_ssh_keys
+ - *install_build_dependencies
+ - *install_node_ci
+ - *install_neo4j
  - *prepare_environment
  - *secrets
- - *create_id_set
- - *merge_public_and_private_id_sets
  - *validate_files_and_yaml
  - run:
  name: Spell Checks
@@ -402,21 +391,28 @@ jobs:
 
  python3 Tests/Marketplace/validate_landing_page_sections.py -i $UNZIP_PATH
  - *store_artifacts
+ - store_artifacts:
+ path: $ARTIFACTS_FOLDER
 
 
 
 workflows:
  version: 2
  commit:
  when:
- not:
- or:
- - << pipeline.parameters.nightly >>
+ matches:
+ # matching the environment variable << pipeline.git.branch >> to contributions branch pattern.
+ pattern: pull/[0-9]+
+ value: << pipeline.git.branch >>
  jobs:
  - Setup Environment
  - Run Unit Testing And Lint:
  requires:
  - Setup Environment
+ matrix:
+ parameters:
+ dockerimageflag: [ "native:ga", "native:maintenance", "native:dev", "native:candidate", "from-yml" ]
+ name: Run Unit Testing And Lint - Docker Image:<< matrix.dockerimageflag >>
  - Run Validations:
  requires:
  - Setup Environment
@@ -437,5 +433,4 @@ workflows:
  # will initiate when using the trigger script.
  when: << pipeline.parameters.nightly >>
  jobs:
- *nightly_jobs
-
+ *nightly_jobs

.circleci/gitlab-ci-env-variables.sh

@@ -2,6 +2,7 @@ echo 'export CI_BUILD_ID="$CIRCLE_BUILD_NUM"' >> $BASH_ENV
 echo 'export CI_PIPELINE_ID="$CIRCLE_WORKFLOW_ID"' >> $BASH_ENV
 echo 'export CI_COMMIT_BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV
 echo 'export ARTIFACTS_FOLDER=/home/circleci/project/artifacts' >> $BASH_ENV
+echo 'export PIPELINE_JOBS_FOLDER=/home/circleci/project/pipeline_jobs_folder' >> $BASH_ENV
 echo 'export CI_COMMIT_SHA="$CIRCLE_SHA1"' >> $BASH_ENV
 echo 'export CI_JOB_URL="$CIRCLE_BUILD_URL"' >> $BASH_ENV
 echo 'export CI_JOB_NAME="$CIRCLE_JOB"' >> $BASH_ENV