diff --git a/CHANGELOG.md b/CHANGELOG.md index e8b61a85..65ca7f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,7 @@ ### citustools v0.7.2 (October 5, 2017) ### -* Changes packaging default PostgreSQL versions from 9.5,9.6 to 9.6,10 +* Changes packaging default PostgreSQL versions from 9.5, 9.6 to 9.6, 10 ### citustools v0.7.1 (August 30, 2017) ### diff --git a/automated_packaging/Makefile b/automated_packaging/Makefile index 1b9e2aad..943ed8a9 100644 --- a/automated_packaging/Makefile +++ b/automated_packaging/Makefile @@ -4,7 +4,7 @@ INSTALL := install -c INSTALL_SCRIPT := $(INSTALL) -m 755 AUTOMATION_SCRIPTS := $(filter-out Makefile,$(wildcard *)) -all: +all: clean: @@ -13,5 +13,5 @@ installdirs: install: all installdirs $(INSTALL_SCRIPT) $(AUTOMATION_SCRIPTS) $(DESTDIR)$(bindir) - + .PHONY: clean installdirs install diff --git a/automated_packaging/common_functions.pm b/automated_packaging/common_functions.pm index 0c96e0d7..c1d7977e 100644 --- a/automated_packaging/common_functions.pm +++ b/automated_packaging/common_functions.pm @@ -150,7 +150,7 @@ sub create_release_changelog { foreach $line (@log_output) { if ($line =~ /^DESCRIPTION: */) { $description_part = substr($line, length($&), -1); - + if (length($description_part) > 78) { print("You have to shorten PR message $description_part of $pr_url\n"); print("Description should not be longer than 78 charachters, please manually shorten this description\n"); diff --git a/citus_dev/README.md b/citus_dev/README.md index 3c5a861e..75bac092 100644 --- a/citus_dev/README.md +++ b/citus_dev/README.md @@ -1,15 +1,19 @@ # Setup -To install dependencies run: + +To install dependencies run: + ```bash pip install -r requirements.txt ``` Add `citus_dev` to your PATH: + ```bash export PATH=$PATH: ``` You can also add this to your profile: + ```bash echo 'export PATH=$PATH:' >>~/.profile ``` @@ -21,6 +25,7 @@ citus_dev make clusterName ``` For the full command list: + ```bash citus_dev --help ``` diff --git a/citus_dev/bash_completion b/citus_dev/bash_completion index 3507db13..21480f6a 100644 --- a/citus_dev/bash_completion +++ b/citus_dev/bash_completion @@ -12,7 +12,7 @@ _citus_dev() make) return 0 ;; - + start|stop|restart) clusters=$(for x in `find . -type d -name 'coordinator' | awk -F '/' '{print $2}'`; do echo ${x} ; done ); COMPREPLY=($( compgen -W "${clusters}" -- "${cur}" )); diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index 6398c906..7adc34a2 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -38,6 +38,7 @@ fixopen = distutils.spawn.find_executable("postgres.fixopen") if fixopen: pgctl_flags += f' -p "{fixopen}"' + def run(command, *args, **kwargs): print(command) result = subprocess.run(command, *args, check=True, shell=True, **kwargs) @@ -61,18 +62,23 @@ def createNodeCommands(clustername, role, index=None, usessl=False, mx=False): if usessl: run(f'echo "ssl = on" >> {clustername}/{nodename}/postgresql.conf') - run(f"echo \"citus.node_conninfo = 'sslmode=require'\" >> {clustername}/{nodename}/postgresql.conf") - run(f"openssl req -new -x509 -days 365 -nodes -text -out {clustername}/{nodename}/server.crt -keyout {clustername}/{nodename}/server.key -subj '/CN={nodename}'") + run( + f"echo \"citus.node_conninfo = 'sslmode=require'\" >> {clustername}/{nodename}/postgresql.conf") + run( + f"openssl req -new -x509 -days 365 -nodes -text -out {clustername}/{nodename}/server.crt -keyout {clustername}/{nodename}/server.key -subj '/CN={nodename}'") run(f"chmod 0600 {clustername}/{nodename}/server.key") if mx: - run(f"echo \"citus.replication_model = 'streaming'\" >> {clustername}/{nodename}/postgresql.conf") + run( + f"echo \"citus.replication_model = 'streaming'\" >> {clustername}/{nodename}/postgresql.conf") + def createPgBouncerUsers(clustername): username = getpass.getuser() with open(f"{clustername}/users.txt", "w") as f: f.write(f'"{username}" ""') + def createPgBouncerConfig(clustername, port, index): workerPort = port + index + 1 bouncerPort = port + index + 101 @@ -111,7 +117,6 @@ def main(arguments): stopCluster(clustername, True) run(f'rm -rf {clustername}') - createNodeCommands( clustername, "coordinator", @@ -177,25 +182,24 @@ def main(arguments): # need to start pgbouncers and configure pg_dist_poolinfo for i in range(size): coordinatorPort = port - workerPort = port + i + 1 + workerPort = port + i + 1 bouncerPort = port + i + 101 run(f'pgbouncer -d {clustername}/worker{i}.pgbouncer.ini') run(f"psql -p {coordinatorPort} -c \"INSERT INTO pg_dist_poolinfo SELECT nodeid, 'host=localhost port={bouncerPort}' AS poolinfo FROM pg_dist_node WHERE nodeport = {workerPort};\"") - if arguments['--init-with']: run(f'psql -p {cport} -f {arguments["--init-with"]} -v ON_ERROR_STOP=1') if arguments['--init-worker-with']: for i in range(size): workerport = port + 1 + i - run(f'psql -p {workerport} -f {arguments["--init-worker-with"]} -v ON_ERROR_STOP=1') + run( + f'psql -p {workerport} -f {arguments["--init-worker-with"]} -v ON_ERROR_STOP=1') elif arguments["stop"]: clusterName = arguments[""] always = arguments["--force"] stopCluster(clusterName, always) - elif arguments["start"]: clustername = arguments[""] port = int(arguments["--port"]) @@ -205,8 +209,6 @@ def main(arguments): for bouncerConfig in getPgBouncerConfigs(clustername): run(f'pgbouncer -d {clustername}/{bouncerConfig}') - - elif arguments["restart"]: clustername = arguments[""] port = int(arguments["--port"]) @@ -222,17 +224,18 @@ def main(arguments): run(f'pg_ctl {pgctl_flags} restart -D {clustername}/{role} -o "-p {cport}" -l {role}_logfile') cport += 1 - else: print("unknown command") exit(1) + def getPgBouncerPort(clustername, configfile): with open(f"{clustername}/{configfile}") as f: - for line in f.readlines(): - if line.find("listen_port") >= 0: - lineParts = line.split("=") - return int(lineParts[1]) + for line in f.readlines(): + if line.find("listen_port") >= 0: + lineParts = line.split("=") + return int(lineParts[1]) + def stopCluster(clustername, always=False): for bouncerConfig in getPgBouncerConfigs(clustername): @@ -247,9 +250,11 @@ def stopCluster(clustername, always=False): for role in getRoles(clustername): run(f"pg_ctl {pgctl_flags} stop -D {clustername}/{role} {pipeTrue}") + def getPgBouncerConfigs(clustername): try: - bouncerFiles = [f.name for f in os.scandir(clustername) if f.name.find("pgbouncer.ini") >= 0] + bouncerFiles = [f.name for f in os.scandir( + clustername) if f.name.find("pgbouncer.ini") >= 0] return bouncerFiles except FileNotFoundError: return [] diff --git a/dashboard/README.md b/dashboard/README.md index d1da746d..c3d18f3c 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -4,11 +4,11 @@ These scripts are intended for use within daily cron scripts (presently running ## Getting Started -You probably shouldn't need to do anything with this, but if anything changes, log into that box, use `curl` or `wget` to get the latest version of the `citusdata/tools` repository, then `sudo make -C dashboard install`. Check the latest cron configuration using `crontab -l`, and see whether any cron failures have sent mail using `mail`. +You probably shouldn't need to do anything with this, but if anything changes, log into that box, use `curl` or `wget` to get the latest version of the `citusdata/tools` repository, then `sudo make -C dashboard install` . Check the latest cron configuration using `crontab -l` , and see whether any cron failures have sent mail using `mail` . ## What's Included -All non `update_*` scripts emit (to standard out) CSV files suitable for loading into PostgreSQL tables defined in `schema.dll`. Most have pretty good usage messages, so check those out if needed. +All non `update_*` scripts emit (to standard out) CSV files suitable for loading into PostgreSQL tables defined in `schema.dll` . Most have pretty good usage messages, so check those out if needed. `pkg.jq` is a huge mishmash of `jq` helper normalization functions for massaging API data into the right state before ingest. diff --git a/dashboard/docker_pulls b/dashboard/docker_pulls index 7854a50f..e60a8a35 100755 --- a/dashboard/docker_pulls +++ b/dashboard/docker_pulls @@ -14,7 +14,7 @@ jq=$(which jq) # outputs usage message on specified device before exiting with provided status usage() { - cat << 'E_O_USAGE' >&"$1" + cat <<'E_O_USAGE' >&"$1" usage: docker_pulls repo [since] repo : a citusdata Docker Hub repository name @@ -27,7 +27,7 @@ recent time point's date as the 'since' parameter will cause this program to exit without emitting any rows if that parameter is on or after today's date. E_O_USAGE - exit "${2}"; + exit "${2}" } if [ "$#" -eq 1 ]; then @@ -53,4 +53,4 @@ repo=${1} repourl="https://hub.docker.com/v2/repositories/citusdata/${repo}/" curl -sf "${repourl}" | -${jq} -r "include \"pkg\"; makepullrow(\"${repo}\") | @csv" + ${jq} -r "include \"pkg\"; makepullrow(\"${repo}\") | @csv" diff --git a/dashboard/github_clones b/dashboard/github_clones index b9c5f0a9..43356cb5 100755 --- a/dashboard/github_clones +++ b/dashboard/github_clones @@ -17,7 +17,7 @@ jq=$(which jq) # outputs usage message on specified device before exiting with provided status usage() { - cat << 'E_O_USAGE' >&"$1" + cat <<'E_O_USAGE' >&"$1" usage: github_clones repo [since] repo : a citusdata GitHub repository name @@ -29,7 +29,7 @@ custom start date: without one, all known data is requested, though GitHub's maximum window is only two weeks long. E_O_USAGE - exit "${2}"; + exit "${2}" } if [ "$#" -eq 1 ]; then @@ -49,4 +49,4 @@ repo=${1} trafficurl="https://api.github.com/repos/citusdata/${repo}/traffic/clones" curl -sf -H "${hubauth}" -H "${preview}" "${trafficurl}" | -${jq} -r "include \"pkg\"; .clones[] | filterdate(\"${since}\") | makeclonerows(\"${repo}\") | @csv" + ${jq} -r "include \"pkg\"; .clones[] | filterdate(\"${since}\") | makeclonerows(\"${repo}\") | @csv" diff --git a/dashboard/homebrew_downloads b/dashboard/homebrew_downloads index 9da95d56..992d2da1 100755 --- a/dashboard/homebrew_downloads +++ b/dashboard/homebrew_downloads @@ -14,7 +14,7 @@ jq=$(which jq) # outputs usage message on specified device before exiting with provided status usage() { - cat << 'E_O_USAGE' >&"$1" + cat <<'E_O_USAGE' >&"$1" usage: homebrew_downloads package [since] package : a package name, such as citus @@ -27,7 +27,7 @@ present (i.e. "6.0.0_1"), they are normalized to the upstream version (in the previous case, "6.0.0") and have their installs aggregated under that. E_O_USAGE - exit "${2}"; + exit "${2}" } if [ "$#" -eq 1 ]; then @@ -42,12 +42,12 @@ else fi case "${OSTYPE}" in - darwin*) - enddate=$(date -v-1d "+%Y-%m-%d") - ;; - *) - enddate=$(date -d "1 days ago" "+%Y-%m-%d") - ;; +darwin*) + enddate=$(date -v-1d "+%Y-%m-%d") + ;; +*) + enddate=$(date -d "1 days ago" "+%Y-%m-%d") + ;; esac echo '"os","release","name","pg_version","version","date","downloads"' diff --git a/dashboard/packagecloud_downloads b/dashboard/packagecloud_downloads index 5b96f031..d355375a 100755 --- a/dashboard/packagecloud_downloads +++ b/dashboard/packagecloud_downloads @@ -16,7 +16,7 @@ jq=$(which jq) # outputs usage message on specified device before exiting with provided status usage() { - cat << 'E_O_USAGE' >&"$1" + cat <<'E_O_USAGE' >&"$1" usage: packagecloud_downloads repo [since] repo : 'community' or 'enterprise' @@ -30,7 +30,7 @@ must queried. Debian/Red Hat versions are normalized back to a presumed git tag in order to better facilitate grouping and aggregation. E_O_USAGE - exit "${2}"; + exit "${2}" } if [ "$#" -eq 1 ]; then @@ -49,24 +49,22 @@ echo '"os","release","name","pg_version","version","date","downloads"' repo=${1} pkgapiurl="https://packagecloud.io/api/v1/repos/citusdata/${repo}/packages.json?per_page=100" -for ((page=1; ; page++)) -do +for ((page = 1; ; page++)); do totalresults=$(curl -sf -u "${pkgauth}" "${pkgapiurl}&page=${page}") totalcount=$(echo ${totalresults} | ${jq} 'length') if [ "${totalcount}" -le 0 ]; then - break + break fi results=$(echo "${totalresults}" | ${jq} 'include "pkg"; stripdebuginfo') count=$(echo ${results} | ${jq} 'length') - for ((i=0; i<${count}; i++)) - do + for ((i = 0; i < ${count}; i++)); do result=$(echo ${results} | ${jq} -r --argjson i "${i}" '.[$i]') series_url=$(echo ${result} | ${jq} -r "include \"pkg\"; extracturl(${since})") curl -sf -u "${pkgauth}" "https://packagecloud.io${series_url}" | - ${jq} --argjson r "${result}" -r 'include "pkg"; stripzeros | .[] | makerow($r) | @csv' + ${jq} --argjson r "${result}" -r 'include "pkg"; stripzeros | .[] | makerow($r) | @csv' done done diff --git a/dashboard/rubygem_installs b/dashboard/rubygem_installs index 03ec7763..478d73fd 100755 --- a/dashboard/rubygem_installs +++ b/dashboard/rubygem_installs @@ -14,7 +14,7 @@ jq=$(which jq) # outputs usage message on specified device before exiting with provided status usage() { - cat << 'E_O_USAGE' >&"$1" + cat <<'E_O_USAGE' >&"$1" usage: rubygem_installs repo [since] repo : a RubyGems.org gem name @@ -27,7 +27,7 @@ most recent time point's date as the 'since' parameter will cause this program to exit without emitting data if that parameter is on or after today's date. E_O_USAGE - exit "${2}"; + exit "${2}" } if [ "$#" -eq 1 ]; then @@ -53,4 +53,4 @@ gem=${1} versionsurl="https://rubygems.org/api/v1/versions/${gem}.json" curl -sf "${versionsurl}" | -${jq} -r "include \"pkg\"; makegemrows(\"${gem}\") | .[] | @csv" + ${jq} -r "include \"pkg\"; makegemrows(\"${gem}\") | .[] | @csv" diff --git a/dashboard/travis_builds b/dashboard/travis_builds index 51033156..57e0c106 100755 --- a/dashboard/travis_builds +++ b/dashboard/travis_builds @@ -19,7 +19,7 @@ jq=$(which jq) # outputs usage message on specified device before exiting with provided status usage() { - cat << 'E_O_USAGE' >&"$1" + cat <<'E_O_USAGE' >&"$1" usage: travis_builds repo [since] repo : a citusdata GitHub repository name @@ -35,7 +35,7 @@ script does not do daily aggregation of build statistics and emits a row for each build. E_O_USAGE - exit "${2}"; + exit "${2}" } if [ "$#" -eq 1 ]; then @@ -52,7 +52,7 @@ fi # first we need to request a Travis auth token using our GitHub token authurl='https://api.travis-ci.org/auth/github' tvstoken=$(curl -sf -A "${agent}" -H "${format}" -H "${content}" \ - -d "${hubtoken}" ${authurl} | jq -r '.access_token') + -d "${hubtoken}" ${authurl} | jq -r '.access_token') tvsauth="Authorization: token \"${tvstoken}\"" echo '"name","number","date","job_count"' @@ -64,12 +64,11 @@ repo=${1} buildsurl="https://api.travis-ci.org/repos/citusdata/${repo}/builds" # exit once we see the page with our last known build, 'since' -for (( sinceidx=-1; sinceidx<0; )) -do +for ((sinceidx = -1; sinceidx < 0; )); do # get page after 'after'-th build. Only care about 'builds' field pageurl="${buildsurl}?after_number=${after}" results=$(curl -sf -A "${agent}" -H "${format}" -H "${tvsauth}" "${pageurl}" | - ${jq} '.builds') + ${jq} '.builds') # sort build numbers; find lowest as well as possible index of 'since' numbers=$(echo ${results} | ${jq} 'map(.number | tonumber) | sort') @@ -78,6 +77,6 @@ do # print out CSV rows for valid builds greater than 'since' echo "${results}" | - ${jq} -r --argjson s "${since}" --argjson r "\"${repo}\"" \ - 'include "pkg"; .[] | filterbuilds($s) | maketravisrows($r) | @csv' + ${jq} -r --argjson s "${since}" --argjson r "\"${repo}\"" \ + 'include "pkg"; .[] | filterbuilds($s) | maketravisrows($r) | @csv' done diff --git a/dashboard/update_stats b/dashboard/update_stats index 093efe8a..ba786c57 100755 --- a/dashboard/update_stats +++ b/dashboard/update_stats @@ -15,7 +15,7 @@ if [ -e ~/.dashboard_secrets ]; then fi while read -r line; do - IFS=',' read -r script repo_or_pkg table qual <<< "$line" + IFS=',' read -r script repo_or_pkg table qual <<<"$line" since=$(psql -qXtAc "SELECT COALESCE((SELECT max(date + 1) FROM ${table} WHERE ${qual}), '2016-01-01'::date);" "${DATABASE_URL}") diff --git a/packaging/README.md b/packaging/README.md index 097c5ed9..61ebea0b 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -10,10 +10,10 @@ ## Usage -First, please read `man citus_package`, we have a man page for it :) +First, please read `man citus_package` , we have a man page for it :) Ensure your `GITHUB_TOKEN` environment variable is properly set (see the man page if you're not sure how to do that). Make sure Docker is running, then you're off to the races! For example, to build a `citus` community "release" on Debian Jessie and Ubuntu Xenial, first change your directory into "citusdata/packaging" repo directory and then checkout the `all-citus` (would be `all-enterprise` for enterprise) branch as this branch has the specific `pkgvars` for community packages. Then execute the following: -`citus_package -p debian/jessie -p ubuntu/focal local release` + `citus_package -p debian/jessie -p ubuntu/focal local release` [1]: https://www.docker.com diff --git a/packaging_automation/README.md b/packaging_automation/README.md index 40e29df2..b220f98e 100644 --- a/packaging_automation/README.md +++ b/packaging_automation/README.md @@ -8,7 +8,7 @@ git clone https://github.com/citusdata/tools.git Enter 'tools' directory -``` console +```console cd tools ``` @@ -16,25 +16,25 @@ cd tools Verify pip installation -``` console +```console python -m pip --version ``` Output should be like following -``` console +```console pip 21.1.2 from /home/vagrant/.local/lib/python3.8/site-packages/pip (python 3.8) ``` If you get error, you should first install pip -``` console +```console sudo apt install python3-pip ``` Install the required libraries to execute the script -``` console +```console python -m pip install -r packaging_automation/requirements.txt ``` @@ -66,7 +66,7 @@ citus-enterprise' 'enterprise-master' **--is_test:** If used, branches would not be pushed remote repository and created release branches would be prefixed with 'test'. Default value is False -**--cherry_pick_enabled:** Available only for patch release. If used, --earliest_pr_date flag also should be used.Gets +**--cherry_pick_enabled:** Available only for patch release. If used, --earliest_pr_date flag also should be used. Gets all PR's with 'backport' label created after earliest_pr_date **--earliest_pr_date:** Used with --cherry-pick-enabled flag. Date format is 'Y.m.d' e.g 2012.01.21. PR's merged after @@ -78,17 +78,18 @@ this date would be listed and cherry-picked. #### Major -``` console +```console python -m packaging_automation.prepare_release --gh_token --prj_name citus --prj_ver 10.1.0 ``` #### Patch -``` console +```console python -m packaging_automation.prepare_release --gh_token --prj_name citus-enterprise --prj_ver 10.0.4 --schema_version 10.0-5 ``` -## Update Package Properties +## Update Package Properties + Update package properties script updates debian and redhat package configuration files. ## Script Usage @@ -96,6 +97,7 @@ Update package properties script updates debian and redhat package configuration Script can be used in projects following: citus, citus-enterprise, pg-auto-failover, pg-auto-failover-enterprise ## Available flags + **--gh_token:** Personal access token that is authorized to commit citus/citus-enterprise projects. (Required) **--prj_name:** Project to be released. Allowed values 'citus' and 'citus-enterprise (Required) @@ -127,8 +129,7 @@ python -m packaging_automation.update_package_properties --gh_token=${{ secrets. --exec_path "$(pwd)" ``` - -## Update Docker +## Update Docker Update docker script updates the docker and changelog files in docker repository required for new release of docker images after citus/postgres release @@ -149,21 +150,21 @@ Script can be used for both citus version upgrades and PostgreSQL updates. **--is_test:** If used, branches would not be pushed remote repository and PR would not be created (Optional) -### Example +### Example #### Citus Upgrade -``` console +```console python -m packaging_automation.update_docker --gh_token --prj_ver 10.0.4 ``` #### Citus and PostgreSQL version upgrade -``` console +```console python -m packaging_automation.update_docker --gh_token --prj_ver 10.0.4 --postgres-version 14.0 ``` -## Update Pgxn +## Update Pgxn Update pgxn script updates the files related to pgxn in all-pgxn branch in packaging repo. @@ -183,15 +184,17 @@ Script can be used for citus version upgrades. ### Example -``` console +```console python -m packaging_automation.update_pgxn --gh_token --prj_ver 10.0.4 ``` -## Upload to package cloud -This script uploads built deb and rpm packages. +## Upload to package cloud + +This script uploads built deb and rpm packages. ## Script usage -This script uploads all the rpm and deb packages from given directory into package cloud,if current branch equals to main branch . + +This script uploads all the rpm and deb packages from given directory into package cloud, if current branch equals to main branch . ### Available flags @@ -199,7 +202,7 @@ This script uploads all the rpm and deb packages from given directory into packa **--package_cloud_api_token:** Token required to get authorization from package cloud to upload (Required) -**--repository_name:** Packagecloud repository name to upload Available repos: "sample","citusdata/enterprise","citusdata/community","citusdata/community-nightlies","citusdata/enterprise-nightlies","citusdata/azure" (Required) +**--repository_name:** Packagecloud repository name to upload Available repos: "sample", "citusdata/enterprise", "citusdata/community", "citusdata/community-nightlies", "citusdata/enterprise-nightlies", "citusdata/azure" (Required) **--output_file_path:** Directory that contains deb and rpm files (Required) @@ -209,7 +212,7 @@ This script uploads all the rpm and deb packages from given directory into packa ### Example -``` console +```console python -m tools.packaging_automation.upload_to_package_cloud \ --platform ${{ matrix.platform }} \ --package_cloud_api_token ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} \ @@ -220,28 +223,29 @@ This script uploads all the rpm and deb packages from given directory into packa ``` ## Publish docker + This script builds and publishes given docker image type ## Script Usage + Script executes docker build on given image type and publishes the docker image with related tags ### Available flags **--github_ref:** Github Action parameter denoting tag or branch name depending on trigger type . (Required) -**--pipeline_trigger_type:** Pipeline trigger type. Available option: push,schedule, workflow_dispatch (Required) +**--pipeline_trigger_type:** Pipeline trigger type. Available option: push, schedule, workflow_dispatch (Required) **--tag_name:** Tag name if trigger type is push and -**--manual_trigger_type:** Trigger type when executing the script manually. Available options: main,tags,nightly (Required) +**--manual_trigger_type:** Trigger type when executing the script manually. Available options: main, tags, nightly (Required) -**--image_type:** Image type to be published. Available options: latest,alpine,nightly, postgre12 +**--image_type:** Image type to be published. Available options: latest, alpine, nightly, postgre12 ### Example -``` console +```console python -m tools.packaging_automation.publish_docker --pipeline_trigger_type "${GITHUB_EVENT_NAME}" \ --exec_path "$(pwd)" --tag_name ${{ github.event.inputs.tag_name }} \ --manual_trigger_type ${{ github.event.inputs.trigger_type }} ``` - diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index ce2978fc..b7da5850 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -82,7 +82,8 @@ def decode_os_and_release(platform_name: str) -> Tuple[str, str]: parts = platform_name.split("/") if len(parts) == 0 or len(parts) > 2 or (len(parts) == 1 and parts[0] != "pgxn"): - raise ValueError("Platforms should have two parts divided by '/' or should be 'pgxn' ") + raise ValueError( + "Platforms should have two parts divided by '/' or should be 'pgxn' ") if len(parts) == 1 and parts[0] == "pgxn": os_name = "pgxn" os_release = "" @@ -104,7 +105,7 @@ def is_docker_running() -> bool: docker_client = docker.from_env() docker_client.ping() return True - ## Exception type is not defined in API so I keep as is + # Exception type is not defined in API so I keep as is except: # noqa: E722 # pylint: disable=bare-except return False @@ -146,7 +147,8 @@ def get_signing_credentials(packaging_secret_key: str, gpg = gnupg.GPG() - private_key = gpg.export_keys(fingerprints[0], secret=True, passphrase=packaging_passphrase) + private_key = gpg.export_keys( + fingerprints[0], secret=True, passphrase=packaging_passphrase) secret_key = transform_key_into_base64_str(private_key) passphrase = packaging_passphrase @@ -156,7 +158,8 @@ def get_signing_credentials(packaging_secret_key: str, def write_postgres_versions_into_file(input_files_dir: str, package_version: str): release_versions = get_supported_postgres_release_versions(f"{input_files_dir}/{POSTGRES_MATRIX_FILE_NAME}", package_version) - nightly_versions = get_supported_postgres_nightly_versions(f"{input_files_dir}/{POSTGRES_MATRIX_FILE_NAME}") + nightly_versions = get_supported_postgres_nightly_versions( + f"{input_files_dir}/{POSTGRES_MATRIX_FILE_NAME}") release_version_str = ','.join(release_versions) nightly_version_str = ','.join(nightly_versions) with open(f"{input_files_dir}/{POSTGRES_VERSION_FILE}", 'w', encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, @@ -181,9 +184,11 @@ def sign_packages(sub_folder: str, signing_credentials: SigningCredentials, print(f"Result:{output}") if result.returncode != 0: - raise ValueError(f"Error while signing rpm files.Err:{result.stderr}") + raise ValueError( + f"Error while signing rpm files.Err:{result.stderr}") if input_output_parameters.output_validation: - validate_output(output, f"{input_output_parameters.input_files_dir}/packaging_ignore.yml", PackageType.rpm) + validate_output( + output, f"{input_output_parameters.input_files_dir}/packaging_ignore.yml", PackageType.rpm) print("RPM signing finished successfully.") @@ -200,7 +205,8 @@ def sign_packages(sub_folder: str, signing_credentials: SigningCredentials, print(f"Result:{output}") if result.returncode != 0: - raise ValueError(f"Error while signing deb files.Err:{result.stdout}") + raise ValueError( + f"Error while signing deb files.Err:{result.stdout}") if input_output_parameters.output_validation: validate_output(result.stdout, f"{input_output_parameters.input_files_dir}/packaging_ignore.yml", @@ -217,7 +223,8 @@ def get_postgres_versions(os_name: str, input_files_dir: str) -> Tuple[List[str] package_version = get_package_version_from_pkgvars(input_files_dir) release_versions = get_supported_postgres_release_versions(f"{input_files_dir}/{POSTGRES_MATRIX_FILE_NAME}", package_version) - nightly_versions = get_supported_postgres_nightly_versions(f"{input_files_dir}/{POSTGRES_MATRIX_FILE_NAME}") + nightly_versions = get_supported_postgres_nightly_versions( + f"{input_files_dir}/{POSTGRES_MATRIX_FILE_NAME}") return release_versions, nightly_versions @@ -267,11 +274,15 @@ def build_packages(github_token: non_empty(non_blank(str)), build_type: BuildType, signing_credentials: SigningCredentials, input_output_parameters: InputOutputParameters, is_test: bool = False) -> None: os_name, os_version = decode_os_and_release(platform) - release_versions, nightly_versions = get_postgres_versions(os_name, input_output_parameters.input_files_dir) - signing_credentials = get_signing_credentials(signing_credentials.secret_key, signing_credentials.passphrase) + release_versions, nightly_versions = get_postgres_versions( + os_name, input_output_parameters.input_files_dir) + signing_credentials = get_signing_credentials( + signing_credentials.secret_key, signing_credentials.passphrase) - package_version = get_package_version_from_pkgvars(input_output_parameters.input_files_dir) - write_postgres_versions_into_file(input_output_parameters.input_files_dir, package_version) + package_version = get_package_version_from_pkgvars( + input_output_parameters.input_files_dir) + write_postgres_versions_into_file( + input_output_parameters.input_files_dir, package_version) if not signing_credentials.passphrase: raise ValueError("PACKAGING_PASSPHRASE should not be null or empty") @@ -282,12 +293,15 @@ def build_packages(github_token: non_empty(non_blank(str)), output_sub_folder = get_release_package_folder_name(os_name, os_version) input_output_parameters.output_dir = f"{input_output_parameters.output_dir}/{output_sub_folder}" for postgres_version in postgres_versions: - print(f"Package build for {os_name}-{os_version} for postgres {postgres_version} started... ") + print( + f"Package build for {os_name}-{os_version} for postgres {postgres_version} started... ") build_package(github_token, build_type, docker_image_name, postgres_version, input_output_parameters, is_test) - print(f"Package build for {os_name}-{os_version} for postgres {postgres_version} finished ") + print( + f"Package build for {os_name}-{os_version} for postgres {postgres_version} finished ") - sign_packages(output_sub_folder, signing_credentials, input_output_parameters) + sign_packages(output_sub_folder, signing_credentials, + input_output_parameters) def get_build_platform(packaging_platform: str, packaging_docker_platform: str) -> str: @@ -303,7 +317,8 @@ def get_package_version_from_pkgvars(input_files_dir: str): pkg_name = pkgvars_config["pkgname"] if len(version_parts) < 3: - raise ValueError("Version should at least contains three parts seperated with '.'. e.g 10.0.2-1") + raise ValueError( + "Version should at least contains three parts seperated with '.'. e.g 10.0.2-1") third_part_splitted = version_parts[2].split("-") if pkg_name == 'hll': @@ -317,7 +332,8 @@ def get_package_version_from_pkgvars(input_files_dir: str): parser = argparse.ArgumentParser() parser.add_argument('--gh_token', required=True) parser.add_argument('--platform', required=False, choices=platform_names()) - parser.add_argument('--packaging_docker_platform', required=False, choices=package_docker_platform_dict.keys()) + parser.add_argument('--packaging_docker_platform', required=False, + choices=package_docker_platform_dict.keys()) parser.add_argument('--build_type', choices=[b.name for b in BuildType]) parser.add_argument('--secret_key', required=True) parser.add_argument('--passphrase', required=True) @@ -329,10 +345,13 @@ def get_package_version_from_pkgvars(input_files_dir: str): args = parser.parse_args() if args.platform and args.packaging_docker_platform: - raise ValueError("Either platform or packaging_docker_platform should be set.") - build_platform = get_build_platform(args.platform, args.packaging_docker_platform) + raise ValueError( + "Either platform or packaging_docker_platform should be set.") + build_platform = get_build_platform( + args.platform, args.packaging_docker_platform) - io_parameters = InputOutputParameters.build(args.input_files_dir, args.output_dir, args.output_validation) + io_parameters = InputOutputParameters.build( + args.input_files_dir, args.output_dir, args.output_validation) sign_credentials = SigningCredentials(args.secret_key, args.passphrase) build_packages(args.gh_token, build_platform, BuildType[args.build_type], sign_credentials, io_parameters, args.is_test) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 10981ba9..0166715a 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -43,6 +43,7 @@ "ubuntu": ["focal", "bionic", "trusty"] } + def platform_names() -> List[str]: platforms = [] for platform_os, platform_releases in supported_platforms.items(): @@ -140,7 +141,8 @@ def run_with_output(command, *args, **kwargs): # this method's main objective is to return output. Therefore it is caller's responsibility to handle # success status # pylint: disable=subprocess-run-check - result = subprocess.run(shlex.split(command), *args, capture_output=True, **kwargs) + result = subprocess.run(shlex.split(command), *args, + capture_output=True, **kwargs) return result @@ -149,7 +151,8 @@ def cherry_pick_prs(prs: List[PullRequest.PullRequest]): commits = pr.get_commits() for single_commit in commits: if not is_merge_commit(single_commit): - cp_result = run(f"git cherry-pick -x {single_commit.commit.sha}") + cp_result = run( + f"git cherry-pick -x {single_commit.commit.sha}") print( f"Cherry pick result for PR no {pr.number} and commit sha {single_commit.commit.sha}: {cp_result}") @@ -204,7 +207,8 @@ def str_array_to_str(str_array: List[str]) -> str: def get_prs_for_patch_release(repo: Repository.Repository, earliest_date: datetime, base_branch: str, last_date: datetime = None): - pull_requests = repo.get_pulls(state="closed", base=base_branch, sort="created", direction="desc") + pull_requests = repo.get_pulls( + state="closed", base=base_branch, sort="created", direction="desc") # filter pull requests according to given time interval filtered_pull_requests = [] @@ -220,7 +224,8 @@ def get_prs_for_patch_release(repo: Repository.Repository, earliest_date: dateti filtered_pull_requests.append(pull_request) # finally, sort the pr's by their merge date - sorted_pull_requests = sorted(filtered_pull_requests, key=lambda p: p.merged_at) + sorted_pull_requests = sorted( + filtered_pull_requests, key=lambda p: p.merged_at) return sorted_pull_requests @@ -284,7 +289,8 @@ def append_line_in_file(file: str, match_regex: str, append_str: str) -> bool: # increment of appended_line_index is 2 since copy_lines appended_line_index+1 includes # append_str lines_to_be_shifted = lines[line_number + 1:] - copy_lines = copy_lines[0:appended_line_index + 2] + lines_to_be_shifted + copy_lines = copy_lines[0:appended_line_index + + 2] + lines_to_be_shifted else: copy_lines.append(append_str) appended_line_index = appended_line_index + 1 @@ -309,7 +315,8 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: # Since line is added before matched string shift index start with line_number # increment of prepend_line_index is 1 line after prepended_line_index should be shifted lines_to_be_shifted = lines[line_number:] - copy_lines = copy_lines[0:prepended_line_index + 1] + lines_to_be_shifted + copy_lines = copy_lines[0:prepended_line_index + + 1] + lines_to_be_shifted prepended_line_index = prepended_line_index + 1 edited_content = str_array_to_str(copy_lines) with open(file, "w", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as writer: @@ -321,7 +328,8 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: def is_tag_on_branch(tag_name: str, branch_name: str): g = git.Git(os.getcwd()) try: - branches_str = g.execute(["git", "branch", "--contains", f"tags/{tag_name}"]) + branches_str = g.execute( + ["git", "branch", "--contains", f"tags/{tag_name}"]) branches = remove_prefix(branches_str, "*").split("\n") print("Branches str:" + branches_str) if len(branches) > 0: @@ -401,7 +409,8 @@ def write_to_file(content: str, dest_file_name: str): def get_gpg_fingerprints_by_name(name: str) -> List[str]: '''Returns GPG fingerprint by its unique key name. We use this function to determine the fingerprint that we should use when signing packages''' - result = subprocess.run(shlex.split("gpg --list-keys"), check=True, stdout=subprocess.PIPE) + result = subprocess.run(shlex.split("gpg --list-keys"), + check=True, stdout=subprocess.PIPE) lines = result.stdout.decode("ascii").splitlines() finger_prints = [] previous_line = "" @@ -454,7 +463,8 @@ def delete_all_gpg_keys_by_name(name: str): def get_private_key_by_fingerprint_without_passphrase(fingerprint: str) -> str: gpg = gnupg.GPG() - private_key = gpg.export_keys(fingerprint, secret=True, expect_passphrase=False) + private_key = gpg.export_keys( + fingerprint, secret=True, expect_passphrase=False) if not private_key: raise ValueError( "Error while getting key. Most probably packaging key is stored with passphrase. " @@ -465,7 +475,8 @@ def get_private_key_by_fingerprint_without_passphrase(fingerprint: str) -> str: def get_private_key_by_fingerprint_with_passphrase(fingerprint: str, passphrase: str) -> str: gpg = gnupg.GPG() - private_key = gpg.export_keys(fingerprint, secret=True, passphrase=passphrase) + private_key = gpg.export_keys( + fingerprint, secret=True, passphrase=passphrase) if not private_key: raise ValueError( "Error while getting key. Most probably packaging key is stored with passphrase. " @@ -482,7 +493,8 @@ def transform_key_into_base64_str(key: str) -> str: def define_rpm_public_key_to_machine(fingerprint: str): with open("rpm_public.key", "w", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as writer: - subprocess.run(shlex.split(f"gpg --export -a {fingerprint}"), stdout=writer, check=True) + subprocess.run(shlex.split( + f"gpg --export -a {fingerprint}"), stdout=writer, check=True) run("rpm --import rpm_public.key") os.remove("rpm_public.key") @@ -524,7 +536,8 @@ def verify_rpm_signature_in_dir(rpm_dir_path: str): rpm_files = filter(lambda file_name: file_name.endswith("rpm"), files) for file in rpm_files: if not is_rpm_file_signed(f"{file}"): - raise ValueError(f"File {file} is not signed or there is a signature check problem") + raise ValueError( + f"File {file} is not signed or there is a signature check problem") def remove_prefix(text, prefix): @@ -546,13 +559,15 @@ def remove_suffix(initial_str: str, suffix: str) -> str: def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): - run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") + run( + f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr(gh_token: str, pr_branch: str, pr_title: str, repo_owner: str, project_name: str, base_branch: str): g = Github(gh_token) repository = g.get_repo(f"{repo_owner}/{project_name}") - create_pr_with_repo(repo=repository, pr_branch=pr_branch, pr_title=pr_title, base_branch=base_branch) + create_pr_with_repo(repo=repository, pr_branch=pr_branch, + pr_title=pr_title, base_branch=base_branch) def create_pr_with_repo(repo: Repository, pr_branch: str, pr_title: str, base_branch: str): @@ -560,7 +575,8 @@ def create_pr_with_repo(repo: Repository, pr_branch: str, pr_title: str, base_br def stat_get_request(request_address: str, request_type: RequestType, session): - request_log = RequestLog(request_time=datetime.now(), request_type=request_type) + request_log = RequestLog( + request_time=datetime.now(), request_type=request_type) session.add(request_log) session.commit() try: @@ -570,7 +586,8 @@ def stat_get_request(request_address: str, request_type: RequestType, session): except requests.exceptions.RequestException as e: result = e.response request_log.status_code = -1 - request_log.response = e.response.content.decode("ascii") if e.response.content.decode("ascii") else str(e) + request_log.response = e.response.content.decode( + "ascii") if e.response.content.decode("ascii") else str(e) finally: session.commit() return result @@ -586,7 +603,8 @@ def get_supported_postgres_release_versions(postgres_matrix_conf_file_path: str, for version_info in yaml_content['version_matrix']: versions_dictionary[list(version_info.keys())[0]] = \ version_info[list(version_info.keys())[0]]['postgres_versions'] - release_versions = match_release_version(versions_dictionary, package_version) + release_versions = match_release_version( + versions_dictionary, package_version) return release_versions @@ -598,7 +616,8 @@ def get_supported_postgres_nightly_versions(postgres_matrix_conf_file_path: str) # nightly version is the last element in the postgres matrix latest_version_info = yaml_content['version_matrix'][-1] - nightly_versions = latest_version_info[list(latest_version_info.keys())[0]]["postgres_versions"] + nightly_versions = latest_version_info[list(latest_version_info.keys())[ + 0]]["postgres_versions"] return nightly_versions @@ -606,8 +625,10 @@ def match_release_version(versions_dictionary, package_version: str): versions = list(versions_dictionary.keys()) numeric_versions_of_config: Dict[int, str] = {} for version in versions: - numeric_versions_of_config[get_numeric_counterpart_of_version(version)] = version - package_version_numeric = get_numeric_counterpart_of_version(package_version) + numeric_versions_of_config[get_numeric_counterpart_of_version( + version)] = version + package_version_numeric = get_numeric_counterpart_of_version( + package_version) if package_version_numeric in numeric_versions_of_config: version_in_str = numeric_versions_of_config[package_version_numeric] diff --git a/packaging_automation/docker_statistics_collector.py b/packaging_automation/docker_statistics_collector.py index a72226b6..e42920a2 100644 --- a/packaging_automation/docker_statistics_collector.py +++ b/packaging_automation/docker_statistics_collector.py @@ -25,19 +25,24 @@ def fetch_and_store_docker_statistics(repository_name: str, db_parameters: DbPar test_day_shift_index: int = 0, test_total_pull_count: int = 0): if repository_name not in docker_repositories: - raise ValueError(f"Repository name should be in {str_array_to_str(docker_repositories)}") + raise ValueError( + f"Repository name should be in {str_array_to_str(docker_repositories)}") if not is_test and (test_day_shift_index != 0 or test_total_pull_count != 0): raise ValueError("test_day_shift_index and test_total_pull_count parameters are test " "parameters. Please don't use these parameters other than testing.") - result = requests.get(f"https://hub.docker.com/v2/repositories/citusdata/{repository_name}/") - total_pull_count = int(result.json()["pull_count"]) if test_total_pull_count == 0 else test_total_pull_count + result = requests.get( + f"https://hub.docker.com/v2/repositories/citusdata/{repository_name}/") + total_pull_count = int(result.json()[ + "pull_count"]) if test_total_pull_count == 0 else test_total_pull_count - session = db_session(db_params=db_parameters, is_test=is_test, create_db_objects=True) + session = db_session(db_params=db_parameters, + is_test=is_test, create_db_objects=True) fetch_date = datetime.now() + timedelta(days=test_day_shift_index) validate_same_day_record_existence(fetch_date, session) - day_diff, mod_pull_diff, pull_diff = calculate_diff_params(fetch_date, session, total_pull_count) + day_diff, mod_pull_diff, pull_diff = calculate_diff_params( + fetch_date, session, total_pull_count) for i in range(0, day_diff): daily_pull_count = ((pull_diff - mod_pull_diff) / day_diff if i > 0 else (pull_diff - mod_pull_diff) / day_diff + mod_pull_diff) @@ -49,17 +54,22 @@ def fetch_and_store_docker_statistics(repository_name: str, db_parameters: DbPar def calculate_diff_params(fetch_date, session, total_pull_count): - last_stat_record = session.query(DockerStats).order_by(desc(DockerStats.stat_date)).first() - day_diff = (fetch_date.date() - last_stat_record.stat_date).days if last_stat_record else 1 - pull_diff = total_pull_count - last_stat_record.total_pull_count if last_stat_record else total_pull_count + last_stat_record = session.query(DockerStats).order_by( + desc(DockerStats.stat_date)).first() + day_diff = (fetch_date.date() - + last_stat_record.stat_date).days if last_stat_record else 1 + pull_diff = total_pull_count - \ + last_stat_record.total_pull_count if last_stat_record else total_pull_count mod_pull_diff = pull_diff % day_diff return day_diff, mod_pull_diff, pull_diff def validate_same_day_record_existence(fetch_date, session): - same_day_record = session.query(DockerStats).filter_by(stat_date=fetch_date.date()).first() + same_day_record = session.query(DockerStats).filter_by( + stat_date=fetch_date.date()).first() if same_day_record: - print(f"Docker download record for date {fetch_date.date()} already exists. No need to add record.") + print( + f"Docker download record for date {fetch_date.date()} already exists. No need to add record.") sys.exit(0) @@ -79,5 +89,6 @@ def validate_same_day_record_existence(fetch_date, session): host_and_port=arguments.db_host_and_port, db_name=arguments.db_name) fetch_and_store_docker_statistics(repository_name=arguments.repo_name, is_test=arguments.is_test, - db_parameters=db_params, test_day_shift_index=int(arguments.test_day_shift_index), + db_parameters=db_params, test_day_shift_index=int( + arguments.test_day_shift_index), test_total_pull_count=int(arguments.test_total_pull_count)) diff --git a/packaging_automation/get_postgres_versions.py b/packaging_automation/get_postgres_versions.py index 525f22fd..e9e6eb5f 100644 --- a/packaging_automation/get_postgres_versions.py +++ b/packaging_automation/get_postgres_versions.py @@ -8,5 +8,6 @@ parser.add_argument('--project_version', required=True) args = parser.parse_args() - postgres_versions = get_postgres_versions_from_matrix_file(args.project_version) + postgres_versions = get_postgres_versions_from_matrix_file( + args.project_version) print(json.dumps(postgres_versions)) diff --git a/packaging_automation/github_statistics_collector.py b/packaging_automation/github_statistics_collector.py index 55d4c098..688d8c9f 100644 --- a/packaging_automation/github_statistics_collector.py +++ b/packaging_automation/github_statistics_collector.py @@ -34,7 +34,8 @@ class GithubCloneStatsTransactionsDetail(Base): clone_date = Column(DATE, nullable=False) count = Column(INTEGER, nullable=False) uniques = Column(INTEGER, nullable=False) - parent_id = Column(INTEGER, ForeignKey('github_stats_clone_transactions_main.id'), nullable=False) + parent_id = Column(INTEGER, ForeignKey( + 'github_stats_clone_transactions_main.id'), nullable=False) class GithubCloneStats(Base): @@ -45,7 +46,8 @@ class GithubCloneStats(Base): clone_date = Column(DATE, nullable=False) count = Column(INTEGER, nullable=False) uniques = Column(INTEGER, nullable=False) - __table_args__ = (UniqueConstraint('repo_name', 'clone_date', name='repo_name_clone_date_uq'),) + __table_args__ = (UniqueConstraint( + 'repo_name', 'clone_date', name='repo_name_clone_date_uq'),) class GitHubReleases(Base): @@ -58,12 +60,14 @@ class GitHubReleases(Base): def clone_record_exists(record_time: datetime.date, session) -> bool: - db_record = session.query(GithubCloneStats).filter_by(clone_date=record_time).first() + db_record = session.query(GithubCloneStats).filter_by( + clone_date=record_time).first() return db_record is not None def release_record_exists(tag_name: str, session) -> bool: - db_record = session.query(GitHubReleases).filter_by(tag_name=tag_name).first() + db_record = session.query(GitHubReleases).filter_by( + tag_name=tag_name).first() return db_record is not None @@ -81,8 +85,10 @@ def github_releases(github_token: str, organization_name: str, repo_name: str): def fetch_and_store_github_stats(organization_name: str, repo_name: str, db_parameters: DbParams, github_token: str, is_test: bool): - fetch_and_store_github_clones(organization_name, repo_name, db_parameters, github_token, is_test) - fetch_and_store_github_releases(organization_name, repo_name, db_parameters, github_token, is_test) + fetch_and_store_github_clones( + organization_name, repo_name, db_parameters, github_token, is_test) + fetch_and_store_github_releases( + organization_name, repo_name, db_parameters, github_token, is_test) def fetch_and_store_github_releases(organization_name: str, repo_name: str, db_parameters: DbParams, github_token: str, diff --git a/packaging_automation/homebrew_statistics_collector.py b/packaging_automation/homebrew_statistics_collector.py index b480d7bb..284e756a 100644 --- a/packaging_automation/homebrew_statistics_collector.py +++ b/packaging_automation/homebrew_statistics_collector.py @@ -14,7 +14,7 @@ class HomebrewStats(Base): __tablename__ = "homebrew_stats" id = Column(INTEGER, primary_key=True, autoincrement=True) fetch_time = Column(TIMESTAMP, nullable=False) - stat_date = Column(DATE, nullable=False,unique=True) + stat_date = Column(DATE, nullable=False, unique=True) stat_30d = Column(INTEGER, nullable=False, default=0,) stat_90d = Column(INTEGER, nullable=False, default=0) stat_365d = Column(INTEGER, nullable=False, default=0) @@ -23,9 +23,11 @@ class HomebrewStats(Base): def fetch_and_save_homebrew_stats(db_params: DbParams, is_test: bool) -> None: session = db_session(db_params=db_params, is_test=is_test) - result = stat_get_request(HOMEBREW_STATS_ADDRESS, RequestType.homebrew_download, session) + result = stat_get_request(HOMEBREW_STATS_ADDRESS, + RequestType.homebrew_download, session) stat_details = json.loads(result.content) - record = session.query(HomebrewStats).filter_by(stat_date=date.today()).first() + record = session.query(HomebrewStats).filter_by( + stat_date=date.today()).first() if record is None: hb_stat = HomebrewStats(fetch_time=datetime.now(), stat_date=date.today(), stat_30d=stat_details["analytics"]["install"]["30d"]["citus"], diff --git a/packaging_automation/package_cloud_statistics_collector.py b/packaging_automation/package_cloud_statistics_collector.py index 6678516a..ce163a78 100644 --- a/packaging_automation/package_cloud_statistics_collector.py +++ b/packaging_automation/package_cloud_statistics_collector.py @@ -48,7 +48,8 @@ class PackageCloudDownloadStats(Base): download_date = Column(DATE, nullable=False) download_count = Column(INTEGER, nullable=False) detail_url = Column(String, nullable=False) - UniqueConstraint('package_full_name', 'download_date', 'distro_version', name='ux_package_cloud_download_stats') + UniqueConstraint('package_full_name', 'download_date', + 'distro_version', name='ux_package_cloud_download_stats') class PackageCloudDownloadDetails(Base): @@ -80,7 +81,8 @@ def package_count(organization: PackageCloudOrganization, repo_name: PackageClou for repo in repo_list: if repo["fqname"] == f"{organization.name}/{repo_name.value}": return int(remove_suffix(repo['package_count_human'], PC_PACKAGE_COUNT_SUFFIX)) - raise ValueError(f"Repo name with the name {repo_name.value} could not be found on package cloud") + raise ValueError( + f"Repo name with the name {repo_name.value} could not be found on package cloud") @dataclass @@ -142,13 +144,16 @@ def fetch_and_save_package_stats(package_info, package_cloud_api_token: str, ses repo_name: PackageCloudRepo): '''Gets and saves the package statistics of the given packages''' request_result = stat_get_request( - package_statistics_request_address(package_cloud_api_token, package_info['downloads_series_url']), + package_statistics_request_address( + package_cloud_api_token, package_info['downloads_series_url']), RequestType.package_cloud_download_series_query, session) if request_result.status_code != HTTPStatus.OK: - raise ValueError(f"Error while getting package stat for package {package_info['filename']}") + raise ValueError( + f"Error while getting package stat for package {package_info['filename']}") download_stats = json.loads(request_result.content) for stat_date in download_stats['value']: - download_date = datetime.strptime(stat_date, PC_DOWNLOAD_DATE_FORMAT).date() + download_date = datetime.strptime( + stat_date, PC_DOWNLOAD_DATE_FORMAT).date() download_count = int(download_stats['value'][stat_date]) if (download_date != date.today() and not is_ignored_package(package_info['name']) and not stat_records_exists(download_date, package_info['filename'], package_info['distro_version'], @@ -171,7 +176,8 @@ def fetch_and_save_package_stats(package_info, package_cloud_api_token: str, ses def fetch_and_save_package_download_details(package_info, package_cloud_admin_api_token: str, session, repo_name: PackageCloudRepo): - print(f"Download Detail Query for {package_info['filename']}: {package_info['downloads_detail_url']}") + print( + f"Download Detail Query for {package_info['filename']}: {package_info['downloads_detail_url']}") page_number = 1 record_count = DEFAULT_PAGE_RECORD_COUNT while record_count == DEFAULT_PAGE_RECORD_COUNT: @@ -188,14 +194,15 @@ def fetch_and_save_package_download_details(package_info, package_cloud_admin_ap record_count = len(download_details) for download_detail in download_details: - downloaded_at = datetime.strptime(download_detail['downloaded_at'], PC_DOWNLOAD_DETAIL_DATE_FORMAT) + downloaded_at = datetime.strptime( + download_detail['downloaded_at'], PC_DOWNLOAD_DETAIL_DATE_FORMAT) download_date = downloaded_at.date() if (download_date != date.today() and not is_ignored_package(package_info['name']) and not stat_records_exists(download_date, package_info['filename'], package_info['distro_version'], session)): download_detail_record = PackageCloudDownloadDetails(fetch_date=datetime.now(), repo=repo_name, package_full_name=package_info['filename'], - package_name=package_info['name'], + package_name=package_info['name'], distro_version=package_info['distro_version'], package_version=package_info['version'], package_release=package_info['release'], @@ -260,17 +267,22 @@ def is_ignored_package(package_name: str) -> bool: if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--organization', choices=[r.value for r in PackageCloudOrganization]) - parser.add_argument('--repo_name', choices=[r.value for r in PackageCloudRepo]) + parser.add_argument( + '--organization', choices=[r.value for r in PackageCloudOrganization]) + parser.add_argument( + '--repo_name', choices=[r.value for r in PackageCloudRepo]) parser.add_argument('--db_user_name', required=True) parser.add_argument('--db_password', required=True) parser.add_argument('--db_host_and_port', required=True) parser.add_argument('--db_name', required=True) parser.add_argument('--package_cloud_api_token', required=True) parser.add_argument('--package_cloud_admin_api_token', required=True) - parser.add_argument('--parallel_count', type=int, choices=range(1, 30), required=True, default=1) - parser.add_argument('--parallel_exec_index', type=int, choices=range(0, 30), required=True, default=0) - parser.add_argument('--page_record_count', type=int, choices=range(5, 101), required=True, default=0) + parser.add_argument('--parallel_count', type=int, + choices=range(1, 30), required=True, default=1) + parser.add_argument('--parallel_exec_index', type=int, + choices=range(0, 30), required=True, default=0) + parser.add_argument('--page_record_count', type=int, + choices=range(5, 101), required=True, default=0) parser.add_argument('--is_test', action="store_true") arguments = parser.parse_args() @@ -280,7 +292,8 @@ def is_ignored_package(package_name: str) -> bool: package_cloud_parameters = PackageCloudParams(admin_api_token=arguments.package_cloud_admin_api_token, standard_api_token=arguments.package_cloud_api_token, - organization=PackageCloudOrganization(arguments.organization), + organization=PackageCloudOrganization( + arguments.organization), repo_name=PackageCloudRepo(arguments.repo_name)) parallel_execution_params = ParallelExecutionParams(parallel_count=arguments.parallel_count, parallel_exec_index=arguments.parallel_exec_index, diff --git a/packaging_automation/packaging_warning_handler.py b/packaging_automation/packaging_warning_handler.py index 88eafeef..25edd4a3 100644 --- a/packaging_automation/packaging_warning_handler.py +++ b/packaging_automation/packaging_warning_handler.py @@ -5,7 +5,8 @@ import yaml -from .common_tool_methods import (PackageType, DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER) +from .common_tool_methods import ( + PackageType, DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER) class PackagingWarningIgnoreType(Enum): @@ -15,20 +16,26 @@ class PackagingWarningIgnoreType(Enum): def validate_output(output: str, ignore_file_path: str, package_type: PackageType): - base_ignore_list, package_type_specific_ignore_list = parse_ignore_lists(ignore_file_path, package_type) + base_ignore_list, package_type_specific_ignore_list = parse_ignore_lists( + ignore_file_path, package_type) output_lines = output.splitlines() - warning_lines, package_type_specific_warning_lines = filter_warning_lines(output_lines, package_type) + warning_lines, package_type_specific_warning_lines = filter_warning_lines( + output_lines, package_type) print("Checking build output for warnings") print("Package Type:" + package_type.name) - print(f"Package type specific warnings:{package_type_specific_warning_lines}") + print( + f"Package type specific warnings:{package_type_specific_warning_lines}") - base_warnings_to_be_raised = get_warnings_to_be_raised(base_ignore_list, warning_lines) + base_warnings_to_be_raised = get_warnings_to_be_raised( + base_ignore_list, warning_lines) package_type_specific_warnings_to_be_raised = get_warnings_to_be_raised(package_type_specific_ignore_list, package_type_specific_warning_lines) - print(f"Package type specific ignore list:{package_type_specific_ignore_list}") - print(f"Package type specific warnings to be raised:{package_type_specific_warnings_to_be_raised}") + print( + f"Package type specific ignore list:{package_type_specific_ignore_list}") + print( + f"Package type specific warnings to be raised:{package_type_specific_warnings_to_be_raised}") print(f"Base warnings to be raised:{base_warnings_to_be_raised}") if len(base_warnings_to_be_raised) > 0 or len(package_type_specific_warnings_to_be_raised) > 0: @@ -56,7 +63,8 @@ def filter_warning_lines(output_lines: List[str], package_type: PackageType) -> is_deb_warning_line = True elif "warning" in output_line.lower() or is_deb_warning_line: if is_deb_warning_line: - match = re.match(lintian_warning_error_pattern, output_line) + match = re.match( + lintian_warning_error_pattern, output_line) if match: package_specific_warning_lines.append(output_line) else: diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 295ba616..f745db46 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -17,7 +17,8 @@ get_template_environment, get_upcoming_minor_version, remove_cloned_code, initialize_env, create_pr_with_repo, DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER) -from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN) +from .common_validations import ( + CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" CITUS_CONTROL = "src/backend/distributed/citus.control" @@ -157,6 +158,8 @@ class MigrationFiles: # disabled since this is related to parameter_validations library methods # pylint: disable=no-value-for-parameter + + def update_release(github_token: non_blank(non_empty(str)), project_params: ProjectParams, earliest_pr_date: datetime, exec_path: non_blank(non_empty(str)), is_test: bool = False, cherry_pick_enabled: bool = False) -> UpdateReleaseReturnValue: @@ -168,11 +171,13 @@ def update_release(github_token: non_blank(non_empty(str)), project_params: Proj downgrades_dir_path=f"{exec_path}/{DOWNGRADES_DIR_PATH}") version_params = VersionParams(project_version_details=get_version_details(project_params.project_version), - upcoming_minor_version=get_upcoming_minor_version(project_params.project_version), + upcoming_minor_version=get_upcoming_minor_version( + project_params.project_version), upcoming_devel_version=f"{get_upcoming_minor_version(project_params.project_version)}devel") branch_params = BranchParams( - release_branch_name=get_release_branch_name(is_test, version_params.project_version_details), + release_branch_name=get_release_branch_name( + is_test, version_params.project_version_details), upcoming_version_branch=f"master-update-version-{uuid.uuid4()}" ) @@ -183,7 +188,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_params: Proj migration_files = MigrationFiles("", "") # major release if is_major_release(project_params.project_version): - print(f"### {project_params.project_version} is a major release. Executing Major release flow... ###") + print( + f"### {project_params.project_version} is a major release. Executing Major release flow... ###") major_release_params = MajorReleaseParams(configure_in_path=path_params.configure_in_path, devel_version=version_params.upcoming_devel_version, is_test=is_test, main_branch=project_params.main_branch, @@ -209,8 +215,10 @@ def update_release(github_token: non_blank(non_empty(str)), project_params: Proj multi_extension_sql_path=path_params.multi_extension_sql_path) upcoming_version_branch = upcoming_version_branch_params.upcoming_version_branch - migration_files = prepare_upcoming_version_branch(upcoming_version_branch_params) - print(f"### Done {project_params.project_version} Major release flow executed successfully. ###") + migration_files = prepare_upcoming_version_branch( + upcoming_version_branch_params) + print( + f"### Done {project_params.project_version} Major release flow executed successfully. ###") # patch release else: patch_release_params = PatchReleaseParams(cherry_pick_enabled=cherry_pick_enabled, @@ -249,9 +257,12 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar # In this case create one if patchReleaseParams.is_test: - non_test_release_branch = patchReleaseParams.release_branch_name.rstrip("-test") - release_branch_exist = remote_branch_exists(non_test_release_branch, os.getcwd()) - test_release_branch_exist = local_branch_exists(patchReleaseParams.release_branch_name, os.getcwd()) + non_test_release_branch = patchReleaseParams.release_branch_name.rstrip( + "-test") + release_branch_exist = remote_branch_exists( + non_test_release_branch, os.getcwd()) + test_release_branch_exist = local_branch_exists( + patchReleaseParams.release_branch_name, os.getcwd()) if release_branch_exist: run(f"git checkout {non_test_release_branch}") @@ -261,7 +272,8 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar else: run(f"git checkout -b {patchReleaseParams.release_branch_name}") else: - checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) + checkout_branch(patchReleaseParams.release_branch_name, + patchReleaseParams.is_test) # change version info in configure.in file update_version_in_configure_in(patchReleaseParams.project_name, patchReleaseParams.configure_in_path, patchReleaseParams.project_version) @@ -279,7 +291,8 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar cherrypick_prs_with_backport_labels(patchReleaseParams.earliest_pr_date_value, patchReleaseParams.main_branch, patchReleaseParams.release_branch_name, patchReleaseParams.repository) # commit all changes - commit_changes_for_version_bump(patchReleaseParams.project_name, patchReleaseParams.project_version) + commit_changes_for_version_bump( + patchReleaseParams.project_name, patchReleaseParams.project_version) # create and push release-$minor_version-push-$curTime branch release_pr_branch = f"{patchReleaseParams.release_branch_name}_{uuid.uuid4()}" create_and_checkout_branch(release_pr_branch) @@ -290,7 +303,8 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams): - print(f"### {upcoming_params.upcoming_version_branch} flow is being executed... ###") + print( + f"### {upcoming_params.upcoming_version_branch} flow is being executed... ###") # checkout master checkout_branch(upcoming_params.main_branch, upcoming_params.is_test) # create master-update-version-$curtime branch @@ -310,7 +324,8 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams update_detail_strings_in_multi_extension_out(upcoming_params.multi_extension_out_path, upcoming_params.upcoming_minor_version) # get current schema version from citus.control - current_schema_version = get_current_schema_from_citus_control(upcoming_params.citus_control_file_path) + current_schema_version = get_current_schema_from_citus_control( + upcoming_params.citus_control_file_path) # add downgrade script in multi_extension.sql file add_downgrade_script_in_multi_extension_file(current_schema_version, upcoming_params.multi_extension_sql_path, @@ -333,7 +348,8 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams update_schema_version_in_citus_control(upcoming_params.citus_control_file_path, default_upcoming_schema_version) # commit and push changes on master-update-version-$curtime branch - commit_changes_for_version_bump(upcoming_params.project_name, upcoming_params.upcoming_devel_version) + commit_changes_for_version_bump( + upcoming_params.project_name, upcoming_params.upcoming_devel_version) if not upcoming_params.is_test: push_branch(upcoming_params.upcoming_version_branch) @@ -341,12 +357,14 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams create_pull_request_for_upcoming_version_branch(upcoming_params.repository, upcoming_params.main_branch, upcoming_params.upcoming_version_branch, upcoming_params.upcoming_devel_version) - print(f"### Done {upcoming_params.upcoming_version_branch} flow executed. ###") + print( + f"### Done {upcoming_params.upcoming_version_branch} flow executed. ###") return MigrationFiles(upgrade_file=upgrade_file, downgrade_file=downgrade_file) def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleaseParams): - print(f"### {majorReleaseParams.release_branch_name} release branch flow is being executed... ###") + print( + f"### {majorReleaseParams.release_branch_name} release branch flow is being executed... ###") # checkout master checkout_branch(majorReleaseParams.main_branch, majorReleaseParams.is_test) # create release branch in release-X.Y format @@ -360,37 +378,44 @@ def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleasePar update_version_in_multi_extension_out(majorReleaseParams.multi_extension_out_path, majorReleaseParams.project_version) # commit all changes - commit_changes_for_version_bump(majorReleaseParams.project_name, majorReleaseParams.project_version) + commit_changes_for_version_bump( + majorReleaseParams.project_name, majorReleaseParams.project_version) # push release branch (No PR creation!!!) if not majorReleaseParams.is_test: push_branch(majorReleaseParams.release_branch_name) - print(f"### Done {majorReleaseParams.release_branch_name} release branch flow executed .###") + print( + f"### Done {majorReleaseParams.release_branch_name} release branch flow executed .###") def cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository): print( f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}... ### ") - prs_with_earliest_date = get_prs_for_patch_release(repository, earliest_pr_date, main_branch) + prs_with_earliest_date = get_prs_for_patch_release( + repository, earliest_pr_date, main_branch) # get commits for selected prs with backport label prs_with_backport = filter_prs_by_label(prs_with_earliest_date, "backport") - print(f"### Done {len(prs_with_backport)} PRs with backport label found. PR list is as below. ###") + print( + f"### Done {len(prs_with_backport)} PRs with backport label found. PR list is as below. ###") for pr in prs_with_backport: print(f"\tNo:{pr.number} Title:{pr.title}") # cherrypick all commits with backport label print(f"### Cherry-picking PRs to {release_branch_name}... ###") cherry_pick_prs(prs_with_backport) - print(f"### Done Cherry pick completed for all PRs on branch {release_branch_name}. ###") + print( + f"### Done Cherry pick completed for all PRs on branch {release_branch_name}. ###") def create_pull_request_for_upcoming_version_branch(repository, main_branch, upcoming_version_branch, upcoming_version): print(f"### Creating pull request for {upcoming_version_branch}... ###") pr_result = create_pr_with_repo(repo=repository, pr_branch=upcoming_version_branch, pr_title=f"Bump Citus to {upcoming_version}", base_branch=main_branch) - print(f"### Done Pull request created. PR no:{pr_result.number} PR URL: {pr_result.url}. ### ") + print( + f"### Done Pull request created. PR no:{pr_result.number} PR URL: {pr_result.url}. ### ") def push_branch(upcoming_version_branch): - print(f"Pushing changes for {upcoming_version_branch} into remote origin... ###") + print( + f"Pushing changes for {upcoming_version_branch} into remote origin... ###") run(f"git push --set-upstream origin {upcoming_version_branch}") print(f"### Done Changes pushed for {upcoming_version_branch}. ###") @@ -404,11 +429,14 @@ def commit_changes_for_version_bump(project_name, project_version): def update_schema_version_in_citus_control(citus_control_file_path, schema_version): - print(f"### Updating {citus_control_file_path} file with the version {schema_version}... ###") + print( + f"### Updating {citus_control_file_path} file with the version {schema_version}... ###") if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, f"default_version = '{schema_version}'"): - raise ValueError(f"{citus_control_file_path} does not have match for version") - print(f"### Done {citus_control_file_path} file is updated with the schema version {schema_version}. ###") + raise ValueError( + f"{citus_control_file_path} does not have match for version") + print( + f"### Done {citus_control_file_path} file is updated with the schema version {schema_version}. ###") def add_downgrade_script_in_multi_extension_file(current_schema_version, @@ -427,16 +455,19 @@ def add_downgrade_script_in_multi_extension_file(current_schema_version, string_to_prepend): raise ValueError(f"Downgrade scripts could not be added in {multi_extension_out_path} since " f"'DROP TABLE prev_objects, extension_diff;' script could not be found ") - print(f"### Done Test downgrade scripts successfully added in {multi_extension_out_path}. ###") + print( + f"### Done Test downgrade scripts successfully added in {multi_extension_out_path}. ###") def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: - print(f"### Reading current schema version from {citus_control_file_path}... ###") + print( + f"### Reading current schema version from {citus_control_file_path}... ###") current_schema_version = "" with open(citus_control_file_path, "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as cc_reader: cc_file_content = cc_reader.read() - _, cc_line = find_nth_matching_line_and_line_number(cc_file_content, CITUS_CONTROL_SEARCH_PATTERN, 1) + _, cc_line = find_nth_matching_line_and_line_number( + cc_file_content, CITUS_CONTROL_SEARCH_PATTERN, 1) schema_not_found = False if len(cc_line) > 0: line_parts = cc_line.split("=") @@ -448,7 +479,8 @@ def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: schema_not_found = True if schema_not_found: - raise ValueError("Version info could not be found in citus.control file") + raise ValueError( + "Version info could not be found in citus.control file") current_schema_version = current_schema_version.strip(" '") print(f"### Done Schema version is {current_schema_version}. ###") @@ -456,25 +488,30 @@ def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: def update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_minor_version): - print(f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}... ###") + print( + f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}... ###") if not replace_line_in_file(config_py_path, CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN, f"MASTER_VERSION = '{upcoming_minor_version}'"): raise ValueError(f"{config_py_path} does not have match for version") - print(f"### Done {config_py_path} file updated with the upcoming version {upcoming_minor_version}. ###") + print( + f"### Done {config_py_path} file updated with the upcoming version {upcoming_minor_version}. ###") def update_version_in_multi_extension_out(multi_extension_out_path, project_version): - print(f"### Updating {multi_extension_out_path} file with the project version {project_version}... ###") + print( + f"### Updating {multi_extension_out_path} file with the project version {project_version}... ###") if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DEVEL_SEARCH_PATTERN, f" {project_version}"): raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {MULTI_EXT_DEVEL_SEARCH_PATTERN}") - print(f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") + print( + f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") def update_detail_strings_in_multi_extension_out(multi_extension_out_path, minor_version): - print(f"### Updating {multi_extension_out_path} detail lines file with the project version {minor_version}... ###") + print( + f"### Updating {multi_extension_out_path} detail lines file with the project version {minor_version}... ###") if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DETAIL1_PATTERN, f"{MULTI_EXT_DETAIL_PREFIX}{minor_version}{MULTI_EXT_DETAIL1_SUFFIX}"): @@ -486,18 +523,21 @@ def update_detail_strings_in_multi_extension_out(multi_extension_out_path, minor raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {MULTI_EXT_DETAIL2_PATTERN}") - print(f"### Done {multi_extension_out_path} detail lines updated with project version {minor_version}. ###") + print( + f"### Done {multi_extension_out_path} detail lines updated with project version {minor_version}. ###") def update_version_in_multi_extension_out_for_patch(multi_extension_out_path, project_version): - print(f"### Updating {multi_extension_out_path} file with the project version {project_version}... ###") + print( + f"### Updating {multi_extension_out_path} file with the project version {project_version}... ###") if not replace_line_in_file(multi_extension_out_path, get_patch_version_regex(project_version), f" {project_version}"): raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {get_patch_version_regex(project_version)}") - print(f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") + print( + f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") def execute_autoconf_f(): @@ -510,12 +550,15 @@ def update_version_in_configure_in(project_name, configure_in_path, project_vers print(f"### Updating version on file {configure_in_path}... ###") if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([{repo_details[project_name]['configure-in-str']}], [{project_version}])"): - raise ValueError(f"{configure_in_path} does not have match for version") - print(f"### Done {configure_in_path} file is updated with project version {project_version}. ###") + raise ValueError( + f"{configure_in_path} does not have match for version") + print( + f"### Done {configure_in_path} file is updated with project version {project_version}. ###") def create_and_checkout_branch(release_branch_name): - print(f"### Creating release branch with name {release_branch_name} from {get_current_branch(os.getcwd())}... ###") + print( + f"### Creating release branch with name {release_branch_name} from {get_current_branch(os.getcwd())}... ###") run(f'git checkout -b {release_branch_name}') print(f"### Done {release_branch_name} created. ###") @@ -535,13 +578,15 @@ def upgrade_sql_file_name(current_schema_version, upcoming_minor_version): def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path, upcoming_minor_version): - newly_created_sql_file = upgrade_sql_file_name(current_schema_version, upcoming_minor_version) + newly_created_sql_file = upgrade_sql_file_name( + current_schema_version, upcoming_minor_version) print(f"### Creating upgrade file {newly_created_sql_file}... ###") with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as f_writer: content = f"-- citus--{current_schema_version}--{upcoming_minor_version}-1" content = content + "\n\n" - content = content + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" + content = content + \ + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" f_writer.write(content) print(f"### Done {newly_created_sql_file} created. ###") return newly_created_sql_file @@ -556,9 +601,9 @@ def create_new_sql_for_downgrade_path(current_schema_version, distributed_dir_pa content = f"-- citus--{upcoming_minor_version}-1--{current_schema_version}" content = content + "\n" content = ( - content + f"-- this is an empty downgrade path since " - f"{upgrade_sql_file_name(current_schema_version, upcoming_minor_version)} " - f"is empty for now" + "\n") + content + f"-- this is an empty downgrade path since " + f"{upgrade_sql_file_name(current_schema_version, upcoming_minor_version)} " + f"is empty for now" + "\n") f_writer.write(content) print(f"### Done {newly_created_sql_file} created. ###") return newly_created_sql_file @@ -572,7 +617,8 @@ def validate_parameters(major_release_flag: bool): raise ValueError("Cherry pick could be enabled only for patch release") if major_release_flag and arguments.earliest_pr_date: - raise ValueError("earliest_pr_date could not be used for major releases") + raise ValueError( + "earliest_pr_date could not be used for major releases") if major_release_flag and arguments.schema_version: raise ValueError("schema_version could not be set for major releases") @@ -586,7 +632,8 @@ def validate_parameters(major_release_flag: bool): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--gh_token', required=True) - parser.add_argument('--prj_name', choices=["citus", "citus-enterprise"], required=True) + parser.add_argument( + '--prj_name', choices=["citus", "citus-enterprise"], required=True) parser.add_argument('--prj_ver', required=True) parser.add_argument('--main_branch') parser.add_argument('--earliest_pr_date') @@ -602,8 +649,10 @@ def validate_parameters(major_release_flag: bool): initialize_env(execution_path, arguments.prj_name, CHECKOUT_DIR) is_cherry_pick_enabled = arguments.cherry_pick_enabled - main_branch = arguments.main_branch if arguments.main_branch else repo_details[arguments.prj_name]["branch"] - print(f"Using main branch {main_branch} for the repo {arguments.prj_name}.") + main_branch = arguments.main_branch if arguments.main_branch else repo_details[ + arguments.prj_name]["branch"] + print( + f"Using main branch {main_branch} for the repo {arguments.prj_name}.") os.chdir(execution_path) print(f"Executing in path {execution_path}") earliest_pr_date_value = (None if major_release or not is_cherry_pick_enabled diff --git a/packaging_automation/publish-into-ms-packages.py b/packaging_automation/publish-into-ms-packages.py index e14f28e3..17091dfe 100644 --- a/packaging_automation/publish-into-ms-packages.py +++ b/packaging_automation/publish-into-ms-packages.py @@ -23,7 +23,8 @@ def publish_single_package(package_path: str, repo): - result = run_with_output(f"repoclient package add --repoID {repo['id']} {package_path}") + result = run_with_output( + f"repoclient package add --repoID {repo['id']} {package_path}") return json.loads(result.stdout) @@ -46,14 +47,16 @@ def get_citus_repos(): name = re.sub(r"(\d+)", r"-\1", name) repos[name] = repo # Adding ubuntu-focal manually because list does not include ubuntu-focal - repos["ubuntu-focal"] = {"url": "ubuntu-focal", "distribution": "focal", "id": UBUNTU_FOCAL_REPO_ID} + repos["ubuntu-focal"] = {"url": "ubuntu-focal", + "distribution": "focal", "id": UBUNTU_FOCAL_REPO_ID} return repos # Ensure deb packages contain the distribution, so they do not conflict def suffix_deb_package_with_distribution(repository, package_file_path): if not package_file_path.endswith(AMD_SUFFIX): - raise ValueError(f"Package should have ended with {AMD_SUFFIX}: {package_file_path}") + raise ValueError( + f"Package should have ended with {AMD_SUFFIX}: {package_file_path}") old_package_path = package_file_path package_prefix = package_file_path[: -len(AMD_SUFFIX)] package_file_path = f"{package_prefix}+{repository['distribution']}_{AMD_SUFFIX}" @@ -73,7 +76,8 @@ def publish_packages(target_platform, citus_repos, packages_dir: str): # Ensure deb packages contain the distribution, so they do not conflict if repo["url"] in DEB_BASED_REPOS: if repo["distribution"] not in package_file: - package_path = suffix_deb_package_with_distribution(repo, package_path) + package_path = suffix_deb_package_with_distribution( + repo, package_path) # Publish packages if os.path.isfile(package_path) and package_file.endswith((".rpm", ".deb")): @@ -126,6 +130,7 @@ def check_submissions(all_responses): citus_repos = get_citus_repos() - submission_responses = publish_packages(args.platform, citus_repos, args.packages_dir) + submission_responses = publish_packages( + args.platform, citus_repos, args.packages_dir) check_submissions(submission_responses) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index cdc211bc..f8b1df33 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -94,12 +94,15 @@ def get_image_tag(tag_prefix: str, docker_image_type: DockerImageType) -> str: def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, will_image_be_published: bool): - triggering_event_info, resource_name = decode_triggering_event_info(github_ref) + triggering_event_info, resource_name = decode_triggering_event_info( + github_ref) for regular_image_type in regular_images_to_be_built(docker_image_type): if triggering_event_info == GithubTriggerEventSource.branch_push: - publish_main_docker_images(regular_image_type, will_image_be_published) + publish_main_docker_images( + regular_image_type, will_image_be_published) else: - publish_tagged_docker_images(regular_image_type, resource_name, will_image_be_published) + publish_tagged_docker_images( + regular_image_type, resource_name, will_image_be_published) def publish_docker_image_on_schedule(docker_image_type: DockerImageType, will_image_be_published: bool): @@ -107,7 +110,8 @@ def publish_docker_image_on_schedule(docker_image_type: DockerImageType, will_im publish_nightly_docker_image(will_image_be_published) else: for regular_image_type in regular_images_to_be_built(docker_image_type): - publish_main_docker_images(regular_image_type, will_image_be_published) + publish_main_docker_images( + regular_image_type, will_image_be_published) def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, will_image_be_published: bool, @@ -132,8 +136,10 @@ def publish_main_docker_images(docker_image_type: DockerImageType, will_image_be print(f"Main docker image for {docker_image_type.name} built.") if will_image_be_published: print(f"Publishing main docker image for {docker_image_type.name}...") - docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_type.name) - print(f"Publishing main docker image for {docker_image_type.name} finished") + docker_client.images.push( + DOCKER_IMAGE_NAME, tag=docker_image_type.name) + print( + f"Publishing main docker image for {docker_image_type.name} finished") else: current_branch = get_current_branch(os.getcwd()) if current_branch != DEFAULT_BRANCH_NAME: @@ -150,7 +156,8 @@ def flush_logs(logs): def publish_tagged_docker_images(docker_image_type, tag_name: str, will_image_be_published: bool): - print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name}...") + print( + f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name}...") tag_parts = decode_tag_parts(tag_name) tag_version_part = "" docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_type.name}" @@ -164,19 +171,23 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, will_image_be image_tag = get_image_tag(tag_version_part, docker_image_type) print(f"Tagging {docker_image_name} with the tag {image_tag}...") docker_api_client.tag(docker_image_name, docker_image_name, image_tag) - print(f"Tagging {docker_image_name} with the tag {image_tag} finished.") + print( + f"Tagging {docker_image_name} with the tag {image_tag} finished.") if will_image_be_published: print(f"Pushing {docker_image_name} with the tag {image_tag}...") - push_logs = docker_client.images.push(DOCKER_IMAGE_NAME, tag=image_tag) + push_logs = docker_client.images.push( + DOCKER_IMAGE_NAME, tag=image_tag) print("Push logs:") print(push_logs) - print(f"Pushing {docker_image_name} with the tag {image_tag} finished") + print( + f"Pushing {docker_image_name} with the tag {image_tag} finished") else: print( f"Skipped pushing {docker_image_type} with the tag {image_tag} since will_image_be_published flag is false") tag_version_part = tag_version_part + "." - print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name} finished.") + print( + f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name} finished.") def publish_nightly_docker_image(will_image_be_published: bool): @@ -190,14 +201,16 @@ def publish_nightly_docker_image(will_image_be_published: bool): if will_image_be_published: print("Pushing nightly image...") - docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_info_dict[DockerImageType.nightly]['docker-tag']) + docker_client.images.push( + DOCKER_IMAGE_NAME, tag=docker_image_info_dict[DockerImageType.nightly]['docker-tag']) print("Nightly image push finished.") else: - print("Nightly image will not be pushed since will_image_be_published flag is false") + print( + "Nightly image will not be pushed since will_image_be_published flag is false") def validate_and_extract_general_parameters(docker_image_type_param: str, pipeline_trigger_type_param: str) -> Tuple[ - GithubPipelineTriggerType, DockerImageType]: + GithubPipelineTriggerType, DockerImageType]: try: trigger_type_param = GithubPipelineTriggerType[pipeline_trigger_type_param] except KeyError: @@ -235,7 +248,8 @@ def validate_and_extract_manual_exec_params(manual_trigger_type_param: str, tag_ def get_image_publish_status(github_ref: str, is_test: bool): if is_test: return False - triggering_event_info, resource_name = decode_triggering_event_info(github_ref) + triggering_event_info, resource_name = decode_triggering_event_info( + github_ref) if triggering_event_info == GithubTriggerEventSource.tag_push: if not is_tag_on_branch(tag_name=resource_name, branch_name=DEFAULT_BRANCH_NAME): return False @@ -249,10 +263,13 @@ def get_image_publish_status(github_ref: str, is_test: bool): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--github_ref', required=True) - parser.add_argument('--pipeline_trigger_type', choices=[e.name for e in GithubPipelineTriggerType], required=True) + parser.add_argument('--pipeline_trigger_type', + choices=[e.name for e in GithubPipelineTriggerType], required=True) parser.add_argument('--tag_name', nargs='?', default="") - parser.add_argument('--manual_trigger_type', choices=[e.name for e in ManualTriggerType]) - parser.add_argument('--image_type', choices=[e.name for e in DockerImageType]) + parser.add_argument('--manual_trigger_type', + choices=[e.name for e in ManualTriggerType]) + parser.add_argument( + '--image_type', choices=[e.name for e in DockerImageType]) parser.add_argument('--is_test', action="store_true") args = parser.parse_args() @@ -263,11 +280,13 @@ def get_image_publish_status(github_ref: str, is_test: bool): publish_status = get_image_publish_status(args.github_ref, args.is_test) if pipeline_trigger_type == GithubPipelineTriggerType.workflow_dispatch: - manual_trigger_type = validate_and_extract_manual_exec_params(args.manual_trigger_type, args.tag_name) + manual_trigger_type = validate_and_extract_manual_exec_params( + args.manual_trigger_type, args.tag_name) publish_docker_image_manually(manual_trigger_type_param=manual_trigger_type, will_image_be_published=publish_status, docker_image_type=image_type, tag_name=args.tag_name) elif pipeline_trigger_type == GithubPipelineTriggerType.push: - publish_docker_image_on_push(image_type, args.github_ref, publish_status) + publish_docker_image_on_push( + image_type, args.github_ref, publish_status) else: publish_docker_image_on_schedule(image_type, publish_status) diff --git a/packaging_automation/templates/citus.spec.tmpl b/packaging_automation/templates/citus.spec.tmpl index 2b4d0163..f9ac6798 100644 --- a/packaging_automation/templates/citus.spec.tmpl +++ b/packaging_automation/templates/citus.spec.tmpl @@ -56,7 +56,7 @@ echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list - + # Columnar does not exist in Citus versions < 10.0 # At this point, we don't have %{pginstdir}, # so first check build directory for columnar. diff --git a/packaging_automation/templates/pgxn/META.tmpl.json b/packaging_automation/templates/pgxn/META.tmpl.json index f920c907..861789a5 100644 --- a/packaging_automation/templates/pgxn/META.tmpl.json +++ b/packaging_automation/templates/pgxn/META.tmpl.json @@ -1,54 +1,54 @@ { - "name": "citus", - "abstract": "Scalable PostgreSQL for real-time workloads", - "description": "Citus horizontally scales PostgreSQL across commodity servers using sharding and replication. Its query engine parallelizes incoming SQL queries across these servers to enable real-time responses on large datasets.", - "version": "{{project_version}}", - "maintainer": "\"Citus Data\" ", - "license": "agpl_3", - "provides": { - "citus": { - "abstract": "Citus Distributed Database", - "file": "citus.so", - "docfile": "README.md", - "version": "{{project_version}}" + "name": "citus", + "abstract": "Scalable PostgreSQL for real-time workloads", + "description": "Citus horizontally scales PostgreSQL across commodity servers using sharding and replication. Its query engine parallelizes incoming SQL queries across these servers to enable real-time responses on large datasets.", + "version": "{{project_version}}", + "maintainer": "\"Citus Data\" ", + "license": "agpl_3", + "provides": { + "citus": { + "abstract": "Citus Distributed Database", + "file": "citus.so", + "docfile": "README.md", + "version": "{{project_version}}" + } + }, + "prereqs": { + "runtime": { + "requires": { + "PostgreSQL": "11.0.0" } - }, - "prereqs": { - "runtime": { - "requires": { - "PostgreSQL": "11.0.0" - } - }, - "test": { - "requires": { - "plpgsql": 0 - } + }, + "test": { + "requires": { + "plpgsql": 0 } - }, - "release_status": "stable", - "resources": { - "homepage": "https://www.citusdata.com", - "bugtracker": { - "web": "https://github.com/citusdata/citus/issues", - "mailto": "support@citusdata.com" - }, - "repository": { - "url": "git://github.com/citusdata/citus.git", - "web": "https://github.com/citusdata/citus", - "type": "git" - } - }, - "generated_by": "David E. Wheeler", - "meta-spec": { - "version": "1.0.0", - "url": "http://pgxn.org/meta/spec.txt" - }, - "tags": [ - "sharding", - "replication", - "parallel", - "distributed", - "horizontal", - "analytics" - ] + } + }, + "release_status": "stable", + "resources": { + "homepage": "https://www.citusdata.com", + "bugtracker": { + "web": "https://github.com/citusdata/citus/issues", + "mailto": "support@citusdata.com" + }, + "repository": { + "url": "git://github.com/citusdata/citus.git", + "web": "https://github.com/citusdata/citus", + "type": "git" + } + }, + "generated_by": "David E. Wheeler", + "meta-spec": { + "version": "1.0.0", + "url": "http://pgxn.org/meta/spec.txt" + }, + "tags": [ + "sharding", + "replication", + "parallel", + "distributed", + "horizontal", + "analytics" + ] } diff --git a/packaging_automation/test_citus_package.py b/packaging_automation/test_citus_package.py index c6017fe8..e05b5c7a 100644 --- a/packaging_automation/test_citus_package.py +++ b/packaging_automation/test_citus_package.py @@ -7,7 +7,8 @@ import sys from typing import List -from .common_tool_methods import (get_supported_postgres_release_versions, get_minor_version) +from .common_tool_methods import ( + get_supported_postgres_release_versions, get_minor_version) POSTGRES_MATRIX_FILE = "postgres-matrix.yml" POSTGRES_MATRIX_WEB_ADDRESS = "https://raw.githubusercontent.com/citusdata/packaging/all-citus-unit-tests/postgres-matrix.yml" @@ -28,11 +29,16 @@ class TestPlatform(Enum): centos_7 = {"name": "centos/7", "docker_image_name": "centos-7"} ol_7 = {"name": "ol/7", "docker_image_name": "ol-7"} ol_8 = {"name": "ol/8", "docker_image_name": "ol-8"} - debian_buster = {"name": "debian/buster", "docker_image_name": "debian-buster", } - debian_bullseye = {"name": "debian/bullseye", "docker_image_name": "debian-bullseye"} - debian_stretch = {"name": "debian/stretch", "docker_image_name": "debian-stretch"} - ubuntu_bionic = {"name": "ubuntu/bionic", "docker_image_name": "ubuntu-bionic"} - ubuntu_focal = {"name": "ubuntu/focal", "docker_image_name": "ubuntu-focal"} + debian_buster = {"name": "debian/buster", + "docker_image_name": "debian-buster", } + debian_bullseye = {"name": "debian/bullseye", + "docker_image_name": "debian-bullseye"} + debian_stretch = {"name": "debian/stretch", + "docker_image_name": "debian-stretch"} + ubuntu_bionic = {"name": "ubuntu/bionic", + "docker_image_name": "ubuntu-bionic"} + ubuntu_focal = {"name": "ubuntu/focal", + "docker_image_name": "ubuntu-focal"} undefined = {"name": "undefined", "docker_image_name": "undefined"} @@ -49,7 +55,8 @@ def get_postgres_versions_from_matrix_file(project_version: str) -> List[str]: with open(POSTGRES_MATRIX_FILE, 'wb') as writer: writer.write(r.content) - pg_versions = get_supported_postgres_release_versions(POSTGRES_MATRIX_FILE, project_version) + pg_versions = get_supported_postgres_release_versions( + POSTGRES_MATRIX_FILE, project_version) return pg_versions @@ -58,7 +65,8 @@ def get_postgres_versions_from_matrix_file(project_version: str) -> List[str]: parser = argparse.ArgumentParser() parser.add_argument('--project_version', required=True) parser.add_argument('--pg_major_version') - parser.add_argument("--os_release", choices=[t.value["name"] for t in TestPlatform]) + parser.add_argument( + "--os_release", choices=[t.value["name"] for t in TestPlatform]) args = parser.parse_args() test_platform = get_test_platform_for_os_release(args.os_release) @@ -66,16 +74,18 @@ def get_postgres_versions_from_matrix_file(project_version: str) -> List[str]: platform = args.os_release - postgres_versions = get_postgres_versions_from_matrix_file(args.project_version) + postgres_versions = get_postgres_versions_from_matrix_file( + args.project_version) - print(f'This version of Citus supports following pg versions: {postgres_versions}') + print( + f'This version of Citus supports following pg versions: {postgres_versions}') os.chdir("test-images") return_codes = {} if args.pg_major_version: - postgres_versions = [p for p in postgres_versions if p == args.pg_major_version] - + postgres_versions = [ + p for p in postgres_versions if p == args.pg_major_version] if len(postgres_versions) == 0: raise ValueError("At least one supported postgres version is required") diff --git a/packaging_automation/tests/files/citus.spec b/packaging_automation/tests/files/citus.spec index 753a9946..b076a0f2 100644 --- a/packaging_automation/tests/files/citus.spec +++ b/packaging_automation/tests/files/citus.spec @@ -56,7 +56,7 @@ echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list - + # Columnar does not exist in Citus versions < 10.0 # At this point, we don't have %{pginstdir}, # so first check build directory for columnar. diff --git a/packaging_automation/tests/files/citus_include_10_0_3.spec b/packaging_automation/tests/files/citus_include_10_0_3.spec index d1c25c25..ee5cca73 100644 --- a/packaging_automation/tests/files/citus_include_10_0_3.spec +++ b/packaging_automation/tests/files/citus_include_10_0_3.spec @@ -56,7 +56,7 @@ echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list - + # Columnar does not exist in Citus versions < 10.0 # At this point, we don't have %{pginstdir}, # so first check build directory for columnar. diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 3beeeb07..bb18578d 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -20,7 +20,8 @@ from dotenv import dotenv_values -TEST_BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) +TEST_BASE_PATH = os.getenv( + "BASE_PATH", default=pathlib2.Path(__file__).parents[2]) PACKAGING_SOURCE_FOLDER = "packaging_test" PACKAGING_EXEC_FOLDER = f"{TEST_BASE_PATH}/{PACKAGING_SOURCE_FOLDER}" @@ -43,7 +44,8 @@ GH_TOKEN = os.getenv("GH_TOKEN") PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN") REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET") -PLATFORM = get_build_platform(os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM")) +PLATFORM = get_build_platform( + os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM")) def get_required_package_count(input_files_dir: str, platform: str): @@ -71,17 +73,20 @@ def teardown_module(): def test_build_packages(): delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME) delete_rpm_key_by_name(TEST_GPG_KEY_NAME) - generate_new_gpg_key(f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging_with_passphrase.gpg") + generate_new_gpg_key( + f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging_with_passphrase.gpg") gpg_fingerprints = get_gpg_fingerprints_by_name(TEST_GPG_KEY_NAME) assert len(gpg_fingerprints) > 0 secret_key = transform_key_into_base64_str( get_private_key_by_fingerprint_with_passphrase(gpg_fingerprints[0], TEST_GPG_KEY_PASSPHRASE)) define_rpm_public_key_to_machine(gpg_fingerprints[0]) - signing_credentials = SigningCredentials(secret_key, TEST_GPG_KEY_PASSPHRASE) + signing_credentials = SigningCredentials( + secret_key, TEST_GPG_KEY_PASSPHRASE) input_output_parameters = InputOutputParameters.build(PACKAGING_EXEC_FOLDER, BASE_OUTPUT_FOLDER, output_validation=False) - build_packages(GH_TOKEN, PLATFORM, BuildType.release, signing_credentials, input_output_parameters, is_test=True) + build_packages(GH_TOKEN, PLATFORM, BuildType.release, + signing_credentials, input_output_parameters, is_test=True) verify_rpm_signature_in_dir(BASE_OUTPUT_FOLDER) os_name, os_version = decode_os_and_release(PLATFORM) sub_folder = get_release_package_folder_name(os_name, os_version) @@ -97,7 +102,8 @@ def test_build_packages(): def test_get_required_package_count(): - assert get_required_package_count(PACKAGING_EXEC_FOLDER, platform="el/8") == 9 + assert get_required_package_count( + PACKAGING_EXEC_FOLDER, platform="el/8") == 9 def test_decode_os_packages(): @@ -106,7 +112,8 @@ def test_decode_os_packages(): def test_upload_to_package_cloud(): - platform = get_build_platform(os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM")) + platform = get_build_platform( + os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM")) current_branch = "all-citus" main_branch = "all-citus" output = upload_files_in_directory_to_package_cloud(BASE_OUTPUT_FOLDER, platform, PACKAGE_CLOUD_API_TOKEN, @@ -114,13 +121,15 @@ def test_upload_to_package_cloud(): current_branch, main_branch) distro_parts = platform.split("/") if len(distro_parts) != 2: - raise ValueError("Platform should consist of two parts splitted with '/' e.g. el/8") + raise ValueError( + "Platform should consist of two parts splitted with '/' e.g. el/8") for return_value in output.return_values: exists = package_exists(PACKAGE_CLOUD_API_TOKEN, "citus-bot", "sample", os.path.basename(return_value.file_name), platform) if not exists: - raise ValueError(f"{os.path.basename(return_value.file_name)} could not be found on package cloud") + raise ValueError( + f"{os.path.basename(return_value.file_name)} could not be found on package cloud") for return_value in output.return_values: delete_output = delete_package_from_package_cloud(PACKAGE_CLOUD_API_TOKEN, "citus-bot", "sample", @@ -129,4 +138,5 @@ def test_upload_to_package_cloud(): if delete_output.success_status: print(f"{os.path.basename(return_value.file_name)} deleted successfully") else: - print(f"{os.path.basename(return_value.file_name)} can not be deleted. Message: {delete_output.message}") + print( + f"{os.path.basename(return_value.file_name)} can not be deleted. Message: {delete_output.message}") diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index b18651e0..5683f3ba 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -12,7 +12,8 @@ delete_rpm_key_by_name, get_private_key_by_fingerprint_with_passphrase, verify_rpm_signature_in_dir, transform_key_into_base64_str) -TEST_BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) +TEST_BASE_PATH = os.getenv( + "BASE_PATH", default=pathlib2.Path(__file__).parents[2]) TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = "Citus123" GH_TOKEN = os.getenv("GH_TOKEN") @@ -26,7 +27,8 @@ def setup_module(): if not os.path.exists("packaging_test"): - run(f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}") + run( + f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}") def teardown_module(): @@ -61,12 +63,14 @@ def test_get_signing_credentials(): delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME) - generate_new_gpg_key(f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging.gpg") + generate_new_gpg_key( + f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging.gpg") os.environ["PACKAGING_PASSPHRASE"] = TEST_GPG_KEY_PASSPHRASE signing_credentials = get_signing_credentials("", TEST_GPG_KEY_PASSPHRASE) fingerprints = get_gpg_fingerprints_by_name(TEST_GPG_KEY_NAME) assert len(fingerprints) > 0 - expected_gpg_key = get_private_key_by_fingerprint_without_passphrase(fingerprints[0]) + expected_gpg_key = get_private_key_by_fingerprint_without_passphrase( + fingerprints[0]) delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME) assert signing_credentials.secret_key == transform_key_into_base64_str( expected_gpg_key) and signing_credentials.passphrase == TEST_GPG_KEY_PASSPHRASE @@ -74,7 +78,8 @@ def test_get_signing_credentials(): def test_delete_rpm_key_by_name(): delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME) - generate_new_gpg_key(f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging_with_passphrase.gpg") + generate_new_gpg_key( + f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging_with_passphrase.gpg") fingerprints = get_gpg_fingerprints_by_name(TEST_GPG_KEY_NAME) assert len(fingerprints) > 0 define_rpm_public_key_to_machine(fingerprints[0]) @@ -85,7 +90,8 @@ def test_delete_rpm_key_by_name(): output = subprocess.run(["rpm", "-q gpg-pubkey", "--qf %{NAME}-%{VERSION}-%{RELEASE}\t%{SUMMARY}\n"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - assert TEST_GPG_KEY_NAME not in output.stdout.decode("ascii") and output.returncode == 1 + assert TEST_GPG_KEY_NAME not in output.stdout.decode( + "ascii") and output.returncode == 1 def test_get_postgres_versions(): @@ -95,7 +101,8 @@ def test_get_postgres_versions(): release_versions, nightly_versions = get_postgres_versions("el", f"{TEST_BASE_PATH}/packaging_automation/tests/files") - assert release_versions == ["11", "12", "13"] and nightly_versions == ["12", "13", "14"] + assert release_versions == [ + "11", "12", "13"] and nightly_versions == ["12", "13", "14"] def test_build_package_debian(): @@ -118,13 +125,16 @@ def test_build_package_rpm(): def test_sign_packages(): delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME) delete_rpm_key_by_name(TEST_GPG_KEY_NAME) - generate_new_gpg_key(f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging_with_passphrase.gpg") + generate_new_gpg_key( + f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging_with_passphrase.gpg") gpg_fingerprints = get_gpg_fingerprints_by_name(TEST_GPG_KEY_NAME) assert len(gpg_fingerprints) > 0 - private_key = get_private_key_by_fingerprint_with_passphrase(gpg_fingerprints[0], TEST_GPG_KEY_PASSPHRASE) + private_key = get_private_key_by_fingerprint_with_passphrase( + gpg_fingerprints[0], TEST_GPG_KEY_PASSPHRASE) secret_key = transform_key_into_base64_str(private_key) define_rpm_public_key_to_machine(gpg_fingerprints[0]) - signing_credentials = SigningCredentials(secret_key=secret_key, passphrase=TEST_GPG_KEY_PASSPHRASE) + signing_credentials = SigningCredentials( + secret_key=secret_key, passphrase=TEST_GPG_KEY_PASSPHRASE) input_output_parameters = InputOutputParameters.build(PACKAGING_EXEC_FOLDER, f"{OUTPUT_FOLDER}", output_validation=False) sign_packages(sub_folder="centos-8", signing_credentials=signing_credentials, diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 0f7f94a5..d2dcb3cf 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -51,8 +51,10 @@ def test_find_nth_occurrence_position(): def test_find_nth_matching_line_number_by_regex(): - assert find_nth_matching_line_and_line_number("citusx\n citusx\ncitusx", "^citusx$", 2)[0] == 2 - assert find_nth_matching_line_and_line_number("citusx\n citusx\ncitusx", "^citusy$", 2)[0] == -1 + assert find_nth_matching_line_and_line_number( + "citusx\n citusx\ncitusx", "^citusx$", 2)[0] == 2 + assert find_nth_matching_line_and_line_number( + "citusx\n citusx\ncitusx", "^citusy$", 2)[0] == -1 def test_is_major_release(): @@ -75,11 +77,13 @@ def test_run(): def test_remove_paranthesis_from_string(): - assert remove_text_with_parenthesis("out of paranthesis (inside paranthesis)") == "out of paranthesis " + assert remove_text_with_parenthesis( + "out of paranthesis (inside paranthesis)") == "out of paranthesis " def test_get_version_details(): - assert get_version_details("10.0.1") == {"major": "10", "minor": "0", "patch": "1"} + assert get_version_details("10.0.1") == { + "major": "10", "minor": "0", "patch": "1"} def test_is_tag_on_branch(): @@ -235,7 +239,8 @@ def test_getprs_with_backlog_label(): def test_process_template_file(): - content = process_template_file("10.0.3", f"{BASE_PATH}/templates", "docker/alpine/alpine.tmpl.dockerfile", "13.2") + content = process_template_file( + "10.0.3", f"{BASE_PATH}/templates", "docker/alpine/alpine.tmpl.dockerfile", "13.2") with open(f"{TEST_BASE_PATH}/files/verify/expected_alpine_10.0.3.txt", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: expected_content = reader.read() @@ -249,7 +254,8 @@ def test_remove_prefix(): def test_delete_rpm_key_by_name(): delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME) - generate_new_gpg_key(f"{TEST_BASE_PATH}/files/gpg/packaging_with_passphrase.gpg") + generate_new_gpg_key( + f"{TEST_BASE_PATH}/files/gpg/packaging_with_passphrase.gpg") fingerprints = get_gpg_fingerprints_by_name(TEST_GPG_KEY_NAME) assert len(fingerprints) > 0 define_rpm_public_key_to_machine(fingerprints[0]) diff --git a/packaging_automation/tests/test_docker_statistics_collector.py b/packaging_automation/tests/test_docker_statistics_collector.py index 75b1f805..dbfd9768 100644 --- a/packaging_automation/tests/test_docker_statistics_collector.py +++ b/packaging_automation/tests/test_docker_statistics_collector.py @@ -16,7 +16,8 @@ def test_docker_statistics_collector(): test_day_shift_index = -2 test_pull_count_shift = 205 - db_params = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD,host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) + db_params = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD, + host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) db = create_engine(db_connection_string(db_params=db_params, is_test=True)) sql = text(f'DROP TABLE IF EXISTS {DockerStats.__tablename__}') db.execute(sql) @@ -25,7 +26,8 @@ def test_docker_statistics_collector(): fetch_and_store_docker_statistics("citus", db_parameters=db_params, is_test=True, test_day_shift_index=test_day_shift_index) first_day = datetime.today() + timedelta(days=test_day_shift_index) - first_day_record = session.query(DockerStats).filter_by(stat_date=first_day.date()).first() + first_day_record = session.query(DockerStats).filter_by( + stat_date=first_day.date()).first() fetch_and_store_docker_statistics("citus", db_parameters=db_params, is_test=True, test_total_pull_count=first_day_record.total_pull_count + test_pull_count_shift) Base.metadata.create_all(db) @@ -33,12 +35,15 @@ def test_docker_statistics_collector(): second_day = datetime.today() + timedelta(days=test_day_shift_index + 1) third_day = datetime.today() - second_day_record = session.query(DockerStats).filter_by(stat_date=second_day.date()).first() - third_day_record = session.query(DockerStats).filter_by(stat_date=third_day.date()).first() + second_day_record = session.query(DockerStats).filter_by( + stat_date=second_day.date()).first() + third_day_record = session.query(DockerStats).filter_by( + stat_date=third_day.date()).first() - pull_count_diff = third_day_record.total_pull_count - first_day_record.total_pull_count + pull_count_diff = third_day_record.total_pull_count - \ + first_day_record.total_pull_count assert third_day_record and second_day_record and ( - third_day_record.total_pull_count == second_day_record.total_pull_count) and ( - third_day_record.daily_pull_count + second_day_record.daily_pull_count == pull_count_diff) and ( - pull_count_diff == test_pull_count_shift) + third_day_record.total_pull_count == second_day_record.total_pull_count) and ( + third_day_record.daily_pull_count + second_day_record.daily_pull_count == pull_count_diff) and ( + pull_count_diff == test_pull_count_shift) diff --git a/packaging_automation/tests/test_github_statistics_collector.py b/packaging_automation/tests/test_github_statistics_collector.py index 67be0735..308eda32 100644 --- a/packaging_automation/tests/test_github_statistics_collector.py +++ b/packaging_automation/tests/test_github_statistics_collector.py @@ -18,10 +18,13 @@ def test_github_stats_collector(): - db_params = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD, host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) + db_params = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD, + host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) db = create_engine(db_connection_string(db_params=db_params, is_test=True)) - db.execute(text(f'DROP TABLE IF EXISTS {GithubCloneStatsTransactionsDetail.__tablename__}')) - db.execute(text(f'DROP TABLE IF EXISTS {GithubCloneStatsTransactionsMain.__tablename__}')) + db.execute( + text(f'DROP TABLE IF EXISTS {GithubCloneStatsTransactionsDetail.__tablename__}')) + db.execute( + text(f'DROP TABLE IF EXISTS {GithubCloneStatsTransactionsMain.__tablename__}')) db.execute(text(f'DROP TABLE IF EXISTS {GithubCloneStats.__tablename__}')) fetch_and_store_github_stats(organization_name=ORGANIZATION_NAME, repo_name=REPO_NAME, github_token=GH_TOKEN, @@ -49,9 +52,11 @@ def test_github_stats_collector(): records = session.query(GithubCloneStats).all() assert len(records) == previous_record_length - today_record = session.query(GithubCloneStats).filter_by(clone_date=datetime.today()) + today_record = session.query(GithubCloneStats).filter_by( + clone_date=datetime.today()) assert not today_record.first() - release_records = session.query(GitHubReleases).filter_by(tag_name="v10.0.3").all() + release_records = session.query( + GitHubReleases).filter_by(tag_name="v10.0.3").all() assert len(release_records) > 0 diff --git a/packaging_automation/tests/test_homebrew_statistics_collector.py b/packaging_automation/tests/test_homebrew_statistics_collector.py index 72e5e6b0..b7a1a2f0 100644 --- a/packaging_automation/tests/test_homebrew_statistics_collector.py +++ b/packaging_automation/tests/test_homebrew_statistics_collector.py @@ -3,18 +3,21 @@ from sqlalchemy import text, create_engine from ..dbconfig import (db_session, DbParams, db_connection_string) -from ..homebrew_statistics_collector import (fetch_and_save_homebrew_stats, HomebrewStats) +from ..homebrew_statistics_collector import ( + fetch_and_save_homebrew_stats, HomebrewStats) DB_USER_NAME = os.getenv("DB_USER_NAME") DB_PASSWORD = os.getenv("DB_PASSWORD") DB_HOST_AND_PORT = os.getenv("DB_HOST_AND_PORT") DB_NAME = os.getenv("DB_NAME") -db_parameters = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD, host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) +db_parameters = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD, + host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) def test_fetch_and_save_homebrew_stats(): - db = create_engine(db_connection_string(db_params=db_parameters, is_test=True)) + db = create_engine(db_connection_string( + db_params=db_parameters, is_test=True)) db.execute(text(f'DROP TABLE IF EXISTS {HomebrewStats.__tablename__}')) session = db_session(db_params=db_parameters, is_test=True) diff --git a/packaging_automation/tests/test_package_cloud_statistics_collector.py b/packaging_automation/tests/test_package_cloud_statistics_collector.py index 261fd36a..e027a519 100644 --- a/packaging_automation/tests/test_package_cloud_statistics_collector.py +++ b/packaging_automation/tests/test_package_cloud_statistics_collector.py @@ -18,7 +18,8 @@ PACKAGE_CLOUD_ADMIN_API_TOKEN = os.getenv("PACKAGE_CLOUD_ADMIN_API_TOKEN") REPO = PackageCloudRepo.azure ORGANIZATION = PackageCloudOrganization.citusdata -db_parameters = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD, host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) +db_parameters = DbParams(user_name=DB_USER_NAME, password=DB_PASSWORD, + host_and_port=DB_HOST_AND_PORT, db_name=DB_NAME) # 7 Records are fetched for each package from package cloud. To check the record count, we need to multiply package # count with 7 PACKAGE_SAVED_HISTORIC_RECORD_COUNT = 7 @@ -29,8 +30,10 @@ def test_fetch_and_save_package_cloud_stats(): - db = create_engine(db_connection_string(db_params=db_parameters, is_test=True)) - db.execute(text(f'DROP TABLE IF EXISTS {PackageCloudDownloadStats.__tablename__}')) + db = create_engine(db_connection_string( + db_params=db_parameters, is_test=True)) + db.execute( + text(f'DROP TABLE IF EXISTS {PackageCloudDownloadStats.__tablename__}')) session = db_session(db_params=db_parameters, is_test=True) page_record_count = 3 parallel_count = 3 @@ -46,14 +49,16 @@ def test_fetch_and_save_package_cloud_stats(): records = session.query(PackageCloudDownloadStats).all() - assert len(records) == filtered_package_count * PACKAGE_SAVED_HISTORIC_RECORD_COUNT + assert len(records) == filtered_package_count * \ + PACKAGE_SAVED_HISTORIC_RECORD_COUNT def get_filtered_package_count(session) -> int: # Since package count for our test repo is lower than 100, we get the total package details by getting all the # packages in one call result = stat_get_request( - package_list_with_pagination_request_address(PACKAGE_CLOUD_PARAMETERS, 1, 100), + package_list_with_pagination_request_address( + PACKAGE_CLOUD_PARAMETERS, 1, 100), RequestType.package_cloud_list_package, session) package_info_list = json.loads(result.content) package_list = list(filter( diff --git a/packaging_automation/tests/test_packaging_warning_handler.py b/packaging_automation/tests/test_packaging_warning_handler.py index 7f4ef1e4..acce081c 100644 --- a/packaging_automation/tests/test_packaging_warning_handler.py +++ b/packaging_automation/tests/test_packaging_warning_handler.py @@ -23,8 +23,10 @@ def test_deb_filter_warning_lines(): encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: lines = reader.read().splitlines() - base_warning_lines, package_specific_warning_lines = filter_warning_lines(lines, PackageType.deb) - assert len(base_warning_lines) == 11 and len(package_specific_warning_lines) == 7 + base_warning_lines, package_specific_warning_lines = filter_warning_lines( + lines, PackageType.deb) + assert len(base_warning_lines) == 11 and len( + package_specific_warning_lines) == 7 def test_rpm_filter_warning_lines(): @@ -32,8 +34,10 @@ def test_rpm_filter_warning_lines(): encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: lines = reader.read().splitlines() - base_warning_lines, package_specific_warning_lines = filter_warning_lines(lines, PackageType.rpm) - assert len(base_warning_lines) == 10 and len(package_specific_warning_lines) == 1 + base_warning_lines, package_specific_warning_lines = filter_warning_lines( + lines, PackageType.rpm) + assert len(base_warning_lines) == 10 and len( + package_specific_warning_lines) == 1 def test_get_base_warnings_to_be_raised(): @@ -44,7 +48,8 @@ def test_get_base_warnings_to_be_raised(): base_warning_lines, _ = filter_warning_lines(lines, PackageType.deb) base_ignore_list, _ = parse_ignore_lists( f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml", PackageType.deb) - base_warnings_to_be_raised = get_warnings_to_be_raised(base_ignore_list, base_warning_lines) + base_warnings_to_be_raised = get_warnings_to_be_raised( + base_ignore_list, base_warning_lines) assert len(base_warnings_to_be_raised) == 1 @@ -53,7 +58,8 @@ def test_get_debian_warnings_to_be_raised(): encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: lines = reader.read().splitlines() - _, package_specific_warning_lines = filter_warning_lines(lines, PackageType.deb) + _, package_specific_warning_lines = filter_warning_lines( + lines, PackageType.deb) _, debian_ignore_list = parse_ignore_lists( f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml", PackageType.deb) debian_warnings_to_be_raised = get_warnings_to_be_raised(debian_ignore_list, @@ -66,14 +72,16 @@ def test_get_error_message(): encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: lines = reader.read().splitlines() - base_warning_lines, debian_warning_lines = filter_warning_lines(lines, PackageType.deb) + base_warning_lines, debian_warning_lines = filter_warning_lines( + lines, PackageType.deb) base_ignore_list, debian_ignore_list = parse_ignore_lists( f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml", PackageType.deb) base_warnings_to_be_raised = get_warnings_to_be_raised(base_ignore_list, base_warning_lines) debian_warnings_to_be_raised = get_warnings_to_be_raised(debian_ignore_list, debian_warning_lines) - error_message = get_error_message(base_warnings_to_be_raised, debian_warnings_to_be_raised, PackageType.deb) + error_message = get_error_message( + base_warnings_to_be_raised, debian_warnings_to_be_raised, PackageType.deb) assert error_message == "Warning lines:\nWarning: Unhandled\nDebian Warning lines:\n" \ "citus-enterprise100_11.x86_64: W: invalid-date-format\n" \ "citus-enterprise100_11.x86_64: E: zero-length /usr/pgsql-/usr/lib/share/extension/\n" @@ -84,14 +92,16 @@ def test_get_error_message_empty_package_specific_errors(): encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: lines = reader.read().splitlines() - base_warning_lines, debian_warning_lines = filter_warning_lines(lines, PackageType.deb) + base_warning_lines, debian_warning_lines = filter_warning_lines( + lines, PackageType.deb) base_ignore_list, debian_ignore_list = parse_ignore_lists( f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml", PackageType.deb) base_warnings_to_be_raised = get_warnings_to_be_raised(base_ignore_list, base_warning_lines) debian_warnings_to_be_raised = get_warnings_to_be_raised(debian_ignore_list, debian_warning_lines) - error_message = get_error_message(base_warnings_to_be_raised, debian_warnings_to_be_raised, PackageType.deb) + error_message = get_error_message( + base_warnings_to_be_raised, debian_warnings_to_be_raised, PackageType.deb) assert error_message == "Warning lines:\nWarning: Unhandled\n" diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 586ad7ed..c2ad7918 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -11,7 +11,8 @@ github_token = os.getenv("GH_TOKEN") -BASE_PATH = pathlib2.Path(__file__).parents[2] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") +BASE_PATH = pathlib2.Path(__file__).parents[2] if os.getenv( + "BASE_PATH") is None else os.getenv("BASE_PATH") MAIN_BRANCH = "test-tools-scripts" TEST_CHECKOUT_DIR = "citus_test" @@ -23,7 +24,8 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") + run( + f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major @@ -40,25 +42,33 @@ def test_major_release(): schema_version="") update_release_return_value = update_release( github_token=github_token, project_params=project_params, - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + earliest_pr_date=datetime.strptime( + '2021.03.25 00:00', '%Y.%m.%d %H:%M'), exec_path=test_base_path_major, is_test=True) run(f"git checkout {update_release_return_value.release_branch_name}") - assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, " 10.1.0") - assert file_includes_line(test_base_path_major, CONFIGURE_IN, "AC_INIT([Citus], [10.1.0])") - assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.1.0'") - assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_STRING='Citus 10.1.0'") + assert file_includes_line(test_base_path_major, + MULTI_EXTENSION_OUT, " 10.1.0") + assert file_includes_line(test_base_path_major, + CONFIGURE_IN, "AC_INIT([Citus], [10.1.0])") + assert file_includes_line(test_base_path_major, + CONFIGURE, "PACKAGE_VERSION='10.1.0'") + assert file_includes_line(test_base_path_major, + CONFIGURE, "PACKAGE_STRING='Citus 10.1.0'") assert file_includes_line(test_base_path_major, CONFIGURE, r"\`configure' configures Citus 10.1.0 to adapt to many kinds of systems.") assert file_includes_line(test_base_path_major, CONFIGURE, ' short | recursive ) echo "Configuration of Citus 10.1.0:";;') - assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.1.0'") - assert get_last_commit_message(test_base_path_major) == "Bump citus version to 10.1.0\n" + assert file_includes_line(test_base_path_major, + CONFIGURE, "PACKAGE_VERSION='10.1.0'") + assert get_last_commit_message( + test_base_path_major) == "Bump citus version to 10.1.0\n" run(f"git checkout {update_release_return_value.upcoming_version_branch}") - assert file_includes_line(test_base_path_major, CITUS_CONTROL, "default_version = '10.2-1'") + assert file_includes_line(test_base_path_major, + CITUS_CONTROL, "default_version = '10.2-1'") assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, "-- Test downgrade to 10.1-1 from 10.2-1") assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, @@ -71,22 +81,31 @@ def test_major_release(): "SELECT * FROM print_extension_changes();") - previous_print_extension_changes == 2 assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, "-- Snapshot of state at 10.2-1") - assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, " 10.2devel") + assert file_includes_line(test_base_path_major, + MULTI_EXTENSION_OUT, " 10.2devel") assert count_line_in_file(test_base_path_major, MULTI_EXTENSION_SQL, "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 - assert file_includes_line(test_base_path_major, CONFIG_PY, "MASTER_VERSION = '10.2'") - assert file_includes_line(test_base_path_major, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert file_includes_line(test_base_path_major, + CONFIG_PY, "MASTER_VERSION = '10.2'") + assert file_includes_line(test_base_path_major, + CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert file_includes_line(test_base_path_major, + CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_includes_line(test_base_path_major, + CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") assert file_includes_line(test_base_path_major, CONFIGURE, r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") assert file_includes_line(test_base_path_major, CONFIGURE, ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert os.path.exists(f"{test_base_path_major}/{update_release_return_value.upgrade_path_sql_file}") - assert os.path.exists(f"{test_base_path_major}/{update_release_return_value.downgrade_path_sql_file}") - assert get_last_commit_message(test_base_path_major) == 'Bump citus version to 10.2devel\n' + assert file_includes_line(test_base_path_major, + CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert os.path.exists( + f"{test_base_path_major}/{update_release_return_value.upgrade_path_sql_file}") + assert os.path.exists( + f"{test_base_path_major}/{update_release_return_value.downgrade_path_sql_file}") + assert get_last_commit_message( + test_base_path_major) == 'Bump citus version to 10.2devel\n' run(f"git checkout {MAIN_BRANCH}") @@ -100,9 +119,11 @@ def test_patch_release(): update_release( github_token=github_token, project_params=project_params, - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + earliest_pr_date=datetime.strptime( + '2021.03.25 00:00', '%Y.%m.%d %H:%M'), exec_path=test_base_path_patch, is_test=True) - assert file_includes_line(test_base_path_patch, MULTI_EXTENSION_OUT, f" {project_params.project_version}") + assert file_includes_line( + test_base_path_patch, MULTI_EXTENSION_OUT, f" {project_params.project_version}") assert file_includes_line(test_base_path_patch, CONFIGURE_IN, f"AC_INIT([Citus], [{project_params.project_version}])") assert file_includes_line(test_base_path_patch, CONFIGURE, diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 820b8bcf..34ae78ed 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -26,26 +26,31 @@ def initialize_env(): def test_decode_triggering_event_info(): - event_source, branch_name = decode_triggering_event_info(f"refs/heads/{NON_DEFAULT_BRANCH_NAME}") + event_source, branch_name = decode_triggering_event_info( + f"refs/heads/{NON_DEFAULT_BRANCH_NAME}") assert event_source == GithubTriggerEventSource.branch_push and branch_name == NON_DEFAULT_BRANCH_NAME - event_source, tag_name = decode_triggering_event_info(f"refs/tags/{TAG_NAME}") + event_source, tag_name = decode_triggering_event_info( + f"refs/tags/{TAG_NAME}") assert event_source == GithubTriggerEventSource.tag_push and tag_name == TAG_NAME def test_decode_tag_parts(): tag_parts = decode_tag_parts(TAG_NAME) - assert len(tag_parts) == 3 and tag_parts[0] == "10" and tag_parts[1] == "0" and tag_parts[2] == "3" + assert len( + tag_parts) == 3 and tag_parts[0] == "10" and tag_parts[1] == "0" and tag_parts[2] == "3" with pytest.raises(ValueError): decode_tag_parts(INVALID_TAG_NAME) def test_get_image_tag(): - image_name = get_image_tag(remove_prefix(TAG_NAME, "v"), DockerImageType.latest) + image_name = get_image_tag(remove_prefix( + TAG_NAME, "v"), DockerImageType.latest) assert image_name == "10.0.3" - image_name = get_image_tag(remove_prefix(TAG_NAME, "v"), DockerImageType.postgres_12) + image_name = get_image_tag(remove_prefix( + TAG_NAME, "v"), DockerImageType.postgres_12) assert image_name == "10.0.3-pg12" diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 80106bfc..fc99c534 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -31,7 +31,8 @@ def teardown_module(): def test_update_docker_file_for_latest_postgres(): - update_docker_file_for_latest_postgres(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_VERSION) + update_docker_file_for_latest_postgres( + PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_VERSION) with open(f"{TEST_BASE_PATH}/Dockerfile", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: content = reader.read() @@ -44,7 +45,8 @@ def test_update_docker_file_for_latest_postgres(): def test_update_regular_docker_compose_file(): - update_regular_docker_compose_file(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH) + update_regular_docker_compose_file( + PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH) parameterized_str = f' image: "citusdata/{PROJECT_NAME}:{PROJECT_VERSION}"' with open(f"{TEST_BASE_PATH}/docker-compose.yml", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: @@ -56,7 +58,8 @@ def test_update_regular_docker_compose_file(): def test_update_docker_file_alpine(): - update_docker_file_alpine(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_VERSION) + update_docker_file_alpine( + PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_VERSION) with open(f"{TEST_BASE_PATH}/alpine/Dockerfile", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: content = reader.read() @@ -67,7 +70,8 @@ def test_update_docker_file_alpine(): def test_update_docker_file_for_postgres12(): - update_docker_file_for_postgres12(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH) + update_docker_file_for_postgres12( + PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH) with open(f"{TEST_BASE_PATH}/postgres-12/Dockerfile", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: content = reader.read() @@ -125,5 +129,6 @@ def test_pkgvar_postgres_version_existence(): for line in lines: if line.startswith("latest_postgres_version"): has_match = True - assert line.strip() == f"latest_postgres_version={POSTGRES_VERSION}" + assert line.strip( + ) == f"latest_postgres_version={POSTGRES_VERSION}" assert has_match diff --git a/packaging_automation/tests/test_update_package_properties.py b/packaging_automation/tests/test_update_package_properties.py index d1439708..e4a3c74e 100644 --- a/packaging_automation/tests/test_update_package_properties.py +++ b/packaging_automation/tests/test_update_package_properties.py @@ -25,10 +25,13 @@ PROJECT_VERSION = os.getenv("PROJECT_VERSION", default="10.0.3") TAG_NAME = os.getenv("TAG_NAME", default="v10.0.3") PROJECT_NAME = os.getenv("PROJECT_NAME", default="citus") -MICROSOFT_EMAIL = os.getenv("MICROSOFT_EMAIL", default="gindibay@microsoft.com") +MICROSOFT_EMAIL = os.getenv( + "MICROSOFT_EMAIL", default="gindibay@microsoft.com") NAME_SURNAME = os.getenv("NAME_SURNAME", default="Gurkan Indibay") -CHANGELOG_DATE_STR = os.getenv("CHANGELOG_DATE", 'Thu, 18 Mar 2021 01:40:08 +0000') -CHANGELOG_DATE = datetime.strptime(CHANGELOG_DATE_STR, '%a, %d %b %Y %H:%M:%S %z') +CHANGELOG_DATE_STR = os.getenv( + "CHANGELOG_DATE", 'Thu, 18 Mar 2021 01:40:08 +0000') +CHANGELOG_DATE = datetime.strptime( + CHANGELOG_DATE_STR, '%a, %d %b %Y %H:%M:%S %z') def default_changelog_param_for_test(changelog_date): @@ -39,7 +42,8 @@ def default_changelog_param_for_test(changelog_date): return changelog_param -DEFAULT_CHANGELOG_PARAM_FOR_TEST = default_changelog_param_for_test(CHANGELOG_DATE) +DEFAULT_CHANGELOG_PARAM_FOR_TEST = default_changelog_param_for_test( + CHANGELOG_DATE) def test_get_version_number(): @@ -63,7 +67,8 @@ def test_prepend_latest_changelog_into_debian_changelog(): changelog_param = default_changelog_param_for_test(CHANGELOG_DATE) try: - prepend_latest_changelog_into_debian_changelog(changelog_param, changelog_file_path) + prepend_latest_changelog_into_debian_changelog( + changelog_param, changelog_file_path) verify_prepend_debian_changelog(changelog_file_path) finally: os.remove(changelog_file_path) @@ -138,7 +143,8 @@ def test_update_pkg_vars(): copyfile(pkgvars_path, pkgvars_copy_path) try: - update_pkgvars(DEFAULT_CHANGELOG_PARAM_FOR_TEST, templates_path, f"{TEST_BASE_PATH}/files/") + update_pkgvars(DEFAULT_CHANGELOG_PARAM_FOR_TEST, + templates_path, f"{TEST_BASE_PATH}/files/") verify_pkgvars(pkgvars_path) finally: copyfile(pkgvars_copy_path, pkgvars_path) @@ -188,4 +194,5 @@ def test_update_all_changes(): def test_regex(): - print(re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", "### citus v10.0.3 (March 16, 2021) ###")) + print(re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", + "### citus v10.0.3 (March 16, 2021) ###")) diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 84ec94aa..8daab5f5 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -2,7 +2,8 @@ import pathlib2 -from ..common_tool_methods import (run, DEFAULT_UNICODE_ERROR_HANDLER, DEFAULT_ENCODING_FOR_FILE_HANDLING) +from ..common_tool_methods import ( + run, DEFAULT_UNICODE_ERROR_HANDLER, DEFAULT_ENCODING_FOR_FILE_HANDLING) from ..update_pgxn import update_meta_json, update_pkgvars BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) @@ -26,7 +27,7 @@ def teardown_module(): def test_update_meta_json(): update_meta_json(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH) with open(f"{TEST_BASE_PATH}/META.json", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, - errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: + errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: content = reader.read() lines = content.splitlines() assert lines[4] == f' "version": "{PROJECT_VERSION}",' @@ -37,7 +38,7 @@ def test_update_meta_json(): def test_update_pkgvars(): update_pkgvars(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH) with open(f"{TEST_BASE_PATH}/pkgvars", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, - errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: + errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: content = reader.read() lines = content.splitlines() assert lines[2] == f'pkglatest={PROJECT_VERSION}' diff --git a/packaging_automation/tests/test_utils.py b/packaging_automation/tests/test_utils.py index 201f8017..f30881e6 100644 --- a/packaging_automation/tests/test_utils.py +++ b/packaging_automation/tests/test_utils.py @@ -5,15 +5,18 @@ from ..common_tool_methods import (run) -TEST_BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) +TEST_BASE_PATH = os.getenv( + "BASE_PATH", default=pathlib2.Path(__file__).parents[2]) def are_strings_equal(expected_string: str, actual_str: str) -> bool: - output_list = [li for li in difflib.ndiff(expected_string, actual_str) if li[0] != ' '] + output_list = [li for li in difflib.ndiff( + expected_string, actual_str) if li[0] != ' '] for output in output_list: if not (output.strip() == '+' or output.strip() == '-'): - raise Exception(f"Actual and expected string are not same Diff:{''.join(output_list)} ") + raise Exception( + f"Actual and expected string are not same Diff:{''.join(output_list)} ") return True diff --git a/packaging_automation/update_docker.py b/packaging_automation/update_docker.py index 097e08bf..0a16281e 100644 --- a/packaging_automation/update_docker.py +++ b/packaging_automation/update_docker.py @@ -89,7 +89,8 @@ def get_new_changelog_entry(project_version: str, postgres_version: str = ""): def update_changelog(project_version: str, exec_path: str, postgres_version: str = ""): - latest_changelog = get_new_changelog_entry(project_version, postgres_version) + latest_changelog = get_new_changelog_entry( + project_version, postgres_version) changelog_file_path = f"{exec_path}/CHANGELOG.md" with open(changelog_file_path, "r+", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: @@ -100,7 +101,8 @@ def update_changelog(project_version: str, exec_path: str, postgres_version: str reader.seek(0, 0) reader.write(changelog) else: - raise ValueError(f"Already using version {project_version} in the changelog") + raise ValueError( + f"Already using version {project_version} in the changelog") def update_all_docker_files(project_version: str, exec_path: str, postgres_version: str): @@ -108,15 +110,21 @@ def update_all_docker_files(project_version: str, exec_path: str, postgres_versi pkgvars_file = f"{exec_path}/pkgvars" if postgres_version: - update_pkgvars(project_version, template_path, pkgvars_file, postgres_version) + update_pkgvars(project_version, template_path, + pkgvars_file, postgres_version) persisted_postgres_version = read_postgres_version(pkgvars_file) - update_docker_file_for_latest_postgres(project_version, template_path, exec_path, persisted_postgres_version) - update_regular_docker_compose_file(project_version, template_path, exec_path) - update_docker_file_alpine(project_version, template_path, exec_path, persisted_postgres_version) - update_docker_file_for_postgres12(project_version, template_path, exec_path) - update_docker_file_for_postgres14(project_version, template_path, exec_path) + update_docker_file_for_latest_postgres( + project_version, template_path, exec_path, persisted_postgres_version) + update_regular_docker_compose_file( + project_version, template_path, exec_path) + update_docker_file_alpine( + project_version, template_path, exec_path, persisted_postgres_version) + update_docker_file_for_postgres12( + project_version, template_path, exec_path) + update_docker_file_for_postgres14( + project_version, template_path, exec_path) update_changelog(project_version, exec_path, postgres_version) @@ -129,11 +137,13 @@ def read_postgres_version(pkgvars_file: str) -> str: if line.startswith("latest_postgres_version"): line_parts = line.split("=") if len(line_parts) != 2: - raise ValueError("keys and values should be seperated with '=' sign") + raise ValueError( + "keys and values should be seperated with '=' sign") postgres_version = line_parts[1].rstrip("\n") break if not postgres_version: - raise ValueError("pkgvars file should include a line with key latest_postgres_version") + raise ValueError( + "pkgvars file should include a line with key latest_postgres_version") else: # Setting it because pkgvars does not exist initially postgres_version = "13.4" @@ -141,7 +151,8 @@ def read_postgres_version(pkgvars_file: str) -> str: def update_pkgvars(project_version: str, template_path: str, pkgvars_file: str, postgres_version: str): - content = process_template_file(project_version, template_path, "docker-pkgvars.tmpl", postgres_version) + content = process_template_file( + project_version, template_path, "docker-pkgvars.tmpl", postgres_version) with open(pkgvars_file, "w", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as writer: writer.write(content) @@ -170,12 +181,14 @@ def update_pkgvars(project_version: str, template_path: str, pkgvars_file: str, os.chdir(execution_path) pr_branch = f"release-{args.prj_ver}-{uuid.uuid4()}" run(f"git checkout -b {pr_branch}") - update_all_docker_files(args.prj_ver, execution_path, args.postgres_version) + update_all_docker_files( + args.prj_ver, execution_path, args.postgres_version) run("git add .") commit_message = f"Bump docker to version {args.prj_ver}" run(f'git commit -m "{commit_message}"') if not args.is_test: run(f'git push --set-upstream origin {pr_branch}') - create_pr(args.gh_token, pr_branch, commit_message, REPO_OWNER, PROJECT_NAME, MAIN_BRANCH) + create_pr(args.gh_token, pr_branch, commit_message, + REPO_OWNER, PROJECT_NAME, MAIN_BRANCH) remove_cloned_code(execution_path) diff --git a/packaging_automation/update_package_properties.py b/packaging_automation/update_package_properties.py index f6530163..4aa2775e 100644 --- a/packaging_automation/update_package_properties.py +++ b/packaging_automation/update_package_properties.py @@ -73,7 +73,7 @@ class PackagePropertiesParams: changelog_entry: str = "" @property - def changelog_version_entry(self)->str: + def changelog_version_entry(self) -> str: return f"{self.project_version}-{self.fancy_version_number}" @property @@ -120,7 +120,8 @@ def rpm_header(self): @property def debian_trailer(self): - formatted_date = self.changelog_date.strftime("%a, %d %b %Y %H:%M:%S %z") + formatted_date = self.changelog_date.strftime( + "%a, %d %b %Y %H:%M:%S %z") return f" -- {self.name_surname} <{self.microsoft_email}> {formatted_date}\n" @@ -128,7 +129,8 @@ def get_enum_from_changelog_project_name(project_name) -> SupportedProject: for e in SupportedProject: if e.value.changelog_project_name == project_name: return e - raise ValueError(f"{project_name} could not be found in supported project changelog names.") + raise ValueError( + f"{project_name} could not be found in supported project changelog names.") def spec_file_name(project_name: str) -> str: @@ -136,7 +138,8 @@ def spec_file_name(project_name: str) -> str: def get_last_changelog_content(all_changelog_content: str) -> str: - second_changelog_index = find_nth_occurrence_position(all_changelog_content, "###", 3) + second_changelog_index = find_nth_occurrence_position( + all_changelog_content, "###", 3) changelogs = all_changelog_content[:second_changelog_index] lines = changelogs.splitlines() if len(lines) < 1: @@ -184,7 +187,8 @@ def get_debian_latest_changelog(package_properties_params: PackagePropertiesPara def prepend_latest_changelog_into_debian_changelog(package_properties_params: PackagePropertiesParams, changelog_file_path: str) -> None: - debian_latest_changelog = get_debian_latest_changelog(package_properties_params) + debian_latest_changelog = get_debian_latest_changelog( + package_properties_params) with open(changelog_file_path, mode="r+", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: if not (package_properties_params.changelog_version_entry in reader.readline()): @@ -203,7 +207,8 @@ def update_pkgvars(package_properties_params: PackagePropertiesParams, templates version_str = package_properties_params.version_number_with_project_name - template = env.get_template(package_properties_params.pkgvars_template_file_name) + template = env.get_template( + package_properties_params.pkgvars_template_file_name) pkgvars_content = f"{template.render(version=version_str)}\n" with open(f'{pkgvars_path}/pkgvars', "w", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, @@ -216,7 +221,8 @@ def rpm_changelog_history(spec_file_path: str) -> str: errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: spec_content = reader.read() changelog_index = spec_content.find("%changelog") - changelog_content = spec_content[changelog_index + len("%changelog") + 1:] + changelog_content = spec_content[changelog_index + + len("%changelog") + 1:] return changelog_content @@ -243,12 +249,14 @@ def update_rpm_spec(package_properties_params: PackagePropertiesParams, spec_ful env = get_template_environment(templates_path) rpm_version = package_properties_params.rpm_version - template = env.get_template(package_properties_params.rpm_spec_template_file_name) + template = env.get_template( + package_properties_params.rpm_spec_template_file_name) history_lines = rpm_changelog_history(spec_full_path).splitlines() if len(history_lines) > 0 and package_properties_params.version_number_with_project_name in history_lines[0]: - raise ValueError(f"{package_properties_params.project_version} already exists in rpm spec file") + raise ValueError( + f"{package_properties_params.project_version} already exists in rpm spec file") latest_changelog = get_rpm_changelog(package_properties_params) changelog = f"{latest_changelog}\n\n{rpm_changelog_history(spec_full_path)}" @@ -274,11 +282,13 @@ def validate_package_properties_params_for_update_all_changes(package_props: Pac @validate_parameters def update_all_changes(package_properties_params: PackagePropertiesParams, packaging_path: str): - validate_package_properties_params_for_update_all_changes(package_properties_params) + validate_package_properties_params_for_update_all_changes( + package_properties_params) templates_path = f"{BASE_PATH}/templates" update_pkgvars(package_properties_params, templates_path, f"{packaging_path}") - prepend_latest_changelog_into_debian_changelog(package_properties_params, f"{packaging_path}/debian/changelog") + prepend_latest_changelog_into_debian_changelog( + package_properties_params, f"{packaging_path}/debian/changelog") spec_full_path = f"{packaging_path}/{package_properties_params.spec_file_name}" update_rpm_spec(package_properties_params, spec_full_path, templates_path) @@ -287,9 +297,11 @@ def update_all_changes(package_properties_params: PackagePropertiesParams, if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--gh_token', required=True) - parser.add_argument('--prj_name', choices=[r.name for r in SupportedProject]) + parser.add_argument( + '--prj_name', choices=[r.name for r in SupportedProject]) parser.add_argument('--tag_name', required=True) - parser.add_argument('--fancy_ver_no', type=int, choices=range(1, 10), default=1) + parser.add_argument('--fancy_ver_no', type=int, + choices=range(1, 10), default=1) parser.add_argument('--email', required=True) parser.add_argument('--name', required=True) parser.add_argument('--date') diff --git a/packaging_automation/update_pgxn.py b/packaging_automation/update_pgxn.py index cec9b77a..d7f61c5d 100644 --- a/packaging_automation/update_pgxn.py +++ b/packaging_automation/update_pgxn.py @@ -63,6 +63,7 @@ def update_pgxn_files(project_version: str, template_path: str, exec_path: str): run(f'git commit -a -m "{commit_message}"') if not args.is_test: run(f'git push --set-upstream origin {pr_branch}') - create_pr(github_token, pr_branch, commit_message, REPO_OWNER, PROJECT_NAME, main_branch) + create_pr(github_token, pr_branch, commit_message, + REPO_OWNER, PROJECT_NAME, main_branch) if not args.is_test and not args.pipeline: remove_cloned_code(execution_path) diff --git a/packaging_automation/upload_to_package_cloud.py b/packaging_automation/upload_to_package_cloud.py index 9270fe31..51d14068 100644 --- a/packaging_automation/upload_to_package_cloud.py +++ b/packaging_automation/upload_to_package_cloud.py @@ -75,7 +75,8 @@ def upload_files_in_directory_to_package_cloud(directoryName: str, distro_name: if not main_branch: raise ValueError("main_branch should be defined") if main_branch != current_branch: - print(f"Package publishing skipped since current branch is not equal to {main_branch}") + print( + f"Package publishing skipped since current branch is not equal to {main_branch}") return MultipleReturnValue(ret_vals=[]) ret_status: List[ReturnValue] = [] @@ -105,7 +106,8 @@ def package_exists(package_cloud_token: str, repo_owner: str, repo_name: str, pa platform: str) -> bool: query_url = (f"https://packagecloud.io/api/v1/repos/{repo_owner}/{repo_name}/search?" f"q={package_name}&filter=all&dist={urllib.parse.quote(platform, safe='')}") - response = requests.get(query_url, auth=HTTPBasicAuth(package_cloud_token, '')) + response = requests.get( + query_url, auth=HTTPBasicAuth(package_cloud_token, '')) return response.ok @@ -113,7 +115,8 @@ def package_exists(package_cloud_token: str, repo_owner: str, repo_name: str, pa parser = argparse.ArgumentParser() parser.add_argument('--platform', choices=supported_distros.keys()) parser.add_argument('--package_cloud_api_token', required=True) - parser.add_argument('--repository_name', required=True, choices=supported_repos) + parser.add_argument('--repository_name', required=True, + choices=supported_repos) parser.add_argument('--output_file_path', required=True) parser.add_argument('--current_branch', required=True) parser.add_argument('--main_branch', required=True) @@ -136,4 +139,5 @@ def package_exists(package_cloud_token: str, repo_owner: str, repo_name: str, pa print(f'File successfully uploaded. Distro: {rv.distro} File Name: {os.path.basename(rv.file_name)} ' f'Repo Name: {rv.repo}') if not multiple_return_value.success_status(): - raise ValueError("There were some errors while uploading some packages") + raise ValueError( + "There were some errors while uploading some packages") diff --git a/test-images/scripts/test.py b/test-images/scripts/test.py index 80ead82e..239900ea 100644 --- a/test-images/scripts/test.py +++ b/test-images/scripts/test.py @@ -12,7 +12,8 @@ def run_with_output(command, *args, **kwargs): # this method's main objective is to return output. Therefore it is caller's responsibility to handle # success status # pylint: disable=subprocess-run-check - result = subprocess.run(shlex.split(command), *args, capture_output=True, **kwargs) + result = subprocess.run(shlex.split(command), *args, + capture_output=True, **kwargs) return result @@ -51,7 +52,8 @@ def verify_output(result, expected_result) -> bool: def test_citus(): assert verify_output(run_with_output('pg_ctl -D citus -o "-p 9700" -l citus/citus_logfile start'), r"^'waiting for server to start.... done\\nserver started\\n'$") - assert verify_output(run_with_output('psql -p 9700 -c "CREATE EXTENSION citus;"'), r"^'CREATE EXTENSION\\n'$") + assert verify_output(run_with_output( + 'psql -p 9700 -c "CREATE EXTENSION citus;"'), r"^'CREATE EXTENSION\\n'$") assert verify_output(run_with_output('psql -p 9700 -c "select version();"'), rf".*PostgreSQL {POSTGRES_VERSION}.* on x86_64-pc-linux-gnu, compiled by gcc \(.*") # Since version info for ol and el 7 contains undefined, undefined was needed to add as expected param for pc diff --git a/test-images/scripts/test_internal.sh b/test-images/scripts/test_internal.sh index 6c6db111..ac7e4470 100755 --- a/test-images/scripts/test_internal.sh +++ b/test-images/scripts/test_internal.sh @@ -1,2 +1,2 @@ #!/bin/bash -python3.8 -m pytest -q test.py -s \ No newline at end of file +python3.8 -m pytest -q test.py -s diff --git a/travis/README.md b/travis/README.md index beb0c203..dca1059e 100644 --- a/travis/README.md +++ b/travis/README.md @@ -8,9 +8,9 @@ Travis CI itself has [_fantastic_ documentation](https://docs.travis-ci.com) cov ## What's Included -With the provided scripts, it's fairly easy to set up a matrix build (on all PostgreSQL and CitusDB versions) to run tests against an installed extension in a "system" cluster (`make installcheck`) or against a one-off cluster started for the test itself (often `make check`). +With the provided scripts, it's fairly easy to set up a matrix build (on all PostgreSQL and CitusDB versions) to run tests against an installed extension in a "system" cluster ( `make installcheck` ) or against a one-off cluster started for the test itself (often `make check` ). -These scripts were inspired by [a similar workflow](https://gist.github.com/petere) used by Peter Eisentraut to test his own PostgreSQL extensions (e.g. [`pguri`](https://github.com/petere/pguri/blob/1.20151224/.travis.yml)). Jason Petersen adapted that workflow into a set of scripts, which lived in [a Gist](https://gist.github.com/jasonmp85/9963879) before the creation of this repository. +These scripts were inspired by [a similar workflow](https://gist.github.com/petere) used by Peter Eisentraut to test his own PostgreSQL extensions (e.g. [ `pguri` ](https://github.com/petere/pguri/blob/1.20151224/.travis.yml)). Jason Petersen adapted that workflow into a set of scripts, which lived in [a Gist](https://gist.github.com/jasonmp85/9963879) before the creation of this repository. ### `setup_apt` @@ -22,20 +22,21 @@ It is possible that your Travis CI virtual machine starts with a PostgreSQL inst ### `install_pg` -A PostgreSQL version will be installed based on the value of the `PGVERSION` environment variable. In addition, common extension dependencies and development headers are installed. Suitable values for `PGVERSION` might be `9.5`, `9.6`, or `10`. +A PostgreSQL version will be installed based on the value of the `PGVERSION` environment variable. In addition, common extension dependencies and development headers are installed. Suitable values for `PGVERSION` might be `9.5` , `9.6` , or `10` . ### `install_uncrustify` Installs a recent `uncrustify` version from a Citus-specific Trusty package stored in S3. + ### `install_citus` -Similar to `install_pg`, but installs CitusDB instead of vanilla PostgreSQL. The convention is that `PGVERSION` should be set to ten times the CitusDB version (to avoid possible collisions with existing PostgreSQL versions). For instance, a value of `40.0` installs CitusDB 4.0, and `41.0` installs CitusDB 4.1. +Similar to `install_pg` , but installs CitusDB instead of vanilla PostgreSQL. The convention is that `PGVERSION` should be set to ten times the CitusDB version (to avoid possible collisions with existing PostgreSQL versions). For instance, a value of `40.0` installs CitusDB 4.0, and `41.0` installs CitusDB 4.1. ### `config_and_start_cluster` Since the "nuke" script removed the only running PostgreSQL cluster, you'll probably want to have one up and running to build, install, and test an extension. This script starts just such a cluster. If the `PG_PRELOAD` environment variable is set, the cluster uses its value for the `shared_preload_libraries` setting. -Note that starting your own cluster is necessary only if you want to build, install, and test your extension in a system-wide cluster. This is the case for most extensions using the PostgreSQL extension build system (i.e. `make install` followed by `make installcheck`), but some advanced use cases may not need a cluster created. +Note that starting your own cluster is necessary only if you want to build, install, and test your extension in a system-wide cluster. This is the case for most extensions using the PostgreSQL extension build system (i.e. `make install` followed by `make installcheck` ), but some advanced use cases may not need a cluster created. ### `pg_travis_test` @@ -45,20 +46,20 @@ Runs regression tests against a system-wide cluster, echoing any failures to sta Runs regression tests against a more "complex" extension, usually one that: - * Requires that `./configure` be run before `make` can proceed - * Starts its own PostgreSQL instances within a `check` Make target. + + Requires that `./configure` be run before `make` can proceed + + Starts its own PostgreSQL instances within a `check` Make target. If this script is used, it is likely that `config_and_start_cluster` is unnecessary. ### `build_new_nightly` -Checks packagecloud.io for the last nightly for the project being built. If any commits have been made to that project's GitHub development branch since the last nightly upload, this script builds a new nightly release (using `citus_package`). +Checks packagecloud.io for the last nightly for the project being built. If any commits have been made to that project's GitHub development branch since the last nightly upload, this script builds a new nightly release (using `citus_package` ). If no nightly is needed, exits immediately. ### `build_new_release` -Downloads the packaging files for the project being built. If the packaging files specify a version that is not yet in packagecloud.io, this script builds a new official release (using `citus_package`). +Downloads the packaging files for the project being built. If the packaging files specify a version that is not yet in packagecloud.io, this script builds a new official release (using `citus_package` ). If no new release is needed, exits immediately. @@ -70,13 +71,13 @@ Does not presently check ahead of time if PGXN already contains the archive; ins ### `sync_to_enterprise` -Pushes branches from the open-source Citus GitHub repository to the closed-source Citus Enterprise repository. Intended for use with the `master` branch and any branches beginning with `release-`. +Pushes branches from the open-source Citus GitHub repository to the closed-source Citus Enterprise repository. Intended for use with the `master` branch and any branches beginning with `release-` . ### `trigger_docker_nightly` Pairs with `build_new_nightly` to trigger a new Docker Hub nightly image build. Only runs if the following conditions are met: - * Project is `citus` - * OS is `debian` - * Release is `stretch` - * New nightly was produced + + Project is `citus` + + OS is `debian` + + Release is `stretch` + + New nightly was produced diff --git a/travis/build_new_nightly b/travis/build_new_nightly index 2494c27b..ecfb5e15 100755 --- a/travis/build_new_nightly +++ b/travis/build_new_nightly @@ -39,22 +39,22 @@ if { [[ "$PLATFORM_TYPE" == "el" ]] || [[ "$PLATFORM_TYPE" == "ol" ]]; } && [[ " fi case "${PLATFORM_TYPE}" in - debian|ubuntu) - pkgflavor='deb' - pkgname="${deb_pkgname:-${pkgname}}" - pkgfull="postgresql-${latestpg}-${pkgname}" - pkgarch="amd64" - ;; - el|ol) - pkgflavor='rpm' - pkgname="${rpm_pkgname:-${pkgname}}" - pkgfull="${pkgname}_${latestpg//./}" - pkgarch="x86_64" - ;; - *) - echo "$0: unknown package flavor -- ${pkgflavor}" >&2 - usage $stderr $badusage - ;; +debian | ubuntu) + pkgflavor='deb' + pkgname="${deb_pkgname:-${pkgname}}" + pkgfull="postgresql-${latestpg}-${pkgname}" + pkgarch="amd64" + ;; +el | ol) + pkgflavor='rpm' + pkgname="${rpm_pkgname:-${pkgname}}" + pkgfull="${pkgname}_${latestpg//./}" + pkgarch="x86_64" + ;; +*) + echo "$0: unknown package flavor -- ${pkgflavor}" >&2 + usage $stderr $badusage + ;; esac hubproj="${hubproj:-${pkgname}}" @@ -66,16 +66,16 @@ response=$(curl -w '\n%{http_code}\n' -s -u "${pkgauth}" "${pkgapiurl}?per_page= httpcode=$(echo "${response}" | tail -n1) case "${httpcode}" in - 404) - httpbody='[]' - ;; - 200) - httpbody=$(echo "${response}" | sed '$d') - ;; - *) - echo "$0: bad response code from packagecloud -- ${httpcode}" >&2 - exit $noservice - ;; +404) + httpbody='[]' + ;; +200) + httpbody=$(echo "${response}" | sed '$d') + ;; +*) + echo "$0: bad response code from packagecloud -- ${httpcode}" >&2 + exit $noservice + ;; esac epochstr='"1970-01-01T00:00:00.000Z"' @@ -87,7 +87,7 @@ hubapiurl+="sha=${nightlyref}&since=${lastnightly}&per_page=1" newcommitcount=$(curl -sf -H "${hubauth}" "${hubapiurl}" | jq -r 'length') if [ "${newcommitcount}" -gt 0 ]; then - citus_package -p "${TARGET_PLATFORM}" 'local' nightly 2>&1 > citus_package.log + citus_package -p "${TARGET_PLATFORM}" 'local' nightly 2>&1 >citus_package.log echo "Warnings generated by nightly docker build :" grep -Ei '(warning|\bi|\be|\bw):' citus_package.log || true mkdir -p pkgs/nightlies diff --git a/travis/build_new_release b/travis/build_new_release index ee1eb632..40ff0f9c 100755 --- a/travis/build_new_release +++ b/travis/build_new_release @@ -41,39 +41,39 @@ fi versioning="${versioning:-simple}" case "${PLATFORM_TYPE}" in - debian|ubuntu) - pkgflavor='deb' - pkgname="${deb_pkgname:-${pkgname}}" - pkgfull="postgresql-${latestpg}-${pkgname}" - - # add minor/major version to package name if using fancy versioning - if [ "${versioning}" == 'fancy' ]; then - suffix=$(echo "${pkglatest}" | grep -oE '^[0-9]+\.[0-9]+') - pkgfull="postgresql-${latestpg}-${pkgname}-${suffix}" - fi - - pkgarch="amd64" - jqfilter='map(.version + "-" + .release)' - ;; - el|ol) - pkgflavor='rpm' - pkgname="${rpm_pkgname:-${pkgname}}" - - # add minor/major version to package name if using fancy versioning - if [ "${versioning}" == 'fancy' ]; then - infix=$(echo "${pkglatest}" | grep -oE '^[0-9]+\.[0-9]+' | tr -d '.') - pkgfull="${pkgname}${infix}_${latestpg//./}" - else - pkgfull="${pkgname}_${latestpg//./}" - fi - - pkgarch="x86_64" - jqfilter='map(.version + "-" + .release | gsub("\\.centos$";"") | gsub("\\.[^.]*$";""))' - ;; - *) - echo "$0: unknown package flavor -- ${pkgflavor}" >&2 - usage $stderr $badusage - ;; +debian | ubuntu) + pkgflavor='deb' + pkgname="${deb_pkgname:-${pkgname}}" + pkgfull="postgresql-${latestpg}-${pkgname}" + + # add minor/major version to package name if using fancy versioning + if [ "${versioning}" == 'fancy' ]; then + suffix=$(echo "${pkglatest}" | grep -oE '^[0-9]+\.[0-9]+') + pkgfull="postgresql-${latestpg}-${pkgname}-${suffix}" + fi + + pkgarch="amd64" + jqfilter='map(.version + "-" + .release)' + ;; +el | ol) + pkgflavor='rpm' + pkgname="${rpm_pkgname:-${pkgname}}" + + # add minor/major version to package name if using fancy versioning + if [ "${versioning}" == 'fancy' ]; then + infix=$(echo "${pkglatest}" | grep -oE '^[0-9]+\.[0-9]+' | tr -d '.') + pkgfull="${pkgname}${infix}_${latestpg//./}" + else + pkgfull="${pkgname}_${latestpg//./}" + fi + + pkgarch="x86_64" + jqfilter='map(.version + "-" + .release | gsub("\\.centos$";"") | gsub("\\.[^.]*$";""))' + ;; +*) + echo "$0: unknown package flavor -- ${pkgflavor}" >&2 + usage $stderr $badusage + ;; esac hubproj="${hubproj:-${pkgname}}" @@ -85,21 +85,21 @@ response=$(curl -w '\n%{http_code}\n' -s -u "${pkgauth}" "${pkgapiurl}?per_page= httpcode=$(echo "${response}" | tail -n1) case "${httpcode}" in - 404) - httpbody='[]' - ;; - 200) - httpbody=$(echo "${response}" | sed '$d') - ;; - *) - echo "$0: bad response code from packagecloud -- ${httpcode}" >&2 - exit $noservice - ;; +404) + httpbody='[]' + ;; +200) + httpbody=$(echo "${response}" | sed '$d') + ;; +*) + echo "$0: bad response code from packagecloud -- ${httpcode}" >&2 + exit $noservice + ;; esac needrelease=$(echo "${httpbody}" | jq -r "${jqfilter} | index(\"${pkglatest}\") < 0") if [ "${needrelease}" == "true" ]; then - citus_package -p "${TARGET_PLATFORM}" 'local' release 2>&1 > citus_package.log + citus_package -p "${TARGET_PLATFORM}" 'local' release 2>&1 >citus_package.log echo "Warnings generated by release docker build :" grep -Ei '(warning|\bi|\be|\bw):' citus_package.log || true mkdir -p pkgs/releases diff --git a/travis/config_and_start_cluster b/travis/config_and_start_cluster index b46fb282..119358ff 100755 --- a/travis/config_and_start_cluster +++ b/travis/config_and_start_cluster @@ -12,8 +12,7 @@ make all PG_CONFIG="/usr/lib/postgresql/${PGVERSION}/bin/pg_config" sudo make install PG_CONFIG="/usr/lib/postgresql/${PGVERSION}/bin/pg_config" # Preload library if asked to do so -if [ "${PG_PRELOAD+1}" ] -then +if [ "${PG_PRELOAD+1}" ]; then echo "shared_preload_libraries = '$PG_PRELOAD'" >> \ "/etc/postgresql/${PGVERSION}/test/postgresql.conf" fi diff --git a/travis/install_custom_pg b/travis/install_custom_pg index 13dc8d81..21fbba7e 100755 --- a/travis/install_custom_pg +++ b/travis/install_custom_pg @@ -46,15 +46,15 @@ mjobs="$((procs + 1))" # configure, build, and install PostgreSQL cd postgresql ./configure --enable-cassert --enable-debug --enable-depend --with-openssl \ - --mandir="/usr/share/postgresql/${PGVERSION}/man" \ - --docdir="/usr/share/doc/postgresql-doc-${PGVERSION}" \ - --sysconfdir=/etc/postgresql-common \ - --datarootdir=/usr/share/ \ - --datadir="/usr/share/postgresql/${PGVERSION}" \ - --bindir="/usr/lib/postgresql/${PGVERSION}/bin" \ - --libdir=/usr/lib/x86_64-linux-gnu/ \ - --libexecdir=/usr/lib/postgresql/ \ - --includedir=/usr/include/postgresql/ \ + --mandir="/usr/share/postgresql/${PGVERSION}/man" \ + --docdir="/usr/share/doc/postgresql-doc-${PGVERSION}" \ + --sysconfdir=/etc/postgresql-common \ + --datarootdir=/usr/share/ \ + --datadir="/usr/share/postgresql/${PGVERSION}" \ + --bindir="/usr/lib/postgresql/${PGVERSION}/bin" \ + --libdir=/usr/lib/x86_64-linux-gnu/ \ + --libexecdir=/usr/lib/postgresql/ \ + --includedir=/usr/include/postgresql/ make -j "${mjobs}" -s all make -j "${mjobs}" -s -C src/test/isolation diff --git a/travis/release_pgxn b/travis/release_pgxn index e3cbfc9a..d570b422 100755 --- a/travis/release_pgxn +++ b/travis/release_pgxn @@ -21,8 +21,8 @@ cd pgxn # TODO: use pgxnclient in Travis to only release on new versions httpcode=$(curl -sL -u "${pgxnuser}:${PGXN_PASSWORD}" \ - -F "archive=@${archive};type=application/zip" \ - -o /dev/null -w '%{http_code}' "${pgxnurl}") + -F "archive=@${archive};type=application/zip" \ + -o /dev/null -w '%{http_code}' "${pgxnurl}") if [ "${httpcode}" -eq 200 ]; then echo "Successfully uploaded ${archive} to PGXN!" >&2 diff --git a/travis/sync_to_enterprise b/travis/sync_to_enterprise index e95f3602..0569a084 100755 --- a/travis/sync_to_enterprise +++ b/travis/sync_to_enterprise @@ -10,8 +10,8 @@ success=0 if [ "${TRAVIS_REPO_SLUG}" != 'citusdata/citus' ]; then echo "Not building citusdata/citus repo; skipping enterprise sync..." >&2 exit $success -elif [[ "${TRAVIS_BRANCH}" != 'master' ]] && - [[ "${TRAVIS_BRANCH}" != 'release-'* ]]; then +elif [[ "${TRAVIS_BRANCH}" != 'master' ]] && + [[ "${TRAVIS_BRANCH}" != 'release-'* ]]; then echo "Not on master or release branch; skipping enterprise sync..." >&2 exit $success fi @@ -20,7 +20,7 @@ creds="citus-github:${GITHUB_TOKEN}" ghurl=https://${creds}@github.com/citusdata/citus-enterprise refspec="${TRAVIS_BRANCH}:open-${TRAVIS_BRANCH}" -git push -q "${ghurl}" "${refspec}" > /dev/null 2>&1 +git push -q "${ghurl}" "${refspec}" >/dev/null 2>&1 echo "Successfully synced ${TRAVIS_BRANCH} to enterprise repo!" >&2 diff --git a/travis/trigger_docker_nightly b/travis/trigger_docker_nightly index 2732762c..071bef09 100755 --- a/travis/trigger_docker_nightly +++ b/travis/trigger_docker_nightly @@ -24,7 +24,7 @@ type="Content-Type: application/json" data='{"docker_tag": "nightly"}' httpcode=$(curl -s -H "${type}" --data "${data}" -X POST \ - -w "%{http_code}" -o /dev/null "${dockertriggerurl}") + -w "%{http_code}" -o /dev/null "${dockertriggerurl}") if [ "${httpcode}" -ne 200 ]; then echo "$0: could not trigger docker build" >&2 diff --git a/uncrustify/README.md b/uncrustify/README.md index 72ccf3ac..bc7d596e 100644 --- a/uncrustify/README.md +++ b/uncrustify/README.md @@ -1,6 +1,6 @@ # Uncrustify -`citus_indent` wraps [`uncrustify`][1], a popular C source code beautifier. When invoked, it immediately applies Citus C style on any git-tracked C files under the current working directory, though a `--check` flag is implemented to check style without modifying any files. +`citus_indent` wraps [ `uncrustify` ][1], a popular C source code beautifier. When invoked, it immediately applies Citus C style on any git-tracked C files under the current working directory, though a `--check` flag is implemented to check style without modifying any files. ## Getting Started @@ -10,7 +10,7 @@ ## Usage -Apply the `citus-style` git attribute to any files that need the Citus C style applied. After that, just ensure you're within the project's directory hierarchy and run `citus_indent` to format all files. Add style changes with `git add -p`. +Apply the `citus-style` git attribute to any files that need the Citus C style applied. After that, just ensure you're within the project's directory hierarchy and run `citus_indent` to format all files. Add style changes with `git add -p` . `citus_indent --check` is useful for scripts: it will not modify any files and simply exits with a non-zero status if any files marked with `citus-style` are non-compliant. diff --git a/uncrustify/citus-style.cfg b/uncrustify/citus-style.cfg index 14ceb2ed..c4c13b43 100644 --- a/uncrustify/citus-style.cfg +++ b/uncrustify/citus-style.cfg @@ -5,33 +5,33 @@ # # The type of line endings -newlines = lf # auto/lf/crlf/cr +newlines = lf # auto/lf/crlf/cr # The original size of tabs in the input -input_tab_size = 4 # number +input_tab_size = 4 # number # The size of tabs in the output (only used if align_with_tabs=true) -output_tab_size = 4 # number +output_tab_size = 4 # number # The ASCII value of the string escape char, usually 92 (\) or 94 (^). (Pawn) -string_escape_char = 92 # number +string_escape_char = 92 # number # Alternate string escape char for Pawn. Only works right before the quote char. -string_escape_char2 = 0 # number +string_escape_char2 = 0 # number # Allow interpreting '>=' and '>>=' as part of a template in 'void f(list>=val);'. # If true (default), 'assert(x<0 && y>=3)' will be broken. # Improvements to template detection may make this option obsolete. -tok_split_gte = false # false/true +tok_split_gte = false # false/true # Control what to do with the UTF-8 BOM (recommend 'remove') -utf8_bom = remove # ignore/add/remove/force +utf8_bom = remove # ignore/add/remove/force # If the file contains bytes with values between 128 and 255, but is not UTF-8, then output as UTF-8 -utf8_byte = true # false/true +utf8_byte = true # false/true # Force the output encoding to UTF-8 -utf8_force = false # false/true +utf8_force = false # false/true # # Indenting @@ -39,1615 +39,1615 @@ utf8_force = false # false/true # The number of columns to indent per level. # Usually 2, 3, 4, or 8. -indent_columns = 4 # number +indent_columns = 4 # number # The continuation indent. If non-zero, this overrides the indent of '(' and '=' continuation indents. # For FreeBSD, this is set to 4. Negative value is absolute and not increased for each ( level -indent_continue = 0 # number +indent_continue = 0 # number # How to use tabs when indenting code # 0=spaces only # 1=indent with tabs to brace level, align with spaces # 2=indent and align with tabs, using spaces when not on a tabstop -indent_with_tabs = 2 # number +indent_with_tabs = 2 # number # Comments that are not a brace level are indented with tabs on a tabstop. # Requires indent_with_tabs=2. If false, will use spaces. -indent_cmt_with_tabs = false # false/true +indent_cmt_with_tabs = false # false/true # Whether to indent strings broken by '\' so that they line up -indent_align_string = true # false/true +indent_align_string = true # false/true # The number of spaces to indent multi-line XML strings. # Requires indent_align_string=True -indent_xml_string = 0 # number +indent_xml_string = 0 # number # Spaces to indent '{' from level -indent_brace = 0 # number +indent_brace = 0 # number # Whether braces are indented to the body level -indent_braces = false # false/true +indent_braces = false # false/true # Disabled indenting function braces if indent_braces is true -indent_braces_no_func = false # false/true +indent_braces_no_func = false # false/true # Disabled indenting class braces if indent_braces is true -indent_braces_no_class = false # false/true +indent_braces_no_class = false # false/true # Disabled indenting struct braces if indent_braces is true -indent_braces_no_struct = false # false/true +indent_braces_no_struct = false # false/true # Indent based on the size of the brace parent, i.e. 'if' => 3 spaces, 'for' => 4 spaces, etc. -indent_brace_parent = false # false/true +indent_brace_parent = false # false/true # Indent based on the paren open instead of the brace open in '({\n', default is to indent by brace. -indent_paren_open_brace = false # false/true +indent_paren_open_brace = false # false/true # Whether the 'namespace' body is indented -indent_namespace = false # false/true +indent_namespace = false # false/true # Only indent one namespace and no sub-namepaces. # Requires indent_namespace=true. -indent_namespace_single_indent = false # false/true +indent_namespace_single_indent = false # false/true # The number of spaces to indent a namespace block -indent_namespace_level = 0 # number +indent_namespace_level = 0 # number # If the body of the namespace is longer than this number, it won't be indented. # Requires indent_namespace=true. Default=0 (no limit) -indent_namespace_limit = 0 # number +indent_namespace_limit = 0 # number # Whether the 'extern "C"' body is indented -indent_extern = false # false/true +indent_extern = false # false/true # Whether the 'class' body is indented -indent_class = false # false/true +indent_class = false # false/true # Whether to indent the stuff after a leading base class colon -indent_class_colon = false # false/true +indent_class_colon = false # false/true # Whether to indent the stuff after a leading class initializer colon -indent_constr_colon = false # false/true +indent_constr_colon = false # false/true # Virtual indent from the ':' for member initializers. Default is 2 -indent_ctor_init_leading = 2 # number +indent_ctor_init_leading = 2 # number # Additional indenting for constructor initializer list -indent_ctor_init = 0 # number +indent_ctor_init = 0 # number # False=treat 'else\nif' as 'else if' for indenting purposes # True=indent the 'if' one level -indent_else_if = false # false/true +indent_else_if = false # false/true # Amount to indent variable declarations after a open brace. neg=relative, pos=absolute -indent_var_def_blk = 0 # number +indent_var_def_blk = 0 # number # Indent continued variable declarations instead of aligning. -indent_var_def_cont = false # false/true +indent_var_def_cont = false # false/true # True: force indentation of function definition to start in column 1 # False: use the default behavior -indent_func_def_force_col1 = false # false/true +indent_func_def_force_col1 = false # false/true # True: indent continued function call parameters one indent level # False: align parameters under the open paren -indent_func_call_param = false # false/true +indent_func_call_param = false # false/true # Same as indent_func_call_param, but for function defs -indent_func_def_param = false # false/true +indent_func_def_param = false # false/true # Same as indent_func_call_param, but for function protos -indent_func_proto_param = false # false/true +indent_func_proto_param = false # false/true # Same as indent_func_call_param, but for class declarations -indent_func_class_param = false # false/true +indent_func_class_param = false # false/true # Same as indent_func_call_param, but for class variable constructors -indent_func_ctor_var_param = false # false/true +indent_func_ctor_var_param = false # false/true # Same as indent_func_call_param, but for templates -indent_template_param = false # false/true +indent_template_param = false # false/true # Double the indent for indent_func_xxx_param options -indent_func_param_double = false # false/true +indent_func_param_double = false # false/true # Indentation column for standalone 'const' function decl/proto qualifier -indent_func_const = 0 # number +indent_func_const = 0 # number # Indentation column for standalone 'throw' function decl/proto qualifier -indent_func_throw = 0 # number +indent_func_throw = 0 # number # The number of spaces to indent a continued '->' or '.' # Usually set to 0, 1, or indent_columns. -indent_member = 0 # number +indent_member = 0 # number # Spaces to indent single line ('//') comments on lines before code -indent_sing_line_comments = 0 # number +indent_sing_line_comments = 0 # number # If set, will indent trailing single line ('//') comments relative # to the code instead of trying to keep the same absolute column -indent_relative_single_line_comments = false # false/true +indent_relative_single_line_comments = false # false/true # Spaces to indent 'case' from 'switch' # Usually 0 or indent_columns. -indent_switch_case = indent_columns # number +indent_switch_case = indent_columns # number # Spaces to shift the 'case' line, without affecting any other lines # Usually 0. -indent_case_shift = 0 # number +indent_case_shift = 0 # number # Spaces to indent '{' from 'case'. # By default, the brace will appear under the 'c' in case. # Usually set to 0 or indent_columns. -indent_case_brace = 0 # number +indent_case_brace = 0 # number # Whether to indent comments found in first column -indent_col1_comment = false # false/true +indent_col1_comment = false # false/true # How to indent goto labels # >0 : absolute column where 1 is the leftmost column # <=0 : subtract from brace indent -indent_label = 1 # number +indent_label = 1 # number # Same as indent_label, but for access specifiers that are followed by a colon -indent_access_spec = 1 # number +indent_access_spec = 1 # number # Indent the code after an access specifier by one level. # If set, this option forces 'indent_access_spec=0' -indent_access_spec_body = false # false/true +indent_access_spec_body = false # false/true # If an open paren is followed by a newline, indent the next line so that it lines up after the open paren (not recommended) -indent_paren_nl = false # false/true +indent_paren_nl = false # false/true # Controls the indent of a close paren after a newline. # 0: Indent to body level # 1: Align under the open paren # 2: Indent to the brace level -indent_paren_close = 0 # number +indent_paren_close = 0 # number # Controls the indent of a comma when inside a paren.If TRUE, aligns under the open paren -indent_comma_paren = false # false/true +indent_comma_paren = false # false/true # Controls the indent of a BOOL operator when inside a paren.If TRUE, aligns under the open paren -indent_bool_paren = false # false/true +indent_bool_paren = false # false/true # If 'indent_bool_paren' is true, controls the indent of the first expression. If TRUE, aligns the first expression to the following ones -indent_first_bool_expr = false # false/true +indent_first_bool_expr = false # false/true # If an open square is followed by a newline, indent the next line so that it lines up after the open square (not recommended) -indent_square_nl = false # false/true +indent_square_nl = false # false/true # Don't change the relative indent of ESQL/C 'EXEC SQL' bodies -indent_preserve_sql = false # false/true +indent_preserve_sql = false # false/true # Align continued statements at the '='. Default=True # If FALSE or the '=' is followed by a newline, the next line is indent one tab. -indent_align_assign = true # false/true +indent_align_assign = true # false/true # Indent OC blocks at brace level instead of usual rules. -indent_oc_block = false # false/true +indent_oc_block = false # false/true # Indent OC blocks in a message relative to the parameter name. # 0=use indent_oc_block rules, 1+=spaces to indent -indent_oc_block_msg = 0 # number +indent_oc_block_msg = 0 # number # Minimum indent for subsequent parameters -indent_oc_msg_colon = 0 # number +indent_oc_msg_colon = 0 # number # If true, prioritize aligning with initial colon (and stripping spaces from lines, if necessary). # Default is true. -indent_oc_msg_prioritize_first_colon = true # false/true +indent_oc_msg_prioritize_first_colon = true # false/true # If indent_oc_block_msg and this option are on, blocks will be indented the way that Xcode does by default (from keyword if the parameter is on its own line; otherwise, from the previous indentation level). -indent_oc_block_msg_xcode_style = false # false/true +indent_oc_block_msg_xcode_style = false # false/true # If indent_oc_block_msg and this option are on, blocks will be indented from where the brace is relative to a msg keyword. -indent_oc_block_msg_from_keyword = false # false/true +indent_oc_block_msg_from_keyword = false # false/true # If indent_oc_block_msg and this option are on, blocks will be indented from where the brace is relative to a msg colon. -indent_oc_block_msg_from_colon = false # false/true +indent_oc_block_msg_from_colon = false # false/true # If indent_oc_block_msg and this option are on, blocks will be indented from where the block caret is. -indent_oc_block_msg_from_caret = false # false/true +indent_oc_block_msg_from_caret = false # false/true # If indent_oc_block_msg and this option are on, blocks will be indented from where the brace is. -indent_oc_block_msg_from_brace = false # false/true +indent_oc_block_msg_from_brace = false # false/true # # Spacing options # # Add or remove space around arithmetic operator '+', '-', '/', '*', etc -sp_arith = force # ignore/add/remove/force +sp_arith = force # ignore/add/remove/force # Add or remove space around assignment operator '=', '+=', etc -sp_assign = force # ignore/add/remove/force +sp_assign = force # ignore/add/remove/force # Add or remove space around '=' in C++11 lambda capture specifications. Overrides sp_assign -sp_cpp_lambda_assign = ignore # ignore/add/remove/force +sp_cpp_lambda_assign = ignore # ignore/add/remove/force # Add or remove space after the capture specification in C++11 lambda. -sp_cpp_lambda_paren = ignore # ignore/add/remove/force +sp_cpp_lambda_paren = ignore # ignore/add/remove/force # Add or remove space around assignment operator '=' in a prototype -sp_assign_default = ignore # ignore/add/remove/force +sp_assign_default = ignore # ignore/add/remove/force # Add or remove space before assignment operator '=', '+=', etc. Overrides sp_assign. -sp_before_assign = ignore # ignore/add/remove/force +sp_before_assign = ignore # ignore/add/remove/force # Add or remove space after assignment operator '=', '+=', etc. Overrides sp_assign. -sp_after_assign = ignore # ignore/add/remove/force +sp_after_assign = ignore # ignore/add/remove/force # Add or remove space in 'NS_ENUM (' -sp_enum_paren = ignore # ignore/add/remove/force +sp_enum_paren = ignore # ignore/add/remove/force # Add or remove space around assignment '=' in enum -sp_enum_assign = force # ignore/add/remove/force +sp_enum_assign = force # ignore/add/remove/force # Add or remove space before assignment '=' in enum. Overrides sp_enum_assign. -sp_enum_before_assign = ignore # ignore/add/remove/force +sp_enum_before_assign = ignore # ignore/add/remove/force # Add or remove space after assignment '=' in enum. Overrides sp_enum_assign. -sp_enum_after_assign = ignore # ignore/add/remove/force +sp_enum_after_assign = ignore # ignore/add/remove/force # Add or remove space around preprocessor '##' concatenation operator. Default=Add -sp_pp_concat = force # ignore/add/remove/force +sp_pp_concat = force # ignore/add/remove/force # Add or remove space after preprocessor '#' stringify operator. Also affects the '#@' charizing operator. -sp_pp_stringify = force # ignore/add/remove/force +sp_pp_stringify = force # ignore/add/remove/force # Add or remove space before preprocessor '#' stringify operator as in '#define x(y) L#y'. -sp_before_pp_stringify = ignore # ignore/add/remove/force +sp_before_pp_stringify = ignore # ignore/add/remove/force # Add or remove space around boolean operators '&&' and '||' -sp_bool = force # ignore/add/remove/force +sp_bool = force # ignore/add/remove/force # Add or remove space around compare operator '<', '>', '==', etc -sp_compare = force # ignore/add/remove/force +sp_compare = force # ignore/add/remove/force # Add or remove space inside '(' and ')' -sp_inside_paren = remove # ignore/add/remove/force +sp_inside_paren = remove # ignore/add/remove/force # Add or remove space between nested parens: '((' vs ') )' -sp_paren_paren = remove # ignore/add/remove/force +sp_paren_paren = remove # ignore/add/remove/force # Add or remove space between back-to-back parens: ')(' vs ') (' -sp_cparen_oparen = ignore # ignore/add/remove/force +sp_cparen_oparen = ignore # ignore/add/remove/force # Whether to balance spaces inside nested parens -sp_balance_nested_parens = false # false/true +sp_balance_nested_parens = false # false/true # Add or remove space between ')' and '{' -sp_paren_brace = force # ignore/add/remove/force +sp_paren_brace = force # ignore/add/remove/force # Add or remove space before pointer star '*' -sp_before_ptr_star = force # ignore/add/remove/force +sp_before_ptr_star = force # ignore/add/remove/force # Add or remove space before pointer star '*' that isn't followed by a variable name # If set to 'ignore', sp_before_ptr_star is used instead. -sp_before_unnamed_ptr_star = ignore # ignore/add/remove/force +sp_before_unnamed_ptr_star = ignore # ignore/add/remove/force # Add or remove space between pointer stars '*' -sp_between_ptr_star = remove # ignore/add/remove/force +sp_between_ptr_star = remove # ignore/add/remove/force # Add or remove space after pointer star '*', if followed by a word. -sp_after_ptr_star = remove # ignore/add/remove/force +sp_after_ptr_star = remove # ignore/add/remove/force # Add or remove space after pointer star '*', if followed by a qualifier. -sp_after_ptr_star_qualifier = ignore # ignore/add/remove/force +sp_after_ptr_star_qualifier = ignore # ignore/add/remove/force # Add or remove space after a pointer star '*', if followed by a func proto/def. -sp_after_ptr_star_func = add # ignore/add/remove/force +sp_after_ptr_star_func = add # ignore/add/remove/force # Add or remove space after a pointer star '*', if followed by an open paren (function types). -sp_ptr_star_paren = remove # ignore/add/remove/force +sp_ptr_star_paren = remove # ignore/add/remove/force # Add or remove space before a pointer star '*', if followed by a func proto/def. -sp_before_ptr_star_func = ignore # ignore/add/remove/force +sp_before_ptr_star_func = ignore # ignore/add/remove/force # Add or remove space before a reference sign '&' -sp_before_byref = ignore # ignore/add/remove/force +sp_before_byref = ignore # ignore/add/remove/force # Add or remove space before a reference sign '&' that isn't followed by a variable name # If set to 'ignore', sp_before_byref is used instead. -sp_before_unnamed_byref = ignore # ignore/add/remove/force +sp_before_unnamed_byref = ignore # ignore/add/remove/force # Add or remove space after reference sign '&', if followed by a word. -sp_after_byref = remove # ignore/add/remove/force +sp_after_byref = remove # ignore/add/remove/force # Add or remove space after a reference sign '&', if followed by a func proto/def. -sp_after_byref_func = remove # ignore/add/remove/force +sp_after_byref_func = remove # ignore/add/remove/force # Add or remove space before a reference sign '&', if followed by a func proto/def. -sp_before_byref_func = ignore # ignore/add/remove/force +sp_before_byref_func = ignore # ignore/add/remove/force # Add or remove space between type and word. Default=Force -sp_after_type = force # ignore/add/remove/force +sp_after_type = force # ignore/add/remove/force # Add or remove space before the paren in the D constructs 'template Foo(' and 'class Foo('. -sp_before_template_paren = ignore # ignore/add/remove/force +sp_before_template_paren = ignore # ignore/add/remove/force # Add or remove space in 'template <' vs 'template<'. # If set to ignore, sp_before_angle is used. -sp_template_angle = ignore # ignore/add/remove/force +sp_template_angle = ignore # ignore/add/remove/force # Add or remove space before '<>' -sp_before_angle = ignore # ignore/add/remove/force +sp_before_angle = ignore # ignore/add/remove/force # Add or remove space inside '<' and '>' -sp_inside_angle = ignore # ignore/add/remove/force +sp_inside_angle = ignore # ignore/add/remove/force # Add or remove space after '<>' -sp_after_angle = ignore # ignore/add/remove/force +sp_after_angle = ignore # ignore/add/remove/force # Add or remove space between '<>' and '(' as found in 'new List();' -sp_angle_paren = ignore # ignore/add/remove/force +sp_angle_paren = ignore # ignore/add/remove/force # Add or remove space between '<>' and a word as in 'List m;' -sp_angle_word = ignore # ignore/add/remove/force +sp_angle_word = ignore # ignore/add/remove/force # Add or remove space between '>' and '>' in '>>' (template stuff C++/C# only). Default=Add -sp_angle_shift = add # ignore/add/remove/force +sp_angle_shift = add # ignore/add/remove/force # Permit removal of the space between '>>' in 'foo >' (C++11 only). Default=False # sp_angle_shift cannot remove the space without this option. -sp_permit_cpp11_shift = false # false/true +sp_permit_cpp11_shift = false # false/true # Add or remove space before '(' of 'if', 'for', 'switch', and 'while' -sp_before_sparen = force # ignore/add/remove/force +sp_before_sparen = force # ignore/add/remove/force # Add or remove space inside if-condition '(' and ')' -sp_inside_sparen = remove # ignore/add/remove/force +sp_inside_sparen = remove # ignore/add/remove/force # Add or remove space before if-condition ')'. Overrides sp_inside_sparen. -sp_inside_sparen_close = ignore # ignore/add/remove/force +sp_inside_sparen_close = ignore # ignore/add/remove/force # Add or remove space before if-condition '('. Overrides sp_inside_sparen. -sp_inside_sparen_open = ignore # ignore/add/remove/force +sp_inside_sparen_open = ignore # ignore/add/remove/force # Add or remove space after ')' of 'if', 'for', 'switch', and 'while' -sp_after_sparen = ignore # ignore/add/remove/force +sp_after_sparen = ignore # ignore/add/remove/force # Add or remove space between ')' and '{' of 'if', 'for', 'switch', and 'while' -sp_sparen_brace = ignore # ignore/add/remove/force +sp_sparen_brace = ignore # ignore/add/remove/force # Add or remove space between 'invariant' and '(' in the D language. -sp_invariant_paren = ignore # ignore/add/remove/force +sp_invariant_paren = ignore # ignore/add/remove/force # Add or remove space after the ')' in 'invariant (C) c' in the D language. -sp_after_invariant_paren = ignore # ignore/add/remove/force +sp_after_invariant_paren = ignore # ignore/add/remove/force # Add or remove space before empty statement ';' on 'if', 'for' and 'while' -sp_special_semi = ignore # ignore/add/remove/force +sp_special_semi = ignore # ignore/add/remove/force # Add or remove space before ';'. Default=Remove -sp_before_semi = remove # ignore/add/remove/force +sp_before_semi = remove # ignore/add/remove/force # Add or remove space before ';' in non-empty 'for' statements -sp_before_semi_for = ignore # ignore/add/remove/force +sp_before_semi_for = ignore # ignore/add/remove/force # Add or remove space before a semicolon of an empty part of a for statement. -sp_before_semi_for_empty = ignore # ignore/add/remove/force +sp_before_semi_for_empty = ignore # ignore/add/remove/force # Add or remove space after ';', except when followed by a comment. Default=Add -sp_after_semi = add # ignore/add/remove/force +sp_after_semi = add # ignore/add/remove/force # Add or remove space after ';' in non-empty 'for' statements. Default=Force -sp_after_semi_for = force # ignore/add/remove/force +sp_after_semi_for = force # ignore/add/remove/force # Add or remove space after the final semicolon of an empty part of a for statement: for ( ; ; ). -sp_after_semi_for_empty = remove # ignore/add/remove/force +sp_after_semi_for_empty = remove # ignore/add/remove/force # Add or remove space before '[' (except '[]') -sp_before_square = ignore # ignore/add/remove/force +sp_before_square = ignore # ignore/add/remove/force # Add or remove space before '[]' -sp_before_squares = ignore # ignore/add/remove/force +sp_before_squares = ignore # ignore/add/remove/force # Add or remove space inside a non-empty '[' and ']' -sp_inside_square = remove # ignore/add/remove/force +sp_inside_square = remove # ignore/add/remove/force # Add or remove space after ',' -sp_after_comma = force # ignore/add/remove/force +sp_after_comma = force # ignore/add/remove/force # Add or remove space before ',' -sp_before_comma = remove # ignore/add/remove/force +sp_before_comma = remove # ignore/add/remove/force # Add or remove space between an open paren and comma: '(,' vs '( ,' -sp_paren_comma = force # ignore/add/remove/force +sp_paren_comma = force # ignore/add/remove/force # Add or remove space before the variadic '...' when preceded by a non-punctuator -sp_before_ellipsis = ignore # ignore/add/remove/force +sp_before_ellipsis = ignore # ignore/add/remove/force # Add or remove space after class ':' -sp_after_class_colon = ignore # ignore/add/remove/force +sp_after_class_colon = ignore # ignore/add/remove/force # Add or remove space before class ':' -sp_before_class_colon = ignore # ignore/add/remove/force +sp_before_class_colon = ignore # ignore/add/remove/force # Add or remove space after class constructor ':' -sp_after_constr_colon = ignore # ignore/add/remove/force +sp_after_constr_colon = ignore # ignore/add/remove/force # Add or remove space before class constructor ':' -sp_before_constr_colon = ignore # ignore/add/remove/force +sp_before_constr_colon = ignore # ignore/add/remove/force # Add or remove space before case ':'. Default=Remove -sp_before_case_colon = remove # ignore/add/remove/force +sp_before_case_colon = remove # ignore/add/remove/force # Add or remove space between 'operator' and operator sign -sp_after_operator = ignore # ignore/add/remove/force +sp_after_operator = ignore # ignore/add/remove/force # Add or remove space between the operator symbol and the open paren, as in 'operator ++(' -sp_after_operator_sym = ignore # ignore/add/remove/force +sp_after_operator_sym = ignore # ignore/add/remove/force # Add or remove space after C/D cast, i.e. 'cast(int)a' vs 'cast(int) a' or '(int)a' vs '(int) a' -sp_after_cast = force # ignore/add/remove/force +sp_after_cast = force # ignore/add/remove/force # Add or remove spaces inside cast parens -sp_inside_paren_cast = remove # ignore/add/remove/force +sp_inside_paren_cast = remove # ignore/add/remove/force # Add or remove space between the type and open paren in a C++ cast, i.e. 'int(exp)' vs 'int (exp)' -sp_cpp_cast_paren = ignore # ignore/add/remove/force +sp_cpp_cast_paren = ignore # ignore/add/remove/force # Add or remove space between 'sizeof' and '(' -sp_sizeof_paren = remove # ignore/add/remove/force +sp_sizeof_paren = remove # ignore/add/remove/force # Add or remove space after the tag keyword (Pawn) -sp_after_tag = ignore # ignore/add/remove/force +sp_after_tag = ignore # ignore/add/remove/force # Add or remove space inside enum '{' and '}' -sp_inside_braces_enum = ignore # ignore/add/remove/force +sp_inside_braces_enum = ignore # ignore/add/remove/force # Add or remove space inside struct/union '{' and '}' -sp_inside_braces_struct = ignore # ignore/add/remove/force +sp_inside_braces_struct = ignore # ignore/add/remove/force # Add or remove space inside '{' and '}' -sp_inside_braces = force # ignore/add/remove/force +sp_inside_braces = force # ignore/add/remove/force # Add or remove space inside '{}' -sp_inside_braces_empty = force # ignore/add/remove/force +sp_inside_braces_empty = force # ignore/add/remove/force # Add or remove space between return type and function name # A minimum of 1 is forced except for pointer return types. -sp_type_func = force # ignore/add/remove/force +sp_type_func = force # ignore/add/remove/force # Add or remove space between function name and '(' on function declaration -sp_func_proto_paren = remove # ignore/add/remove/force +sp_func_proto_paren = remove # ignore/add/remove/force # Add or remove space between function name and '(' on function definition -sp_func_def_paren = remove # ignore/add/remove/force +sp_func_def_paren = remove # ignore/add/remove/force # Add or remove space inside empty function '()' -sp_inside_fparens = remove # ignore/add/remove/force +sp_inside_fparens = remove # ignore/add/remove/force # Add or remove space inside function '(' and ')' -sp_inside_fparen = remove # ignore/add/remove/force +sp_inside_fparen = remove # ignore/add/remove/force # Add or remove space inside the first parens in the function type: 'void (*x)(...)' -sp_inside_tparen = remove # ignore/add/remove/force +sp_inside_tparen = remove # ignore/add/remove/force # Add or remove between the parens in the function type: 'void (*x)(...)' -sp_after_tparen_close = ignore # ignore/add/remove/force +sp_after_tparen_close = ignore # ignore/add/remove/force # Add or remove space between ']' and '(' when part of a function call. -sp_square_fparen = ignore # ignore/add/remove/force +sp_square_fparen = ignore # ignore/add/remove/force # Add or remove space between ')' and '{' of function -sp_fparen_brace = ignore # ignore/add/remove/force +sp_fparen_brace = ignore # ignore/add/remove/force # Java: Add or remove space between ')' and '{{' of double brace initializer. -sp_fparen_dbrace = ignore # ignore/add/remove/force +sp_fparen_dbrace = ignore # ignore/add/remove/force # Add or remove space between function name and '(' on function calls -sp_func_call_paren = remove # ignore/add/remove/force +sp_func_call_paren = remove # ignore/add/remove/force # Add or remove space between function name and '()' on function calls without parameters. # If set to 'ignore' (the default), sp_func_call_paren is used. -sp_func_call_paren_empty = ignore # ignore/add/remove/force +sp_func_call_paren_empty = ignore # ignore/add/remove/force # Add or remove space between the user function name and '(' on function calls # You need to set a keyword to be a user function, like this: 'set func_call_user _' in the config file. -sp_func_call_user_paren = ignore # ignore/add/remove/force +sp_func_call_user_paren = ignore # ignore/add/remove/force # Add or remove space between a constructor/destructor and the open paren -sp_func_class_paren = ignore # ignore/add/remove/force +sp_func_class_paren = ignore # ignore/add/remove/force # Add or remove space between 'return' and '(' -sp_return_paren = ignore # ignore/add/remove/force +sp_return_paren = ignore # ignore/add/remove/force # Add or remove space between '__attribute__' and '(' -sp_attribute_paren = remove # ignore/add/remove/force +sp_attribute_paren = remove # ignore/add/remove/force # Add or remove space between 'defined' and '(' in '#if defined (FOO)' -sp_defined_paren = ignore # ignore/add/remove/force +sp_defined_paren = ignore # ignore/add/remove/force # Add or remove space between 'throw' and '(' in 'throw (something)' -sp_throw_paren = ignore # ignore/add/remove/force +sp_throw_paren = ignore # ignore/add/remove/force # Add or remove space between 'throw' and anything other than '(' as in '@throw [...];' -sp_after_throw = ignore # ignore/add/remove/force +sp_after_throw = ignore # ignore/add/remove/force # Add or remove space between 'catch' and '(' in 'catch (something) { }' # If set to ignore, sp_before_sparen is used. -sp_catch_paren = ignore # ignore/add/remove/force +sp_catch_paren = ignore # ignore/add/remove/force # Add or remove space between 'version' and '(' in 'version (something) { }' (D language) # If set to ignore, sp_before_sparen is used. -sp_version_paren = ignore # ignore/add/remove/force +sp_version_paren = ignore # ignore/add/remove/force # Add or remove space between 'scope' and '(' in 'scope (something) { }' (D language) # If set to ignore, sp_before_sparen is used. -sp_scope_paren = ignore # ignore/add/remove/force +sp_scope_paren = ignore # ignore/add/remove/force # Add or remove space between macro and value -sp_macro = force # ignore/add/remove/force +sp_macro = force # ignore/add/remove/force # Add or remove space between macro function ')' and value -sp_macro_func = force # ignore/add/remove/force +sp_macro_func = force # ignore/add/remove/force # Add or remove space between 'else' and '{' if on the same line -sp_else_brace = ignore # ignore/add/remove/force +sp_else_brace = ignore # ignore/add/remove/force # Add or remove space between '}' and 'else' if on the same line -sp_brace_else = ignore # ignore/add/remove/force +sp_brace_else = ignore # ignore/add/remove/force # Add or remove space between '}' and the name of a typedef on the same line -sp_brace_typedef = ignore # ignore/add/remove/force +sp_brace_typedef = ignore # ignore/add/remove/force # Add or remove space between 'catch' and '{' if on the same line -sp_catch_brace = ignore # ignore/add/remove/force +sp_catch_brace = ignore # ignore/add/remove/force # Add or remove space between '}' and 'catch' if on the same line -sp_brace_catch = ignore # ignore/add/remove/force +sp_brace_catch = ignore # ignore/add/remove/force # Add or remove space between 'finally' and '{' if on the same line -sp_finally_brace = ignore # ignore/add/remove/force +sp_finally_brace = ignore # ignore/add/remove/force # Add or remove space between '}' and 'finally' if on the same line -sp_brace_finally = ignore # ignore/add/remove/force +sp_brace_finally = ignore # ignore/add/remove/force # Add or remove space between 'try' and '{' if on the same line -sp_try_brace = ignore # ignore/add/remove/force +sp_try_brace = ignore # ignore/add/remove/force # Add or remove space between get/set and '{' if on the same line -sp_getset_brace = ignore # ignore/add/remove/force +sp_getset_brace = ignore # ignore/add/remove/force # Add or remove space between a variable and '{' for C++ uniform initialization -sp_word_brace = add # ignore/add/remove/force +sp_word_brace = add # ignore/add/remove/force # Add or remove space between a variable and '{' for a namespace -sp_word_brace_ns = add # ignore/add/remove/force +sp_word_brace_ns = add # ignore/add/remove/force # Add or remove space before the '::' operator -sp_before_dc = ignore # ignore/add/remove/force +sp_before_dc = ignore # ignore/add/remove/force # Add or remove space after the '::' operator -sp_after_dc = ignore # ignore/add/remove/force +sp_after_dc = ignore # ignore/add/remove/force # Add or remove around the D named array initializer ':' operator -sp_d_array_colon = ignore # ignore/add/remove/force +sp_d_array_colon = ignore # ignore/add/remove/force # Add or remove space after the '!' (not) operator. Default=Remove -sp_not = remove # ignore/add/remove/force +sp_not = remove # ignore/add/remove/force # Add or remove space after the '~' (invert) operator. Default=Remove -sp_inv = remove # ignore/add/remove/force +sp_inv = remove # ignore/add/remove/force # Add or remove space after the '&' (address-of) operator. Default=Remove # This does not affect the spacing after a '&' that is part of a type. -sp_addr = remove # ignore/add/remove/force +sp_addr = remove # ignore/add/remove/force # Add or remove space around the '.' or '->' operators. Default=Remove -sp_member = remove # ignore/add/remove/force +sp_member = remove # ignore/add/remove/force # Add or remove space after the '*' (dereference) operator. Default=Remove # This does not affect the spacing after a '*' that is part of a type. -sp_deref = remove # ignore/add/remove/force +sp_deref = remove # ignore/add/remove/force # Add or remove space after '+' or '-', as in 'x = -5' or 'y = +7'. Default=Remove -sp_sign = remove # ignore/add/remove/force +sp_sign = remove # ignore/add/remove/force # Add or remove space before or after '++' and '--', as in '(--x)' or 'y++;'. Default=Remove -sp_incdec = remove # ignore/add/remove/force +sp_incdec = remove # ignore/add/remove/force # Add or remove space before a backslash-newline at the end of a line. Default=Add -sp_before_nl_cont = force # ignore/add/remove/force +sp_before_nl_cont = force # ignore/add/remove/force # Add or remove space after the scope '+' or '-', as in '-(void) foo;' or '+(int) bar;' -sp_after_oc_scope = ignore # ignore/add/remove/force +sp_after_oc_scope = ignore # ignore/add/remove/force # Add or remove space after the colon in message specs # '-(int) f:(int) x;' vs '-(int) f: (int) x;' -sp_after_oc_colon = ignore # ignore/add/remove/force +sp_after_oc_colon = ignore # ignore/add/remove/force # Add or remove space before the colon in message specs # '-(int) f: (int) x;' vs '-(int) f : (int) x;' -sp_before_oc_colon = ignore # ignore/add/remove/force +sp_before_oc_colon = ignore # ignore/add/remove/force # Add or remove space after the colon in immutable dictionary expression # 'NSDictionary *test = @{@"foo" :@"bar"};' -sp_after_oc_dict_colon = ignore # ignore/add/remove/force +sp_after_oc_dict_colon = ignore # ignore/add/remove/force # Add or remove space before the colon in immutable dictionary expression # 'NSDictionary *test = @{@"foo" :@"bar"};' -sp_before_oc_dict_colon = ignore # ignore/add/remove/force +sp_before_oc_dict_colon = ignore # ignore/add/remove/force # Add or remove space after the colon in message specs # '[object setValue:1];' vs '[object setValue: 1];' -sp_after_send_oc_colon = ignore # ignore/add/remove/force +sp_after_send_oc_colon = ignore # ignore/add/remove/force # Add or remove space before the colon in message specs # '[object setValue:1];' vs '[object setValue :1];' -sp_before_send_oc_colon = ignore # ignore/add/remove/force +sp_before_send_oc_colon = ignore # ignore/add/remove/force # Add or remove space after the (type) in message specs # '-(int)f: (int) x;' vs '-(int)f: (int)x;' -sp_after_oc_type = ignore # ignore/add/remove/force +sp_after_oc_type = ignore # ignore/add/remove/force # Add or remove space after the first (type) in message specs # '-(int) f:(int)x;' vs '-(int)f:(int)x;' -sp_after_oc_return_type = ignore # ignore/add/remove/force +sp_after_oc_return_type = ignore # ignore/add/remove/force # Add or remove space between '@selector' and '(' # '@selector(msgName)' vs '@selector (msgName)' # Also applies to @protocol() constructs -sp_after_oc_at_sel = ignore # ignore/add/remove/force +sp_after_oc_at_sel = ignore # ignore/add/remove/force # Add or remove space between '@selector(x)' and the following word # '@selector(foo) a:' vs '@selector(foo)a:' -sp_after_oc_at_sel_parens = ignore # ignore/add/remove/force +sp_after_oc_at_sel_parens = ignore # ignore/add/remove/force # Add or remove space inside '@selector' parens # '@selector(foo)' vs '@selector( foo )' # Also applies to @protocol() constructs -sp_inside_oc_at_sel_parens = ignore # ignore/add/remove/force +sp_inside_oc_at_sel_parens = ignore # ignore/add/remove/force # Add or remove space before a block pointer caret # '^int (int arg){...}' vs. ' ^int (int arg){...}' -sp_before_oc_block_caret = ignore # ignore/add/remove/force +sp_before_oc_block_caret = ignore # ignore/add/remove/force # Add or remove space after a block pointer caret # '^int (int arg){...}' vs. '^ int (int arg){...}' -sp_after_oc_block_caret = ignore # ignore/add/remove/force +sp_after_oc_block_caret = ignore # ignore/add/remove/force # Add or remove space between the receiver and selector in a message. # '[receiver selector ...]' -sp_after_oc_msg_receiver = ignore # ignore/add/remove/force +sp_after_oc_msg_receiver = ignore # ignore/add/remove/force # Add or remove space after @property. -sp_after_oc_property = ignore # ignore/add/remove/force +sp_after_oc_property = ignore # ignore/add/remove/force # Add or remove space around the ':' in 'b ? t : f' -sp_cond_colon = force # ignore/add/remove/force +sp_cond_colon = force # ignore/add/remove/force # Add or remove space before the ':' in 'b ? t : f'. Overrides sp_cond_colon. -sp_cond_colon_before = ignore # ignore/add/remove/force +sp_cond_colon_before = ignore # ignore/add/remove/force # Add or remove space after the ':' in 'b ? t : f'. Overrides sp_cond_colon. -sp_cond_colon_after = ignore # ignore/add/remove/force +sp_cond_colon_after = ignore # ignore/add/remove/force # Add or remove space around the '?' in 'b ? t : f' -sp_cond_question = force # ignore/add/remove/force +sp_cond_question = force # ignore/add/remove/force # Add or remove space before the '?' in 'b ? t : f'. Overrides sp_cond_question. -sp_cond_question_before = ignore # ignore/add/remove/force +sp_cond_question_before = ignore # ignore/add/remove/force # Add or remove space after the '?' in 'b ? t : f'. Overrides sp_cond_question. -sp_cond_question_after = ignore # ignore/add/remove/force +sp_cond_question_after = ignore # ignore/add/remove/force # In the abbreviated ternary form (a ?: b), add/remove space between ? and :.'. Overrides all other sp_cond_* options. -sp_cond_ternary_short = ignore # ignore/add/remove/force +sp_cond_ternary_short = ignore # ignore/add/remove/force # Fix the spacing between 'case' and the label. Only 'ignore' and 'force' make sense here. -sp_case_label = force # ignore/add/remove/force +sp_case_label = force # ignore/add/remove/force # Control the space around the D '..' operator. -sp_range = ignore # ignore/add/remove/force +sp_range = ignore # ignore/add/remove/force # Control the spacing after ':' in 'for (TYPE VAR : EXPR)' (Java) -sp_after_for_colon = ignore # ignore/add/remove/force +sp_after_for_colon = ignore # ignore/add/remove/force # Control the spacing before ':' in 'for (TYPE VAR : EXPR)' (Java) -sp_before_for_colon = ignore # ignore/add/remove/force +sp_before_for_colon = ignore # ignore/add/remove/force # Control the spacing in 'extern (C)' (D) -sp_extern_paren = ignore # ignore/add/remove/force +sp_extern_paren = ignore # ignore/add/remove/force # Control the space after the opening of a C++ comment '// A' vs '//A' -sp_cmt_cpp_start = ignore # ignore/add/remove/force +sp_cmt_cpp_start = ignore # ignore/add/remove/force # Controls the spaces between #else or #endif and a trailing comment -sp_endif_cmt = ignore # ignore/add/remove/force +sp_endif_cmt = ignore # ignore/add/remove/force # Controls the spaces after 'new', 'delete', and 'delete[]' -sp_after_new = ignore # ignore/add/remove/force +sp_after_new = ignore # ignore/add/remove/force # Controls the spaces before a trailing or embedded comment -sp_before_tr_emb_cmt = ignore # ignore/add/remove/force +sp_before_tr_emb_cmt = ignore # ignore/add/remove/force # Number of spaces before a trailing or embedded comment -sp_num_before_tr_emb_cmt = 0 # number +sp_num_before_tr_emb_cmt = 0 # number # Control space between a Java annotation and the open paren. -sp_annotation_paren = ignore # ignore/add/remove/force +sp_annotation_paren = ignore # ignore/add/remove/force # # Code alignment (not left column spaces/tabs) # # Whether to keep non-indenting tabs -align_keep_tabs = false # false/true +align_keep_tabs = false # false/true # Whether to use tabs for aligning -align_with_tabs = false # false/true +align_with_tabs = false # false/true # Whether to bump out to the next tab when aligning -align_on_tabstop = false # false/true +align_on_tabstop = false # false/true # Whether to right-align numbers -align_number_right = false # false/true +align_number_right = false # false/true # Whether to keep whitespace not required for alignment. -align_keep_extra_space = false # false/true +align_keep_extra_space = false # false/true # Align variable definitions in prototypes and functions -align_func_params = false # false/true +align_func_params = false # false/true # Align parameters in single-line functions that have the same name. # The function names must already be aligned with each other. -align_same_func_call_params = false # false/true +align_same_func_call_params = false # false/true # The span for aligning variable definitions (0=don't align) -align_var_def_span = 0 # number +align_var_def_span = 0 # number # How to align the star in variable definitions. # 0=Part of the type 'void * foo;' # 1=Part of the variable 'void *foo;' # 2=Dangling 'void *foo;' -align_var_def_star_style = 0 # number +align_var_def_star_style = 0 # number # How to align the '&' in variable definitions. # 0=Part of the type # 1=Part of the variable # 2=Dangling -align_var_def_amp_style = 0 # number +align_var_def_amp_style = 0 # number # The threshold for aligning variable definitions (0=no limit) -align_var_def_thresh = 0 # number +align_var_def_thresh = 0 # number # The gap for aligning variable definitions -align_var_def_gap = 0 # number +align_var_def_gap = 0 # number # Whether to align the colon in struct bit fields -align_var_def_colon = false # false/true +align_var_def_colon = false # false/true # Whether to align any attribute after the variable name -align_var_def_attribute = false # false/true +align_var_def_attribute = false # false/true # Whether to align inline struct/enum/union variable definitions -align_var_def_inline = false # false/true +align_var_def_inline = false # false/true # The span for aligning on '=' in assignments (0=don't align) -align_assign_span = 0 # number +align_assign_span = 0 # number # The threshold for aligning on '=' in assignments (0=no limit) -align_assign_thresh = 0 # number +align_assign_thresh = 0 # number # The span for aligning on '=' in enums (0=don't align) -align_enum_equ_span = 0 # number +align_enum_equ_span = 0 # number # The threshold for aligning on '=' in enums (0=no limit) -align_enum_equ_thresh = 0 # number +align_enum_equ_thresh = 0 # number # The span for aligning struct/union (0=don't align) -align_var_struct_span = 0 # number +align_var_struct_span = 0 # number # The threshold for aligning struct/union member definitions (0=no limit) -align_var_struct_thresh = 0 # number +align_var_struct_thresh = 0 # number # The gap for aligning struct/union member definitions -align_var_struct_gap = 0 # number +align_var_struct_gap = 0 # number # The span for aligning struct initializer values (0=don't align) -align_struct_init_span = 0 # number +align_struct_init_span = 0 # number # The minimum space between the type and the synonym of a typedef -align_typedef_gap = 0 # number +align_typedef_gap = 0 # number # The span for aligning single-line typedefs (0=don't align) -align_typedef_span = 0 # number +align_typedef_span = 0 # number # How to align typedef'd functions with other typedefs # 0: Don't mix them at all # 1: align the open paren with the types # 2: align the function type name with the other type names -align_typedef_func = 0 # number +align_typedef_func = 0 # number # Controls the positioning of the '*' in typedefs. Just try it. # 0: Align on typedef type, ignore '*' # 1: The '*' is part of type name: typedef int *pint; # 2: The '*' is part of the type, but dangling: typedef int *pint; -align_typedef_star_style = 0 # number +align_typedef_star_style = 0 # number # Controls the positioning of the '&' in typedefs. Just try it. # 0: Align on typedef type, ignore '&' # 1: The '&' is part of type name: typedef int &pint; # 2: The '&' is part of the type, but dangling: typedef int &pint; -align_typedef_amp_style = 0 # number +align_typedef_amp_style = 0 # number # The span for aligning comments that end lines (0=don't align) -align_right_cmt_span = 0 # number +align_right_cmt_span = 0 # number # If aligning comments, mix with comments after '}' and #endif with less than 3 spaces before the comment -align_right_cmt_mix = false # false/true +align_right_cmt_mix = false # false/true # If a trailing comment is more than this number of columns away from the text it follows, # it will qualify for being aligned. This has to be > 0 to do anything. -align_right_cmt_gap = 0 # number +align_right_cmt_gap = 0 # number # Align trailing comment at or beyond column N; 'pulls in' comments as a bonus side effect (0=ignore) -align_right_cmt_at_col = 0 # number +align_right_cmt_at_col = 0 # number # The span for aligning function prototypes (0=don't align) -align_func_proto_span = 0 # number +align_func_proto_span = 0 # number # Minimum gap between the return type and the function name. -align_func_proto_gap = 0 # number +align_func_proto_gap = 0 # number # Align function protos on the 'operator' keyword instead of what follows -align_on_operator = false # false/true +align_on_operator = false # false/true # Whether to mix aligning prototype and variable declarations. # If true, align_var_def_XXX options are used instead of align_func_proto_XXX options. -align_mix_var_proto = false # false/true +align_mix_var_proto = false # false/true # Align single-line functions with function prototypes, uses align_func_proto_span -align_single_line_func = false # false/true +align_single_line_func = false # false/true # Aligning the open brace of single-line functions. # Requires align_single_line_func=true, uses align_func_proto_span -align_single_line_brace = false # false/true +align_single_line_brace = false # false/true # Gap for align_single_line_brace. -align_single_line_brace_gap = 0 # number +align_single_line_brace_gap = 0 # number # The span for aligning ObjC msg spec (0=don't align) -align_oc_msg_spec_span = 0 # number +align_oc_msg_spec_span = 0 # number # Whether to align macros wrapped with a backslash and a newline. # This will not work right if the macro contains a multi-line comment. -align_nl_cont = false # false/true +align_nl_cont = false # false/true # # Align macro functions and variables together -align_pp_define_together = false # false/true +align_pp_define_together = false # false/true # The minimum space between label and value of a preprocessor define -align_pp_define_gap = 0 # number +align_pp_define_gap = 0 # number # The span for aligning on '#define' bodies (0=don't align, other=number of lines including comments between blocks) -align_pp_define_span = 0 # number +align_pp_define_span = 0 # number # Align lines that start with '<<' with previous '<<'. Default=true -align_left_shift = true # false/true +align_left_shift = true # false/true # Span for aligning parameters in an Obj-C message call on the ':' (0=don't align) -align_oc_msg_colon_span = 0 # number +align_oc_msg_colon_span = 0 # number # If true, always align with the first parameter, even if it is too short. -align_oc_msg_colon_first = false # false/true +align_oc_msg_colon_first = false # false/true # Aligning parameters in an Obj-C '+' or '-' declaration on the ':' -align_oc_decl_colon = false # false/true +align_oc_decl_colon = false # false/true # # Newline adding and removing options # # Whether to collapse empty blocks between '{' and '}' -nl_collapse_empty_body = true # false/true +nl_collapse_empty_body = true # false/true # Don't split one-line braced assignments - 'foo_t f = { 1, 2 };' -nl_assign_leave_one_liners = true # false/true +nl_assign_leave_one_liners = true # false/true # Don't split one-line braced statements inside a class xx { } body -nl_class_leave_one_liners = false # false/true +nl_class_leave_one_liners = false # false/true # Don't split one-line enums: 'enum foo { BAR = 15 };' -nl_enum_leave_one_liners = false # false/true +nl_enum_leave_one_liners = false # false/true # Don't split one-line get or set functions -nl_getset_leave_one_liners = false # false/true +nl_getset_leave_one_liners = false # false/true # Don't split one-line function definitions - 'int foo() { return 0; }' -nl_func_leave_one_liners = false # false/true +nl_func_leave_one_liners = false # false/true # Don't split one-line C++11 lambdas - '[]() { return 0; }' -nl_cpp_lambda_leave_one_liners = false # false/true +nl_cpp_lambda_leave_one_liners = false # false/true # Don't split one-line if/else statements - 'if(a) b++;' -nl_if_leave_one_liners = false # false/true +nl_if_leave_one_liners = false # false/true # Don't split one-line OC messages -nl_oc_msg_leave_one_liner = false # false/true +nl_oc_msg_leave_one_liner = false # false/true # Add or remove newlines at the start of the file -nl_start_of_file = ignore # ignore/add/remove/force +nl_start_of_file = ignore # ignore/add/remove/force # The number of newlines at the start of the file (only used if nl_start_of_file is 'add' or 'force' -nl_start_of_file_min = 0 # number +nl_start_of_file_min = 0 # number # Add or remove newline at the end of the file -nl_end_of_file = force # ignore/add/remove/force +nl_end_of_file = force # ignore/add/remove/force # The number of newlines at the end of the file (only used if nl_end_of_file is 'add' or 'force') -nl_end_of_file_min = 1 # number +nl_end_of_file_min = 1 # number # Add or remove newline between '=' and '{' -nl_assign_brace = ignore # ignore/add/remove/force +nl_assign_brace = ignore # ignore/add/remove/force # Add or remove newline between '=' and '[' (D only) -nl_assign_square = ignore # ignore/add/remove/force +nl_assign_square = ignore # ignore/add/remove/force # Add or remove newline after '= [' (D only). Will also affect the newline before the ']' -nl_after_square_assign = ignore # ignore/add/remove/force +nl_after_square_assign = ignore # ignore/add/remove/force # The number of blank lines after a block of variable definitions at the top of a function body # 0 = No change (default) -nl_func_var_def_blk = 0 # number +nl_func_var_def_blk = 0 # number # The number of newlines before a block of typedefs # 0 = No change (default) -nl_typedef_blk_start = 0 # number +nl_typedef_blk_start = 0 # number # The number of newlines after a block of typedefs # 0 = No change (default) -nl_typedef_blk_end = 0 # number +nl_typedef_blk_end = 0 # number # The maximum consecutive newlines within a block of typedefs # 0 = No change (default) -nl_typedef_blk_in = 0 # number +nl_typedef_blk_in = 0 # number # The number of newlines before a block of variable definitions not at the top of a function body # 0 = No change (default) -nl_var_def_blk_start = 0 # number +nl_var_def_blk_start = 0 # number # The number of newlines after a block of variable definitions not at the top of a function body # 0 = No change (default) -nl_var_def_blk_end = 0 # number +nl_var_def_blk_end = 0 # number # The maximum consecutive newlines within a block of variable definitions # 0 = No change (default) -nl_var_def_blk_in = 0 # number +nl_var_def_blk_in = 0 # number # Add or remove newline between a function call's ')' and '{', as in: # list_for_each(item, &list) { } -nl_fcall_brace = force # ignore/add/remove/force +nl_fcall_brace = force # ignore/add/remove/force # Add or remove newline between 'enum' and '{' -nl_enum_brace = force # ignore/add/remove/force +nl_enum_brace = force # ignore/add/remove/force # Add or remove newline between 'struct and '{' -nl_struct_brace = force # ignore/add/remove/force +nl_struct_brace = force # ignore/add/remove/force # Add or remove newline between 'union' and '{' -nl_union_brace = force # ignore/add/remove/force +nl_union_brace = force # ignore/add/remove/force # Add or remove newline between 'if' and '{' -nl_if_brace = force # ignore/add/remove/force +nl_if_brace = force # ignore/add/remove/force # Add or remove newline between '}' and 'else' -nl_brace_else = force # ignore/add/remove/force +nl_brace_else = force # ignore/add/remove/force # Add or remove newline between 'else if' and '{' # If set to ignore, nl_if_brace is used instead -nl_elseif_brace = ignore # ignore/add/remove/force +nl_elseif_brace = ignore # ignore/add/remove/force # Add or remove newline between 'else' and '{' -nl_else_brace = force # ignore/add/remove/force +nl_else_brace = force # ignore/add/remove/force # Add or remove newline between 'else' and 'if' -nl_else_if = remove # ignore/add/remove/force +nl_else_if = remove # ignore/add/remove/force # Add or remove newline between '}' and 'finally' -nl_brace_finally = ignore # ignore/add/remove/force +nl_brace_finally = ignore # ignore/add/remove/force # Add or remove newline between 'finally' and '{' -nl_finally_brace = ignore # ignore/add/remove/force +nl_finally_brace = ignore # ignore/add/remove/force # Add or remove newline between 'try' and '{' -nl_try_brace = ignore # ignore/add/remove/force +nl_try_brace = ignore # ignore/add/remove/force # Add or remove newline between get/set and '{' -nl_getset_brace = ignore # ignore/add/remove/force +nl_getset_brace = ignore # ignore/add/remove/force # Add or remove newline between 'for' and '{' -nl_for_brace = force # ignore/add/remove/force +nl_for_brace = force # ignore/add/remove/force # Add or remove newline between 'catch' and '{' -nl_catch_brace = ignore # ignore/add/remove/force +nl_catch_brace = ignore # ignore/add/remove/force # Add or remove newline between '}' and 'catch' -nl_brace_catch = ignore # ignore/add/remove/force +nl_brace_catch = ignore # ignore/add/remove/force # Add or remove newline between '}' and ']' -nl_brace_square = ignore # ignore/add/remove/force +nl_brace_square = ignore # ignore/add/remove/force # Add or remove newline between '}' and ')' in a function invocation -nl_brace_fparen = ignore # ignore/add/remove/force +nl_brace_fparen = ignore # ignore/add/remove/force # Add or remove newline between 'while' and '{' -nl_while_brace = force # ignore/add/remove/force +nl_while_brace = force # ignore/add/remove/force # Add or remove newline between 'scope (x)' and '{' (D) -nl_scope_brace = ignore # ignore/add/remove/force +nl_scope_brace = ignore # ignore/add/remove/force # Add or remove newline between 'unittest' and '{' (D) -nl_unittest_brace = ignore # ignore/add/remove/force +nl_unittest_brace = ignore # ignore/add/remove/force # Add or remove newline between 'version (x)' and '{' (D) -nl_version_brace = ignore # ignore/add/remove/force +nl_version_brace = ignore # ignore/add/remove/force # Add or remove newline between 'using' and '{' -nl_using_brace = ignore # ignore/add/remove/force +nl_using_brace = ignore # ignore/add/remove/force # Add or remove newline between two open or close braces. # Due to general newline/brace handling, REMOVE may not work. -nl_brace_brace = force # ignore/add/remove/force +nl_brace_brace = force # ignore/add/remove/force # Add or remove newline between 'do' and '{' -nl_do_brace = remove # ignore/add/remove/force +nl_do_brace = remove # ignore/add/remove/force # Add or remove newline between '}' and 'while' of 'do' statement -nl_brace_while = remove # ignore/add/remove/force +nl_brace_while = remove # ignore/add/remove/force # Add or remove newline between 'switch' and '{' -nl_switch_brace = force # ignore/add/remove/force +nl_switch_brace = force # ignore/add/remove/force # Add a newline between ')' and '{' if the ')' is on a different line than the if/for/etc. # Overrides nl_for_brace, nl_if_brace, nl_switch_brace, nl_while_switch, and nl_catch_brace. -nl_multi_line_cond = false # false/true +nl_multi_line_cond = false # false/true # Force a newline in a define after the macro name for multi-line defines. -nl_multi_line_define = false # false/true +nl_multi_line_define = false # false/true # Whether to put a newline before 'case' statement -nl_before_case = true # false/true +nl_before_case = true # false/true # Add or remove newline between ')' and 'throw' -nl_before_throw = ignore # ignore/add/remove/force +nl_before_throw = ignore # ignore/add/remove/force # Whether to put a newline after 'case' statement -nl_after_case = true # false/true +nl_after_case = true # false/true # Add or remove a newline between a case ':' and '{'. Overrides nl_after_case. -nl_case_colon_brace = force # ignore/add/remove/force +nl_case_colon_brace = force # ignore/add/remove/force # Newline between namespace and { -nl_namespace_brace = ignore # ignore/add/remove/force +nl_namespace_brace = ignore # ignore/add/remove/force # Add or remove newline between 'template<>' and whatever follows. -nl_template_class = ignore # ignore/add/remove/force +nl_template_class = ignore # ignore/add/remove/force # Add or remove newline between 'class' and '{' -nl_class_brace = ignore # ignore/add/remove/force +nl_class_brace = ignore # ignore/add/remove/force # Add or remove newline after each ',' in the class base list -nl_class_init_args = ignore # ignore/add/remove/force +nl_class_init_args = ignore # ignore/add/remove/force # Add or remove newline after each ',' in the constructor member initialization -nl_constr_init_args = ignore # ignore/add/remove/force +nl_constr_init_args = ignore # ignore/add/remove/force # Add or remove newline between return type and function name in a function definition -nl_func_type_name = force # ignore/add/remove/force +nl_func_type_name = force # ignore/add/remove/force # Add or remove newline between return type and function name inside a class {} # Uses nl_func_type_name or nl_func_proto_type_name if set to ignore. -nl_func_type_name_class = ignore # ignore/add/remove/force +nl_func_type_name_class = ignore # ignore/add/remove/force # Add or remove newline between function scope and name in a definition # Controls the newline after '::' in 'void A::f() { }' -nl_func_scope_name = ignore # ignore/add/remove/force +nl_func_scope_name = ignore # ignore/add/remove/force # Add or remove newline between return type and function name in a prototype -nl_func_proto_type_name = remove # ignore/add/remove/force +nl_func_proto_type_name = remove # ignore/add/remove/force # Add or remove newline between a function name and the opening '(' -nl_func_paren = remove # ignore/add/remove/force +nl_func_paren = remove # ignore/add/remove/force # Add or remove newline between a function name and the opening '(' in the definition -nl_func_def_paren = remove # ignore/add/remove/force +nl_func_def_paren = remove # ignore/add/remove/force # Add or remove newline after '(' in a function declaration -nl_func_decl_start = remove # ignore/add/remove/force +nl_func_decl_start = remove # ignore/add/remove/force # Add or remove newline after '(' in a function definition -nl_func_def_start = remove # ignore/add/remove/force +nl_func_def_start = remove # ignore/add/remove/force # Overrides nl_func_decl_start when there is only one parameter. -nl_func_decl_start_single = ignore # ignore/add/remove/force +nl_func_decl_start_single = ignore # ignore/add/remove/force # Overrides nl_func_def_start when there is only one parameter. -nl_func_def_start_single = ignore # ignore/add/remove/force +nl_func_def_start_single = ignore # ignore/add/remove/force # Add or remove newline after each ',' in a function declaration -nl_func_decl_args = ignore # ignore/add/remove/force +nl_func_decl_args = ignore # ignore/add/remove/force # Add or remove newline after each ',' in a function definition -nl_func_def_args = ignore # ignore/add/remove/force +nl_func_def_args = ignore # ignore/add/remove/force # Add or remove newline before the ')' in a function declaration -nl_func_decl_end = ignore # ignore/add/remove/force +nl_func_decl_end = ignore # ignore/add/remove/force # Add or remove newline before the ')' in a function definition -nl_func_def_end = remove # ignore/add/remove/force +nl_func_def_end = remove # ignore/add/remove/force # Overrides nl_func_decl_end when there is only one parameter. -nl_func_decl_end_single = ignore # ignore/add/remove/force +nl_func_decl_end_single = ignore # ignore/add/remove/force # Overrides nl_func_def_end when there is only one parameter. -nl_func_def_end_single = ignore # ignore/add/remove/force +nl_func_def_end_single = ignore # ignore/add/remove/force # Add or remove newline between '()' in a function declaration. -nl_func_decl_empty = ignore # ignore/add/remove/force +nl_func_decl_empty = ignore # ignore/add/remove/force # Add or remove newline between '()' in a function definition. -nl_func_def_empty = ignore # ignore/add/remove/force +nl_func_def_empty = ignore # ignore/add/remove/force # Whether to put each OC message parameter on a separate line # See nl_oc_msg_leave_one_liner -nl_oc_msg_args = false # false/true +nl_oc_msg_args = false # false/true # Add or remove newline between function signature and '{' -nl_fdef_brace = force # ignore/add/remove/force +nl_fdef_brace = force # ignore/add/remove/force # Add or remove newline between C++11 lambda signature and '{' -nl_cpp_ldef_brace = ignore # ignore/add/remove/force +nl_cpp_ldef_brace = ignore # ignore/add/remove/force # Add or remove a newline between the return keyword and return expression. -nl_return_expr = remove # ignore/add/remove/force +nl_return_expr = remove # ignore/add/remove/force # Whether to put a newline after semicolons, except in 'for' statements -nl_after_semicolon = true # false/true +nl_after_semicolon = true # false/true # Java: Control the newline between the ')' and '{{' of the double brace initializer. -nl_paren_dbrace_open = ignore # ignore/add/remove/force +nl_paren_dbrace_open = ignore # ignore/add/remove/force # Whether to put a newline after brace open. # This also adds a newline before the matching brace close. -nl_after_brace_open = true # false/true +nl_after_brace_open = true # false/true # If nl_after_brace_open and nl_after_brace_open_cmt are true, a newline is # placed between the open brace and a trailing single-line comment. -nl_after_brace_open_cmt = false # false/true +nl_after_brace_open_cmt = false # false/true # Whether to put a newline after a virtual brace open with a non-empty body. # These occur in un-braced if/while/do/for statement bodies. -nl_after_vbrace_open = false # false/true +nl_after_vbrace_open = false # false/true # Whether to put a newline after a virtual brace open with an empty body. # These occur in un-braced if/while/do/for statement bodies. -nl_after_vbrace_open_empty = false # false/true +nl_after_vbrace_open_empty = false # false/true # Whether to put a newline after a brace close. # Does not apply if followed by a necessary ';'. -nl_after_brace_close = true # false/true +nl_after_brace_close = true # false/true # Whether to put a newline after a virtual brace close. # Would add a newline before return in: 'if (foo) a++; return;' -nl_after_vbrace_close = false # false/true +nl_after_vbrace_close = false # false/true # Control the newline between the close brace and 'b' in: 'struct { int a; } b;' # Affects enums, unions, and structures. If set to ignore, uses nl_after_brace_close -nl_brace_struct_var = ignore # ignore/add/remove/force +nl_brace_struct_var = ignore # ignore/add/remove/force # Whether to alter newlines in '#define' macros -nl_define_macro = false # false/true +nl_define_macro = false # false/true # Whether to not put blanks after '#ifxx', '#elxx', or before '#endif' -nl_squeeze_ifdef = false # false/true +nl_squeeze_ifdef = false # false/true # Add or remove blank line before 'if' -nl_before_if = ignore # ignore/add/remove/force +nl_before_if = ignore # ignore/add/remove/force # Add or remove blank line after 'if' statement -nl_after_if = ignore # ignore/add/remove/force +nl_after_if = ignore # ignore/add/remove/force # Add or remove blank line before 'for' -nl_before_for = ignore # ignore/add/remove/force +nl_before_for = ignore # ignore/add/remove/force # Add or remove blank line after 'for' statement -nl_after_for = ignore # ignore/add/remove/force +nl_after_for = ignore # ignore/add/remove/force # Add or remove blank line before 'while' -nl_before_while = ignore # ignore/add/remove/force +nl_before_while = ignore # ignore/add/remove/force # Add or remove blank line after 'while' statement -nl_after_while = ignore # ignore/add/remove/force +nl_after_while = ignore # ignore/add/remove/force # Add or remove blank line before 'switch' -nl_before_switch = ignore # ignore/add/remove/force +nl_before_switch = ignore # ignore/add/remove/force # Add or remove blank line after 'switch' statement -nl_after_switch = ignore # ignore/add/remove/force +nl_after_switch = ignore # ignore/add/remove/force # Add or remove blank line before 'do' -nl_before_do = ignore # ignore/add/remove/force +nl_before_do = ignore # ignore/add/remove/force # Add or remove blank line after 'do/while' statement -nl_after_do = ignore # ignore/add/remove/force +nl_after_do = ignore # ignore/add/remove/force # Whether to double-space commented-entries in struct/enum -nl_ds_struct_enum_cmt = false # false/true +nl_ds_struct_enum_cmt = false # false/true # Whether to double-space before the close brace of a struct/union/enum # (lower priority than 'eat_blanks_before_close_brace') -nl_ds_struct_enum_close_brace = false # false/true +nl_ds_struct_enum_close_brace = false # false/true # Add or remove a newline around a class colon. # Related to pos_class_colon, nl_class_init_args, and pos_class_comma. -nl_class_colon = ignore # ignore/add/remove/force +nl_class_colon = ignore # ignore/add/remove/force # Add or remove a newline around a class constructor colon. # Related to pos_constr_colon, nl_constr_init_args, and pos_constr_comma. -nl_constr_colon = ignore # ignore/add/remove/force +nl_constr_colon = ignore # ignore/add/remove/force # Change simple unbraced if statements into a one-liner # 'if(b)\n i++;' => 'if(b) i++;' -nl_create_if_one_liner = false # false/true +nl_create_if_one_liner = false # false/true # Change simple unbraced for statements into a one-liner # 'for (i=0;i<5;i++)\n foo(i);' => 'for (i=0;i<5;i++) foo(i);' -nl_create_for_one_liner = false # false/true +nl_create_for_one_liner = false # false/true # Change simple unbraced while statements into a one-liner # 'while (i<5)\n foo(i++);' => 'while (i<5) foo(i++);' -nl_create_while_one_liner = false # false/true +nl_create_while_one_liner = false # false/true # # Positioning options # # The position of arithmetic operators in wrapped expressions -pos_arith = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_arith = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of assignment in wrapped expressions. # Do not affect '=' followed by '{' -pos_assign = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_assign = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of boolean operators in wrapped expressions -pos_bool = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_bool = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of comparison operators in wrapped expressions -pos_compare = join # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_compare = join # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of conditional (b ? t : f) operators in wrapped expressions -pos_conditional = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_conditional = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of the comma in wrapped expressions -pos_comma = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_comma = trail # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of the comma in the class base list -pos_class_comma = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_class_comma = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of the comma in the constructor initialization list -pos_constr_comma = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_constr_comma = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of colons between class and base class list -pos_class_colon = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_class_colon = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # The position of colons between constructor and member initialization -pos_constr_colon = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force +pos_constr_colon = ignore # ignore/join/lead/lead_break/lead_force/trail/trail_break/trail_force # # Line Splitting options # # Try to limit code width to N number of columns -code_width = 90 # number +code_width = 90 # number # Whether to fully split long 'for' statements at semi-colons -ls_for_split_full = false # false/true +ls_for_split_full = false # false/true # Whether to fully split long function protos/calls at commas -ls_func_split_full = false # false/true +ls_func_split_full = false # false/true # Whether to split lines as close to code_width as possible and ignore some groupings -ls_code_width = true # false/true +ls_code_width = true # false/true # # Blank line options # # The maximum consecutive newlines -nl_max = 3 # number +nl_max = 3 # number # The number of newlines after a function prototype, if followed by another function prototype -nl_after_func_proto = 0 # number +nl_after_func_proto = 0 # number # The number of newlines after a function prototype, if not followed by another function prototype -nl_after_func_proto_group = 0 # number +nl_after_func_proto_group = 0 # number # The number of newlines after '}' of a multi-line function body -nl_after_func_body = 3 # number +nl_after_func_body = 3 # number # The number of newlines after '}' of a multi-line function body in a class declaration -nl_after_func_body_class = 0 # number +nl_after_func_body_class = 0 # number # The number of newlines after '}' of a single line function body -nl_after_func_body_one_liner = 3 # number +nl_after_func_body_one_liner = 3 # number # The minimum number of newlines before a multi-line comment. # Doesn't apply if after a brace open or another multi-line comment. -nl_before_block_comment = 2 # number +nl_before_block_comment = 2 # number # The minimum number of newlines before a single-line C comment. # Doesn't apply if after a brace open or other single-line C comments. -nl_before_c_comment = 2 # number +nl_before_c_comment = 2 # number # The minimum number of newlines before a CPP comment. # Doesn't apply if after a brace open or other CPP comments. -nl_before_cpp_comment = 0 # number +nl_before_cpp_comment = 0 # number # Whether to force a newline after a multi-line comment. -nl_after_multiline_comment = false # false/true +nl_after_multiline_comment = false # false/true # The number of newlines after '}' or ';' of a struct/enum/union definition -nl_after_struct = 1 # number +nl_after_struct = 1 # number # The number of newlines after '}' or ';' of a class definition -nl_after_class = 0 # number +nl_after_class = 0 # number # The number of newlines before a 'private:', 'public:', 'protected:', 'signals:', or 'slots:' label. # Will not change the newline count if after a brace open. # 0 = No change. -nl_before_access_spec = 0 # number +nl_before_access_spec = 0 # number # The number of newlines after a 'private:', 'public:', 'protected:', 'signals:', or 'slots:' label. # 0 = No change. -nl_after_access_spec = 0 # number +nl_after_access_spec = 0 # number # The number of newlines between a function def and the function comment. # 0 = No change. -nl_comment_func_def = 1 # number +nl_comment_func_def = 1 # number # The number of newlines after a try-catch-finally block that isn't followed by a brace close. # 0 = No change. -nl_after_try_catch_finally = 0 # number +nl_after_try_catch_finally = 0 # number # The number of newlines before and after a property, indexer or event decl. # 0 = No change. -nl_around_cs_property = 0 # number +nl_around_cs_property = 0 # number # The number of newlines between the get/set/add/remove handlers in C#. # 0 = No change. -nl_between_get_set = 0 # number +nl_between_get_set = 0 # number # Add or remove newline between C# property and the '{' -nl_property_brace = ignore # ignore/add/remove/force +nl_property_brace = ignore # ignore/add/remove/force # Whether to remove blank lines after '{' -eat_blanks_after_open_brace = true # false/true +eat_blanks_after_open_brace = true # false/true # Whether to remove blank lines before '}' -eat_blanks_before_close_brace = true # false/true +eat_blanks_before_close_brace = true # false/true # How aggressively to remove extra newlines not in preproc. # 0: No change # 1: Remove most newlines not handled by other config # 2: Remove all newlines and reformat completely by config -nl_remove_extra_newlines = 0 # number +nl_remove_extra_newlines = 0 # number # Whether to put a blank line before 'return' statements, unless after an open brace. -nl_before_return = false # false/true +nl_before_return = false # false/true # Whether to put a blank line after 'return' statements, unless followed by a close brace. -nl_after_return = false # false/true +nl_after_return = false # false/true # Whether to put a newline after a Java annotation statement. # Only affects annotations that are after a newline. -nl_after_annotation = ignore # ignore/add/remove/force +nl_after_annotation = ignore # ignore/add/remove/force # Controls the newline between two annotations. -nl_between_annotation = ignore # ignore/add/remove/force +nl_between_annotation = ignore # ignore/add/remove/force # # Code modifying options (non-whitespace) # # Add or remove braces on single-line 'do' statement -mod_full_brace_do = ignore # ignore/add/remove/force +mod_full_brace_do = ignore # ignore/add/remove/force # Add or remove braces on single-line 'for' statement -mod_full_brace_for = force # ignore/add/remove/force +mod_full_brace_for = force # ignore/add/remove/force # Add or remove braces on single-line function definitions. (Pawn) -mod_full_brace_function = ignore # ignore/add/remove/force +mod_full_brace_function = ignore # ignore/add/remove/force # Add or remove braces on single-line 'if' statement. Will not remove the braces if they contain an 'else'. -mod_full_brace_if = force # ignore/add/remove/force +mod_full_brace_if = force # ignore/add/remove/force # Make all if/elseif/else statements in a chain be braced or not. Overrides mod_full_brace_if. # If any must be braced, they are all braced. If all can be unbraced, then the braces are removed. -mod_full_brace_if_chain = false # false/true +mod_full_brace_if_chain = false # false/true # Don't remove braces around statements that span N newlines -mod_full_brace_nl = 0 # number +mod_full_brace_nl = 0 # number # Add or remove braces on single-line 'while' statement -mod_full_brace_while = force # ignore/add/remove/force +mod_full_brace_while = force # ignore/add/remove/force # Add or remove braces on single-line 'using ()' statement -mod_full_brace_using = ignore # ignore/add/remove/force +mod_full_brace_using = ignore # ignore/add/remove/force # Add or remove unnecessary paren on 'return' statement -mod_paren_on_return = ignore # ignore/add/remove/force +mod_paren_on_return = ignore # ignore/add/remove/force # Whether to change optional semicolons to real semicolons -mod_pawn_semicolon = false # false/true +mod_pawn_semicolon = false # false/true # Add parens on 'while' and 'if' statement around bools -mod_full_paren_if_bool = false # false/true +mod_full_paren_if_bool = false # false/true # Whether to remove superfluous semicolons -mod_remove_extra_semicolon = true # false/true +mod_remove_extra_semicolon = true # false/true # If a function body exceeds the specified number of newlines and doesn't have a comment after # the close brace, a comment will be added. -mod_add_long_function_closebrace_comment = 0 # number +mod_add_long_function_closebrace_comment = 0 # number # If a namespace body exceeds the specified number of newlines and doesn't have a comment after # the close brace, a comment will be added. -mod_add_long_namespace_closebrace_comment = 0 # number +mod_add_long_namespace_closebrace_comment = 0 # number # If a switch body exceeds the specified number of newlines and doesn't have a comment after # the close brace, a comment will be added. -mod_add_long_switch_closebrace_comment = 0 # number +mod_add_long_switch_closebrace_comment = 0 # number # If an #ifdef body exceeds the specified number of newlines and doesn't have a comment after # the #endif, a comment will be added. -mod_add_long_ifdef_endif_comment = 0 # number +mod_add_long_ifdef_endif_comment = 0 # number # If an #ifdef or #else body exceeds the specified number of newlines and doesn't have a comment after # the #else, a comment will be added. -mod_add_long_ifdef_else_comment = 0 # number +mod_add_long_ifdef_else_comment = 0 # number # If TRUE, will sort consecutive single-line 'import' statements [Java, D] -mod_sort_import = false # false/true +mod_sort_import = false # false/true # If TRUE, will sort consecutive single-line 'using' statements [C#] -mod_sort_using = false # false/true +mod_sort_using = false # false/true # If TRUE, will sort consecutive single-line '#include' statements [C/C++] and '#import' statements [Obj-C] # This is generally a bad idea, as it may break your code. -mod_sort_include = false # false/true +mod_sort_include = false # false/true # If TRUE, it will move a 'break' that appears after a fully braced 'case' before the close brace. -mod_move_case_break = true # false/true +mod_move_case_break = true # false/true # Will add or remove the braces around a fully braced case statement. # Will only remove the braces if there are no variable declarations in the block. -mod_case_brace = force # ignore/add/remove/force +mod_case_brace = force # ignore/add/remove/force # If TRUE, it will remove a void 'return;' that appears as the last statement in a function. -mod_remove_empty_return = true # false/true +mod_remove_empty_return = true # false/true # # Comment modifications # # Try to wrap comments at cmt_width columns -cmt_width = 0 # number +cmt_width = 0 # number # Set the comment reflow mode (default: 0) # 0: no reflowing (apart from the line wrapping due to cmt_width) # 1: no touching at all # 2: full reflow -cmt_reflow_mode = 0 # number +cmt_reflow_mode = 0 # number # Whether to convert all tabs to spaces in comments. Default is to leave tabs inside comments alone, unless used for indenting. -cmt_convert_tab_to_spaces = false # false/true +cmt_convert_tab_to_spaces = false # false/true # If false, disable all multi-line comment changes, including cmt_width. keyword substitution, and leading chars. # Default is true. -cmt_indent_multi = true # false/true +cmt_indent_multi = true # false/true # Whether to group c-comments that look like they are in a block -cmt_c_group = false # false/true +cmt_c_group = false # false/true # Whether to put an empty '/*' on the first line of the combined c-comment -cmt_c_nl_start = false # false/true +cmt_c_nl_start = false # false/true # Whether to put a newline before the closing '*/' of the combined c-comment -cmt_c_nl_end = false # false/true +cmt_c_nl_end = false # false/true # Whether to group cpp-comments that look like they are in a block -cmt_cpp_group = false # false/true +cmt_cpp_group = false # false/true # Whether to put an empty '/*' on the first line of the combined cpp-comment -cmt_cpp_nl_start = false # false/true +cmt_cpp_nl_start = false # false/true # Whether to put a newline before the closing '*/' of the combined cpp-comment -cmt_cpp_nl_end = false # false/true +cmt_cpp_nl_end = false # false/true # Whether to change cpp-comments into c-comments -cmt_cpp_to_c = true # false/true +cmt_cpp_to_c = true # false/true # Whether to put a star on subsequent comment lines -cmt_star_cont = true # false/true +cmt_star_cont = true # false/true # The number of spaces to insert at the start of subsequent comment lines -cmt_sp_before_star_cont = 0 # number +cmt_sp_before_star_cont = 0 # number # The number of spaces to insert after the star on subsequent comment lines -cmt_sp_after_star_cont = 0 # number +cmt_sp_after_star_cont = 0 # number # For multi-line comments with a '*' lead, remove leading spaces if the first and last lines of # the comment are the same length. Default=True -cmt_multi_check_last = true # false/true +cmt_multi_check_last = true # false/true # The filename that contains text to insert at the head of a file if the file doesn't start with a C/C++ comment. # Will substitute $(filename) with the current file's name. -cmt_insert_file_header = "" # string +cmt_insert_file_header = "" # string # The filename that contains text to insert at the end of a file if the file doesn't end with a C/C++ comment. # Will substitute $(filename) with the current file's name. -cmt_insert_file_footer = "" # string +cmt_insert_file_footer = "" # string # The filename that contains text to insert before a function implementation if the function isn't preceded with a C/C++ comment. # Will substitute $(function) with the function name and $(javaparam) with the javadoc @param and @return stuff. # Will also substitute $(fclass) with the class name: void CFoo::Bar() { ... } -cmt_insert_func_header = "" # string +cmt_insert_func_header = "" # string # The filename that contains text to insert before a class if the class isn't preceded with a C/C++ comment. # Will substitute $(class) with the class name. -cmt_insert_class_header = "" # string +cmt_insert_class_header = "" # string # The filename that contains text to insert before a Obj-C message specification if the method isn't preceded with a C/C++ comment. # Will substitute $(message) with the function name and $(javaparam) with the javadoc @param and @return stuff. -cmt_insert_oc_msg_header = "" # string +cmt_insert_oc_msg_header = "" # string # If a preprocessor is encountered when stepping backwards from a function name, then # this option decides whether the comment should be inserted. # Affects cmt_insert_oc_msg_header, cmt_insert_func_header and cmt_insert_class_header. -cmt_insert_before_preproc = false # false/true +cmt_insert_before_preproc = false # false/true # # Preprocessor options # # Control indent of preprocessors inside #if blocks at brace level 0 (file-level) -pp_indent = ignore # ignore/add/remove/force +pp_indent = ignore # ignore/add/remove/force # Whether to indent #if/#else/#endif at the brace level (true) or from column 1 (false) -pp_indent_at_level = false # false/true +pp_indent_at_level = false # false/true # Specifies the number of columns to indent preprocessors per level at brace level 0 (file-level). # If pp_indent_at_level=false, specifies the number of columns to indent preprocessors per level at brace level > 0 (function-level). # Default=1. -pp_indent_count = 1 # number +pp_indent_count = 1 # number # Add or remove space after # based on pp_level of #if blocks -pp_space = ignore # ignore/add/remove/force +pp_space = ignore # ignore/add/remove/force # Sets the number of spaces added with pp_space -pp_space_count = 0 # number +pp_space_count = 0 # number # The indent for #region and #endregion in C# and '#pragma region' in C/C++ -pp_indent_region = 0 # number +pp_indent_region = 0 # number # Whether to indent the code between #region and #endregion -pp_region_indent_code = false # false/true +pp_region_indent_code = false # false/true # If pp_indent_at_level=true, sets the indent for #if, #else, and #endif when not at file-level. # 0: indent preprocessors using output_tab_size. # >0: column at which all preprocessors will be indented. -pp_indent_if = 0 # number +pp_indent_if = 0 # number # Control whether to indent the code between #if, #else and #endif. -pp_if_indent_code = false # false/true +pp_if_indent_code = false # false/true # Whether to indent '#define' at the brace level (true) or from column 1 (false) -pp_define_at_level = false # false/true +pp_define_at_level = false # false/true # You can force a token to be a type with the 'type' option. # Example: diff --git a/valgrind/launch-test-instance b/valgrind/launch-test-instance index 2640d69f..effc0f3b 100755 --- a/valgrind/launch-test-instance +++ b/valgrind/launch-test-instance @@ -2,13 +2,7 @@ set -euo pipefail -REPORT_EMAIL=${REPORT_EMAIL:-burak@citusdata.com \ - furkan@citusdata.com \ - metin@citusdata.com \ - Burak.Yucesoy@microsoft.com \ - Furkan.Sahin@microsoft.com \ - Hanefi.Onaldi@microsoft.com \ - Metin.Doslu@microsoft.com } +REPORT_EMAIL=${REPORT_EMAIL:-burak@citusdata.com furkan@citusdata.com metin@citusdata.com Burak.Yucesoy@microsoft.com Furkan.Sahin@microsoft.com Hanefi.Onaldi@microsoft.com Metin.Doslu@microsoft.com } POSTGRES_GITREF=${POSTGRES_GITREF:-REL_11_STABLE} CITUS_GITREF=${CITUS_GITREF:-master} @@ -19,14 +13,14 @@ echo " CITUS_GITREF:" $CITUS_GITREF # when running in a tty wait for the human to press enter, this is to allow the human to verify the settings # when running as a crontab it will just continue -if [ -t 1 ] ; then +if [ -t 1 ]; then read -p "Press enter to continue" fi # create a key pair just for valgrind tests and store it in valgrind-test.pem echo "Creating key pair..." key_name=valgrind_$RANDOM -aws ec2 create-key-pair --key-name $key_name --query 'KeyMaterial' --output text > $key_name.pem +aws ec2 create-key-pair --key-name $key_name --query 'KeyMaterial' --output text >$key_name.pem chmod 600 $key_name.pem # start an instance with ami-f4cc1de2 image(i.e. Ubuntu Xenial 16.04) diff --git a/valgrind/run-valgrind-tests b/valgrind/run-valgrind-tests index db956c11..360eb23f 100755 --- a/valgrind/run-valgrind-tests +++ b/valgrind/run-valgrind-tests @@ -2,13 +2,7 @@ set -euo pipefail -REPORT_EMAIL=${REPORT_EMAIL:-burak@citusdata.com \ - furkan@citusdata.com \ - metin@citusdata.com \ - Burak.Yucesoy@microsoft.com \ - Furkan.Sahin@microsoft.com \ - Hanefi.Onaldi@microsoft.com \ - Metin.Doslu@microsoft.com } +REPORT_EMAIL=${REPORT_EMAIL:-burak@citusdata.com furkan@citusdata.com metin@citusdata.com Burak.Yucesoy@microsoft.com Furkan.Sahin@microsoft.com Hanefi.Onaldi@microsoft.com Metin.Doslu@microsoft.com } CITUS_GITREF=${CITUS_GITREF:-master} POSTGRES_GITREF=${POSTGRES_GITREF:-REL_11_STABLE} @@ -76,9 +70,9 @@ if [ -s regression.diffs ]; then fi if [ -z "$attachments" ]; then - mail -aFrom:valgrind-test@citusdata.com -s "[Valgrind Test Results] - Success" $REPORT_EMAIL < /dev/null + mail -aFrom:valgrind-test@citusdata.com -s "[Valgrind Test Results] - Success" $REPORT_EMAIL