Run formatters for all the files in the repo

CHANGELOG.md

@@ -66,7 +66,7 @@
 
 ### citustools v0.7.2 (October 5, 2017) ###
 
-* Changes packaging default PostgreSQL versions from 9.5,9.6 to 9.6,10
+* Changes packaging default PostgreSQL versions from 9.5, 9.6 to 9.6, 10
 
 ### citustools v0.7.1 (August 30, 2017) ###
 

automated_packaging/Makefile

@@ -4,7 +4,7 @@ INSTALL := install -c
 INSTALL_SCRIPT := $(INSTALL) -m 755
 AUTOMATION_SCRIPTS := $(filter-out Makefile,$(wildcard *))
 
-all: 
+all:
 
 clean:
 
@@ -13,5 +13,5 @@ installdirs:
 
 install: all installdirs
 	$(INSTALL_SCRIPT) $(AUTOMATION_SCRIPTS) $(DESTDIR)$(bindir)
-	
+
 .PHONY: clean installdirs install

automated_packaging/common_functions.pm

@@ -150,7 +150,7 @@ sub create_release_changelog {
             foreach $line (@log_output) {
                 if ($line =~ /^DESCRIPTION: */) {
                     $description_part = substr($line, length($&), -1);
-                    
+
                     if (length($description_part) > 78) {
                         print("You have to shorten PR message $description_part of $pr_url\n");
                         print("Description should not be longer than 78 charachters, please manually shorten this description\n");

citus_dev/README.md

@@ -1,15 +1,19 @@
 # Setup
-To install dependencies run: 
+
+To install dependencies run:
+
 ```bash
 pip install -r requirements.txt
 ```
 
 Add `citus_dev` to your PATH:
+
 ```bash
 export PATH=$PATH:<your path to citus_dev folder>
 ```
 
 You can also add this to your profile:
+
 ```bash
 echo 'export PATH=$PATH:<your path to citus_dev folder>' >>~/.profile
 ```
@@ -21,6 +25,7 @@ citus_dev make clusterName
 ```
 
 For the full command list:
+
 ```bash
 citus_dev --help
 ```

citus_dev/bash_completion

@@ -12,7 +12,7 @@ _citus_dev()
         make)
             return 0
             ;;
-        
+
         start|stop|restart)
             clusters=$(for x in `find . -type d -name 'coordinator' | awk -F '/' '{print $2}'`; do echo ${x} ; done );
             COMPREPLY=($( compgen -W "${clusters}" -- "${cur}" ));

citus_dev/citus_dev

@@ -38,6 +38,7 @@ fixopen = distutils.spawn.find_executable("postgres.fixopen")
 if fixopen:
     pgctl_flags += f' -p "{fixopen}"'
 
+
 def run(command, *args, **kwargs):
     print(command)
     result = subprocess.run(command, *args, check=True, shell=True, **kwargs)
@@ -61,18 +62,23 @@ def createNodeCommands(clustername, role, index=None, usessl=False, mx=False):
 
     if usessl:
         run(f'echo "ssl = on" >> {clustername}/{nodename}/postgresql.conf')
-        run(f"echo \"citus.node_conninfo = 'sslmode=require'\" >> {clustername}/{nodename}/postgresql.conf")
-        run(f"openssl req -new -x509 -days 365 -nodes -text -out {clustername}/{nodename}/server.crt -keyout {clustername}/{nodename}/server.key -subj '/CN={nodename}'")
+        run(
+            f"echo \"citus.node_conninfo = 'sslmode=require'\" >> {clustername}/{nodename}/postgresql.conf")
+        run(
+            f"openssl req -new -x509 -days 365 -nodes -text -out {clustername}/{nodename}/server.crt -keyout {clustername}/{nodename}/server.key -subj '/CN={nodename}'")
         run(f"chmod 0600 {clustername}/{nodename}/server.key")
 
     if mx:
-        run(f"echo \"citus.replication_model = 'streaming'\" >> {clustername}/{nodename}/postgresql.conf")
+        run(
+            f"echo \"citus.replication_model = 'streaming'\" >> {clustername}/{nodename}/postgresql.conf")
+
 
 def createPgBouncerUsers(clustername):
     username = getpass.getuser()
     with open(f"{clustername}/users.txt", "w") as f:
         f.write(f'"{username}" ""')
 
+
 def createPgBouncerConfig(clustername, port, index):
     workerPort = port + index + 1
     bouncerPort = port + index + 101
@@ -111,7 +117,6 @@ def main(arguments):
             stopCluster(clustername, True)
             run(f'rm -rf {clustername}')
 
-
         createNodeCommands(
             clustername,
             "coordinator",
@@ -177,25 +182,24 @@ def main(arguments):
             # need to start pgbouncers and configure pg_dist_poolinfo
             for i in range(size):
                 coordinatorPort = port
-                workerPort = port + i + 1 
+                workerPort = port + i + 1
                 bouncerPort = port + i + 101
                 run(f'pgbouncer -d {clustername}/worker{i}.pgbouncer.ini')
                 run(f"psql -p {coordinatorPort} -c \"INSERT INTO pg_dist_poolinfo SELECT nodeid, 'host=localhost port={bouncerPort}' AS poolinfo FROM pg_dist_node WHERE nodeport = {workerPort};\"")
 
-
         if arguments['--init-with']:
             run(f'psql -p {cport} -f {arguments["--init-with"]} -v ON_ERROR_STOP=1')
         if arguments['--init-worker-with']:
             for i in range(size):
                 workerport = port + 1 + i
-                run(f'psql -p {workerport} -f {arguments["--init-worker-with"]} -v ON_ERROR_STOP=1')
+                run(
+                    f'psql -p {workerport} -f {arguments["--init-worker-with"]} -v ON_ERROR_STOP=1')
 
     elif arguments["stop"]:
         clusterName = arguments["<name>"]
         always = arguments["--force"]
         stopCluster(clusterName, always)
 
-
     elif arguments["start"]:
         clustername = arguments["<name>"]
         port = int(arguments["--port"])
@@ -205,8 +209,6 @@ def main(arguments):
         for bouncerConfig in getPgBouncerConfigs(clustername):
             run(f'pgbouncer -d {clustername}/{bouncerConfig}')
 
-
-
     elif arguments["restart"]:
         clustername = arguments["<name>"]
         port = int(arguments["--port"])
@@ -222,17 +224,18 @@ def main(arguments):
                 run(f'pg_ctl {pgctl_flags} restart -D {clustername}/{role} -o "-p {cport}" -l {role}_logfile')
                 cport += 1
 
-
     else:
         print("unknown command")
         exit(1)
 
+
 def getPgBouncerPort(clustername, configfile):
     with open(f"{clustername}/{configfile}") as f:
-            for line in f.readlines():
-                if line.find("listen_port") >= 0:
-                    lineParts = line.split("=")
-                    return int(lineParts[1])
+        for line in f.readlines():
+            if line.find("listen_port") >= 0:
+                lineParts = line.split("=")
+                return int(lineParts[1])
+
 
 def stopCluster(clustername, always=False):
     for bouncerConfig in getPgBouncerConfigs(clustername):
@@ -247,9 +250,11 @@ def stopCluster(clustername, always=False):
     for role in getRoles(clustername):
         run(f"pg_ctl {pgctl_flags} stop -D {clustername}/{role} {pipeTrue}")
 
+
 def getPgBouncerConfigs(clustername):
     try:
-        bouncerFiles = [f.name for f in os.scandir(clustername) if f.name.find("pgbouncer.ini") >= 0]
+        bouncerFiles = [f.name for f in os.scandir(
+            clustername) if f.name.find("pgbouncer.ini") >= 0]
         return bouncerFiles
     except FileNotFoundError:
         return []

dashboard/README.md

@@ -4,11 +4,11 @@ These scripts are intended for use within daily cron scripts (presently running
 
 ## Getting Started
 
-You probably shouldn't need to do anything with this, but if anything changes, log into that box, use `curl` or `wget` to get the latest version of the `citusdata/tools` repository, then `sudo make -C dashboard install`. Check the latest cron configuration using `crontab -l`, and see whether any cron failures have sent mail using `mail`.
+You probably shouldn't need to do anything with this, but if anything changes, log into that box, use `curl` or `wget` to get the latest version of the `citusdata/tools` repository, then `sudo make -C dashboard install` . Check the latest cron configuration using `crontab -l` , and see whether any cron failures have sent mail using `mail` .
 
 ## What's Included
 
-All non `update_*` scripts emit (to standard out) CSV files suitable for loading into PostgreSQL tables defined in `schema.dll`. Most have pretty good usage messages, so check those out if needed.
+All non `update_*` scripts emit (to standard out) CSV files suitable for loading into PostgreSQL tables defined in `schema.dll` . Most have pretty good usage messages, so check those out if needed.
 
 `pkg.jq` is a huge mishmash of `jq` helper normalization functions for massaging API data into the right state before ingest.
 

dashboard/docker_pulls

@@ -14,7 +14,7 @@ jq=$(which jq)
 
 # outputs usage message on specified device before exiting with provided status
 usage() {
-    cat << 'E_O_USAGE' >&"$1"
+  cat <<'E_O_USAGE' >&"$1"
 usage: docker_pulls repo [since]
 
     repo : a citusdata Docker Hub repository name
@@ -27,7 +27,7 @@ recent time point's date as the 'since' parameter will cause this program to
 exit without emitting any rows if that parameter is on or after today's date.
 E_O_USAGE
 
-    exit "${2}";
+  exit "${2}"
 }
 
 if [ "$#" -eq 1 ]; then
@@ -53,4 +53,4 @@ repo=${1}
 repourl="https://hub.docker.com/v2/repositories/citusdata/${repo}/"
 
 curl -sf "${repourl}" |
-${jq} -r "include \"pkg\"; makepullrow(\"${repo}\") | @csv"
+  ${jq} -r "include \"pkg\"; makepullrow(\"${repo}\") | @csv"

dashboard/github_clones

@@ -17,7 +17,7 @@ jq=$(which jq)
 
 # outputs usage message on specified device before exiting with provided status
 usage() {
-    cat << 'E_O_USAGE' >&"$1"
+  cat <<'E_O_USAGE' >&"$1"
 usage: github_clones repo [since]
 
     repo : a citusdata GitHub repository name
@@ -29,7 +29,7 @@ custom start date: without one, all known data is requested, though GitHub's
 maximum window is only two weeks long.
 E_O_USAGE
 
-    exit "${2}";
+  exit "${2}"
 }
 
 if [ "$#" -eq 1 ]; then
@@ -49,4 +49,4 @@ repo=${1}
 trafficurl="https://api.github.com/repos/citusdata/${repo}/traffic/clones"
 
 curl -sf -H "${hubauth}" -H "${preview}" "${trafficurl}" |
-${jq} -r "include \"pkg\"; .clones[] | filterdate(\"${since}\") | makeclonerows(\"${repo}\") | @csv"
+  ${jq} -r "include \"pkg\"; .clones[] | filterdate(\"${since}\") | makeclonerows(\"${repo}\") | @csv"

dashboard/homebrew_downloads

@@ -14,7 +14,7 @@ jq=$(which jq)
 
 # outputs usage message on specified device before exiting with provided status
 usage() {
-    cat << 'E_O_USAGE' >&"$1"
+  cat <<'E_O_USAGE' >&"$1"
 usage: homebrew_downloads package [since]
 
     package : a package name, such as citus
@@ -27,7 +27,7 @@ present (i.e. "6.0.0_1"), they are normalized to the upstream version (in the
 previous case, "6.0.0") and have their installs aggregated under that.
 E_O_USAGE
 
-    exit "${2}";
+  exit "${2}"
 }
 
 if [ "$#" -eq 1 ]; then
@@ -42,12 +42,12 @@ else
 fi
 
 case "${OSTYPE}" in
-  darwin*)
-    enddate=$(date -v-1d "+%Y-%m-%d")
-    ;;
-  *)
-    enddate=$(date -d "1 days ago" "+%Y-%m-%d")
-    ;;
+darwin*)
+  enddate=$(date -v-1d "+%Y-%m-%d")
+  ;;
+*)
+  enddate=$(date -d "1 days ago" "+%Y-%m-%d")
+  ;;
 esac
 
 echo '"os","release","name","pg_version","version","date","downloads"'

dashboard/packagecloud_downloads

@@ -16,7 +16,7 @@ jq=$(which jq)
 
 # outputs usage message on specified device before exiting with provided status
 usage() {
-    cat << 'E_O_USAGE' >&"$1"
+  cat <<'E_O_USAGE' >&"$1"
 usage: packagecloud_downloads repo [since]
 
     repo : 'community' or 'enterprise'
@@ -30,7 +30,7 @@ must queried. Debian/Red Hat versions are normalized back to a presumed git tag
 in order to better facilitate grouping and aggregation.
 E_O_USAGE
 
-    exit "${2}";
+  exit "${2}"
 }
 
 if [ "$#" -eq 1 ]; then
@@ -49,24 +49,22 @@ echo '"os","release","name","pg_version","version","date","downloads"'
 repo=${1}
 pkgapiurl="https://packagecloud.io/api/v1/repos/citusdata/${repo}/packages.json?per_page=100"
 
-for ((page=1; ; page++))
-do
+for ((page = 1; ; page++)); do
   totalresults=$(curl -sf -u "${pkgauth}" "${pkgapiurl}&page=${page}")
   totalcount=$(echo ${totalresults} | ${jq} 'length')
 
   if [ "${totalcount}" -le 0 ]; then
-      break
+    break
   fi
 
   results=$(echo "${totalresults}" | ${jq} 'include "pkg"; stripdebuginfo')
   count=$(echo ${results} | ${jq} 'length')
 
-  for ((i=0; i<${count}; i++))
-  do
+  for ((i = 0; i < ${count}; i++)); do
     result=$(echo ${results} | ${jq} -r --argjson i "${i}" '.[$i]')
     series_url=$(echo ${result} | ${jq} -r "include \"pkg\"; extracturl(${since})")
 
     curl -sf -u "${pkgauth}" "https://packagecloud.io${series_url}" |
-    ${jq} --argjson r "${result}" -r 'include "pkg"; stripzeros | .[] | makerow($r) | @csv'
+      ${jq} --argjson r "${result}" -r 'include "pkg"; stripzeros | .[] | makerow($r) | @csv'
   done
 done

dashboard/rubygem_installs

@@ -14,7 +14,7 @@ jq=$(which jq)
 
 # outputs usage message on specified device before exiting with provided status
 usage() {
-    cat << 'E_O_USAGE' >&"$1"
+  cat <<'E_O_USAGE' >&"$1"
 usage: rubygem_installs repo [since]
 
     repo : a RubyGems.org gem name
@@ -27,7 +27,7 @@ most recent time point's date as the 'since' parameter will cause this program
 to exit without emitting data if that parameter is on or after today's date.
 E_O_USAGE
 
-    exit "${2}";
+  exit "${2}"
 }
 
 if [ "$#" -eq 1 ]; then
@@ -53,4 +53,4 @@ gem=${1}
 versionsurl="https://rubygems.org/api/v1/versions/${gem}.json"
 
 curl -sf "${versionsurl}" |
-${jq} -r "include \"pkg\"; makegemrows(\"${gem}\") | .[] | @csv"
+  ${jq} -r "include \"pkg\"; makegemrows(\"${gem}\") | .[] | @csv"