Move to poetry for dep management, ruff formatting/linting, bump all dependencies

.github/workflows/acronym-ci.yml

@@ -15,9 +15,10 @@ jobs:
     name: acronym-lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
+          ruby-version: '.ruby-version'
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
       - name: Run csvlint on acronyms file
         run: bundle exec csvlint acronyms.csv

.github/workflows/script-ci.yml

@@ -18,12 +18,12 @@ jobs:
     name: build and test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version-file: '.python-version' # Read python version from a file
-      - name: Install dependencies
-        run: make python-install
+      - name: Set up poetry
+        uses: abatilo/actions-poetry@v3
       - name: Run tests
         run: make test

.gitignore

@@ -16,3 +16,5 @@ __pycache__/
 # test results
 results/
 .coverage
+
+.install.stamp

.python-version

@@ -1 +1 @@
-3.10.12
+3.12.4

.ruby-version

@@ -1 +1 @@
-3.2.2
+3.2.4

Gemfile

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
 
 source "https://rubygems.org"
-gem 'csvlint'
-
+gem 'csvlint', "~>1.3.0"

Gemfile.lock

@@ -1,18 +1,20 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.7.2)
+    activesupport (7.0.8.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.5)
-      public_suffix (>= 2.0.2, < 6.0)
-    concurrent-ruby (1.2.2)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
+    concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
-    csvlint (1.2.0)
-      activesupport
+    csv (3.3.0)
+    csvlint (1.3.0)
+      activesupport (< 7.1.0)
       addressable
+      csv
       escape_utils
       net-http-persistent
       open_uri_redirections
@@ -24,31 +26,48 @@ GEM
     escape_utils (1.3.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.15.5)
-    i18n (1.14.1)
+    ffi (1.17.0)
+    ffi (1.17.0-aarch64-linux-gnu)
+    ffi (1.17.0-aarch64-linux-musl)
+    ffi (1.17.0-arm-linux-gnu)
+    ffi (1.17.0-arm-linux-musl)
+    ffi (1.17.0-arm64-darwin)
+    ffi (1.17.0-x86-linux-gnu)
+    ffi (1.17.0-x86-linux-musl)
+    ffi (1.17.0-x86_64-darwin)
+    ffi (1.17.0-x86_64-linux-gnu)
+    ffi (1.17.0-x86_64-linux-musl)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    minitest (5.19.0)
+    minitest (5.24.0)
     net-http-persistent (4.0.2)
       connection_pool (~> 2.2)
     open_uri_redirections (0.2.1)
-    public_suffix (5.0.3)
-    rack (3.0.9.1)
+    public_suffix (6.0.0)
+    rack (3.1.3)
     rainbow (3.1.1)
-    thor (1.2.2)
-    typhoeus (1.4.0)
+    thor (1.3.1)
+    typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     uri_template (0.7.0)
 
 PLATFORMS
-  x86_64-darwin-20
-  x86_64-darwin-21
-  x86_64-darwin-22
-  x86_64-linux
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
+  ruby
+  x86-linux-gnu
+  x86-linux-musl
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
 
 DEPENDENCIES
-  csvlint
+  csvlint (~> 1.3.0)
 
 BUNDLED WITH
-   2.3.17
+   2.5.14

Makefile

@@ -1,97 +1,47 @@
-# Makefile for Performance dashboard
+# Makefile for acronyms
 # Tested with GNU Make 3.81
 MAKEFLAGS += --warn-undefined-variables
 SHELL        	:= /usr/bin/env bash -e
-CI_ARG      	:= $(CI)
 .DEFAULT_GOAL := help
 
+INSTALL_STAMP := .install.stamp
+POETRY := $(shell command -v poetry 2> /dev/null)
 # cribbed from https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html and https://news.ycombinator.com/item?id=11195539
 help:  ## Prints out documentation for available commands
 	@awk -F ':|##' \
 		'/^[^\t].+?:.*?##/ {\
 			printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF \
 		}' $(MAKEFILE_LIST)
 
-# spelling
-.PHONY: spelling-tool-install
-spelling-tool-install:  ## Installs spelling tools (only works on OSX for now)
-ifneq ($(CI_ARG), true)
-	./scripts/install-spelling-tools.sh
-endif
-
-.PHONY: spelling-check
-spelling-check:  ## Checks spelling with hunspell
-	@cd scripts; \
-	./check-spelling.sh
-
-## Pip / Python
-.PHONY: python-install
-# python-install recipe all has to run in a single shell because it's running inside a virtualenv
-python-install:  ## Sets up your python environment for the first time (only need to run once)
-ifeq ($(CI_ARG), true)
-	pip --no-cache-dir install -r scripts/requirements.txt -r scripts/dev-requirements.txt
-else
-	python3 -m venv ./venv ;\
-	source venv/bin/activate ;\
-	echo shell venv activated ;\
-	pip install -r scripts/requirements.txt -r scripts/dev-requirements.txt ;\
-	echo Finished python install ;\
-	echo Please activate the virtualenvironment with: ;\
-	echo source venv/bin/activate
-endif
-
-# Errors out if VIRTUAL_ENV is not defined and we aren't in a CI environment.
-.PHONY: check-env
-check-env:
-ifndef VIRTUAL_ENV
-ifneq ($(CI_ARG), true)
-	$(error VIRTUAL_ENV is undefined, meaning you aren't running in a virtual environment. Fix by running: 'source venv/bin/activate')
-endif
-endif
-
-# Because of this bug we have to turn an absolute path to relative one. Relies on the repository name
-# https://github.com/jazzband/pip-tools/issues/204
-scripts/requirements.txt: scripts/requirements.in
-	pip-compile scripts/requirements.in --output-file $@
-
-scripts/dev-requirements.txt: scripts/dev-requirements.in
-	pip-compile scripts/dev-requirements.in --output-file $@
-
-.PHONY: pip-upgrade
-pip-upgrade:  ## Upgrade all python dependencies
-	pip-compile --upgrade scripts/requirements.in --output-file scripts/requirements.txt
-	pip-compile --upgrade scripts/dev-requirements.in --output-file scripts/dev-requirements.txt
-
-SITE_PACKAGES := $(shell pip show pip | grep '^Location' | cut -f2 -d ':')
-$(SITE_PACKAGES): scripts/requirements.txt scripts/dev-requirements.txt check-env
-ifeq ($(CI_ARG), true)
-	@echo "Do nothing; assume python dependencies were installed already"
-else
-	pip-sync scripts/requirements.txt scripts/dev-requirements.txt
-endif
-
-.PHONY: pip-install
-pip-install: $(SITE_PACKAGES)
+install: $(INSTALL_STAMP)  ## Install dependencies
+$(INSTALL_STAMP): pyproject.toml poetry.lock
+	@echo $(POETRY)
+	@if [ -z $(POETRY) ]; then echo "Poetry could not be found. See https://python-poetry.org/docs/"; exit 2; fi
+	"$(POETRY)" --version
+	"$(POETRY)" install
+	touch $(INSTALL_STAMP)
 
 ## Test targets
-.PHONY: unit-test
-unit-test:  pip-install   ## Run python unit tests
-	PYTHONPATH=scripts \
-	python -m pytest -v --cov --cov-report term --cov-report xml --cov-report html
+.PHONY: test
+test: $(INSTALL_STAMP) unit-test lint  ## Run tests and lint checks
 
-.PHONY: flake8
-flake8: pip-install 	## Run Flake8 python static style checking and linting
-	@echo "flake8 comments:"
-	flake8 --statistics scripts
+.PHONY:  unit-test
+unit-test: $(INSTALL_STAMP)  ## Run python unit tests
+	"$(POETRY)" run pytest -v --cov --cov-report term --cov-report html
 
-.PHONY: test
-test: unit-test flake8   ## Run unit tests, static analysis
-	@echo "All tests passed."  # This should only be printed if all of the other targets succeed
+.PHONY: lint
+lint: $(INSTALL_STAMP)  ## Lint code base
+	"$(POETRY)" run ruff check
+	"$(POETRY)" run ruff format --check
+
+.PHONY: format
+format: $(INSTALL_STAMP)  ## Format code base
+	"$(POETRY)" run ruff check --fix
+	"$(POETRY)" run ruff format
 
 .PHONY: format-acronyms
 format-acronyms:  ## Formats acronyms file, cleaning up smart quotes and capitalization. Results written to STDOUT
-	@cd scripts; \
-	./format_acronyms.py
+	"$(POETRY)" run format_acronyms
 
 #Ruby
 .PHONY: csvlint-install
@@ -100,7 +50,7 @@ csvlint-install:  ## Install csvlint
 
 .PHONY: csvlint
 csvlint:  ## Runs csvlint on acronyms file
-	csvlint acronyms.csv
+	bundle exec csvlint acronyms.csv
 
 # Other tools
 .PHONY: dupe-acronyms
@@ -113,12 +63,18 @@ dupe-definitions:  ## prints out duplicated definitions
 	@cd scripts; \
 	./print-dupe-definitions.sh
 
+# spelling
+.PHONY: spelling-tool-install
+spelling-tool-install:  ## Installs spelling tools (only works on OSX for now)
+	./scripts/install-spelling-tools.sh
+
+.PHONY: spelling-check
+spelling-check:  ## Checks spelling with hunspell
+	@cd scripts; \
+	./check-spelling.sh
+
 .PHONY: clean
 clean: ## Delete any directories, files or logs that are auto-generated and python packages
-	rm -rf venv
-	rm -rf results
-	rm -rf .pytest_cache
-	rm -f .coverage
-	find python -name '__pycache__' -type d | xargs rm -rf
-	@echo virtualenvironment was deleted. Type 'deactivate' to deactivate the shims.
-	@echo Run 'make python-install' to reinstall the virtual environment.
+	rm -rf results .ruff_cache .pytest_cache
+	rm -f $(INSTALL_STAMP) .coverage
+	find . -type d -name "__pycache__" | xargs rm -rf {};

README.md

@@ -21,7 +21,7 @@ The terms can be cleaned up for duplicates and sorted via the cleanup script.
 Run it via `cd scripts && ./clean.sh`. An output.csv file will be generated that you can replace the acronyms.csv file with.
 
 ### csvlint
-You can (optionally) install [csvlint](https://github.com/Data-Liberation-Front/csvlint.rb) to check the format of the acronyms file. It requires Ruby v3.2.2. You can install it with `make csvlint-install` and then run it with `make csvlint`
+You can (optionally) install [csvlint](https://github.com/Data-Liberation-Front/csvlint.rb) to check the format of the acronyms file. You can install it with `make csvlint-install` and then run it with `make csvlint`
 
 ### Other cleanup scripts
 * `make format-acronyms` to fix other issues with the acronyms file, such as moving any all-lower-case or all-upper-case definition strings to title case, and turning smart quotes (e.g. “”‘’) to regular quotes. It will output to stdout.

scripts/format_acronyms.py → format_acronyms/format_acronyms.py

@@ -26,10 +26,12 @@ def fix_all_upper(row: list) -> list:
 
 # borrowed from https://stackoverflow.com/questions/40330953/dict-to-remove-smart-quotes
 def fix_smart_quotes(row: list) -> list:
-    charmap = {0x201c: u'"',  # double left
-               0x201d: u'"',  # double right
-               0x2018: u"'",  # single left
-               0x2019: u"'"}  # single right
+    charmap = {
+        0x201C: '"',  # double left
+        0x201D: '"',  # double right
+        0x2018: "'",  # single left
+        0x2019: "'",  # single right
+    }
 
     for i, elem in enumerate(row):
         row[i] = elem.translate(charmap)
@@ -44,28 +46,30 @@ def process_row(row: list) -> list:
     return row
 
 
-def main(csv_filename: str):
-    csv.register_dialect('my_format', lineterminator='\n', strict=True)
-    with open(csv_filename, newline='') as csv_file:
+def main():
+    parser = argparse.ArgumentParser(description="Fixes and outputs to stdout various issues with " + "acronyms file.")
+    parser.add_argument(
+        "-f",
+        "--file",
+        required=False,
+        action="store",
+        default="acronyms.csv",
+        help="File to use. Default: acronyms.csv",
+    )
+    args = parser.parse_args()
+    csv.register_dialect("my_format", lineterminator="\n", strict=True)
+    csv_filename = args.file
+    with open(csv_filename, newline="") as csv_file:
         csv_reader = csv.reader(csv_file)
-        csv_writer = csv.writer(sys.stdout, quoting=csv.QUOTE_MINIMAL,
-                                dialect='my_format')
-        line_count = 0
-        for row in csv_reader:
+        csv_writer = csv.writer(sys.stdout, quoting=csv.QUOTE_MINIMAL, dialect="my_format")
+
+        for line_count, row in enumerate(csv_reader):
             if line_count == 0:
                 pass
             else:
                 row = process_row(row)
-            line_count += 1
             csv_writer.writerow(row)
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description="Fixes and outputs to stdout various issues with " +
-        "acronyms file.")
-    parser.add_argument('-f', '--file', required=False, action='store',
-                        default='../acronyms.csv',
-                        help="File to use. Default: ../acronyms.csv")
-    args = parser.parse_args()
-    main(args.file)
+if __name__ == "__main__":
+    main()