From cf6373fd479ab1e44880cac38ce200b2d09226d6 Mon Sep 17 00:00:00 2001 From: Simon Mathis Date: Wed, 13 Jul 2022 21:23:38 +0200 Subject: [PATCH 1/7] [hotfix] change default requirements to production requirements --- requirements/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index f563457..9bcbc5d 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -6,4 +6,4 @@ dependencies: - python=3.8 - pip - pip: - - -r dev-requirements.txt + - -r requirements.txt From 5dcdd4076e4e2e6d61740c14359a80702c61b2c6 Mon Sep 17 00:00:00 2001 From: Simon Mathis Date: Fri, 23 Sep 2022 11:59:45 +0100 Subject: [PATCH 2/7] [feature] add optional mamba support in makefile --- Makefile | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 0e09b12..3a8440f 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,12 @@ else HAS_CONDA=True endif +ifeq (,$(shell which mamba)) +HAS_MAMBA=False +else +HAS_MAMBA=True +endif + ################################################################################# # COMMANDS # ################################################################################# @@ -30,6 +36,7 @@ lint: ## Format geograph directory using black format: + isort geograph black geograph ## Set up pre-commit hooks @@ -39,20 +46,22 @@ precommit: ## Set up python interpreter environment and install basic dependencies env: -ifeq (True,$(HAS_CONDA)) - @echo ">>> Detected conda, creating conda environment." +ifeq (True, $(HAS_MAMBA)) + @echo ">>> Detected mamba, creating conda environment via mamba." + + # Create the conda environment + mamba env create --prefix=./env -f requirements/environment.yml + @echo ">>> New mamba env created. Activate from project directory with:\nconda activate ./env" +else ifeq (True,$(HAS_CONDA)) + @echo ">>> Detected conda, creating conda environment." + # Create the conda environment conda env create --prefix=./env -f requirements/environment.yml @echo ">>> New conda env created. Activate from project directory with:\nconda activate ./env" else - @echo ">>> No conda detected. Falling back to virtualenv instead. The python verison will be that of your python3 interpreter." - $(PYTHON_INTERPRETER) -m pip install -q virtualenv virtualenvwrapper - @echo ">>> Installing virtualenvwrapper if not already installed.\nMake sure the following lines are in shell startup file\n\ - export WORKON_HOME=$$HOME/.virtualenvs\nexport PROJECT_HOME=$$HOME/Devel\nsource /usr/local/bin/virtualenvwrapper.sh\n" - @bash -c "source `which virtualenvwrapper.sh`;mkvirtualenv $(PROJECT_NAME) --python=$(PYTHON_INTERPRETER)" - @echo ">>> New virtualenv created. Activate with:\nworkon $(PROJECT_NAME)" + @echo ">>> No conda detected. Please install conda or manually install requirements in your preferred python version." endif ################################################################################# From adb232df0c9ddf67838b5b48a5d91db35710d85c Mon Sep 17 00:00:00 2001 From: Simon Mathis Date: Fri, 23 Sep 2022 13:47:26 +0100 Subject: [PATCH 3/7] [fix] bump binder environment and fix dependencies to fix binder bugs --- binder/environment.yml | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/binder/environment.yml b/binder/environment.yml index 5a9f837..49f7699 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -1,8 +1,38 @@ +prefix: ./env channels: - conda-forge - defaults dependencies: - - python=3.8 - - pip + - python=3.10 + - pip==21.2.4 + # linear algebra and general data analysis + - numpy ==1.23.3 # arrays, linear algebra + - pandas ==1.5.0 # tabular data analysis + + # plotting + - folium ==0.12.1.post1 # plotting maps + - ipyleaflet ==0.17.1 # plotting ipywidget maps + + # interactive computing + - tqdm ==4.64.1 # progress bars + + # geospatial analysis requirements + # vector data + - geopandas ==0.11.1 # manipulating geospatial vector data + - shapely ==1.8.4 # working with vector shapes + - rtree ==1.0.0 # efficiently querying polygon data + # raster data + - rasterio ==1.3.2 # opening and loading raster data + - xarray ==2022.6.0 # useful data structures + + # graph requirements + - networkx ==2.8.6 # manipulating graph data + + # For binder + - rioxarray ==0.12.2 + - seaborn ==0.12.0 + - netcdf4 ==1.6.1 + - pip: - - -r requirements.txt + - pylandstats ==2.4.2 + - -e .. From 343e1ffbf04b607ae6dafc36c59546869f8389b7 Mon Sep 17 00:00:00 2001 From: Simon Mathis Date: Fri, 23 Sep 2022 14:02:52 +0100 Subject: [PATCH 4/7] [fix] fix problems with default requirements --- requirements/environment.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index 9bcbc5d..367f3d9 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -3,7 +3,30 @@ channels: - conda-forge - defaults dependencies: - - python=3.8 + - python=3.10 - pip + # linear algebra and general data analysis + - numpy 1.* # arrays, linear algebra + - pandas 1.5.* # tabular data analysis + + # plotting + - folium ==0.12.1.post1 # plotting maps + - ipyleaflet # plotting ipywidget maps + + # interactive computing + - tqdm # progress bars + + # geospatial analysis requirements + # vector data + - geopandas # manipulating geospatial vector data + - shapely # working with vector shapes + - rtree # efficiently querying polygon data + # raster data + - rasterio # opening and loading raster data + - xarray # useful data structures + + # graph requirements + - networkx # manipulating graph data + - pip: - - -r requirements.txt + - -e .. From d8b1df83c4a549163cef53c838788c778297911f Mon Sep 17 00:00:00 2001 From: Simon Mathis Date: Fri, 23 Sep 2022 14:22:42 +0100 Subject: [PATCH 5/7] [temp] remove rasterio pin to 1.1.8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 56abad2..460ebc8 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "geopandas", "shapely", "rtree", - "rasterio==1.1.8", + "rasterio", "xarray", "networkx", ] From efbf476d0dc6f0f6fe5ca469a0d6c96645804a25 Mon Sep 17 00:00:00 2001 From: Simon Mathis Date: Fri, 23 Sep 2022 15:10:45 +0100 Subject: [PATCH 6/7] [fix] loose fix requirements for main version --- binder/requirements.txt | 38 ------------------------------------ requirements/environment.yml | 18 ++++++++--------- 2 files changed, 9 insertions(+), 47 deletions(-) delete mode 100644 binder/requirements.txt diff --git a/binder/requirements.txt b/binder/requirements.txt deleted file mode 100644 index cdbd27d..0000000 --- a/binder/requirements.txt +++ /dev/null @@ -1,38 +0,0 @@ -# NOTE: Your final requirements for production/publishing the repo go here. - -# local package --e ./.. - -# linear algebra and general data analysis -numpy==1.20.1 # arrays, linear algebra -pandas==1.2.3 # tabular data analysis - -# interactive computing -jupyter==1.0.0 # for opening jupyter notebooks in binder -tqdm==4.59.0 # progress bars - -# plotting -matplotlib==3.3.4 # general python plotting -seaborn==0.11.1 # fancier plotting styles # TODO: Check for removal -folium==0.12.1 # plotting maps -ipyleaflet==0.13.6 # plotting ipywidget maps - -# geospatial analysis requirements -# vector data -geopandas==0.9.0 # manipulating geospatial vector data -shapely==1.7.1 # working with vector shapes -rtree==0.9.7 # efficiently querying polygon data -descartes==1.1.0 # plotting geopandas vector data -# raster data -rasterio==1.1.8 # opening and loading raster data (Note: version >= 1.2 requires unsetting PROJ_LIB environment variable which is set by fiona -xarray==0.17.0 # useful data structures -rioxarray==0.3.1 # adaptation of xarray for raterio. # TODO: Check for removal - -# graph requirements -networkx==2.5 # manipulating graph data - -# comparison to fragmentation metrics in binder -pylandstats==2.2.1 - -# saving and loading netcdf4 files -netcdf4 \ No newline at end of file diff --git a/requirements/environment.yml b/requirements/environment.yml index 367f3d9..0c2b0db 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -10,23 +10,23 @@ dependencies: - pandas 1.5.* # tabular data analysis # plotting - - folium ==0.12.1.post1 # plotting maps - - ipyleaflet # plotting ipywidget maps + - folium 0.12.* # plotting maps + - ipyleaflet 0.* # plotting ipywidget maps # interactive computing - - tqdm # progress bars + - tqdm 4.* # progress bars # geospatial analysis requirements # vector data - - geopandas # manipulating geospatial vector data - - shapely # working with vector shapes - - rtree # efficiently querying polygon data + - geopandas 0.11.* # manipulating geospatial vector data + - shapely 1.* # working with vector shapes + - rtree 1.* # efficiently querying polygon data # raster data - - rasterio # opening and loading raster data - - xarray # useful data structures + - rasterio 1.3.* # opening and loading raster data + - xarray 2022.* # useful data structures # graph requirements - - networkx # manipulating graph data + - networkx 2.8.* # manipulating graph data - pip: - -e .. From 2518dffa8f3b9ee576fa9d3e0baefb582139c1b3 Mon Sep 17 00:00:00 2001 From: Simon Mathis Date: Fri, 23 Sep 2022 15:19:06 +0100 Subject: [PATCH 7/7] [clean] clean up requirements files --- .pre-commit-config.yaml | 4 +- requirements/dev-requirements.txt | 70 ++-------------------------- requirements/doc-requirements.txt | 76 +------------------------------ requirements/requirements.txt | 27 ----------- 4 files changed, 7 insertions(+), 170 deletions(-) delete mode 100644 requirements/requirements.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 538b35e..b8dd309 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,9 @@ repos: - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 22.8.0 hooks: - id: black - language_version: python3.8 + language_version: python3.10 - repo: https://github.com/pycqa/pylint rev: pylint-2.6.0 hooks: diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 4406113..b557b90 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,75 +1,11 @@ # NOTE: These requirements are used for developing code on the repo. # As a standard they include certain formatters and linters. -# local package --e ../. - # external requirements (mostly linters and formatters) -pylint # pylint linter +pylint==2.6.0 # pylint linter mypy # python type checker -black # automatic formatting provider +black==22.8.0 # automatic formatting provider pre-commit # for git precommit hooks isort # automatic import sorter python-dotenv # environment variable manager -pydocstyle # set pydocstyle - -# linear algebra and general data analysis -numpy # arrays, linear algebra -scipy # linear algebra and numerical mathematics -numba # speeding up array operations -pandas # tabular data analysis - -# plotting -matplotlib # general python plotting -seaborn # fancier plotting styles -descartes # geospatial plotting of shapefiles -folium # plotting maps -ipyleaflet # plotting ipywidget maps - -# interactive computing -jupyterlab # jupyter notebooks -tqdm # progress bars - -# geospatial analysis requirements -rasterio # opening and loading raster data -fiona # manipulating geospatial vector data -geopandas # manipulating geospatial vector data -shapely # working with vector shapes -pycrs # working with coordinate reference systems -geopy # convenient API requests to geocoders -xarray # useful data structures -rioxarray # adaptation of xarray for raterio. -dask[array] # allows to composite multiple satellite images stored in different shards -dask[dataframe] # allows more lazy operation for xarray. -dask[dataframe] # allows more lazy operation for xarray. -dask[distributed] # allows distributed computing -netCDF4 # makes sure that the default driver is netCDF4. -bottleneck # needed for fill forward (xarray.DataArray.ffil) - -# additional -networkx # manipulating graph data -rtree # rtree library - -# gdrive functionality -google-api-python-client -google-auth-httplib2 -google-auth-oauthlib - -# make videos for animating timeseries etc. -imageio -imageio-ffmpeg - -# xgboost -xgboost # gradient boosted regression -sklearn # sklearn -graphviz # can plot the decission tree - -# -pillow -torch==1.8 -torchvision -wandb -pytorch-lightning -git+https://github.com/qubvel/segmentation_models.pytorch -hydra-core -twine # for publishing to PyPI +pydocstyle # set pydocstyle \ No newline at end of file diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index f30109f..eb6fae8 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,77 +1,5 @@ -# NOTE: These requirements are used for developing code on the repo. -# As a standard they include certain formatters and linters. - -# local package -# -e ../. - -# external requirements (mostly linters and formatters) -pylint # pylint linter -mypy # python type checker -black # automatic formatting provider -pre-commit # for git precommit hooks -isort # automatic import sorter -python-dotenv # environment variable manager -pydocstyle # set pydocstyle - -# linear algebra and general data analysis -numpy # arrays, linear algebra -scipy # linear algebra and numerical mathematics -numba # speeding up array operations -pandas # tabular data analysis - -# plotting -matplotlib # general python plotting -seaborn # fancier plotting styles -descartes # geospatial plotting of shapefiles -folium # plotting maps -ipyleaflet # plotting ipywidget maps - -# interactive computing -jupyterlab # jupyter notebooks -tqdm # progress bars - -# geospatial analysis requirements -rasterio # opening and loading raster data -fiona # manipulating geospatial vector data -geopandas # manipulating geospatial vector data -shapely # working with vector shapes -pycrs # working with coordinate reference systems -geopy # convenient API requests to geocoders -xarray # useful data structures -rioxarray # adaptation of xarray for raterio. -dask[array] # allows to composite multiple satellite images stored in different shards -dask[dataframe] # allows more lazy operation for xarray. -dask[dataframe] # allows more lazy operation for xarray. -dask[distributed] # allows distributed computing -netCDF4 # makes sure that the default driver is netCDF4. -bottleneck # needed for fill forward (xarray.DataArray.ffil) - -# additional -networkx # manipulating graph data -rtree # rtree library - -# gdrive functionality -google-api-python-client -google-auth-httplib2 -google-auth-oauthlib - -# make videos for animating timeseries etc. -imageio -imageio-ffmpeg - -# xgboost -xgboost # gradient boosted regression -sklearn # sklearn -graphviz # can plot the decission tree - -# -pillow -torch==1.8 -torchvision -wandb -pytorch-lightning -git+https://github.com/qubvel/segmentation_models.pytorch -hydra-core +# NOTE: These requirements are used by readthedocs.io to autogenerate +# the documentation for this repo. #docs sphinx diff --git a/requirements/requirements.txt b/requirements/requirements.txt deleted file mode 100644 index c30b1e1..0000000 --- a/requirements/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -# NOTE: Your final requirements for production/publishing the repo go here. - -# local package --e . - -# linear algebra and general data analysis -numpy # arrays, linear algebra -pandas # tabular data analysis - -# plotting -folium # plotting maps -ipyleaflet # plotting ipywidget maps - -# interactive computing -tqdm # progress bars - -# geospatial analysis requirements -# vector data -geopandas # manipulating geospatial vector data -shapely # working with vector shapes -rtree # efficiently querying polygon data -# raster data -rasterio==1.1.8 # opening and loading raster data -xarray # useful data structures - -# graph requirements -networkx # manipulating graph data