updated v0.5.8

- Adding dependency on GDAL again to handle custom geotiffs correctly.
- Added back rename tool and improvements made in v0.5.6
- Updated language to notify users of images in a collection or tables in a folder

README.md

@@ -46,6 +46,49 @@ This assumes that you have native python & pip installed in your system, you can
 
 **This also needs earthengine cli to be [installed and authenticated on your system](https://developers.google.com/earth-engine/python_install_manual) and earthengine to be callable in your command line or terminal**
 
+**This command line tool is dependent on functionality from GDAL**
+For installing GDAL in Ubuntu
+
+```
+sudo add-apt-repository ppa:ubuntugis/ppa && sudo apt-get update
+sudo apt-get install gdal-bin
+sudo apt-get install python-gdal
+```
+
+## Windows Setup
+
+Shapely and a few other libraries are notoriously difficult to install on windows machines so follow the steps mentioned here **before installing porder**. You can download and install shapely and other libraries from the [Unofficial Wheel files from here](https://www.lfd.uci.edu/~gohlke/pythonlibs) download depending on the python version you have. **Do this only once you have install GDAL**. I would recommend the steps mentioned above to get the GDAL properly installed. However I am including instructions to using a precompiled version of GDAL similar to the other libraries on windows. You can test to see if you have gdal by simply running
+
+`gdalinfo`
+
+in your command prompt. If you get a read out and not an error message you are good to go. If you don't have gdal try Option 1,2 or 3 in that order and that will install gdal along with the other libraries
+
+#### Option 1:
+
+Starting from geeup v0.3.4 onwards:
+
+Simply run `geeup -h` after installation. This should go fetch the extra libraries you need and install them. Once installation is complete, the porder help page will show up. This should save you from the few steps below.
+
+#### Option 2:
+
+If this does not work or you get an unexpected error try the following commands. You can also use these commands if you simply want to update these libraries.
+
+```
+pipwin refresh
+pipwin install gdal
+```
+
+#### Option 3
+
+For Windows I also found this [guide](https://webcache.googleusercontent.com/search?q=cache:UZWc-pnCgwsJ:https://sandbox.idre.ucla.edu/sandbox/tutorials/installing-gdal-for-windows+&cd=4&hl=en&ct=clnk&gl=us) from UCLA
+
+Also for Ubuntu Linux I saw that this is necessary before the install
+
+`sudo apt install libcurl4-openssl-dev libssl-dev`
+**geeup now only support Python v3.7 or higher from geeup version 0.5.6**
+
+**This also needs earthengine cli to be [installed and authenticated on your system](https://developers.google.com/earth-engine/python_install_manual) and earthengine to be callable in your command line or terminal**
+
 To install **geeup: Simple CLI for Earth Engine Uploads** you can install using two methods.
 
 `pip install geeup`
@@ -291,6 +334,11 @@ optional arguments:
 
 # Changelog
 
+### 0.5.8
+- Adding dependency on GDAL again to handle custom geotiffs correctly.
+- Added back rename tool and improvements made in v0.5.6
+- Updated language to notify users of images in a collection or tables in a folder
+
 ### 0.5.7
 - Getmeta tool now generates crs and bounding box
 

docs/changelog.md

@@ -1,5 +1,10 @@
 # Changelog
 
+### 0.5.8
+- Adding dependency on GDAL again to handle custom geotiffs correctly.
+- Added back rename tool and improvements made in v0.5.6
+- Updated language to notify users of images in a collection or tables in a folder
+
 ### 0.5.7
 - Getmeta tool now generates crs and bounding box
 

geeup/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Samapriya Roy"
 __email__ = "[email protected]"
-__version__ = "0.5.7"
+__version__ = "0.5.8"

geeup/batch_uploader.py

@@ -286,7 +286,7 @@ def __find_remaining_assets_for_upload(path_to_local_assets, path_remote):
                 sys.exit(1)
             elif len(assets_left_for_upload) > 0:
                 print(
-                    f"Total of {len(assets_left_for_upload)} assets remaining : {len(set(remote_assets))} assets with {len(set(tasked_assets))} associated tasks running or submitted"
+                    f"Total of {len(assets_left_for_upload)} assets remaining : Total of {len(set(remote_assets))} already in collection with {len(set(tasked_assets))} associated tasks running or submitted"
                 )
 
             assets_left_for_upload_full_path = [

geeup/geeup.py

@@ -1,7 +1,8 @@
-from .zipfiles import zipshape
-from .tuploader import tabup
-from .getmeta import getmeta
 from .batch_uploader import upload
+from .getmeta import getmeta
+from .tuploader import tabup
+from .zipfiles import zipshape
+
 __copyright__ = """
 
     Copyright 2021 Samapriya Roy
@@ -29,6 +30,7 @@
 import sys
 import time
 import webbrowser
+from datetime import datetime
 from os.path import expanduser
 
 import ee
@@ -40,6 +42,75 @@
 
 os.chdir(os.path.dirname(os.path.realpath(__file__)))
 
+if str(platform.system().lower()) == "windows":
+    version = sys.version_info[0]
+    try:
+        import pipwin
+
+        if pipwin.__version__ == "0.5.0":
+            pass
+        else:
+            subprocess.call(
+                "python" + str(version) + " -m pip install pipwin==0.5.0", shell=True
+            )
+            subprocess.call("pipwin refresh", shell=True)
+        """Check if the pipwin cache is old: useful if you are upgrading porder on windows
+        [This section looks if the pipwin cache is older than two weeks]
+        """
+        home_dir = expanduser("~")
+        fullpath = os.path.join(home_dir, ".pipwin")
+        file_mod_time = os.stat(fullpath).st_mtime
+        if int((time.time() - file_mod_time) / 60) > 90000:
+            print("Refreshing your pipwin cache")
+            subprocess.call("pipwin refresh", shell=True)
+    except ImportError:
+        subprocess.call(
+            "python" + str(version) + " -m pip install pipwin==0.5.0", shell=True
+        )
+        subprocess.call("pipwin refresh", shell=True)
+    except Exception as e:
+        logger.exception(e)
+    try:
+        import gdal
+    except ImportError:
+        try:
+            from osgeo import gdal
+        except ModuleNotFoundError:
+            subprocess.call("pipwin install gdal", shell=True)
+    except ModuleNotFoundError or ImportError:
+        subprocess.call("pipwin install gdal", shell=True)
+    except Exception as e:
+        logger.exception(e)
+    try:
+        import pandas
+    except ImportError:
+        subprocess.call("pipwin install pandas", shell=True)
+    except Exception as e:
+        logger.exception(e)
+    try:
+        import pyproj
+    except ImportError:
+        subprocess.call("pipwin install pyproj", shell=True)
+    except Exception as e:
+        logger.exception(e)
+    try:
+        import shapely
+    except ImportError:
+        subprocess.call("pipwin install shapely", shell=True)
+    except Exception as e:
+        logger.exception(e)
+    try:
+        import fiona
+    except ImportError:
+        subprocess.call("pipwin install fiona", shell=True)
+    except Exception as e:
+        logger.exception(e)
+    try:
+        import geopandas
+    except ImportError:
+        subprocess.call("pipwin install geopandas", shell=True)
+    except Exception as e:
+        logger.exception(e)
 
 lpath = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(lpath)
@@ -123,7 +194,24 @@ def read_from_parser(args):
     readme()
 
 
+def rename(directory):
+    file_list = [file for file in os.listdir(directory)]
+    for i, file_original in enumerate(file_list):
+        file_name, file_extension = os.path.splitext(file_original)
+        string = re.sub(r'[^a-zA-Z0-9 _-]', r'', file_name)
+        string = re.sub(r"\s+", "_", string)
+        if file_original != string:
+            print(f'Renaming {file_original} to {string}{file_extension}')
+            os.rename(os.path.join(directory, file_original),
+                      os.path.join(directory, f'{string}{file_extension}'))
+
+
+def rename_from_parser(args):
+    rename(directory=args.input)
+
 # cookie setup
+
+
 def cookie_setup():
     platform_info = platform.system().lower()
     if str(platform_info) == "linux" or str(platform_info) == "darwin":
@@ -275,21 +363,47 @@ def tabup_from_parser(args):
     )
 
 
-def tasks():
+def tasks(state):
     ee.Initialize()
-    statuses = ee.data.listOperations()
-    st = []
-    for status in statuses:
-        st.append(status["metadata"]["state"])
-    print(f"Tasks Running: {st.count('RUNNING')}")
-    print(f"Tasks Pending: {st.count('PENDING')}")
-    print(f"Tasks Completed: {st.count('SUCCEEDED')}")
-    print(f"Tasks Failed: {st.count('FAILED')}")
-    print(f"Tasks Cancelled: {st.count('CANCELLED') + st.count('CANCELLING')}")
+    if state is not None:
+        task_bundle = []
+        operations = [status
+                      for status in ee.data.listOperations() if status["metadata"]["state"] == state]
+        for operation in operations:
+            task_id = operation['name'].split('/')[-1]
+            description = operation['metadata']['description'].split(
+                ':')[-1].strip().replace('"', '')
+            op_type = operation['metadata']['type']
+            attempt_count = str(operation['metadata']['attempt'])
+            start = datetime.strptime(
+                operation['metadata']["startTime"], "%Y-%m-%dT%H:%M:%S.%fZ")
+            end = datetime.strptime(
+                operation['metadata']["updateTime"], "%Y-%m-%dT%H:%M:%S.%fZ")
+            time_difference = end-start
+            item = {
+                "task_id": task_id,
+                "operation_type": op_type,
+                "description/path": description,
+                "time_difference": str(time_difference),
+                "attempt": attempt_count
+            }
+            task_bundle.append(item)
+        print(json.dumps(task_bundle, indent=2))
+    else:
+        statuses = ee.data.listOperations()
+        st = []
+        for status in statuses:
+            st.append(status["metadata"]["state"])
+        print(f"Tasks Running: {st.count('RUNNING')}")
+        print(f"Tasks Pending: {st.count('PENDING')}")
+        print(f"Tasks Completed: {st.count('SUCCEEDED')}")
+        print(f"Tasks Failed: {st.count('FAILED')}")
+        print(
+            f"Tasks Cancelled: {st.count('CANCELLED') + st.count('CANCELLING')}")
 
 
 def tasks_from_parser(args):
-    tasks()
+    tasks(state=args.state)
 
 
 def cancel_tasks(tasks):
@@ -417,6 +531,19 @@ def main(args=None):
     )
     parser_quota.set_defaults(func=quota_from_parser)
 
+    parser_rename = subparsers.add_parser(
+        "rename",
+        help="Renames filename to adhere to EE naming rules: Caution this is in place renaming",
+    )
+    required_named = parser_rename.add_argument_group(
+        "Required named arguments.")
+    required_named.add_argument(
+        "--input",
+        help="Path to the input directory with all files to be uploaded",
+        required=True,
+    )
+    parser_rename.set_defaults(func=rename_from_parser)
+
     parser_zipshape = subparsers.add_parser(
         "zipshape",
         help="Zips all shapefiles and subsidary files in folder into individual zip files",
@@ -523,6 +650,12 @@ def main(args=None):
         "tasks",
         help="Queries current task status [completed,running,ready,failed,cancelled]",
     )
+    optional_named = parser_tasks.add_argument_group(
+        "Optional named arguments")
+    optional_named.add_argument(
+        "--state",
+        help="Query by state type SUCCEEDED|PENDING|RUNNING|FAILED",
+    )
     parser_tasks.set_defaults(func=tasks_from_parser)
 
     parser_cancel = subparsers.add_parser(

geeup/getmeta.py

@@ -3,37 +3,41 @@
 import csv
 import os
 
-from geotiff import GeoTiff
-
 
 def getmeta(indir, mfile):
+    try:
+        from osgeo import gdal
+    except ImportError:
+        import gdal
     i = 1
     flength = len([name for name in os.listdir(
         indir) if name.endswith(".tif")])
     with open(mfile, "w") as csvfile:
         writer = csv.DictWriter(
             csvfile,
-            fieldnames=["id_no", "crs", "bbox"],
+            fieldnames=["id_no", "xsize", "ysize", "num_bands"],
             delimiter=",",
             lineterminator="\n",
         )
         writer.writeheader()
     for filename in os.listdir(indir):
         if filename.endswith(".tif"):
-            gtif = GeoTiff(os.path.join(indir, filename))
+            gtif = gdal.Open(os.path.join(indir, filename))
             try:
                 print("Processed: " + str(i) + " of " + str(flength), end="\r")
-                fname = os.path.basename(filename).split(".tif")[0]
-                crs = gtif.as_crs
-                bbox = gtif.tif_bBox_wgs_84
+                fname = os.path.basename(
+                    gtif.GetDescription()).split(".tif")[0]
+                xsize = gtif.RasterXSize
+                ysize = gtif.RasterYSize
+                bsize = gtif.RasterCount
                 with open(mfile, "a") as csvfile:
                     writer = csv.writer(
                         csvfile, delimiter=",", lineterminator="\n")
-                    writer.writerow([fname, crs, bbox])
+                    writer.writerow([fname, xsize, ysize, bsize])
                 csvfile.close()
                 i = i + 1
-            except Exception as error:
-                print(error)
+            except Exception as e:
+                print(e)
                 i = i + 1
 
 

geeup/tuploader.py

@@ -187,7 +187,7 @@ def tabup(dirc, uname, destination, x, y):
         set(gee_table_exists), set(tasked_assets))
     if len(diff_set) > 0:
         print(
-            f"Total of {len(diff_set)} assets remaining : {len(set(gee_table_exists))} assets with {len(set(tasked_assets))} associated tasks running or submitted"
+            f"Total of {len(diff_set)} assets remaining : Total of {len(set(gee_table_exists))} already in folder with {len(set(tasked_assets))} associated tasks running or submitted"
         )
         status = ["RUNNING", "PENDING"]
         task_count = len(

requirements.txt

@@ -2,7 +2,6 @@ earthengine_api>=0.1.274
 logzero>=1.5.0
 requests >= 2.22.0
 cerberus>=1.3.4
-geotiff>=0.2.4
 retrying >= 1.3.3
 natsort >= 8.1.0
 beautifulsoup4 >= 4.9.0
@@ -12,6 +11,5 @@ future >= 0.16.0
 google-cloud-storage >= 1.1.1
 pandas>=0.23.0
 psutil>=5.4.5
-pySmartDL>=1.2.5
 pathlib>=1.0.1
 lxml>=4.1.1