updated v0.1.8

- Multipart encoder using requests toolbelt for streaming upload
- Changed manifest upload methodology to match changes in earthengine-api

.gitignore

@@ -21,3 +21,5 @@ pack/geeup/__init__.py
 pack/dist/geeup-0.1.6.tar.gz
 pack/dist/geeup-0.1.7-py2.py3-none-any.whl
 pack/dist/geeup-0.1.7.tar.gz
+pack/dist/geeup-0.1.8.tar.gz
+pack/dist/geeup-0.1.8-py2.py3-none-any.whl

README.md

@@ -1,13 +1,13 @@
 # geeup: Simple CLI for Earth Engine Uploads with Selenium Support   [![Tweet](https://img.shields.io/twitter/url/http/shields.io.svg?style=social)](https://twitter.com/intent/tweet?text=Use%20porder%20CLI%20with%20@planetlabs%20new%20ordersv2%20API&url=https://github.com/samapriya/geeup)
 
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2653281.svg)](https://doi.org/10.5281/zenodo.2653281)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2527157.svg)](https://doi.org/10.5281/zenodo.2527157)
 [![PyPI version](https://badge.fury.io/py/geeup.svg)](https://badge.fury.io/py/geeup)
 ![Build Status](https://img.shields.io/badge/dynamic/json.svg?label=downloads&url=https%3A%2F%2Fpypistats.org%2Fapi%2Fpackages%2Fgeeup%2Frecent%3Fperiod%3Dmonth&query=%24.data.last_month&colorB=blue&suffix=%2fmonth)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 This tool came of the simple need to handle batch uploads of both image assets to collections but also thanks to the new table feature the possibility of batch uploading shapefiles into a folder. Though a lot of these tools including batch image uploader is part of my other project [geeadd](https://github.com/samapriya/gee_asset_manager_addon) which also includes additional features to add to the python CLI, this tool was designed to be minimal so as to allow the user to simply query their quota, upload images or tables and also to query ongoing tasks and delete assets. I am hoping this tool with a simple objective proves useful to a few users of Google Earth Engine.
 
-If you find this tool useful, star and cite it as below
+-If you find this tool useful, star and cite it as below
 
 ```
 Samapriya Roy. (2019, April 29). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (Version 0.1.6). Zenodo.
@@ -225,6 +225,11 @@ optional arguments:
 ```
 # Changelog
 
+### v0.1.8
+
+- Multipart encoder using requests toolbelt for streaming upload
+- Changed manifest upload methodology to match changes in earthengine-api
+
 ### v0.1.6
 
 - Fixed issue with [module locations](https://github.com/samapriya/geeup/issues/2)

geeup/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = 'Samapriya Roy'
 __email__ = '[email protected]'
-__version__ = '0.1.6'
+__version__ = '0.1.8'

geeup/batch_uploader.py

@@ -67,7 +67,7 @@
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
-#from google.cloud import storage
+from requests_toolbelt import MultipartEncoder
 from metadata_loader import load_metadata_from_csv, validate_metadata_from_csv
 os.chdir(os.path.dirname(os.path.realpath(__file__)))
 lp=os.path.dirname(os.path.realpath(__file__))
@@ -281,11 +281,16 @@ def __get_upload_url(session):
 @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3)
 def __upload_file_gee(session, file_path):
     with open(file_path, 'rb') as f:
+        file_name=os.path.basename(file_path)
         upload_url = __get_upload_url(session)
         files = {'file': f}
-        resp = session.post(upload_url, files=files)
-        gsid = resp.json()[0]
-        return gsid
+        m=MultipartEncoder( fields={'image_file':(file_name, f)})
+        try:
+            resp = session.post(upload_url, data=m, headers={'Content-Type': m.content_type})
+            gsid = resp.json()[0]
+            return gsid
+        except Exception as e:
+            print(e)
 
 @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3)
 def __upload_file_gcs(storage_client, bucket_name, image_path):

geeup/metadata_ingest.py

@@ -43,13 +43,16 @@
 - Removed multipart upload
 - Added poster for streaming upload
 '''
+
 import ast
 import csv
 import getpass
 import glob
 import logging
 import os
 import sys
+lp=os.path.dirname(os.path.realpath(__file__))
+sys.path.append(lp)
 import time
 import subprocess
 import json
@@ -68,9 +71,8 @@
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
-os.chdir(os.path.dirname(os.path.realpath(__file__)))
-lp=os.path.dirname(os.path.realpath(__file__))
-sys.path.append(lp)
+from requests_toolbelt import MultipartEncoder
+
 ee.Initialize()
 def selupload(user, source_path, destination_path, manifest=None,metadata_path=None, nodata_value=None, bucket_name=None):
     submitted_tasks_id = {}
@@ -177,7 +179,7 @@ def selupload(user, source_path, destination_path, manifest=None,metadata_path=N
                         json_data = json.dumps(data)
                         with open(os.path.join(lp,'data.json'), 'w') as outfile:
                             json.dump(data, outfile)
-                        subprocess.call("earthengine upload_manifest "+'"'+os.path.join(lp,'data.json')+'"',shell=True)
+                        subprocess.call("earthengine upload image --manifest "+'"'+os.path.join(lp,'data.json')+'"',shell=True)
         except Exception as e:
             print('Upload of '+str(filename)+' has failed.')
             failed_asset_writer.writerow([filename, 0, str(e)])
@@ -314,11 +316,16 @@ def __get_upload_url(session):
 @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3)
 def __upload_file_gee(session, file_path):
     with open(file_path, 'rb') as f:
+        file_name=os.path.basename(file_path)
         upload_url = __get_upload_url(session)
         files = {'file': f}
-        resp = session.post(upload_url, files=files)
-        gsid = resp.json()[0]
-        return gsid
+        m=MultipartEncoder( fields={'image_file':(file_name, f)})
+        try:
+            resp = session.post(upload_url, data=m, headers={'Content-Type': m.content_type})
+            gsid = resp.json()[0]
+            return gsid
+        except Exception as e:
+            print(e)
 
 @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3)
 def __upload_file_gcs(storage_client, bucket_name, image_path):

geeup/sel_tuploader.py

@@ -28,6 +28,7 @@
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
+from requests_toolbelt import MultipartEncoder
 import time,os,getpass,subprocess
 
 
@@ -78,15 +79,18 @@ def seltabup(dirc,uname,destination):
                 d = ast.literal_eval(r.text)
                 upload_url = d['url']
                 file_path=os.path.join(dirc,item)
+                file_name=os.path.basename(file_path)
                 with open(file_path, 'rb') as f:
                     upload_url = d['url']
-                    files = {'file': f}
-                    resp = s.post(upload_url, files=files)
-                    gsid = resp.json()[0]
-                    asset_full_path=destination+'/'+item.split('.')[0]
-                    #print(asset_full_path)
-                    output=subprocess.check_output('earthengine upload table --asset_id '+str(asset_full_path)+' '+str(gsid),shell=True)
-                    print('Ingesting '+str(i)+' of '+str(file_count)+' '+str(os.path.basename(asset_full_path))+' task ID: '+str(output).strip())
+                    try:
+                        m=MultipartEncoder( fields={'zip_file':(file_name, f)})
+                        resp = session.post(upload_url, data=m, headers={'Content-Type': m.content_type})
+                        gsid = resp.json()[0]
+                        asset_full_path=destination+'/'+item.split('.')[0]
+                        output=subprocess.check_output('earthengine upload table --asset_id '+str(asset_full_path)+' '+str(gsid),shell=True)
+                        print('Ingesting '+str(i)+' of '+str(file_count)+' '+str(os.path.basename(asset_full_path))+' task ID: '+str(output).strip())
+                    except Exception as e:
+                        print(e)
                     i=i+1
     except Exception as e:
         print(e)

setup.py

@@ -18,7 +18,7 @@ def readme():
         return f.read()
 setuptools.setup(
     name='geeup',
-    version='0.1.6',
+    version='0.1.8',
     packages=find_packages(),
     url='https://github.com/samapriya/geeup',
     install_requires=['earthengine_api >= 0.1.87','requests >= 2.10.0','retrying >= 1.3.3','beautifulsoup4 >= 4.5.1','pandas>=0.23.0','psutil>=5.4.5',