From d685327e44b43ad26f9c8e8d547f06f13335b42d Mon Sep 17 00:00:00 2001 From: samapriya Date: Wed, 8 May 2019 23:10:24 -0400 Subject: [PATCH] updated v0.1.8 - Multipart encoder using requests toolbelt for streaming upload - Changed manifest upload methodology to match changes in earthengine-api --- .gitignore | 2 ++ README.md | 9 +++++++-- geeup/__init__.py | 2 +- geeup/batch_uploader.py | 13 +++++++++---- geeup/metadata_ingest.py | 21 ++++++++++++++------- geeup/sel_tuploader.py | 18 +++++++++++------- setup.py | 2 +- 7 files changed, 45 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 3d0cf34..3ec15ed 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ pack/geeup/__init__.py pack/dist/geeup-0.1.6.tar.gz pack/dist/geeup-0.1.7-py2.py3-none-any.whl pack/dist/geeup-0.1.7.tar.gz +pack/dist/geeup-0.1.8.tar.gz +pack/dist/geeup-0.1.8-py2.py3-none-any.whl diff --git a/README.md b/README.md index dfa20ba..e8d9ecb 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # geeup: Simple CLI for Earth Engine Uploads with Selenium Support   [![Tweet](https://img.shields.io/twitter/url/http/shields.io.svg?style=social)](https://twitter.com/intent/tweet?text=Use%20porder%20CLI%20with%20@planetlabs%20new%20ordersv2%20API&url=https://github.com/samapriya/geeup) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2653281.svg)](https://doi.org/10.5281/zenodo.2653281) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2527157.svg)](https://doi.org/10.5281/zenodo.2527157) [![PyPI version](https://badge.fury.io/py/geeup.svg)](https://badge.fury.io/py/geeup) ![Build Status](https://img.shields.io/badge/dynamic/json.svg?label=downloads&url=https%3A%2F%2Fpypistats.org%2Fapi%2Fpackages%2Fgeeup%2Frecent%3Fperiod%3Dmonth&query=%24.data.last_month&colorB=blue&suffix=%2fmonth) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) This tool came of the simple need to handle batch uploads of both image assets to collections but also thanks to the new table feature the possibility of batch uploading shapefiles into a folder. Though a lot of these tools including batch image uploader is part of my other project [geeadd](https://github.com/samapriya/gee_asset_manager_addon) which also includes additional features to add to the python CLI, this tool was designed to be minimal so as to allow the user to simply query their quota, upload images or tables and also to query ongoing tasks and delete assets. I am hoping this tool with a simple objective proves useful to a few users of Google Earth Engine. -If you find this tool useful, star and cite it as below +-If you find this tool useful, star and cite it as below ``` Samapriya Roy. (2019, April 29). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (Version 0.1.6). Zenodo. @@ -225,6 +225,11 @@ optional arguments: ``` # Changelog +### v0.1.8 + +- Multipart encoder using requests toolbelt for streaming upload +- Changed manifest upload methodology to match changes in earthengine-api + ### v0.1.6 - Fixed issue with [module locations](https://github.com/samapriya/geeup/issues/2) diff --git a/geeup/__init__.py b/geeup/__init__.py index 0d61c3e..89fb8fa 100644 --- a/geeup/__init__.py +++ b/geeup/__init__.py @@ -2,4 +2,4 @@ __author__ = 'Samapriya Roy' __email__ = 'samapriya.roy@gmail.com' -__version__ = '0.1.6' +__version__ = '0.1.8' diff --git a/geeup/batch_uploader.py b/geeup/batch_uploader.py index 647d469..d049815 100644 --- a/geeup/batch_uploader.py +++ b/geeup/batch_uploader.py @@ -67,7 +67,7 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By -#from google.cloud import storage +from requests_toolbelt import MultipartEncoder from metadata_loader import load_metadata_from_csv, validate_metadata_from_csv os.chdir(os.path.dirname(os.path.realpath(__file__))) lp=os.path.dirname(os.path.realpath(__file__)) @@ -281,11 +281,16 @@ def __get_upload_url(session): @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3) def __upload_file_gee(session, file_path): with open(file_path, 'rb') as f: + file_name=os.path.basename(file_path) upload_url = __get_upload_url(session) files = {'file': f} - resp = session.post(upload_url, files=files) - gsid = resp.json()[0] - return gsid + m=MultipartEncoder( fields={'image_file':(file_name, f)}) + try: + resp = session.post(upload_url, data=m, headers={'Content-Type': m.content_type}) + gsid = resp.json()[0] + return gsid + except Exception as e: + print(e) @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3) def __upload_file_gcs(storage_client, bucket_name, image_path): diff --git a/geeup/metadata_ingest.py b/geeup/metadata_ingest.py index 8c402f5..56a108a 100644 --- a/geeup/metadata_ingest.py +++ b/geeup/metadata_ingest.py @@ -43,6 +43,7 @@ - Removed multipart upload - Added poster for streaming upload ''' + import ast import csv import getpass @@ -50,6 +51,8 @@ import logging import os import sys +lp=os.path.dirname(os.path.realpath(__file__)) +sys.path.append(lp) import time import subprocess import json @@ -68,9 +71,8 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By -os.chdir(os.path.dirname(os.path.realpath(__file__))) -lp=os.path.dirname(os.path.realpath(__file__)) -sys.path.append(lp) +from requests_toolbelt import MultipartEncoder + ee.Initialize() def selupload(user, source_path, destination_path, manifest=None,metadata_path=None, nodata_value=None, bucket_name=None): submitted_tasks_id = {} @@ -177,7 +179,7 @@ def selupload(user, source_path, destination_path, manifest=None,metadata_path=N json_data = json.dumps(data) with open(os.path.join(lp,'data.json'), 'w') as outfile: json.dump(data, outfile) - subprocess.call("earthengine upload_manifest "+'"'+os.path.join(lp,'data.json')+'"',shell=True) + subprocess.call("earthengine upload image --manifest "+'"'+os.path.join(lp,'data.json')+'"',shell=True) except Exception as e: print('Upload of '+str(filename)+' has failed.') failed_asset_writer.writerow([filename, 0, str(e)]) @@ -314,11 +316,16 @@ def __get_upload_url(session): @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3) def __upload_file_gee(session, file_path): with open(file_path, 'rb') as f: + file_name=os.path.basename(file_path) upload_url = __get_upload_url(session) files = {'file': f} - resp = session.post(upload_url, files=files) - gsid = resp.json()[0] - return gsid + m=MultipartEncoder( fields={'image_file':(file_name, f)}) + try: + resp = session.post(upload_url, data=m, headers={'Content-Type': m.content_type}) + gsid = resp.json()[0] + return gsid + except Exception as e: + print(e) @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3) def __upload_file_gcs(storage_client, bucket_name, image_path): diff --git a/geeup/sel_tuploader.py b/geeup/sel_tuploader.py index 51ce0f7..dbac61a 100644 --- a/geeup/sel_tuploader.py +++ b/geeup/sel_tuploader.py @@ -28,6 +28,7 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By +from requests_toolbelt import MultipartEncoder import time,os,getpass,subprocess @@ -78,15 +79,18 @@ def seltabup(dirc,uname,destination): d = ast.literal_eval(r.text) upload_url = d['url'] file_path=os.path.join(dirc,item) + file_name=os.path.basename(file_path) with open(file_path, 'rb') as f: upload_url = d['url'] - files = {'file': f} - resp = s.post(upload_url, files=files) - gsid = resp.json()[0] - asset_full_path=destination+'/'+item.split('.')[0] - #print(asset_full_path) - output=subprocess.check_output('earthengine upload table --asset_id '+str(asset_full_path)+' '+str(gsid),shell=True) - print('Ingesting '+str(i)+' of '+str(file_count)+' '+str(os.path.basename(asset_full_path))+' task ID: '+str(output).strip()) + try: + m=MultipartEncoder( fields={'zip_file':(file_name, f)}) + resp = session.post(upload_url, data=m, headers={'Content-Type': m.content_type}) + gsid = resp.json()[0] + asset_full_path=destination+'/'+item.split('.')[0] + output=subprocess.check_output('earthengine upload table --asset_id '+str(asset_full_path)+' '+str(gsid),shell=True) + print('Ingesting '+str(i)+' of '+str(file_count)+' '+str(os.path.basename(asset_full_path))+' task ID: '+str(output).strip()) + except Exception as e: + print(e) i=i+1 except Exception as e: print(e) diff --git a/setup.py b/setup.py index 21e876e..1356072 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def readme(): return f.read() setuptools.setup( name='geeup', - version='0.1.6', + version='0.1.8', packages=find_packages(), url='https://github.com/samapriya/geeup', install_requires=['earthengine_api >= 0.1.87','requests >= 2.10.0','retrying >= 1.3.3','beautifulsoup4 >= 4.5.1','pandas>=0.23.0','psutil>=5.4.5',