From ca57c5502b6c75b980dfc4acfcc350ddc6ae6e41 Mon Sep 17 00:00:00 2001 From: samapriya Date: Wed, 2 Jan 2019 20:21:27 -0500 Subject: [PATCH] Updated v0.1.1 Fixed dependency general improvements --- README.md | 9 ++++++-- geeup/__init__.py | 2 +- geeup/batch_uploader.py | 50 ++++------------------------------------ geeup/metadata_ingest.py | 48 ++++---------------------------------- requirements.txt | 4 +++- setup.py | 6 +++-- 6 files changed, 24 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 0523763..523bb27 100644 --- a/README.md +++ b/README.md @@ -216,12 +216,17 @@ positional arguments: optional arguments: -h, --help show this help message and exit ``` - # Changelog +### v0.1.1 + +- fixed dependency issues +- Upload post issues resolved +- Removed dependency on poster for now + ### v0.0.9 -- fixed attribution and dependency issues +- fixed attribution and dependecy issues - Included poster to improve streaming uploads - All uploads now use selenium diff --git a/geeup/__init__.py b/geeup/__init__.py index 09bd9a7..32c1e77 100644 --- a/geeup/__init__.py +++ b/geeup/__init__.py @@ -2,4 +2,4 @@ __author__ = 'Samapriya Roy' __email__ = 'samapriya.roy@gmail.com' -__version__ = '0.0.9' +__version__ = '0.1.1' diff --git a/geeup/batch_uploader.py b/geeup/batch_uploader.py index ff379e4..4c9e8e9 100644 --- a/geeup/batch_uploader.py +++ b/geeup/batch_uploader.py @@ -57,9 +57,6 @@ import ee import requests import retrying -import poster -from poster.encode import multipart_encode -from poster.streaminghttp import register_openers from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver import Firefox @@ -69,7 +66,7 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By -from google.cloud import storage +#from google.cloud import storage from metadata_loader import load_metadata_from_csv, validate_metadata_from_csv os.chdir(os.path.dirname(os.path.realpath(__file__))) pathway=os.path.dirname(os.path.realpath(__file__)) @@ -280,48 +277,11 @@ def __get_upload_url(session): print(e) @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3) -def __upload_file_gee(session, file_path): +def __upload_file_gee(session, file_path, use_multipart): + with open(file_path, 'rb') as f: upload_url = __get_upload_url(session) - class IterableToFileAdapter(object): - def __init__(self, iterable): - self.iterator = iter(iterable) - self.length = iterable.total - - def read(self, size=-1): - return next(self.iterator, b'') - - def __len__(self): - return self.length - - # define a helper function simulating the interface of posters multipart_encode()-function - # but wrapping its generator with the file-like adapter - def multipart_encode_for_requests(params, boundary=None, cb=None): - datagen, headers = multipart_encode(params, boundary, cb) - return IterableToFileAdapter(datagen), headers - - - - # this is your progress callback - def progress(param, current, total): - if not param: - return - - # check out http://tcd.netinf.eu/doc/classnilib_1_1encode_1_1MultipartParam.html - # for a complete list of the properties param provides to you - calc=float(current)/float(total)*100 - print ('Uploading '+str(os.path.basename(file_path).split('.')[0])+': '+str(format(float(calc),'.2f'))+" %", end='\r') - - # generate headers and gata-generator an a requests-compatible format - # and provide our progress-callback - datagen, headers = multipart_encode_for_requests({ - "file": open(file_path, 'rb'), - "composite": "NONE", - }, cb=progress) - - # use the requests-lib to issue a post-request with out data attached - resp = session.post(upload_url, data=datagen,headers=headers) - #print(resp.content) - + files = {'file': f} + resp = session.post(upload_url, files=files) gsid = resp.json()[0] return gsid diff --git a/geeup/metadata_ingest.py b/geeup/metadata_ingest.py index ae19c2b..6d9fc0b 100644 --- a/geeup/metadata_ingest.py +++ b/geeup/metadata_ingest.py @@ -59,9 +59,6 @@ import requests import retrying from google.cloud import storage -import poster -from poster.encode import multipart_encode -from poster.streaminghttp import register_openers from metadata_loader import load_metadata_from_csv, validate_metadata_from_csv from selenium import webdriver from selenium.webdriver import Firefox @@ -312,48 +309,11 @@ def __get_upload_url(session): print(e) @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3) -def __upload_file_gee(session, file_path): +def __upload_file_gee(session, file_path, use_multipart): + with open(file_path, 'rb') as f: upload_url = __get_upload_url(session) - class IterableToFileAdapter(object): - def __init__(self, iterable): - self.iterator = iter(iterable) - self.length = iterable.total - - def read(self, size=-1): - return next(self.iterator, b'') - - def __len__(self): - return self.length - - # define a helper function simulating the interface of posters multipart_encode()-function - # but wrapping its generator with the file-like adapter - def multipart_encode_for_requests(params, boundary=None, cb=None): - datagen, headers = multipart_encode(params, boundary, cb) - return IterableToFileAdapter(datagen), headers - - - - # this is your progress callback - def progress(param, current, total): - if not param: - return - - # check out http://tcd.netinf.eu/doc/classnilib_1_1encode_1_1MultipartParam.html - # for a complete list of the properties param provides to you - calc=float(current)/float(total)*100 - print ('Uploading '+str(os.path.basename(file_path).split('.')[0])+': '+str(format(float(calc),'.2f'))+" %", end='\r') - - # generate headers and gata-generator an a requests-compatible format - # and provide our progress-callback - datagen, headers = multipart_encode_for_requests({ - "file": open(file_path, 'rb'), - "composite": "NONE", - }, cb=progress) - - # use the requests-lib to issue a post-request with out data attached - resp = session.post(upload_url, data=datagen,headers=headers) - #print(resp.content) - + files = {'file': f} + resp = session.post(upload_url, files=files) gsid = resp.json()[0] return gsid diff --git a/requirements.txt b/requirements.txt index dc0f558..a846d6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ pandas>=0.23.0 psutil>=5.4.5 poster>=0.8.1 selenium>=3.13.0 -pySmartDL>=1.2.5 \ No newline at end of file +pySmartDL>=1.2.5 +pathlib>=1.0.1 +lxml>=4.1.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 43c0756..f0606f8 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,12 @@ def readme(): return f.read() setuptools.setup( name='geeup', - version='0.0.9', + version='0.1.1', packages=find_packages(), url='https://github.com/samapriya/geeup', install_requires=['earthengine_api >= 0.1.87','requests >= 2.10.0','retrying >= 1.3.3','beautifulsoup4 >= 4.5.1','pandas>=0.23.0','psutil>=5.4.5', 'requests_toolbelt >= 0.7.0','pytest >= 3.0.0','future >= 0.16.0','google-cloud-storage >= 1.1.1','selenium>=3.13.0', - 'pySmartDL>=1.2.5','poster>=0.8.1'], + 'pySmartDL>=1.2.5','pathlib>=1.0.1','lxml>=4.1.1'], license='Apache 2.0', long_description=open('README.md').read(), long_description_content_type='text/markdown', @@ -35,6 +35,8 @@ def readme(): 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Operating System :: OS Independent', 'Topic :: Scientific/Engineering :: GIS', ),