Updated v0.1.1

Fixed dependency
general improvements

README.md

@@ -216,12 +216,17 @@ positional arguments:
 optional arguments:
   -h, --help  show this help message and exit
 ```
-
 # Changelog
 
+### v0.1.1
+
+- fixed dependency issues
+- Upload post issues resolved
+- Removed dependency on poster for now
+
 ### v0.0.9
 
-- fixed attribution and dependency issues
+- fixed attribution and dependecy issues
 - Included poster to improve streaming uploads
 - All uploads now use selenium
 

geeup/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = 'Samapriya Roy'
 __email__ = '[email protected]'
-__version__ = '0.0.9'
+__version__ = '0.1.1'

geeup/batch_uploader.py

@@ -57,9 +57,6 @@
 import ee
 import requests
 import retrying
-import poster
-from poster.encode import multipart_encode
-from poster.streaminghttp import register_openers
 from bs4 import BeautifulSoup
 from selenium import webdriver
 from selenium.webdriver import Firefox
@@ -69,7 +66,7 @@
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
-from google.cloud import storage
+#from google.cloud import storage
 from metadata_loader import load_metadata_from_csv, validate_metadata_from_csv
 os.chdir(os.path.dirname(os.path.realpath(__file__)))
 pathway=os.path.dirname(os.path.realpath(__file__))
@@ -280,48 +277,11 @@ def __get_upload_url(session):
         print(e)
 
 @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3)
-def __upload_file_gee(session, file_path):
+def __upload_file_gee(session, file_path, use_multipart):
+    with open(file_path, 'rb') as f:
         upload_url = __get_upload_url(session)
-        class IterableToFileAdapter(object):
-            def __init__(self, iterable):
-                self.iterator = iter(iterable)
-                self.length = iterable.total
-
-            def read(self, size=-1):
-                return next(self.iterator, b'')
-
-            def __len__(self):
-                return self.length
-
-        # define a helper function simulating the interface of posters multipart_encode()-function
-        # but wrapping its generator with the file-like adapter
-        def multipart_encode_for_requests(params, boundary=None, cb=None):
-            datagen, headers = multipart_encode(params, boundary, cb)
-            return IterableToFileAdapter(datagen), headers
-
-
-
-        # this is your progress callback
-        def progress(param, current, total):
-            if not param:
-                return
-
-            # check out http://tcd.netinf.eu/doc/classnilib_1_1encode_1_1MultipartParam.html
-            # for a complete list of the properties param provides to you
-            calc=float(current)/float(total)*100
-            print ('Uploading '+str(os.path.basename(file_path).split('.')[0])+':   '+str(format(float(calc),'.2f'))+" %", end='\r')
-
-        # generate headers and gata-generator an a requests-compatible format
-        # and provide our progress-callback
-        datagen, headers = multipart_encode_for_requests({
-            "file": open(file_path, 'rb'),
-            "composite": "NONE",
-        }, cb=progress)
-
-        # use the requests-lib to issue a post-request with out data attached
-        resp = session.post(upload_url, data=datagen,headers=headers)
-        #print(resp.content)
-
+        files = {'file': f}
+        resp = session.post(upload_url, files=files)
         gsid = resp.json()[0]
         return gsid
 

geeup/metadata_ingest.py

@@ -59,9 +59,6 @@
 import requests
 import retrying
 from google.cloud import storage
-import poster
-from poster.encode import multipart_encode
-from poster.streaminghttp import register_openers
 from metadata_loader import load_metadata_from_csv, validate_metadata_from_csv
 from selenium import webdriver
 from selenium.webdriver import Firefox
@@ -312,48 +309,11 @@ def __get_upload_url(session):
         print(e)
 
 @retrying.retry(retry_on_exception=retry_if_ee_error, wait_exponential_multiplier=1000, wait_exponential_max=4000, stop_max_attempt_number=3)
-def __upload_file_gee(session, file_path):
+def __upload_file_gee(session, file_path, use_multipart):
+    with open(file_path, 'rb') as f:
         upload_url = __get_upload_url(session)
-        class IterableToFileAdapter(object):
-            def __init__(self, iterable):
-                self.iterator = iter(iterable)
-                self.length = iterable.total
-
-            def read(self, size=-1):
-                return next(self.iterator, b'')
-
-            def __len__(self):
-                return self.length
-
-        # define a helper function simulating the interface of posters multipart_encode()-function
-        # but wrapping its generator with the file-like adapter
-        def multipart_encode_for_requests(params, boundary=None, cb=None):
-            datagen, headers = multipart_encode(params, boundary, cb)
-            return IterableToFileAdapter(datagen), headers
-
-
-
-        # this is your progress callback
-        def progress(param, current, total):
-            if not param:
-                return
-
-            # check out http://tcd.netinf.eu/doc/classnilib_1_1encode_1_1MultipartParam.html
-            # for a complete list of the properties param provides to you
-            calc=float(current)/float(total)*100
-            print ('Uploading '+str(os.path.basename(file_path).split('.')[0])+':   '+str(format(float(calc),'.2f'))+" %", end='\r')
-
-        # generate headers and gata-generator an a requests-compatible format
-        # and provide our progress-callback
-        datagen, headers = multipart_encode_for_requests({
-            "file": open(file_path, 'rb'),
-            "composite": "NONE",
-        }, cb=progress)
-
-        # use the requests-lib to issue a post-request with out data attached
-        resp = session.post(upload_url, data=datagen,headers=headers)
-        #print(resp.content)
-
+        files = {'file': f}
+        resp = session.post(upload_url, files=files)
         gsid = resp.json()[0]
         return gsid
 

requirements.txt

@@ -10,4 +10,6 @@ pandas>=0.23.0
 psutil>=5.4.5
 poster>=0.8.1
 selenium>=3.13.0
-pySmartDL>=1.2.5
+pySmartDL>=1.2.5
+pathlib>=1.0.1
+lxml>=4.1.1

setup.py

@@ -18,12 +18,12 @@ def readme():
         return f.read()
 setuptools.setup(
     name='geeup',
-    version='0.0.9',
+    version='0.1.1',
     packages=find_packages(),
     url='https://github.com/samapriya/geeup',
     install_requires=['earthengine_api >= 0.1.87','requests >= 2.10.0','retrying >= 1.3.3','beautifulsoup4 >= 4.5.1','pandas>=0.23.0','psutil>=5.4.5',
                       'requests_toolbelt >= 0.7.0','pytest >= 3.0.0','future >= 0.16.0','google-cloud-storage >= 1.1.1','selenium>=3.13.0',
-                      'pySmartDL>=1.2.5','poster>=0.8.1'],
+                      'pySmartDL>=1.2.5','pathlib>=1.0.1','lxml>=4.1.1'],
     license='Apache 2.0',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
@@ -35,6 +35,8 @@ def readme():
         'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Operating System :: OS Independent',
         'Topic :: Scientific/Engineering :: GIS',
     ),