Update create-many-transfers script

README.md

@@ -17,6 +17,7 @@ While these are primarily intended for use in development, Archivematica systems
 - [Installation](#installation)
 - [Tools Provided](#tools-provided)
   - [graph-links](#graph-links)
+  - [create-many-transfers](#create-many-transfers)
   - [get-fpr-changes](#get-fpr-changes)
   - [rebuild-elasticsearch-aip-index-from-files](#rebuild-elasticsearch-aip-index-from-files)
   - [rebuild-transfer-backlog](#rebuild-transfer-backlog)
@@ -118,6 +119,21 @@ Edges are labelled with the user choice or exit code that connects those nodes,
   * Source: `MicroServiceChainLinks.pk` where taskType is 'goto magic link' (`6fe259c2-459d-4d4b-81a4-1b9daf7ee2e9`)
   * Destination: `Transfer.magicLink`. This is set by the most recent 'assign magic link' (`3590f73d-5eb0-44a0-91a6-5b2db6655889`)
 
+### create-many-transfers
+
+*Versions*: Archivematica 1.4, 1.5, 1.6
+
+Creates many transfers at once, for scalability testing and stress testing Archivematica.
+This creates the transfers directly from locally available files without involving the storage service.
+
+There is one required parameter: The path of the directory or file to make transfers from.
+
+There are three optional parameters.
+`-n` or `--num-transfers` sets the number of transfers to create, with a default of 1.
+`--start-number` sets the number to start indexing transfers from, useful if this is not the first time running this script.
+`--name` sets the name of the transfers, with a default of 'test'.
+The name will have the transfer number appended to it.
+
 ### get-fpr-changes
 
 *Versions*: Independent of Archivematica

tools/create-many-transfers

@@ -18,11 +18,11 @@
 # You should have received a copy of the GNU General Public License
 # along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import print_function
 import argparse
-import tempfile
+import errno
 import os
 import shutil
-import subprocess
 import sys
 import time
 
@@ -36,14 +36,15 @@ django.setup()
 from main.models import Transfer, Job
 
 
-def rsync_copy(source, destination):
-    subprocess.call([
-        'rsync',
-        '-r',
-        '-t',
-        source,
-        destination
-    ])
+def copy_anything(src, dst):
+    # http://stackoverflow.com/questions/1994488/copy-file-or-directory-in-python
+    try:
+        shutil.copytree(src, dst)
+    except OSError as exc: # python >2.5
+        if exc.errno == errno.ENOTDIR:
+            shutil.copy(src, dst)
+        else:
+            raise
 
 
 def make_filepath_unique(filepath, original=None, attempt=0):
@@ -57,47 +58,38 @@ def make_filepath_unique(filepath, original=None, attempt=0):
 
 
 def make_transfer(path, transfer_name):
-    # url: '/filesystem/get_temp_directory/',
-    temp_dir = tempfile.mkdtemp()
-    transfer_dir = os.path.join(temp_dir, transfer_name)
-    # url: '/filesystem/copy_transfer_component/',
-    if not os.path.isdir(transfer_dir):
-        os.mkdir(transfer_dir)
-    # cycle through each path copying files/dirs inside it to transfer dir
-    for entry in os.listdir(path):
-        entry_path = os.path.join(path, entry)
-        rsync_copy(entry_path, transfer_dir)
-
-    # var url = '/filesystem/ransfer/'
-    filepath = os.path.join(temp_dir, transfer_name)
-
-    destination = os.path.join('/', 'var', 'archivematica', 'sharedDirectory', 'watchedDirectories', 'activeTransfers', 'standardTransfer', transfer_name)
-    destination = make_filepath_unique(destination)
-    print 'Creating transfer at', destination
+    transfer_path = os.path.join('/', 'var', 'archivematica', 'sharedDirectory', 'watchedDirectories', 'activeTransfers', 'standardTransfer', transfer_name)
+    transfer_path = make_filepath_unique(transfer_path)
+    print('Creating transfer at', transfer_path)
     try:
-        shutil.move(filepath, destination)
+        os.mkdir(transfer_path)
+        for entry in os.listdir(path):
+            entry_path = os.path.join(path, entry)
+            transfer_entry_path = os.path.join(transfer_path, entry)
+            copy_anything(entry_path, transfer_entry_path)
     except OSError:
-        print 'Error copying from ' + filepath + ' to ' + destination + '. (' + str(sys.exc_info()[0]) + ')'
+        print('Error copying from', entry_path, 'to', transfer_entry_path, '. (', sys.exc_info()[0], ')')
+    return os.path.basename(transfer_path)
 
 
 def approve_transfer(transfer_name):
-    # Approve transfer
+    # TODO replace this with something like automation_tools.transfer.approve_transfer
     approve_chain_uuid = '6953950b-c101-4f4c-a0c3-0cd0684afe5e'
     user_id = 1
     path = os.path.join('activeTransfers', 'standardTransfer', transfer_name)+'/'
     transfer = Transfer.objects.get(currentlocation__endswith=path)
     transfer_name = transfer.currentlocation.split('/')[-2]
-    print 'Approving transfer UUID:', transfer_name
+    print('Approving transfer:', transfer_name)
     job_uuid = Job.objects.get(jobtype='Approve standard transfer', sipuuid=transfer.uuid).jobuuid
     client = MCPClient()
     client.execute(job_uuid, approve_chain_uuid, user_id)
 
 
 def make_transfers(num_transfers, start_number, path, transfer_name):
     names = ['{name}_{number}'.format(name=transfer_name, number=i) for i in range(start_number, start_number+num_transfers)]
-    for name in names:
-        make_transfer(path, name)
-    print "Waiting 15 seconds for Archivematica to process transfers..."
+    # Start transfers, updating stored name
+    names = [make_transfer(path, name) for name in names]
+    print("Waiting 15 seconds for Archivematica to process transfers...")
     time.sleep(15)
     for name in names:
         approve_transfer(name)