From edd187cc20f6162fe9b723a8a64617f4b8ce349c Mon Sep 17 00:00:00 2001 From: Dan Feder Date: Wed, 14 Dec 2022 17:55:59 -0500 Subject: [PATCH] Switch to simple built-in FileFetcher on import (#3881) --- composer.json | 1 - modules/common/common.services.yml | 5 - modules/common/src/FileFetcher/Factory.php | 56 ------ modules/common/tests/src/Traits/CleanUp.php | 4 +- .../tests/src/Unit/Storage/JobStoreTest.php | 18 +- modules/datastore/datastore.services.yml | 1 - .../src/Plugin/QueueWorker/FileFetcherJob.php | 165 ++++++++++++++++++ .../src/Plugin/QueueWorker/ImportJob.php | 2 +- modules/datastore/src/Service.php | 11 +- .../datastore/src/Service/Info/ImportInfo.php | 3 +- .../src/Service/Info/ImportInfoList.php | 4 +- .../src/Service/ResourceLocalizer.php | 28 +-- modules/datastore/tests/data/tiny.csv | 2 + .../datastore/tests/src/Mock/Container.php | 2 - .../Plugin/QueueWorker/FileFetcherJobTest.php | 102 +++++++++++ .../Unit/Service/Info/ImportInfoListTest.php | 4 +- .../Unit/Service/ResourceLocalizerTest.php | 24 +-- .../datastore/tests/src/Unit/ServiceTest.php | 5 +- .../tests/src/Unit/Storage/JobStoreTest.php | 18 +- modules/sample_content/sample_content.json | 26 +-- 20 files changed, 331 insertions(+), 150 deletions(-) delete mode 100644 modules/common/src/FileFetcher/Factory.php create mode 100644 modules/datastore/src/Plugin/QueueWorker/FileFetcherJob.php create mode 100644 modules/datastore/tests/data/tiny.csv create mode 100644 modules/datastore/tests/src/Unit/Plugin/QueueWorker/FileFetcherJobTest.php diff --git a/composer.json b/composer.json index 58d3d04f0e..84ac4e3e40 100644 --- a/composer.json +++ b/composer.json @@ -18,7 +18,6 @@ "fmizzell/maquina": "^1.1.0", "getdkan/contracts": "^1.0.0", "getdkan/csv-parser": "^1.2.3", - "getdkan/file-fetcher" : "^4.1.0", "getdkan/harvest": "^1.0.0", "getdkan/json-schema-provider": "^0.1.2", "getdkan/locker": "^1.1.0", diff --git a/modules/common/common.services.yml b/modules/common/common.services.yml index d3760afea9..dfda5b0944 100644 --- a/modules/common/common.services.yml +++ b/modules/common/common.services.yml @@ -24,11 +24,6 @@ services: factory: entity_type.manager:getStorage arguments: ['node'] - dkan.common.file_fetcher: - class: \Drupal\common\FileFetcher\Factory - arguments: - - '@dkan.common.job_store' - dkan.common.dataset_info: class: \Drupal\common\DatasetInfo calls: diff --git a/modules/common/src/FileFetcher/Factory.php b/modules/common/src/FileFetcher/Factory.php deleted file mode 100644 index 31ef1b43e3..0000000000 --- a/modules/common/src/FileFetcher/Factory.php +++ /dev/null @@ -1,56 +0,0 @@ - TRUE, - ]; - - /** - * Constructor. - */ - public function __construct(JobStoreFactory $factory) { - $this->factory = $factory; - } - - /** - * Inherited. - * - * @inheritdoc - */ - public function getInstance(string $identifier, array $config = []) { - $config = array_merge($this->configDefault, $config); - return FileFetcher::get($identifier, $this->getFileFetcherJobStore(), $config); - } - - /** - * Private. - */ - private function getFileFetcherJobStore() { - /** @var \Drupal\common\Storage\JobStoreFactory $jobStoreFactory */ - $jobStoreFactory = $this->factory; - return $jobStoreFactory->getInstance(FileFetcher::class); - } - -} diff --git a/modules/common/tests/src/Traits/CleanUp.php b/modules/common/tests/src/Traits/CleanUp.php index 0a9dda8eb0..9ddf633aa8 100644 --- a/modules/common/tests/src/Traits/CleanUp.php +++ b/modules/common/tests/src/Traits/CleanUp.php @@ -3,7 +3,7 @@ namespace Drupal\Tests\common\Traits; use Drupal\node\Entity\Node; -use FileFetcher\FileFetcher; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; /** * @@ -50,7 +50,7 @@ private function removeAllFileFetchingJobs() { $jobStoreFactory = \Drupal::service('dkan.common.job_store'); /** @var \Drupal\common\Storage\JobStore $jobStore */ - $jobStore = $jobStoreFactory->getInstance(FileFetcher::class); + $jobStore = $jobStoreFactory->getInstance(FileFetcherJob::class); foreach ($jobStore->retrieveAll() as $id) { $jobStore->remove($id); } diff --git a/modules/common/tests/src/Unit/Storage/JobStoreTest.php b/modules/common/tests/src/Unit/Storage/JobStoreTest.php index 38d20e8622..0126d5a323 100644 --- a/modules/common/tests/src/Unit/Storage/JobStoreTest.php +++ b/modules/common/tests/src/Unit/Storage/JobStoreTest.php @@ -9,9 +9,9 @@ use Drupal\Core\Database\Schema; use Drupal\Core\Database\StatementWrapper; use Drupal\common\Storage\JobStore; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; use Contracts\Mock\Storage\Memory; -use FileFetcher\FileFetcher; use MockChain\Chain; use MockChain\Sequence; use PHPUnit\Framework\TestCase; @@ -30,7 +30,7 @@ public function testConstruction() { ->add(Connection::class, "schema", Schema::class) ->add(Schema::class, "tableExists", FALSE); - $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); + $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock()); $this->assertTrue(is_object($jobStore)); } @@ -59,8 +59,8 @@ public function testRetrieve() { ->add(Connection::class, 'query', StatementWrapper::class) ->add(StatementWrapper::class, 'fetchAll', $fieldInfo); - $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); - $this->assertEquals($job_data, $jobStore->retrieve("1", FileFetcher::class)); + $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock()); + $this->assertEquals($job_data, $jobStore->retrieve("1", FileFetcherJob::class)); } /** @@ -90,7 +90,7 @@ public function testRetrieveAll() { ->add(Connection::class, 'query', StatementWrapper::class) ->add(StatementWrapper::class, 'fetchAll', $sequence); - $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); + $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock()); $this->assertTrue(is_array($jobStore->retrieveAll())); } @@ -127,7 +127,7 @@ public function testStore() { ->add(StatementWrapper::class, 'fetchAll', $fieldInfo) ->getMock(); - $jobStore = new JobStore(FileFetcher::class, $connection); + $jobStore = new JobStore(FileFetcherJob::class, $connection); $this->assertEquals("1", $jobStore->store(json_encode($jobObject), "1")); } @@ -151,16 +151,16 @@ public function testRemove() { ->add(StatementWrapper::class, 'fetchAll', $fieldInfo) ->getMock(); - $jobStore = new JobStore(FileFetcher::class, $connection); + $jobStore = new JobStore(FileFetcherJob::class, $connection); - $this->assertEquals("", $jobStore->remove("1", FileFetcher::class)); + $this->assertEquals("", $jobStore->remove("1", FileFetcherJob::class)); } /** * Private. */ private function getFileFetcher() { - return FileFetcher::get("1", new Memory(), ["filePath" => "file://" . __DIR__ . "/../../data/countries.csv"]); + return FileFetcherJob::get("1", new Memory(), ["filePath" => "file://" . __DIR__ . "/../../data/countries.csv"]); } } diff --git a/modules/datastore/datastore.services.yml b/modules/datastore/datastore.services.yml index 6ff5b2ad1a..05906fbc57 100644 --- a/modules/datastore/datastore.services.yml +++ b/modules/datastore/datastore.services.yml @@ -17,7 +17,6 @@ services: class: \Drupal\datastore\Service\ResourceLocalizer arguments: - '@dkan.metastore.resource_mapper' - - '@dkan.common.file_fetcher' - '@dkan.common.drupal_files' - '@dkan.common.job_store' diff --git a/modules/datastore/src/Plugin/QueueWorker/FileFetcherJob.php b/modules/datastore/src/Plugin/QueueWorker/FileFetcherJob.php new file mode 100644 index 0000000000..fdabf1c195 --- /dev/null +++ b/modules/datastore/src/Plugin/QueueWorker/FileFetcherJob.php @@ -0,0 +1,165 @@ + $config['filePath'], + 'total_bytes' => 0, + 'total_bytes_copied' => 0, + 'temporary' => FALSE, + 'temporary_directory' => $config['temporaryDirectory'] ?? '/tmp', + ]; + + $this->getResult()->setData(json_encode($state)); + } + + /** + * {@inheritdoc} + */ + protected function runIt() { + $state = $this->setupState($this->getState()); + $this->getResult()->setData(json_encode($state)); + $info = $this->copy($this->getState(), $this->getResult(), $this->getTimeLimit()); + $this->setState($info['state']); + return $info['result']; + } + + /** + * Set up the job state. + * + * @param array $state + * Incoming state array. + * + * @return array + * Modified state array. + */ + protected function setupState(array $state): array { + $state['total_bytes'] = PHP_INT_MAX; + $state['temporary'] = TRUE; + $state['destination'] = $this->getTemporaryFilePath($state); + + return $state; + } + + /** + * Get temporary file path, depending on flag keep_original_filename value. + * + * @param array $state + * State. + * + * @return string + * Temporary file path. + */ + private function getTemporaryFilePath(array $state): string { + $file_name = basename($state['source']); + return "{$state['temporary_directory']}/{$file_name}"; + } + + /** + * Copy the file to local storage. + * + * @param array $state + * State array. + * @param \Procrastinator\Result $result + * Job result object. + * + * @return array + * Array with two elements: state and result. + */ + public function copy(array $state, Result $result): array { + if (stream_is_local($state['source'])) { + return $this->copyLocal($state, $result); + } + else { + return $this->copyRemote($state, $result); + } + } + + /** + * Copy local file to proper local storage. + * + * @param array $state + * State array. + * @param \Procrastinator\Result $result + * Job result object. + * + * @return array + * Array with two elements: state and result. + */ + protected function copyLocal(array $state, Result $result): array { + $this->ensureCreatingForWriting($state['destination']); + if (copy($state['source'], $state['destination'])) { + $result->setStatus(Result::DONE); + } + else { + throw new \Exception("File copy failed."); + } + $state['total_bytes_copied'] = $state['total_bytes'] = filesize($state['destination']); + return ['state' => $state, 'result' => $result]; + } + + /** + * Copy remote file to local storage. + * + * @param array $state + * State array. + * @param \Procrastinator\Result $result + * Job result object. + * + * @return array + * Array with two elements: state and result. + */ + protected function copyRemote(array $state, Result $result): array { + $client = new Client(); + try { + $fout = $this->ensureCreatingForWriting($state['destination']); + $client->get($state['source'], ['sink' => $fout]); + $result->setStatus(Result::DONE); + } + catch (\Exception $e) { + $result->setStatus(Result::ERROR); + $result->setError($e->getMessage()); + } + + $state['total_bytes_copied'] = $state['total_bytes'] = filesize($state['destination']); + return ['state' => $state, 'result' => $result]; + } + + /** + * Ensure the destination file can be created. + * + * @param string $to + * The destination filename. + * + * @return false|resource + * File resource. + */ + private function ensureCreatingForWriting(string $to) { + // Delete destination first to avoid appending if existing. + if (file_exists($to)) { + unlink($to); + } + $fout = fopen($to, "w"); + return $fout; + } + +} diff --git a/modules/datastore/src/Plugin/QueueWorker/ImportJob.php b/modules/datastore/src/Plugin/QueueWorker/ImportJob.php index 2625b2a23f..ca72035886 100644 --- a/modules/datastore/src/Plugin/QueueWorker/ImportJob.php +++ b/modules/datastore/src/Plugin/QueueWorker/ImportJob.php @@ -361,7 +361,7 @@ protected function setStorageSchema(array $header) { * Verify headers are unique. * * @param array $header - * List of strings + * List of strings. * * @throws \Exception */ diff --git a/modules/datastore/src/Service.php b/modules/datastore/src/Service.php index df7df8d778..dbc8d4747f 100644 --- a/modules/datastore/src/Service.php +++ b/modules/datastore/src/Service.php @@ -3,6 +3,7 @@ namespace Drupal\datastore; use Drupal\common\DataResource; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; use Drupal\common\Storage\JobStoreFactory; use Procrastinator\Result; use Symfony\Component\DependencyInjection\ContainerInterface; @@ -12,7 +13,6 @@ use Drupal\datastore\Service\ResourceLocalizer; use Drupal\datastore\Service\Factory\ImportFactoryInterface; use Drupal\datastore\Service\Info\ImportInfoList; -use FileFetcher\FileFetcher; /** * Main services for the datastore. @@ -33,6 +33,13 @@ class Service implements ContainerInjectionInterface { */ private $importServiceFactory; + /** + * Import info list service. + * + * @var \Drupal\datastore\Service\Info\ImportInfoList + */ + private ImportInfoList $importInfoList; + /** * Drupal queue. * @@ -204,7 +211,7 @@ public function drop(string $identifier, ?string $version = NULL, bool $local_re if ($local_resource) { $this->resourceLocalizer->remove($identifier, $version); $this->jobStoreFactory - ->getInstance(FileFetcher::class) + ->getInstance(FileFetcherJob::class) ->remove(substr(str_replace('__', '_', $resource_id), 0, -11)); } } diff --git a/modules/datastore/src/Service/Info/ImportInfo.php b/modules/datastore/src/Service/Info/ImportInfo.php index cd81587361..b9c84baf1f 100644 --- a/modules/datastore/src/Service/Info/ImportInfo.php +++ b/modules/datastore/src/Service/Info/ImportInfo.php @@ -6,7 +6,6 @@ use Drupal\common\Storage\JobStoreFactory; use Drupal\datastore\Service\Factory\ImportFactoryInterface; use Drupal\datastore\Service\ResourceLocalizer; -use FileFetcher\FileFetcher; use Procrastinator\Job\Job; /** @@ -38,7 +37,7 @@ class ImportInfo { /** * FileFetcher service. * - * @var \FileFetcher\FileFetcher + * @var \Drupal\datastore\Plugin\QueueWorker\FileFetcherJob */ private $fileFetcher; diff --git a/modules/datastore/src/Service/Info/ImportInfoList.php b/modules/datastore/src/Service/Info/ImportInfoList.php index d17aaef238..6d2e226537 100644 --- a/modules/datastore/src/Service/Info/ImportInfoList.php +++ b/modules/datastore/src/Service/Info/ImportInfoList.php @@ -2,9 +2,9 @@ namespace Drupal\datastore\Service\Info; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; use Drupal\common\Storage\JobStoreFactory; use Drupal\Core\DependencyInjection\ContainerInjectionInterface; -use FileFetcher\FileFetcher; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -58,7 +58,7 @@ public function __construct(JobStoreFactory $jobStoreFactory, ImportInfo $import public function buildList() { $list = []; - $store = $this->jobStoreFactory->getInstance(FileFetcher::class); + $store = $this->jobStoreFactory->getInstance(FileFetcherJob::class); foreach ($store->retrieveAll() as $id) { $pieces = explode("_", $id); diff --git a/modules/datastore/src/Service/ResourceLocalizer.php b/modules/datastore/src/Service/ResourceLocalizer.php index ea946ecead..5f7c97c31e 100644 --- a/modules/datastore/src/Service/ResourceLocalizer.php +++ b/modules/datastore/src/Service/ResourceLocalizer.php @@ -7,14 +7,13 @@ use Drupal\common\Storage\JobStoreFactory; use Drupal\common\UrlHostTokenResolver; use Drupal\common\Util\DrupalFiles; -use Contracts\FactoryInterface; use Drupal\Core\File\FileSystemInterface; use Drupal\metastore\Exception\AlreadyRegistered; use Drupal\metastore\Reference\Referencer; use Drupal\metastore\ResourceMapper; -use FileFetcher\FileFetcher; use Procrastinator\Result; use Drupal\common\EventDispatcherTrait; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; /** * Resource localizer. @@ -46,13 +45,6 @@ class ResourceLocalizer { */ private $fileMapper; - /** - * DKAN resource file fetcher factory. - * - * @var \Contracts\FactoryInterface - */ - private $fileFetcherFactory; - /** * Drupal files utility service. * @@ -70,9 +62,8 @@ class ResourceLocalizer { /** * Constructor. */ - public function __construct(ResourceMapper $fileMapper, FactoryInterface $fileFetcherFactory, DrupalFiles $drupalFiles, JobStoreFactory $jobStoreFactory) { + public function __construct(ResourceMapper $fileMapper, DrupalFiles $drupalFiles, JobStoreFactory $jobStoreFactory) { $this->fileMapper = $fileMapper; - $this->fileFetcherFactory = $fileFetcherFactory; $this->drupalFiles = $drupalFiles; $this->jobStoreFactory = $jobStoreFactory; } @@ -114,7 +105,7 @@ public function get($identifier, $version = NULL, $perpective = self::LOCAL_FILE /** * Private. */ - private function registerNewPerspectives(DataResource $resource, FileFetcher $fileFetcher) { + private function registerNewPerspectives(DataResource $resource, FileFetcherJob $fileFetcher) { $localFilePath = $fileFetcher->getStateProperty('destination'); $dir = "file://" . $this->drupalFiles->getPublicFilesDirectory(); @@ -179,7 +170,7 @@ private function removeLocalUrl(DataResource $resource) { */ private function removeJob($uuid) { if ($uuid) { - $this->getJobStoreFactory()->getInstance(FileFetcher::class)->remove($uuid); + $this->jobStoreFactory->getInstance(FileFetcherJob::class)->remove($uuid); } } @@ -193,7 +184,7 @@ private function getResourceSource($identifier, $version = NULL): ?DataResource /** * Get FileFetcher. */ - public function getFileFetcher(DataResource $resource): FileFetcher { + public function getFileFetcher(DataResource $resource): FileFetcherJob { $uuid = "{$resource->getIdentifier()}_{$resource->getVersion()}"; $directory = "public://resources/{$uuid}"; $this->getFilesystem()->prepareDirectory($directory, FileSystemInterface::CREATE_DIRECTORY); @@ -201,7 +192,7 @@ public function getFileFetcher(DataResource $resource): FileFetcher { 'filePath' => UrlHostTokenResolver::resolveFilePath($resource->getFilePath()), 'temporaryDirectory' => $directory, ]; - return $this->fileFetcherFactory->getInstance($uuid, $config); + return FileFetcherJob::get($uuid, $this->jobStoreFactory->getInstance(FileFetcherJob::class), $config); } /** @@ -211,13 +202,6 @@ private function getFileMapper(): ResourceMapper { return $this->fileMapper; } - /** - * Private. - */ - private function getJobStoreFactory() { - return $this->jobStoreFactory; - } - /** * Get the Drupal filesystem service. * diff --git a/modules/datastore/tests/data/tiny.csv b/modules/datastore/tests/data/tiny.csv new file mode 100644 index 0000000000..9cc135f368 --- /dev/null +++ b/modules/datastore/tests/data/tiny.csv @@ -0,0 +1,2 @@ +"hello", "friend" +"goodbye", "enemy" \ No newline at end of file diff --git a/modules/datastore/tests/src/Mock/Container.php b/modules/datastore/tests/src/Mock/Container.php index 52db31c52f..5e4da5e4f4 100644 --- a/modules/datastore/tests/src/Mock/Container.php +++ b/modules/datastore/tests/src/Mock/Container.php @@ -14,7 +14,6 @@ use Drupal\Core\TypedData\TypedDataInterface; use Drupal\datastore\Service\Service; use Drupal\node\NodeInterface; -use FileFetcher\Processor\Local; use PHPUnit\Framework\TestCase; use Symfony\Component\DependencyInjection\ContainerInterface; @@ -38,7 +37,6 @@ public function __construct(TestCase $testCase) { $fileFetcherContent = file_get_contents(__DIR__ . '/../../../data/filefetcher.json'); $fileFetcherObject = json_decode($fileFetcherContent); $data = json_decode($fileFetcherObject->result->data); - $data->processor = Local::class; $fileFetcherObject->result->data = json_encode($data); $this->jobStoreData = (object) [ diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/FileFetcherJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/FileFetcherJobTest.php new file mode 100644 index 0000000000..81ec20baf7 --- /dev/null +++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/FileFetcherJobTest.php @@ -0,0 +1,102 @@ +getJobstore(); + $config = [ + 'temporaryDirectory' => '/tmp', + 'filePath' => __DIR__ . '/../../../../data/tiny.csv', + ]; + $fetcher = new FileFetcherJob('abc', $jobStore, $config); + $fetcher->run(); + + $state = $fetcher->getState(); + $this->assertEquals( + file_get_contents($state['source']), + file_get_contents($state['destination']) + ); + } + + /** + * Test bad path. + */ + public function testCopyMissingFile() { + $jobStore = $this->getJobstore(); + $config = [ + 'temporaryDirectory' => '/tmp', + 'filePath' => __DIR__ . '/../../../../data/missing.csv', + ]; + $fetcher = new FileFetcherJob('abc', $jobStore, $config); + $fetcher->run(); + + $result = $fetcher->getResult(); + $this->assertStringContainsString("failed to open stream", $result->getError()); + } + + /** + * Test bad url. + */ + public function testCopyBadUrl() { + $jobStore = $this->getJobstore(); + $config = [ + 'temporaryDirectory' => '/tmp', + 'filePath' => 'http://something.fakeurl', + ]; + $fetcher = new FileFetcherJob('abc', $jobStore, $config); + $fetcher->run(); + + $result = $fetcher->getResult(); + $this->assertStringContainsString("Could not resolve host", $result->getError()); + } + + /** + * Test bad destination. + */ + public function testBadDestinationPath() { + $jobStore = $this->getJobstore(); + $config = [ + 'temporaryDirectory' => '/badTempDir', + 'filePath' => __DIR__ . '/../../../../data/tiny.csv', + ]; + $fetcher = new FileFetcherJob('abc', $jobStore, $config); + $fetcher->run(); + + $result = $fetcher->getResult(); + $this->assertStringContainsString("No such file or directory", $result->getError()); + } + + /** + * Test empty config. + */ + public function testInvalidConfig() { + $jobStore = $this->getJobstore(); + $this->expectExceptionMessage("Constructor missing expected config"); + (new FileFetcherJob('abc', $jobStore, [])); + } + + /** + * Get a mock JobStore object. + */ + protected function getJobstore() { + $chain = (new Chain($this)) + ->add(JobStore::class, "setTable", TRUE) + ->add(JobStore::class, "store", "foo"); + + return $chain->getMock(); + } + +} diff --git a/modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php b/modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php index 7f4805ef0a..4c1b9756e8 100644 --- a/modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php +++ b/modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php @@ -7,7 +7,7 @@ use Drupal\common\Storage\JobStoreFactory; use Drupal\datastore\Service\Info\ImportInfo; use Drupal\datastore\Service\Info\ImportInfoList; -use FileFetcher\FileFetcher; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; use MockChain\Chain; use MockChain\Options; use PHPUnit\Framework\TestCase; @@ -17,7 +17,7 @@ class ImportInfoListTest extends TestCase { public function test() { - $ff = FileFetcher::hydrate('{}'); + $ff = FileFetcherJob::hydrate('{}'); $result = Result::hydrate('{"status":"error","data":"","error":"File import error"}'); diff --git a/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php b/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php index ff5ad9d269..8ddbc220c2 100644 --- a/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php +++ b/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php @@ -12,7 +12,7 @@ use Drupal\Core\StreamWrapper\StreamWrapperManager; use Drupal\datastore\Service\ResourceLocalizer; use Drupal\metastore\ResourceMapper; -use FileFetcher\FileFetcher; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; use MockChain\Chain; use MockChain\Options; use PHPUnit\Framework\TestCase; @@ -40,7 +40,6 @@ public function testNoResourceFound() { $service = new ResourceLocalizer( $this->getFileMapperChain()->getMock(), - $this->getFileFetcherFactoryChain()->getMock(), $this->getDrupalFilesChain()->getMock(), $this->getJobStoreFactoryChain()->getMock() ); @@ -68,13 +67,8 @@ private function doTestResourceLocalizerRemove(string $file_path): void { ->add(ResourceMapper::class, 'get', $resource) ->getMock(); - $fileFetcher = $this->getFileFetcherFactoryChain() - ->add(FileFetcher::class, 'getStateProperty', $file_path) - ->getMock(); - $service = new ResourceLocalizer( $fileMapper, - $fileFetcher, $this->getDrupalFilesChain()->getMock(), $this->getJobStoreFactoryChain()->getMock() ); @@ -123,16 +117,6 @@ private function getFileMapperChain() { ->add(ResourceMapper::class, 'remove', NULL); } - /** - * - */ - private function getFileFetcherFactoryChain() { - return (new Chain($this)) - ->add(JobStoreFactory::class, 'getInstance', FileFetcher::class) - ->add(FileFetcher::class, 'getResult', Result::class) - ->add(Result::class, 'getStatus', Result::DONE); - } - /** * */ @@ -151,7 +135,11 @@ private function getDrupalFilesChain() { private function getJobStoreFactoryChain() { return (new Chain($this)) ->add(JobStoreFactory::class, 'getInstance', JobStore::class) - ->add(JobStore::class, 'remove', NULL); + ->add(JobStore::class, 'remove', NULL) + ->add(JobStore::class, 'tableExist', FALSE) + ->add(JobStore::class, 'setTable', TRUE) + ->add(JobStore::class, 'retrieve', []) + ->add(JobStore::class, 'store', "abc"); } /** diff --git a/modules/datastore/tests/src/Unit/ServiceTest.php b/modules/datastore/tests/src/Unit/ServiceTest.php index 40ab3e67be..7b13d018c6 100644 --- a/modules/datastore/tests/src/Unit/ServiceTest.php +++ b/modules/datastore/tests/src/Unit/ServiceTest.php @@ -15,11 +15,10 @@ use Drupal\datastore\Service\ResourceLocalizer; use Drupal\datastore\Storage\DatabaseTable; use Drupal\metastore\ResourceMapper; -use FileFetcher\FileFetcher; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; use MockChain\Chain; use MockChain\Options; use PHPUnit\Framework\TestCase; -use Procrastinator\Job\AbstractPersistentJob; use Procrastinator\Result; use Symfony\Component\DependencyInjection\Container; use TypeError; @@ -38,7 +37,7 @@ public function testImport() { $chain = $this->getContainerChainForService('dkan.datastore.service') ->add(ResourceLocalizer::class, 'get', $resource) ->add(ResourceLocalizer::class, 'getResult', Result::class) - ->add(FileFetcher::class, 'run', Result::class) + ->add(FileFetcherJob::class, 'run', Result::class) ->add(ResourceMapper::class, 'get', $resource) ->add(ImportServiceFactory::class, "getInstance", ImportService::class) ->add(ImportService::class, "import", NULL) diff --git a/modules/datastore/tests/src/Unit/Storage/JobStoreTest.php b/modules/datastore/tests/src/Unit/Storage/JobStoreTest.php index d0b02d4699..9703976cf0 100644 --- a/modules/datastore/tests/src/Unit/Storage/JobStoreTest.php +++ b/modules/datastore/tests/src/Unit/Storage/JobStoreTest.php @@ -12,7 +12,7 @@ use MockChain\Chain; use MockChain\Sequence; use Drupal\common\Storage\JobStore; -use FileFetcher\FileFetcher; +use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob; use PHPUnit\Framework\TestCase; /** @@ -29,7 +29,7 @@ public function testConstruction() { ->add(Connection::class, "schema", Schema::class) ->add(Schema::class, "tableExists", FALSE); - $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); + $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock()); $this->assertTrue(is_object($jobStore)); } @@ -58,8 +58,8 @@ public function testRetrieve() { ->add(Connection::class, 'query', StatementWrapper::class) ->add(StatementWrapper::class, 'fetchAll', $fieldInfo); - $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); - $this->assertEquals($job_data, $jobStore->retrieve("1", FileFetcher::class)); + $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock()); + $this->assertEquals($job_data, $jobStore->retrieve("1", FileFetcherJob::class)); } /** @@ -89,7 +89,7 @@ public function testRetrieveAll() { ->add(Connection::class, 'query', StatementWrapper::class) ->add(StatementWrapper::class, 'fetchAll', $sequence); - $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); + $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock()); $this->assertTrue(is_array($jobStore->retrieveAll())); } @@ -126,7 +126,7 @@ public function testStore() { ->add(StatementWrapper::class, 'fetchAll', $fieldInfo) ->getMock(); - $jobStore = new JobStore(FileFetcher::class, $connection); + $jobStore = new JobStore(FileFetcherJob::class, $connection); $this->assertEquals("1", $jobStore->store(json_encode($jobObject), "1")); } @@ -150,16 +150,16 @@ public function testRemove() { ->add(StatementWrapper::class, 'fetchAll', $fieldInfo) ->getMock(); - $jobStore = new JobStore(FileFetcher::class, $connection); + $jobStore = new JobStore(FileFetcherJob::class, $connection); - $this->assertEquals("", $jobStore->remove("1", FileFetcher::class)); + $this->assertEquals("", $jobStore->remove("1", FileFetcherJob::class)); } /** * Private. */ private function getFileFetcher() { - return FileFetcher::get("1", new Memory(), ["filePath" => "file://" . __DIR__ . "/../../data/countries.csv"]); + return FileFetcherJob::get("1", new Memory(), ["filePath" => "file://" . __DIR__ . "/../../data/countries.csv"]); } } diff --git a/modules/sample_content/sample_content.json b/modules/sample_content/sample_content.json index 39137987dc..e3f1402586 100644 --- a/modules/sample_content/sample_content.json +++ b/modules/sample_content/sample_content.json @@ -16,7 +16,7 @@ "distribution": [ { "@type": "dcat:Distribution", - "downloadURL": "file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/Bike_Lane.csv", + "downloadURL": "file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/Bike_Lane.csv", "mediaType": "text\/csv", "format": "csv", "title": "Florida Bike Lanes" @@ -49,21 +49,21 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/pittsmva2016data.zip", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/pittsmva2016data.zip", "mediaType":"application\/zip", "format":"zip", "title":"MVA Geographic Data 2016" }, { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/2016mvadatadictionary.pdf", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/2016mvadatadictionary.pdf", "mediaType":"application\/pdf", "format":"pdf", "title":"2016 MVA Data Dictionary" }, { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/IMD-MAPS.zip", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/IMD-MAPS.zip", "mediaType":"application\/zip", "format":"zip", "title":"Map files" @@ -94,7 +94,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/ViolentCrimeRates.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/ViolentCrimeRates.csv", "mediaType":"text\/csv", "format":"csv", "description":"Includes information on the frequency of violent crimes in general, murder and non-negligent manslaughter, aggravated assault, rape and robbery. All crime rates are per 100,000 people.", @@ -124,7 +124,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/Asthma-Prevalence-Map-2017.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/Asthma-Prevalence-Map-2017.csv", "mediaType":"text\/csv", "format":"csv", "description":"Mock data for demonstration purposes.", @@ -156,7 +156,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/CDCSmokingRates.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/CDCSmokingRates.csv", "mediaType":"text\/csv", "format":"csv", "description":"This dataset highlights critical trends in adult total and per capita consumption of both combustible (cigarettes, little cigars, small cigars, pipe tobacco, roll-your-own tobacco) tobacco products and smokeless (chewing tobacco and snuff) tobacco.", @@ -188,7 +188,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/demo_varicelladeaths_1991_2007.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/demo_varicelladeaths_1991_2007.csv", "mediaType":"text\/csv", "format":"csv", "description":"Mock data for demonstration purposes.", @@ -196,7 +196,7 @@ }, { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/demo_varicelladeaths_1974_1990.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/demo_varicelladeaths_1974_1990.csv", "mediaType":"text\/csv", "format":"csv", "description":"Mock data for demonstration purposes.", @@ -228,7 +228,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL": "file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/polling-places.csv", + "downloadURL": "file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/polling-places.csv", "mediaType":"text\/csv", "format":"csv", "description":"List of polling places in Madison, WI.", @@ -259,7 +259,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/us_foreclosures_jan_2012_by_state.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/us_foreclosures_jan_2012_by_state.csv", "mediaType":"text\/csv", "format":"csv", "description":"US National Foreclosure Statistics - By State - January 2012", @@ -287,7 +287,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/data.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/data.csv", "mediaType":"text\/csv", "format":"csv", "title":"Table of Gold Prices" @@ -318,7 +318,7 @@ "distribution":[ { "@type":"dcat:Distribution", - "downloadURL":"file:\/\//var/www/docroot/modules/contrib/dkan/modules/sample_content/files\/district_centerpoints.csv", + "downloadURL":"file:\/\//var/www/html/docroot/modules/contrib/dkan/modules/sample_content/files\/district_centerpoints.csv", "mediaType":"text\/csv", "format":"csv", "title":"District Names"