ImportPagesCSV.module

<?php namespace ProcessWire;

/**
 * ProcessWire Process module to import pages from a CSV file.
 *
 * Copyright 2011-2020 by Ryan Cramer 
 * Licensed under MPL 2.0
 * 
 * https://processwire.com
 *
 * 
 * Importing file/image fields
 * ===========================
 * CSV column should contain full URL (or diskpath and filename) to the file you want to import. 
 * For fields that support multiple files, place each filename or URL on its own line, OR separate 
 * them by | (pipe) OR tab.
 * 
 * Importing page reference fields
 * ===============================
 * Please make sure that your Page reference field has one or more pages selected for the "Parent" 
 * setting on the Details tab. If you want the import to be able to create paes, there must also
 * be a single Template selected on the "Template" setting. Column values for page references 
 * may be page IDs, Titles, or Names separated by newline or pipe "|". 
 * 
 * 
 * @method bool isAllowedField(Field $field)
 * @method bool importPageValue(Page $page, $name, $value)
 * 
 *
 */

class ImportPagesCSV extends Process implements Module {

	/**
	 * getModuleInfo is a module required by all modules to tell ProcessWire about them
	 *
	 * @return array
	 *
	 */
	public static function getModuleInfo() {
		return array(
			'title' => 'Import Pages from CSV', 
			'version' => 108, 
			'summary' => 'Import CSV files to create ProcessWire pages.',
			'author' => 'Ryan Cramer',
			'icon' => 'table',
			'page' => array(
				'name' => 'import-pages-csv',
				'parent' => 'setup',
				'title' => 'Import Pages CSV'
			),
			'requires' => 'ProcessWire>=3.0.123'
		);
	}

	/**
	 * Constants for the csvDuplicate session var
	 *
	 */
	const csvDuplicateSkip = 0;	
	const csvDuplicateNew = 1; 
	const csvDuplicateModify = 2; 

	/**
	 * Filename with path to CSV file
	 *
	 */
	protected $csvFilename = '';

	/**
	 * Instance of Template, used for imported pages
	 * 
	 * @var Template|null
	 *
	 */
	protected $template = null;

	/**
	 * Instance of Page, representing the parent Page for imported pages
	 * 
	 * @var Page|null
	 *
	 */
	protected $parent = null;

	/**
	 * List of Fieldtypes that we support importing to
	 *
	 */
	protected $allowedFieldtypes = array(
		'Checkbox',
		'Datetime',
		'Email',
		'File',
		'Float',
		'Integer',
		'Options',
		'Page',
		'PageTitle',
		'Text',
		'Textarea',
		'Toggle',
		'URL',
	);

	/**
	 * Initialize the module
	 *
	 */
	public function init() {
		parent::init();
		// ini_set('auto_detect_line_endings', true); // depreciated php 8
	}

	/**
	 * Executed when root url for module is accessed
	 *
	 */
	public function ___execute() {
		$form = $this->buildForm1(); 
		if($this->input->post('submit')) {
			if($this->processForm1($form)) $this->session->redirect('./fields/'); 
		}
		return $form->render();
	}

	/**
	 * Executed when ./fields/ url for module is accessed
	 *
	 */
	public function ___executeFields() {

		$this->template = $this->templates->get($this->sessionGet('csvTemplate')); 
		$this->parent = $this->pages->get($this->sessionGet('csvParent', new NullPage())); 
		$this->csvFilename = $this->sessionGet('csvFilename'); 
		$error = '';

		if(!$this->template || !$this->parent->id || !$this->csvFilename) {
			$error = "Missing required fields"; 
		} else if(!$this->parent->editable()) {
			$error = "Selected parent page is not editable";
		}  

		if($error) {
			$this->error($error); 
			$this->session->redirect("../");
		}
		$this->message("Using template: {$this->template}"); 
		$this->message("Using parent: {$this->parent->path}"); 

		$form = $this->buildForm2();	

		if($this->input->post('submit')) {
			return $this->processForm2($form); 
		} else { 
			return $form->render();
		}
	}

	/**
	 * Build the "Step 1" form
	 * 
	 * @return InputfieldForm
	 *
	 */
	protected function buildForm1() {

		/** @var InputfieldForm $form */
		$form = $this->modules->get("InputfieldForm"); 
		$form->description = "Step 1: Define source and destination";

		/** @var InputfieldSelect $f */
		$f = $this->modules->get("InputfieldSelect"); 	
		$f->name = 'template';
		$f->label = 'Template';
		$f->description = 'The pages you import will use the selected template.';
		$f->required = true; 
		$f->icon = 'cubes';
		$f->addOption(''); 
		foreach($this->templates as $t) {
			$f->addOption($t->id, $t->name);
		}
		$value = $this->sessionGet('csvTemplate');
		if($value) $f->attr('value', $value);
		$form->add($f); 	

		/** @var InputfieldPageListSelect $f */
		$f = $this->modules->get("InputfieldPageListSelect"); 
		$f->name = 'parent_id';
		$f->label = 'Parent Page';
		$f->icon = 'sitemap';
		$f->required = true; 
		$f->description = "The pages you import will be given this parent.";
		$value = $this->sessionGet('csvParent');
		if($value) $f->attr('value', $value); 
		$form->add($f); 

		/** @var InputfieldFile $f */
		$f = $this->modules->get("InputfieldFile"); 
		$f->name = 'csv_file';
		$f->label = 'CSV File';
		$f->icon = 'file-text';
		$f->extensions = 'csv txt';
		$f->maxFiles = 1; 
		$f->descriptionRows = 0; 
		$f->overwrite = true; 
		$f->required = false; 
		$f->description = 
			"The list of field names must be provided as the first row in the CSV file. " . 
			"UTF-8 compatible encoding is assumed. File must have the extension '.csv' or '.txt'. " . 
			"If you prefer, you may instead paste in CSV data in the 'More Options' section below. ";
		$form->add($f); 

		/** @var InputfieldFieldset $fieldset */
		$fieldset = $this->modules->get("InputfieldFieldset"); 
		$fieldset->attr('id', 'csv_advanced_options'); 
		$fieldset->label = "More Options";
		$fieldset->collapsed = Inputfield::collapsedYes; 
		$fieldset->icon = 'sliders';
		$form->add($fieldset);
		
		/** @var InputfieldRadios $f */
		$f = $this->modules->get("InputfieldRadios");
		$f->name = 'csv_delimeter';
		$f->label = 'Fields delimited by';
		$f->addOption(1, 'Commas');
		$f->addOption(2, 'Tabs');
		$value = (string) $this->sessionGet('csvDelimeter');
		if(strlen($value)) {
			$f->attr('value', $value === "\t" ? 2 : 1);
		} else {
			$f->attr('value', 1);
		}
		$f->columnWidth = 33;
		$fieldset->add($f);

		/** @var InputfieldText $f */
		$f = $this->modules->get("InputfieldText");
		$f->name = 'csv_enclosure';
		$f->label = 'Fields enclosed by';
		$f->description = "If you aren't sure, it is recommended you leave it at the default (\").";
		$f->attr('value', $this->sessionGet('csvEnclosure', '"'));
		$f->attr('maxlength', 1);
		$f->attr('size', 1);
		$f->columnWidth = 33;
		$fieldset->add($f);

		/** @var InputfieldInteger $f */
		$f = $this->modules->get("InputfieldInteger");
		$f->name = 'csv_max_rows';
		$f->label = 'Max rows to import';
		$f->description = "0 = no limit";
		$f->attr('value', (int) $this->sessionGet('csvMaxRows', 0));
		$f->attr('size', 5);
		$f->columnWidth = 34;
		$fieldset->add($f);

		/** @var InputfieldRadios $f */
		$f = $this->modules->get("InputfieldRadios");
		$f->name = 'csv_duplicate';
		$f->label = 'What to do with duplicate page names';
		$f->description = "When a row in a CSV file will result in a page with the same 'name' as one that's already there, what do you want to do?";
		$f->addOption(self::csvDuplicateSkip, 'Skip it');
		$f->addOption(self::csvDuplicateNew, 'Make the name unique and import new page');
		$f->addOption(self::csvDuplicateModify, 'Modify the existing page');
		$f->attr('value', (int) $this->sessionGet('csvDuplicate'));
		$fieldset->add($f);

		/** @var InputfieldRadios $f */
		$f = $this->modules->get("InputfieldRadios");
		$f->name = 'csv_add_page_refs';
		$f->label = 'Create new pages for Page references that do not exist?';
		$f->description =
			"When importing, if an existing Page for a FieldtypePage field cannot be found by title or name, " .
			"it can optionally be created during import. This requires that the column being imported to the " . 
			"FieldtypePage field contains a title or name for the Page. It also requires that the FieldtypePage " .
			"field is already configured to specify both the parent and template that it should use."; 
		$f->notes = 
			"Note that only the title and name properties are populated to created Page reference pages. " . 
			"If there are more properties you want to populate, create or import those pages ahead of time.";
		$f->addOption(1, 'Yes, create new pages for Page references that do not already exist');
		$f->addOption(0, 'No, do not create new pages for missing page references');
		$f->attr('value', (int) $this->sessionGet('csvAddPageRefs'));
		$fieldset->add($f);

		/** @var InputfieldTextarea $f */
		$f = $this->modules->get("InputfieldTextarea"); 
		$f->name = 'csv_data';
		$f->label = 'Paste in CSV Data';
		$f->icon = 'code';
		$f->description = 
			"If you prefer, you may paste in the CSV data here rather than uploading a file above. " . 
			"You should use one or the other, but not both.";
		$f->collapsed = Inputfield::collapsedBlank; 
		$fieldset->add($f); 

		$this->addSubmit($form, 'Continue to Step 2'); 

		return $form; 
	}

	/**
	 * Process the "Step 1" form and populate session variables with the results
	 * 
	 * @param InputfieldForm $form
	 * @return bool
	 *
	 */
	protected function processForm1(InputfieldForm $form) {

		$form->processInput($this->input->post); 
		if(count($form->getErrors())) return false;

		$this->sessionSet('csvTemplate', (int) $form->getChildByName('template')->value);
		$this->sessionSet('csvParent', (int) $form->getChildByName('parent_id')->value); 
		$this->sessionSet('csvDelimeter', $form->getChildByName('csv_delimeter')->value == 2 ? "\t" : ",");
		$this->sessionSet('csvEnclosure', substr($form->getChildByName('csv_enclosure')->value, 0, 1)); 
		$this->sessionSet('csvDuplicate', (int) $form->getChildByName('csv_duplicate')->value);
		$this->sessionSet('csvMaxRows', (int) $form->getChildByName('csv_max_rows')->value);
		$this->sessionSet('csvAddPageRefs', (int) $form->getChildByName('csv_add_page_refs')->value); 

		/** @var Pagefiles|Pagefile $csvFile */
		$csvFile = $form->getChildByName('csv_file')->value; 
		$csvData = $form->getChildByName('csv_data')->value; 
		
		$csvBasename = 'data-' . $this->user->id . '.csv';
		$csvFilename = $this->page->filesManager()->path() . $csvBasename;

		if(count($csvFile)) {
			$csvFile = $csvFile->first();
			$csvFile->rename($csvBasename);
			$csvFilename = $csvFile->filename; 

		} else if(strlen($csvData)) {
			file_put_contents($csvFilename, $csvData); 
			$this->wire('files')->chmod($csvFilename);

		} else {
			$csvFilename = '';
		}

		if(!$csvFilename || !is_file($csvFilename)) {
			$this->error("Missing required CSV file/data"); 
			return false;
		}

		$this->sessionSet('csvFilename', $csvFilename);
		
		return true; 
	}

	/**
	 * Build the "Step 2" form to connect the fields
	 * 
	 * @return InputfieldForm
	 *
	 */
	protected function buildForm2() {

		/** @var InputfieldForm $form */
		$form = $this->modules->get("InputfieldForm"); 
		$form->description = "Step 2: Connect the fields"; 
		$form->value = "<p>" . 
			"Below is a list of columns found in in the header of your CSV file. " . 
			"For each of them, select the field it should import to. " . 
			"Leave any fields you want to exclude blank. " . 
			"Once finished, click “Start Import” at the bottom of this page. " . 
			"Note: any field names in your CSV file that match those in your site " . 
			"will be automatically selected." . 
			"</p>";

		$fp = fopen($this->csvFilename, "r"); 
		if($fp === false) throw new WireException("Unable to open CSV file"); 
		
		$data = fgetcsv($fp, 0, $this->sessionGet('csvDelimeter'), $this->sessionGet('csvEnclosure'));

		foreach($data as $key => $value) {

			/** @var InputfieldSelect $f */
			$f = $this->modules->get('InputfieldSelect'); 
			$f->name = "csv" . $key;
			$f->label = $value; 
			$f->addOption(''); 

			foreach($this->template->fieldgroup as $field) {
				if(!$this->isAllowedField($field)) continue;
				$label = "$field->name – $field->label (" . $field->type->shortName . ")"; 
				$f->addOption($field->name, $label); 
				if($field->name == $value) $f->attr('value', $field->name); 
			}

			$form->add($f);
		}

		fclose($fp); 

		$this->addSubmit($form, 'Start Import');

		return $form;
	}

	/**
	 * Process the "Step 2" form and perform the import
	 * 
	 * @param InputfieldForm $form
	 * @return string
	 *
	 */
	protected function processForm2(InputfieldForm $form) {

		$form->processInput($this->input->post); 
		
		$csvFilename = $this->csvFilename;
		$fp = fopen($csvFilename, "r"); 
		if($fp === false) throw new WireException('Unable to open CSV file');
		
		$numImported = 0;
		$rowNum = 0;
		$maxRows = $this->sessionGet('csvMaxRows'); 
		$csvDelimeter = $this->sessionGet('csvDelimeter', ',');
		$csvEnclosure = $this->sessionGet('csvEnclosure', '"');

		while(($data = fgetcsv($fp, 0, $csvDelimeter, $csvEnclosure)) !== false) {
			$cnt = count($data); 

			// skip blank lines
			if(!$cnt || ($cnt == 1 && empty($data[0]))) continue;
			
			$rowNum++;
			
			// only start importing on second line (if $n)
			if($rowNum > 1 && $this->importPage($data, $form)) {
				$numImported++;
			}
			
			if($maxRows && $rowNum > $maxRows) break;
		}

		fclose($fp); 
		
		$this->wire('files')->unlink($csvFilename); 
		
		return $this->processForm2Markup($numImported); 
	}

	/**
	 * Provide the completion output markup for processForm2
	 * 
	 * @param int $numImported
	 * @return string
	 *
	 */
	protected function processForm2Markup($numImported) {
		return 	
			"<h2>Imported $numImported pages</h2>" . 
			"<p><a href='{$this->config->urls->admin}page/list/?open={$this->parent->id}'>See the imported pages</a></p>" . 
			"<p><a href='../'>Import more pages</a></p>";
	}

	/**
	 * Import a single page
	 * 
	 * @param array $data
	 * @param InputfieldForm $form
	 * @return bool
	 *
	 */
	protected function importPage(array $data, InputfieldForm $form) {
		
		$page = $this->wire('pages')->newPage(array('template' => $this->template));
		$page->parent = $this->parent; 
		$page->set('ImportPagesCSVData', array()); // data to set after page is saved
		$page->setTrackChanges(true);
		$fieldNames = array();

		foreach($form as $f) {
			if(!preg_match('/^csv(\d+)$/', $f->name, $matches)) continue; 
			$key = (int) $matches[1]; 
			$value = $data[$key]; 
			$name = $f->value; 
			if(!$name) continue; 
			$this->importPageValue($page, $name, $value); 
			$fieldNames[] = $name; 
		}	

		if(!$page->name) {
			$this->error(
				"Unable to import page because it has no required 'title' field or it is blank.<br />" . 
				"<pre>" . print_r($data, true) . "</pre>", 
				Notice::allowMarkup
			);
			return false;
		}
		
		$existingPage = $this->wire('pages')->get("parent_id=$this->parent, name=$page->name"); 
		
		if($existingPage->id) {
			// existing page
			if($this->sessionGet('csvDuplicate') == self::csvDuplicateNew) {
				$page->name = $this->getUniquePageName($page->name);
				$page = $this->savePage($page, true);

			} else if($this->sessionGet('csvDuplicate') == self::csvDuplicateModify) {
				$page = $this->modifyPage($existingPage, $page, $fieldNames);

			} else {
				$this->message("Skipping row with duplicate name '$page->name'");
			}
		} else {
			// new page
			$page = $this->savePage($page, true);
		}

		// import post-save data, like files
		if($page->id && $page->if('ImportPagesCSVData')) {
			foreach($page->get('ImportPagesCSVData') as $name => $value) {
				$page->set($name, $value);
			}
			$page->save();
		}

		return $page->id > 0; 
	}

	/**
	 * Assign a value to a page field 
	 * 
	 * @param Page $page Page being imported to
	 * @param string $name Field name or page property name
	 * @param mixed $value Value to set
	 * @return bool
	 *
	 */
	protected function ___importPageValue(Page $page, $name, $value) {

		$field = $this->fields->get($name); 
		if(!$field) return false;
		
		if($field->type instanceof FieldtypeFile) {

			$value = trim($value);
			// split delimeted data to an array
			$value = preg_split('/[\r\n\t|]+/', $value);
			if($field->get('maxFiles') == 1) $value = array_shift($value);
			$data = $page->get('ImportPagesCSVData');
			$data[$name] = $value;
			$page->set('ImportPagesCSVData', $data);

		} else if($field->type instanceof FieldtypePage) {

			// $oldValue = $page->id ? (string) $page->get($name) : null;

			if($this->sessionGet('csvAddPageRefs')) {
				$field->setQuietly('_sanitizeValueString', 'create');
				$page->set($name, $value);
				$field->offsetUnset('_sanitizeValueString');
			} else {
				$page->set($name, $value);
			}

		} else if($name === 'title') {
			$page->set($name, $value);
			if(!$page->name) $page->name = $this->sanitizer->pageName($value, Sanitizer::translate);

		} else { 
			$page->set($name, $value); 
		}
		
		return true;
	}

	/**
	 * Modify an existing page with CSV data
	 * 
	 * @param Page $existingPage
	 * @param Page $newPage
	 * @param array $fieldNames
	 * @return bool|Page
	 *
	 */
	protected function modifyPage(Page $existingPage, Page $newPage, array $fieldNames) {

		if($existingPage->template->id != $newPage->template->id) {
			$this->error("Unable to modify '$existingPage->name' because it uses a different template '$existingPage->template'"); 
			return false;
		}
	
		/** @var array $data */
		$data = $newPage->get('ImportPagesCSVData'); 

		foreach($fieldNames as $fieldName) {
			if(isset($data[$fieldName])) {
				$value = $data[$fieldName]; 
			} else {
				$value = $newPage->get($fieldName); 
			}
		
			$field = $this->wire('fields')->get($fieldName); 
			$existingValue = $existingPage->get($fieldName); 
			$existingPage->set($fieldName, $value); 
			
			if($field->type instanceof FieldtypePage) {
				if(((string) $existingValue) === ((string) $newPage->get($fieldName))) {
					$existingPage->untrackChange($fieldName); 
				}
			}
			
		}

		return $this->savePage($existingPage);
	}

	/**
	 * Wrapper to PW's page save to capture exceptions so importPage can try name variations if necessary
	 * 
	 * @param Page $page
	 * @param bool $reportErrors
	 * @return Page
	 *
	 */
	protected function savePage(Page $page, $reportErrors = true) {

		try {
			$label = $page->id ? "Modified" : "Created";
			$changes = implode(', ', $page->getChanges());
			if(strlen($changes)) {
				$changes = "($changes)";
				$page->save();
				$this->message("$label $page->path $changes");
				$page->setQuietly('_csvSaved', true); 
			} else {
				$page->setQuietly('_csvSaved', false); 
			}

		} catch(\Exception $e) {
			if($reportErrors) $this->error($e->getMessage()); 
		}

		return $page; 
	}

	/**
	 * Given a page name, check that it is unique and return it or a unique numbered variation of it 
	 * 
	 * @param string $pageName
	 * @return string
	 *
	 */
	protected function getUniquePageName($pageName) {

		return $this->wire('pages')->names()->uniquePageName(array(
			'name' => $pageName, 
			'parent' => $this->parent
		));
	
		/*
		 * Original method for reference
		$n = 0;
		do {
			$testName = $pageName . "-" . (++$n);
			$test = $this->parent->child("name=$testName, include=all");
			if(!$test->id) break;
		} while(1); 
		return $testName; 
		*/
	}

	/**
	 * Add a submit button, moved to a function so we don't have to do this twice
	 * 
	 * @param InputfieldForm $form
	 * @param string $value
	 *
	 */
	protected function addSubmit(InputfieldForm $form, $value = 'Submit') {
		/** @var InputfieldSubmit $f */
		$f = $this->modules->get("InputfieldSubmit"); 
		$f->name = 'submit';
		$f->value = $value; 
		$form->add($f); 
	}

	/**
	 * Is given Field allowed for importing?
	 * 
	 * @param Field $field
	 * @return bool
	 * 
	 */
	protected function ___isAllowedField(Field $field) {
		$valid = false;
		foreach($this->allowedFieldtypes as $name) {
			if(wireInstanceOf($field->type, "Fieldtype$name")) $valid = true;
			if($valid) break;
		}
		return $valid;
	}

	/**
	 * Get session value 
	 * 
	 * @param string $key
	 * @param null $fallback
	 * @return string|int|null
	 * 
	 */
	protected function sessionGet($key, $fallback = null) {
		$value = $this->session->getFor($this, $key);
		if($value === null) $value = $fallback;
		return $value;
	}

	/**
	 * Set session value 
	 * 
	 * @param string $key
	 * @param string|int $value
	 * 
	 */
	protected function sessionSet($key, $value) {
		$this->session->setFor($this, $key, $value);
	}
}