Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Bugfix] Too many embedded layers cause php to hit max_execution_time #4093

Merged
merged 2 commits into from
Feb 5, 2024
Merged

[Bugfix] Too many embedded layers cause php to hit max_execution_time #4093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e2ac5a3
speed-up loading of embedded layers
mind84 Jan 24, 2024
ccb7f0e
embedded relations loading, embedded projects loading when the main p…
mind84 Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 23 additions & 7 deletions lizmap/modules/lizmap/lib/Project/Project.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ public function __construct($key, Repository $rep, App\AppContextInterface $appC
}
}
$rewriteCache = false;
// embedded layers
$embeddedProjects = array();
foreach ($data['qgis']['layers'] as $index => $layer) {
if (array_key_exists('embedded', $layer)
&& $layer['embedded'] == '1'
Expand All @@ -172,16 +174,30 @@ public function __construct($key, Repository $rep, App\AppContextInterface $appC
|| $layer['qgsmtime'] < filemtime($layer['file'])
)
) {
$qgsProj = new QgisProject($layer['file'], $services, $this->appContext);
$newLayer = $qgsProj->getLayerDefinition($layer['id']);
$newLayer['qgsmtime'] = filemtime($layer['file']);
$newLayer['file'] = $layer['file'];
$newLayer['embedded'] = 1;
$newLayer['projectPath'] = $layer['projectPath'];
$data['qgis']['layers'][$index] = $newLayer;
if (!array_key_exists($layer['file'], $embeddedProjects)) {
$embeddedProjects[$layer['file']] = array();
}
// populate array of embedded layers
$embeddedProjects[$layer['file']][$index] = $layer;
$rewriteCache = true;
}
}

// loop through the embedded projects if any, to get the embedded layers definition
foreach ($embeddedProjects as $projectPath => $embeddedLayers) {
if (is_array($embeddedLayers)) {
$embeddedProject = new QgisProject($projectPath, $this->services, $this->appContext);
foreach ($embeddedLayers as $index => $embeddedLayer) {
$newLayer = $embeddedProject->getLayerDefinition($embeddedLayer['id']);
$newLayer['qgsmtime'] = filemtime($embeddedLayer['file']);
$newLayer['file'] = $embeddedLayer['file'];
$newLayer['embedded'] = 1;
$newLayer['projectPath'] = $embeddedLayer['projectPath'];
$data['qgis']['layers'][$index] = $newLayer;
}
}
}

if ($rewriteCache) {
$this->cacheHandler->storeProjectData($data);
}
Expand Down
60 changes: 47 additions & 13 deletions lizmap/modules/lizmap/lib/Project/QgisProject.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,15 +581,28 @@ public function xpathQuery($query)
*
* @deprecated
*
* @param mixed $layerId
* @param mixed $layerId
* @param null|array $embeddedRelationsProjects reference to associative array path(key) - QgisProject instance (value)
*
* @return \SimpleXMLElement[]
*/
public function getXmlLayer($layerId)
public function getXmlLayer($layerId, &$embeddedRelationsProjects = null)
{
$layer = $this->getLayerDefinition($layerId);
if ($layer && array_key_exists('embedded', $layer) && $layer['embedded'] == 1) {
$qgsProj = new QgisProject(realpath(dirname($this->path).DIRECTORY_SEPARATOR.$layer['projectPath']), $this->services, $this->appContext);
// avoid reloading the same qgis project multiple times while reading relations by checking embeddedRelationsProjects param
// If this array is null or does not contains the corresponding qgis project, then the function if forced to load a new qgis project
if ($embeddedRelationsProjects && array_key_exists($layer['projectPath'], $embeddedRelationsProjects)) {
// use QgisProject instance already created
$qgsProj = $embeddedRelationsProjects[$layer['projectPath']];
} else {
// create new QgisProject instance
$qgsProj = new QgisProject(realpath(dirname($this->path).DIRECTORY_SEPARATOR.$layer['projectPath']), $this->services, $this->appContext);
// update the array, if exists
if ($embeddedRelationsProjects) {
$embeddedRelationsProjects[$layer['projectPath']] = $qgsProj;
}
}

return $qgsProj->getXml()->xpath("//maplayer[id='{$layerId}']");
}
Expand Down Expand Up @@ -1336,11 +1349,15 @@ protected function readRelations($xml)
$pivotGather = array();
$pivot = array();
if ($xmlRelations) {
// Store qgisProjects references in a key=>value array and pass it by reference along the methods that loads and validates relations.
// This avoid to reload the same QgisProject instance multiple times, if there are many "embedded relations" referencing the same (embedded) qgis project
$embeddedRelationsProjects = array();

/** @var \SimpleXMLElement $relation */
foreach ($xmlRelations[0] as $relation) {
$relationObj = $relation->attributes();

$relationField = $this->readRelationField($relation);
$relationField = $this->readRelationField($relation, $embeddedRelationsProjects);
if ($relationField === null) {
// no corresponding layer
continue;
Expand Down Expand Up @@ -1384,12 +1401,13 @@ protected function readRelations($xml)

/**
* @param \SimpleXMLElement $relationXml
* @param null|array $embeddedRelationsProjects
*/
protected function readRelationField($relationXml)
protected function readRelationField($relationXml, &$embeddedRelationsProjects = null)
{
$referencedLayerId = $relationXml->attributes()->referencedLayer;

$_referencedLayerXml = $this->getXmlLayer($referencedLayerId);
$_referencedLayerXml = $this->getXmlLayer($referencedLayerId, $embeddedRelationsProjects);
if (count($_referencedLayerXml) == 0) {
return null;
}
Expand Down Expand Up @@ -1486,18 +1504,20 @@ protected function readLayers($xml)
if (!$xmlLayers) {
return $layers;
}
// Associative array that stores the embedded projects path as key and the list of embedded layers attributes as value.
// The the embedded layers definition are retreived outside the main foreach loop to avoid loading the same embedded qgis project multiple times
// for each embedded layer
$embeddedProjects = array();

foreach ($xmlLayers as $xmlLayer) {
$attributes = $xmlLayer->attributes();
if (isset($attributes['embedded']) && (string) $attributes->embedded == '1') {
$xmlFile = realpath(dirname($this->path).DIRECTORY_SEPARATOR.(string) $attributes->project);
$qgsProj = new QgisProject($xmlFile, $this->services, $this->appContext);
$layer = $qgsProj->getLayerDefinition((string) $attributes->id);
$layer['qgsmtime'] = filemtime($xmlFile);
$layer['file'] = $xmlFile;
$layer['embedded'] = 1;
$layer['projectPath'] = (string) $attributes->project;
$layers[] = $layer;
if (!array_key_exists($xmlFile, $embeddedProjects)) {
$embeddedProjects[$xmlFile] = array();
}
// populate array of embedded layers
$embeddedProjects[$xmlFile][] = $attributes;
} else {
$layer = array(
'type' => (string) $attributes->type,
Expand Down Expand Up @@ -1659,6 +1679,20 @@ protected function readLayers($xml)
$layers[] = $layer;
}
}
// loop through the embedded projects if any, to get the embedded layers definition
foreach ($embeddedProjects as $projectPath => $layersAttributes) {
if (is_array($layersAttributes)) {
$embeddedProject = new QgisProject($projectPath, $this->services, $this->appContext);
foreach ($layersAttributes as $attributes) {
$layer = $embeddedProject->getLayerDefinition((string) $attributes->id);
$layer['qgsmtime'] = filemtime($projectPath);
$layer['file'] = $projectPath;
$layer['embedded'] = 1;
$layer['projectPath'] = (string) $attributes->project;
$layers[] = $layer;
}
}
}

return $layers;
}
Expand Down
Loading