Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Download all WordPress assets on boot #1532

Merged
merged 31 commits into from
Jul 10, 2024
Merged
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
6d9e165
ZIP WP assets
bgrgicak Jun 20, 2024
75bc2bb
Add afterWordPressInstall hook to boot
bgrgicak Jun 21, 2024
6c2b0ad
Download assets after WordPress is installed
bgrgicak Jun 21, 2024
57daebb
Merge branch 'trunk' into add/wp-static-asset-zip
bgrgicak Jun 21, 2024
42be4fc
Use getWordPressVersionFromPhp
bgrgicak Jun 21, 2024
8beee0c
Remove bootWordPress hook
bgrgicak Jun 24, 2024
6cf5fef
Use wordPressSiteUrl
bgrgicak Jun 24, 2024
1f2347e
Stream static asset response instead of awaiting
bgrgicak Jun 24, 2024
f1e2784
Use request handler to get document root
bgrgicak Jun 24, 2024
3c98912
Add warning if assets weren't downloaded
bgrgicak Jun 25, 2024
e703eff
Merge branch 'trunk' into add/wp-static-asset-zip
bgrgicak Jun 25, 2024
2e3910c
Async fetch
bgrgicak Jun 25, 2024
d2ca9c2
Update error messages
bgrgicak Jun 25, 2024
4a1f8d0
Use wpVersionToStaticAssetsDirectory
bgrgicak Jun 25, 2024
efb4b57
Use getLoadedWordPressVersion
bgrgicak Jun 25, 2024
b2ba22b
downloadWordPressAssets on load
bgrgicak Jun 26, 2024
1aad0d0
Remove logs
bgrgicak Jun 26, 2024
db8c25f
Merge branch 'trunk' into add/wp-static-asset-zip
bgrgicak Jun 26, 2024
de206a1
Merge branch 'trunk' into add/wp-static-asset-zip
bgrgicak Jun 28, 2024
a6f20bd
Empty remote asset paths after done instead of deleting it, update co…
bgrgicak Jun 28, 2024
ebfa224
Merge branch 'trunk' into add/wp-static-asset-zip
bgrgicak Jul 1, 2024
37c0b91
Update packages/playground/remote/src/lib/worker-thread.ts
bgrgicak Jul 2, 2024
ee044ae
Linter fixes
bgrgicak Jul 2, 2024
f8dab44
Build static assets
bgrgicak Jul 2, 2024
6f48337
Use unzipFile
bgrgicak Jul 4, 2024
d34f066
Fix dependencies
bgrgicak Jul 4, 2024
1feb41a
Update static asset zips
bgrgicak Jul 4, 2024
f3fe08d
Merge branch 'trunk' into add/wp-static-asset-zip
bgrgicak Jul 4, 2024
f0378d0
Remove unused zip delete
bgrgicak Jul 10, 2024
7c205fa
Update remote asset PR comment
bgrgicak Jul 10, 2024
e03c9ad
Merge branch 'trunk' into add/wp-static-asset-zip
bgrgicak Jul 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/playground/remote/src/lib/boot-playground-remote.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,13 @@ export async function bootPlaygroundRemote() {
) {
return await workerApi.bindOpfs(options, onProgress);
},

/**
* Download WordPress assets.
*/
async backfillStaticFilesRemovedFromMinifiedBuild() {
await workerApi.backfillStaticFilesRemovedFromMinifiedBuild();
},
};

await workerApi.isConnected();
Expand Down Expand Up @@ -232,6 +239,9 @@ export async function bootPlaygroundRemote() {
throw e;
}

wpFrame.addEventListener('load', () => {
webApi.backfillStaticFilesRemovedFromMinifiedBuild();
});
/*
* An assertion to make sure Playground Client is compatible
* with Remote<PlaygroundClient>
Expand Down
1 change: 1 addition & 0 deletions packages/playground/remote/src/lib/playground-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export interface WebClientMixin extends ProgressReceiver {
replayFSJournal: PlaygroundWorkerEndpoint['replayFSJournal'];
addEventListener: PlaygroundWorkerEndpoint['addEventListener'];
removeEventListener: PlaygroundWorkerEndpoint['removeEventListener'];
backfillStaticFilesRemovedFromMinifiedBuild: PlaygroundWorkerEndpoint['backfillStaticFilesRemovedFromMinifiedBuild'];

/** @inheritDoc @php-wasm/universal!UniversalPHP.onMessage */
onMessage: PlaygroundWorkerEndpoint['onMessage'];
Expand Down
94 changes: 92 additions & 2 deletions packages/playground/remote/src/lib/worker-thread.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { SyncProgressCallback, exposeAPI } from '@php-wasm/web';
import { EmscriptenDownloadMonitor } from '@php-wasm/progress';
import { setURLScope } from '@php-wasm/scopes';
import { joinPaths } from '@php-wasm/util';
import { joinPaths, phpVar } from '@php-wasm/util';
import { wordPressSiteUrl } from './config';
import {
getWordPressModuleDetails,
Expand Down Expand Up @@ -35,7 +35,7 @@ import transportFetch from './playground-mu-plugin/playground-includes/wp_http_f
import transportDummy from './playground-mu-plugin/playground-includes/wp_http_dummy.php?raw';
/** @ts-ignore */
import playgroundWebMuPlugin from './playground-mu-plugin/0-playground.php?raw';
import { PHPWorker } from '@php-wasm/universal';
import { PHP, PHPWorker } from '@php-wasm/universal';
import {
bootWordPress,
getLoadedWordPressVersion,
Expand Down Expand Up @@ -186,6 +186,96 @@ export class PlaygroundWorkerEndpoint extends PHPWorker {
async replayFSJournal(events: FilesystemOperation[]) {
return replayFSJournal(this.__internal_getPHP()!, events);
}

async backfillStaticFilesRemovedFromMinifiedBuild() {
await backfillStaticFilesRemovedFromMinifiedBuild(
this.__internal_getPHP()!
);
}
}

async function backfillStaticFilesRemovedFromMinifiedBuild(php: PHP) {
if (!php.requestHandler) {
logger.warn('No PHP request handler available');
return;
}

try {
const remoteAssetListPath = joinPaths(
php.requestHandler.documentRoot,
'wordpress-remote-asset-paths'
);

/**
* Don't download static assets if they're already downloaded.
* WordPress may be loaded either from a production release or a minified bundle.
* Minified bundles are shipped without most CSS files, JS files, and other static assets.
* Instead, they contain a list of remote assets in wordpress-remote-asset-paths.
* We use this list to determine if we should fetch them on demand or if they are already downloaded.
* If the list is empty, we assume the assets are already downloaded.
* See https://github.com/WordPress/wordpress-playground/pull/1531
bgrgicak marked this conversation as resolved.
Show resolved Hide resolved
*/
if (
!php.fileExists(remoteAssetListPath) ||
(await php.readFileAsText(remoteAssetListPath)) === ''
adamziel marked this conversation as resolved.
Show resolved Hide resolved
) {
return;
}
const wpVersion = await getLoadedWordPressVersion(php.requestHandler);
const staticAssetsDirectory =
wpVersionToStaticAssetsDirectory(wpVersion);
adamziel marked this conversation as resolved.
Show resolved Hide resolved
if (!staticAssetsDirectory) {
return;
}
const response = await fetch(
joinPaths('/', staticAssetsDirectory, 'wordpress-static.zip')
);

if (!response.ok) {
throw new Error(
`Failed to fetch WordPress static assets: ${response.status} ${response.statusText}`
);
}

const zipPath = '/tmp/wordpress-static-assets.zip';
bgrgicak marked this conversation as resolved.
Show resolved Hide resolved
await php.writeFile(
bgrgicak marked this conversation as resolved.
Show resolved Hide resolved
zipPath,
new Uint8Array(await response.arrayBuffer())
);
await php.run({
code: `<?php
$document_root = ${phpVar(php.requestHandler.documentRoot)};
$zip_path = ${phpVar(zipPath)};
$zip = new ZipArchive;
$res = $zip->open($zip_path);
if ($res !== TRUE) {
return;
}
for ($i = 0; $i < $zip->numFiles; $i++) {
$filename = $zip->getNameIndex($i);
$extractPath = str_replace('wordpress-static', $document_root, $filename);
// Create directories if they don't exist
if (substr($filename, -1) === '/') {
if (is_dir($extractPath)) {
continue;
}
mkdir($extractPath, 0777, true);
} else {
// Extract files
copy("zip://$zip_path#".$filename, $extractPath);
}
}
$zip->close();
`,
});
if (await php.fileExists(zipPath)) {
await php.unlink(zipPath);
}
Copy link
Collaborator

@adamziel adamziel Jun 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this can be replaced by the unzipFile helper:

Suggested change
const zipPath = '/tmp/wordpress-static-assets.zip';
await php.writeFile(
zipPath,
new Uint8Array(await response.arrayBuffer())
);
await php.run({
code: `<?php
$document_root = ${phpVar(php.requestHandler.documentRoot)};
$zip_path = ${phpVar(zipPath)};
$zip = new ZipArchive;
$res = $zip->open($zip_path);
if ($res !== TRUE) {
return;
}
for ($i = 0; $i < $zip->numFiles; $i++) {
$filename = $zip->getNameIndex($i);
$extractPath = str_replace('wordpress-static', $document_root, $filename);
// Create directories if they don't exist
if (substr($filename, -1) === '/') {
if (is_dir($extractPath)) {
continue;
}
mkdir($extractPath, 0777, true);
} else {
// Extract files
copy("zip://$zip_path#".$filename, $extractPath);
}
}
$zip->close();
`,
});
if (await php.fileExists(zipPath)) {
await php.unlink(zipPath);
}
await unzipFile(
php.requestHandler.documentRoot,
new File([ new Uint8Array(await response.arrayBuffer()) ], "assets.zip")
);

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried, but it didn't work. I also couldn't get it to work with just unzip in PHP or JS.
The only way unzipping works for me is file by file, but I might be doing something wrong.

Copy link
Collaborator

@adamziel adamziel Jul 3, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to be the problem:

$extractPath = str_replace('wordpress-static', $document_root, $filename);

If wp-content, wp-include etc was already at the zip root, you could unzip it into /wordpress and the helper would do the right thing.

// Clear the remote asset list to indicate that the assets are downloaded.
await php.writeFile(remoteAssetListPath, '');
} catch (e) {
logger.warn('Failed to download WordPress assets', e);
}
}

const apiEndpoint = new PlaygroundWorkerEndpoint(
Expand Down
10 changes: 8 additions & 2 deletions packages/playground/wordpress-builds/build/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,17 @@ RUN rm -rf wordpress-static/wp-content/mu-plugins
# to request a remote asset or delegate the request for a missing file to PHP.
RUN find wordpress-static -type f | sed 's#^wordpress-static/##'> wordpress-remote-asset-paths

# Make the remote asset listing available remotely so it can be downloaded
# Make the remote asset listing available remotely so it can be downloaded
# directly in cases where an older minified WordPress build without this file
# has been saved to browser storage.
RUN cp wordpress-remote-asset-paths wordpress-static/

# ZIP the static files
RUN zip -r wordpress-static.zip wordpress-static/
bgrgicak marked this conversation as resolved.
Show resolved Hide resolved

# Move ZIP to the public output directory
RUN cp wordpress-static.zip wordpress-static/

# Move the static files to the final output directory
RUN mkdir /root/output/$OUT_FILENAME
RUN mv wordpress-static/* /root/output/$OUT_FILENAME/
Expand Down Expand Up @@ -135,6 +141,6 @@ RUN cd wordpress && \
# Build the final wp.zip file
RUN mv wordpress /wordpress && \
cp wordpress-remote-asset-paths /wordpress/ && \
cp wordpress-static.zip /wordpress/ && \
cd /wordpress && \
zip /root/output/$OUT_FILENAME.zip -r .

Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
2 changes: 1 addition & 1 deletion packages/playground/wordpress/src/boot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ async function isWordPressInstalled(php: PHP) {
return (
(
await php.run({
code: `<?php
code: `<?php
require '${php.documentRoot}/wp-load.php';
echo is_blog_installed() ? '1' : '0';
`,
Expand Down
18 changes: 7 additions & 11 deletions packages/playground/wordpress/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,7 @@ import { PHP, UniversalPHP } from '@php-wasm/universal';
import { joinPaths, phpVar } from '@php-wasm/util';
import { unzipFile } from '@wp-playground/common';
export { bootWordPress } from './boot';
export {
getLoadedWordPressVersion,
isSupportedWordPressVersion,
} from './version-detect';

export * from './version-detect';
bgrgicak marked this conversation as resolved.
Show resolved Hide resolved
export * from './rewrite-rules';

/**
Expand All @@ -21,7 +17,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
await php.writeFile(
'/internal/shared/preload/env.php',
`<?php

// Allow adding filters/actions prior to loading WordPress.
// $function_to_add MUST be a string.
function playground_add_filter( $tag, $function_to_add, $priority = 10, $accepted_args = 1 ) {
Expand All @@ -31,7 +27,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
function playground_add_action( $tag, $function_to_add, $priority = 10, $accepted_args = 1 ) {
playground_add_filter( $tag, $function_to_add, $priority, $accepted_args );
}

// Load our mu-plugins after customer mu-plugins
// NOTE: this means our mu-plugins can't use the muplugins_loaded action!
playground_add_action( 'muplugins_loaded', 'playground_load_mu_plugins', 0 );
Expand Down Expand Up @@ -60,7 +56,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
}
return $redirect_url;
} );

// Needed because gethostbyname( 'wordpress.org' ) returns
// a private network IP address for some reason.
add_filter( 'allowed_redirect_hosts', function( $deprecated = '' ) {
Expand All @@ -78,7 +74,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
if(!file_exists(WP_CONTENT_DIR . '/fonts')) {
mkdir(WP_CONTENT_DIR . '/fonts');
}

$log_file = WP_CONTENT_DIR . '/debug.log';
define('ERROR_LOG_FILE', $log_file);
ini_set('error_log', $log_file);
Expand All @@ -91,7 +87,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
await php.writeFile(
'/internal/shared/preload/error-handler.php',
`<?php
(function() {
(function() {
$playground_consts = [];
if(file_exists('/internal/shared/consts.json')) {
$playground_consts = @json_decode(file_get_contents('/internal/shared/consts.json'), true) ?: [];
Expand Down Expand Up @@ -207,7 +203,7 @@ export async function preloadSqliteIntegration(

/**
* Loads the SQLite integration plugin before WordPress is loaded
* and without creating a drop-in "db.php" file.
* and without creating a drop-in "db.php" file.
*
* Technically, it creates a global $wpdb object whose only two
* purposes are to:
Expand Down
1 change: 0 additions & 1 deletion packages/playground/wordpress/src/version-detect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ export async function getLoadedWordPressVersion(
if (!versionString) {
throw new Error('Unable to read loaded WordPress version.');
}

return versionStringToLoadedWordPressVersion(versionString);
}

Expand Down
Loading