Download all WordPress assets on boot (#1532)

## Motivation for the change, related issues

To achieve [offline
support](#1483) we
need to download all WordPress files into the Playground filesystem.
Today Playground downloads WordPress partially on boot and downloads
assets as they are needed using fetch requests.

This PR downloads all assets into the Playground filesystem without
blocking the boot process.

## Implementation details

Zip files are generated using the WordPress build script and are
accessible in the `/wp-VERSION/` folder like any other assets.

The download is triggered after WordPress is installed. The fetch is
async and won't block the boot process.

In case some of the assets are needed immediately on boot they will
still be downloaded using fetch to ensure they are available to
WordPress.

Once the assets are downloaded they are stored in the Playground
filesystem and will be served from there.

## Testing Instructions (or ideally a Blueprint)

- Run `npx nx bundle-wordpress:nightly playground-wordpress-builds`
- Confirm that a
`packages/playground/wordpress-builds/public/wp-nightly/wordpress-static.zip`
file was created
- [Load Playground using this
blueprint](http://127.0.0.1:5400/website-server/?php=8.0&wp=nightly)
- In network tools see that the `wordpress-static.zip` file was
downloaded
- Open `/wp-admin/` 
- In network tools confirm that static assets like _thickbox.js_ were
loaded from the service worker instead of a fetch

packages/playground/remote/src/lib/boot-playground-remote.ts

@@ -197,6 +197,13 @@ export async function bootPlaygroundRemote() {
  ) {
  return await workerApi.bindOpfs(options, onProgress);
  },
+
+ /**
+ * Download WordPress assets.
+ */
+ async backfillStaticFilesRemovedFromMinifiedBuild() {
+ await workerApi.backfillStaticFilesRemovedFromMinifiedBuild();
+ },
  };
 
  await workerApi.isConnected();
@@ -232,6 +239,9 @@ export async function bootPlaygroundRemote() {
  throw e;
  }
 
+ wpFrame.addEventListener('load', () => {
+ webApi.backfillStaticFilesRemovedFromMinifiedBuild();
+ });
  /*
  * An assertion to make sure Playground Client is compatible
  * with Remote<PlaygroundClient>

packages/playground/remote/src/lib/playground-client.ts

@@ -52,6 +52,7 @@ export interface WebClientMixin extends ProgressReceiver {
  replayFSJournal: PlaygroundWorkerEndpoint['replayFSJournal'];
  addEventListener: PlaygroundWorkerEndpoint['addEventListener'];
  removeEventListener: PlaygroundWorkerEndpoint['removeEventListener'];
+ backfillStaticFilesRemovedFromMinifiedBuild: PlaygroundWorkerEndpoint['backfillStaticFilesRemovedFromMinifiedBuild'];
 
  /** @inheritDoc @php-wasm/universal!UniversalPHP.onMessage */
  onMessage: PlaygroundWorkerEndpoint['onMessage'];

packages/playground/remote/src/lib/worker-thread.ts

@@ -35,13 +35,14 @@ import transportFetch from './playground-mu-plugin/playground-includes/wp_http_f
 import transportDummy from './playground-mu-plugin/playground-includes/wp_http_dummy.php?raw';
 /** @ts-ignore */
 import playgroundWebMuPlugin from './playground-mu-plugin/0-playground.php?raw';
-import { PHPWorker } from '@php-wasm/universal';
+import { PHP, PHPWorker } from '@php-wasm/universal';
 import {
  bootWordPress,
  getLoadedWordPressVersion,
 } from '@wp-playground/wordpress';
 import { wpVersionToStaticAssetsDirectory } from '@wp-playground/wordpress-builds';
 import { logger } from '@php-wasm/logger';
+import { unzipFile } from '@wp-playground/common';
 
 const scope = Math.random().toFixed(16);
 
@@ -186,6 +187,67 @@ export class PlaygroundWorkerEndpoint extends PHPWorker {
  async replayFSJournal(events: FilesystemOperation[]) {
  return replayFSJournal(this.__internal_getPHP()!, events);
  }
+
+ async backfillStaticFilesRemovedFromMinifiedBuild() {
+ await backfillStaticFilesRemovedFromMinifiedBuild(
+ this.__internal_getPHP()!
+ );
+ }
+}
+
+async function backfillStaticFilesRemovedFromMinifiedBuild(php: PHP) {
+ if (!php.requestHandler) {
+ logger.warn('No PHP request handler available');
+ return;
+ }
+
+ try {
+ const remoteAssetListPath = joinPaths(
+ php.requestHandler.documentRoot,
+ 'wordpress-remote-asset-paths'
+ );
+
+ /**
+ * Don't download static assets if they're already downloaded.
+ * WordPress may be loaded either from a production release or a minified bundle.
+ * Minified bundles are shipped without most CSS files, JS files, and other static assets.
+ * Instead, they contain a list of remote assets in wordpress-remote-asset-paths.
+ * We use this list to determine if we should fetch them on demand or if they are already downloaded.
+ * If the list is empty, we assume the assets are already downloaded.
+ * See https://github.com/WordPress/wordpress-playground/pull/1531 to understand how we use the remote asset list to backfill assets on demand.
+ */
+ if (
+ !php.fileExists(remoteAssetListPath) ||
+ (await php.readFileAsText(remoteAssetListPath)) === ''
+ ) {
+ return;
+ }
+ const wpVersion = await getLoadedWordPressVersion(php.requestHandler);
+ const staticAssetsDirectory =
+ wpVersionToStaticAssetsDirectory(wpVersion);
+ if (!staticAssetsDirectory) {
+ return;
+ }
+ const response = await fetch(
+ joinPaths('/', staticAssetsDirectory, 'wordpress-static.zip')
+ );
+
+ if (!response.ok) {
+ throw new Error(
+ `Failed to fetch WordPress static assets: ${response.status} ${response.statusText}`
+ );
+ }
+
+ await unzipFile(
+ php,
+ new File([await response.blob()], 'wordpress-static.zip'),
+ php.requestHandler.documentRoot
+ );
+ // Clear the remote asset list to indicate that the assets are downloaded.
+ await php.writeFile(remoteAssetListPath, '');
+ } catch (e) {
+ logger.warn('Failed to download WordPress assets', e);
+ }
 }
 
 const apiEndpoint = new PlaygroundWorkerEndpoint(

packages/playground/wordpress-builds/build/Dockerfile

@@ -44,11 +44,19 @@ RUN rm -rf wordpress-static/wp-content/mu-plugins
 # to request a remote asset or delegate the request for a missing file to PHP.
 RUN find wordpress-static -type f | sed 's#^wordpress-static/##'> wordpress-remote-asset-paths
 
-# Make the remote asset listing available remotely so it can be downloaded 
+# Make the remote asset listing available remotely so it can be downloaded
 # directly in cases where an older minified WordPress build without this file
 # has been saved to browser storage.
 RUN cp wordpress-remote-asset-paths wordpress-static/
 
+# ZIP the static files
+RUN cd wordpress-static/ && \
+ zip -r ../wordpress-static.zip . && \
+ cd ..
+
+# Move ZIP to the public output directory
+RUN cp wordpress-static.zip wordpress-static/
+
 # Move the static files to the final output directory
 RUN mkdir /root/output/$OUT_FILENAME
 RUN mv wordpress-static/* /root/output/$OUT_FILENAME/
@@ -135,6 +143,6 @@ RUN cd wordpress && \
 # Build the final wp.zip file
 RUN mv wordpress /wordpress && \
  cp wordpress-remote-asset-paths /wordpress/ && \
+ cp wordpress-static.zip /wordpress/ && \
  cd /wordpress && \
  zip /root/output/$OUT_FILENAME.zip -r .
-

packages/playground/wordpress/src/boot.ts

@@ -219,7 +219,7 @@ async function isWordPressInstalled(php: PHP) {
  return (
  (
  await php.run({
- code: `<?php 
+ code: `<?php
  require '${php.documentRoot}/wp-load.php';
  echo is_blog_installed() ? '1' : '0';
  `,

packages/playground/wordpress/src/index.ts

@@ -2,8 +2,8 @@ import { PHP, UniversalPHP } from '@php-wasm/universal';
 import { joinPaths, phpVar } from '@php-wasm/util';
 import { unzipFile } from '@wp-playground/common';
 export { bootWordPress } from './boot';
-export { getLoadedWordPressVersion } from './version-detect';
 
+export * from './version-detect';
 export * from './rewrite-rules';
 
 /**
@@ -18,7 +18,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
  await php.writeFile(
  '/internal/shared/preload/env.php',
  `<?php
- 
+
  // Allow adding filters/actions prior to loading WordPress.
  // $function_to_add MUST be a string.
  function playground_add_filter( $tag, $function_to_add, $priority = 10, $accepted_args = 1 ) {
@@ -28,7 +28,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
  function playground_add_action( $tag, $function_to_add, $priority = 10, $accepted_args = 1 ) {
  playground_add_filter( $tag, $function_to_add, $priority, $accepted_args );
  }
- 
+
  // Load our mu-plugins after customer mu-plugins
  // NOTE: this means our mu-plugins can't use the muplugins_loaded action!
  playground_add_action( 'muplugins_loaded', 'playground_load_mu_plugins', 0 );
@@ -57,7 +57,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
  }
  return $redirect_url;
  } );
- 
+
  // Needed because gethostbyname( 'wordpress.org' ) returns
  // a private network IP address for some reason.
  add_filter( 'allowed_redirect_hosts', function( $deprecated = '' ) {
@@ -75,7 +75,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
  if(!file_exists(WP_CONTENT_DIR . '/fonts')) {
  mkdir(WP_CONTENT_DIR . '/fonts');
  }
- 
+
  $log_file = WP_CONTENT_DIR . '/debug.log';
  define('ERROR_LOG_FILE', $log_file);
  ini_set('error_log', $log_file);
@@ -88,7 +88,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
  await php.writeFile(
  '/internal/shared/preload/error-handler.php',
  `<?php
- (function() { 
+ (function() {
  $playground_consts = [];
  if(file_exists('/internal/shared/consts.json')) {
  $playground_consts = @json_decode(file_get_contents('/internal/shared/consts.json'), true) ?: [];
@@ -204,7 +204,7 @@ export async function preloadSqliteIntegration(
 
 /**
  * Loads the SQLite integration plugin before WordPress is loaded
- * and without creating a drop-in "db.php" file. 
+ * and without creating a drop-in "db.php" file.
  *
  * Technically, it creates a global $wpdb object whose only two
  * purposes are to:

packages/playground/wordpress/src/version-detect.ts

@@ -15,7 +15,6 @@ export async function getLoadedWordPressVersion(
  if (!versionString) {
  throw new Error('Unable to read loaded WordPress version.');
  }
-
  return versionStringToLoadedWordPressVersion(versionString);
 }