Speed up fixture loading drastically to improve the setup experience, f…

…ixes #1457

README.md

@@ -33,7 +33,7 @@ These steps are provided for development purposes only.
  composer install
  npm install
  ```
- Ensure env vars are set up correctly, you probably need to set `APP_MAILER_FROM_EMAIL`, `APP_MAILER_FROM_NAME` and `APP_DEV_EMAIL_RECIPIENT` in .env.local and possibly `MAILER_DSN` if you need to receive email.
+ Ensure env vars are set up correctly, you probably need to set `APP_MAILER_FROM_EMAIL`, `APP_MAILER_FROM_NAME` and `APP_DEV_EMAIL_RECIPIENT` in `.env.local`. Set also `MAILER_DSN` if you'd like to receive email.
 
 3. Start the web server:
  ```bash
@@ -70,12 +70,19 @@ You should now be able to access the site, create a user, etc.
 You can get test data by running the fixtures:
 
 ```bash
-bin/console doctrine:fixtures:load
+bin/console doctrine:fixtures:load --group base
+bin/console doctrine:fixtures:load --group downloads --append
  ```
 
-This will create 100 packages from packagist.org, update them from GitHub,
-populate them with fake download stats, and assign a user named `dev`
-(with password: `dev`) as their maintainer.
+This will create some packages, update them from GitHub, populate them
+with fake download stats, and assign a user named `dev` (with password: `dev`)
+as their maintainer.
+
+There is also a user `user` (with password: `user`) that has no access if you
+need to check readonly views.
+
+Finally there is a user `admin` (with password: `admin`) that has super admin
+permissions.
 
 ### Search
 

src/Command/MigrateDownloadCountsCommand.php

@@ -61,7 +61,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
  try {
  // might be a large-ish dataset coming through here
- ini_set('memory_limit', '1G');
+ ini_set('memory_limit', -1);
 
  $now = new \DateTimeImmutable();
  $todaySuffix = ':'.$now->format('Ymd');

src/Command/MigratePhpStatsCommand.php

@@ -64,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
  try {
  // might be a large-ish dataset coming through here
- ini_set('memory_limit', '2G');
+ ini_set('memory_limit', -1);
 
  $now = new \DateTimeImmutable();
  $yesterday = new \DateTimeImmutable('yesterday');

src/DataFixtures/DownloadFixtures.php

@@ -4,12 +4,15 @@
 
 namespace App\DataFixtures;
 
+use App\Command\CompileStatsCommand;
 use App\Command\MigrateDownloadCountsCommand;
+use App\Command\MigratePhpStatsCommand;
 use App\Entity\Package;
 use App\Entity\Version;
 use DateInterval;
 use DateTimeImmutable;
 use Doctrine\Bundle\FixturesBundle\Fixture;
+use Doctrine\Bundle\FixturesBundle\FixtureGroupInterface;
 use Doctrine\Common\DataFixtures\DependentFixtureInterface;
 use Doctrine\ORM\EntityManagerInterface;
 use Doctrine\Persistence\ObjectManager;
@@ -21,19 +24,19 @@
 /**
  * Creates fake download statistics for each package.
  */
-class DownloadFixtures extends Fixture implements DependentFixtureInterface
+class DownloadFixtures extends Fixture implements FixtureGroupInterface
 {
  public function __construct(
  private Client $redis,
- private MigrateDownloadCountsCommand $migrateDownloadCountsCommand
+ private MigrateDownloadCountsCommand $migrateDownloadCountsCommand,
+ private readonly MigratePhpStatsCommand $migratePhpStatsCommand,
+ private readonly CompileStatsCommand $compileStatsCommand,
  ) {
  }
 
- public function getDependencies(): array
+ public static function getGroups(): array
  {
- return [
- PackageFixtures::class,
- ];
+ return ['downloads'];
  }
 
  public function load(ObjectManager $manager): void
@@ -43,36 +46,55 @@ public function load(ObjectManager $manager): void
 
  $output->writeln('Generating downloads...');
 
- /** @var Package[] $packages */
- $packages = $manager->getRepository(Package::class)->findAll();
+ $pkgNames = ['composer/pcre', 'monolog/monolog', 'twig/twig'];
+ $packages = $manager->getRepository(Package::class)->findBy(['name' => $pkgNames]);
+
+ $versions = [];
+ foreach ($packages as $index => $package) {
+ if ($package->getName() === 'composer/pcre') {
+ $versions[$index] = $package->getVersions();
+ } else {
+ assert($manager instanceof EntityManagerInterface);
+ $latestVersion = $this->getLatestPackageVersion($manager, $package);
+ $versions[$index][] = $latestVersion;
+ }
+ }
+
+ if ($versions === []) {
+ echo 'No packages found, make sure to run "bin/console doctrine:fixtures:load --group base" before the download fixtures' . PHP_EOL;
+ return;
+ }
 
- $progressBar = new ProgressBar($output, count($packages));
+ echo 'Creating download fixtures for packages: '.implode(', ', $pkgNames).PHP_EOL;
+
+ $progressBar = new ProgressBar($output, array_sum(array_map('count', $versions)));
  $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% (%remaining% left) %message%');
 
  $progressBar->setMessage('');
  $progressBar->start();
 
  // Set the Redis keys that would normally be set by the DownloadManager, for the whole period.
-
- foreach ($packages as $package) {
+ foreach ($packages as $index => $package) {
  $progressBar->setMessage($package->getName());
  $progressBar->display();
 
- /** @var EntityManagerInterface $manager */
- $latestVersion = $this->getLatestPackageVersion($manager, $package);
-
- $this->populateDownloads($package, $latestVersion);
+ foreach ($versions[$index] as $version) {
+ $this->populateDownloads($package, $version);
+ }
 
  $progressBar->advance();
  }
 
  $progressBar->finish();
  $output->writeln('');
 
- // Then migrate the Redis keys to the db
+ $manager->clear();
 
+ // Then migrate the Redis keys to the db
  $output->writeln('Migrating downloads to db... (this may take some time)');
  $this->migrateDownloadCountsCommand->run($input, $output);
+ $this->migratePhpStatsCommand->run($input, $output);
+ $this->compileStatsCommand->run($input, $output);
  }
 
  /**
@@ -115,19 +137,22 @@ private function populateDownloads(Package $package, Version $version): void
  $day = $date->format('Ymd');
  $month = $date->format('Ym');
 
+ $phpMinorPlatform = random_int(7, 8).'.'.random_int(0, 4);
+
  $keys = [
- 'downloads',
- 'downloads:' . $day,
- 'downloads:' . $month,
+ 'downloads' => $downloads,
+ 'downloads:' . $day => $downloads,
+ 'downloads:' . $month => $downloads,
+
+ 'dl:' . $package->getId() => $downloads,
+ 'dl:' . $package->getId() . ':' . $day => $downloads,
+ 'dl:' . $package->getId() . '-' . $version->getId() . ':' . $day => $downloads,
 
- 'dl:' . $package->getId(),
- 'dl:' . $package->getId() . ':' . $day,
- 'dl:' . $package->getId() . '-' . $version->getId() . ':' . $day
+ 'phpplatform:'.$phpMinorPlatform.':' => $downloads,
+ 'phpplatform:'.$package->getId() . '-' . $version->getId().':'.$phpMinorPlatform.':'.$day => $downloads,
  ];
 
- foreach ($keys as $key) {
- $this->redis->incrby($key, $downloads);
- }
+ $this->redis->mset($keys);
 
  $date = $date->add(new DateInterval('P1D'));