static build using crawler

composer.json

@@ -18,6 +18,7 @@
  "require": {
  "php": "^8.1",
  "illuminate/contracts": "^9.0",
+ "spatie/crawler": "^7.1",
  "spatie/laravel-package-tools": "^1.13.0",
  "voku/html-min": "^4.5"
  },

config/static.php

@@ -11,17 +11,26 @@
  */
  'fallback_cache' => 'memcached',
 
- /**
- * Clear static files before warming up static cache.
- * When disabled, the cache is warmed up rather by updating and overwriting files instead of starting without an existing cache.
- */
- 'clear_before_warm_up' => false,
+ 'build' => [
+ /**
+ * Clear static files before building static cache.
+ * When disabled, the cache is warmed up rather by updating and overwriting files instead of starting without an existing cache.
+ */
+ 'clear_before_start' => false,
 
- /**
- * HTTP Header that is being sent to web server by warm up command, to recognize and pass through static cache and
- * hit the Laravel application on the server.
- */
- 'warm_up_http_header' => 'X-Laravel-Static',
+ /**
+ * Concurrency for crawling to warm up static cache.
+ */
+ 'concurrency' => 5,
+
+ /**
+ * HTTP header that can be used to bypass the cache. Useful for updating the cache without needing to clear it first,
+ * or to monitor the performance of your application.
+ */
+ 'bypass_header' => [
+ 'X-Laravel-Static' => 'off',
+ ],
+ ],
 
  /**
  * Different caches per domain.
@@ -37,7 +46,7 @@
  /**
  * Define if you want to save the static cache after response has been sent to browser.
  */
- 'on_termination' => false,
+ 'on_termination' => true,
 
  /**
  * This setting prevents saving the same static cache file twice (with and without trailing slash) using a 302 redirect.

src/Commands/StaticBuildCommand.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace Vormkracht10\LaravelStatic\Commands;
+
+use GuzzleHttp\RequestOptions;
+use Illuminate\Config\Repository;
+use Illuminate\Console\Command;
+use Spatie\Crawler\Crawler;
+use Spatie\Crawler\CrawlProfiles\CrawlInternalUrls;
+use Vormkracht10\LaravelStatic\Crawler\StaticCrawlObserver;
+use Vormkracht10\LaravelStatic\LaravelStatic;
+
+class StaticBuildCommand extends Command
+{
+ public $signature = 'static:build';
+
+ public $description = 'Build Static version';
+
+ protected Repository $config;
+
+ protected LaravelStatic $static;
+
+ public function __construct(Repository $config, LaravelStatic $static)
+ {
+ parent::__construct();
+
+ $this->config = $config;
+ $this->static = $static;
+ }
+
+ public function handle(): void
+ {
+ if ($this->config->get('static.build.clear_before_start')) {
+ $this->call(StaticClearCommand::class);
+ }
+
+ $bypassHeader = $this->config->get('static.build.bypass_header');
+
+ Crawler::create([
+ RequestOptions::VERIFY => ! app()->environment('local'),
+ RequestOptions::ALLOW_REDIRECTS => true,
+ RequestOptions::HEADERS => [
+ array_key_first($bypassHeader) => array_shift($bypassHeader),
+ 'User-Agent' => 'LaravelStatic/1.0',
+ ],
+ ])
+ ->setCrawlObserver(new StaticCrawlObserver)
+ ->setCrawlProfile(new CrawlInternalUrls($this->config->get('app.url')))
+ ->acceptNofollowLinks()
+ ->setConcurrency($this->config->get('static.build.concurrency'))
+ ->setDefaultScheme('https')
+ // ->setParseableMimeTypes(['text/html', 'text/plain'])
+ ->startCrawling($this->config->get('app.url'));
+ }
+}

src/Commands/StaticClearCommand.php

@@ -2,6 +2,7 @@
 
 namespace Vormkracht10\LaravelStatic\Commands;
 
+use Illuminate\Config\Repository;
 use Illuminate\Console\Command;
 use Vormkracht10\LaravelStatic\LaravelStatic;
 
@@ -11,8 +12,22 @@ class StaticClearCommand extends Command
 
  public $description = 'Clear static cached files';
 
- public function handle(LaravelStatic $static): void
+ protected Repository $config;
+
+ protected LaravelStatic $static;
+
+ public function __construct(Repository $config, LaravelStatic $static)
  {
- $static->clear();
+ parent::__construct();
+
+ $this->config = $config;
+ $this->static = $static;
+ }
+
+ public function handle(): void
+ {
+ $this->static->clear();
+
+ $this->info('✔ Static cache cleared!');
  }
 }

src/Console/Terminal.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace Vormkracht10\LaravelStatic\Console;
+
+use Illuminate\Console\Command;
+
+class Terminal extends Command
+{
+ protected $name = 'NONEXISTENT';
+
+ protected $hidden = true;
+
+ public $outputSymfony;
+
+ public $outputStyle;
+
+ public function __construct($argInput = '')
+ {
+ parent::__construct();
+
+ $this->input = new \Symfony\Component\Console\Input\StringInput($argInput);
+
+ $this->outputSymfony = new \Symfony\Component\Console\Output\ConsoleOutput();
+ $this->outputStyle = new \Illuminate\Console\OutputStyle($this->input, $this->outputSymfony);
+
+ $this->output = $this->outputStyle;
+ }
+}

src/Crawler/StaticCrawlObserver.php

@@ -0,0 +1,59 @@
+<?php
+
+namespace Vormkracht10\LaravelStatic\Crawler;
+
+use GuzzleHttp\Exception\RequestException;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\UriInterface;
+use Spatie\Crawler\CrawlObservers\CrawlObserver;
+
+class StaticCrawlObserver extends CrawlObserver
+{
+ /**
+ * Called when the crawler will crawl the url.
+ *
+ * @param \Psr\Http\Message\UriInterface $url
+ */
+ public function willCrawl(UriInterface $url): void
+ {
+ // ...
+ }
+
+ /**
+ * Called when the crawler has crawled the given url successfully.
+ *
+ * @param \Psr\Http\Message\UriInterface $url
+ * @param \Psr\Http\Message\ResponseInterface $response
+ * @param \Psr\Http\Message\UriInterface|null $foundOnUrl
+ */
+ public function crawled(
+ UriInterface $url,
+ ResponseInterface $response,
+ ?UriInterface $foundOnUrl = null
+ ): void {
+ console()->info('✔ '.$url);
+ }
+
+ /**
+ * Called when the crawler had a problem crawling the given url.
+ *
+ * @param \Psr\Http\Message\UriInterface $url
+ * @param \GuzzleHttp\Exception\RequestException $requestException
+ * @param \Psr\Http\Message\UriInterface|null $foundOnUrl
+ */
+ public function crawlFailed(
+ UriInterface $url,
+ RequestException $requestException,
+ ?UriInterface $foundOnUrl = null
+ ): void {
+ console()->error('✘ '.$url);
+ }
+
+ /**
+ * Called when the crawl has ended.
+ */
+ public function finishedCrawling(): void
+ {
+ console()->info('✔ Static build completed');
+ }
+}

src/LaravelStaticServiceProvider.php

@@ -4,6 +4,7 @@
 
 use Spatie\LaravelPackageTools\Package;
 use Spatie\LaravelPackageTools\PackageServiceProvider;
+use Vormkracht10\LaravelStatic\Commands\StaticBuildCommand;
 use Vormkracht10\LaravelStatic\Commands\StaticClearCommand;
 
 class LaravelStaticServiceProvider extends PackageServiceProvider
@@ -13,6 +14,9 @@ public function configurePackage(Package $package): void
  $package
  ->name('laravel-static')
  ->hasConfigFile()
- ->hasCommand(StaticClearCommand::class);
+ ->hasCommands([
+ StaticClearCommand::class,
+ StaticBuildCommand::class,
+ ]);
  }
 }

src/Middleware/StaticResponse.php

@@ -16,10 +16,13 @@ class StaticResponse
 
  protected Filesystem $files;
 
+ protected array $bypassHeader;
+
  public function __construct(Repository $config, Filesystem $files)
  {
  $this->config = $config;
  $this->files = $files;
+ $this->bypassHeader = $this->config->get('static.build.bypass_header');
  }
 
  /**
@@ -93,7 +96,8 @@ protected function shouldBeStatic(Request $request, Response $response): bool
  {
  return
  $request->isMethod('GET') &&
- $response->getStatusCode() == 200;
+ $response->getStatusCode() == 200 &&
+ $request->header(array_key_first($this->bypassHeader)) != array_shift($this->bypassHeader);
  }
 
  /**