Fix bin/publish: copy docs.dist from project root

Fix bin/publish: use correct .env path for rspade_system
Fix bin/publish script: prevent grep exit code 1 from terminating script

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

vendor/spatie/crawler/src/CrawlerRobots.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace Spatie\Crawler;
+
+use Spatie\Robots\RobotsHeaders;
+use Spatie\Robots\RobotsMeta;
+
+class CrawlerRobots
+{
+    protected RobotsHeaders $robotsHeaders;
+
+    protected RobotsMeta $robotsMeta;
+
+    protected bool $mustRespectRobots;
+
+    public function __construct(array $headers, string $body, bool $mustRespectRobots)
+    {
+        $this->robotsHeaders = RobotsHeaders::create($headers);
+
+        $this->robotsMeta = RobotsMeta::create($body);
+
+        $this->mustRespectRobots = $mustRespectRobots;
+    }
+
+    public function mayIndex(): bool
+    {
+        if (! $this->mustRespectRobots) {
+            return true;
+        }
+
+        if (! $this->robotsHeaders->mayIndex()) {
+            return false;
+        }
+
+        if (! $this->robotsMeta->mayIndex()) {
+            return false;
+        }
+
+        return true;
+    }
+
+    public function mayFollow(): bool
+    {
+        if (! $this->mustRespectRobots) {
+            return true;
+        }
+
+        if (! $this->robotsHeaders->mayFollow()) {
+            return false;
+        }
+
+        if (! $this->robotsMeta->mayFollow()) {
+            return false;
+        }
+
+        return true;
+    }
+}