Append removed URLs to robots.txt

.gitignore

@@ -5,6 +5,7 @@
 !/**/.gitkeep
 !/.github/**
 !/.idea/
+!/assets/**
 !/bin/
 !/config/**
 !/Makefile

assets/robots-additions.txt

@@ -0,0 +1,8 @@
+Disallow: /drupal-tailwindcss/
+Disallow: /drupalorg/
+Disallow: /github/
+Disallow: /linkedin/
+Disallow: /rebuilding-pantheon/
+Disallow: /twitter/
+Disallow: /webpack-encore-pcss-regex/
+Disallow: /workshop-drupal-testing/

composer.json

@@ -77,6 +77,9 @@
                 "[web-root]/sites/README.txt": false,
                 "[web-root]/modules/README.txt": false,
                 "[web-root]/profiles/README.txt": false,
+                "[web-root]/robots.txt": {
+                    "append": "assets/robots-additions.txt"
+                },
                 "[web-root]/themes/README.txt": false,
                 "[web-root]/web.config": false
             },