📝 Added new scripts, aliases, etc.

2023-06-20 01:11:01 -07:00 · 2023-06-20 01:11:01 -07:00 · 99dac59d82
commit 99dac59d82
parent f67d7b627e
5 changed files with 76 additions and 0 deletions
--- a/robots_examples.txt
+++ b/robots_examples.txt
@ -0,0 +1,20 @@
+# Allows all:
+User-agent: *
+Disallow:
+
+# Disallow all (you don't want ANY site traffic here...):
+User-agent: *
+Disallow: /
+
+# Disallow specific files and folders
+User-agent: *
+Disallow: /topsy/
+Disallow: /crets/
+Disallow: /hidden/file.html
+
+# Disallow Specific Bots
+User-agent: Bingbot
+Disallow: /
+
+User-agent: *
+Disallow: