diff --git a/robots-txt-tests/index.html b/robots-txt-tests/index.html new file mode 100644 index 0000000000000000000000000000000000000000..d95f3ad14dee633a758d2e331151e950dd13e4ed --- /dev/null +++ b/robots-txt-tests/index.html @@ -0,0 +1 @@ +content diff --git a/robots-txt-tests/robots.txt b/robots-txt-tests/robots.txt new file mode 100644 index 0000000000000000000000000000000000000000..ee05f3f1f19c12259e2ff7892e1e6f0f5766affe --- /dev/null +++ b/robots-txt-tests/robots.txt @@ -0,0 +1,2 @@ +User-agent: Custom Agent Just for Tests +Disallow: /