diff --git a/handler/handler.go b/handler/handler.go
index e682f12..938327d 100644
--- a/handler/handler.go
+++ b/handler/handler.go
@@ -5,8 +5,8 @@ import (
"net/http"
)
-//go:embed static/index.html
-var indexPage string
+//go:embed static/robots.txt
+var robotsTxt []byte
type Params struct {
MatrixHostname string
@@ -27,7 +27,7 @@ func New(params Params) http.Handler {
h.Handle("GET /.well-known/matrix/server", http.HandlerFunc(h.getMatrixServer))
h.Handle("GET /.well-known/matrix/client", http.HandlerFunc(h.getMatrixClient))
- h.Handle("GET /{$}", http.HandlerFunc(h.getHomepage))
+ h.Handle("GET /robots.txt", http.HandlerFunc(h.getRobotsTxt))
h.Handle("GET /", http.FileServer(http.Dir(params.RootPath)))
return h
@@ -45,10 +45,8 @@ func (h *handler) getMatrixClient(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"m.homeserver": {"base_url": "` + h.params.MatrixBaseURL + `"}}`))
}
-func (h *handler) getHomepage(w http.ResponseWriter, r *http.Request) {
- w.Header().Add("content-type", "text/html")
+func (h *handler) getRobotsTxt(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("content-type", "text/plain")
w.WriteHeader(http.StatusOK)
- if r.Method == http.MethodGet {
- w.Write([]byte(indexPage))
- }
+ w.Write(robotsTxt)
}
diff --git a/handler/handler_test.go b/handler/handler_test.go
index 28aa533..797832c 100644
--- a/handler/handler_test.go
+++ b/handler/handler_test.go
@@ -41,6 +41,14 @@ func TestHandler(t *testing.T) {
wantStatusCode: http.StatusOK,
wantBody: `{"m.homeserver": {"base_url": "https://foo.example.com"}}`,
},
+ {
+ name: "GET /robots.txt",
+ method: http.MethodGet,
+ path: "/robots.txt",
+ wantContentType: "text/plain",
+ wantStatusCode: http.StatusOK,
+ wantBody: "Allow: /",
+ },
{
name: "GET /test.html",
method: http.MethodGet,
@@ -53,15 +61,15 @@ func TestHandler(t *testing.T) {
name: "GET /",
method: http.MethodGet,
path: "/",
- wantContentType: "text/html",
+ wantContentType: "text/html; charset=utf-8",
wantStatusCode: http.StatusOK,
- wantBody: "Welcome to netflux.io",
+ wantBody: "\n \n Homepage\n \n\n",
},
{
name: "HEAD /",
method: http.MethodHead,
path: "/",
- wantContentType: "text/html",
+ wantContentType: "text/html; charset=utf-8",
wantStatusCode: http.StatusOK,
wantBody: "",
},
@@ -105,6 +113,7 @@ func TestHandler(t *testing.T) {
respBody, err := io.ReadAll(resp.Body)
require.NoError(t, err)
+
assert.Contains(t, string(respBody), tc.wantBody)
})
}
diff --git a/handler/static/robots.txt b/handler/static/robots.txt
new file mode 100644
index 0000000..e6729da
--- /dev/null
+++ b/handler/static/robots.txt
@@ -0,0 +1,44 @@
+User-agent: AI2Bot
+User-agent: Ai2Bot-Dolma
+User-agent: Amazonbot
+User-agent: anthropic-ai
+User-agent: Applebot
+User-agent: Applebot-Extended
+User-agent: Bytespider
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: Claude-Web
+User-agent: ClaudeBot
+User-agent: cohere-ai
+User-agent: Diffbot
+User-agent: DuckAssistBot
+User-agent: FacebookBot
+User-agent: FriendlyCrawler
+User-agent: Google-Extended
+User-agent: GoogleOther
+User-agent: GoogleOther-Image
+User-agent: GoogleOther-Video
+User-agent: GPTBot
+User-agent: iaskspider/2.0
+User-agent: ICC-Crawler
+User-agent: ImagesiftBot
+User-agent: img2dataset
+User-agent: ISSCyberRiskCrawler
+User-agent: Kangaroo Bot
+User-agent: Meta-ExternalAgent
+User-agent: Meta-ExternalFetcher
+User-agent: OAI-SearchBot
+User-agent: omgili
+User-agent: omgilibot
+User-agent: PerplexityBot
+User-agent: PetalBot
+User-agent: Scrapy
+User-agent: Sidetrade indexer bot
+User-agent: Timpibot
+User-agent: VelenPublicWebCrawler
+User-agent: Webzio-Extended
+User-agent: YouBot
+Disallow: /
+
+User-agent: *
+Allow: /
diff --git a/handler/testdata/static/index.html b/handler/testdata/static/index.html
new file mode 100644
index 0000000..3479682
--- /dev/null
+++ b/handler/testdata/static/index.html
@@ -0,0 +1,5 @@
+
+
+ Homepage
+
+