rob
/
netflux-homepage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: add robots.txt
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Rob Watson 2024-11-20 20:45:19 +01:00
parent 881f37f89c
commit 7ee67d3f2d
4 changed files with 67 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
handler/handler.go
View File

@ -5,8 +5,8 @@ import (
"net/http"
)
//go:embed static/index.html
var indexPage string
//go:embed static/robots.txt
var robotsTxt []byte
type Params struct {
MatrixHostname string
@ -27,7 +27,7 @@ func New(params Params) http.Handler {
h.Handle("GET /.well-known/matrix/server", http.HandlerFunc(h.getMatrixServer))
h.Handle("GET /.well-known/matrix/client", http.HandlerFunc(h.getMatrixClient))
h.Handle("GET /{$}", http.HandlerFunc(h.getHomepage))
h.Handle("GET /robots.txt", http.HandlerFunc(h.getRobotsTxt))
h.Handle("GET /", http.FileServer(http.Dir(params.RootPath)))
return h
@ -45,10 +45,8 @@ func (h *handler) getMatrixClient(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"m.homeserver": {"base_url": "` + h.params.MatrixBaseURL + `"}}`))
}
func (h *handler) getHomepage(w http.ResponseWriter, r *http.Request) {
w.Header().Add("content-type", "text/html")
func (h *handler) getRobotsTxt(w http.ResponseWriter, r *http.Request) {
w.Header().Add("content-type", "text/plain")
w.WriteHeader(http.StatusOK)
if r.Method == http.MethodGet {
w.Write([]byte(indexPage))
}
w.Write(robotsTxt)
}

15
handler/handler_test.go
View File

@ -41,6 +41,14 @@ func TestHandler(t *testing.T) {
wantStatusCode: http.StatusOK,
wantBody: `{"m.homeserver": {"base_url": "https://foo.example.com"}}`,
},
{
name: "GET /robots.txt",
method: http.MethodGet,
path: "/robots.txt",
wantContentType: "text/plain",
wantStatusCode: http.StatusOK,
wantBody: "Allow: /",
},
{
name: "GET /test.html",
method: http.MethodGet,
@ -53,15 +61,15 @@ func TestHandler(t *testing.T) {
name: "GET /",
method: http.MethodGet,
path: "/",
wantContentType: "text/html",
wantContentType: "text/html; charset=utf-8",
wantStatusCode: http.StatusOK,
wantBody: "Welcome to netflux.io",
wantBody: "<html>\n <head>\n <title>Homepage</title>\n </head>\n</html>\n",
},
{
name: "HEAD /",
method: http.MethodHead,
path: "/",
wantContentType: "text/html",
wantContentType: "text/html; charset=utf-8",
wantStatusCode: http.StatusOK,
wantBody: "",
},
@ -105,6 +113,7 @@ func TestHandler(t *testing.T) {
respBody, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Contains(t, string(respBody), tc.wantBody)
})
}

44
handler/static/robots.txt Normal file
View File

@ -0,0 +1,44 @@
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /
User-agent: *
Allow: /

5
handler/testdata/static/index.html vendored Normal file
View File

@ -0,0 +1,5 @@
<html>
<head>
<title>Homepage</title>
</head>
</html>