rob
/
netflux-kubernetes
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

feat(nginx): add robots.txt

This commit is contained in:
Rob Watson 2024-01-13 14:17:34 +01:00
parent b77383586d
commit 3a3bbe2ab7
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
deploy/base/ingress.yaml
View File

@ -5,8 +5,9 @@ metadata:
annotations: annotations:
nginx.ingress.kubernetes.io/proxy-body-size: 10m nginx.ingress.kubernetes.io/proxy-body-size: 10m
nginx.ingress.kubernetes.io/server-snippet: | nginx.ingress.kubernetes.io/server-snippet: |
if ($http_user_agent ~* "BLEXBot|SemrushBot|AhrefsBot" ) { location = /robots.txt {
return 403; add_header Content-Type text/plain;
return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n";
} }
spec: spec:
ingressClassName: nginx ingressClassName: nginx