rob
/
netflux-kubernetes
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

feat(nginx): add robots.txt

This commit is contained in:
Rob Watson 2024-01-13 14:17:34 +01:00
parent b77383586d
commit 3a3bbe2ab7
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
deploy/base/ingress.yaml
View File

@ -5,8 +5,9 @@ metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: 10m
nginx.ingress.kubernetes.io/server-snippet: |
if ($http_user_agent ~* "BLEXBot|SemrushBot|AhrefsBot" ) {
return 403;
location = /robots.txt {
add_header Content-Type text/plain;
return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n";
}
spec:
ingressClassName: nginx