rob
/
netflux-kubernetes
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

chore(nginx): update robots.txt

This commit is contained in:
Rob Watson 2024-01-20 20:43:24 +01:00
parent 386a02c4f5
commit 563fafa7ee
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
deploy/base/ingress.yaml
View File

@ -7,7 +7,8 @@ metadata:
nginx.ingress.kubernetes.io/server-snippet: |
location = /robots.txt {
add_header Content-Type text/plain;
return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: MJ12bot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n";
return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: MJ12bot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n";
}
spec:
ingressClassName: nginx