From 563fafa7ee04980998a119c13b8a8d833762308b Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Sat, 20 Jan 2024 20:43:24 +0100 Subject: [PATCH] chore(nginx): update robots.txt --- deploy/base/ingress.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/base/ingress.yaml b/deploy/base/ingress.yaml index ab0abdc..1aedfa0 100644 --- a/deploy/base/ingress.yaml +++ b/deploy/base/ingress.yaml @@ -7,7 +7,8 @@ metadata: nginx.ingress.kubernetes.io/server-snippet: | location = /robots.txt { add_header Content-Type text/plain; - return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: MJ12bot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n"; + return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: MJ12bot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n"; + } spec: ingressClassName: nginx