From 3a3bbe2ab7758098f62c54400e1a95546f0c880d Mon Sep 17 00:00:00 2001
From: Rob Watson <rob@netflux.io>
Date: Sat, 13 Jan 2024 14:17:34 +0100
Subject: [PATCH] feat(nginx): add robots.txt

---
 deploy/base/ingress.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/deploy/base/ingress.yaml b/deploy/base/ingress.yaml
index 5d497dd..6f6ce56 100644
--- a/deploy/base/ingress.yaml
+++ b/deploy/base/ingress.yaml
@@ -5,8 +5,9 @@ metadata:
   annotations:
     nginx.ingress.kubernetes.io/proxy-body-size: 10m
     nginx.ingress.kubernetes.io/server-snippet: |
-      if ($http_user_agent ~* "BLEXBot|SemrushBot|AhrefsBot" ) {
-        return 403;
+      location = /robots.txt {
+        add_header Content-Type text/plain;
+        return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n";
       }
 spec:
   ingressClassName: nginx