README.md

@@ -85,12 +85,6 @@ To connect locally to the Nginx ingress controller:
 sudo -E kubectl port-forward dev-ingress-nginx-controller-abcdef1234-ab123 80:80 443:443 22:22
 ```
 
-Or
-
-```
-sudo -E kubectl port-forward $(kubectl get pods | grep "dev-ingress-nginx-controller.*Running" | awk '{print $1}') 80:80 443:443 22:22
-```
-
 ### Additional dependencies
 
 - envfilesubst [link](https://git.netflux.io/rob/envfilesubst)

deploy/base/ingress.yaml

@@ -5,9 +5,8 @@ metadata:
   annotations:
     nginx.ingress.kubernetes.io/proxy-body-size: 10m
     nginx.ingress.kubernetes.io/server-snippet: |
-      location = /robots.txt {
-        add_header Content-Type text/plain;
-        return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n";
+      if ($http_user_agent ~* "BLEXBot|SemrushBot|AhrefsBot" ) {
+        return 403;
       }
 spec:
   ingressClassName: nginx