docs: update README.md

README.md

@@ -85,6 +85,12 @@ To connect locally to the Nginx ingress controller:
 sudo -E kubectl port-forward dev-ingress-nginx-controller-abcdef1234-ab123 80:80 443:443 22:22
 ```
 
+Or
+
+```
+sudo -E kubectl port-forward $(kubectl get pods | grep "dev-ingress-nginx-controller.*Running" | awk '{print $1}') 80:80 443:443 22:22
+```
+
 ### Additional dependencies
 
 - envfilesubst [link](https://git.netflux.io/rob/envfilesubst)

deploy/base/ingress.yaml

@@ -5,8 +5,9 @@ metadata:
   annotations:
     nginx.ingress.kubernetes.io/proxy-body-size: 10m
     nginx.ingress.kubernetes.io/server-snippet: |
-      if ($http_user_agent ~* "BLEXBot|SemrushBot|AhrefsBot" ) {
-        return 403;
+      location = /robots.txt {
+        add_header Content-Type text/plain;
+        return 200 "User-agent: Amazonbot\nDisallow: /\n\nUser-agent: BLEXBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: DotBot\nDisallow: /\n\nUser-agent: BingBot\nCrawl-delay: 30\n";
       }
 spec:
   ingressClassName: nginx