[Staging] Increase proxy-read-timeout to avoid early external request disconnects

deployment/helm-templates/templates/frontend-ingress.yaml

@@ -3,7 +3,6 @@ kind: Ingress
 metadata:
   annotations:
     cert-manager.io/cluster-issuer: cert-manager
-    kubernetes.io/ingress.class: nginx
     meta.helm.sh/release-name: dx-helm-wri-staging-release
     meta.helm.sh/release-namespace: wri-odp-staging
     nginx.ingress.kubernetes.io/configuration-snippet: |
@@ -38,13 +37,38 @@ metadata:
       location ~ ^/dataset/school_bus_fleets {
          rewrite ^/dataset/school_bus_fleets$ /datasets/usa-school-bus-fleets permanent;
         }
+      if ($http_user_agent ~* "(GPTBot|360Spider|80legs\.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis\.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS\.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(g00g1e|Genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go!Zilla|Go-Ahead-Got-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|heritrix|HMView|HomePageBot|HTTP\:\:Lite|HTTrack|HubSpot|ia_archiver|icarus6|IDBot|id-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|Java|JetCar|JobdiggerSpider)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp-trivial|Mail\.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon\$|Mediatoolkitbot|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla\.\*Indy|Mozilla\.\*NEWT|MSFrontPage|msnbot|Navroad)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN\.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Rippers\ 0|rogerbot|RSSingBot|rv:1\.9\.1|RyzeCrawler|SafeSearch|SBIder|Scrapy|Screaming|SeaMonkey\$|search\.goo\.ne\.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer\.info|SiteSnagger|skygrid|Slackbot|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|Twitterbot|URI\:\:Fetch|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(woobot|woopingbot|worldwebheritage\.org|Wotbox|WPScan|WWWOFFLE|WWW-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg)") {
+        return 444;
+      }
     nginx.ingress.kubernetes.io/use-regex: "true"
   generation: 1
   labels:
     app.kubernetes.io/managed-by: Helm
   name: wri-staging-ingress-fe-internal
   namespace: wri-odp-staging
 spec:
+  ingressClassName: nginx
   rules:
   - host: wri.staging.frontend.datopian.com
     http:

deployment/helm-templates/templates/wri-staging-ingress-fe-internal-api.yaml

@@ -3,7 +3,6 @@ kind: Ingress
 metadata:
   annotations:
     cert-manager.io/cluster-issuer: cert-manager
-    kubernetes.io/ingress.class: nginx
     meta.helm.sh/release-name: dx-helm-wri-staging-release
     meta.helm.sh/release-namespace: wri-odp-staging
     nginx.ingress.kubernetes.io/configuration-snippet: |
@@ -17,14 +16,15 @@ metadata:
     nginx.ingress.kubernetes.io/limit-rps: "50"
     nginx.ingress.kubernetes.io/proxy-body-size: 1000M
     nginx.ingress.kubernetes.io/proxy-connect-timeout: "60"
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "60"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "1800"
     nginx.ingress.kubernetes.io/rewrite-target: /api/action/$1
     nginx.ingress.kubernetes.io/use-regex: "true"
   labels:
     app.kubernetes.io/managed-by: Helm
   name: wri-staging-ingress-fe-internal-api
   namespace: wri-odp-staging
 spec:
+  ingressClassName: nginx
   rules:
   - host: wri.staging.frontend.datopian.com
     http: