✨ Response size and time limit - fixes #30

Author: makew0rld

NOTES.md

@@ -3,7 +3,6 @@
 ## Issues
 - URL for each tab should not be stored as a string - in the current code there's lots of reparsing the URL
 - Can't go back or do other things while page is loading - need a way to stop `handleURL`
-- dlChoiceModal doesn't go away when portal is selected, and freezes on Cancel
 
 ## Upstream Bugs
 - Wrapping messes up on brackets

config/config.go

@@ -149,6 +149,8 @@ func Init() error {
 	viper.SetDefault("a-general.left_margin", 0.15)
 	viper.SetDefault("a-general.max_width", 100)
 	viper.SetDefault("a-general.downloads", "")
+	viper.SetDefault("a-general.page_max_size", 2097152)
+	viper.SetDefault("a-general.page_max_time", 10)
 	viper.SetDefault("cache.max_size", 0)
 	viper.SetDefault("cache.max_pages", 20)
 

config/default.go

@@ -1,6 +1,5 @@
 package config
 
-//go:generate ./default.sh
 var defaultConf = []byte(`# This is the default config file.
 # It also shows all the default values, if you don't create the file.
 
@@ -29,6 +28,11 @@ max_width = 100  # The max number of columns to wrap a page's text to. Preformat
 # An empty value means the code will find the default downloads folder for your system.
 # If the path does not exist it will be created.
 downloads = "" 
+# Max size for displayable content in bytes - after that size a download window pops up
+page_max_size = 2097152  # 2 MiB
+# Max time it takes to load a page in seconds - after that a download window pops up
+page_max_time = 10
+
 # Options for page cache - which is only for text/gemini pages
 # Increase the cache size to speed up browsing at the expense of memory
 [cache]

config/default.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-head -n 3 default.go | tee default.go > /dev/null
+head -n 1 default.go | tee default.go > /dev/null
 echo -n 'var defaultConf = []byte(`' >> default.go
 cat ../default-config.toml >> default.go
 echo '`)' >> default.go

default-config.toml

@@ -26,6 +26,11 @@ max_width = 100  # The max number of columns to wrap a page's text to. Preformat
 # An empty value means the code will find the default downloads folder for your system.
 # If the path does not exist it will be created.
 downloads = "" 
+# Max size for displayable content in bytes - after that size a download window pops up
+page_max_size = 2097152  # 2 MiB
+# Max time it takes to load a page in seconds - after that a download window pops up
+page_max_time = 10
+
 # Options for page cache - which is only for text/gemini pages
 # Increase the cache size to speed up browsing at the expense of memory
 [cache]

display/download.go

@@ -24,7 +24,6 @@ import (
 // For choosing between download and the portal - copy of YesNo basically
 var dlChoiceModal = cview.NewModal().
 	SetTextColor(tcell.ColorWhite).
-	SetText("That file could not be displayed. What would you like to do?").
 	AddButtons([]string{"Download", "Open in portal", "Cancel"})
 
 // Channel to indicate what choice they made using the button text
@@ -72,7 +71,7 @@ func dlInit() {
 
 // dlChoice displays the download choice modal and acts on the user's choice.
 // It should run in a goroutine.
-func dlChoice(u string, resp *gemini.Response) {
+func dlChoice(text, u string, resp *gemini.Response) {
 	defer resp.Body.Close()
 
 	parsed, err := url.Parse(u)
@@ -81,6 +80,7 @@ func dlChoice(u string, resp *gemini.Response) {
 		return
 	}
 
+	dlChoiceModal.SetText(text)
 	tabPages.ShowPage("dlChoice")
 	tabPages.SendToFront("dlChoice")
 	App.SetFocus(dlChoiceModal)

display/private.go

@@ -309,11 +309,27 @@ func handleURL(t *tab, u string) (string, bool) {
 			return ret("", false)
 		}
 
-		page.Width = termW
+		// Make new request for downloading purposes
+		res, clientErr := client.Fetch(u)
+		if clientErr != nil && clientErr != client.ErrTofu {
+			Error("URL Fetch Error", err.Error())
+			return ret("", false)
+		}
+
+		if err == renderer.ErrTooLarge {
+			go dlChoice("That page is too large. What would you like to do?", u, res)
+			return ret("", false)
+		}
+		if err == renderer.ErrTimedOut {
+			go dlChoice("Loading that page timed out. What would you like to do?", u, res)
+			return ret("", false)
+		}
 		if err != nil {
 			Error("Page Error", "Issuing creating page: "+err.Error())
 			return ret("", false)
 		}
+
+		page.Width = termW
 		go cache.Add(page)
 		setPage(t, page)
 		return ret(u, true)
@@ -359,7 +375,7 @@ func handleURL(t *tab, u string) (string, bool) {
 		return ret("", false)
 	}
 	// Status code 20, but not a document that can be displayed
-	go dlChoice(u, res)
+	go dlChoice("That file could not be displayed. What would you like to do?", u, res)
 	return ret("", false)
 }
 

renderer/page.go

@@ -1,16 +1,22 @@
 package renderer
 
 import (
+	"bytes"
 	"errors"
-	"io/ioutil"
+	"io"
 	"mime"
 	"strings"
+	"time"
 
 	"github.com/makeworld-the-better-one/amfora/structs"
 	"github.com/makeworld-the-better-one/go-gemini"
+	"github.com/spf13/viper"
 	"golang.org/x/text/encoding/ianaindex"
 )
 
+var ErrTooLarge = errors.New("page content would be too large")
+var ErrTimedOut = errors.New("page download timed out")
+
 // isUTF8 returns true for charsets that are compatible with UTF-8 and don't need to be decoded.
 func isUTF8(charset string) bool {
 	utfCharsets := []string{"", "utf-8", "us-ascii"}
@@ -53,11 +59,27 @@ func MakePage(url string, res *gemini.Response, width, leftMargin int) (*structs
 		return nil, errors.New("not valid content for a Page")
 	}
 
-	rawText, err := ioutil.ReadAll(res.Body) // TODO: Don't use all memory on large pages
-	if err != nil {
+	buf := new(bytes.Buffer)
+	go func() {
+		time.Sleep(time.Duration(viper.GetInt("a-general.page_max_time")) * time.Second)
+		res.Body.Close()
+	}()
+
+	_, err := io.CopyN(buf, res.Body, viper.GetInt64("a-general.page_max_size")) // 2 MiB max
+	res.Body.Close()
+	rawText := buf.Bytes()
+	if err == nil {
+		// Content was larger than 2 MiB
+		return nil, ErrTooLarge
+	} else if err != io.EOF {
+		if strings.HasSuffix(err.Error(), "use of closed network connection") {
+			// Timed out
+			return nil, ErrTimedOut
+		}
+		// Some other error
 		return nil, err
 	}
-	res.Body.Close()
+	// Otherwise, the error is EOF, which is what we want.
 
 	mediatype, params, _ := mime.ParseMediaType(res.Meta)