Add js.Batch

Fixes gohugoio#12626
Closes gohugoio#7499
Closes gohugoio#9978
Closes gohugoio#12879
Closes gohugoio#13113
Fixes gohugoio#13116

Author: bep

commands/hugobuilder.go

@@ -920,7 +920,11 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
 
 			changed := c.changeDetector.changed()
 			if c.changeDetector != nil {
-				lrl.Logf("build changed %d files", len(changed))
+				if len(changed) >= 10 {
+					lrl.Logf("build changed %d files", len(changed))
+				} else {
+					lrl.Logf("build changed %d files: %q", len(changed), changed)
+				}
 				if len(changed) == 0 {
 					// Nothing has changed.
 					return

commands/server.go

@@ -32,6 +32,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -210,16 +211,17 @@ func (f *fileChangeDetector) changed() []string {
 		}
 	}
 
-	return f.filterIrrelevant(c)
+	return f.filterIrrelevantAndSort(c)
 }
 
-func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
+func (f *fileChangeDetector) filterIrrelevantAndSort(in []string) []string {
 	var filtered []string
 	for _, v := range in {
 		if !f.irrelevantRe.MatchString(v) {
 			filtered = append(filtered, v)
 		}
 	}
+	sort.Strings(filtered)
 	return filtered
 }
 

common/herrors/errors.go

@@ -133,6 +133,21 @@ func IsNotExist(err error) bool {
 	return false
 }
 
+// IsExist returns true if the error is a file exists error.
+// Unlike os.IsExist, this also considers wrapped errors.
+func IsExist(err error) bool {
+	if os.IsExist(err) {
+		return true
+	}
+
+	// os.IsExist does not consider wrapped errors.
+	if os.IsExist(errors.Unwrap(err)) {
+		return true
+	}
+
+	return false
+}
+
 var nilPointerErrRe = regexp.MustCompile(`at <(.*)>: error calling (.*?): runtime error: invalid memory address or nil pointer dereference`)
 
 const deferredPrefix = "__hdeferred/"

common/herrors/file_error.go

@@ -384,7 +384,7 @@ func extractPosition(e error) (pos text.Position) {
 	case godartsass.SassError:
 		span := v.Span
 		start := span.Start
-		filename, _ := paths.UrlToFilename(span.Url)
+		filename, _ := paths.UrlStringToFilename(span.Url)
 		pos.Filename = filename
 		pos.Offset = start.Offset
 		pos.ColumnNumber = start.Column

common/hreflect/helpers.go

@@ -223,6 +223,27 @@ func AsTime(v reflect.Value, loc *time.Location) (time.Time, bool) {
 	return time.Time{}, false
 }
 
+// ToSliceAny converts the given value to a slice of any if possible.
+func ToSliceAny(v any) ([]any, bool) {
+	if v == nil {
+		return nil, false
+	}
+	switch vv := v.(type) {
+	case []any:
+		return vv, true
+	default:
+		vvv := reflect.ValueOf(v)
+		if vvv.Kind() == reflect.Slice {
+			out := make([]any, vvv.Len())
+			for i := 0; i < vvv.Len(); i++ {
+				out[i] = vvv.Index(i).Interface()
+			}
+			return out, true
+		}
+	}
+	return nil, false
+}
+
 func CallMethodByName(cxt context.Context, name string, v reflect.Value) []reflect.Value {
 	fn := v.MethodByName(name)
 	var args []reflect.Value

common/hreflect/helpers_test.go

@@ -50,6 +50,19 @@ func TestIsContextType(t *testing.T) {
 	c.Assert(IsContextType(reflect.TypeOf(valueCtx)), qt.IsTrue)
 }
 
+func TestToSliceAny(t *testing.T) {
+	c := qt.New(t)
+
+	checkOK := func(in any, expected []any) {
+		out, ok := ToSliceAny(in)
+		c.Assert(ok, qt.Equals, true)
+		c.Assert(out, qt.DeepEquals, expected)
+	}
+
+	checkOK([]any{1, 2, 3}, []any{1, 2, 3})
+	checkOK([]int{1, 2, 3}, []any{1, 2, 3})
+}
+
 func BenchmarkIsContextType(b *testing.B) {
 	type k string
 	b.Run("value", func(b *testing.B) {

common/maps/cache.go

@@ -113,11 +113,14 @@ func (c *Cache[K, T]) set(key K, value T) {
 }
 
 // ForEeach calls the given function for each key/value pair in the cache.
-func (c *Cache[K, T]) ForEeach(f func(K, T)) {
+// If the function returns false, the iteration stops.
+func (c *Cache[K, T]) ForEeach(f func(K, T) bool) {
 	c.RLock()
 	defer c.RUnlock()
 	for k, v := range c.m {
-		f(k, v)
+		if !f(k, v) {
+			return
+		}
 	}
 }
 

common/paths/url.go

@@ -1,4 +1,4 @@
-// Copyright 2021 The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -159,9 +159,92 @@ func Uglify(in string) string {
 	return path.Clean(in)
 }
 
+// URLEscape escapes unicode letters.
+func URLEscape(uri string) string {
+	// escape unicode letters
+	u, err := url.Parse(uri)
+	if err != nil {
+		panic(err)
+	}
+	return u.String()
+}
+
+// TrimExt trims the extension from a path..
+func TrimExt(in string) string {
+	return strings.TrimSuffix(in, path.Ext(in))
+}
+
+// From https://github.com/golang/go/blob/e0c76d95abfc1621259864adb3d101cf6f1f90fc/src/cmd/go/internal/web/url.go#L45
+func UrlFromFilename(filename string) (*url.URL, error) {
+	if !filepath.IsAbs(filename) {
+		return nil, fmt.Errorf("filepath must be absolute")
+	}
+
+	// If filename has a Windows volume name, convert the volume to a host and prefix
+	// per https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/.
+	if vol := filepath.VolumeName(filename); vol != "" {
+		if strings.HasPrefix(vol, `\\`) {
+			filename = filepath.ToSlash(filename[2:])
+			i := strings.IndexByte(filename, '/')
+
+			if i < 0 {
+				// A degenerate case.
+				// \\host.example.com (without a share name)
+				// becomes
+				// file://host.example.com/
+				return &url.URL{
+					Scheme: "file",
+					Host:   filename,
+					Path:   "/",
+				}, nil
+			}
+
+			// \\host.example.com\Share\path\to\file
+			// becomes
+			// file://host.example.com/Share/path/to/file
+			return &url.URL{
+				Scheme: "file",
+				Host:   filename[:i],
+				Path:   filepath.ToSlash(filename[i:]),
+			}, nil
+		}
+
+		// C:\path\to\file
+		// becomes
+		// file:///C:/path/to/file
+		return &url.URL{
+			Scheme: "file",
+			Path:   "/" + filepath.ToSlash(filename),
+		}, nil
+	}
+
+	// /path/to/file
+	// becomes
+	// file:///path/to/file
+	return &url.URL{
+		Scheme: "file",
+		Path:   filepath.ToSlash(filename),
+	}, nil
+}
+
+// UrlStringToFilename converts a URL string to a filename.
+// Returns the filename and a boolean indicating success.
+// If ParseRequestURI fails, the input is just converted to OS specific slashes and returned.
+func UrlStringToFilename(s string) (string, bool) {
+	u, err := url.ParseRequestURI(s)
+	if err != nil {
+		return filepath.FromSlash(s), false
+	}
+	u.Scheme = fileURLSchemeReplacer.Replace(u.Scheme)
+	return UrlToFilename(u)
+}
+
+// hugostdin is used in the Dart Sass integration.
+var fileURLSchemeReplacer = strings.NewReplacer("hugostdin", "file")
+
 // UrlToFilename converts the URL s to a filename.
 // If ParseRequestURI fails, the input is just converted to OS specific slashes and returned.
-func UrlToFilename(s string) (string, bool) {
+func doUrlStringToFilename(s string) (string, bool) {
 	u, err := url.ParseRequestURI(s)
 	if err != nil {
 		return filepath.FromSlash(s), false
@@ -184,12 +267,29 @@ func UrlToFilename(s string) (string, bool) {
 	return p, true
 }
 
-// URLEscape escapes unicode letters.
-func URLEscape(uri string) string {
-	// escape unicode letters
-	u, err := url.Parse(uri)
+// Based on https://github.com/golang/go/blob/e0c76d95abfc1621259864adb3d101cf6f1f90fc/src/cmd/go/internal/web/url.go#L45
+func UrlToFilename(u *url.URL) (string, bool) {
+	if u.Scheme != "file" {
+		return "", false
+	}
+
+	checkAbs := func(path string) (string, bool) {
+		if !filepath.IsAbs(path) {
+			return "", false
+		}
+		return path, true
+	}
+
+	if u.Path == "" {
+		if u.Host != "" || u.Opaque == "" {
+			return "", false
+		}
+		return checkAbs(filepath.FromSlash(u.Opaque))
+	}
+
+	path, err := convertFileURLPath(u.Host, u.Path)
 	if err != nil {
-		panic(err)
+		return path, false
 	}
-	return u.String()
+	return checkAbs(path)
 }

common/paths/url_other.go

@@ -0,0 +1,31 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !windows
+
+package paths
+
+import (
+	"errors"
+	"path/filepath"
+)
+
+// From https://github.com/golang/go/blob/6c25cf1c5fc063cc9ea27aa850ef0c4345f3a5b4/src/cmd/go/internal/web/url_other.go#L14
+func convertFileURLPath(host, path string) (string, error) {
+	switch host {
+	case "", "localhost":
+	default:
+		return "", errors.New("file URL specifies non-local host")
+	}
+	return filepath.FromSlash(path), nil
+}

common/paths/url_windows.go

@@ -0,0 +1,53 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package paths
+
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+)
+
+// From https://github.com/golang/go/blob/6c25cf1c5fc063cc9ea27aa850ef0c4345f3a5b4/src/cmd/go/internal/web/url_windows.go#L13
+func convertFileURLPath(host, path string) (string, error) {
+	if len(path) == 0 || path[0] != '/' {
+		return "", errors.New("file URL path must start with /")
+	}
+
+	path = filepath.FromSlash(path)
+
+	// We interpret Windows file URLs per the description in
+	// https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/.
+
+	// The host part of a file URL (if any) is the UNC volume name,
+	// but RFC 8089 reserves the authority "localhost" for the local machine.
+	if host != "" && host != "localhost" {
+		// A common "legacy" format omits the leading slash before a drive letter,
+		// encoding the drive letter as the host instead of part of the path.
+		// (See https://blogs.msdn.microsoft.com/freeassociations/2005/05/19/the-bizarre-and-unhappy-story-of-file-urls/.)
+		// We do not support that format, but we should at least emit a more
+		// helpful error message for it.
+		if filepath.VolumeName(host) != "" {
+			return "", errors.New("file URL encodes volume in host field: too few slashes?")
+		}
+		return `\\` + host + path, nil
+	}
+
+	// If host is empty, path must contain an initial slash followed by a
+	// drive letter and path. Remove the slash and verify that the path is valid.
+	if vol := filepath.VolumeName(path[1:]); vol == "" || strings.HasPrefix(vol, `\\`) {
+		return "", errors.New("file URL missing drive letter")
+	}
+	return path[1:], nil
+}

common/types/closer.go

@@ -19,6 +19,13 @@ type Closer interface {
 	Close() error
 }
 
+// CloserFunc is a convenience type to create a Closer from a function.
+type CloserFunc func() error
+
+func (f CloserFunc) Close() error {
+	return f()
+}
+
 type CloseAdder interface {
 	Add(Closer)
 }

config/allconfig/configlanguage.go

@@ -137,11 +137,11 @@ func (c ConfigLanguage) Watching() bool {
 	return c.m.Base.Internal.Watch
 }
 
-func (c ConfigLanguage) NewIdentityManager(name string) identity.Manager {
+func (c ConfigLanguage) NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager {
 	if !c.Watching() {
 		return identity.NopManager
 	}
-	return identity.NewManager(name)
+	return identity.NewManager(name, opts...)
 }
 
 func (c ConfigLanguage) ContentTypes() config.ContentTypesProvider {

config/configProvider.go

@@ -58,7 +58,7 @@ type AllProvider interface {
 	BuildDrafts() bool
 	Running() bool
 	Watching() bool
-	NewIdentityManager(name string) identity.Manager
+	NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager
 	FastRenderMode() bool
 	PrintUnusedTemplates() bool
 	EnableMissingTranslationPlaceholders() bool