Skip to content

Commit 565c30e

Browse files
bepbep
committed
js: Fix js.Batch for multihost setups
Note that this is an unreleased feature. Fixes #13151
1 parent 48dd6a9 commit 565c30e

File tree

8 files changed

+190
-71
lines changed

8 files changed

+190
-71
lines changed

‎deps/deps.go

+7
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/gohugoio/hugo/helpers"
2525
"github.com/gohugoio/hugo/hugofs"
2626
"github.com/gohugoio/hugo/identity"
27+
"github.com/gohugoio/hugo/internal/js"
2728
"github.com/gohugoio/hugo/internal/warpc"
2829
"github.com/gohugoio/hugo/media"
2930
"github.com/gohugoio/hugo/resources/page"
@@ -105,6 +106,12 @@ type Deps struct {
105106
// TODO(bep) rethink this re. a plugin setup, but this will have to do for now.
106107
WasmDispatchers *warpc.Dispatchers
107108

109+
// The JS batcher client.
110+
JSBatcherClient js.BatcherClient
111+
112+
// The JS batcher client.
113+
// JSBatcherClient *esbuild.BatcherClient
114+
108115
isClosed bool
109116

110117
*globalErrHandler

‎hugolib/paths/paths.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func New(fs *hugofs.Fs, cfg config.AllProvider) (*Paths, error) {
6767
var multihostTargetBasePaths []string
6868
if cfg.IsMultihost() && len(cfg.Languages()) > 1 {
6969
for _, l := range cfg.Languages() {
70-
multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang)
70+
multihostTargetBasePaths = append(multihostTargetBasePaths, hpaths.ToSlashPreserveLeading(l.Lang))
7171
}
7272
}
7373

‎hugolib/site.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242
"github.com/gohugoio/hugo/deps"
4343
"github.com/gohugoio/hugo/hugolib/doctree"
4444
"github.com/gohugoio/hugo/hugolib/pagesfromdata"
45+
"github.com/gohugoio/hugo/internal/js/esbuild"
4546
"github.com/gohugoio/hugo/internal/warpc"
4647
"github.com/gohugoio/hugo/langs/i18n"
4748
"github.com/gohugoio/hugo/modules"
@@ -205,6 +206,12 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
205206
return nil, err
206207
}
207208

209+
batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps)
210+
if err != nil {
211+
return nil, err
212+
}
213+
firstSiteDeps.JSBatcherClient = batcherClient
214+
208215
confm := cfg.Configs
209216
if err := confm.Validate(logger); err != nil {
210217
return nil, err
@@ -313,7 +320,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
313320
return li.Lang < lj.Lang
314321
})
315322

316-
var err error
317323
h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
318324
if err == nil && h == nil {
319325
panic("hugo: newHugoSitesNew returned nil error and nil HugoSites")

‎internal/js/api.go

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright 2024 The Hugo Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
package js
15+
16+
import (
17+
"context"
18+
19+
"github.com/gohugoio/hugo/common/maps"
20+
"github.com/gohugoio/hugo/resources/resource"
21+
)
22+
23+
// BatcherClient is used to do JS batch operations.
24+
type BatcherClient interface {
25+
New(id string) (Batcher, error)
26+
Store() *maps.Cache[string, Batcher]
27+
}
28+
29+
// BatchPackage holds a group of JavaScript resources.
30+
type BatchPackage interface {
31+
Groups() map[string]resource.Resources
32+
}
33+
34+
// Batcher is used to build JavaScript packages.
35+
type Batcher interface {
36+
Build(context.Context) (BatchPackage, error)
37+
Config(ctx context.Context) OptionsSetter
38+
Group(ctx context.Context, id string) BatcherGroup
39+
}
40+
41+
// BatcherGroup is a group of scripts and instances.
42+
type BatcherGroup interface {
43+
Instance(sid, iid string) OptionsSetter
44+
Runner(id string) OptionsSetter
45+
Script(id string) OptionsSetter
46+
}
47+
48+
// OptionsSetter is used to set options for a batch, script or instance.
49+
type OptionsSetter interface {
50+
SetOptions(map[string]any) string
51+
}

‎internal/js/esbuild/batch.go

+56-49
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
_ "embed"
2121
"encoding/json"
2222
"fmt"
23+
"io"
2324
"path"
2425
"path/filepath"
2526
"reflect"
@@ -34,19 +35,20 @@ import (
3435
"github.com/gohugoio/hugo/common/maps"
3536
"github.com/gohugoio/hugo/common/paths"
3637
"github.com/gohugoio/hugo/deps"
38+
"github.com/gohugoio/hugo/helpers"
3739
"github.com/gohugoio/hugo/identity"
40+
"github.com/gohugoio/hugo/internal/js"
3841
"github.com/gohugoio/hugo/lazy"
3942
"github.com/gohugoio/hugo/media"
4043
"github.com/gohugoio/hugo/resources"
4144
"github.com/gohugoio/hugo/resources/resource"
4245
"github.com/gohugoio/hugo/resources/resource_factories/create"
4346
"github.com/gohugoio/hugo/tpl"
4447
"github.com/mitchellh/mapstructure"
45-
"github.com/spf13/afero"
4648
"github.com/spf13/cast"
4749
)
4850

49-
var _ Batcher = (*batcher)(nil)
51+
var _ js.Batcher = (*batcher)(nil)
5052

5153
const (
5254
NsBatch = "_hugo-js-batch"
@@ -58,7 +60,7 @@ const (
5860
//go:embed batch-esm-runner.gotmpl
5961
var runnerTemplateStr string
6062

61-
var _ BatchPackage = (*Package)(nil)
63+
var _ js.BatchPackage = (*Package)(nil)
6264

6365
var _ buildToucher = (*optsHolder[scriptOptions])(nil)
6466

@@ -67,16 +69,17 @@ var (
6769
_ isBuiltOrTouchedProvider = (*scriptGroup)(nil)
6870
)
6971

70-
func NewBatcherClient(deps *deps.Deps) (*BatcherClient, error) {
72+
func NewBatcherClient(deps *deps.Deps) (js.BatcherClient, error) {
7173
c := &BatcherClient{
7274
d: deps,
7375
buildClient: NewBuildClient(deps.BaseFs.Assets, deps.ResourceSpec),
7476
createClient: create.New(deps.ResourceSpec),
75-
bundlesCache: maps.NewCache[string, BatchPackage](),
77+
batcherStore: maps.NewCache[string, js.Batcher](),
78+
bundlesStore: maps.NewCache[string, js.BatchPackage](),
7679
}
7780

7881
deps.BuildEndListeners.Add(func(...any) bool {
79-
c.bundlesCache.Reset()
82+
c.bundlesStore.Reset()
8083
return false
8184
})
8285

@@ -125,7 +128,7 @@ func (o *opts[K, C]) Reset() {
125128
o.h.resetCounter++
126129
}
127130

128-
func (o *opts[K, C]) Get(id uint32) OptionsSetter {
131+
func (o *opts[K, C]) Get(id uint32) js.OptionsSetter {
129132
var b *optsHolder[C]
130133
o.once.Do(func() {
131134
b = o.h
@@ -184,18 +187,6 @@ func newOpts[K any, C optionsCompiler[C]](key K, optionsID string, defaults defa
184187
}
185188
}
186189

187-
// BatchPackage holds a group of JavaScript resources.
188-
type BatchPackage interface {
189-
Groups() map[string]resource.Resources
190-
}
191-
192-
// Batcher is used to build JavaScript packages.
193-
type Batcher interface {
194-
Build(context.Context) (BatchPackage, error)
195-
Config(ctx context.Context) OptionsSetter
196-
Group(ctx context.Context, id string) BatcherGroup
197-
}
198-
199190
// BatcherClient is a client for building JavaScript packages.
200191
type BatcherClient struct {
201192
d *deps.Deps
@@ -206,12 +197,13 @@ type BatcherClient struct {
206197
createClient *create.Client
207198
buildClient *BuildClient
208199

209-
bundlesCache *maps.Cache[string, BatchPackage]
200+
batcherStore *maps.Cache[string, js.Batcher]
201+
bundlesStore *maps.Cache[string, js.BatchPackage]
210202
}
211203

212204
// New creates a new Batcher with the given ID.
213205
// This will be typically created once and reused across rebuilds.
214-
func (c *BatcherClient) New(id string) (Batcher, error) {
206+
func (c *BatcherClient) New(id string) (js.Batcher, error) {
215207
var initErr error
216208
c.once.Do(func() {
217209
// We should fix the initialization order here (or use the Go template package directly), but we need to wait
@@ -288,6 +280,10 @@ func (c *BatcherClient) New(id string) (Batcher, error) {
288280
return b, nil
289281
}
290282

283+
func (c *BatcherClient) Store() *maps.Cache[string, js.Batcher] {
284+
return c.batcherStore
285+
}
286+
291287
func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTemplateContext) (resource.Resource, string, error) {
292288
var buf bytes.Buffer
293289

@@ -304,18 +300,6 @@ func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTempla
304300
return r, s, nil
305301
}
306302

307-
// BatcherGroup is a group of scripts and instances.
308-
type BatcherGroup interface {
309-
Instance(sid, iid string) OptionsSetter
310-
Runner(id string) OptionsSetter
311-
Script(id string) OptionsSetter
312-
}
313-
314-
// OptionsSetter is used to set options for a batch, script or instance.
315-
type OptionsSetter interface {
316-
SetOptions(map[string]any) string
317-
}
318-
319303
// Package holds a group of JavaScript resources.
320304
type Package struct {
321305
id string
@@ -353,9 +337,9 @@ type batcher struct {
353337
}
354338

355339
// Build builds the batch if not already built or if it's stale.
356-
func (b *batcher) Build(ctx context.Context) (BatchPackage, error) {
340+
func (b *batcher) Build(ctx context.Context) (js.BatchPackage, error) {
357341
key := dynacache.CleanKey(b.id + ".js")
358-
p, err := b.client.bundlesCache.GetOrCreate(key, func() (BatchPackage, error) {
342+
p, err := b.client.bundlesStore.GetOrCreate(key, func() (js.BatchPackage, error) {
359343
return b.build(ctx)
360344
})
361345
if err != nil {
@@ -364,11 +348,11 @@ func (b *batcher) Build(ctx context.Context) (BatchPackage, error) {
364348
return p, nil
365349
}
366350

367-
func (b *batcher) Config(ctx context.Context) OptionsSetter {
351+
func (b *batcher) Config(ctx context.Context) js.OptionsSetter {
368352
return b.configOptions.Get(b.buildCount)
369353
}
370354

371-
func (b *batcher) Group(ctx context.Context, id string) BatcherGroup {
355+
func (b *batcher) Group(ctx context.Context, id string) js.BatcherGroup {
372356
if err := ValidateBatchID(id, false); err != nil {
373357
panic(err)
374358
}
@@ -419,7 +403,7 @@ func (b *batcher) isStale() bool {
419403
return false
420404
}
421405

422-
func (b *batcher) build(ctx context.Context) (BatchPackage, error) {
406+
func (b *batcher) build(ctx context.Context) (js.BatchPackage, error) {
423407
b.mu.Lock()
424408
defer b.mu.Unlock()
425409
defer func() {
@@ -463,6 +447,8 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) {
463447
pathGroup: maps.NewCache[string, string](),
464448
}
465449

450+
multihostBasePaths := b.client.d.ResourceSpec.MultihostTargetBasePaths
451+
466452
// Entry points passed to ESBuid.
467453
var entryPoints []string
468454
addResource := func(group, pth string, r resource.Resource, isResult bool) {
@@ -701,15 +687,36 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) {
701687

702688
if !handled {
703689
// Copy to destination.
704-
p := strings.TrimPrefix(o.Path, outDir)
705-
targetFilename := filepath.Join(b.id, p)
706-
fs := b.client.d.BaseFs.PublishFs
707-
if err := fs.MkdirAll(filepath.Dir(targetFilename), 0o777); err != nil {
708-
return nil, fmt.Errorf("failed to create dir %q: %w", targetFilename, err)
690+
// In a multihost setup, we will have multiple targets.
691+
var targetFilenames []string
692+
if len(multihostBasePaths) > 0 {
693+
for _, base := range multihostBasePaths {
694+
p := strings.TrimPrefix(o.Path, outDir)
695+
targetFilename := filepath.Join(base, b.id, p)
696+
targetFilenames = append(targetFilenames, targetFilename)
697+
}
698+
} else {
699+
p := strings.TrimPrefix(o.Path, outDir)
700+
targetFilename := filepath.Join(b.id, p)
701+
targetFilenames = append(targetFilenames, targetFilename)
709702
}
710703

711-
if err := afero.WriteFile(fs, targetFilename, o.Contents, 0o666); err != nil {
712-
return nil, fmt.Errorf("failed to write to %q: %w", targetFilename, err)
704+
fs := b.client.d.BaseFs.PublishFs
705+
706+
if err := func() error {
707+
fw, err := helpers.OpenFilesForWriting(fs, targetFilenames...)
708+
if err != nil {
709+
return err
710+
}
711+
defer fw.Close()
712+
713+
fr := bytes.NewReader(o.Contents)
714+
715+
_, err = io.Copy(fw, fr)
716+
717+
return err
718+
}(); err != nil {
719+
return nil, fmt.Errorf("failed to copy to %q: %w", targetFilenames, err)
713720
}
714721
}
715722
}
@@ -845,7 +852,7 @@ type optionsGetSetter[K, C any] interface {
845852
Key() K
846853
Reset()
847854

848-
Get(uint32) OptionsSetter
855+
Get(uint32) js.OptionsSetter
849856
isStale() bool
850857
currPrev() (map[string]any, map[string]any)
851858
}
@@ -975,7 +982,7 @@ func (b *scriptGroup) IdentifierBase() string {
975982
return b.id
976983
}
977984

978-
func (s *scriptGroup) Instance(sid, id string) OptionsSetter {
985+
func (s *scriptGroup) Instance(sid, id string) js.OptionsSetter {
979986
if err := ValidateBatchID(sid, false); err != nil {
980987
panic(err)
981988
}
@@ -1014,7 +1021,7 @@ func (g *scriptGroup) Reset() {
10141021
}
10151022
}
10161023

1017-
func (s *scriptGroup) Runner(id string) OptionsSetter {
1024+
func (s *scriptGroup) Runner(id string) js.OptionsSetter {
10181025
if err := ValidateBatchID(id, false); err != nil {
10191026
panic(err)
10201027
}
@@ -1043,7 +1050,7 @@ func (s *scriptGroup) Runner(id string) OptionsSetter {
10431050
return s.runnersOptions[sid].Get(s.b.buildCount)
10441051
}
10451052

1046-
func (s *scriptGroup) Script(id string) OptionsSetter {
1053+
func (s *scriptGroup) Script(id string) js.OptionsSetter {
10471054
if err := ValidateBatchID(id, false); err != nil {
10481055
panic(err)
10491056
}

0 commit comments

Comments
 (0)