Add 'importbin' statement

Add `importbin` statement.  Similar to `importstr` but the result is
an array of numbers (all integers 0-255).

Author: anguslees

ast/ast.go

@@ -423,6 +423,14 @@ type ImportStr struct {
 
 // ---------------------------------------------------------------------------
 
+// ImportBin represents importbin "file".
+type ImportBin struct {
+	NodeBase
+	File *LiteralString
+}
+
+// ---------------------------------------------------------------------------
+
 // Index represents both e[e] and the syntax sugar e.f.
 //
 // One of index and id will be nil before desugaring.  After desugaring id

ast/clone.go

@@ -169,6 +169,13 @@ func clone(astPtr *Node) {
 		r.File = new(LiteralString)
 		*r.File = *node.File
 
+	case *ImportBin:
+		r := new(ImportBin)
+		*astPtr = r
+		*r = *node
+		r.File = new(LiteralString)
+		*r.File = *node.File
+
 	case *Index:
 		r := new(Index)
 		*astPtr = r

imports.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"unsafe"
 
 	"github.com/google/go-jsonnet/ast"
 	"github.com/google/go-jsonnet/internal/program"
@@ -58,19 +59,33 @@ type Importer interface {
 }
 
 // Contents is a representation of imported data. It is a simple
-// string wrapper, which makes it easier to enforce the caching policy.
+// byte wrapper, which makes it easier to enforce the caching policy.
 type Contents struct {
-	data *string
+	data *[]byte
 }
 
 func (c Contents) String() string {
+	// Construct string without copying underlying bytes.
+	// NB: This only works because c.data is not modified.
+	return *(*string)(unsafe.Pointer(c.data))
+}
+
+func (c Contents) Data() []byte {
 	return *c.data
 }
 
 // MakeContents creates Contents from a string.
 func MakeContents(s string) Contents {
+	data := []byte(s)
 	return Contents{
-		data: &s,
+		data: &data,
+	}
+}
+
+// MakeContentsRaw creates Contents from (possibly non-utf8) []byte data.
+func MakeContentsRaw(bytes []byte) Contents {
+	return Contents{
+		data: &bytes,
 	}
 }
 
@@ -139,6 +154,20 @@ func (cache *importCache) importString(importedFrom, importedPath string, i *int
 	return makeValueString(data.String()), nil
 }
 
+// ImportString imports an array of bytes, caches it and then returns it.
+func (cache *importCache) importBinary(importedFrom, importedPath string, i *interpreter) (*valueArray, error) {
+	data, _, err := cache.importData(importedFrom, importedPath)
+	if err != nil {
+		return nil, i.Error(err.Error())
+	}
+	bytes := data.Data()
+	elements := make([]*cachedThunk, len(bytes))
+	for i := range bytes {
+		elements[i] = readyThunk(intToValue(int(bytes[i])))
+	}
+	return makeValueArray(elements), nil
+}
+
 func nodeToPV(i *interpreter, filename string, node ast.Node) *cachedThunk {
 	env := makeInitialEnv(filename, i.baseStd)
 	return &cachedThunk{
@@ -223,7 +252,7 @@ func (importer *FileImporter) tryPath(dir, importedPath string) (found bool, con
 		} else {
 			entry = &fsCacheEntry{
 				exists:   true,
-				contents: MakeContents(string(contentBytes)),
+				contents: MakeContentsRaw(contentBytes),
 			}
 		}
 		importer.fsCache[absPath] = entry

internal/formatter/fix_indentation.go

@@ -574,6 +574,11 @@ func (c *FixIndentation) Visit(expr ast.Node, currIndent indent, crowded bool) {
 		newIndent := c.newIndent(*openFodder(node.File), currIndent, c.column+1)
 		c.Visit(node.File, newIndent, true)
 
+	case *ast.ImportBin:
+		c.column += 9 // importbin
+		newIndent := c.newIndent(*openFodder(node.File), currIndent, c.column+1)
+		c.Visit(node.File, newIndent, true)
+
 	case *ast.InSuper:
 		c.Visit(node.Index, currIndent, crowded)
 		c.fill(node.InFodder, true, true, currIndent.lineUp)

internal/formatter/unparser.go

@@ -370,6 +370,10 @@ func (u *unparser) unparse(expr ast.Node, crowded bool) {
 		u.write("importstr")
 		u.unparse(node.File, true)
 
+	case *ast.ImportBin:
+		u.write("importbin")
+		u.unparse(node.File, true)
+
 	case *ast.Index:
 		u.unparse(node.Target, crowded)
 		u.fill(node.LeftBracketFodder, false, false) // Can also be DotFodder

internal/parser/context.go

@@ -64,9 +64,7 @@ func DirectChildren(node ast.Node) []ast.Node {
 		return []ast.Node{node.Expr}
 	case *ast.Function:
 		return nil
-	case *ast.Import:
-		return nil
-	case *ast.ImportStr:
+	case *ast.Import, *ast.ImportStr, *ast.ImportBin:
 		return nil
 	case *ast.Index:
 		if node.Id != nil {
@@ -181,9 +179,7 @@ func thunkChildren(node ast.Node) []ast.Node {
 		return nil
 	case *ast.Function:
 		return nil
-	case *ast.Import:
-		return nil
-	case *ast.ImportStr:
+	case *ast.Import, *ast.ImportStr, *ast.ImportBin:
 		return nil
 	case *ast.Index:
 		return nil
@@ -304,9 +300,7 @@ func specialChildren(node ast.Node) []ast.Node {
 			}
 		}
 		return children
-	case *ast.Import:
-		return nil
-	case *ast.ImportStr:
+	case *ast.Import, *ast.ImportStr, *ast.ImportBin:
 		return nil
 	case *ast.Index:
 		return nil

internal/parser/lexer.go

@@ -65,6 +65,7 @@ const (
 	tokenIf
 	tokenImport
 	tokenImportStr
+	tokenImportBin
 	tokenIn
 	tokenLocal
 	tokenNullLit
@@ -112,6 +113,7 @@ var tokenKindStrings = []string{
 	tokenIf:         "if",
 	tokenImport:     "import",
 	tokenImportStr:  "importstr",
+	tokenImportBin:  "importbin",
 	tokenIn:         "in",
 	tokenLocal:      "local",
 	tokenNullLit:    "null",
@@ -580,6 +582,8 @@ func getTokenKindFromID(str string) tokenKind {
 		return tokenImport
 	case "importstr":
 		return tokenImportStr
+	case "importbin":
+		return tokenImportBin
 	case "in":
 		return tokenIn
 	case "local":

internal/parser/lexer_test.go

@@ -424,6 +424,12 @@ func TestImportstr(t *testing.T) {
 	})
 }
 
+func TestImportbin(t *testing.T) {
+	SingleTest(t, "importbin", "", Tokens{
+		{kind: tokenImportBin, data: "importbin"},
+	})
+}
+
 func TestIn(t *testing.T) {
 	SingleTest(t, "in", "", Tokens{
 		{kind: tokenIn, data: "in"},

internal/parser/parser.go

@@ -870,7 +870,7 @@ func (p *parser) parseTerminal() (ast.Node, errors.StaticError) {
 	tok := p.pop()
 	switch tok.kind {
 	case tokenAssert, tokenBraceR, tokenBracketR, tokenComma, tokenDot, tokenElse,
-		tokenError, tokenFor, tokenFunction, tokenIf, tokenIn, tokenImport, tokenImportStr,
+		tokenError, tokenFor, tokenFunction, tokenIf, tokenIn, tokenImport, tokenImportStr, tokenImportBin,
 		tokenLocal, tokenOperator, tokenParenR, tokenSemicolon, tokenTailStrict, tokenThen:
 		return nil, makeUnexpectedError(tok, "parsing terminal")
 
@@ -1122,6 +1122,23 @@ func (p *parser) parse(prec precedence) (ast.Node, errors.StaticError) {
 		}
 		return nil, errors.MakeStaticError("Computed imports are not allowed", *body.Loc())
 
+	case tokenImportBin:
+		p.pop()
+		body, err := p.parse(maxPrecedence)
+		if err != nil {
+			return nil, err
+		}
+		if lit, ok := body.(*ast.LiteralString); ok {
+			if lit.Kind == ast.StringBlock {
+				return nil, errors.MakeStaticError("Block string literals not allowed in imports", *body.Loc())
+			}
+			return &ast.ImportBin{
+				NodeBase: ast.NewNodeBaseLoc(locFromTokenAST(begin, body), begin.fodder),
+				File:     lit,
+			}, nil
+		}
+		return nil, errors.MakeStaticError("Computed imports are not allowed", *body.Loc())
+
 	case tokenLocal:
 		p.pop()
 		var binds ast.LocalBinds

internal/parser/parser_test.go

@@ -98,6 +98,7 @@ var tests = []string{
 
 	`import 'foo.jsonnet'`,
 	`importstr 'foo.text'`,
+	`importbin 'foo.bin'`,
 
 	`{a: b} + {c: d}`,
 	`{a: b}{c: d}`,
@@ -230,6 +231,8 @@ var errorTests = []testError{
 	{`import (a+b)`, `test:1:8-13 Computed imports are not allowed`},
 	{`importstr (a b)`, `test:1:14-15 Expected token ")" but got (IDENTIFIER, "b")`},
 	{`importstr (a+b)`, `test:1:11-16 Computed imports are not allowed`},
+	{`importbin (a b)`, `test:1:14-15 Expected token ")" but got (IDENTIFIER, "b")`},
+	{`importbin (a+b)`, `test:1:11-16 Computed imports are not allowed`},
 
 	{`local a = b ()`, `test:1:15 Expected , or ; but got end of file`},
 	{`local a = b; (a b)`, `test:1:17-18 Expected token ")" but got (IDENTIFIER, "b")`},

internal/pass/pass.go

@@ -48,6 +48,7 @@ type ASTPass interface {
 	Function(ASTPass, *ast.Function, Context)
 	Import(ASTPass, *ast.Import, Context)
 	ImportStr(ASTPass, *ast.ImportStr, Context)
+	ImportBin(ASTPass, *ast.ImportBin, Context)
 	Index(ASTPass, *ast.Index, Context)
 	Slice(ASTPass, *ast.Slice, Context)
 	Local(ASTPass, *ast.Local, Context)
@@ -282,6 +283,12 @@ func (*Base) ImportStr(p ASTPass, node *ast.ImportStr, ctx Context) {
 	p.LiteralString(p, node.File, ctx)
 }
 
+// ImportBin traverses that kind of node
+func (*Base) ImportBin(p ASTPass, node *ast.ImportBin, ctx Context) {
+	p.Fodder(p, &node.File.Fodder, ctx)
+	p.LiteralString(p, node.File, ctx)
+}
+
 // Index traverses that kind of node
 func (*Base) Index(p ASTPass, node *ast.Index, ctx Context) {
 	p.Visit(p, &node.Target, ctx)
@@ -419,6 +426,8 @@ func (*Base) Visit(p ASTPass, node *ast.Node, ctx Context) {
 		p.Import(p, node, ctx)
 	case *ast.ImportStr:
 		p.ImportStr(p, node, ctx)
+	case *ast.ImportBin:
+		p.ImportBin(p, node, ctx)
 	case *ast.Index:
 		p.Index(p, node, ctx)
 	case *ast.InSuper:

internal/program/desugarer.go

@@ -441,6 +441,14 @@ func desugar(astPtr *ast.Node, objLevel int) (err error) {
 			return
 		}
 
+	case *ast.ImportBin:
+		// See comment in ast.Import.
+		var file ast.Node = node.File
+		err = desugar(&file, objLevel)
+		if err != nil {
+			return
+		}
+
 	case *ast.Index:
 		err = desugar(&node.Target, objLevel)
 		if err != nil {

internal/program/static_analyzer.go

@@ -89,9 +89,7 @@ func analyzeVisit(a ast.Node, inObject bool, vars ast.IdentifierSet) error {
 		for _, param := range a.Parameters {
 			s.freeVars.Remove(param.Name)
 		}
-	case *ast.Import:
-		//nothing to do here
-	case *ast.ImportStr:
+	case *ast.Import, *ast.ImportStr, *ast.ImportBin:
 		//nothing to do here
 	case *ast.InSuper:
 		if !inObject {

interpreter.go

@@ -488,6 +488,10 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) {
 		codePath := node.Loc().FileName
 		return i.importCache.importString(codePath, node.File.Value, i)
 
+	case *ast.ImportBin:
+		codePath := node.Loc().FileName
+		return i.importCache.importBinary(codePath, node.File.Value, i)
+
 	case *ast.LiteralBoolean:
 		return makeValueBoolean(node.Value), nil
 

jsonnet_test.go

@@ -109,10 +109,11 @@ func TestCustomImporter(t *testing.T) {
 		map[string]Contents{
 			"a.jsonnet": MakeContents("2 + 2"),
 			"b.jsonnet": MakeContents("3 + 3"),
+			"c.bin":     MakeContentsRaw([]byte{0xff, 0xfe, 0xfd}),
 		},
 	})
-	input := `[import "a.jsonnet", importstr "b.jsonnet"]`
-	expected := `[ 4, "3 + 3" ]`
+	input := `[import "a.jsonnet", importstr "b.jsonnet", importbin "c.bin"]`
+	expected := `[ 4, "3 + 3", [ 255, 254, 253 ] ]`
 	actual, err := vm.EvaluateSnippet("custom_import.jsonnet", input)
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
@@ -159,7 +160,7 @@ func TestExtVarImportedFrom(t *testing.T) {
 	if actual != expected {
 		t.Errorf("Expected %q, but got %q", expected, actual)
 	}
-	expectedImportHistory := []importHistoryEntry{importHistoryEntry{"", "a.jsonnet"}}
+	expectedImportHistory := []importHistoryEntry{{"", "a.jsonnet"}}
 	if !reflect.DeepEqual(importer.history, expectedImportHistory) {
 		t.Errorf("Expected %q, but got %q", expectedImportHistory, importer.history)
 	}
@@ -186,7 +187,7 @@ func TestTLAImportedFrom(t *testing.T) {
 	if actual != expected {
 		t.Errorf("Expected %q, but got %q", expected, actual)
 	}
-	expectedImportHistory := []importHistoryEntry{importHistoryEntry{"", "a.jsonnet"}}
+	expectedImportHistory := []importHistoryEntry{{"", "a.jsonnet"}}
 	if !reflect.DeepEqual(importer.history, expectedImportHistory) {
 		t.Errorf("Expected %q, but got %q", expectedImportHistory, importer.history)
 	}
@@ -212,7 +213,7 @@ func TestAnonymousImportedFrom(t *testing.T) {
 	if actual != expected {
 		t.Errorf("Expected %q, but got %q", expected, actual)
 	}
-	expectedImportHistory := []importHistoryEntry{importHistoryEntry{"", "a.jsonnet"}}
+	expectedImportHistory := []importHistoryEntry{{"", "a.jsonnet"}}
 	if !reflect.DeepEqual(importer.history, expectedImportHistory) {
 		t.Errorf("Expected %q, but got %q", expectedImportHistory, importer.history)
 	}

linter/internal/types/build_graph.go

@@ -217,6 +217,8 @@ func calcTP(node ast.Node, varAt map[ast.Node]*common.Variable, g *typeGraph) ty
 		return tpRef(g.getExprPlaceholder(imported))
 	case *ast.ImportStr:
 		return tpRef(stringType)
+	case *ast.ImportBin:
+		return tpRef(anyArrayType)
 	case *ast.LiteralBoolean:
 		return tpRef(boolType)
 	case *ast.LiteralNull:

linter/linter.go

@@ -124,6 +124,12 @@ func getImports(vm *jsonnet.VM, node nodeWithLocation, roots map[string]ast.Node
 		if err != nil {
 			errWriter.writeError(vm, errors.MakeStaticError(err.Error(), *node.Loc()))
 		}
+	case *ast.ImportBin:
+		p := node.File.Value
+		_, err := vm.ResolveImport(currentPath, p)
+		if err != nil {
+			errWriter.writeError(vm, errors.MakeStaticError(err.Error(), *node.Loc()))
+		}
 	default:
 		for _, c := range parser.Children(node) {
 			getImports(vm, nodeWithLocation{c, currentPath}, roots, errWriter)

testdata/importbin_block_literal.golden

@@ -0,0 +1,7 @@
+testdata/importbin_block_literal:(1:11)-(3:4) Block string literals not allowed in imports
+
+importbin |||
+	block_literals_for_imports_are_not_allowed_and_make_exactly_zero_sense
+|||
+
+

testdata/importbin_block_literal.jsonnet

@@ -0,0 +1,3 @@
+importbin |||
+	block_literals_for_imports_are_not_allowed_and_make_exactly_zero_sense
+|||

testdata/importbin_block_literal.linter.golden

@@ -0,0 +1,7 @@
+../testdata/importbin_block_literal:(1:11)-(3:4) Block string literals not allowed in imports
+
+importbin |||
+	block_literals_for_imports_are_not_allowed_and_make_exactly_zero_sense
+|||
+
+