Release wrap up v0.47.0 (#377)

* Misc cleanup pre-release

Author: neilotoole

.images/sq_inspect_remote_s3.png

@@ -10,9 +10,9 @@ Breaking changes are annotated with ☢️, and alpha/beta features with 🐥.
 ## Upcoming
 
 This is a significant release, focused on improving i/o, responsiveness,
-and performance. The headline features are caching of ingested data for
-document sources such as CSV or Excel, and download caching for remote
-document sources.
+and performance. The headline features are [caching](https://sq.io/docs/source#cache)
+of [ingested](https://sq.io/docs/source#ingest) data for [document sources](https://sq.io/docs/source#document-source)
+such as CSV or Excel, and [download](https://sq.io/docs/source#download) caching for remote document sources.
 
 ### Added
 
@@ -21,13 +21,16 @@ document sources.
   by the new config options [`progress`](https://sq.io/docs/config#progress)
   and [`progress.delay`](https://sq.io/docs/config#progressdelay). You can also use
   the `--no-progress` flag to disable the progress bar.
-- Ingested [document sources](https://sq.io/docs/concepts#document-source) (such as
+  - Note: the progress bar is rendered
+    on `stderr` and is always zapped from the terminal when command output begins. It won't corrupt the output.
+- Ingested [document sources](https://sq.io/docs/source#document-source) (such as
   [CSV](https://sq.io/docs/drivers/csv) or [Excel](https://sq.io/docs/drivers/xlsx))
-  now make use of an ingest cache DB. Previously, ingestion of document source data occurred
-  on each `sq` command. It is now a one-time cost; subsequent use of the document source utilizes
-  the cache DB. If the source document changes, the ingest cache DB is invalidated and
+  now make use of an [ingest](https://sq.io/docs/source#ingest) cache DB. Previously, ingestion 
+  of document source data occurred  on each `sq` command. It is now a one-time cost; subsequent
+  use of the document source utilizes
+  the cache DB. Until, that is, the source document changes: then the ingest cache DB is invalidated and
   ingested again. This is a significantly improved experience for large document sources.
-- There's several new commands to interact with the cache.
+- There are several new commands to interact with the cache (although you shouldn't need to):
   - [`sq cache enable`](https://sq.io/docs/cmd/cache_enable) and
   [`sq cache disable`](https://sq.io/docs/cmd/cache_disable) control cache usage.
   You can also instead use the new [`ingest.cache`](https://sq.io/docs/config#ingestcache)
@@ -36,11 +39,11 @@ document sources.
   - [`sq cache location`](https://sq.io/docs/cmd/cache_location) prints the cache location on disk.
   - [`sq cache stat`](https://sq.io/docs/cmd/cache_stat) shows stats about the cache.
   - [`sq cache tree`](https://sq.io/docs/cmd/cache_location) shows a tree view of the cache.
-- Downloading of remote document sources (e.g. a CSV file at
+- The [download](https://sq.io/docs/source#download) mechanism for remote document sources (e.g. a CSV file at
   [`https://sq.io/testdata/actor.csv`](https://sq.io/testdata/actor.csv)) has been completely
   overhauled. Previously, `sq` would re-download the remote file on every command. Now, the
-  remote file is downloaded and cached locally. Subsequent commands check for staleness of
-  the cached download, and re-download if necessary.
+  remote file is downloaded and [cached](https://sq.io/docs/source#cache) locally.
+  Subsequent `sq` invocations check for staleness of the cached download, and re-download if necessary.
 - As part of the download revamp, new config options have been introduced:
   - [`http.request.timeout`](https://sq.io/docs/config#httprequesttimeout) is the timeout for the initial response from the server, and
     [`http.response.timeout`](https://sq.io/docs/config#httpresponsetimeout) is the timeout for reading the entire response body. We separate

CHANGELOG.md

@@ -8,6 +8,7 @@
 
 `sq` is a command line tool that provides jq-style access to
 structured data sources: SQL databases, or document formats like CSV or Excel.
+It is the lovechild of sql+jq.
 
 ![sq](.images/splash.png)
 
@@ -84,7 +85,9 @@ $ sq ls
 
 Let's [add](https://sq.io/docs/cmd/add) a source. First we'll add a [SQLite](https://sq.io/docs/drivers/sqlite)
 database, but this could also be [Postgres](https://sq.io/docs/drivers/postgres),
-[SQL Server](https://sq.io/docs/drivers/sqlserver), [Excel](https://sq.io/docs/drivers/xlsx), etc.
+[SQL Server](https://sq.io/docs/drivers/sqlserver) etc., or a document source such [Excel](https://sq.io/docs/drivers/xlsx) or
+[CSV](https://sq.io/docs/drivers/csv).
+
 Download the sample DB, and `sq add` the source. 
 
 ```shell
@@ -118,6 +121,19 @@ $ sq ping
 @sakila  1ms  pong
 ```
 
+> [!TIP]
+> Document sources such as CSV or Excel can be added from the local filesystem, or
+> from a remote HTTP URL. 
+> 
+> ```shell
+> $ sq add https://acme.s3.amazonaws.com/sales.csv
+> ```
+> 
+> See the [sources](https://sq.io/docs/source#download) docs for more.
+> 
+> ![sq inspect remote](./.images/sq_inspect_remote_s3.png)
+
+
 ### Query
 
 Fundamentally, `sq` is for querying data. The jq-style syntax is covered in

README.md

@@ -262,7 +262,6 @@ func newCommandTree(ru *run.Run) (rootCmd *cobra.Command) {
 	addCmd(ru, xCmd, newXLockSrcCmd())
 	addCmd(ru, xCmd, newXLockConfigCmd())
 	addCmd(ru, xCmd, newXProgressCmd())
-	addCmd(ru, xCmd, newXDownloadCmd())
 
 	return rootCmd
 }

cli/cli.go

@@ -45,11 +45,10 @@ func newCacheCmd() *cobra.Command {
 
 func newCacheLocationCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:     "location",
-		Aliases: []string{"loc"},
-		Short:   "Print cache location",
-		Long:    "Print cache location.",
-		Args:    cobra.ExactArgs(0),
+		Use:   "location",
+		Short: "Print cache location",
+		Long:  "Print cache location.",
+		Args:  cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ru := run.FromContext(cmd.Context())
 			return ru.Writers.Config.CacheLocation(ru.Files.CacheDir())

cli/cmd_cache.go

@@ -59,12 +59,11 @@ func newConfigCmd() *cobra.Command {
 
 func newConfigLocationCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:     "location",
-		Aliases: []string{"loc"},
-		Short:   "Print config location",
-		Long:    "Print config location. Use --verbose for more detail.",
-		Args:    cobra.ExactArgs(0),
-		RunE:    execConfigLocation,
+		Use:   "location",
+		Short: "Print config location",
+		Long:  "Print config location. Use --verbose for more detail.",
+		Args:  cobra.ExactArgs(0),
+		RunE:  execConfigLocation,
 		Example: `  # Print config location
   $ sq config location
   /Users/neilotoole/.config/sq

cli/cmd_config.go

@@ -104,6 +104,9 @@ formats both show extensive detail.`,
 	panicOn(cmd.RegisterFlagCompletionFunc(flag.ActiveSchema,
 		activeSchemaCompleter{getActiveSourceViaArgs}.complete))
 	addOptionFlag(cmd.Flags(), driver.OptIngestCache)
+
+	cmd.Flags().StringP(flag.Input, flag.InputShort, "", flag.InputUsage)
+	panicOn(cmd.Flags().MarkHidden(flag.Input)) // Hide for now; this is mostly used for testing.
 	return cmd
 }
 

cli/cmd_inspect.go

@@ -327,6 +327,9 @@ func addQueryCmdFlags(cmd *cobra.Command) {
 
 	cmd.Flags().StringP(flag.Output, flag.OutputShort, "", flag.OutputUsage)
 
+	cmd.Flags().StringP(flag.Input, flag.InputShort, "", flag.InputUsage)
+	panicOn(cmd.Flags().MarkHidden(flag.Input)) // Hide for now; this is mostly used for testing.
+
 	cmd.Flags().String(flag.Insert, "", flag.InsertUsage)
 	panicOn(cmd.RegisterFlagCompletionFunc(flag.Insert,
 		(&handleTableCompleter{onlySQL: true, handleRequired: true}).complete))

cli/cmd_slq.go

@@ -3,22 +3,15 @@ package cli
 import (
 	"bufio"
 	"fmt"
-	"net/url"
 	"os"
 	"time"
 
 	"github.com/spf13/cobra"
 
-	"github.com/neilotoole/sq/cli/flag"
 	"github.com/neilotoole/sq/cli/run"
-	"github.com/neilotoole/sq/libsq/core/errz"
-	"github.com/neilotoole/sq/libsq/core/ioz/checksum"
-	"github.com/neilotoole/sq/libsq/core/ioz/httpz"
 	"github.com/neilotoole/sq/libsq/core/lg"
 	"github.com/neilotoole/sq/libsq/core/progress"
 	"github.com/neilotoole/sq/libsq/files"
-	"github.com/neilotoole/sq/libsq/files/downloader"
-	"github.com/neilotoole/sq/libsq/source"
 )
 
 // newXCmd returns the root "x" command, which is the container
@@ -122,71 +115,6 @@ func execXProgress(cmd *cobra.Command, _ []string) error {
 	return ctx.Err()
 }
 
-func newXDownloadCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:    "download URL",
-		Short:  "Download a file",
-		Hidden: true,
-		Args:   cobra.ExactArgs(1),
-		RunE:   execXDownloadCmd,
-		Example: `  $ sq x download https://sq.io/testdata/actor.csv
-
-  # Download a big-ass file
-  $ sq x download https://sqio-public.s3.amazonaws.com/testdata/payment-large.gen.csv
-`,
-	}
-	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
-	return cmd
-}
-
-func execXDownloadCmd(cmd *cobra.Command, args []string) error {
-	ctx := cmd.Context()
-	log := lg.FromContext(ctx)
-	ru := run.FromContext(ctx)
-
-	u, err := url.ParseRequestURI(args[0])
-	if err != nil {
-		return err
-	}
-
-	sum := checksum.Sum([]byte(u.String()))
-	fakeSrc := &source.Source{Handle: "@download_" + sum}
-	cacheDir, err := ru.Files.CacheDirFor(fakeSrc)
-	if err != nil {
-		return err
-	}
-
-	c := httpz.NewClient(
-		httpz.DefaultUserAgent,
-		httpz.OptResponseTimeout(time.Second*15),
-		httpz.OptRequestDelay(time.Second*5),
-	)
-	dl, err := downloader.New(fakeSrc.Handle, c, u.String(), cacheDir)
-	if err != nil {
-		return err
-	}
-
-	h := downloader.NewSinkHandler(log.With("origin", "handler"))
-	if cacheFile := dl.Get(ctx, h.Handler); cacheFile == "" {
-		fmt.Fprintf(ru.Out, "No cache file: %s\n", err)
-	} else {
-		fmt.Fprintf(ru.Out, "Returned cache file: %s\n", cacheFile)
-	}
-
-	switch {
-	case len(h.Errors) > 0:
-		err1 := errz.Err(h.Errors[0])
-		return err1
-	case len(h.Downloaded) > 0:
-		fmt.Fprintf(ru.Out, "Cached: %s\n", h.Downloaded[0])
-		return nil
-	case len(h.Streams) > 0:
-		fmt.Fprintf(ru.Out, "Uncached: %d bytes\n", h.Streams[0].Size())
-	}
-
-	return nil
-}
-
 func pressEnter() <-chan struct{} {
 	done := make(chan struct{})
 	go func() {

cli/cmd_x.go

@@ -40,11 +40,12 @@ var OptShellCompletionLog = options.NewBool(
 	0,
 	false,
 	"Enable logging of shell completion activity",
-	`Enable logging of shell completion activity. It is frequently the case
-that shell completion handlers will trigger work (such as inspecting
-the schema) that doesn't complete by the shell completion timeout. This can result
-in the logs being filled with uninteresting junk when the timeout triggers
-logging of errors due to the cancellation.`,
+	`Enable logging of shell completion activity. This is really only useful
+for debugging shell completion functionality. It's disabled by default,
+because it's frequently the case that shell completion handlers will trigger
+work (such as inspecting the schema) that doesn't complete by the shell
+completion timeout. This can result in the logs being filled with uninteresting
+junk when the timeout triggers logging of errors.`,
 )
 
 // completionFunc is a shell completion function.

cli/complete.go

@@ -71,6 +71,7 @@ var OptConfigLockTimeout = options.NewDuration(
 	0,
 	time.Second*5,
 	"Wait timeout to acquire config lock",
-	`Wait timeout to acquire the config lock. During this period, retry will occur
+	`Wait timeout to acquire the config lock (which prevents multiple sq instances
+stepping on each other's config changes). During this period, retry will occur
 if the lock is already held by another process. If zero, no retry occurs.`,
 )

cli/config/store.go

@@ -74,6 +74,15 @@ const (
 	OutputShort = "o"
 	OutputUsage = "Write output to <file> instead of stdout"
 
+	// Input sets Run.Stdin to the named file. At this time, this is used
+	// mainly for debugging, so it's marked hidden by the CLI. I'm not
+	// sure if this will ever be generally useful. Also, there's been no
+	// testing done to see how this flag would interact with, say,
+	// flag.PasswordPrompt, which also reads from stdin.
+	Input      = "input"
+	InputShort = "i"
+	InputUsage = "Read input from <file> instead of stdin"
+
 	InspectOverview      = "overview"
 	InspectOverviewShort = "O"
 	InspectOverviewUsage = "Show metadata only (no schema)"

cli/flag/flag.go

@@ -219,7 +219,7 @@ func getLogEnabled(ctx context.Context, osArgs []string, cfg *config.Config) boo
 
 	val, ok, err := getBootstrapFlagValue(flag.LogEnabled, "", flag.LogEnabledUsage, osArgs)
 	if err != nil {
-		bootLog.Error("Reading log 'enabled' from flag", lga.Flag, flag.LogEnabled, lga.Err, err)
+		bootLog.Warn("Reading log 'enabled' from flag", lga.Flag, flag.LogEnabled, lga.Err, err)
 	}
 	if ok {
 		bootLog.Debug("Using log 'enabled' specified via flag", lga.Flag, flag.LogEnabled, lga.Val, val)
@@ -277,7 +277,7 @@ func getLogLevel(ctx context.Context, osArgs []string, cfg *config.Config) slog.
 
 	val, ok, err := getBootstrapFlagValue(flag.LogLevel, "", flag.LogLevelUsage, osArgs)
 	if err != nil {
-		bootLog.Error("Reading log level from flag", lga.Flag, flag.LogLevel, lga.Err, err)
+		bootLog.Warn("Reading log level from flag", lga.Flag, flag.LogLevel, lga.Err, err)
 	}
 	if ok {
 		bootLog.Debug("Using log level specified via flag", lga.Flag, flag.LogLevel, lga.Val, val)
@@ -321,7 +321,7 @@ func getLogFormat(ctx context.Context, osArgs []string, cfg *config.Config) form
 
 	val, ok, err := getBootstrapFlagValue(flag.LogFormat, "", flag.LogFormatUsage, osArgs)
 	if err != nil {
-		bootLog.Error("Error reading log format from flag", lga.Flag, flag.LogFormat, lga.Err, err)
+		bootLog.Warn("Error reading log format from flag", lga.Flag, flag.LogFormat, lga.Err, err)
 	}
 	if ok {
 		bootLog.Debug("Using log format specified via flag", lga.Flag, flag.LogFormat, lga.Val, val)
@@ -373,7 +373,7 @@ func getLogFilePath(ctx context.Context, osArgs []string, cfg *config.Config) st
 
 	fp, ok, err := getBootstrapFlagValue(flag.LogFile, "", flag.LogFileUsage, osArgs)
 	if err != nil {
-		bootLog.Error("Reading log file from flag", lga.Flag, flag.LogFile, lga.Err, err)
+		bootLog.Warn("Reading log file from flag", lga.Flag, flag.LogFile, lga.Err, err)
 	}
 	if ok {
 		bootLog.Debug("Log file specified via flag", lga.Flag, flag.LogFile, lga.Path, fp)

cli/logging.go

@@ -18,7 +18,6 @@ import (
 	"github.com/neilotoole/sq/libsq/core/timez"
 	"github.com/neilotoole/sq/libsq/driver"
 	"github.com/neilotoole/sq/libsq/files"
-	"github.com/neilotoole/sq/libsq/files/downloader"
 	"github.com/neilotoole/sq/libsq/source"
 	"github.com/neilotoole/sq/libsq/source/drivertype"
 )
@@ -184,8 +183,8 @@ func RegisterDefaultOpts(reg *options.Registry) {
 		files.OptHTTPRequestTimeout,
 		files.OptHTTPResponseTimeout,
 		files.OptHTTPSInsecureSkipVerify,
-		downloader.OptCache,
-		downloader.OptContinueOnError,
+		files.OptDownloadCache,
+		files.OptDownloadContinueOnError,
 		driver.OptConnMaxOpen,
 		driver.OptConnMaxIdle,
 		driver.OptConnMaxIdleTime,

cli/options.go

@@ -140,6 +140,23 @@ func preRun(cmd *cobra.Command, ru *run.Run) error {
 		ru.Cleanup = cleanup.New()
 	}
 
+	// If the --input=some/file flag is set, then we need
+	// to override ru.Stdin (which is typically stdin) to point
+	// it at the input source file.
+	if cmdFlagChanged(ru.Cmd, flag.Input) {
+		fpath, _ := ru.Cmd.Flags().GetString(flag.Input)
+		fpath, err := filepath.Abs(fpath)
+		if err != nil {
+			return errz.Wrapf(err, "failed to get absolute path for --%s", flag.Input)
+		}
+
+		f, err := os.Open(fpath)
+		if err != nil {
+			return errz.Wrapf(err, "failed to open file specified by flag --%s", flag.Input)
+		}
+		ru.Stdin = f
+	}
+
 	// If the --output=/some/file flag is set, then we need to
 	// override ru.Out (which is typically stdout) to point it at
 	// the output destination file.

cli/run.go

@@ -16,6 +16,7 @@ import (
 	"github.com/neilotoole/sq/testh"
 	"github.com/neilotoole/sq/testh/fixt"
 	"github.com/neilotoole/sq/testh/sakila"
+	"github.com/neilotoole/sq/testh/tu"
 )
 
 // typeTestTableDDLPath is the location of the SQL CREATE statement
@@ -165,6 +166,8 @@ func createTypeTestTbls(th *testh.Helper, src *source.Source, nTimes int, withDa
 // the returned data matches the inserted data, including verifying
 // that NULL is handled correctly.
 func TestDatabaseTypes(t *testing.T) {
+	tu.SkipIssueWindows(t, tu.GH355SQLiteDecimalWin)
+
 	th := testh.New(t)
 	src := th.Source(sakila.SL3)
 	actualTblName := createTypeTestTbls(th, src, 1, true)[0]

drivers/sqlite3/db_type_test.go

@@ -67,7 +67,7 @@ func CopyFile(dst, src string, mkdir bool) error {
 		return errz.Err(err)
 	}
 	defer in.Close()
-	tmp, err := os.CreateTemp(filepath.Dir(dst), "")
+	tmp, err := os.CreateTemp(filepath.Dir(dst), "*_"+filepath.Base(src))
 	if err != nil {
 		return errz.Err(err)
 	}