Merge pull request #6160 from kmuriki/r3.8.2

R3.8.2

Author: kmuriki

.github/workflows/ci.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16.4
+          go-version: 1.16.7
 
       - name: Build Singularity
         run: |
@@ -126,7 +126,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16.4
+          go-version: 1.16.7
 
       - name: Fetch deps
         run: sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup
@@ -148,7 +148,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16.4
+          go-version: 1.16.7
 
       - name: Fetch deps
         run: sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup
@@ -192,7 +192,7 @@ jobs:
         if: env.run_tests
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16.4
+          go-version: 1.16.7
 
       - name: Fetch deps
         if: env.run_tests

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Singularity Changelog
 
+## v3.8.2 - [2021-08-31]
+
+### Bug fixes
+
+  - Fix regression when files `source`d from `%environment` contain `\` escaped
+    shell builtins (fixes issue with `source` of conda profile.d script).
+  - `singularity delete` will use the correct library service when the hostname
+    is specified in the `library://` URI.
+  - `singularity build` will use the correct library service when the hostname
+    is specified in the `library://` URI / definition file.
+  - Call `debootstrap` with correct Debian arch when it is not identical to the
+    value of `runtime.GOARCH`. E.g. `ppc64el -> ppc64le`.
+  - When destination is ommitted in `%files` entry in definition file, ensure
+    globbed files are copied to correct resolved path.
+  - Return an error if `--tokenfile` used for `remote login` to an OCI registry,
+    as this is not supported.
+  - Ensure repeated `remote login` to same URI does not create duplicate entries
+    in `~/.singularity/remote.yaml`.
+  - Properly escape single quotes in Docker `CMD` / `ENTRYPOINT` translation.
+  - Use host uid when choosing unsquashfs flags, to avoid selinux xattr errors
+    with `--fakeroot` on non-EL/Fedora distributions with recent squashfs-tools.
+  - Updated the modified golang-x-crypto module with the latest upstream 
+    version.
+
 ## v3.8.1 - [2021-08-12]
 
 ### Bug Fixes
@@ -13,6 +37,7 @@
   - Prevent garbage collection from closing the container image file
     descriptor.
   - Update to Arch Linux pacman.conf URL and remove file size verification.
+  - Avoid panic when mountinfo line has a blank field.
 
 ## v3.8.0 - [2021-06-15]
 

INSTALL.md

@@ -40,8 +40,8 @@ First, download the Golang archive to `/tmp`, then extract the archive to `/usr/
 _**NOTE:** if you are updating Go from a older version, make sure you remove `/usr/local/go` before
 reinstalling it._
 
-```
-$ export VERSION=1.16.6 OS=linux ARCH=amd64  # change this as you need
+```sh
+$ export VERSION=1.16.7 OS=linux ARCH=amd64  # change this as you need
 
 $ wget -O /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz https://dl.google.com/go/go${VERSION}.${OS}-${ARCH}.tar.gz && \
   sudo tar -C /usr/local -xzf /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz
@@ -87,7 +87,7 @@ $ mkdir -p ${GOPATH}/src/github.com/hpcng && \
 To build a stable version of Singularity, check out a [release tag](https://github.com/hpcng/singularity/tags) before compiling:
 
 ```
-$ git checkout v3.8.1
+$ git checkout v3.8.2
 ```
 
 ## Compiling Singularity
@@ -130,7 +130,7 @@ as shown above.  Then download the latest
 and use it to install the RPM like this: 
 
 ```
-$ export VERSION=3.8.1  # this is the singularity version, change as you need
+$ export VERSION=3.8.2  # this is the singularity version, change as you need
 
 $ wget https://github.com/hpcng/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \
     rpmbuild -tb singularity-${VERSION}.tar.gz && \
@@ -146,7 +146,7 @@ tarball and use it to install Singularity:
 $ cd $GOPATH/src/github.com/hpcng/singularity && \
   ./mconfig && \
   make -C builddir rpm && \
-  sudo rpm -ivh ~/rpmbuild/RPMS/x86_64/singularity-3.8.1*.x86_64.rpm # or whatever version you built
+  sudo rpm -ivh ~/rpmbuild/RPMS/x86_64/singularity-3.8.2*.x86_64.rpm # or whatever version you built
 ```
 
 To build an rpm with an alternative install prefix set RPMPREFIX on the

cmd/internal/cli/delete.go

@@ -97,6 +97,18 @@ var deleteImageCmd = &cobra.Command{
 			sylog.Fatalf("Error parsing library ref: %v", err)
 		}
 
+		if deleteLibraryURI != "" && imageRef.Host != "" {
+			sylog.Fatalf("Conflicting arguments; do not use --library with a library URI containing host name")
+		}
+
+		var libraryURI string
+		if deleteLibraryURI != "" {
+			libraryURI = deleteLibraryURI
+		} else if imageRef.Host != "" {
+			// override libraryURI if ref contains host name
+			libraryURI = "https://" + imageRef.Host
+		}
+
 		r := fmt.Sprintf("%s:%s", imageRef.Path, imageRef.Tags[0])
 
 		if !deleteForce {
@@ -109,7 +121,7 @@ var deleteImageCmd = &cobra.Command{
 			}
 		}
 
-		libraryConfig, err := getLibraryClientConfig(deleteLibraryURI)
+		libraryConfig, err := getLibraryClientConfig(libraryURI)
 		if err != nil {
 			sylog.Fatalf("Error while getting library client config: %v", err)
 		}

cmd/internal/cli/remote.go

@@ -67,7 +67,7 @@ var remoteTokenFileFlag = cmdline.Flag{
 	Value:        &loginTokenFile,
 	DefaultValue: "",
 	Name:         "tokenfile",
-	Usage:        "path to the file holding token",
+	Usage:        "path to the file holding auth token for login (remote endpoints only)",
 }
 
 // --no-login
@@ -86,7 +86,7 @@ var remoteLoginUsernameFlag = cmdline.Flag{
 	DefaultValue: "",
 	Name:         "username",
 	ShortHand:    "u",
-	Usage:        "username to authenticate with (leave it empty for token authentication)",
+	Usage:        "username to authenticate with (required for Docker/OCI registry login)",
 	EnvKeys:      []string{"LOGIN_USERNAME"},
 }
 
@@ -97,7 +97,7 @@ var remoteLoginPasswordFlag = cmdline.Flag{
 	DefaultValue: "",
 	Name:         "password",
 	ShortHand:    "p",
-	Usage:        "password to authenticate with",
+	Usage:        "password / token to authenticate with",
 	EnvKeys:      []string{"LOGIN_PASSWORD"},
 }
 

docs/remote.go

@@ -14,23 +14,43 @@ const (
 	RemoteUse   string = `remote [remote options...]`
 	RemoteShort string = `Manage singularity remote endpoints, keyservers and OCI/Docker registry credentials`
 	RemoteLong  string = `
-  The 'remote' commands allow you to manage Singularity remote endpoints, keyservers
-  and OCI/Docker registry credentials through its subcommands. The remote configuration
-  is stored in $HOME/.singularity/remotes.yaml by default.`
+  The 'remote' command allows you to manage Singularity remote endpoints,
+  standalone keyservers and OCI/Docker registry credentials through its
+  subcommands.
+
+  A 'remote endpoint' is the Sylabs Cloud, a Singularity Enterprise installation,
+  or a compatible group of services. The remote endpoint is a single address,
+  e.g. 'cloud.sylabs.io' through which linked library, builder and keystore
+  sevices will be automatically discovered.
+
+  To configure a remote endpoint you must 'remote add' it. You can 'remote login' if
+  you will be performing actions needing authentication. Switch between
+  configured remote endpoints with the 'remote use' command. The active remote
+  endpoint will be used for remote builds, key operations, and 'library://' pull
+  and push. You can also 'remote logout' from and 'remote remove' an endpoint that
+  is no longer required.
+
+  To configure credentials for OCI registries that should be used when pulling or
+  pushing from/to 'docker://'' or 'oras://' URIs, use the 'remote login' command
+  only. You do not have to 'remote add' OCI registries. To remove credentials
+  'remote logout' with the same URI. You do not need to 'remote remove' OCI
+  credentials.
+
+  The remote configuration is stored in $HOME/.singularity/remotes.yaml by default.`
 	RemoteExample string = `
   All group commands have their own help output:
 
-	$ singularity help remote list
-	$ singularity remote list`
+    $ singularity help remote list
+    $ singularity remote list`
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	// remote add command
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	RemoteAddUse   string = `add [add options...] <remote_name> <remote_URI>`
-	RemoteAddShort string = `Create a new singularity remote endpoint`
+	RemoteAddShort string = `Add a new singularity remote endpoint`
 	RemoteAddLong  string = `
-	The 'remote add' command allows you to create a new remote endpoint to be
-	be used for singularity remote services. Authentication with a newly created
-	endpoint will occur automatically.`
+  The 'remote add' command allows you to add a new remote endpoint to be
+  be used for singularity remote services. Authentication with a newly created
+  endpoint will occur automatically.`
 	RemoteAddExample string = `
   $ singularity remote add SylabsCloud cloud.sylabs.io`
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -57,28 +77,39 @@ const (
 	// remote list command
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	RemoteListUse   string = `list`
-	RemoteListShort string = `List all singularity remote endpoints and services that are configured`
+	RemoteListShort string = `List all singularity remote endpoints, keyservers, and OCI credentials that are configured`
 	RemoteListLong  string = `
-  The 'remote list' command lists all remote endpoints configured for use. If a remote
-  is in use, its name will be encompassed by brackets.`
+  The 'remote list' command lists all remote endpoints, keyservers, and OCI registry
+  credentials configured for use.
+
+  The current remote is indicated by 'YES' in the 'ACTIVE' column and can be changed
+  with the 'remote use' command.`
 	RemoteListExample string = `
   $ singularity remote list`
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	// remote login command
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	RemoteLoginUse   string = `login [login options...] <remote_name|registry_uri>`
-	RemoteLoginShort string = `Log into a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials`
+	RemoteLoginShort string = `Login to a singularity remote endpoint, an OCI/Docker registry or a keyserver using credentials`
 	RemoteLoginLong  string = `
   The 'remote login' command allows you to set credentials for a specific endpoint,
-  an OCI/Docker registry or a keyserver. This command can produce a link directing you to
-  the token service you can use to generate a valid token. If no endpoint or registry is
-  specified, it will try the default remote endpoint (SylabsCloud).`
+  an OCI/Docker registry or a keyserver.
+
+  If no endpoint or registry is specified, the command will login to the currently
+  active remote endpoint. This is cloud.sylabs.io by default.`
 	RemoteLoginExample string = `
   To log in to an endpoint:
   $ singularity remote login SylabsCloud
 
   To login in to a docker/OCI registry:
-  $ singularity remote login --username foo --password bar docker://docker.io`
+  $ singularity remote login --username foo docker://docker.io
+  $ singularity remote login --username foo oras://myregistry.example.com
+
+  Note that many cloud OCI registries use token based authentication. The token
+  should be specified as the password for login. A username is still required. E.g.
+  when using a standard Azure identity and token to login to an ACR registry the
+  username '00000000-0000-0000-0000-000000000000' is required. Consult your provider
+  documentation for detail of their login requirements.`
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	// remote logout command
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -87,7 +118,8 @@ const (
 	RemoteLogoutLong  string = `
   The 'remote logout' command allows you to log out from a singularity specific endpoint,
   an OCI/Docker registry or a keyserver. If no endpoint or service is specified, it will
-  try the default remote endpoint (SylabsCloud).`
+  logout from the current active remote endpoint.
+  `
 	RemoteLogoutExample string = `
   To log out from an endpoint
   $ singularity remote logout SylabsCloud

e2e/delete/delete.go

@@ -23,6 +23,7 @@ func (c ctx) testDeleteCmd(t *testing.T) {
 		args       []string
 		agree      string
 		expectExit int
+		expect     e2e.SingularityCmdResultOp
 	}{
 		{
 			name:       "delete unauthorized arch",
@@ -66,6 +67,13 @@ func (c ctx) testDeleteCmd(t *testing.T) {
 			agree:      "y",
 			expectExit: 255,
 		},
+		{
+			name:       "delete host in uri",
+			args:       []string{"library://library.example.com/test/default/test:v0.0.3"},
+			agree:      "y",
+			expectExit: 255,
+			expect:     e2e.ExpectError(e2e.ContainMatch, "dial tcp: lookup library.example.com: no such host"),
+		},
 	}
 
 	for _, tt := range tests {
@@ -76,7 +84,7 @@ func (c ctx) testDeleteCmd(t *testing.T) {
 			e2e.WithCommand("delete"),
 			e2e.WithArgs(tt.args...),
 			e2e.WithStdin(bytes.NewBufferString(tt.agree)),
-			e2e.ExpectExit(tt.expectExit),
+			e2e.ExpectExit(tt.expectExit, tt.expect),
 		)
 	}
 }

e2e/docker/docker.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2019, Sylabs Inc. All rights reserved.
+// Copyright (c) 2019-2021 Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -397,6 +397,19 @@ func (c ctx) testDockerRegistry(t *testing.T) {
 	}
 }
 
+// https://github.com/sylabs/singularity/issues/233
+func (c ctx) testDockerCMDQuotes(t *testing.T) {
+	c.env.RunSingularity(
+		t,
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("run"),
+		e2e.WithArgs("docker://sylabsio/issue233"),
+		e2e.ExpectExit(0,
+			e2e.ExpectOutput(e2e.ContainMatch, "Test run"),
+		),
+	)
+}
+
 // E2ETests is the main func to trigger the test suite
 func E2ETests(env e2e.TestEnv) testhelper.Tests {
 	c := ctx{
@@ -410,5 +423,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 		"pulls":            c.testDockerPulls,
 		"registry":         c.testDockerRegistry,
 		"whiteout symlink": c.testDockerWhiteoutSymlink,
+		"cmd quotes":       c.testDockerCMDQuotes,
 	}
 }

e2e/env/env.go

@@ -444,8 +444,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 		"environment manipulation": c.singularityEnv,
 		"environment option":       c.singularityEnvOption,
 		"environment file":         c.singularityEnvFile,
-		"issue 5057":               c.issue5057, // https://github.com/sylabs/hpcng/issues/5057
-		"issue 5426":               c.issue5426, // https://github.com/sylabs/hpcng/issues/5426
+		"issue 5057":               c.issue5057, // https://github.com/hpcng/issues/5057
+		"issue 5426":               c.issue5426, // https://github.com/hpcng/issues/5426
 		"issue 43":                 c.issue43,   // https://github.com/sylabs/singularity/issues/43
+		"issue 274":                c.issue274,  // https://github.com/sylabs/singularity/issues/274
 	}
 }