oci: support --overlay of bare images (sylabs#1699)

OCI-mode: support mounting of squashfs images (read-only) and extfs images (read-write or read-only) with --overlay, alongside the already existing functionality of directory-based overlays.

Squashed merge of:

* prepare data structures for non-dir overlays

* fix e2e test (no more "auto-create overlay dir" functionality)

* switch to using image.Init() to analyze image files

* added caching mechanism to FindBin

* first working version with squashfs support

* refactor funcs into methods of OverlayItem where appropriate

* standardized naming + added lots of comments

* addressing first round of review comments

* refactor: move non-OCI-dep. code to internal/pkg/util/fs/overlay

* initial support for extfs overlay (readonly)

* resurrect testing of write to persistent overlay

* fix bug in writable overlay, introduced in course of refactor

* removed caching mechanism from FindBin

* addressing second round of review comments, except tests

* move calling of prepareWritableOverlay() into Item.Mount()

* removed redundant error-string wrapping

* overlay.Item unit-test

* overlay.Set unit-test

* e2e tests for oci image overlays, misc. testing improvements

CHANGELOG.md

@@ -38,10 +38,11 @@
   `--fakeroot`, for example).
 - The `remote status` command will now print the username, realname, and email
   of the logged-in user, if available.
-- OCI-mode now supports `--overlay <dir>` flag, allowing writes to the
-  filesystem to persist across runs of the OCI container. If specified dir does
-  not exist, Singularity will attempt to create it. Multiple overlays can be
-  specified, but all but one must be read-only (`--overlay <dir>:ro`).
+- OCI-mode now supports an `--overlay <arg>` flag. `<arg>` can be a writable
+  directory, in which case changes to the filesystem will persist across runs of
+  the OCI container. Alternatively, `<arg>` can be `<dir>:ro` or the path of a
+  squashfs or extfs image, to be mounted as a read-only overlay. Multiple
+  overlays can be specified, but all but one must be read-only.
 - The `tap` CNI plugin, new to github.com/containernetworking/plugins v1.3.0,
   is now provided.
 - OCI-mode now supports the `--workdir <workdir>` option. If this option is

e2e/actions/oci.go

@@ -23,6 +23,18 @@ import (
 	"gotest.tools/v3/assert"
 )
 
+const (
+	imgTestFilePath    string = "file-for-testing"
+	squashfsTestString string = "squashfs-test-string"
+	extfsTestString    string = "extfs-test-string"
+)
+
+var (
+	imgsPath        = filepath.Join("..", "internal", "pkg", "util", "fs", "overlay", "testdata")
+	squashfsImgPath = filepath.Join(imgsPath, "squashfs.img")
+	extfsImgPath    = filepath.Join(imgsPath, "extfs.img")
+)
+
 func (c actionTests) actionOciRun(t *testing.T) {
 	e2e.EnsureOCIArchive(t, c.env)
 	e2e.EnsureDockerArchive(t, c.env)
@@ -1018,6 +1030,8 @@ func (c actionTests) actionOciCompat(t *testing.T) {
 }
 
 // actionOciOverlay checks that --overlay functions correctly in OCI mode.
+//
+//nolint:maintidx
 func (c actionTests) actionOciOverlay(t *testing.T) {
 	e2e.EnsureOCIArchive(t, c.env)
 	imageRef := "oci-archive:" + c.env.OCIArchivePath
@@ -1033,6 +1047,20 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
 			}
 		})
 
+		// Create a few read-only overlay subdirs under testDir
+		for i := 0; i < 3; i++ {
+			dirName := fmt.Sprintf("my_rw_ol_dir%d", i)
+			fullPath := filepath.Join(testDir, dirName)
+			if err = os.Mkdir(fullPath, 0o755); err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() {
+				if !t.Failed() {
+					os.RemoveAll(fullPath)
+				}
+			})
+		}
+
 		// Create a few read-only overlay subdirs under testDir
 		for i := 0; i < 3; i++ {
 			dirName := fmt.Sprintf("my_ro_ol_dir%d", i)
@@ -1060,47 +1088,164 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
 		}
 
 		tests := []struct {
-			name        string
-			args        []string
-			exitCode    int
-			wantOutputs []e2e.SingularityCmdResultOp
+			name         string
+			args         []string
+			exitCode     int
+			requiredCmds []string
+			wantOutputs  []e2e.SingularityCmdResultOp
 		}{
 			{
-				name:     "NewWritable",
-				args:     []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir"), imageRef, "sh", "-c", "echo my_test_string > /my_test_file"},
+				name:     "ExistRWDir",
+				args:     []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "sh", "-c", "echo my_test_string > /my_test_file"},
 				exitCode: 0,
 			},
 			{
-				name:     "ExistWritable",
-				args:     []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir"), imageRef, "cat", "/my_test_file"},
+				name:     "ExistRWDirRevisit",
+				args:     []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/my_test_file"},
 				exitCode: 0,
 				wantOutputs: []e2e.SingularityCmdResultOp{
 					e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
 				},
 			},
 			{
-				name:     "NonExistReadonly",
-				args:     []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir_nonexistent:ro"), imageRef, "echo", "hi"},
+				name:     "RWOverlayMissing",
+				args:     []string{"--overlay", filepath.Join(testDir, "something_nonexistent"), imageRef, "echo", "hi"},
+				exitCode: 255,
+			},
+			{
+				name:     "ROOverlayMissing",
+				args:     []string{"--overlay", filepath.Join(testDir, "something_nonexistent:ro"), imageRef, "echo", "hi"},
 				exitCode: 255,
 			},
 			{
-				name:     "ReadonlyAddsTmpfs",
+				name:     "AutoAddTmpfs",
 				args:     []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "sh", "-c", "echo this_should_disappear > /my_test_file"},
 				exitCode: 0,
 			},
 			{
-				name:     "SeveralReadonly",
-				args:     []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "cat", "/testfile.1", "/maskable_testfile"},
+				name: "SeveralRODirs",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"),
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile",
+				},
 				exitCode: 0,
 				wantOutputs: []e2e.SingularityCmdResultOp{
 					e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
 					e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"),
 				},
 			},
+			{
+				name: "AllTypesAtOnce",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", extfsImgPath + ":ro",
+					"--overlay", squashfsImgPath,
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
+				},
+				requiredCmds: []string{"squashfuse", "fuse2fs"},
+				exitCode:     0,
+				wantOutputs: []e2e.SingularityCmdResultOp{
+					e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
+					e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"),
+					e2e.ExpectOutput(e2e.ContainMatch, extfsTestString),
+				},
+			},
+			{
+				name: "SquashfsAndDirs",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", squashfsImgPath,
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
+				},
+				requiredCmds: []string{"squashfuse"},
+				exitCode:     0,
+				wantOutputs: []e2e.SingularityCmdResultOp{
+					e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
+					e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"),
+					e2e.ExpectOutput(e2e.ContainMatch, squashfsTestString),
+				},
+			},
+			{
+				name: "ExtfsAndDirs",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", extfsImgPath + ":ro",
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
+				},
+				requiredCmds: []string{"fuse2fs"},
+				exitCode:     0,
+				wantOutputs: []e2e.SingularityCmdResultOp{
+					e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
+					e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"),
+					e2e.ExpectOutput(e2e.ContainMatch, extfsTestString),
+				},
+			},
+			{
+				name: "SquashfsAndDirsAndMissingRO",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", squashfsImgPath,
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					"--overlay", filepath.Join(testDir, "something_nonexistent:ro"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
+				},
+				requiredCmds: []string{"squashfuse"},
+				exitCode:     255,
+			},
+			{
+				name: "SquashfsAndDirsAndMissingRW",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", squashfsImgPath,
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					"--overlay", filepath.Join(testDir, "something_nonexistent"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
+				},
+				requiredCmds: []string{"squashfuse"},
+				exitCode:     255,
+			},
+			{
+				name: "TwoWritables",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir1"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
+				},
+				requiredCmds: []string{"squashfuse"},
+				exitCode:     255,
+			},
+			{
+				name: "ThreeWritables",
+				args: []string{
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
+					"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir1"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
+					"--overlay", filepath.Join(testDir, "my_rw_ol_dir2"),
+					imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
+				},
+				requiredCmds: []string{"squashfuse"},
+				exitCode:     255,
+			},
 		}
 
 		t.Run(profile.String(), func(t *testing.T) {
 			for _, tt := range tests {
+				if !haveAllCommands(t, tt.requiredCmds) {
+					continue
+				}
+
 				c.env.RunSingularity(
 					t,
 					e2e.AsSubtest(tt.name),
@@ -1170,6 +1315,16 @@ func countLines(path string) (int, error) {
 	return lines, nil
 }
 
+func haveAllCommands(t *testing.T, cmds []string) bool {
+	for _, c := range cmds {
+		if _, err := exec.LookPath(c); err != nil {
+			return false
+		}
+	}
+
+	return true
+}
+
 // Make sure --workdir and --scratch work together nicely even when workdir is a
 // relative path. Test needs to be run in non-parallel mode, because it changes
 // the current working directory of the host.

internal/pkg/runtime/launcher/oci/oci_overlay.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/sylabs/singularity/internal/pkg/util/fs/overlay"
 	"github.com/sylabs/singularity/pkg/ocibundle/tools"
 	"github.com/sylabs/singularity/pkg/sylog"
 	"github.com/sylabs/singularity/pkg/util/singularityconf"
@@ -39,30 +40,28 @@ func WrapWithWritableTmpFs(f func() error, bundleDir string) error {
 // WrapWithOverlays runs a function wrapped with prep / cleanup steps for overlays.
 func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) error {
 	writableOverlayFound := false
-	ovs := tools.OverlaySet{}
+	s := overlay.Set{}
 	for _, p := range overlayPaths {
-		writable := true
-		splitted := strings.SplitN(p, ":", 2)
-		barePath := splitted[0]
-		if len(splitted) > 1 {
-			if splitted[1] == "ro" {
-				writable = false
-			}
+		item, err := overlay.NewItemFromString(p)
+		if err != nil {
+			return err
 		}
 
-		if writable && writableOverlayFound {
-			return fmt.Errorf("you can't specify more than one writable overlay; %#v has already been specified as a writable overlay; use '--overlay %s:ro' instead", ovs.WritableLoc, barePath)
+		item.SetParentDir(bundleDir)
+
+		if item.Writable && writableOverlayFound {
+			return fmt.Errorf("you can't specify more than one writable overlay; %#v has already been specified as a writable overlay; use '--overlay %s:ro' instead", s.WritableOverlay, item.SourcePath)
 		}
-		if writable {
+		if item.Writable {
 			writableOverlayFound = true
-			ovs.WritableLoc = barePath
+			s.WritableOverlay = item
 		} else {
-			ovs.ReadonlyLocs = append(ovs.ReadonlyLocs, barePath)
+			s.ReadonlyOverlays = append(s.ReadonlyOverlays, item)
 		}
 	}
 
 	rootFsDir := tools.RootFs(bundleDir).Path()
-	err := tools.ApplyOverlay(rootFsDir, ovs)
+	err := s.Mount(rootFsDir)
 	if err != nil {
 		return err
 	}
@@ -74,7 +73,7 @@ func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) e
 	}
 
 	// Cleanup actions log errors, but don't return - so we get as much cleanup done as possible.
-	if cleanupErr := tools.UnmountOverlay(rootFsDir, ovs); cleanupErr != nil {
+	if cleanupErr := s.Unmount(rootFsDir); cleanupErr != nil {
 		sylog.Errorf("While unmounting rootfs overlay: %v", cleanupErr)
 	}
 

internal/pkg/util/bin/bin.go

@@ -50,11 +50,16 @@ func FindBin(name string) (path string, err error) {
 	// ldconfig is invoked by nvidia-container-cli, so must be trusted also.
 	case "cryptsetup", "ldconfig", "nvidia-container-cli":
 		return findFromConfigOnly(name)
-	// distro provided squashfuse & fusermount for unpriv SIF mount
+	// distro provided squashfuse and fusermount for unpriv SIF mount and
+	// OCI-mode bare-image overlay
 	case "squashfuse", "fusermount":
 		return findOnPath(name)
+	// fuse2fs for OCI-mode bare-image overlay
+	case "fuse2fs":
+		return findOnPath(name)
+	default:
+		return "", fmt.Errorf("unknown executable name %q", name)
 	}
-	return "", fmt.Errorf("unknown executable name %q", name)
 }
 
 // findOnPath performs a simple search on PATH for the named executable, returning its full path.