Skip to content

Commit

Permalink
oci: support for writable extfs img overlay via fuse-overlayfs (sylab…
Browse files Browse the repository at this point in the history 
…s#1740)

* oci: support for writable extfs img overlay via fuse-overlayfs

* added unit- and e2e-tests

* fixed exposure of "upper" & "work" subdir in readonly overlays

* added fix for file-ownership in FUSE-mounted images + e2e test of fix

* fix handling of "upper" in :ro overlay dirs, adjust e2e tests
  • Loading branch information
preminger authored Jun 8, 2023
1 parent 54afd2c commit 836ab96
Show file tree
Hide file tree
Showing 10 changed files with 474 additions and 143 deletions.
11 changes: 6 additions & 5 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,12 @@
`--fakeroot`, for example).
- The `remote status` command will now print the username, realname, and email
of the logged-in user, if available.
- OCI-mode now supports an `--overlay <arg>` flag. `<arg>` can be a writable
directory, in which case changes to the filesystem will persist across runs of
the OCI container. Alternatively, `<arg>` can be `<dir>:ro` or the path of a
squashfs or extfs image, to be mounted as a read-only overlay. Multiple
overlays can be specified, but all but one must be read-only.
- OCI-mode now supports the `--overlay <arg>` flag. `<arg>` can be the path to a
writable directory or writable extfs image, in which case changes to the
filesystem will persist across runs of the OCI container. Alternatively,
`--overlay <arg>:ro` can be used, where `<arg>` is the path to a directory, to
a squashfs image, or to an extfs image, to be mounted as a read-only overlay.
Multiple overlays can be specified, but all but one must be read-only.
- The `tap` CNI plugin, new to github.com/containernetworking/plugins v1.3.0,
is now provided.
- OCI-mode now supports the `--workdir <workdir>` option. If this option is
Expand Down
23 changes: 12 additions & 11 deletions e2e/actions/actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2602,16 +2602,17 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
//
// OCI Runtime Mode
//
"ociRun": c.actionOciRun, // singularity run --oci
"ociExec": c.actionOciExec, // singularity exec --oci
"ociShell": c.actionOciShell, // singularity shell --oci
"ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci
"ociNetwork": c.actionOciNetwork, // singularity exec --oci --net
"ociBinds": c.actionOciBinds, // singularity exec --oci --bind / --mount
"ociCdi": c.actionOciCdi, // singularity exec --oci --cdi
"ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot
"ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat
"ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode
"ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode
"ociRun": c.actionOciRun, // singularity run --oci
"ociExec": c.actionOciExec, // singularity exec --oci
"ociShell": c.actionOciShell, // singularity shell --oci
"ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci
"ociNetwork": c.actionOciNetwork, // singularity exec --oci --net
"ociBinds": c.actionOciBinds, // singularity exec --oci --bind / --mount
"ociCdi": c.actionOciCdi, // singularity exec --oci --cdi
"ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot
"ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat
"ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode
"ociOverlayExtfsPerms": (c.actionOciOverlayExtfsPerms), // permissions in writable extfs overlays mounted with FUSE in OCI mode
"ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode
}
}
218 changes: 192 additions & 26 deletions e2e/actions/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (

cdispecs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
"github.com/sylabs/singularity/e2e/internal/e2e"
"github.com/sylabs/singularity/internal/pkg/test/tool/dirs"
"github.com/sylabs/singularity/internal/pkg/test/tool/require"
"github.com/sylabs/singularity/internal/pkg/util/fs"
"gotest.tools/v3/assert"
)
Expand Down Expand Up @@ -1048,13 +1050,14 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
}
})

// Create a few read-only overlay subdirs under testDir
// Create a few writable overlay subdirs under testDir
for i := 0; i < 3; i++ {
dirName := fmt.Sprintf("my_rw_ol_dir%d", i)
fullPath := filepath.Join(testDir, dirName)
if err = os.Mkdir(fullPath, 0o755); err != nil {
t.Fatal(err)
}
dirs.MkdirOrFatal(t, fullPath, 0o755)
upperPath := filepath.Join(fullPath, "upper")
dirs.MkdirOrFatal(t, upperPath, 0o777)
dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777)
t.Cleanup(func() {
if !t.Failed() {
os.RemoveAll(fullPath)
Expand All @@ -1066,28 +1069,45 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
for i := 0; i < 3; i++ {
dirName := fmt.Sprintf("my_ro_ol_dir%d", i)
fullPath := filepath.Join(testDir, dirName)
if err = os.Mkdir(fullPath, 0o755); err != nil {
t.Fatal(err)
}
dirs.MkdirOrFatal(t, fullPath, 0o755)
upperPath := filepath.Join(fullPath, "upper")
dirs.MkdirOrFatal(t, upperPath, 0o777)
dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777)
t.Cleanup(func() {
if !t.Failed() {
os.RemoveAll(fullPath)
}
})
if err = os.WriteFile(
filepath.Join(fullPath, fmt.Sprintf("testfile.%d", i)),
filepath.Join(upperPath, fmt.Sprintf("testfile.%d", i)),
[]byte(fmt.Sprintf("test_string_%d\n", i)),
0o644); err != nil {
t.Fatal(err)
}
if err = os.WriteFile(
filepath.Join(fullPath, "maskable_testfile"),
filepath.Join(upperPath, "maskable_testfile"),
[]byte(fmt.Sprintf("maskable_string_%d\n", i)),
0o644); err != nil {
t.Fatal(err)
}
}

// Create a copy of the extfs test image to be used for testing readonly
// extfs image overlays
readonlyExtfsImgPath := filepath.Join(testDir, "readonly-extfs.img")
err = fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444)
if err != nil {
t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err)
}

// Create a copy of the extfs test image to be used for testing writable
// extfs image overlays
writableExtfsImgPath := filepath.Join(testDir, "writable-extfs.img")
err = fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755)
if err != nil {
t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err)
}

tests := []struct {
name string
args []string
Expand All @@ -1108,6 +1128,14 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "ExistRWDirRevisitAsRO",
args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0:ro"), imageRef, "cat", "/my_test_file"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "RWOverlayMissing",
args: []string{"--overlay", filepath.Join(testDir, "something_nonexistent"), imageRef, "echo", "hi"},
Expand Down Expand Up @@ -1141,13 +1169,13 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
name: "AllTypesAtOnce",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
"--overlay", extfsImgPath + ":ro",
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", squashfsImgPath,
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse", "fuse2fs"},
requiredCmds: []string{"squashfuse", "fuse2fs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
Expand All @@ -1164,7 +1192,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
requiredCmds: []string{"squashfuse", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
Expand All @@ -1176,12 +1204,12 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
name: "ExtfsAndDirs",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
"--overlay", extfsImgPath + ":ro",
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"fuse2fs"},
requiredCmds: []string{"fuse2fs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
Expand All @@ -1199,7 +1227,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
requiredCmds: []string{"squashfuse", "fusermount"},
exitCode: 255,
},
{
Expand All @@ -1211,7 +1239,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "something_nonexistent"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
requiredCmds: []string{"squashfuse", "fusermount"},
exitCode: 255,
},
{
Expand All @@ -1223,8 +1251,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
exitCode: 255,
exitCode: 255,
},
{
name: "ThreeWritables",
Expand All @@ -1236,8 +1263,75 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir2"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
exitCode: 255,
exitCode: 255,
},
{
name: "WritableExtfs",
args: []string{"--overlay", writableExtfsImgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
},
{
name: "WritableExtfsRevisit",
args: []string{"--overlay", writableExtfsImgPath, imageRef, "cat", "/my_test_file"},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsRevisitAsRO",
args: []string{"--overlay", writableExtfsImgPath + ":ro", imageRef, "cat", "/my_test_file"},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsWithDirs",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"),
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", writableExtfsImgPath,
imageRef, "cat", "/my_test_file",
},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsWithMix",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"),
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", writableExtfsImgPath,
imageRef, "cat", "/my_test_file",
},
exitCode: 0,
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsWithAll",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"),
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", writableExtfsImgPath,
imageRef, "cat", "/my_test_file",
},
exitCode: 0,
requiredCmds: []string{"squashfuse", "fuse2fs", "fuse-overlayfs", "fusermount"},
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
}

Expand All @@ -1263,6 +1357,16 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
}
}

func haveAllCommands(t *testing.T, cmds []string) bool {
for _, c := range cmds {
if _, err := exec.LookPath(c); err != nil {
return false
}
}

return true
}

// actionOciOverlayTeardown checks that OCI-mode overlays are correctly
// unmounted even in root mode (i.e., when user namespaces are not involved).
func (c actionTests) actionOciOverlayTeardown(t *testing.T) {
Expand All @@ -1282,6 +1386,9 @@ func (c actionTests) actionOciOverlayTeardown(t *testing.T) {
}
})

dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "upper"), 0o777)
dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "work"), 0o777)

c.env.RunSingularity(
t,
e2e.WithProfile(e2e.OCIRootProfile),
Expand Down Expand Up @@ -1316,14 +1423,73 @@ func countLines(path string) (int, error) {
return lines, nil
}

func haveAllCommands(t *testing.T, cmds []string) bool {
for _, c := range cmds {
if _, err := exec.LookPath(c); err != nil {
return false
// Check that write permissions are indeed available for writable FUSE-mounted
// extfs image overlays.
func (c actionTests) actionOciOverlayExtfsPerms(t *testing.T) {
require.Command(t, "fuse2fs")
require.Command(t, "fuse-overlayfs")
require.Command(t, "fusermount")

for _, profile := range e2e.OCIProfiles {
// First, create a writable extfs overlay with `singularity overlay create`.
tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "oci_overlay_extfs_perms-", "")
t.Cleanup(func() {
if !t.Failed() {
cleanup(t)
}
})

imgPath := filepath.Join(tmpDir, "extfs-perms-test.img")

c.env.RunSingularity(
t,
e2e.WithProfile(e2e.UserProfile),
e2e.WithCommand("overlay"),
e2e.WithArgs("create", "--size", "64", imgPath),
e2e.ExpectExit(0),
)

// Now test whether we can write to, and subsequently read from, the image
// we created.
e2e.EnsureOCIArchive(t, c.env)
imageRef := "oci-archive:" + c.env.OCIArchivePath

tests := []struct {
name string
args []string
exitCode int
wantOutputs []e2e.SingularityCmdResultOp
}{
{
name: "FirstWrite",
args: []string{"--overlay", imgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"},
exitCode: 0,
},
{
name: "ThenRead",
args: []string{"--overlay", imgPath, imageRef, "cat", "/my_test_file"},
exitCode: 0,
wantOutputs: []e2e.SingularityCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
}
t.Run(profile.String(), func(t *testing.T) {
for _, tt := range tests {
c.env.RunSingularity(
t,
e2e.AsSubtest(tt.name),
e2e.WithProfile(profile),
e2e.WithCommand("exec"),
e2e.WithArgs(tt.args...),
e2e.ExpectExit(
tt.exitCode,
tt.wantOutputs...,
),
)
}
})
}

return true
}

// Make sure --workdir and --scratch work together nicely even when workdir is a
Expand Down
Loading

0 comments on commit 836ab96

Please sign in to comment.