From fb90c1231c0756b6898d10e49b4363df3d73d75b Mon Sep 17 00:00:00 2001 From: Christoph Hartmann Date: Fri, 23 Feb 2024 19:35:25 +0100 Subject: [PATCH] add tests for container connections --- .../connection/container/image_connection.go | 4 +- .../container/image_connection_test.go | 342 +++++++++--------- .../docker/container_connection_test.go | 65 ++++ providers/os/connection/tar/tar_connection.go | 12 +- .../os/resources/processes/docker_test.go | 26 +- 5 files changed, 264 insertions(+), 185 deletions(-) diff --git a/providers/os/connection/container/image_connection.go b/providers/os/connection/container/image_connection.go index f537658c18..68a0fd11a7 100644 --- a/providers/os/connection/container/image_connection.go +++ b/providers/os/connection/container/image_connection.go @@ -61,8 +61,8 @@ func NewContainerRegistryImage(id uint32, conf *inventory.Config, asset *invento if err != nil { return nil, err } - if asset.Connections[0].Options == nil { - asset.Connections[0].Options = map[string]string{} + if conf.Options == nil { + conf.Options = map[string]string{} } conn, err := NewContainerImageConnection(id, conf, asset, img) diff --git a/providers/os/connection/container/image_connection_test.go b/providers/os/connection/container/image_connection_test.go index cb724c3e5d..849e545a44 100644 --- a/providers/os/connection/container/image_connection_test.go +++ b/providers/os/connection/container/image_connection_test.go @@ -22,51 +22,197 @@ import ( ) const ( - alpineImage = "alpine:3.9" + alpineImage = "alpine:3.15" alpineContainerPath = "./alpine-container.tar" centosImage = "centos:7" centosContainerPath = "./centos-container.tar" ) -func TestTarCommand(t *testing.T) { - err := cacheAlpine() - require.NoError(t, err, "should create tar without error") +func cacheImageToTar(source string, filename string) error { + // check if the cache is already there + _, err := os.Stat(filename) + if err == nil { + return nil + } - c, err := container.NewContainerFromTar(0, &inventory.Config{ - Type: "tar", - Options: map[string]string{ - tar.OPTION_FILE: alpineContainerPath, - }, - }, &inventory.Asset{}) - assert.Equal(t, nil, err, "should create tar without error") + tag, err := name.NewTag(source, name.WeakValidation) + if err != nil { + return err + } - cmd, err := c.RunCommand("ls /") - assert.Nil(t, err) - if assert.NotNil(t, cmd) { - assert.Equal(t, nil, err, "should execute without error") - assert.Equal(t, -1, cmd.ExitStatus, "command should not be executed") - stdoutContent, _ := io.ReadAll(cmd.Stdout) - assert.Equal(t, "", string(stdoutContent), "output should be correct") - stderrContent, _ := io.ReadAll(cmd.Stdout) - assert.Equal(t, "", string(stderrContent), "output should be correct") + auth, err := authn.DefaultKeychain.Resolve(tag.Registry) + if err != nil { + return err + } + + img, err := remote.Image(tag, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport)) + if err != nil { + return err } + + return tarball.WriteToFile(filename, tag, img) +} + +func cacheAlpine() error { + return cacheImageToTar(alpineImage, alpineContainerPath) +} + +func cacheCentos() error { + return cacheImageToTar(centosImage, centosContainerPath) +} + +type dockerConnTest struct { + name string + conn *tar.TarConnection + testfile string } -func TestPlatformIdentifier(t *testing.T) { +func TestImageConnections(t *testing.T) { + testConnections := []dockerConnTest{} + + // create a connection to ta downloaded alpine image err := cacheAlpine() require.NoError(t, err, "should create tar without error") - - conn, err := container.NewContainerFromTar(0, &inventory.Config{ + alpineConn, err := container.NewContainerFromTar(0, &inventory.Config{ Type: "tar", Options: map[string]string{ tar.OPTION_FILE: alpineContainerPath, }, }, &inventory.Asset{}) - require.NoError(t, err) - platformId, err := conn.Identifier() - require.NoError(t, err) - assert.True(t, len(platformId) > 0) + testConnections = append(testConnections, dockerConnTest{ + name: "alpine", + conn: alpineConn, + testfile: "/etc/alpine-release", + }) + + // create a connection to ta downloaded centos image + err = cacheCentos() + require.NoError(t, err, "should create tar without error") + centosConn, err := container.NewContainerFromTar(0, &inventory.Config{ + Type: "tar", + Options: map[string]string{ + tar.OPTION_FILE: centosContainerPath, + }, + }, &inventory.Asset{}) + testConnections = append(testConnections, dockerConnTest{ + name: "centos", + conn: centosConn, + testfile: "/etc/centos-release", + }) + + // create a connection to a remote alpine image + alpineRemoteConn, err := container.NewContainerRegistryImage(0, &inventory.Config{ + Type: "docker-image", + Host: alpineImage, + }, &inventory.Asset{}) + require.NoError(t, err, "should create remote connection without error") + testConnections = append(testConnections, dockerConnTest{ + name: "alpine", + conn: alpineRemoteConn, + testfile: "/etc/alpine-release", + }) + + for _, test := range testConnections { + t.Run("Test Connection for "+test.name, func(t *testing.T) { + conn := test.conn + require.NotNil(t, conn) + t.Run("Test Run Command", func(t *testing.T) { + cmd, err := conn.RunCommand("ls /") + assert.Nil(t, err, "should execute without error") + assert.Equal(t, -1, cmd.ExitStatus, "command should not be executed") + stdoutContent, _ := io.ReadAll(cmd.Stdout) + assert.Equal(t, "", string(stdoutContent), "output should be correct") + stderrContent, _ := io.ReadAll(cmd.Stdout) + assert.Equal(t, "", string(stderrContent), "output should be correct") + }) + + t.Run("Test Platform Identifier", func(t *testing.T) { + platformId, err := conn.Identifier() + require.NoError(t, err) + assert.True(t, len(platformId) > 0) + }) + + t.Run("Test File Stat", func(t *testing.T) { + f, err := conn.FileSystem().Open(test.testfile) + assert.Nil(t, err) + assert.Equal(t, nil, err, "should execute without error") + + p := f.Name() + assert.Equal(t, test.testfile, p, "path should be correct") + + stat, err := f.Stat() + assert.True(t, stat.Size() >= 6, "should read file size") + assert.Equal(t, nil, err, "should execute without error") + + content, err := io.ReadAll(f) + assert.Equal(t, nil, err, "should execute without error") + assert.True(t, len(content) >= 6, "should read the full content") + }) + + t.Run("Test File Permissions", func(t *testing.T) { + path := test.testfile + details, err := conn.FileInfo(path) + require.NoError(t, err) + assert.Equal(t, int64(0), details.Uid) + assert.Equal(t, int64(0), details.Gid) + assert.True(t, details.Size >= 0) + assert.Equal(t, false, details.Mode.IsDir()) + assert.Equal(t, true, details.Mode.IsRegular()) + assert.Equal(t, "-rw-r--r--", details.Mode.String()) + assert.True(t, details.Mode.UserReadable()) + assert.True(t, details.Mode.UserWriteable()) + assert.False(t, details.Mode.UserExecutable()) + assert.True(t, details.Mode.GroupReadable()) + assert.False(t, details.Mode.GroupWriteable()) + assert.False(t, details.Mode.GroupExecutable()) + assert.True(t, details.Mode.OtherReadable()) + assert.False(t, details.Mode.OtherWriteable()) + assert.False(t, details.Mode.OtherExecutable()) + assert.False(t, details.Mode.Suid()) + assert.False(t, details.Mode.Sgid()) + assert.False(t, details.Mode.Sticky()) + + path = "/etc" + details, err = conn.FileInfo(path) + require.NoError(t, err) + assert.Equal(t, int64(0), details.Uid) + assert.Equal(t, int64(0), details.Gid) + assert.True(t, details.Size >= 0) + assert.True(t, details.Mode.IsDir()) + assert.False(t, details.Mode.IsRegular()) + assert.Equal(t, "drwxr-xr-x", details.Mode.String()) + assert.True(t, details.Mode.UserReadable()) + assert.True(t, details.Mode.UserWriteable()) + assert.True(t, details.Mode.UserExecutable()) + assert.True(t, details.Mode.GroupReadable()) + assert.False(t, details.Mode.GroupWriteable()) + assert.True(t, details.Mode.GroupExecutable()) + assert.True(t, details.Mode.OtherReadable()) + assert.False(t, details.Mode.OtherWriteable()) + assert.True(t, details.Mode.OtherExecutable()) + assert.False(t, details.Mode.Suid()) + assert.False(t, details.Mode.Sgid()) + assert.False(t, details.Mode.Sticky()) + }) + + t.Run("Test Files Find", func(t *testing.T) { + fs := conn.FileSystem() + fSearch := fs.(*tar.FS) + + if test.testfile == "/etc/alpine-release" { + infos, err := fSearch.Find("/", regexp.MustCompile(`alpine-release`), "file") + require.NoError(t, err) + assert.Equal(t, 1, len(infos)) + } else if test.testfile == "/etc/centos-release" { + infos, err := fSearch.Find("/", regexp.MustCompile(`centos-release`), "file") + require.NoError(t, err) + assert.Equal(t, 6, len(infos)) + } + }) + }) + } + } func TestTarSymlinkFile(t *testing.T) { @@ -131,145 +277,3 @@ func TestTarRelativeSymlinkFileCentos(t *testing.T) { assert.Equal(t, 37, len(content), "should read the full content") } } - -func TestTarFile(t *testing.T) { - err := cacheAlpine() - require.NoError(t, err, "should create tar without error") - - c, err := container.NewContainerFromTar(0, &inventory.Config{ - Type: "tar", - Options: map[string]string{ - tar.OPTION_FILE: alpineContainerPath, - }, - }, &inventory.Asset{}) - assert.Equal(t, nil, err, "should create tar without error") - - f, err := c.FileSystem().Open("/etc/alpine-release") - assert.Nil(t, err) - if assert.NotNil(t, f) { - assert.Equal(t, nil, err, "should execute without error") - - p := f.Name() - assert.Equal(t, "/etc/alpine-release", p, "path should be correct") - - stat, err := f.Stat() - assert.Equal(t, int64(6), stat.Size(), "should read file size") - assert.Equal(t, nil, err, "should execute without error") - - content, err := io.ReadAll(f) - assert.Equal(t, nil, err, "should execute without error") - assert.Equal(t, 6, len(content), "should read the full content") - } -} - -func TestFilePermissions(t *testing.T) { - err := cacheAlpine() - require.NoError(t, err, "should create tar without error") - - c, err := container.NewContainerFromTar(0, &inventory.Config{ - Type: "tar", - Options: map[string]string{ - tar.OPTION_FILE: alpineContainerPath, - }, - }, &inventory.Asset{}) - require.NoError(t, err) - - path := "/etc/alpine-release" - details, err := c.FileInfo(path) - require.NoError(t, err) - assert.Equal(t, int64(0), details.Uid) - assert.Equal(t, int64(0), details.Gid) - assert.True(t, details.Size >= 0) - assert.Equal(t, false, details.Mode.IsDir()) - assert.Equal(t, true, details.Mode.IsRegular()) - assert.Equal(t, "-rw-r--r--", details.Mode.String()) - assert.True(t, details.Mode.UserReadable()) - assert.True(t, details.Mode.UserWriteable()) - assert.False(t, details.Mode.UserExecutable()) - assert.True(t, details.Mode.GroupReadable()) - assert.False(t, details.Mode.GroupWriteable()) - assert.False(t, details.Mode.GroupExecutable()) - assert.True(t, details.Mode.OtherReadable()) - assert.False(t, details.Mode.OtherWriteable()) - assert.False(t, details.Mode.OtherExecutable()) - assert.False(t, details.Mode.Suid()) - assert.False(t, details.Mode.Sgid()) - assert.False(t, details.Mode.Sticky()) - - path = "/etc" - details, err = c.FileInfo(path) - require.NoError(t, err) - assert.Equal(t, int64(0), details.Uid) - assert.Equal(t, int64(0), details.Gid) - assert.True(t, details.Size >= 0) - assert.True(t, details.Mode.IsDir()) - assert.False(t, details.Mode.IsRegular()) - assert.Equal(t, "drwxr-xr-x", details.Mode.String()) - assert.True(t, details.Mode.UserReadable()) - assert.True(t, details.Mode.UserWriteable()) - assert.True(t, details.Mode.UserExecutable()) - assert.True(t, details.Mode.GroupReadable()) - assert.False(t, details.Mode.GroupWriteable()) - assert.True(t, details.Mode.GroupExecutable()) - assert.True(t, details.Mode.OtherReadable()) - assert.False(t, details.Mode.OtherWriteable()) - assert.True(t, details.Mode.OtherExecutable()) - assert.False(t, details.Mode.Suid()) - assert.False(t, details.Mode.Sgid()) - assert.False(t, details.Mode.Sticky()) -} - -func TestTarFileFind(t *testing.T) { - err := cacheAlpine() - require.NoError(t, err, "should create tar without error") - - c, err := container.NewContainerFromTar(0, &inventory.Config{ - Type: "tar", - Options: map[string]string{ - tar.OPTION_FILE: alpineContainerPath, - }, - }, &inventory.Asset{}) - assert.Equal(t, nil, err, "should create tar without error") - - fs := c.FileSystem() - - fSearch := fs.(*tar.FS) - - infos, err := fSearch.Find("/", regexp.MustCompile(`alpine-release`), "file") - require.NoError(t, err) - - assert.Equal(t, 1, len(infos)) -} - -func cacheAlpine() error { - return cacheImageToTar(alpineImage, alpineContainerPath) -} - -func cacheCentos() error { - return cacheImageToTar(centosImage, centosContainerPath) -} - -func cacheImageToTar(source string, filename string) error { - // check if the cache is already there - _, err := os.Stat(filename) - if err == nil { - return nil - } - - tag, err := name.NewTag(source, name.WeakValidation) - if err != nil { - return err - } - - auth, err := authn.DefaultKeychain.Resolve(tag.Registry) - if err != nil { - return err - } - - img, err := remote.Image(tag, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport)) - if err != nil { - return err - } - - return tarball.WriteToFile(filename, tag, img) -} diff --git a/providers/os/connection/docker/container_connection_test.go b/providers/os/connection/docker/container_connection_test.go index e1c7d80618..e22440f3f1 100644 --- a/providers/os/connection/docker/container_connection_test.go +++ b/providers/os/connection/docker/container_connection_test.go @@ -4,8 +4,17 @@ package docker import ( + "context" + "fmt" + "io" + "os" "testing" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/google/uuid" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.mondoo.com/cnquery/v10/providers-sdk/v1/inventory" @@ -41,3 +50,59 @@ func TestAssetNameForRemoteImages(t *testing.T) { assert.Equal(t, "gcr.io/google-containers/busybox@545e6a6310a2", asset.Name) assert.Contains(t, asset.PlatformIds, "//platformid.api.mondoo.app/runtime/docker/images/545e6a6310a27636260920bc07b994a299b6708a1b26910cfefd335fdfb60d2b") } + +// TestDockerContainerConnection creates a new running container and tests the connection +func TestDockerContainerConnection(t *testing.T) { + image := "docker.io/nginx:stable" + ctx := context.Background() + dClient, err := GetDockerClient() + assert.NoError(t, err) + + // If docker is not available, then skip the test. + _, err = dClient.ServerVersion(ctx) + if err != nil { + t.SkipNow() + } + + responseBody, err := dClient.ImagePull(ctx, image, types.ImagePullOptions{}) + defer responseBody.Close() + require.NoError(t, err) + + _, err = io.Copy(os.Stdout, responseBody) + require.NoError(t, err) + + // Make sure the docker image is cleaned up + defer func() { + _, err := dClient.ImageRemove(ctx, image, types.ImageRemoveOptions{}) + require.NoError(t, err, "failed to cleanup pre-pulled docker image") + }() + + cfg := &container.Config{ + AttachStdin: false, + AttachStdout: false, + AttachStderr: false, + StdinOnce: false, + Image: image, + } + + uuid := uuid.New() + created, err := dClient.ContainerCreate(ctx, cfg, &container.HostConfig{}, &network.NetworkingConfig{}, &specs.Platform{}, uuid.String()) + require.NoError(t, err) + + require.NoError(t, dClient.ContainerStart(ctx, created.ID, types.ContainerStartOptions{})) + + // Make sure the container is cleaned up + defer func() { + err := dClient.ContainerRemove(ctx, created.ID, types.ContainerRemoveOptions{Force: true}) + require.NoError(t, err) + }() + + fmt.Println("inject: " + created.ID) + conn, err := NewDockerContainerConnection(0, nil, nil) + assert.NoError(t, err) + + cmd, err := conn.RunCommand("ls /") + require.NoError(t, err) + assert.NotNil(t, cmd) + assert.Equal(t, 0, cmd.ExitStatus) +} diff --git a/providers/os/connection/tar/tar_connection.go b/providers/os/connection/tar/tar_connection.go index 495162c5ca..bd7388e6ae 100644 --- a/providers/os/connection/tar/tar_connection.go +++ b/providers/os/connection/tar/tar_connection.go @@ -35,7 +35,7 @@ type TarConnection struct { fetchFn func() (string, error) fetchOnce sync.Once - Fs *FS + fs *FS closeFN func() // fields are exposed since the tar backend is re-used for the docker backend PlatformKind string @@ -96,7 +96,7 @@ func (p *TarConnection) EnsureLoaded() { func (p *TarConnection) FileSystem() afero.Fs { p.EnsureLoaded() - return p.Fs + return p.fs } func (c *TarConnection) FileInfo(path string) (shared.FileInfoDetails, error) { @@ -142,9 +142,9 @@ func (c *TarConnection) Load(stream io.Reader) error { } path := Abs(h.Name) - c.Fs.FileMap[path] = h + c.fs.FileMap[path] = h } - log.Debug().Int("files", len(c.Fs.FileMap)).Msg("tar> successfully loaded") + log.Debug().Int("files", len(c.fs.FileMap)).Msg("tar> successfully loaded") return nil } @@ -188,6 +188,8 @@ func WithFetchFn(fetchFn func() (string, error)) tarClientOption { } } +func WithCustomFileSystem() {} + // NewTarConnection is opening a tar file and creating a new tar connection. The tar file is expected to be a valid // tar file and contains a flattened file structure. Nested tar files as used in docker images are not supported and // need to be extracted before using this connection. @@ -213,7 +215,7 @@ func NewTarConnection(id uint32, conf *inventory.Config, asset *inventory.Asset, c := &TarConnection{ Connection: plugin.NewConnection(id, asset), asset: asset, - Fs: NewFs(filename), + fs: NewFs(filename), closeFN: params.closeFn, fetchFn: params.fetchFn, PlatformKind: conf.Type, diff --git a/providers/os/resources/processes/docker_test.go b/providers/os/resources/processes/docker_test.go index 762e2c1373..8d40f14414 100644 --- a/providers/os/resources/processes/docker_test.go +++ b/providers/os/resources/processes/docker_test.go @@ -1,13 +1,11 @@ // Copyright (c) Mondoo, Inc. // SPDX-License-Identifier: BUSL-1.1 -//go:build debugtest -// +build debugtest - package processes import ( "context" + "fmt" "io" "os" "testing" @@ -19,13 +17,14 @@ import ( specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.mondoo.com/cnquery/v10/providers/os/connection" + "go.mondoo.com/cnquery/v10/providers-sdk/v1/inventory" + "go.mondoo.com/cnquery/v10/providers/os/connection/docker" ) func TestDockerProcsList(t *testing.T) { image := "docker.io/nginx:stable" ctx := context.Background() - dClient, err := connection.GetDockerClient() + dClient, err := docker.GetDockerClient() assert.NoError(t, err) // If docker is not available, then skip the test. @@ -67,14 +66,23 @@ func TestDockerProcsList(t *testing.T) { require.NoError(t, err) }() - panic("inject: " + created.ID) - provider, err := connection.NewDockerContainerConnection(0, nil, nil) + fmt.Println("inject: " + created.ID) + conn, err := docker.NewDockerContainerConnection(0, &inventory.Config{ + Host: created.ID, + }, &inventory.Asset{ + // for the test we need to set the platform + Platform: &inventory.Platform{ + Name: "debian", + Version: "11", + Family: []string{"debian", "linux"}, + }, + }) assert.NoError(t, err) pMan, err := ResolveManager(conn) assert.NoError(t, err) - procs, err := pMan.List() + proc, err := pMan.Process(1) assert.NoError(t, err) - assert.NotEmpty(t, procs) + assert.NotEmpty(t, proc) }