feat: Initial Support for Memory Mapped Files

go.mod

@@ -4,12 +4,19 @@ go 1.22.1
 
 require (
 	github.com/consensys/gnark-crypto v0.12.1
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.0
+	github.com/stretchr/testify v1.8.2
+	golang.org/x/sys v0.9.0
 )
 
 require (
 	github.com/bits-and-blooms/bitset v1.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/sys v0.9.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -3,23 +3,41 @@ github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY
 github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/mmap/block_device.go

@@ -0,0 +1,99 @@
+package mmap
+
+import (
+	"errors"
+	"io"
+	"runtime/debug"
+	"syscall"
+
+	pkgErrors "github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+// BlockDevice represents a mmap block device holding a reference to a file descriptor.
+type BlockDevice struct {
+	FileDescriptor int
+	Data           []byte
+}
+
+// NewBlockDevice creates a BlockDevice from a file
+// descriptor referring either to a regular file or UNIX device node. To
+// speed up reads, a memory map is used.
+func NewBlockDevice(fileDescriptor, sizeBytes int) (*BlockDevice, error) {
+	data, err := unix.Mmap(fileDescriptor, 0, sizeBytes, syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		return nil, pkgErrors.Wrap(err, "failed to memory map block device")
+	}
+
+	return &BlockDevice{
+		FileDescriptor: fileDescriptor,
+		Data:           data,
+	}, nil
+}
+
+// ReadAt reads through the memory map at a given offset.
+func (bd *BlockDevice) ReadAt(p []byte, off int64) (n int, err error) {
+	// Let read actions go through the memory map to prevent system
+	// call overhead for commonly requested objects.
+	if off < 0 {
+		return 0, syscall.EINVAL
+	}
+
+	if off > int64(len(bd.Data)) {
+		return 0, io.EOF
+	}
+	// Install a page fault handler, so that I/O errors against the
+	// memory map (e.g., due to disk failure) don't cause us to
+	// crash.
+	old := debug.SetPanicOnFault(true)
+	defer func() {
+		debug.SetPanicOnFault(old)
+
+		if recover() != nil {
+			err = errors.New("page fault occurred while reading from memory map")
+		}
+	}()
+
+	n = copy(p, bd.Data[off:])
+	if n < len(p) {
+		err = io.EOF
+	}
+
+	return
+}
+
+// WriteAt writes at a given offset.
+func (bd *BlockDevice) WriteAt(p []byte, off int64) (int, error) {
+	// Let write actions go through the file descriptor. Doing so
+	// yields better performance, as writes through a memory map
+	// would trigger a page fault that causes data to be read.
+	//
+	// The pwrite() system call cannot return a size and error at
+	// the same time. If an error occurs after one or more bytes are
+	// written, it returns the size without an error (a "short
+	// write"). As WriteAt() must return an error in those cases, we
+	// must invoke pwrite() repeatedly.
+	//
+	// TODO: Maybe it makes sense to let unaligned writes that would
+	// trigger reads anyway to go through the memory map?
+	nTotal := 0
+
+	for len(p) > 0 {
+		n, err := unix.Pwrite(bd.FileDescriptor, p, off)
+		nTotal += n
+
+		if err != nil {
+			return nTotal, err
+		}
+
+		p = p[n:]
+		off += int64(n)
+	}
+
+	return nTotal, nil
+}
+
+// Sync synchronizes a file's in-core state with storage device.
+func (bd *BlockDevice) Sync() error {
+	return unix.Fsync(bd.FileDescriptor)
+}

pkg/mmap/file.go

@@ -0,0 +1,51 @@
+package mmap
+
+import (
+	pkgErrors "github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+// File represents a memory-mapped file.
+type File struct {
+	BlockDevice     *BlockDevice
+	SectorSizeBytes int
+	SectorCount     int64
+}
+
+// NewFile constructs a new instance of File.
+func NewFile(path string, minimumSizeBytes int) (*File, error) {
+	fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR|unix.O_APPEND, 0666)
+	if err != nil {
+		return nil, pkgErrors.Wrapf(err, "failed to open file %#v", path)
+	}
+
+	// Use the block size returned by fstat() to determine the
+	// sector size and the number of sectors needed to store the
+	// desired amount of space.
+	var stat unix.Stat_t
+	if err := unix.Fstat(fd, &stat); err != nil {
+		return nil, pkgErrors.Wrapf(err, "failed to obtain size of file %#v", path)
+	}
+
+	sectorSizeBytes := int(stat.Blksize)
+	sectorCount := int64((uint64(minimumSizeBytes) + uint64(stat.Blksize) - 1) / uint64(stat.Blksize))
+	sizeBytes := int64(sectorSizeBytes) * sectorCount
+
+	if err := unix.Ftruncate(fd, sizeBytes); err != nil {
+		return nil, pkgErrors.Wrapf(err, "failed to truncate file %#v to %d bytes", path, sizeBytes)
+	}
+
+	bd, err := NewBlockDevice(fd, int(sizeBytes))
+
+	if err != nil {
+		return nil, err
+	} else if err := unix.Close(fd); err != nil {
+		return nil, err
+	}
+
+	return &File{
+		BlockDevice:     bd,
+		SectorSizeBytes: sectorSizeBytes,
+		SectorCount:     sectorCount,
+	}, nil
+}

pkg/test/ir_test.go

@@ -1,4 +1,4 @@
-package testA
+package test
 
 import (
 	"bufio"

pkg/test/mmap_test.go

@@ -0,0 +1,75 @@
+package test
+
+import (
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+
+	"github.com/consensys/go-corset/pkg/mmap"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Ignored_TestNewBlockDeviceFromFile(t *testing.T) {
+	minSizeBytes := 123456
+	blockDevicePath := filepath.Join(t.TempDir(), "test_blockdevice")
+
+	println(blockDevicePath)
+
+	mmapFile, err := mmap.NewFile(blockDevicePath, minSizeBytes)
+	require.NoError(t, err)
+
+	sectorSizeBytes := mmapFile.SectorSizeBytes
+	sectorCount := mmapFile.SectorCount
+	blockDevice := mmapFile.BlockDevice
+	// The sector size should be a power of two, and the number of
+	// sectors should be sufficient to hold the required space.
+	require.LessOrEqual(t, 512, sectorSizeBytes)
+	require.Equal(t, 0, sectorSizeBytes&(sectorSizeBytes-1))
+	require.Equal(t, int64((minSizeBytes+sectorSizeBytes-1)/sectorSizeBytes), sectorCount)
+
+	// The file on disk should have a size that corresponds to the
+	// sector size and count.
+	fileInfo, err := os.Stat(blockDevicePath)
+	require.NoError(t, err)
+	require.Equal(t, int64(sectorSizeBytes)*sectorCount, fileInfo.Size())
+
+	// Test read, write and sync operations.
+	n, err := blockDevice.WriteAt([]byte("Hello"), 12345)
+	require.Equal(t, 5, n)
+	require.NoError(t, err)
+
+	var b [16]byte
+	n, err = blockDevice.ReadAt(b[:], 12340)
+	require.Equal(t, 16, n)
+	require.NoError(t, err)
+	require.Equal(t, []byte("\x00\x00\x00\x00\x00Hello\x00\x00\x00\x00\x00\x00"), b[:])
+
+	require.NoError(t, mmapFile.BlockDevice.Sync())
+
+	// Truncating the file will cause future read access to the
+	// memory map underneath the BlockDevice to raise SIGBUS. This
+	// may also occur in case of actual I/O errors. These page
+	// faults should be caught properly.
+	//
+	// To be able to implement this, ReadAt() temporary enables the
+	// debug.SetPanicOnFault() option. Test that the original value
+	// of this option is restored upon completion.
+	require.NoError(t, os.Truncate(blockDevicePath, 0))
+
+	debug.SetPanicOnFault(false)
+
+	n, err = blockDevice.ReadAt(b[:], 12340)
+	require.NoError(t, err)
+
+	require.False(t, debug.SetPanicOnFault(false))
+	require.Equal(t, 0, n)
+	debug.SetPanicOnFault(true)
+
+	n, err = blockDevice.ReadAt(b[:], 12340)
+
+	require.True(t, debug.SetPanicOnFault(false))
+	require.Equal(t, 0, n)
+	require.Error(t, err, "page fault occurred while reading from memory map")
+}