Merge pull request #30 from mandiant/fix/issue-29

swap out yara pattern scanning for binary regexps

.gitignore

@@ -11,7 +11,8 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 .vscode/
-test/
+test/build/
+test/weirdbins/
 GoReSym
 Dockerfile.test
 .idea

README.md

@@ -13,14 +13,12 @@ Refer to https://www.mandiant.com/resources/blog/golang-internals-symbol-recover
 
 You can download pre-built `linux` and `windows` GoReSym binaries from the [Releases tab](https://github.com/mandiant/GoReSym/releases/).
 
-To build from source with a recent Go compiler, install libyara manually [Lib Yara Installation](https://yara.readthedocs.io/en/stable/gettingstarted.html) then invoke the Go compiler:
+To build from source with a recent Go compiler, invoke the Go compiler:
 
 ```
 go build
 ```
 
-If you are cross compiling for other architectures you will need to do some configuration steps to allow `go-yara` to compile successfully for any foreign architectures [Go-Yara Cross Compilation](https://github.com/hillu/go-yara/blob/master/README.cross-building.md). See the build script https://github.com/mandiant/GoReSym/blob/master/build_all.sh for details, only cross compilation for windows on a debian host is supported. If you need to compile for macos, it must be done natively on an apple machine.
-
 Once built invoke GoReSym like this:
 ```
 GoReSym.exe -t -d -p /path/to/input.exe

run_test.sh → build_test_files.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 trap "exit" INT
-sudo rm -rf $(pwd)/test
-versions=("1.19" "1.18" "1.17" "1.16" "1.15" "1.14" "1.13" "1.12" "1.11" "1.10" "1.9" "1.8" "1.7" "1.6" "1.5")
+sudo rm -rf $(pwd)/test/build
+versions=("1.21" "1.20" "1.19" "1.18" "1.17" "1.16" "1.15" "1.14" "1.13" "1.12" "1.11" "1.10" "1.9" "1.8" "1.7" "1.6" "1.5")
 for v in "${versions[@]}"
 do
     GO_TAG=$v
     GO_VER=$(echo "$GO_TAG" | tr -d '.')
-
-    rm Dockerfile.test
+    mkdir -p $(pwd)/test/build/"$ver"/
+
+    rm -f Dockerfile.test
 cat <<EOF >Dockerfile.test
     FROM golang:$GO_TAG-alpine
     ARG ver=$GO_VER
@@ -31,3 +32,4 @@ done
 
 rm Dockerfile.test
 
+unzip $(pwd)/test/weirdbins.zip -d $(pwd)/test/

debug/elf/file.go

@@ -59,6 +59,8 @@ type File struct {
 	closer    io.Closer
 	gnuNeed   []verneed
 	gnuVersym []byte
+
+	dataAfterSectionCache map[uint64][]byte // secVA -> dataAfterSection
 }
 
 // A SectionHeader represents a single ELF section header.
@@ -564,6 +566,7 @@ func NewFile(r io.ReaderAt) (*File, error) {
 		}
 	}
 
+	f.dataAfterSectionCache = make(map[uint64][]byte)
 	return f, nil
 }
 
@@ -686,6 +689,10 @@ func getString(section []byte, start int) (string, bool) {
 }
 
 func (f *File) DataAfterSection(target *Section) []byte {
+	if cached, ok := f.dataAfterSectionCache[uint64(target.Addr)]; ok {
+		return cached
+	}
+
 	data := []byte{}
 	found := false
 	for _, s := range f.Sections {
@@ -704,6 +711,7 @@ func (f *File) DataAfterSection(target *Section) []byte {
 			}
 		}
 	}
+	f.dataAfterSectionCache[uint64(target.Addr)] = data
 	return data
 }
 

debug/macho/file.go

@@ -31,7 +31,8 @@ type File struct {
 	Symtab   *Symtab
 	Dysymtab *Dysymtab
 
-	closer io.Closer
+	closer                io.Closer
+	dataAfterSectionCache map[uint64][]byte // secVA -> dataAfterSection
 }
 
 // A Load represents any Mach-O load command.
@@ -448,6 +449,7 @@ func NewFile(r io.ReaderAt) (*File, error) {
 			s.ReaderAt = s.sr
 		}
 	}
+	f.dataAfterSectionCache = make(map[uint64][]byte)
 	return f, nil
 }
 
@@ -573,6 +575,10 @@ func (f *File) Segment(name string) *Segment {
 }
 
 func (f *File) DataAfterSection(target *Section) []byte {
+	if cached, ok := f.dataAfterSectionCache[target.Addr]; ok {
+		return cached
+	}
+
 	data := []byte{}
 	found := false
 	for _, s := range f.Sections {
@@ -591,6 +597,7 @@ func (f *File) DataAfterSection(target *Section) []byte {
 			}
 		}
 	}
+	f.dataAfterSectionCache[target.Addr] = data
 	return data
 }
 

debug/pe/file.go

@@ -31,7 +31,8 @@ type File struct {
 	COFFSymbols    []COFFSymbol // all COFF symbols (including auxiliary symbol records)
 	StringTable    StringTable
 
-	closer io.Closer
+	closer                io.Closer
+	dataAfterSectionCache map[uint64][]byte // secVA -> dataAfterSection
 }
 
 // Open opens the named file using os.Open and prepares it for use as a PE binary.
@@ -172,6 +173,7 @@ func NewFile(r io.ReaderAt) (*File, error) {
 		}
 	}
 
+	f.dataAfterSectionCache = make(map[uint64][]byte)
 	return f, nil
 }
 
@@ -212,6 +214,10 @@ func (f *File) Section(name string) *Section {
 }
 
 func (f *File) DataAfterSection(target *Section) []byte {
+	if cached, ok := f.dataAfterSectionCache[uint64(target.VirtualAddress)]; ok {
+		return cached
+	}
+
 	data := []byte{}
 	found := false
 	for _, s := range f.Sections {
@@ -230,6 +236,7 @@ func (f *File) DataAfterSection(target *Section) []byte {
 			}
 		}
 	}
+	f.dataAfterSectionCache[uint64(target.VirtualAddress)] = data
 	return data
 }
 

go.mod

@@ -1,13 +1,16 @@
 module github.com/mandiant/GoReSym
 
-go 1.20
+go 1.21
 
 require (
+	github.com/elliotchance/orderedmap v1.4.0
+	github.com/pkg/profile v1.7.0
 	golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff
-	google.golang.org/protobuf v1.28.1
+	rsc.io/binaryregexp v0.2.0
 )
 
 require (
-	github.com/elliotchance/orderedmap v1.4.0
-	github.com/hillu/go-yara/v4 v4.3.2
+	github.com/felixge/fgprof v0.9.3 // indirect
+	github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6 // indirect
+	golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
 )

go.sum

@@ -1,25 +1,36 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
 github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/hillu/go-yara/v4 v4.3.2 h1:HGqUN3ORUduWZbb95RQjut4UzavGDbtt/C6SnGB3Amk=
-github.com/hillu/go-yara/v4 v4.3.2/go.mod h1:AHEs/FXVMQKVVlT6iG9d+q1BRr0gq0WoAWZQaZ0gS7s=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6 h1:ZgoomqkdjGbQ3+qQXCkvYMCDvGDNg2k5JJDjjdTB6jY=
+github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff h1:XmKBi9R6duxOB3lfc72wyrwiOY7X2Jl1wuI+RFOyMDE=
 golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA=
+golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

main_test.go

@@ -1,24 +1,26 @@
-/*Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.*/ 
+/*Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.*/
 package main
 
 import (
 	"errors"
 	"fmt"
 	"os"
 	"testing"
+
+	"github.com/pkg/profile"
 )
 
 var versions = []string{"117", "116", "115", "114", "113", "112", "111", "110", "19", "18", "17", "16", "15"}
 var fileNames = []string{"testproject_lin", "testproject_lin_32", "testproject_lin_stripped", "testproject_lin_stripped_32", "testproject_mac", "testproject_mac_stripped", "testproject_win_32.exe", "testproject_win_stripped_32.exe", "testproject_win_stripped.exe", "testproject_win.exe"}
 
 func TestAllVersions(t *testing.T) {
+	defer profile.Start(profile.ProfilePath(".")).Stop()
+
 	workingDirectory, err := os.Getwd()
 	if err != nil {
 		t.Errorf("Failed to get working directory")
 	}
 
-	fmt.Println(workingDirectory)
-
 	for _, v := range versions {
 		for _, file := range fileNames {
 			versionPath := fmt.Sprintf("%s/%s", v, file)
@@ -80,3 +82,104 @@ func TestAllVersions(t *testing.T) {
 		}
 	}
 }
+
+func TestWeirdBins(t *testing.T) {
+	defer profile.Start(profile.ProfilePath(".")).Stop()
+
+	workingDirectory, err := os.Getwd()
+	if err != nil {
+		t.Errorf("Failed to get working directory")
+	}
+
+	testSymbolRecovery := func(t *testing.T, binaryName string, pclntabVA uint64, moduledataVa uint64, mainVA uint64) {
+		filePath := fmt.Sprintf("%s/test/weirdbins/%s", workingDirectory, binaryName)
+		if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
+			t.Errorf("Test file %s doesn't exist\n", filePath)
+			return
+		}
+
+		data, err := main_impl(filePath, true, true, true, 0, "")
+		if err != nil {
+			t.Errorf("GoReSym failed: %s", err)
+		}
+
+		if data.TabMeta.VA != pclntabVA {
+			t.Errorf("incorrect pclntab VA: %016x", data.TabMeta.VA)
+		}
+
+		if data.ModuleMeta.VA != moduledataVa {
+			t.Errorf("incorrect moduledata VA: %016x", data.ModuleMeta.VA)
+		}
+
+		foundMain := false
+		for _, fn := range data.UserFunctions {
+			if fn.FullName == "main.main" {
+				if fn.Start != mainVA {
+					t.Errorf("main.main has wrong VA: %016x", fn.Start)
+				}
+				foundMain = true
+				break
+			}
+		}
+
+		if !foundMain {
+			t.Errorf("main.main symbol not recovered")
+		}
+	}
+
+	t.Run("bigendian", func(t *testing.T) {
+		testSymbolRecovery(t, "bigendian", 0x1F6500, 0x2A70C0, 0x150c30)
+	})
+
+	t.Run("elf_data_rel_ro_pclntab", func(t *testing.T) {
+		testSymbolRecovery(t, "elf_data_rel_ro_pclntab", 0x412dc0, 0x4be120, 0x17c080)
+	})
+
+	t.Run("fmtisfun_lin", func(t *testing.T) {
+		testSymbolRecovery(t, "fmtisfun_lin", 0x4b1d80, 0x4f9160, 0x47c070)
+	})
+
+	t.Run("fmtisfun_lin_stripped", func(t *testing.T) {
+		testSymbolRecovery(t, "fmtisfun_lin_stripped", 0x4b1ce0, 0x4f9160, 0x47c070)
+	})
+
+	t.Run("fmtisfun_macho", func(t *testing.T) {
+		testSymbolRecovery(t, "fmtisfun_macho", 0x10be128, 0x1109260, 0x10879b0)
+	})
+
+	t.Run("fmtisfun_win", func(t *testing.T) {
+		testSymbolRecovery(t, "fmtisfun_win", 0x4bf940, 0x5082a0, 0x489310)
+	})
+
+	t.Run("fmtisfun_win_stripped", func(t *testing.T) {
+		testSymbolRecovery(t, "fmtisfun_win_stripped", 0x4bf940, 0x5082a0, 0x489310)
+	})
+
+	t.Run("hello", func(t *testing.T) {
+		testSymbolRecovery(t, "hello", 0x4de6e0, 0x544140, 0x499080)
+	})
+
+	t.Run("hello_lin", func(t *testing.T) {
+		testSymbolRecovery(t, "hello_lin", 0x4de6e0, 0x544140, 0x499080)
+	})
+
+	t.Run("hello_stripped_lin", func(t *testing.T) {
+		testSymbolRecovery(t, "hello_stripped_lin", 0x4de5e0, 0x543140, 0x499080)
+	})
+
+	t.Run("windows_rdata_pclntab", func(t *testing.T) {
+		testSymbolRecovery(t, "windows_rdata_pclntab", 0x4ef820, 0x5582c0, 0x4a57a0)
+	})
+
+	t.Run("windows_stripped_rdata_pclntab", func(t *testing.T) {
+		testSymbolRecovery(t, "windows_stripped_rdata_pclntab", 0x4ef820, 0x5582c0, 0x4a57a0)
+	})
+
+	t.Run("GoReSym_garbled", func(t *testing.T) {
+		testSymbolRecovery(t, "GoReSym_garbled", 0x6042c0, 0x71c080, 0x55b800)
+	})
+
+	t.Run("kubectl_macho", func(t *testing.T) {
+		testSymbolRecovery(t, "kubectl_macho", 0x6C6CB20, 0x7F8CB20, 0x5CD9E40)
+	})
+}