From 9942852bde9823ae2a6a4d95f0b893ce22ee5814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Tue, 5 Dec 2023 17:34:22 +0100 Subject: [PATCH 1/2] test: add corrupt WARC file This commit adds a corrupted WARC file to reproduce a nil-pointer exception. --- internal/warc/testdata/corrupt.warc.gz | 3 +++ internal/warc/validate_test.go | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 internal/warc/testdata/corrupt.warc.gz diff --git a/internal/warc/testdata/corrupt.warc.gz b/internal/warc/testdata/corrupt.warc.gz new file mode 100644 index 0000000..5dabe60 --- /dev/null +++ b/internal/warc/testdata/corrupt.warc.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c13fd08626a7564c16058fabbaa3785317268e9ee3f40eac94cc9858e51864b +size 55 diff --git a/internal/warc/validate_test.go b/internal/warc/validate_test.go index e6c556b..8229d40 100644 --- a/internal/warc/validate_test.go +++ b/internal/warc/validate_test.go @@ -21,8 +21,9 @@ import ( ) const ( - validWarcFile = "testdata/valid.warc.gz" - invalidWarcFile = "testdata/invalid.warc.gz" + validWarcFile = "testdata/valid.warc.gz" + invalidWarcFile = "testdata/invalid.warc.gz" + corruptedWarcFile = "testdata/corrupt.warc.gz" ) func TestMain(m *testing.M) { @@ -42,6 +43,10 @@ func TestIsValid(t *testing.T) { file: invalidWarcFile, isValid: false, }, + { + file: corruptedWarcFile, + isValid: false, + }, } for _, test := range tests { From a3447c191b906fbf0c150d77b2b42306d9089870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Tue, 5 Dec 2023 17:36:54 +0100 Subject: [PATCH 2/2] fix: nil-pointer exception This commit fixes a nil-pointer exception that occurs if a corrupted WARC-file is validated. The error is that 'wr' is nil if 'wf.Next' returns an error and using 'wr' than triggers the nil-pointer exception. --- internal/warc/validate.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/warc/validate.go b/internal/warc/validate.go index 514e498..70d5b52 100644 --- a/internal/warc/validate.go +++ b/internal/warc/validate.go @@ -38,7 +38,10 @@ func IsValid(file, tmpDir string) (bool, error) { return validation.Valid(), nil } if err != nil { - *validation = append(*validation, err) + if wr != nil { + defer wr.Close() + } + return false, nil } func() { defer wr.Close()