diff --git a/README.md b/README.md index 6992d9a..827f7d9 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,14 @@ Also, Go is not the de facto language for scientific research, and I think new l This package features the following functionality * ATIS format support +* Prophesee DAT format support (Read only) * Support to N-Caltech and N-MNIST datasets, including saccade stabilization +* Support to N-Cars dataset * Spatio-temporal filtering * Refraction * Additive and degenerative noise generation * Surface of Active Events (SAE) generation +* Basic rendering of event streams and SAE # Installation instructions @@ -83,9 +86,10 @@ type EventCapture struct { } ``` -# Code example +# Code examples -The following code example shows the basic functionality of the event vision library. +## Basic usage on Neuromorphic datasets +The following code example shows the basic functionality of the event vision library using N-Caltech100 dataset. ``` package main @@ -100,7 +104,7 @@ import ( func main() { - //Adjust path to actual location + // Adjust path to actual location reader := neuromorphic.NeuromorphicDataset{ FilePath: "Caltech101/accordion/image_0005.bin", } @@ -123,14 +127,70 @@ func main() { } ``` +## SAE creation and rendering for N-Cars + +The following example reads an entry from N-Cars dataset, builds an additive SAE in map format and renders to an image pointer. +``` +package main + +import ( + "fmt" + "image/png" + "log" + "os" + + "github.com/ffardo/go-event-vision/datasets" + "github.com/ffardo/go-event-vision/datasets/ncars" + "github.com/ffardo/go-event-vision/render" + "github.com/ffardo/go-event-vision/sae" +) + +func main() { + // Adjust path to actual location + reader := ncars.Ncars{ + FilePath: "Prophesee_Dataset_n_cars/n-cars_train/cars/obj_004396_td.dat", + } + + evCap, err := datasets.ReadDataset(reader) + + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Event capture\nWidht=%d\nHeight=%d\nTotal events=%d\n", evCap.Width, evCap.Height, len(evCap.Events)) + + s, err := sae.CreateMap(evCap.Events, "additive") + + // Render SAE to image. Values are automatically normalized. + evImg := render.SaeMap( + s, evCap.Width, evCap.Height, + ) + + // Save resultin image to file + out, err := os.Create("./output.png") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = png.Encode(out, evImg) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + +} + +``` # Roadmap This project is a work in progress and there is no tagged release yet. The following requirements and features are planned * Full test coverage -* Additional formats such as Prophesee -* Additional dataset support such as N-Cars and DDD17 +* Full support to Prophesee DAT format +* Additional dataset support such as DDD17 and N-ImageNet * Feature extraction algorithms such as HATs +* Additional rendering styles for SAE # Additional Information diff --git a/datasets/ncars/ncars.go b/datasets/ncars/ncars.go new file mode 100644 index 0000000..4d94dce --- /dev/null +++ b/datasets/ncars/ncars.go @@ -0,0 +1,25 @@ +package ncars + +import ( + "github.com/ffardo/go-event-vision" + "github.com/ffardo/go-event-vision/format/prophesee" +) + +// Ncars implements DatasetReader interface for N-CARS dataset +type Ncars struct { + FilePath string +} + +// Read event capture for an entry in the dataset +func (n Ncars) Read() (event.EventCapture, error) { + atis := prophesee.Dat{FilePath: n.FilePath} + + return atis.ReadEvents() +} + +// Write capture to a dataset. Should be used only for data augmentation. +func (n Ncars) Write(evCap event.EventCapture) error { + atis := prophesee.Dat{FilePath: n.FilePath} + + return atis.WriteEvents(evCap) +} diff --git a/datasets/neuromorphic/neuromorphic.go b/datasets/neuromorphic/neuromorphic.go index d0b724a..e6f5e88 100644 --- a/datasets/neuromorphic/neuromorphic.go +++ b/datasets/neuromorphic/neuromorphic.go @@ -2,7 +2,7 @@ package neuromorphic import ( "github.com/ffardo/go-event-vision" - "github.com/ffardo/go-event-vision/format" + "github.com/ffardo/go-event-vision/format/atis" ) // NeuromorphicDataset implements DatasetReader interface for N-MNIST and N-Caltech100 datasets @@ -12,14 +12,14 @@ type NeuromorphicDataset struct { // Read event capture for an entry in the dataset func (n NeuromorphicDataset) Read() (event.EventCapture, error) { - atis := format.Atis{FilePath: n.FilePath} + atis := atis.Aer{FilePath: n.FilePath} return atis.ReadEvents() } // Write capture to a dataset. Should be used only for data augmentation. func (n NeuromorphicDataset) Write(evCap event.EventCapture) error { - atis := format.Atis{FilePath: n.FilePath} + atis := atis.Aer{FilePath: n.FilePath} return atis.WriteEvents(evCap) } diff --git a/event.go b/event.go index 38a039b..6b9cbf6 100644 --- a/event.go +++ b/event.go @@ -8,14 +8,14 @@ type Point2D struct { // Event represents a discrete event with coordinates (X,Y), timestamp (Ts) and polarity (P) type Event struct { - Coords Point2D - Ts int - P int + Coords Point2D // event Location + Ts int // timestamp, usually expressed in microsseconds. It might change depending on vendor + P int // polarity (1: Positive event, 0: Negative event) } // EventCapture represents a scene captured with event sensors type EventCapture struct { - Events []Event - Width int - Height int + Events []Event // list of events + Width int // width of the captured scene + Height int // height of the captured scene } diff --git a/format/atis/atis.go b/format/atis/atis.go new file mode 100644 index 0000000..c139429 --- /dev/null +++ b/format/atis/atis.go @@ -0,0 +1,113 @@ +package atis + +import ( + "math" + "os" + + "github.com/ffardo/go-event-vision" +) + +// Aer implements ATIS AER format reading and writing +type Aer struct { + FilePath string +} + +func (a Aer) newEventFromBytes(data []byte) event.Event { + x := int(data[0]) + y := int(data[1]) + ts1 := (int(data[2]) & 127) << 16 + ts2 := int(data[3]) << 8 + ts3 := int(data[4]) + ts := ts1 + ts2 + ts3 + p := (int(data[2]) & 128) >> 7 + + if y == 240 { + ts += int(math.Pow(2, 13)) + } + + return event.Event{ + Coords: event.Point2D{X: x, Y: y}, + P: p, + Ts: ts, + } +} + +func (a Aer) eventToBytes(ev event.Event) []byte { + x := ev.Coords.X + y := ev.Coords.Y + + ts := ev.Ts + + if y == 240 { + ts -= int(math.Pow(2, 13)) + } + + data := make([]byte, 5) + + data[0] = byte(x) + data[1] = byte(y) + + p := ev.P << 7 + data[2] = byte(((ts >> 16) & 127) + p) + data[3] = byte((ts >> 8) & 255) + data[4] = byte(ts & 255) + + return data +} + +// ReadEvents read events in the ATIS AER format from file +func (a Aer) ReadEvents() (event.EventCapture, error) { + f, err := os.Open(a.FilePath) + if err != nil { + return event.EventCapture{}, err + } + + defer f.Close() + + ev := []event.Event{} + + bb := make([]byte, 5) + bc := 5 + + mX, mY := 0, 0 + + for bc == 5 { + bc, err = f.Read(bb) + if bc == 5 && err == nil { + n := a.newEventFromBytes(bb) + if n.Coords.Y == 240 { + continue + } + if n.Coords.X > mX { + mX = n.Coords.X + } + if n.Coords.Y > mY { + mY = n.Coords.Y + } + ev = append(ev, n) + } + } + + return event.EventCapture{ + Events: ev, + Width: mX + 1, + Height: mY + 1, + }, nil +} + +// WriteEvents will write events to file in the ATIS AER format +func (a Aer) WriteEvents(evCap event.EventCapture) error { + f, err := os.Create(a.FilePath) + + defer f.Close() + if err != nil { + return err + } + + for _, ev := range evCap.Events { + + data := a.eventToBytes(ev) + f.Write(data) + } + return nil +} diff --git a/format/format.go b/format/format.go index 4dcec41..e6cbb0d 100644 --- a/format/format.go +++ b/format/format.go @@ -1,112 +1,4 @@ package format -import ( - "math" - "os" - - "github.com/ffardo/go-event-vision" -) - -type Atis struct { - FilePath string -} - -func (a Atis) newEventFromBytes(data []byte) event.Event { - x := int(data[0]) - y := int(data[1]) - ts1 := (int(data[2]) & 127) << 16 - ts2 := int(data[3]) << 8 - ts3 := int(data[4]) - ts := ts1 + ts2 + ts3 - p := (int(data[2]) & 128) >> 7 - - if y == 240 { - ts += int(math.Pow(2, 13)) - } - - return event.Event{ - Coords: event.Point2D{X: x, Y: y}, - P: p, - Ts: ts, - } -} - -func (a Atis) eventToBytes(ev event.Event) []byte { - x := ev.Coords.X - y := ev.Coords.Y - - ts := ev.Ts - - if y == 240 { - ts -= int(math.Pow(2, 13)) - } - - data := make([]byte, 5) - - data[0] = byte(x) - data[1] = byte(y) - - p := ev.P << 7 - data[2] = byte(((ts >> 16) & 127) + p) - data[3] = byte((ts >> 8) & 255) - data[4] = byte(ts & 255) - - return data -} - -// ReadEvents read events in the ATIS AER format from file -func (a Atis) ReadEvents() (event.EventCapture, error) { - f, err := os.Open(a.FilePath) - if err != nil { - return event.EventCapture{}, err - } - - defer f.Close() - - ev := []event.Event{} - - bb := make([]byte, 5) - bc := 5 - - mX, mY := 0, 0 - - for bc == 5 { - bc, err = f.Read(bb) - if bc == 5 && err == nil { - n := a.newEventFromBytes(bb) - if n.Coords.Y == 240 { - continue - } - if n.Coords.X > mX { - mX = n.Coords.X - } - if n.Coords.Y > mY { - mY = n.Coords.Y - } - ev = append(ev, n) - } - } - - return event.EventCapture{ - Events: ev, - Width: mX + 1, - Height: mY + 1, - }, nil -} - -// WriteEvents will write events to file in the ATIS AER format -func (a Atis) WriteEvents(evCap event.EventCapture) error { - f, err := os.Create(a.FilePath) - - defer f.Close() - if err != nil { - return err - } - - for _, ev := range evCap.Events { - - data := a.eventToBytes(ev) - f.Write(data) - } - return nil +type Format interface { } diff --git a/format/prophesee/dat.go b/format/prophesee/dat.go new file mode 100644 index 0000000..dff14a4 --- /dev/null +++ b/format/prophesee/dat.go @@ -0,0 +1,95 @@ +package prophesee + +import ( + "encoding/binary" + "errors" + "os" + + "github.com/ffardo/go-event-vision" +) + +// Dat implements Prophesee RAW DAT format reading and writing +// More information can be found in the official documentation +// https://docs.prophesee.ai/stable/data_formats/file_formats/dat.html +type Dat struct { + FilePath string +} + +func (d Dat) newEventFromBytes(data []byte) event.Event { + + ts := int(binary.LittleEndian.Uint32(data[:4])) + addressValue := int(binary.LittleEndian.Uint32(data[4:])) + + x := int((addressValue & 0x00003FFF)) + y := int((addressValue & 0x0FFFC000) >> 14) + p := int((addressValue & 0x10000000) >> 28) + + return event.Event{Coords: event.Point2D{X: x, Y: y}, P: p, Ts: ts} +} + +// ReadEvents read events in the Prophesee RAW DAT format from file +func (d Dat) ReadEvents() (event.EventCapture, error) { + f, err := os.Open(d.FilePath) + + if err != nil { + return event.EventCapture{}, err + } + + defer f.Close() + + ev := []event.Event{} + bc := 2 + + mX, mY := 0, 0 + evSize := 0 + + evSize, err = d.seekFirstEvent(f) + bc = evSize + + bb := make([]byte, 8) + for bc == evSize { + bc, err = f.Read(bb) + if bc == evSize && err == nil { + + n := d.newEventFromBytes(bb) + if n.Coords.X > mX { + mX = n.Coords.X + } + if n.Coords.Y > mY { + mY = n.Coords.Y + } + ev = append(ev, n) + } + } + + return event.EventCapture{Events: ev, Width: mX + 1, Height: mY + 1}, nil +} + +func (d Dat) seekFirstEvent(f *os.File) (int, error) { + var err error + bh := make([]byte, 2) + bc := 2 + + evSize := 0 + + for bc == 2 { + bc, err = f.Read(bh) + if bh[0] == '%' && bh[1] == ' ' { + bhi := make([]byte, 1) + for bhi[0] != '\n' { + _, err = f.Read(bhi) + } + continue + } + + evSize = int(bh[1]) + break + } + + return evSize, err +} + +// WriteEvents will write events to file in the Prophesee RAW DAT format +func (d Dat) WriteEvents(evCap event.EventCapture) error { + return errors.New("Prophesee.WriteEvents is not implemented") +} diff --git a/render/sae.go b/render/sae.go new file mode 100644 index 0000000..11a16c2 --- /dev/null +++ b/render/sae.go @@ -0,0 +1,35 @@ +package render + +import ( + "image" + "image/color" + + "github.com/ffardo/go-event-vision" +) + +// SaeMap renders normalized SAE data in map format to an image.RGBA pointer +func SaeMap(sae map[event.Point2D]int, width, height int) *image.RGBA { + image := image.NewRGBA(image.Rectangle{image.Pt(0, 0), image.Pt(width, height)}) + + bg := color.RGBA{A: 255} + for i := 0; i < height; i++ { + for j := 0; j < width; j++ { + image.Set(j, i, bg) + } + } + + max := 0 + for _, v := range sae { + if v > max { + max = v + } + } + + maxF := float64(max) + + for pt, v := range sae { + c := uint8((float64(v) / maxF) * 255.0) + image.Set(pt.X, pt.Y, color.RGBA{A: 255, R: c, B: c, G: c}) + } + return image +} diff --git a/render/stream.go b/render/stream.go new file mode 100644 index 0000000..990c28d --- /dev/null +++ b/render/stream.go @@ -0,0 +1,29 @@ +package render + +import ( + "image" + "image/color" + + "github.com/ffardo/go-event-vision" +) + +// Stream renders an event stream to an image.RGBA pointer. +func Stream(events []event.Event, width, height int, background, positive, negative color.RGBA) *image.RGBA { + image := image.NewRGBA(image.Rectangle{image.Pt(0, 0), image.Pt(width, height)}) + + for i := 0; i < height; i++ { + for j := 0; j < width; j++ { + image.Set(j, i, background) + } + } + + for _, e := range events { + if e.P == 1 { + image.Set(e.Coords.X, e.Coords.Y, positive) + } else { + image.Set(e.Coords.X, e.Coords.Y, negative) + } + } + + return image +}