diff --git a/go.mod b/go.mod index a4cf984..440af15 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,27 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.30.5 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.33 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.32 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect + github.com/aws/smithy-go v1.20.4 // indirect github.com/beevik/etree v1.4.0 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect diff --git a/go.sum b/go.sum index e3965de..2664f51 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,39 @@ +github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= +github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= +github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU= +github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beevik/etree v1.4.0 h1:oz1UedHRepuY3p4N5OjE0nK1WLCqtzHf25bxplKOHLs= github.com/beevik/etree v1.4.0/go.mod h1:cyWiXwGoasx60gHvtnEh5x8+uIjUVnjWqBvEnhnqKDA= @@ -15,6 +51,9 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopacket/gopacket v1.3.0 h1:MouZCc+ej0vnqzB0WeiaO/6+tGvb+KU7UczxoQ+X0Yc= github.com/gopacket/gopacket v1.3.0/go.mod h1:WnFrU1Xkf5lWKV38uKNR9+yYtppn+ZYzOyNqMeH4oNE= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= @@ -71,6 +110,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/observability/buffer.go b/pkg/observability/buffer.go new file mode 100644 index 0000000..2993330 --- /dev/null +++ b/pkg/observability/buffer.go @@ -0,0 +1,19 @@ +package observability + +import ( + "fmt" + "io" + "time" +) + +func (o *Observability) WriteBufferToStorage() error { + file, err := o.Storage.OpenFileForWriting("data-" + time.Now().Format("2003-01-02T02T15:04:05")) + if err != nil { + return fmt.Errorf("open file for writing error: %s", err) + } + _, err = io.Copy(file, &o.Buffer) + if err != nil { + return fmt.Errorf("file write error: %s", err) + } + return file.Close() +} diff --git a/pkg/observability/decoding.go b/pkg/observability/decoding.go index f166e66..8aaf6db 100644 --- a/pkg/observability/decoding.go +++ b/pkg/observability/decoding.go @@ -1,9 +1,13 @@ package observability import ( + "encoding/binary" "encoding/json" "fmt" "io" + "math" + "reflect" + "strconv" ) func Decode(r io.Reader) ([]FluentBitMessage, error) { @@ -22,11 +26,23 @@ func Decode(r io.Reader) ([]FluentBitMessage, error) { switch m2 := m1[0].(type) { case map[string]interface{}: var fluentBitMessage FluentBitMessage + fluentBitMessage.Data = make(map[string]string) val, ok := m2["date"] if ok { fluentBitMessage.Date = val.(float64) } - fluentBitMessage.Data = m2 + for key, value := range m2 { + switch valueTyped := value.(type) { + case string: + fluentBitMessage.Data[key] = valueTyped + case float64: + fluentBitMessage.Data[key] = strconv.FormatFloat(valueTyped, 'f', -1, 64) + case []byte: + fluentBitMessage.Data[key] = string(valueTyped) + default: + fmt.Printf("no hit on type: %s", reflect.TypeOf(valueTyped)) + } + } result = append(result, fluentBitMessage) default: return result, fmt.Errorf("invalid type: no map found in array") @@ -36,3 +52,36 @@ func Decode(r io.Reader) ([]FluentBitMessage, error) { } return result, nil } + +func decodeMessage(msgs []byte) []FluentBitMessage { + res := []FluentBitMessage{} + recordOffset := 0 + for k := 0; k < len(msgs); k++ { + if k > recordOffset+8 && msgs[k] == 0xff && msgs[k-1] == 0xff { + bits := binary.LittleEndian.Uint64(msgs[recordOffset : recordOffset+8]) + msg := FluentBitMessage{ + Date: math.Float64frombits(bits), + Data: map[string]string{}, + } + isKey := false + key := "" + start := 8 + recordOffset + for kk := 8 + recordOffset; kk < k; kk++ { + if msgs[kk] == 0xff { + if isKey { + isKey = false + msg.Data[key] = string(msgs[recordOffset+start+1 : recordOffset+kk]) + start = kk + 1 + } else { + isKey = true + key = string(msgs[recordOffset+start : recordOffset+kk]) + start = kk + } + } + } + res = append(res, msg) + recordOffset = k + 1 + } + } + return res +} diff --git a/pkg/observability/decoding_test.go b/pkg/observability/decoding_test.go index 152ef7e..6d0c108 100644 --- a/pkg/observability/decoding_test.go +++ b/pkg/observability/decoding_test.go @@ -2,11 +2,12 @@ package observability import ( "bytes" + "fmt" "testing" ) func TestDecoding(t *testing.T) { - data := `[{"date":1720613813.197045,"rand_value":5523152494216581654}]` + data := `[{"date":1720613813.197045,"rand_value":"rand"}]` messages, err := Decode(bytes.NewBuffer([]byte(data))) if err != nil { t.Fatalf("error: %s", err) @@ -21,7 +22,46 @@ func TestDecoding(t *testing.T) { if !ok { t.Fatalf("rand_value key not found") } - if val.(float64) != 5523152494216581654 { - t.Fatalf("wrong data returned") + if string(val) != "rand" { + t.Fatalf("wrong data returned: %s", val) + } +} + +func TestDecodeMsg(t *testing.T) { + msgs := []FluentBitMessage{ + { + Date: 1720613813.197045, + Data: map[string]string{ + "mykey": "this is myvalue", + "second key": "this is my second value", + "third key": "this is my third value", + }, + }, + /*{ + Date: 1720613813.197099, + Data: map[string]string{ + "second data set": "my value", + }, + },*/ + } + encoded := encodeMessage(msgs) + decoded := decodeMessage(encoded) + fmt.Printf("decoded: %+v\n", decoded) + + if len(msgs) != len(decoded) { + t.Fatalf("length doesn't match") + } + for k := range decoded { + if msgs[k].Date != decoded[k].Date { + t.Fatalf("date doesn't match") + } + if len(msgs[k].Data) != len(decoded[k].Data) { + t.Fatalf("length of data doesn't match") + } + for kk := range decoded[k].Data { + if msgs[k].Data[kk] != decoded[k].Data[kk] { + t.Fatalf("key/value pair in data doesn't match: key: %s. Data: %s vs %s", kk, msgs[k].Data[kk], decoded[k].Data[kk]) + } + } } } diff --git a/pkg/observability/encoding.go b/pkg/observability/encoding.go new file mode 100644 index 0000000..c805b8f --- /dev/null +++ b/pkg/observability/encoding.go @@ -0,0 +1,24 @@ +package observability + +import ( + "bytes" + "encoding/binary" + "math" +) + +func encodeMessage(msgs []FluentBitMessage) []byte { + out := bytes.Buffer{} + for _, msg := range msgs { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], math.Float64bits(msg.Date)) + out.Write(buf[:]) + for key, msgData := range msg.Data { + out.Write([]byte(key)) + out.Write([]byte{0xff}) + out.Write([]byte(msgData)) + out.Write([]byte{0xff}) + } + out.Write([]byte{0xff}) + } + return out.Bytes() +} diff --git a/pkg/observability/handlers.go b/pkg/observability/handlers.go index defd71d..2e7b20d 100644 --- a/pkg/observability/handlers.go +++ b/pkg/observability/handlers.go @@ -10,12 +10,21 @@ func (o *Observability) observabilityHandler(w http.ResponseWriter, r *http.Requ } func (o *Observability) ingestionHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusBadRequest) + return + } msgs, err := Decode(r.Body) if err != nil { w.WriteHeader(http.StatusBadRequest) fmt.Printf("error: %s", err) return } - fmt.Printf("Got msgs: %+v\n", msgs) + _, err = o.Buffer.Write(encodeMessage(msgs)) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + fmt.Printf("cannot store message: %s", err) + return + } w.WriteHeader(http.StatusOK) } diff --git a/pkg/observability/handlers_test.go b/pkg/observability/handlers_test.go new file mode 100644 index 0000000..719386d --- /dev/null +++ b/pkg/observability/handlers_test.go @@ -0,0 +1,54 @@ +package observability + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + memorystorage "github.com/in4it/wireguard-server/pkg/storage/memory" +) + +func TestIngestionHandler(t *testing.T) { + storage := &memorystorage.MockMemoryStorage{} + o := &Observability{ + Storage: storage, + } + payload := IncomingData{ + { + "Date": 1720613813.197045, + "log": "this is a string", + }, + } + + payloadBytes, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal error: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/api/observability/ingestion/json", bytes.NewReader(payloadBytes)) + w := httptest.NewRecorder() + o.ingestionHandler(w, req) + res := w.Result() + + if res.StatusCode != http.StatusOK { + t.Fatalf("expected status code OK. Got: %d", res.StatusCode) + } + + err = o.WriteBufferToStorage() + if err != nil { + t.Fatalf("write buffer to storage error: %s", err) + } + dirlist, err := storage.ReadDir("") + if err != nil { + t.Fatalf("read dir error: %s", err) + } + for _, filename := range dirlist { + filenameOut, err := storage.ReadFile(filename) + if err != nil { + t.Fatalf("read file error: %s", err) + } + fmt.Printf("filenameOut: %s", filenameOut) + } +} diff --git a/pkg/observability/new.go b/pkg/observability/new.go index bcf3147..184b68b 100644 --- a/pkg/observability/new.go +++ b/pkg/observability/new.go @@ -6,9 +6,6 @@ func New() *Observability { return &Observability{} } -type Observability struct { -} - type Iface interface { GetRouter() *http.ServeMux } diff --git a/pkg/observability/types.go b/pkg/observability/types.go index 3190430..2b83935 100644 --- a/pkg/observability/types.go +++ b/pkg/observability/types.go @@ -1,6 +1,19 @@ package observability +import ( + "bytes" + + "github.com/in4it/wireguard-server/pkg/storage" +) + +type IncomingData []map[string]any + type FluentBitMessage struct { - Date float64 `json:"date"` - Data map[string]any `json:"data"` + Date float64 `json:"date"` + Data map[string]string `json:"data"` +} + +type Observability struct { + Storage storage.Iface + Buffer bytes.Buffer } diff --git a/pkg/storage/memory/storage.go b/pkg/storage/memory/storage.go index ef5734e..c18b790 100644 --- a/pkg/storage/memory/storage.go +++ b/pkg/storage/memory/storage.go @@ -97,7 +97,9 @@ func (m *MockMemoryStorage) ReadDir(path string) ([]string, error) { } res := []string{} for k := range m.Data { - if strings.HasPrefix(k, path+"/") { + if path == "" { + res = append(res, strings.ReplaceAll(k, path+"/", "")) + } else if strings.HasPrefix(k, path+"/") { res = append(res, strings.ReplaceAll(k, path+"/", "")) } } diff --git a/pkg/storage/s3/list.go b/pkg/storage/s3/list.go new file mode 100644 index 0000000..5cadfdc --- /dev/null +++ b/pkg/storage/s3/list.go @@ -0,0 +1,25 @@ +package s3storage + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func (s *S3Storage) ReadDir(pathname string) ([]string, error) { + objectList, err := s.s3Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucketname), + Prefix: aws.String(s.prefix + "/" + strings.TrimLeft(pathname, "/")), + }) + if err != nil { + return []string{}, fmt.Errorf("list object error: %s", err) + } + res := make([]string, len(objectList.Contents)) + for k, object := range objectList.Contents { + res[k] = *object.Key + } + return res, nil +} diff --git a/pkg/storage/s3/new.go b/pkg/storage/s3/new.go new file mode 100644 index 0000000..7e0c7f7 --- /dev/null +++ b/pkg/storage/s3/new.go @@ -0,0 +1,23 @@ +package s3storage + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func New(bucketname, prefix string) (*S3Storage, error) { + sdkConfig, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return nil, fmt.Errorf("config load error: %s", err) + } + s3Client := s3.NewFromConfig(sdkConfig) + + return &S3Storage{ + bucketname: bucketname, + prefix: prefix, + s3Client: s3Client, + }, nil +} diff --git a/pkg/storage/s3/path.go b/pkg/storage/s3/path.go new file mode 100644 index 0000000..b8db5ce --- /dev/null +++ b/pkg/storage/s3/path.go @@ -0,0 +1,42 @@ +package s3storage + +import ( + "io/fs" + "strings" +) + +func (l *S3Storage) FileExists(filename string) bool { + return false +} + +func (l *S3Storage) ConfigPath(filename string) string { + return CONFIG_PATH + "/" + strings.TrimLeft(filename, "/") +} + +func (s *S3Storage) GetPath() string { + return s.prefix +} + +func (l *S3Storage) EnsurePath(pathname string) error { + return nil +} + +func (l *S3Storage) EnsureOwnership(filename, login string) error { + return nil +} + +func (l *S3Storage) Remove(name string) error { + return nil +} + +func (l *S3Storage) Rename(oldName, newName string) error { + return nil +} + +func (l *S3Storage) EnsurePermissions(name string, mode fs.FileMode) error { + return nil +} + +func (l *S3Storage) FileInfo(name string) (fs.FileInfo, error) { + return nil, nil +} diff --git a/pkg/storage/s3/read.go b/pkg/storage/s3/read.go new file mode 100644 index 0000000..91b0ffe --- /dev/null +++ b/pkg/storage/s3/read.go @@ -0,0 +1,17 @@ +package s3storage + +import ( + "io" +) + +func (l *S3Storage) ReadFile(name string) ([]byte, error) { + return nil, nil +} + +func (l *S3Storage) OpenFilesFromPos(names []string, pos int64) ([]io.ReadCloser, error) { + return nil, nil +} + +func (l *S3Storage) OpenFile(name string) (io.ReadCloser, error) { + return nil, nil +} diff --git a/pkg/storage/s3/types.go b/pkg/storage/s3/types.go new file mode 100644 index 0000000..3ff10eb --- /dev/null +++ b/pkg/storage/s3/types.go @@ -0,0 +1,11 @@ +package s3storage + +import "github.com/aws/aws-sdk-go-v2/service/s3" + +const CONFIG_PATH = "config" + +type S3Storage struct { + bucketname string + prefix string + s3Client *s3.Client +} diff --git a/pkg/storage/s3/write.go b/pkg/storage/s3/write.go new file mode 100644 index 0000000..815b819 --- /dev/null +++ b/pkg/storage/s3/write.go @@ -0,0 +1,35 @@ +package s3storage + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func (s *S3Storage) WriteFile(name string, data []byte) error { + _, err := s.s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(s.bucketname), + Key: aws.String(name), + Body: bytes.NewReader(data), + }) + if err != nil { + return fmt.Errorf("put object error: %s", err) + } + return nil +} + +func (s *S3Storage) AppendFile(name string, data []byte) error { + return nil +} + +func (s *S3Storage) OpenFileForWriting(name string) (io.WriteCloser, error) { + return nil, nil +} + +func (s *S3Storage) OpenFileForAppending(name string) (io.WriteCloser, error) { + return nil, nil +}