Merge pull request #64 from basenana/feature/document

add document manager & collect doc

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.26
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0
 	github.com/aws/smithy-go v1.13.5
-	github.com/basenana/friday v0.0.0-20231009150247-acf6edbeac1e
+	github.com/basenana/friday v0.0.0-20231024123940-1d872b60eefc
 	github.com/bluele/gcache v0.0.2
 	github.com/getsentry/sentry-go v0.22.0
 	github.com/glebarez/sqlite v1.7.0

go.sum

@@ -46,6 +46,8 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuP
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/basenana/friday v0.0.0-20231009150247-acf6edbeac1e h1:m0C19chhzNJlERNfXUzJyTSIk6RGlw+gEm0xz3tZnt8=
 github.com/basenana/friday v0.0.0-20231009150247-acf6edbeac1e/go.mod h1:D4cDBxHlHz77NH9n144Dwznlq7TfYKVEoHAISUCXeBE=
+github.com/basenana/friday v0.0.0-20231024123940-1d872b60eefc h1:LFgzGVIv455NzUiRdvtuxlGBOKrE31TwjYdnTUEl1IY=
+github.com/basenana/friday v0.0.0-20231024123940-1d872b60eefc/go.mod h1:D4cDBxHlHz77NH9n144Dwznlq7TfYKVEoHAISUCXeBE=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=

pkg/controller/controller.go

@@ -18,16 +18,19 @@ package controller
 
 import (
 	"context"
+	"runtime/trace"
+
+	"go.uber.org/zap"
+
 	"github.com/basenana/nanafs/config"
 	"github.com/basenana/nanafs/pkg/dentry"
+	"github.com/basenana/nanafs/pkg/document"
 	"github.com/basenana/nanafs/pkg/events"
 	"github.com/basenana/nanafs/pkg/metastore"
 	"github.com/basenana/nanafs/pkg/notify"
 	"github.com/basenana/nanafs/pkg/types"
 	"github.com/basenana/nanafs/pkg/workflow"
 	"github.com/basenana/nanafs/utils/logger"
-	"go.uber.org/zap"
-	"runtime/trace"
 )
 
 const (
@@ -67,6 +70,7 @@ type controller struct {
 	entry    dentry.Manager
 	notify   *notify.Notify
 	workflow workflow.Manager
+	document document.Manager
 
 	logger *zap.SugaredLogger
 }
@@ -325,8 +329,13 @@ func New(loader config.Loader, meta metastore.Meta) (Controller, error) {
 		return nil, err
 	}
 
+	ctl.document, err = document.NewManager(meta)
+	if err != nil {
+		return nil, err
+	}
+
 	ctl.notify = notify.NewNotify(meta)
-	ctl.workflow, err = workflow.NewManager(ctl.entry, ctl.notify, meta, cfg.Workflow, cfg.FUSE)
+	ctl.workflow, err = workflow.NewManager(ctl.entry, ctl.document, ctl.notify, meta, cfg.Workflow, cfg.FUSE)
 	if err != nil {
 		return nil, err
 	}

pkg/document/manager.go

@@ -0,0 +1,88 @@
+/*
+  Copyright 2023 NanaFS Authors.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+package document
+
+import (
+	"context"
+	"time"
+
+	"github.com/google/uuid"
+	"go.uber.org/zap"
+
+	"github.com/basenana/nanafs/pkg/metastore"
+	"github.com/basenana/nanafs/pkg/types"
+	"github.com/basenana/nanafs/utils/logger"
+)
+
+type Manager interface {
+	ListDocuments(ctx context.Context) ([]*types.Document, error)
+	CreateDocument(ctx context.Context, doc *types.Document) error
+	UpdateDocument(ctx context.Context, doc *types.Document) error
+	GetDocument(ctx context.Context, id string) (*types.Document, error)
+	DeleteDocument(ctx context.Context, id string) error
+}
+
+type manager struct {
+	logger   *zap.SugaredLogger
+	recorder metastore.DocumentRecorder
+}
+
+var _ Manager = &manager{}
+
+func NewManager(recorder metastore.DocumentRecorder) (Manager, error) {
+	docLogger := logger.NewLogger("document")
+	return &manager{
+		logger:   docLogger,
+		recorder: recorder,
+	}, nil
+}
+
+func (m *manager) ListDocuments(ctx context.Context) ([]*types.Document, error) {
+	result, err := m.recorder.ListDocument(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (m *manager) CreateDocument(ctx context.Context, doc *types.Document) error {
+	if doc.ID == "" {
+		doc.ID = uuid.New().String()
+	}
+	doc.CreatedAt = time.Now()
+	doc.ChangedAt = time.Now()
+	if err := m.recorder.SaveDocument(ctx, doc); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *manager) UpdateDocument(ctx context.Context, doc *types.Document) error {
+	doc.ChangedAt = time.Now()
+	if err := m.recorder.SaveDocument(ctx, doc); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *manager) GetDocument(ctx context.Context, id string) (*types.Document, error) {
+	return m.recorder.GetDocument(ctx, id)
+}
+
+func (m *manager) DeleteDocument(ctx context.Context, id string) error {
+	return m.recorder.DeleteDocument(ctx, id)
+}

pkg/document/manager_test.go

@@ -0,0 +1,67 @@
+/*
+  Copyright 2023 NanaFS Authors.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+package document
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/basenana/nanafs/pkg/types"
+)
+
+var _ = Describe("testDocumentManage", func() {
+	var docId string
+	Context("create document", func() {
+		It("create should be succeed", func() {
+			doc := &types.Document{
+				Name: "test_create_doc",
+			}
+			err := docManager.CreateDocument(context.TODO(), doc)
+			Expect(err).Should(BeNil())
+			docId = doc.ID
+		})
+		It("query should be succeed", func() {
+			doc, err := docManager.GetDocument(context.TODO(), docId)
+			Expect(err).Should(BeNil())
+			Expect(doc.Name).Should(Equal("test_create_doc"))
+		})
+	})
+	Context("update document", func() {
+		It("update should be succeed", func() {
+			err := docManager.UpdateDocument(context.TODO(), &types.Document{
+				ID:      docId,
+				Name:    "test_create_doc",
+				Content: "abc",
+			})
+			Expect(err).Should(BeNil())
+			doc, err := docManager.GetDocument(context.TODO(), docId)
+			Expect(err).Should(BeNil())
+			Expect(doc.Content).Should(Equal("abc"))
+		})
+	})
+	Context("delete document", func() {
+		It("delete should be succeed", func() {
+			err := docManager.DeleteDocument(context.TODO(), docId)
+			Expect(err).Should(BeNil())
+			doc, err := docManager.GetDocument(context.TODO(), docId)
+			Expect(err).ShouldNot(BeNil())
+			Expect(doc).Should(BeNil())
+		})
+	})
+})

pkg/document/suite_test.go

@@ -0,0 +1,65 @@
+/*
+ Copyright 2023 NanaFS Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package document
+
+import (
+	"os"
+	"testing"
+
+	"github.com/basenana/nanafs/config"
+	"github.com/basenana/nanafs/pkg/metastore"
+	"github.com/basenana/nanafs/pkg/plugin"
+	"github.com/basenana/nanafs/pkg/storage"
+	"github.com/basenana/nanafs/pkg/types"
+	"github.com/basenana/nanafs/utils/logger"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	docRecorder metastore.DocumentRecorder
+	docManager  Manager
+	root        *types.Metadata
+
+	workdir string
+)
+
+func TestDEntry(t *testing.T) {
+	logger.InitLogger()
+	defer logger.Sync()
+	RegisterFailHandler(Fail)
+
+	var err error
+	workdir, err = os.MkdirTemp(os.TempDir(), "ut-nanafs-dentry-")
+	Expect(err).Should(BeNil())
+	t.Logf("unit test workdir on: %s", workdir)
+	storage.InitLocalCache(config.Config{CacheDir: workdir, CacheSize: 1})
+
+	RunSpecs(t, "DEntry Suite")
+}
+
+var _ = BeforeSuite(func() {
+	memMeta, err := metastore.NewMetaStorage(storage.MemoryStorage, config.Meta{})
+	Expect(err).Should(BeNil())
+	docRecorder = memMeta
+	docManager, _ = NewManager(docRecorder)
+
+	// init plugin
+	err = plugin.Init(&config.Plugin{}, memMeta)
+	Expect(err).Should(BeNil())
+})

pkg/metastore/db/entity.go

@@ -20,12 +20,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/basenana/nanafs/utils/logger"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+
+	"github.com/basenana/nanafs/utils/logger"
+
 	"github.com/google/uuid"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
@@ -729,6 +732,70 @@ func (e *Entity) UpdateNotificationStatus(ctx context.Context, nid, status strin
 	return res.Error
 }
 
+func (e *Entity) SaveDocument(ctx context.Context, doc *types.Document) error {
+	return e.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		dbDoc := &Document{}
+		res := tx.Where("id = ?", doc.ID).First(dbDoc)
+		if res.Error == gorm.ErrRecordNotFound {
+			if err := dbDoc.Update(doc); err != nil {
+				return err
+			}
+			res = tx.Create(dbDoc)
+			return res.Error
+		}
+		if err := dbDoc.Update(doc); err != nil {
+			return err
+		}
+		res = tx.Save(dbDoc)
+		return res.Error
+	})
+}
+
+func (e *Entity) ListDocument(ctx context.Context) ([]*types.Document, error) {
+	docList := make([]Document, 0)
+	res := e.DB.WithContext(ctx).Order("time desc").Find(&docList)
+	if res.Error != nil {
+		return nil, res.Error
+	}
+
+	result := make([]*types.Document, len(docList))
+	for i, doc := range docList {
+		keyWords := strings.Split(doc.KeyWords, ",")
+		result[i] = &types.Document{
+			ID:        doc.ID,
+			Name:      doc.Name,
+			Uri:       doc.Uri,
+			Source:    doc.Source,
+			KeyWords:  keyWords,
+			Content:   doc.Content,
+			Summary:   doc.Summary,
+			CreatedAt: doc.CreatedAt,
+			ChangedAt: doc.ChangedAt,
+		}
+	}
+	return result, nil
+}
+
+func (e *Entity) GetDocument(ctx context.Context, id string) (*types.Document, error) {
+	doc := &Document{}
+	res := e.DB.WithContext(ctx).Where("id = ?", id).First(doc)
+	if res.Error != nil {
+		return nil, res.Error
+	}
+	return doc.ToDocument()
+}
+
+func (e *Entity) DeleteDocument(ctx context.Context, id string) error {
+	return e.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		res := tx.Where("id = ?", id).First(&Document{})
+		if res.Error != nil {
+			return res.Error
+		}
+		res = tx.Where("id = ?", id).Delete(&Document{})
+		return res.Error
+	})
+}
+
 func (e *Entity) SystemInfo(ctx context.Context) (*types.SystemInfo, error) {
 	info := &SystemInfo{}
 	res := e.WithContext(ctx).First(info)

pkg/metastore/db/migrate.go

@@ -69,6 +69,15 @@ func buildMigrations() []*gormigrate.Migration {
 				return nil
 			},
 		},
+		{
+			ID: "2023101400",
+			Migrate: func(db *gorm.DB) error {
+				return db.AutoMigrate(&Document{})
+			},
+			Rollback: func(db *gorm.DB) error {
+				return nil
+			},
+		},
 	}
 }