Skip to content

Commit

Permalink
update: using workflow collect rss to tgt dir
Browse files Browse the repository at this point in the history
Signed-off-by: hyponet <[email protected]>
  • Loading branch information
hyponet committed Oct 5, 2023
1 parent c6197c5 commit aaaabf7
Show file tree
Hide file tree
Showing 66 changed files with 14,062 additions and 118 deletions.
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/glebarez/go-sqlite v1.20.3 // indirect
Expand All @@ -67,6 +68,7 @@ require (
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/goccy/go-yaml v1.11.2 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
Expand All @@ -83,6 +85,7 @@ require (
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
Expand All @@ -105,6 +108,7 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
Expand Down
16 changes: 16 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbj
github.com/denisenkom/go-mssqldb v0.12.0 h1:VtrkII767ttSPNRfFekePK3sctr+joXgO58stqQbtUA=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
Expand All @@ -119,6 +121,7 @@ github.com/go-gormigrate/gormigrate/v2 v2.0.2 h1:YV4Lc5yMQX8ahVW0ENPq6sPhrhdkGuk
github.com/go-gormigrate/gormigrate/v2 v2.0.2/go.mod h1:vld36QpBTfTzLealsHsmQQJK5lSwJt6wiORv+oFX8/I=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
Expand All @@ -142,6 +145,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe
github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ=
github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
Expand Down Expand Up @@ -239,10 +244,12 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
Expand Down Expand Up @@ -277,10 +284,13 @@ github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
Expand Down Expand Up @@ -308,6 +318,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
Expand All @@ -332,6 +343,7 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
Expand Down Expand Up @@ -363,6 +375,7 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
Expand Down Expand Up @@ -497,6 +510,7 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -539,6 +553,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
16 changes: 8 additions & 8 deletions pkg/dentry/group.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/basenana/nanafs/pkg/plugin/pluginapi"
"github.com/basenana/nanafs/pkg/types"
"github.com/basenana/nanafs/utils/logger"
"go.uber.org/zap"
"path"
"runtime/trace"
"strings"
Expand Down Expand Up @@ -227,6 +228,7 @@ type extGroup struct {
mgr Manager
stdGroup *stdGroup
mirror plugin.MirrorPlugin
logger *zap.SugaredLogger
}

func (e *extGroup) FindEntry(ctx context.Context, name string) (*types.Metadata, error) {
Expand Down Expand Up @@ -258,25 +260,23 @@ func (e *extGroup) CreateEntry(ctx context.Context, attr EntryAttr) (*types.Meta
}

func (e *extGroup) UpdateEntry(ctx context.Context, entryId int64, patch *types.Metadata) error {
group, err := e.stdGroup.cacheStore.getEntry(ctx, e.stdGroup.entryID)
// query old and write back
entry, err := e.stdGroup.cacheStore.getEntry(ctx, entryId)
if err != nil {
return err
}
mirrorEn, err := e.mirror.FindEntry(ctx, group.Name)

mirrorEn, err := e.mirror.FindEntry(ctx, entry.Name)
if err != nil {
e.logger.Warnw("find entry in mirror plugin failed", "name", entry.Name, "err", err)
return err
}

mirrorEn.Size = patch.Size

// query old and write back
entry, err := e.stdGroup.cacheStore.getEntry(ctx, entryId)
if err != nil {
return err
}

err = e.mirror.UpdateEntry(ctx, mirrorEn)
if err != nil {
e.logger.Warnw("update entry to mirror plugin failed", "name", entry.Name, "err", err)
return err
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/dentry/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,8 @@ func (m *manager) OpenGroup(ctx context.Context, groupId int64) (Group, error) {
if err != nil {
return nil, err
}
grp = &extGroup{mgr: m, stdGroup: stdGrp, mirror: mirror}
grp = &extGroup{mgr: m, stdGroup: stdGrp, mirror: mirror,
logger: logger.NewLogger("extLogger").With(zap.Int64("group", groupId))}
} else {
grp = emptyGroup{}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/dispatch/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func (w workflowAction) handleEvent(ctx context.Context, evt *types.EntryEvent)
}

// trigger workflow
job, err = w.manager.TriggerWorkflow(ctx, wf.Id, evt.RefID, workflow.JobAttr{Reason: fmt.Sprintf("event: %s", evt.Type)})
job, err = w.manager.TriggerWorkflow(ctx, wf.Id, types.WorkflowTarget{EntryID: evt.RefID}, workflow.JobAttr{Reason: fmt.Sprintf("event: %s", evt.Type)})
if err != nil {
w.logger.Errorw("[workflowAction] workflow trigger failed", "entry", evt.RefID, "workflow", wf.Id, "err", err)
continue
Expand Down
2 changes: 2 additions & 0 deletions pkg/metastore/sql.go
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,7 @@ func (s *sqlMetaStore) ListWorkflowJob(ctx context.Context, filter types.JobFilt

func (s *sqlMetaStore) SaveWorkflow(ctx context.Context, wf *types.WorkflowSpec) error {
defer trace.StartRegion(ctx, "metastore.sql.SaveWorkflow").End()
wf.UpdatedAt = time.Now()
err := s.dbEntity.SaveWorkflow(ctx, wf)
if err != nil {
return db.SqlError2Error(err)
Expand All @@ -452,6 +453,7 @@ func (s *sqlMetaStore) SaveWorkflow(ctx context.Context, wf *types.WorkflowSpec)

func (s *sqlMetaStore) SaveWorkflowJob(ctx context.Context, wf *types.WorkflowJob) error {
defer trace.StartRegion(ctx, "metastore.sql.SaveWorkflowJob").End()
wf.UpdatedAt = time.Now()
err := s.dbEntity.SaveWorkflowJob(ctx, wf)
if err != nil {
return db.SqlError2Error(err)
Expand Down
24 changes: 18 additions & 6 deletions pkg/plugin/buildin/rss.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"github.com/hyponet/webpage-packer/packer"
"github.com/mmcdole/gofeed"
"go.uber.org/zap"
"io/ioutil"
"net/url"
"os"
"path"
Expand All @@ -41,6 +40,7 @@ const (

archiveFileTypeUrl = "url"
archiveFileTypeHtml = "html"
archiveFileTypeRawHtml = "rawhtml"
archiveFileTypeWebArchive = "webarchive"
)

Expand Down Expand Up @@ -82,19 +82,23 @@ func (r *RssSourcePlugin) SourceInfo() (string, error) {
}

func (r *RssSourcePlugin) Run(ctx context.Context, request *pluginapi.Request) (*pluginapi.Response, error) {
if request.ParentEntryId <= 0 {
return nil, fmt.Errorf("invalid parent entry id: %d", request.ParentEntryId)
}
source, err := r.rssSources(r.scope.Parameters)
if err != nil {
r.logger.Errorw("get rss source failed", "err", err)
return nil, err
}
source.EntryId = request.EntryId
source.EntryId = request.ParentEntryId

entries, err := r.syncRssSource(ctx, source, request.WorkPath)
if err != nil {
r.logger.Warnw("sync rss failed", "source", source.FeedUrl, "err", err)
return pluginapi.NewFailedResponse(fmt.Sprintf("sync rss failed: %s", err)), nil
}
results := []pluginapi.CollectManifest{{BaseEntry: source.EntryId, NewFiles: entries}}
r.logger.Infow("sync rss finish", "baseEntry", source.EntryId, "entries", len(entries))

return pluginapi.NewResponseWithResult(map[string]any{pluginapi.ResCollectManifests: results}), nil
}
Expand All @@ -114,7 +118,7 @@ func (r *RssSourcePlugin) rssSources(pluginParams map[string]string) (src rssSou

src.FileType = pluginParams["file_type"]
if src.FileType == "" {
src.FileType = archiveFileTypeUrl
src.FileType = archiveFileTypeHtml
}

src.Timeout = 120
Expand Down Expand Up @@ -164,12 +168,20 @@ func (r *RssSourcePlugin) syncRssSource(ctx context.Context, source rssSource, w
buf.WriteString("\n")
buf.WriteString(fmt.Sprintf("URL=%s", item.Link))

err = ioutil.WriteFile(filePath, buf.Bytes(), 0655)
err = os.WriteFile(filePath, buf.Bytes(), 0655)
if err != nil {
return nil, fmt.Errorf("pack to url file failed: %s", err)
}

case archiveFileTypeHtml:
filePath += ".html"
htmlContent := readableHtmlContent(item.Link, item.Title, item.Content)
err = os.WriteFile(filePath, []byte(htmlContent), 0655)
if err != nil {
return nil, fmt.Errorf("pack to html file failed: %s", err)
}

case archiveFileTypeRawHtml:
filePath += ".html"
p := packer.NewHtmlPacker()
err = p.Pack(ctx, packer.Option{
Expand All @@ -180,7 +192,7 @@ func (r *RssSourcePlugin) syncRssSource(ctx context.Context, source rssSource, w
Headers: headers,
})
if err != nil {
return nil, fmt.Errorf("pack to html file failed: %s", err)
return nil, fmt.Errorf("pack to raw html file failed: %s", err)
}

case archiveFileTypeWebArchive:
Expand Down Expand Up @@ -216,7 +228,7 @@ func (r *RssSourcePlugin) syncRssSource(ctx context.Context, source rssSource, w
IsGroup: false,
})
}
return nil, nil
return newEntries, nil
}

func BuildRssSourcePlugin(ctx context.Context, recorder metastore.PluginRecorder, spec types.PluginSpec, scope types.PlugScope) *RssSourcePlugin {
Expand Down
60 changes: 60 additions & 0 deletions pkg/plugin/buildin/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
Copyright 2023 NanaFS Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package buildin

import (
"net/url"
"strings"
)

func readableHtmlContent(urlStr, title, content string) string {
var hostStr string
u, err := url.Parse(urlStr)
if err == nil {
hostStr = u.Host
} else {
hostStr = urlStr
}
patched := strings.ReplaceAll(readableHtmlTpl, "{TITLE}", title)
patched = strings.ReplaceAll(patched, "{HOST}", hostStr)
patched = strings.ReplaceAll(patched, "{URL}", urlStr)
patched = strings.ReplaceAll(patched, "{CONTENT}", content)

return patched
}

const readableHtmlTpl = `
<head>
<title>{TITLE}</title>
<meta charset='UTF-8' />
<meta name='viewport' content='width=device-width, initial-scale=1.0, user-scalable=yes'>
<style type='text/css'>body, table { width: 95%; margin: 0 auto; background-color: #FFF; color:#333; font-family: arial, sans-serif; font-weight: 100; font-size: 12pt; margin:2em 2em 2em 2em; }
p, li { line-height: 150%; }
h1, h2, h3 { color: #333; }
a { color: #3366cc; border-bottom: 1px dotted #3366cc; text-decoration: none; }
a:hover { color: #2647a3; border-bottom-color: color: #66ccff; }
img { max-width: 90%; margin: 0 auto; }
pre { overflow: auto; }
blockquote { color: #888888; padding: 10px; }
figure { width: 100%; margin: 0px; }
figure figcaption { display: none; }
iframe { height: auto; width: auto; max-width: 95%; max-height: 100%; }</style>
<body>
<div> <a href="{URL}" target="_blank">{HOST}</a> <h1>{TITLE}</h1> </div>
{CONTENT}
</body>
`
11 changes: 6 additions & 5 deletions pkg/plugin/pluginapi/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@
package pluginapi

type Request struct {
Action string
WorkPath string
EntryId int64
EntryPath string
Parameter map[string]any
Action string
WorkPath string
EntryId int64
ParentEntryId int64
EntryPath string
Parameter map[string]any
}

func NewRequest() *Request {
Expand Down
2 changes: 1 addition & 1 deletion pkg/types/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ const (
type PlugScope struct {
PluginName string `json:"plugin_name"`
Version string `json:"version"`
PluginType PluginType `json:"plugin_type"`
PluginType PluginType `json:"plugin_type,omitempty"`
Action string `json:"action,omitempty"`
Parameters map[string]string `json:"parameters"`
}
Expand Down
Loading

0 comments on commit aaaabf7

Please sign in to comment.