From fb1276740f979c76bf4207cbfe1c2915e5f974d5 Mon Sep 17 00:00:00 2001 From: azimut Date: Sun, 31 Mar 2024 21:10:41 -0300 Subject: [PATCH] backend: refactor --- backend/Makefile | 4 +- backend/src/db.go | 142 +++++++++++++++++++++++++ backend/src/entries.go | 29 ++++++ backend/src/{feed.go => feeds.go} | 168 +----------------------------- backend/src/main.go | 9 +- 5 files changed, 183 insertions(+), 169 deletions(-) create mode 100644 backend/src/db.go create mode 100644 backend/src/entries.go rename backend/src/{feed.go => feeds.go} (52%) diff --git a/backend/Makefile b/backend/Makefile index 04c75a1..a1550c6 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -1,2 +1,4 @@ -feeds.db: src/main.go src/feed.go src/file.go feeds.json +SRCS := src/main.go src/feeds.go src/db.go src/entries.go src/file.go + +feeds.db: $(SRCS) feeds.json go run -tags "sqlite_fts5 sqlite_foreign_keys" ./... diff --git a/backend/src/db.go b/backend/src/db.go new file mode 100644 index 0000000..e54dea5 --- /dev/null +++ b/backend/src/db.go @@ -0,0 +1,142 @@ +package main + +import ( + "database/sql" + "os" + + _ "github.com/mattn/go-sqlite3" +) + +func initDb() (*sql.DB, error) { + os.Remove(DB_NAME) + + db, err := sql.Open("sqlite3", DB_NAME) + if err != nil { + return nil, err + } + + initStmt := ` + pragma journal_mode = delete; + pragma page_size = 1024; + + create table feeds ( + id integer not null primary key, + title text, + url text, + description text + ); + + create table entries ( + id integer not null primary key, + feedid integer not null, + datemillis integer not null, + title text, + content text, + url text, + foreign key(feedid) references feeds(id) + ); + create index entriesindex on entries(feedid); + + create table entries_content ( + entriesid integer not null, + content text, + foreign key(entriesid) references entries(id) + ); + create index entriescindex on entries_content(entriesid); + + create virtual table search using fts5( + entriesid unindexed, + content + ); + ` + _, err = db.Exec(initStmt) + if err != nil { + return nil, err + } + + return db, nil +} + +func insertFeedsAndEntries(db *sql.DB, feeds Feeds) error { + tx, err := db.Begin() + if err != nil { + return err + } + stmt_feeds, err := tx.Prepare("insert into feeds(id,title,url,description) values(?,?,?,?)") + if err != nil { + return err + } + defer stmt_feeds.Close() + stmt_entry, err := tx.Prepare( + "insert into entries(feedid,datemillis,title,url) values(?,?,?,?)", + ) + if err != nil { + return err + } + defer stmt_entry.Close() + stmt_entry_content, err := tx.Prepare( + "insert into entries_content(entriesid,content) values(?,?)", + ) + if err != nil { + return err + } + defer stmt_entry_content.Close() + for feedid, feed := range feeds { + _, err = stmt_feeds.Exec(feedid, feed.Title, feed.Url, feed.Description) + if err != nil { + return err + } + for _, entry := range feed.Entries { + // entries + res, err := stmt_entry.Exec( + feedid, + entry.Date.UnixMilli(), + entry.Title, + entry.Url, + ) + if err != nil { + return err + } + // entries_content + entryid, err := res.LastInsertId() + if err != nil { + return err + } + _, err = stmt_entry_content.Exec( + entryid, + entry.Content, + ) + if err != nil { + return err + } + } + } + err = tx.Commit() + if err != nil { + return err + } + + err = insertSearch(db) + if err != nil { + return err + } + + return nil +} + +// insertSearch populates `search` table. +// Assumes there are already `entries_content` on the db. +func insertSearch(db *sql.DB) error { + sqlStmt := ` + insert into search + select entriesid,content + from entries_content; + insert into search(search) values('optimize'); + vacuum; + ` + _, err := db.Exec(sqlStmt) + if err != nil { + return err + } + return nil +} diff --git a/backend/src/entries.go b/backend/src/entries.go new file mode 100644 index 0000000..6b9bf76 --- /dev/null +++ b/backend/src/entries.go @@ -0,0 +1,29 @@ +package main + +import "time" + +type Entry struct { + Date time.Time + HumanDate string + MachineDate string + Title string + Url string + Description string + Content string +} + +type Entries []Entry + +func (a Entries) Len() int { + return len(a) +} + +func (a Entries) Less(i, j int) bool { + iDate := a[i].Date + jDate := a[j].Date + return iDate.After(jDate) +} + +func (a Entries) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} diff --git a/backend/src/feed.go b/backend/src/feeds.go similarity index 52% rename from backend/src/feed.go rename to backend/src/feeds.go index b9319da..bed94d9 100644 --- a/backend/src/feed.go +++ b/backend/src/feeds.go @@ -1,17 +1,13 @@ package main import ( - "database/sql" - "os" "sort" "strings" - "time" md "github.com/JohannesKaufmann/html-to-markdown" "github.com/adrg/strutil" "github.com/adrg/strutil/metrics" "github.com/dustin/go-humanize" - _ "github.com/mattn/go-sqlite3" "github.com/mmcdole/gofeed" ) @@ -28,16 +24,6 @@ type Feed struct { Url string `json:"url"` } -type Entry struct { - Date time.Time - HumanDate string - MachineDate string - Title string - Url string - Description string - Content string -} - type Feeds []Feed func (a Feeds) Less(i, j int) bool { @@ -60,22 +46,6 @@ func (a Feeds) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -type Entries []Entry - -func (a Entries) Len() int { - return len(a) -} - -func (a Entries) Less(i, j int) bool { - iDate := a[i].Date - jDate := a[j].Date - return iDate.After(jDate) -} - -func (a Entries) Swap(i, j int) { - a[i], a[j] = a[j], a[i] -} - func (feeds Feeds) Sort() { for i := 0; i < len(feeds); i++ { sort.Sort(feeds[i].Entries) @@ -83,154 +53,20 @@ func (feeds Feeds) Sort() { sort.Sort(feeds) } -func initDb() (*sql.DB, error) { - os.Remove(DB_NAME) - - db, err := sql.Open("sqlite3", DB_NAME) - if err != nil { - return nil, err - } - - initStmt := ` - pragma journal_mode = delete; - pragma page_size = 1024; - - create table feeds ( - id integer not null primary key, - title text, - url text, - description text - ); - - create table entries ( - id integer not null primary key, - feedid integer not null, - datemillis integer not null, - title text, - content text, - url text, - foreign key(feedid) references feeds(id) - ); - create index entriesindex on entries(feedid); - - create table entries_content ( - entriesid integer not null, - content text, - foreign key(entriesid) references entries(id) - ); - create index entriescindex on entries_content(entriesid); - - create virtual table search using fts5( - entriesid unindexed, - content - ); - ` - _, err = db.Exec(initStmt) - if err != nil { - return nil, err - } - - return db, nil -} - -func insertSearch(db *sql.DB) error { - sqlStmt := ` - insert into search - select entriesid,content - from entries_content; - insert into search(search) values('optimize'); - vacuum; - ` - _, err := db.Exec(sqlStmt) - if err != nil { - return err - } - return nil -} - -func insertFeeds(db *sql.DB, feeds Feeds) error { - tx, err := db.Begin() - if err != nil { - return err - } - stmt_feeds, err := tx.Prepare("insert into feeds(id,title,url,description) values(?,?,?,?)") - if err != nil { - return err - } - defer stmt_feeds.Close() - stmt_entry, err := tx.Prepare( - "insert into entries(feedid,datemillis,title,url) values(?,?,?,?)", - ) - if err != nil { - return err - } - defer stmt_entry.Close() - stmt_entry_content, err := tx.Prepare( - "insert into entries_content(entriesid,content) values(?,?)", - ) - if err != nil { - return err - } - defer stmt_entry_content.Close() - for feedid, feed := range feeds { - _, err = stmt_feeds.Exec(feedid, feed.Title, feed.Url, feed.Description) - if err != nil { - return err - } - for _, entry := range feed.Entries { - // entries - res, err := stmt_entry.Exec( - feedid, - entry.Date.UnixMilli(), - entry.Title, - entry.Url, - ) - if err != nil { - return err - } - // entries_content - entryid, err := res.LastInsertId() - if err != nil { - return err - } - _, err = stmt_entry_content.Exec( - entryid, - entry.Content, - ) - if err != nil { - return err - } - } - } - err = tx.Commit() - if err != nil { - return err - } - - return nil -} - func (feeds Feeds) Save() error { db, err := initDb() if err != nil { return err } defer db.Close() - - err = insertFeeds(db, feeds) + err = insertFeedsAndEntries(db, feeds) if err != nil { return err } - - err = insertSearch(db) - if err != nil { - return err - } - return nil } -func (feed *Feed) fetch() error { +func (feed *Feed) Fetch() error { rawFeed, err := gofeed.NewParser().ParseURL(feed.Url) if err != nil { return err diff --git a/backend/src/main.go b/backend/src/main.go index 0b97b23..6935185 100644 --- a/backend/src/main.go +++ b/backend/src/main.go @@ -13,8 +13,13 @@ func main() { } for i := range feeds { - if err := feeds[i].fetch(); err != nil { - fmt.Fprintf(os.Stderr, "processing of url (%s) failed (%v)\n", feeds[i].Url, err) + if err := feeds[i].Fetch(); err != nil { + fmt.Fprintf( + os.Stderr, + "processing of url (%s) failed (%v)\n", + feeds[i].Url, + err, + ) continue } }