split sqlite3.entries table content, partially for #28

although I am not 100% sure if it worked it is sure that it speed up the display of individual feeds *shrugs*
azimut · Mar 31, 2024 · 54ac22d · 54ac22d
1 parent ee5a3b8
commit 54ac22d
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 18 deletions.
diff --git a/backend/src/feed.go b/backend/src/feed.go
@@ -94,12 +94,14 @@ func initDb() (*sql.DB, error) {
 	initStmt := `
     pragma journal_mode = delete;
     pragma page_size = 1024;
+
     create table feeds (
         id          integer not null primary key,
         title       text,
         url         text,
         description text
     );
+
     create table entries (
         id          integer not null primary key,
         feedid      integer not null,
@@ -110,6 +112,14 @@ func initDb() (*sql.DB, error) {
         foreign key(feedid) references feeds(id)
     );
     create index entriesindex on entries(feedid);
+
+    create table entries_content (
+        entriesid   integer not null,
+        content     text,
+        foreign key(entriesid) references entries(id)
+    );
+    create index entriescindex on entries_content(entriesid);
+
     create virtual table search using fts5(
         entriesid unindexed,
         content
@@ -126,8 +136,8 @@ func initDb() (*sql.DB, error) {
 func insertSearch(db *sql.DB) error {
 	sqlStmt := `
     insert into search
-    select id,content
-      from entries;
+    select entriesid,content
+      from entries_content;
     insert into search(search) values('optimize');
     vacuum;
     `
@@ -149,28 +159,47 @@ func insertFeeds(db *sql.DB, feeds Feeds) error {
 	}
 	defer stmt_feeds.Close()
 	stmt_entry, err := tx.Prepare(
-		"insert into entries(feedid,datemillis,title,content,url) values(?,?,?,?,?)",
+		"insert into entries(feedid,datemillis,title,url) values(?,?,?,?)",
 	)
 	if err != nil {
 		return err
 	}
 	defer stmt_entry.Close()
+	stmt_entry_content, err := tx.Prepare(
+		"insert into entries_content(entriesid,content) values(?,?)",
+	)
+	if err != nil {
+		return err
+	}
+	defer stmt_entry_content.Close()
 	for feedid, feed := range feeds {
 		_, err = stmt_feeds.Exec(feedid, feed.Title, feed.Url, feed.Description)
 		if err != nil {
 			return err
 		}
 		for _, entry := range feed.Entries {
-			_, err = stmt_entry.Exec(
+			// entries
+			res, err := stmt_entry.Exec(
 				feedid,
 				entry.Date.UnixMilli(),
 				entry.Title,
-				entry.Content,
 				entry.Url,
 			)
 			if err != nil {
 				return err
 			}
+			// entries_content
+			entryid, err := res.LastInsertId()
+			if err != nil {
+				return err
+			}
+			_, err = stmt_entry_content.Exec(
+				entryid,
+				entry.Content,
+			)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	err = tx.Commit()

diff --git a/frontend/index.js b/frontend/index.js
@@ -24,7 +24,8 @@ export async function getFeeds(dbarg) {
   let queue = [];
   await db('exec', {
     sql: `SELECT feeds.id, feeds.title, count(*)
-            FROM feeds JOIN entries ON feeds.id=entries.feedid
+            FROM feeds
+            JOIN entries ON feeds.id=entries.feedid
         GROUP BY entries.feedid
           HAVING count(*) > 0`,
     bind: {},
@@ -47,8 +48,8 @@ export async function getEntries(dbarg, feedid) {
   let queue = [];
   await db('exec', {
     sql: `SELECT id, title, datemillis, url
-          FROM entries
-          WHERE feedid=$fid`,
+            FROM entries
+           WHERE feedid=$fid`,
     bind: {$fid: feedid},
     callback: (msg) => {
       if (msg.row) {
@@ -75,10 +76,10 @@ export async function search(dbarg, needle) {
                  entries.title,
                  entries.url,
                  entries.datemillis
-          FROM search
-          JOIN entries ON search.entriesid=entries.id
-          WHERE search.content MATCH $match
-          ORDER BY entries.feedid,entries.id DESC`,
+            FROM search
+            JOIN entries ON search.entriesid=entries.id
+           WHERE search.content MATCH $match
+           ORDER BY entries.feedid,entries.id DESC`,
     bind: {$match: needle},
     callback: (msg) => {
       if (msg.row) {
@@ -102,9 +103,10 @@ export async function getEntryDetails(dbarg, entryId, needle) {
   if (needle && typeof needle === "string" && needle.length > 0) {
     await db('exec', {
       sql: `SELECT entries.feedid, highlight(search,1,'\`\`\`','\`\`\`')
-            FROM entries
-            JOIN search ON entries.id=search.entriesid
-            WHERE entries.id=$eid AND search.content MATCH $needle`,
+              FROM entries
+              JOIN search ON entries.id=search.entriesid
+             WHERE entries.id=$eid
+               AND search.content MATCH $needle`,
       bind: {$eid: entryId, $needle: needle},
       callback: (msg) => {
         if (msg.row) {
@@ -119,9 +121,10 @@ export async function getEntryDetails(dbarg, entryId, needle) {
     });
   } else {
     await db('exec', {
-      sql: `SELECT feedid, content
-            FROM entries
-            WHERE id=$eid`,
+      sql: `SELECT entries.feedid, entries_content.content
+              FROM entries
+              JOIN entries_content ON entries_content.entriesid=entries.id
+             WHERE entries.id=$eid`,
       bind: {$eid: entryId},
       callback: (msg) => {
         if (msg.row) {