Skip to content

Commit

Permalink
More database fixes (#42)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ice-dionysos
ice-dionysos authored Dec 9, 2024
1 parent a2a41d6 commit 5595a66
Show file tree
Hide file tree
Showing 11 changed files with 218 additions and 76 deletions.
5 changes: 4 additions & 1 deletion database/query/DDL.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ end
CREATE TABLE IF NOT EXISTS event_counters
(
reference_id text not null,
reference_type text not null DEFAULT '',
reference_type text not null DEFAULT '', -- for kind 7 events, it contains the actual reaction to the event, like `+` or `-`.
kind integer not null,
value integer not null DEFAULT 0,
primary key (kind, reference_type, reference_id)
Expand All @@ -334,6 +334,7 @@ begin
when e.kind in (1, 6, 16, 30023) and NEW.event_tag_key = 'e' and NEW.event_tag_value3 in ('reply', 'root') then 'reply'
when e.kind in (1, 6, 16, 30023) and NEW.event_tag_key = 'q' then 'quote'
when e.kind = 3 and NEW.event_tag_key = 'p' then 'follower'
when e.kind = 7 then e.content -- reaction type
else ''
end,
e.kind,
Expand Down Expand Up @@ -368,6 +369,7 @@ begin
end
;
--------
drop trigger if exists trigger_event_tags_after_delete_dec_counter;
create trigger if not exists trigger_event_tags_after_delete_dec_counter
after delete
on event_tags
Expand All @@ -386,6 +388,7 @@ begin
when e.kind in (1, 6, 16, 30023) and OLD.event_tag_key = 'e' and OLD.event_tag_value3 in ('reply', 'root') then 'reply'
when e.kind in (1, 6, 16, 30023) and OLD.event_tag_key = 'q' then 'quote'
when e.kind = 3 and OLD.event_tag_key = 'p' then 'follower'
when e.kind = 7 then e.content
else ''
end
and (
Expand Down
54 changes: 54 additions & 0 deletions database/query/query_count_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,3 +330,57 @@ func TestEventCounters(t *testing.T) {
})
})
}

func TestEventMultiReactions(t *testing.T) {
t.Parallel()

db := helperNewDatabase(t)
defer db.Close()

var ev model.Event
ev.ID = "t1id1"
ev.Content = "hello world"
ev.CreatedAt = 1
ev.PubKey = "t1pub1"
require.NoError(t, db.AcceptEvents(context.TODO(), &ev))
helperMustBePrecalculatedCount(t, db, 0, model.Filter{IDs: []string{ev.ID}, Kinds: []int{nostr.KindReaction}})

for _, r := range []string{"+", "-", "*"} {
t.Run(r, func(t *testing.T) {
const num = 2
for i := range num {
var reaction model.Event
reaction.ID = r + "reaction" + strconv.Itoa(i)
reaction.Kind = nostr.KindReaction
reaction.PubKey = r + "pubkeyr" + strconv.Itoa(i)
reaction.CreatedAt = model.Timestamp(i)
reaction.Content = r
reaction.Tags = model.Tags{{"e", "t1id1"}, {"p", "t1pub1"}}
require.NoError(t, db.AcceptEvents(context.Background(), &reaction))
}
})
}
helperMustBePrecalculatedCount(t, db, 6, model.Filter{IDs: []string{ev.ID}, Kinds: []int{nostr.KindReaction}})

// Remove one `-` reaction.
var deleteEv model.Event
deleteEv.Kind = nostr.KindDeletion
deleteEv.PubKey = "-pubkeyr1"
deleteEv.Tags = model.Tags{{"e", "-reaction1"}}
require.NoError(t, db.AcceptEvents(context.Background(), &deleteEv))
helperMustBePrecalculatedCount(t, db, 5, model.Filter{IDs: []string{ev.ID}, Kinds: []int{nostr.KindReaction}})

// Remove one `*` reaction.
deleteEv.Kind = nostr.KindDeletion
deleteEv.PubKey = "*pubkeyr0"
deleteEv.Tags = model.Tags{{"e", "*reaction0"}}
require.NoError(t, db.AcceptEvents(context.Background(), &deleteEv))
helperMustBePrecalculatedCount(t, db, 4, model.Filter{IDs: []string{ev.ID}, Kinds: []int{nostr.KindReaction}})

// Remove original event.
deleteEv.Kind = nostr.KindDeletion
deleteEv.PubKey = "t1pub1"
deleteEv.Tags = model.Tags{{"e", "t1id1"}}
require.NoError(t, db.AcceptEvents(context.Background(), &deleteEv))
helperMustBePrecalculatedCount(t, db, 0, model.Filter{IDs: []string{ev.ID}, Kinds: []int{nostr.KindReaction}})
}
51 changes: 33 additions & 18 deletions database/query/query_where_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,9 @@ type (
}
}
databaseFilterMarker struct {
Tag string
Marker string
Tag string
Marker string
Exclude bool
}
)

Expand Down Expand Up @@ -202,6 +203,9 @@ func (w *whereBuilder) applyFilterTagMarkers(name string, markers []databaseFilt

for id, marker := range markers {
w.maybeAND()
if marker.Exclude {
w.WriteString("NOT ")
}
w.WriteString("EXISTS (select true from event_tags where event_id in (e.id, e.reference_id) AND event_tag_key = :")
w.WriteString(w.addParam(name, "mtag"+strconv.Itoa(id), marker.Tag))
w.WriteString(" AND event_tag_value3 = :")
Expand Down Expand Up @@ -476,8 +480,13 @@ select
coalesce(evr.pubkey, ''),
coalesce(evr.master_pubkey, ''),
'',
case when f.kind = 7 then json_object('+', f.value) else cast(f.value as text) end as content,
json_array(json_object('kinds', json_array(:` + (filterID + "fkind") + `),:` + (filterID + "ftagname") + `,json_array(f.reference_id))) as d_tag,
`)
if filter.Reduce.Group && len(filter.Reduce.Kinds) > 1 && filter.Reduce.Kinds[1] == nostr.KindReaction {
w.WriteString(`json_group_object(coalesce(nullif(f.reference_type, ''), '+'), f.value) as content,`)
} else {
w.WriteString(`cast(f.value as text) as content,`)
}
w.WriteString(`json_array(json_object('kinds', json_array(:` + (filterID + "fkind") + `),:` + (filterID + "ftagname") + `,json_array(f.reference_id))) as d_tag,
case when
f.kind = 7 then
json_array(
Expand Down Expand Up @@ -639,8 +648,10 @@ group by e.pubkey, e.master_pubkey`)
w.addParam(filterID, "context", filter.Reduce.Tag)
refType = "follower"
}
w.WriteString(" AND f.reference_type = :")
w.WriteString(w.addParam(filterID, "rref", refType))
if !filter.Reduce.Group && filter.Reduce.Kinds[1] != nostr.KindReaction {
w.WriteString(" AND f.reference_type = :")
w.WriteString(w.addParam(filterID, "rref", refType))
}
w.WriteString(" AND f.reference_id IN (")
if filter.Reduce.Kinds[1] == nostr.KindFollowList {
w.WriteString(w.createWhereForDepFilter(filterID, cteName, "pubkey", &filter.Start))
Expand All @@ -650,6 +661,9 @@ group by e.pubkey, e.master_pubkey`)
w.WriteString(w.createWhereForDepFilter(filterID, cteName, "id", &filter.Start))
}
w.WriteString(")")
if filter.Reduce.Group && filter.Reduce.Kinds[1] == nostr.KindReaction {
w.WriteString(" GROUP BY reference_id")
}
}
}

Expand Down Expand Up @@ -792,20 +806,21 @@ func (w *whereBuilder) BuildForPrecalculatedCounters(filters ...model.Filter) (s
w.maybeOR()
w.WriteString("kind = :")
w.WriteString(w.addParam(filterID, "kind"+strconv.Itoa(idx), kinds[idx]))
w.WriteString(" AND reference_type = :")
var referenceType string
switch kinds[idx] {
case nostr.KindFollowList:
referenceType = "follower"
case nostr.KindTextNote, nostr.KindRepost, nostr.KindArticle, nostr.KindGenericRepost:
if _, ok := filter.Tags["q"]; ok {
referenceType = "quote"
} else {
referenceType = "reply"
if kinds[idx] != nostr.KindReaction {
w.WriteString(" AND reference_type = :")
var referenceType string
switch kinds[idx] {
case nostr.KindFollowList:
referenceType = "follower"
case nostr.KindTextNote, nostr.KindRepost, nostr.KindArticle, nostr.KindGenericRepost:
if _, ok := filter.Tags["q"]; ok {
referenceType = "quote"
} else {
referenceType = "reply"
}
}
w.WriteString(w.addParam(filterID, "reference_type"+strconv.Itoa(idx), referenceType))
}
w.WriteString(w.addParam(filterID, "reference_type"+strconv.Itoa(idx), referenceType))

}
w.WriteRune(')')
}
Expand Down
47 changes: 47 additions & 0 deletions database/query/query_where_builder_db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -932,3 +932,50 @@ func TestTagMarkerWithRepost(t *testing.T) {
require.Equal(t, "1", events[0].ID)
})
}

func TestTagMarkerNegative(t *testing.T) {
t.Parallel()

db := helperNewDatabase(t)
defer db.Close()

t.Run("Insert", func(t *testing.T) {
err := db.AcceptEvents(context.TODO(),
&model.Event{
Event: nostr.Event{
ID: "1id",
Kind: nostr.KindTextNote,
CreatedAt: 1,
Content: "foo",
PubKey: "fookey",
Tags: model.Tags{
{"e", "2", "", "root"},
{"q", "2"},
},
},
},
&model.Event{
Event: nostr.Event{
ID: "2id",
Kind: nostr.KindTextNote,
CreatedAt: 1,
Content: "bar",
PubKey: "barkey",
Tags: model.Tags{
{"e", "2", "", "reply"},
{"q", "2"},
},
},
},
)
require.NoError(t, err)
count, err := db.CountEvents(context.Background(), nil)
require.NoError(t, err)
require.Equal(t, int64(2), count)
})
t.Run("Select", func(t *testing.T) {
events := helperSelectEvents(t, db, model.Filter{Search: "!emarker:root"})
require.Len(t, events, 1)
require.Equal(t, "2id", events[0].ID)
})
}
15 changes: 12 additions & 3 deletions database/query/query_where_builder_filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,12 @@ func parseNostrFilterFlags(f *databaseFilterSearch) *databaseFilterSearch {
}

func parseNostrFilterTagMarkers(f *databaseFilterSearch) *databaseFilterSearch {
// Parse tag markers.
// - <tag>marker:<value>
// - !<tag>marker:<value>
const tagMarker = `marker:`
var tagMarkerOffset int

for strings.Contains(f.Search[tagMarkerOffset:], tagMarker) {
tagMarkerStart := strings.Index(f.Search[tagMarkerOffset:], tagMarker)
if tagMarkerStart < 1 {
Expand All @@ -68,14 +72,19 @@ func parseNostrFilterTagMarkers(f *databaseFilterSearch) *databaseFilterSearch {
tagMarkerEnd += tagMarkerStart + len(tagMarker)
}
if x := strings.TrimSpace(f.Search[tagMarkerOffset+tagMarkerStart-1 : tagMarkerOffset+tagMarkerStart]); x != "" {
var excludeOffset int
if (tagMarkerOffset+tagMarkerStart-1 > 0) && (f.Search[tagMarkerOffset+tagMarkerStart-2] == '!') {
excludeOffset = 1
}
f.TagMarkers = append(f.TagMarkers, databaseFilterMarker{
Tag: x,
Marker: f.Search[tagMarkerOffset+tagMarkerStart+len(tagMarker) : tagMarkerOffset+tagMarkerEnd],
Tag: x,
Marker: f.Search[tagMarkerOffset+tagMarkerStart+len(tagMarker) : tagMarkerOffset+tagMarkerEnd],
Exclude: excludeOffset > 0,
})
if tagMarkerEnd < len(f.Search) {
tagMarkerEnd++
}
f.Search = strings.TrimSpace(f.Search[:tagMarkerStart-1] + f.Search[tagMarkerEnd:])
f.Search = strings.TrimSpace(f.Search[:tagMarkerStart-1-excludeOffset] + f.Search[tagMarkerEnd:])
} else {
tagMarkerOffset += tagMarkerStart + len(tagMarker)
}
Expand Down
11 changes: 11 additions & 0 deletions database/query/query_where_builder_stmt_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,17 @@ func TestParseNostrFilter(t *testing.T) {
require.Len(t, f.TagMarkers, 3)
require.Equal(t, []databaseFilterMarker{{Tag: "a", Marker: "aval"}, {Tag: "b", Marker: "bval"}, {Tag: "c", Marker: "cval"}}, f.TagMarkers)
})
t.Run("One negative marker and one positive and images", func(t *testing.T) {
f, err := parseNostrFilter(model.Filter{
Search: "amarker:aval images:true some !bmarker:bval content here marker:invalid x",
})
require.NoError(t, err)
require.NotNil(t, f.Images)
require.True(t, *f.Images)
require.Equal(t, "some content here marker:invalid x", f.Filter.Search)
require.Len(t, f.TagMarkers, 2)
require.Equal(t, []databaseFilterMarker{{Tag: "a", Marker: "aval"}, {Tag: "b", Marker: "bval", Exclude: true}}, f.TagMarkers)
})
}

func TestBuildLiteFilter(t *testing.T) {
Expand Down
Loading

0 comments on commit 5595a66

Please sign in to comment.