From 7f313dcbd4f5cd2f0cb50a37643ccd7b82955b45 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Fri, 18 Aug 2023 08:40:33 -0700 Subject: [PATCH 001/111] nostrscript: add comment about iOS virtual memory allocs I'm really just doing this because I forgot a changelog entry Changelog-Fixed: Fixed nostrscript not working on smaller phones --- damus-c/wasm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/damus-c/wasm.c b/damus-c/wasm.c index 8304f3ec33..8a81d301c9 100644 --- a/damus-c/wasm.c +++ b/damus-c/wasm.c @@ -7100,6 +7100,7 @@ int wasm_interp_init(struct wasm_interp *interp, struct module *module) return 0; } + // keep total memory size small for now, iOS doesn't like like mallocs memory_pages_size = 8 * WASM_PAGE_SIZE; memsize = From 066b3cdde8ee3b285b8ab085dafb95e615639299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=98Aquino?= Date: Tue, 15 Aug 2023 21:33:47 +0000 Subject: [PATCH 002/111] Fix image links appearing with escaped slashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changelog-Fixed: Fix images and links occasionally appearing with escaped slashes Closes: https://github.com/damus-io/damus/issues/1468 Signed-off-by: Daniel D‘Aquino Signed-off-by: William Casarin Rewarded-sats: 50000 --- damus.xcodeproj/project.pbxproj | 4 +++ damus/TestData.swift | 3 ++ damus/Views/NoteContentView.swift | 5 ++++ damusTests/NostrEventTests.swift | 43 +++++++++++++++++++++++++++ damusTests/NoteContentViewTests.swift | 10 +++++++ nostrdb/Test/NdbTests.swift | 8 +++++ nostrdb/nostrdb.c | 1 + 7 files changed, 74 insertions(+) create mode 100644 damusTests/NostrEventTests.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index c1686283d7..21ebe09616 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -383,6 +383,7 @@ BAB68BED29543FA3007BA466 /* SelectWalletView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BAB68BEC29543FA3007BA466 /* SelectWalletView.swift */; }; D2277EEA2A089BD5006C3807 /* Router.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2277EE92A089BD5006C3807 /* Router.swift */; }; D78525252A7B2EA4002FA637 /* NoteContentViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */; }; + D7DEEF2F2A8C021E00E0C99F /* NostrEventTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */; }; E4FA1C032A24BB7F00482697 /* SearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */; }; E990020F2955F837003BBC5A /* EditMetadataView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E990020E2955F837003BBC5A /* EditMetadataView.swift */; }; E9E4ED0B295867B900DD7078 /* ThreadView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E9E4ED0A295867B900DD7078 /* ThreadView.swift */; }; @@ -934,6 +935,7 @@ BAB68BEC29543FA3007BA466 /* SelectWalletView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectWalletView.swift; sourceTree = ""; }; D2277EE92A089BD5006C3807 /* Router.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Router.swift; sourceTree = ""; }; D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NoteContentViewTests.swift; sourceTree = ""; }; + D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NostrEventTests.swift; sourceTree = ""; }; E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchSettingsView.swift; sourceTree = ""; }; E990020E2955F837003BBC5A /* EditMetadataView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EditMetadataView.swift; sourceTree = ""; }; E9E4ED0A295867B900DD7078 /* ThreadView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThreadView.swift; sourceTree = ""; }; @@ -1745,6 +1747,7 @@ D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */, 4C684A542A7E91FE005E6031 /* LongPostTests.swift */, 4C684A562A7FFAE6005E6031 /* UrlTests.swift */, + D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */, ); path = damusTests; sourceTree = ""; @@ -2451,6 +2454,7 @@ 4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */, 3A3040ED29A5CB86008A0F29 /* ReplyDescriptionTests.swift in Sources */, 3AAC7A022A60FE72002B50DF /* LocalizationUtilTests.swift in Sources */, + D7DEEF2F2A8C021E00E0C99F /* NostrEventTests.swift in Sources */, 4C8D00D429E3C5D40036AF10 /* NIP19Tests.swift in Sources */, 3A30410129AB12AA008A0F29 /* EventGroupViewTests.swift in Sources */, 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */, diff --git a/damus/TestData.swift b/damus/TestData.swift index d5d965410b..5e8bfec9c7 100644 --- a/damus/TestData.swift +++ b/damus/TestData.swift @@ -28,6 +28,9 @@ let test_note = createdAt: UInt32(Date().timeIntervalSince1970 - 100) )! +let test_note_json_with_escaped_slash = "{\"tags\":[],\"pubkey\":\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\",\"content\":\"https:\\/\\/cdn.nostr.build\\/i\\/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg\",\"created_at\":1691864981,\"kind\":1,\"sig\":\"fc0033aa3d4df50b692a5b346fa816fdded698de2045e36e0642a021391468c44ca69c2471adc7e92088131872d4aaa1e90ea6e1ad97f3cc748f4aed96dfae18\",\"id\":\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\"}" +let test_encoded_note_with_image = NostrEvent.owned_from_json(json: test_note_json_with_escaped_slash) + let test_encoded_post = "{\"id\": \"8ba545ab96959fe0ce7db31bc10f3ac3aa5353bc4428dbf1e56a7be7062516db\",\"pubkey\": \"7e27509ccf1e297e1df164912a43406218f8bd80129424c3ef798ca3ef5c8444\",\"created_at\": 1677013417,\"kind\": 1,\"tags\": [],\"content\": \"hello\",\"sig\": \"93684f15eddf11f42afbdd81828ee9fc35350344d8650c78909099d776e9ad8d959cd5c4bff7045be3b0b255144add43d0feef97940794a1bc9c309791bebe4a\"}" let test_repost_1 = NostrEvent(content: test_encoded_post, keypair: test_keypair, kind: 6, tags: [], createdAt: 1)! diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index f41483a1b3..82069515ae 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -610,6 +610,11 @@ struct NoteContentView_Previews: PreviewProvider { NoteContentView(damus_state: state, event: test_note, show_images: true, size: .normal, options: []) } .previewDisplayName("Short note") + + VStack { + NoteContentView(damus_state: state, event: test_encoded_note_with_image!, show_images: true, size: .normal, options: []) + } + .previewDisplayName("Note with image") VStack { NoteContentView(damus_state: state2, event: test_longform_event.event, show_images: true, size: .normal, options: [.wide]) diff --git a/damusTests/NostrEventTests.swift b/damusTests/NostrEventTests.swift new file mode 100644 index 0000000000..827e890982 --- /dev/null +++ b/damusTests/NostrEventTests.swift @@ -0,0 +1,43 @@ +// +// NostrEventTests.swift +// damusTests +// +// Created by Daniel D’Aquino on 2023-08-15. +// + +import Foundation +import XCTest +@testable import damus + +final class NostrEventTests: XCTestCase { + + override func setUpWithError() throws { + // Put setup code here. This method is called before the invocation of each test method in the class. + } + + override func tearDownWithError() throws { + // Put teardown code here. This method is called after the invocation of each test method in the class. + } + + /// Based on https://github.com/damus-io/damus/issues/1468 + /// Tests whether `decode_nostr_event` correctly decodes nostr note image content written with optional escaped slashes + func testDecodeNostrEventWithEscapedSlashes() throws { + let testMessageString = "[\"EVENT\",\"A54091AC-D144-49F6-853A-2141A5EA09B6\",{\"content\":\"{\\\"tags\\\":[],\\\"pubkey\\\":\\\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\\\",\\\"content\\\":\\\"https:\\\\/\\\\/cdn.nostr.build\\\\/i\\\\/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg\\\",\\\"created_at\\\":1691864981,\\\"kind\\\":1,\\\"sig\\\":\\\"fc0033aa3d4df50b692a5b346fa816fdded698de2045e36e0642a021391468c44ca69c2471adc7e92088131872d4aaa1e90ea6e1ad97f3cc748f4aed96dfae18\\\",\\\"id\\\":\\\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\\\"}\",\"created_at\":1691866192,\"id\":\"56e3f14cedab76762afef78eeb34b07ce1313543a2f3365a7b99fd5daa65abc9\",\"kind\":6,\"pubkey\":\"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\",\"sig\":\"3c1161c3b03cafe13e2d9d624b158bdb74867caf61df158871633c859cd587e7779d096680de34024d9acfcba9aa3cb76fdbfa20227afb7e03a9ab588e6b77c9\",\"tags\":[[\"e\",\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\",\"\",\"root\"],[\"p\",\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\"]]}]" + let response: NostrResponse = decode_nostr_event(txt: testMessageString)! + guard case .event(_, let testEvent) = response else { + XCTAssert(false, "Could not decode event") + return + } + let urlInContent = "https:\\/\\/cdn.nostr.build\\/i\\/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg" + XCTAssert(testEvent.content.contains(urlInContent), "Issue parsing event. Expected to see '\(urlInContent)' inside \(testEvent.content)") + + let testMessageString2 = "[\"EVENT\",\"A54091AC-D144-49F6-853A-2141A5EA09B6\",{\"content\":\"{\\\"tags\\\":[],\\\"pubkey\\\":\\\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\\\",\\\"content\\\":\\\"https:\\/\\/cdn.nostr.build\\/i\\/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg\\\",\\\"created_at\\\":1691864981,\\\"kind\\\":1,\\\"sig\\\":\\\"fc0033aa3d4df50b692a5b346fa816fdded698de2045e36e0642a021391468c44ca69c2471adc7e92088131872d4aaa1e90ea6e1ad97f3cc748f4aed96dfae18\\\",\\\"id\\\":\\\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\\\"}\",\"created_at\":1691866192,\"id\":\"56e3f14cedab76762afef78eeb34b07ce1313543a2f3365a7b99fd5daa65abc9\",\"kind\":6,\"pubkey\":\"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\",\"sig\":\"3c1161c3b03cafe13e2d9d624b158bdb74867caf61df158871633c859cd587e7779d096680de34024d9acfcba9aa3cb76fdbfa20227afb7e03a9ab588e6b77c9\",\"tags\":[[\"e\",\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\",\"\",\"root\"],[\"p\",\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\"]]}]" + let response2: NostrResponse = decode_nostr_event(txt: testMessageString2)! + guard case .event(_, let testEvent2) = response2 else { + XCTAssert(false, "Could not decode event") + return + } + let urlInContent2 = "https://cdn.nostr.build/i/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg" + XCTAssert(testEvent2.content.contains(urlInContent2), "Issue parsing event. Expected to see '\(urlInContent2)' inside \(testEvent2.content)") + } +} diff --git a/damusTests/NoteContentViewTests.swift b/damusTests/NoteContentViewTests.swift index 078f6500ab..9cd740c90e 100644 --- a/damusTests/NoteContentViewTests.swift +++ b/damusTests/NoteContentViewTests.swift @@ -25,5 +25,15 @@ class NoteContentViewTests: XCTestCase { XCTAssertEqual(runArray[1].link?.absoluteString, "damus:t:cool", "Latin-character hashtag is missing. Runs description :\(runArray.description)") XCTAssertEqual(runArray[3].link?.absoluteString.removingPercentEncoding!, "damus:t:かっこいい", "Non-latin-character hashtag is missing. Runs description :\(runArray.description)") } + + /// Based on https://github.com/damus-io/damus/issues/1468 + /// Tests whether a note content view correctly parses an image block when url in JSON content contains optional escaped slashes + func testParseImageBlockInContentWithEscapedSlashes() { + let testJSONWithEscapedSlashes = "{\"tags\":[],\"pubkey\":\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\",\"content\":\"https:\\/\\/cdn.nostr.build\\/i\\/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg\",\"created_at\":1691864981,\"kind\":1,\"sig\":\"fc0033aa3d4df50b692a5b346fa816fdded698de2045e36e0642a021391468c44ca69c2471adc7e92088131872d4aaa1e90ea6e1ad97f3cc748f4aed96dfae18\",\"id\":\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\"}" + let testNote = NostrEvent.owned_from_json(json: testJSONWithEscapedSlashes)! + let parsed = parse_note_content(content: .init(note: testNote, privkey: test_keypair.privkey)) + + XCTAssertTrue((parsed.blocks[0].is_url != nil), "NoteContentView does not correctly parse an image block when url in JSON content contains optional escaped slashes.") + } } diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift index dede7500ec..2b1e9fb177 100644 --- a/nostrdb/Test/NdbTests.swift +++ b/nostrdb/Test/NdbTests.swift @@ -80,6 +80,14 @@ final class NdbTests: XCTestCase { XCTAssertEqual(tags, 786) XCTAssertEqual(total_count_stored, total_count_iter) } + + /// Based on https://github.com/damus-io/damus/issues/1468 + /// Tests whether a JSON with optional escaped slash characters is correctly unescaped (In accordance to https://datatracker.ietf.org/doc/html/rfc8259#section-7) + func test_decode_json_with_escaped_slashes() { + let testJSONWithEscapedSlashes = "{\"tags\":[],\"pubkey\":\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\",\"content\":\"https:\\/\\/cdn.nostr.build\\/i\\/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg\",\"created_at\":1691864981,\"kind\":1,\"sig\":\"fc0033aa3d4df50b692a5b346fa816fdded698de2045e36e0642a021391468c44ca69c2471adc7e92088131872d4aaa1e90ea6e1ad97f3cc748f4aed96dfae18\",\"id\":\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\"}" + let testNote = NdbNote.owned_from_json(json: testJSONWithEscapedSlashes)! + XCTAssertEqual(testNote.content, "https://cdn.nostr.build/i/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg") + } func test_decode_perf() throws { // This is an example of a performance test case. diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index ea6a60b4b7..689cd56951 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -117,6 +117,7 @@ static int cursor_push_unescaped_char(struct cursor *cur, char c1, char c2) case 'b': return cursor_push_byte(cur, '\b'); case 'f': return cursor_push_byte(cur, '\f'); case '\\': return cursor_push_byte(cur, '\\'); + case '/': return cursor_push_byte(cur, '/'); case '"': return cursor_push_byte(cur, '"'); case 'u': // these aren't handled yet From f9eb669132c3375f81481c9d9ed957c6607352f3 Mon Sep 17 00:00:00 2001 From: "tappu75e@duck.com" Date: Mon, 14 Aug 2023 22:22:31 +0300 Subject: [PATCH 003/111] replies: fix bug where it would sometimes show -1 Changelog-Fixed: Fix bug where it would sometimes show -1 in replies Closes: https://github.com/damus-io/damus/pull/1476 --- damus/Views/Events/Components/ReplyDescription.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/damus/Views/Events/Components/ReplyDescription.swift b/damus/Views/Events/Components/ReplyDescription.swift index 5bc9c89a3f..44a981467a 100644 --- a/damus/Views/Events/Components/ReplyDescription.swift +++ b/damus/Views/Events/Components/ReplyDescription.swift @@ -47,7 +47,7 @@ func reply_desc(profiles: Profiles, event: NostrEvent, replying_to: NostrEvent?, if uniqueNames.count > 1 { let othersCount = n - pubkeys.count - if othersCount == 0 { + if othersCount <= 0 { return String(format: NSLocalizedString("Replying to %@ & %@", bundle: bundle, comment: "Label to indicate that the user is replying to 2 users."), locale: locale, uniqueNames[0], uniqueNames[1]) } else { return String(format: localizedStringFormat(key: "replying_to_two_and_others", locale: locale), locale: locale, othersCount, uniqueNames[0], uniqueNames[1]) From 2e512317e7f5610d66c349f581a880a5303b304d Mon Sep 17 00:00:00 2001 From: William Casarin Date: Fri, 18 Aug 2023 10:09:08 -0700 Subject: [PATCH 004/111] v1.6 (13) --- damus.xcodeproj/project.pbxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 21ebe09616..d0e7c8d8b5 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -2746,7 +2746,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 11; + CURRENT_PROJECT_VERSION = 13; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; @@ -2795,7 +2795,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 11; + CURRENT_PROJECT_VERSION = 13; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; From 9fb1cc5b57016f0b03c4b4d355121b12ec5aff97 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Fri, 18 Aug 2023 11:20:51 -0700 Subject: [PATCH 005/111] v1.16 (13) changelog --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7a190c8b2..b5d28c988c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## [1.6-13] - 2023-08-18 + +### Fixed + +- Fix bug where it would sometimes show -1 in replies (tappu75e@duck.com) +- Fix images and links occasionally appearing with escaped slashes (Daniel D‘Aquino) +- Fixed nostrscript not working on smaller phones (William Casarin) +- Fix zaps sometimes not appearing (William Casarin) +- Fixed issue where reposts would sometimes repost the wrong thing (William Casarin) +- Fixed issue where sometimes there would be empty entries on your profile (William Casarin) + +[1.6-13]: https://github.com/damus-io/damus/releases/tag/v1.6-13 + + ## [1.6-11]: "Bugfix Sunday" - 2023-08-07 ### Added From d5b944170f7d873ae1d08dd10c33f44e1b1418d5 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Fri, 18 Aug 2023 16:40:36 -0700 Subject: [PATCH 006/111] actually build 15 because reasons --- damus.xcodeproj/project.pbxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index d0e7c8d8b5..17fc83a845 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -2746,7 +2746,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 13; + CURRENT_PROJECT_VERSION = 15; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; @@ -2795,7 +2795,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 13; + CURRENT_PROJECT_VERSION = 15; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; From ae2f7255a7972f5d42562b76fdae369981b1659c Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 20 Aug 2023 11:45:04 -0700 Subject: [PATCH 007/111] Mute hellthreads everywhere Changelog-Fixed: Mute hellthreads everywhere Fixes: https://damus.io/note1rn3ckl76myga6xcefr0le52d8czd0wqe8apguewqknyv7m55mmpq3rv3hv --- damus/Models/HomeModel.swift | 23 +++++++++++------------ damus/Models/SearchHomeModel.swift | 5 +++-- damus/Models/SearchModel.swift | 6 ++++-- damus/Views/Events/MutedEventView.swift | 4 ++-- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index d6446bc1c9..30432bd0e0 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -562,15 +562,9 @@ class HomeModel { func handle_notification(ev: NostrEvent) { // don't show notifications from ourselves - guard ev.pubkey != damus_state.pubkey else { - return - } - - guard event_has_our_pubkey(ev, our_pubkey: self.damus_state.pubkey) else { - return - } - - guard should_show_event(contacts: damus_state.contacts, ev: ev) && !damus_state.muted_threads.isMutedThread(ev, privkey: damus_state.keypair.privkey) else { + guard ev.pubkey != damus_state.pubkey, + event_has_our_pubkey(ev, our_pubkey: self.damus_state.pubkey), + should_show_event(privkey: self.damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { return } @@ -608,7 +602,7 @@ class HomeModel { func handle_text_event(sub_id: String, _ ev: NostrEvent) { - guard should_show_event(contacts: damus_state.contacts, ev: ev) else { + guard should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { return } @@ -635,7 +629,7 @@ class HomeModel { } func handle_dm(_ ev: NostrEvent) { - guard should_show_event(contacts: damus_state.contacts, ev: ev) else { + guard should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { return } @@ -1100,10 +1094,15 @@ func event_has_our_pubkey(_ ev: NostrEvent, our_pubkey: Pubkey) -> Bool { } -func should_show_event(contacts: Contacts, ev: NostrEvent) -> Bool { +func should_show_event(privkey: Privkey?, hellthreads: MutedThreadsManager, contacts: Contacts, ev: NostrEvent) -> Bool { if contacts.is_muted(ev.pubkey) { return false } + + if hellthreads.isMutedThread(ev, privkey: privkey) { + return false + } + return ev.should_show_event } diff --git a/damus/Models/SearchHomeModel.swift b/damus/Models/SearchHomeModel.swift index 6931f39507..6493bc71b2 100644 --- a/damus/Models/SearchHomeModel.swift +++ b/damus/Models/SearchHomeModel.swift @@ -35,7 +35,7 @@ class SearchHomeModel: ObservableObject { } func filter_muted() { - events.filter { should_show_event(contacts: damus_state.contacts, ev: $0) } + events.filter { should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: $0) } self.objectWillChange.send() } @@ -60,7 +60,8 @@ class SearchHomeModel: ObservableObject { guard sub_id == self.base_subid || sub_id == self.profiles_subid else { return } - if ev.is_textlike && should_show_event(contacts: damus_state.contacts, ev: ev) && !ev.is_reply(nil) { + if ev.is_textlike && should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) && !ev.is_reply(nil) + { if !damus_state.settings.multiple_events_per_pubkey && seen_pubkey.contains(ev.pubkey) { return } diff --git a/damus/Models/SearchModel.swift b/damus/Models/SearchModel.swift index 7922b41ade..a782b26103 100644 --- a/damus/Models/SearchModel.swift +++ b/damus/Models/SearchModel.swift @@ -27,7 +27,9 @@ class SearchModel: ObservableObject { } func filter_muted() { - self.events.filter { should_show_event(contacts: state.contacts, ev: $0) } + self.events.filter { + should_show_event(privkey: state.keypair.privkey, hellthreads: state.muted_threads, contacts: state.contacts, ev: $0) + } self.objectWillChange.send() } @@ -55,7 +57,7 @@ class SearchModel: ObservableObject { return } - guard should_show_event(contacts: state.contacts, ev: ev) else { + guard should_show_event(privkey: state.keypair.privkey, hellthreads: state.muted_threads, contacts: state.contacts, ev: ev) else { return } diff --git a/damus/Views/Events/MutedEventView.swift b/damus/Views/Events/MutedEventView.swift index 9749ab7364..10140389a5 100644 --- a/damus/Views/Events/MutedEventView.swift +++ b/damus/Views/Events/MutedEventView.swift @@ -18,11 +18,11 @@ struct MutedEventView: View { self.damus_state = damus_state self.event = event self.selected = selected - self._shown = State(initialValue: should_show_event(contacts: damus_state.contacts, ev: event)) + self._shown = State(initialValue: should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: event)) } var should_mute: Bool { - return !should_show_event(contacts: damus_state.contacts, ev: event) + return !should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: event) } var MutedBox: some View { From 1432087edfa66db184b03efadf54ef57734b81db Mon Sep 17 00:00:00 2001 From: Fishcake Date: Sun, 20 Aug 2023 10:08:56 +0900 Subject: [PATCH 008/111] add nostr event 27235 (nip-98) Closes: https://github.com/damus-io/damus/pull/1471 Signed-off-by: William Casarin --- damus/Models/HomeModel.swift | 2 ++ damus/Nostr/NostrKind.swift | 1 + 2 files changed, 3 insertions(+) diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index 30432bd0e0..b06669711f 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -188,6 +188,8 @@ class HomeModel { break case .nwc_response: handle_nwc_response(ev, relay: relay_id) + case .http_auth: + break } } diff --git a/damus/Nostr/NostrKind.swift b/damus/Nostr/NostrKind.swift index 680eebdb4e..217d0978e1 100644 --- a/damus/Nostr/NostrKind.swift +++ b/damus/Nostr/NostrKind.swift @@ -23,4 +23,5 @@ enum NostrKind: UInt32, Codable { case zap_request = 9734 case nwc_request = 23194 case nwc_response = 23195 + case http_auth = 27235 } From 9104ddb051705adf354f4e4d41ad58c9627ece18 Mon Sep 17 00:00:00 2001 From: Fishcake Date: Sun, 20 Aug 2023 10:09:30 +0900 Subject: [PATCH 009/111] add function to create nip98 http authorization header Closes: https://github.com/damus-io/damus/pull/1471 Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 ++++ damus/Nostr/Nip98HTTPAuth.swift | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 damus/Nostr/Nip98HTTPAuth.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 17fc83a845..db5ab935d4 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -36,6 +36,7 @@ 3ACB685F297633BC00C46468 /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = 3ACB685D297633BC00C46468 /* Localizable.strings */; }; 3ACBCB78295FE5C70037388A /* TimeAgoTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3ACBCB77295FE5C70037388A /* TimeAgoTests.swift */; }; 3AE45AF6297BB2E700C1D842 /* LibreTranslateServer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AE45AF5297BB2E700C1D842 /* LibreTranslateServer.swift */; }; + 3CCD1E6A2A874C4E0099A953 /* Nip98HTTPAuth.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CCD1E692A874C4E0099A953 /* Nip98HTTPAuth.swift */; }; 4C06670128FC7C5900038D2A /* RelayView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C06670028FC7C5900038D2A /* RelayView.swift */; }; 4C06670428FC7EC500038D2A /* Kingfisher in Frameworks */ = {isa = PBXBuildFile; productRef = 4C06670328FC7EC500038D2A /* Kingfisher */; }; 4C06670628FCB08600038D2A /* ImageCarousel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C06670528FCB08600038D2A /* ImageCarousel.swift */; }; @@ -534,6 +535,7 @@ 3AF6336829884C6B0005672A /* pt-PT */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "pt-PT"; path = "pt-PT.lproj/InfoPlist.strings"; sourceTree = ""; }; 3AF6336929884C6B0005672A /* pt-PT */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "pt-PT"; path = "pt-PT.lproj/Localizable.strings"; sourceTree = ""; }; 3AF6336A29884C6B0005672A /* pt-PT */ = {isa = PBXFileReference; lastKnownFileType = text.plist.stringsdict; name = "pt-PT"; path = "pt-PT.lproj/Localizable.stringsdict"; sourceTree = ""; }; + 3CCD1E692A874C4E0099A953 /* Nip98HTTPAuth.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Nip98HTTPAuth.swift; sourceTree = ""; }; 4C06670028FC7C5900038D2A /* RelayView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayView.swift; sourceTree = ""; }; 4C06670528FCB08600038D2A /* ImageCarousel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageCarousel.swift; sourceTree = ""; }; 4C06670828FDE64700038D2A /* damus-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "damus-Bridging-Header.h"; sourceTree = ""; }; @@ -1313,6 +1315,7 @@ 4C75EFAE28049D340006080F /* NostrFilter.swift */, 4C75EFB028049D510006080F /* NostrResponse.swift */, 4C75EFB228049D640006080F /* NostrEvent.swift */, + 3CCD1E692A874C4E0099A953 /* Nip98HTTPAuth.swift */, 4C75EFB428049D790006080F /* Relay.swift */, 4C75EFB628049D990006080F /* RelayPool.swift */, 4C75EFBA2804A34C0006080F /* ProofOfWork.swift */, @@ -2331,6 +2334,7 @@ 4C12536A2A76D3850004F4B8 /* RelaysChangedNotify.swift in Sources */, 4C30AC8029A6A53F00E2BD5A /* ProfilePicturesView.swift in Sources */, 5C6E1DAD2A193EC2008FC15A /* GradientButtonStyle.swift in Sources */, + 3CCD1E6A2A874C4E0099A953 /* Nip98HTTPAuth.swift in Sources */, 4C8EC52529D1FA6C0085D9A8 /* DamusColors.swift in Sources */, 3A4647CF2A413ADC00386AD8 /* CondensedProfilePicturesView.swift in Sources */, D2277EEA2A089BD5006C3807 /* Router.swift in Sources */, diff --git a/damus/Nostr/Nip98HTTPAuth.swift b/damus/Nostr/Nip98HTTPAuth.swift new file mode 100644 index 0000000000..c4a8027be9 --- /dev/null +++ b/damus/Nostr/Nip98HTTPAuth.swift @@ -0,0 +1,23 @@ +// +// Nip98HTTPAuth.swift +// damus +// +// Created by Fishcake on 2023/08/12. +// + +import Foundation + +func create_nip98_signature (keypair: Keypair, method: String, url: URL) -> String? { + let tags = [ + ["u", url.standardized.absoluteString], // Ensure that we standardise the URL before extracting string value. + ["method", method] + ] + + guard let ev = NostrEvent(content: "", keypair: keypair, kind: NostrKind.http_auth.rawValue, tags: tags) else { + return nil + } + + let json = event_to_json(ev: ev) + let base64Header = base64_encode(Array(json.utf8)) + return "Nostr " + base64Header // The returned value should be used in Authorization HTTP header +} From c464a26151046e17cd469f390a3212b8af1c9c8e Mon Sep 17 00:00:00 2001 From: Fishcake Date: Sun, 20 Aug 2023 10:10:24 +0900 Subject: [PATCH 010/111] use nostr.build api v2 with optional nip98 support Closes: https://github.com/damus-io/damus/pull/1471 Signed-off-by: William Casarin --- damus/Models/ImageUploadModel.swift | 4 ++-- damus/Views/AttachMediaUtility.swift | 34 +++++++++++++++++++++++++--- damus/Views/PostView.swift | 2 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/damus/Models/ImageUploadModel.swift b/damus/Models/ImageUploadModel.swift index d31081a701..4a3dae8e5f 100644 --- a/damus/Models/ImageUploadModel.swift +++ b/damus/Models/ImageUploadModel.swift @@ -47,8 +47,8 @@ enum MediaUpload { class ImageUploadModel: NSObject, URLSessionTaskDelegate, ObservableObject { @Published var progress: Double? = nil - func start(media: MediaUpload, uploader: MediaUploader) async -> ImageUploadResult { - let res = await create_upload_request(mediaToUpload: media, mediaUploader: uploader, progress: self) + func start(media: MediaUpload, uploader: MediaUploader, keypair: Keypair? = nil) async -> ImageUploadResult { + let res = await create_upload_request(mediaToUpload: media, mediaUploader: uploader, progress: self, keypair: keypair) DispatchQueue.main.async { self.progress = nil } diff --git a/damus/Views/AttachMediaUtility.swift b/damus/Views/AttachMediaUtility.swift index 5702f05c24..88400b2854 100644 --- a/damus/Views/AttachMediaUtility.swift +++ b/damus/Views/AttachMediaUtility.swift @@ -28,7 +28,7 @@ fileprivate func create_upload_body(mediaData: Data, boundary: String, mediaUplo return body as Data } -func create_upload_request(mediaToUpload: MediaUpload, mediaUploader: MediaUploader, progress: URLSessionTaskDelegate) async -> ImageUploadResult { +func create_upload_request(mediaToUpload: MediaUpload, mediaUploader: MediaUploader, progress: URLSessionTaskDelegate, keypair: Keypair? = nil) async -> ImageUploadResult { var mediaData: Data? guard let url = URL(string: mediaUploader.postAPI) else { return .failed(nil) @@ -39,6 +39,15 @@ func create_upload_request(mediaToUpload: MediaUpload, mediaUploader: MediaUploa let boundary = "Boundary-\(UUID().description)" request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type") + // If uploading to a media host that support NIP-98 authorization, add the header + if mediaUploader == .nostrBuild, + let keypair, + let method = request.httpMethod, + let signature = create_nip98_signature(keypair: keypair, method: method, url: url) { + + request.setValue(signature, forHTTPHeaderField: "Authorization") + } + switch mediaToUpload { case .image(let url): do { @@ -139,7 +148,7 @@ enum MediaUploader: String, CaseIterable, Identifiable, StringCodable { var postAPI: String { switch self { case .nostrBuild: - return "https://nostr.build/api/upload/ios.php" + return "https://nostr.build/api/v2/upload/files" case .nostrImg: return "https://nostrimg.com/api/upload" } @@ -149,11 +158,30 @@ enum MediaUploader: String, CaseIterable, Identifiable, StringCodable { switch self { case .nostrBuild: do { - return try JSONSerialization.jsonObject(with: data, options: .allowFragments) as? String + if let jsonObject = try JSONSerialization.jsonObject(with: data, options: .allowFragments) as? [String: Any], + let status = jsonObject["status"] as? String { + + if status == "success", let dataArray = jsonObject["data"] as? [[String: Any]] { + + var urls: [String] = [] + + for dataDict in dataArray { + if let mainUrl = dataDict["url"] as? String { + urls.append(mainUrl) + } + } + + return urls.joined(separator: "\n") + } else if status == "error", let message = jsonObject["message"] as? String { + print("Upload Error: \(message)") + return nil + } + } } catch { print("Failed JSONSerialization") return nil } + return nil case .nostrImg: guard let responseString = String(data: data, encoding: String.Encoding(rawValue: String.Encoding.utf8.rawValue)) else { print("Upload failed getting response string") diff --git a/damus/Views/PostView.swift b/damus/Views/PostView.swift index 8c7897087e..d803bfa05b 100644 --- a/damus/Views/PostView.swift +++ b/damus/Views/PostView.swift @@ -255,7 +255,7 @@ struct PostView: View { let img = getImage(media: media) print("img size w:\(img.size.width) h:\(img.size.height)") async let blurhash = calculate_blurhash(img: img) - let res = await image_upload.start(media: media, uploader: uploader) + let res = await image_upload.start(media: media, uploader: uploader, keypair: damus_state.keypair) switch res { case .success(let url): From dcb94635ea0b202e6828072a490f0ee859164e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=99Aquino?= Date: Sat, 19 Aug 2023 19:04:18 +0000 Subject: [PATCH 011/111] Fix text editing issues on characters added right after mention link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changelog-Fixed: Fix text editing issues on characters added right after mention link Closes: https://github.com/damus-io/damus/issues/1375 Signed-off-by: Daniel D’Aquino Tested-by: William Casarin Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 + damus/Views/PostView.swift | 2 + damus/Views/TextViewWrapper.swift | 68 +++++++++++++- damusTests/PostViewTests.swift | 143 ++++++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 damusTests/PostViewTests.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index db5ab935d4..3621badec0 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -383,6 +383,7 @@ BA693074295D649800ADDB87 /* UserSettingsStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA693073295D649800ADDB87 /* UserSettingsStore.swift */; }; BAB68BED29543FA3007BA466 /* SelectWalletView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BAB68BEC29543FA3007BA466 /* SelectWalletView.swift */; }; D2277EEA2A089BD5006C3807 /* Router.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2277EE92A089BD5006C3807 /* Router.swift */; }; + D71DC1EC2A9129C3006E207C /* PostViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D71DC1EB2A9129C3006E207C /* PostViewTests.swift */; }; D78525252A7B2EA4002FA637 /* NoteContentViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */; }; D7DEEF2F2A8C021E00E0C99F /* NostrEventTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */; }; E4FA1C032A24BB7F00482697 /* SearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */; }; @@ -936,6 +937,7 @@ BA693073295D649800ADDB87 /* UserSettingsStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserSettingsStore.swift; sourceTree = ""; }; BAB68BEC29543FA3007BA466 /* SelectWalletView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectWalletView.swift; sourceTree = ""; }; D2277EE92A089BD5006C3807 /* Router.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Router.swift; sourceTree = ""; }; + D71DC1EB2A9129C3006E207C /* PostViewTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostViewTests.swift; sourceTree = ""; }; D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NoteContentViewTests.swift; sourceTree = ""; }; D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NostrEventTests.swift; sourceTree = ""; }; E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchSettingsView.swift; sourceTree = ""; }; @@ -1751,6 +1753,7 @@ 4C684A542A7E91FE005E6031 /* LongPostTests.swift */, 4C684A562A7FFAE6005E6031 /* UrlTests.swift */, D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */, + D71DC1EB2A9129C3006E207C /* PostViewTests.swift */, ); path = damusTests; sourceTree = ""; @@ -2457,6 +2460,7 @@ 4C9B0DEE2A65A75F00CBDA21 /* AttrStringTestExtensions.swift in Sources */, 4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */, 3A3040ED29A5CB86008A0F29 /* ReplyDescriptionTests.swift in Sources */, + D71DC1EC2A9129C3006E207C /* PostViewTests.swift in Sources */, 3AAC7A022A60FE72002B50DF /* LocalizationUtilTests.swift in Sources */, D7DEEF2F2A8C021E00E0C99F /* NostrEventTests.swift in Sources */, 4C8D00D429E3C5D40036AF10 /* NIP19Tests.swift in Sources */, diff --git a/damus/Views/PostView.swift b/damus/Views/PostView.swift index d803bfa05b..a484d2d42b 100644 --- a/damus/Views/PostView.swift +++ b/damus/Views/PostView.swift @@ -204,6 +204,8 @@ struct PostView: View { TextViewWrapper(attributedText: $post, postTextViewCanScroll: $postTextViewCanScroll, cursorIndex: newCursorIndex, getFocusWordForMention: { word, range in focusWordAttributes = (word, range) self.newCursorIndex = nil + }, updateCursorPosition: { newCursorIndex in + self.newCursorIndex = newCursorIndex }) .environmentObject(tagModel) .focused($focus) diff --git a/damus/Views/TextViewWrapper.swift b/damus/Views/TextViewWrapper.swift index 0f18eeab32..007c0315e4 100644 --- a/damus/Views/TextViewWrapper.swift +++ b/damus/Views/TextViewWrapper.swift @@ -14,6 +14,7 @@ struct TextViewWrapper: UIViewRepresentable { let cursorIndex: Int? var getFocusWordForMention: ((String?, NSRange?) -> Void)? = nil + let updateCursorPosition: ((Int) -> Void) func makeUIView(context: Context) -> UITextView { let textView = UITextView() @@ -34,11 +35,11 @@ struct TextViewWrapper: UIViewRepresentable { func updateUIView(_ uiView: UITextView, context: Context) { uiView.isScrollEnabled = postTextViewCanScroll - let range = uiView.selectedRange uiView.attributedText = attributedText TextViewWrapper.setTextProperties(uiView) setCursorPosition(textView: uiView) + let range = uiView.selectedRange uiView.selectedRange = NSRange(location: range.location + tagModel.diff, length: range.length) tagModel.diff = 0 @@ -52,16 +53,18 @@ struct TextViewWrapper: UIViewRepresentable { } func makeCoordinator() -> Coordinator { - Coordinator(attributedText: $attributedText, getFocusWordForMention: getFocusWordForMention) + Coordinator(attributedText: $attributedText, getFocusWordForMention: getFocusWordForMention, updateCursorPosition: updateCursorPosition) } class Coordinator: NSObject, UITextViewDelegate { @Binding var attributedText: NSMutableAttributedString var getFocusWordForMention: ((String?, NSRange?) -> Void)? = nil + let updateCursorPosition: ((Int) -> Void) - init(attributedText: Binding, getFocusWordForMention: ((String?, NSRange?) -> Void)?) { + init(attributedText: Binding, getFocusWordForMention: ((String?, NSRange?) -> Void)?, updateCursorPosition: @escaping ((Int) -> Void)) { _attributedText = attributedText self.getFocusWordForMention = getFocusWordForMention + self.updateCursorPosition = updateCursorPosition } func textViewDidChange(_ textView: UITextView) { @@ -96,6 +99,65 @@ struct TextViewWrapper: UIViewRepresentable { } return NSRange(location: startOffset, length: length) } + + // This `UITextViewDelegate` method is automatically called by the editor when edits occur, to check whether a change should occur + // We will use this method to manually handle edits concerning mention ("@") links, to avoid manual text edits to attributed mention links + func textView(_ textView: UITextView, shouldChangeTextIn range: NSRange, replacementText text: String) -> Bool { + guard let attributedString = textView.attributedText else { + return true // If we cannot get an attributed string, just fail gracefully and allow changes + } + var mutable = NSMutableAttributedString(attributedString: attributedString) + + let entireRange = NSRange(location: 0, length: attributedString.length) + var shouldAllowChange = true + var performEditActionManually = false + + attributedString.enumerateAttribute(.link, in: entireRange, options: []) { (value, linkRange, stop) in + guard value != nil else { + return // This range is not a link. Skip checking. + } + + if range.contains(linkRange.upperBound) && range.contains(linkRange.lowerBound) { + // Edit range engulfs all of this link's range. + // This link will naturally disappear, so no work needs to be done in this range. + return + } + else if linkRange.intersection(range) != nil { + // If user tries to change an existing link directly, remove the link attribute + mutable.removeAttribute(.link, range: linkRange) + // Perform action manually to flush above changes to the view, and to prevent the character being added from having an attributed link property + performEditActionManually = true + return + } + else if range.location == linkRange.location + linkRange.length && range.length == 0 { + // If we are inserting a character at the right edge of a link, UITextInput tends to include the new character inside the link. + // Therefore, we need to manually append that character outside of the link + performEditActionManually = true + return + } + } + + if performEditActionManually { + shouldAllowChange = false + addUnattributedText(text, to: &mutable, inRange: range) + attributedText = mutable + + // Move caret to the end of the newly changed text. + updateCursorPosition(range.location + text.count) + } + + return shouldAllowChange + } + + func addUnattributedText(_ text: String, to attributedString: inout NSMutableAttributedString, inRange range: NSRange) { + if range.length == 0 { + attributedString.insert(NSAttributedString(string: text, attributes: nil), at: range.location) + } + else { + attributedString.replaceCharacters(in: range, with: text) + } + } + } } diff --git a/damusTests/PostViewTests.swift b/damusTests/PostViewTests.swift new file mode 100644 index 0000000000..bab02ac364 --- /dev/null +++ b/damusTests/PostViewTests.swift @@ -0,0 +1,143 @@ +// +// PostViewTests.swift +// damusTests +// +// Created by Daniel D’Aquino on 2023-08-19. +// +import Foundation +import XCTest +@testable import damus +import SwiftUI + +final class PostViewTests: XCTestCase { + + override func setUpWithError() throws { + // Put setup code here. This method is called before the invocation of each test method in the class. + } + + override func tearDownWithError() throws { + // Put teardown code here. This method is called after the invocation of each test method in the class. + } + + /// Based on https://github.com/damus-io/damus/issues/1375 + /// Tests whether the editor properly handles mention links after they have been added, to avoid manual editing of attributed links + func testMentionLinkEditorHandling() throws { + var content: NSMutableAttributedString + + // Test normal insertion + checkMentionLinkEditorHandling(content: NSMutableAttributedString(string: "Hello"), replacementText: "@", replacementRange: NSRange(location: 0, length: 0), shouldBeAbleToChangeAutomatically: true) + checkMentionLinkEditorHandling(content: NSMutableAttributedString(string: "Hello "), replacementText: "@", replacementRange: NSRange(location: 6, length: 0), shouldBeAbleToChangeAutomatically: true) + checkMentionLinkEditorHandling(content: NSMutableAttributedString(string: "Helo "), replacementText: "l", replacementRange: NSRange(location: 3, length: 0), shouldBeAbleToChangeAutomatically: true) + + // Test normal backspacing + checkMentionLinkEditorHandling(content: NSMutableAttributedString(string: "Hello"), replacementText: "", replacementRange: NSRange(location: 5, length: 1), shouldBeAbleToChangeAutomatically: true) + checkMentionLinkEditorHandling(content: NSMutableAttributedString(string: "Hello "), replacementText: "", replacementRange: NSRange(location: 6, length: 1), shouldBeAbleToChangeAutomatically: true) + checkMentionLinkEditorHandling(content: NSMutableAttributedString(string: "Helo "), replacementText: "", replacementRange: NSRange(location: 3, length: 1), shouldBeAbleToChangeAutomatically: true) + + // Test normal insertion after mention link + content = NSMutableAttributedString(string: "Hello @user ") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 5)) + checkMentionLinkEditorHandling(content: content, replacementText: "a", replacementRange: NSRange(location: 12, length: 0), shouldBeAbleToChangeAutomatically: true) + + // Test insertion right at the end of a mention link, at the end of the text + content = NSMutableAttributedString(string: "Hello @user") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 5)) + checkMentionLinkEditorHandling(content: content, replacementText: ",", replacementRange: NSRange(location: 11, length: 0), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 12, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello @user,") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 11, effectiveRange: nil)) + }) + + // Test insertion right at the end of a mention link, in the middle of the text + content = NSMutableAttributedString(string: "Hello @user how are you?") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 5)) + checkMentionLinkEditorHandling(content: content, replacementText: ",", replacementRange: NSRange(location: 11, length: 0), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 12, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello @user, how are you?") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 11, effectiveRange: nil)) + }) + + // Test insertion in the middle of a mention link to check if the link is removed + content = NSMutableAttributedString(string: "Hello @user how are you?") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 5)) + checkMentionLinkEditorHandling(content: content, replacementText: "a", replacementRange: NSRange(location: 8, length: 0), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 9, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello @uaser how are you?") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 8, effectiveRange: nil)) + }) + + // Test insertion in the middle of a mention link to check if the link is removed, at the end of the text + content = NSMutableAttributedString(string: "Hello @user") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 5)) + checkMentionLinkEditorHandling(content: content, replacementText: "a", replacementRange: NSRange(location: 8, length: 0), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 9, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello @uaser") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 8, effectiveRange: nil)) + }) + + // Test backspacing right at the end of a mention link, at the end of the text + content = NSMutableAttributedString(string: "Hello @user") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 5)) + checkMentionLinkEditorHandling(content: content, replacementText: "", replacementRange: NSRange(location: 10, length: 1), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 10, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello @use") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 6, effectiveRange: nil)) + }) + + // Test adding text right at the start of a mention link, to check that the link is removed + content = NSMutableAttributedString(string: "Hello @user") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 5)) + checkMentionLinkEditorHandling(content: content, replacementText: "a", replacementRange: NSRange(location: 6, length: 0), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 7, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello a@user") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 7, effectiveRange: nil)) + }) + + // Test that removing one link does not affect the other + content = NSMutableAttributedString(string: "Hello @user1 @user2") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 6)) + content.addAttribute(.link, value: "damus:5678", range: NSRange(location: 13, length: 6)) + checkMentionLinkEditorHandling(content: content, replacementText: "", replacementRange: NSRange(location: 18, length: 1), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 18, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello @user1 @user") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 13, effectiveRange: nil)) + XCTAssertNotNil(newManuallyEditedContent.attribute(.link, at: 6, effectiveRange: nil)) + }) + + // Test that replacing a whole range intersecting with two links removes both links + content = NSMutableAttributedString(string: "Hello @user1 @user2") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 6)) + content.addAttribute(.link, value: "damus:5678", range: NSRange(location: 13, length: 6)) + checkMentionLinkEditorHandling(content: content, replacementText: "a", replacementRange: NSRange(location: 10, length: 4), shouldBeAbleToChangeAutomatically: false, expectedNewCursorIndex: 11, handleNewContent: { newManuallyEditedContent in + XCTAssertEqual(newManuallyEditedContent.string, "Hello @useauser2") + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 6, effectiveRange: nil)) + XCTAssertNil(newManuallyEditedContent.attribute(.link, at: 11, effectiveRange: nil)) + }) + + // Test that replacing a whole range including two links removes both links naturally + content = NSMutableAttributedString(string: "Hello @user1 @user2, how are you?") + content.addAttribute(.link, value: "damus:1234", range: NSRange(location: 6, length: 6)) + content.addAttribute(.link, value: "damus:5678", range: NSRange(location: 13, length: 6)) + checkMentionLinkEditorHandling(content: content, replacementText: "", replacementRange: NSRange(location: 5, length: 28), shouldBeAbleToChangeAutomatically: true) + + } +} + +func checkMentionLinkEditorHandling( + content: NSMutableAttributedString, + replacementText: String, + replacementRange: NSRange, + shouldBeAbleToChangeAutomatically: Bool, + expectedNewCursorIndex: Int? = nil, + handleNewContent: ((NSMutableAttributedString) -> Void)? = nil) { + let bindingContent: Binding = Binding(get: { + return content + }, set: { newValue in + handleNewContent?(newValue) + }) + let coordinator: TextViewWrapper.Coordinator = TextViewWrapper.Coordinator(attributedText: bindingContent, getFocusWordForMention: nil, updateCursorPosition: { newCursorIndex in + if let expectedNewCursorIndex { + XCTAssertEqual(newCursorIndex, expectedNewCursorIndex) + } + }) + let textView = UITextView() + textView.attributedText = content + + XCTAssertEqual(coordinator.textView(textView, shouldChangeTextIn: replacementRange, replacementText: replacementText), shouldBeAbleToChangeAutomatically, "Expected shouldChangeTextIn to return \(shouldBeAbleToChangeAutomatically), but was \(!shouldBeAbleToChangeAutomatically)") +} + + + From 9bfb59c4cc39f6acc1a4a8b4041c91fa3eef2ff3 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 21 Aug 2023 13:25:24 -0700 Subject: [PATCH 012/111] docs: people like centralized tools This was confusing people, make it clear that github PRs are fine --- docs/CONTRIBUTING.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index e5a9d66bad..069eeb181f 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,9 +1,5 @@ # Contributing -You can use github PRs to submit code but it is not encouraged. Damus is -a decentralized social media protocol and we prefer to use decentralized -techniques during the code submission process. - [Email patches][git-send-email] to patches@damus.io are preferred, but we accept PRs on GitHub as well. Patches sent via email may include a bolt11 lightning invoice, choosing the price you think the patch is worth, and From 6ab893a617acc9c75cb0dae4d2d530f7b0118a42 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 21 Aug 2023 10:21:40 -0700 Subject: [PATCH 013/111] profile: remove redundant view builder --- damus/Views/Profile/ProfileView.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/damus/Views/Profile/ProfileView.swift b/damus/Views/Profile/ProfileView.swift index 2695e10c66..9860e6635f 100644 --- a/damus/Views/Profile/ProfileView.swift +++ b/damus/Views/Profile/ProfileView.swift @@ -144,7 +144,6 @@ struct ProfileView: View { return 100.0 - (Theme.safeAreaInsets?.top ?? 0) } - @ViewBuilder func navImage(img: String) -> some View { Image(img) .frame(width: 33, height: 33) From 286ae68fd63754fbf5a9f4b271c207e9d2df1728 Mon Sep 17 00:00:00 2001 From: Grimless Date: Mon, 21 Aug 2023 17:17:21 -0400 Subject: [PATCH 014/111] Move the Block helper type to its own file Collapse the various standalone functions for parsing block data, and refactor consumers to initialize a Block with given data and access its members as needed. Closes: https://github.com/damus-io/damus/pull/1496 Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 + damus/ContentParsing.swift | 2 +- damus/Models/Mentions.swift | 227 +----------------------------- damus/Types/Block.swift | 214 ++++++++++++++++++++++++++++ damus/Util/Zap.swift | 2 +- damus/Views/DMChatView.swift | 4 +- damus/Views/NoteContentView.swift | 10 +- nostrdb/NdbNote.swift | 10 +- 8 files changed, 244 insertions(+), 229 deletions(-) create mode 100644 damus/Types/Block.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 3621badec0..3e9d71bd91 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -371,6 +371,7 @@ 643EA5C8296B764E005081BB /* RelayFilterView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 643EA5C7296B764E005081BB /* RelayFilterView.swift */; }; 647D9A8D2968520300A295DE /* SideMenuView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 647D9A8C2968520300A295DE /* SideMenuView.swift */; }; 64FBD06F296255C400D9D3B2 /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64FBD06E296255C400D9D3B2 /* Theme.swift */; }; + 7527271E2A93FF0100214108 /* Block.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7527271D2A93FF0100214108 /* Block.swift */; }; 7C60CAEF298471A1009C80D6 /* CoreSVG.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */; }; 7C902AE32981D55B002AB16E /* ZoomableScrollView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */; }; 7C95CAEE299DCEF1009DCB67 /* KFOptionSetter+.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */; }; @@ -925,6 +926,7 @@ 643EA5C7296B764E005081BB /* RelayFilterView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterView.swift; sourceTree = ""; }; 647D9A8C2968520300A295DE /* SideMenuView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SideMenuView.swift; sourceTree = ""; }; 64FBD06E296255C400D9D3B2 /* Theme.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Theme.swift; sourceTree = ""; }; + 7527271D2A93FF0100214108 /* Block.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Block.swift; sourceTree = ""; }; 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreSVG.swift; sourceTree = ""; }; 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ZoomableScrollView.swift; sourceTree = ""; }; 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KFOptionSetter+.swift"; sourceTree = ""; }; @@ -1578,6 +1580,7 @@ isa = PBXGroup; children = ( 4CC14FED2A73FCBB007AEB17 /* Ids */, + 7527271D2A93FF0100214108 /* Block.swift */, ); path = Types; sourceTree = ""; @@ -2388,6 +2391,7 @@ 7CFF6317299FEFE5005D382A /* SelectableText.swift in Sources */, 4CA352A82A76B37E003BB08B /* NewMutesNotify.swift in Sources */, 4CFF8F6929CC9ED1008DB934 /* ImageContainerView.swift in Sources */, + 7527271E2A93FF0100214108 /* Block.swift in Sources */, 4C54AA0729A540BA003E4487 /* NotificationsModel.swift in Sources */, 4C12536C2A76D4B00004F4B8 /* RepostedNotify.swift in Sources */, 4CB55EF3295E5D59007FD187 /* RecommendedRelayView.swift in Sources */, diff --git a/damus/ContentParsing.swift b/damus/ContentParsing.swift index 6f2f389ddf..1362f39df4 100644 --- a/damus/ContentParsing.swift +++ b/damus/ContentParsing.swift @@ -27,7 +27,7 @@ func parsed_blocks_finish(bs: inout note_blocks, tags: TagsSequence?) -> Blocks while (i < bs.num_blocks) { let block = bs.blocks[i] - if let converted = convert_block(block, tags: tags) { + if let converted = Block(block, tags: tags) { out.append(converted) } diff --git a/damus/Models/Mentions.swift b/damus/Models/Mentions.swift index d6cc781add..2132ca8ed4 100644 --- a/damus/Models/Mentions.swift +++ b/damus/Models/Mentions.swift @@ -123,147 +123,11 @@ struct LightningInvoice { } } -enum Block: Equatable { - static func == (lhs: Block, rhs: Block) -> Bool { - switch (lhs, rhs) { - case (.text(let a), .text(let b)): - return a == b - case (.mention(let a), .mention(let b)): - return a == b - case (.hashtag(let a), .hashtag(let b)): - return a == b - case (.url(let a), .url(let b)): - return a == b - case (.invoice(let a), .invoice(let b)): - return a.string == b.string - case (_, _): - return false - } - } - - case text(String) - case mention(Mention) - case hashtag(String) - case url(URL) - case invoice(Invoice) - case relay(String) - - var is_invoice: Invoice? { - if case .invoice(let invoice) = self { - return invoice - } - return nil - } - - var is_hashtag: String? { - if case .hashtag(let htag) = self { - return htag - } - return nil - } - - var is_url: URL? { - if case .url(let url) = self { - return url - } - - return nil - } - - var is_text: String? { - if case .text(let txt) = self { - return txt - } - return nil - } - - var is_note_mention: Bool { - if case .mention(let mention) = self, - case .note = mention.ref { - return true - } - return false - } - - var is_mention: Mention? { - if case .mention(let m) = self { - return m - } - return nil - } -} - -func render_blocks(blocks: [Block]) -> String { - return blocks.reduce("") { str, block in - switch block { - case .mention(let m): - if let idx = m.index { - return str + "#[\(idx)]" - } - - switch m.ref { - case .pubkey(let pk): return str + "nostr:\(pk.npub)" - case .note(let note_id): return str + "nostr:\(note_id.bech32)" - } - case .relay(let relay): - return str + relay - case .text(let txt): - return str + txt - case .hashtag(let htag): - return str + "#" + htag - case .url(let url): - return str + url.absoluteString - case .invoice(let inv): - return str + inv.string - } - } -} - struct Blocks: Equatable { let words: Int let blocks: [Block] } -func strblock_to_string(_ s: str_block_t) -> String? { - let len = s.end - s.start - let bytes = Data(bytes: s.start, count: len) - return String(bytes: bytes, encoding: .utf8) -} - -func convert_block(_ b: block_t, tags: TagsSequence?) -> Block? { - if b.type == BLOCK_HASHTAG { - guard let str = strblock_to_string(b.block.str) else { - return nil - } - return .hashtag(str) - } else if b.type == BLOCK_TEXT { - guard let str = strblock_to_string(b.block.str) else { - return nil - } - return .text(str) - } else if b.type == BLOCK_MENTION_INDEX { - return convert_mention_index_block(ind: Int(b.block.mention_index), tags: tags) - } else if b.type == BLOCK_URL { - return convert_url_block(b.block.str) - } else if b.type == BLOCK_INVOICE { - return convert_invoice_block(b.block.invoice) - } else if b.type == BLOCK_MENTION_BECH32 { - return convert_mention_bech32_block(b.block.mention_bech32) - } - - return nil -} - -func convert_url_block(_ b: str_block) -> Block? { - guard let str = strblock_to_string(b) else { - return nil - } - guard let url = URL(string: str) else { - return .text(str) - } - return .url(url) -} - func maybe_pointee(_ p: UnsafeMutablePointer!) -> T? { guard p != nil else { return nil @@ -326,75 +190,6 @@ func format_msats(_ msat: Int64, locale: Locale = Locale.current) -> String { return String(format: format, locale: locale, sats.decimalValue as NSDecimalNumber, formattedSats) } -func convert_invoice_block(_ b: invoice_block) -> Block? { - guard let invstr = strblock_to_string(b.invstr) else { - return nil - } - - guard var b11 = maybe_pointee(b.bolt11) else { - return nil - } - - guard let description = convert_invoice_description(b11: b11) else { - return nil - } - - let amount: Amount = maybe_pointee(b11.msat).map { .specific(Int64($0.millisatoshis)) } ?? .any - let payment_hash = Data(bytes: &b11.payment_hash, count: 32) - let created_at = b11.timestamp - - tal_free(b.bolt11) - return .invoice(Invoice(description: description, amount: amount, string: invstr, expiry: b11.expiry, payment_hash: payment_hash, created_at: created_at)) -} - -func convert_mention_bech32_block(_ b: mention_bech32_block) -> Block? -{ - switch b.bech32.type { - case NOSTR_BECH32_NOTE: - let note = b.bech32.data.note; - let note_id = NoteId(Data(bytes: note.event_id, count: 32)) - return .mention(.any(.note(note_id))) - - case NOSTR_BECH32_NEVENT: - let nevent = b.bech32.data.nevent; - let note_id = NoteId(Data(bytes: nevent.event_id, count: 32)) - return .mention(.any(.note(note_id))) - - case NOSTR_BECH32_NPUB: - let npub = b.bech32.data.npub - let pubkey = Pubkey(Data(bytes: npub.pubkey, count: 32)) - return .mention(.any(.pubkey(pubkey))) - - case NOSTR_BECH32_NSEC: - let nsec = b.bech32.data.nsec - let privkey = Privkey(Data(bytes: nsec.nsec, count: 32)) - guard let pubkey = privkey_to_pubkey(privkey: privkey) else { return nil } - return .mention(.any(.pubkey(pubkey))) - - case NOSTR_BECH32_NPROFILE: - let nprofile = b.bech32.data.nprofile - let pubkey = Pubkey(Data(bytes: nprofile.pubkey, count: 32)) - return .mention(.any(.pubkey(pubkey))) - - case NOSTR_BECH32_NRELAY: - let nrelay = b.bech32.data.nrelay - guard let relay_str = strblock_to_string(nrelay.relay) else { - return nil - } - return .relay(relay_str) - - case NOSTR_BECH32_NADDR: - // TODO: wtf do I do with this - guard let naddr = strblock_to_string(b.str) else { - return nil - } - return .text("nostr:" + naddr) - - default: - return nil - } -} - func convert_invoice_description(b11: bolt11) -> InvoiceDescription? { if let desc = b11.description { return .description(String(cString: desc)) @@ -407,24 +202,6 @@ func convert_invoice_description(b11: bolt11) -> InvoiceDescription? { return nil } -func convert_mention_index_block(ind: Int, tags: TagsSequence?) -> Block? -{ - guard let tags, - ind >= 0, - ind + 1 <= tags.count - else { - return .text("#[\(ind)]") - } - - let tag = tags[ind] - - guard let mention = MentionRef.from_tag(tag: tag) else { - return .text("#[\(ind)]") - } - - return .mention(.any(mention, index: ind)) -} - func find_tag_ref(type: String, id: String, tags: [[String]]) -> Int? { var i: Int = 0 for tag in tags { @@ -474,7 +251,9 @@ func post_to_event(post: NostrPost, keypair: FullKeypair) -> NostrEvent? { let tags = post.references.map({ r in r.tag }) + post.tags let post_blocks = parse_post_blocks(content: post.content) let post_tags = make_post_tags(post_blocks: post_blocks, tags: tags) - let content = render_blocks(blocks: post_tags.blocks) + let content = post_tags.blocks + .map(\.asString) + .joined(separator: "") return NostrEvent(content: content, keypair: keypair.to_keypair(), kind: post.kind.rawValue, tags: post_tags.tags) } diff --git a/damus/Types/Block.swift b/damus/Types/Block.swift new file mode 100644 index 0000000000..e478f6007b --- /dev/null +++ b/damus/Types/Block.swift @@ -0,0 +1,214 @@ +// +// Block.swift +// damus +// +// Created by Kyle Roucis on 2023-08-21. +// + +import Foundation + + +fileprivate extension String { + /// Failable initializer to build a Swift.String from a C-backed `str_block_t`. + init?(_ s: str_block_t) { + let len = s.end - s.start + let bytes = Data(bytes: s.start, count: len) + self.init(bytes: bytes, encoding: .utf8) + } +} + +/// Represents a block of data stored by the NOSTR protocol. This can be +/// simple text, a hashtag, a url, a relay reference, a mention ref and +/// potentially more in the future. +enum Block: Equatable { + static func == (lhs: Block, rhs: Block) -> Bool { + switch (lhs, rhs) { + case (.text(let a), .text(let b)): + return a == b + case (.mention(let a), .mention(let b)): + return a == b + case (.hashtag(let a), .hashtag(let b)): + return a == b + case (.url(let a), .url(let b)): + return a == b + case (.invoice(let a), .invoice(let b)): + return a.string == b.string + case (_, _): + return false + } + } + + case text(String) + case mention(Mention) + case hashtag(String) + case url(URL) + case invoice(Invoice) + case relay(String) +} +extension Block { + /// Failable initializer for the C-backed type `block_t`. This initializer will inspect + /// the underlying block type and build the appropriate enum value as needed. + init?(_ block: block_t, tags: TagsSequence? = nil) { + switch block.type { + case BLOCK_HASHTAG: + guard let str = String(block.block.str) else { + return nil + } + self = .hashtag(str) + case BLOCK_TEXT: + guard let str = String(block.block.str) else { + return nil + } + self = .text(str) + case BLOCK_MENTION_INDEX: + guard let b = Block(index: Int(block.block.mention_index), tags: tags) else { + return nil + } + self = b + case BLOCK_URL: + guard let b = Block(block.block.str) else { + return nil + } + self = b + case BLOCK_INVOICE: + guard let b = Block(invoice: block.block.invoice) else { + return nil + } + self = b + case BLOCK_MENTION_BECH32: + guard let b = Block(bech32: block.block.mention_bech32) else { + return nil + } + self = b + default: + return nil + } + } +} +fileprivate extension Block { + /// Failable initializer for the C-backed type `str_block_t`. + init?(_ b: str_block_t) { + guard let str = String(b) else { + return nil + } + + if let url = URL(string: str) { + self = .url(url) + } + else { + self = .text(str) + } + } +} +fileprivate extension Block { + /// Failable initializer for a block index and a tag sequence. + init?(index: Int, tags: TagsSequence? = nil) { + guard let tags, + index >= 0, + index + 1 <= tags.count + else { + self = .text("#[\(index)]") + return + } + + let tag = tags[index] + + if let mention = MentionRef.from_tag(tag: tag) { + self = .mention(.any(mention, index: index)) + } + else { + self = .text("#[\(index)]") + } + } +} +fileprivate extension Block { + /// Failable initializer for the C-backed type `invoice_block_t`. + init?(invoice: invoice_block_t) { + guard let invstr = String(invoice.invstr) else { + return nil + } + + guard var b11 = maybe_pointee(invoice.bolt11) else { + return nil + } + + guard let description = convert_invoice_description(b11: b11) else { + return nil + } + + let amount: Amount = maybe_pointee(b11.msat).map { .specific(Int64($0.millisatoshis)) } ?? .any + let payment_hash = Data(bytes: &b11.payment_hash, count: 32) + let created_at = b11.timestamp + + tal_free(invoice.bolt11) + self = .invoice(Invoice(description: description, amount: amount, string: invstr, expiry: b11.expiry, payment_hash: payment_hash, created_at: created_at)) + } +} +fileprivate extension Block { + /// Failable initializer for the C-backed type `mention_bech32_block_t`. This initializer will inspect the + /// bech32 type code and build the appropriate enum type. + init?(bech32 b: mention_bech32_block_t) { + switch b.bech32.type { + case NOSTR_BECH32_NOTE: + let note = b.bech32.data.note; + let note_id = NoteId(Data(bytes: note.event_id, count: 32)) + self = .mention(.any(.note(note_id))) + case NOSTR_BECH32_NEVENT: + let nevent = b.bech32.data.nevent; + let note_id = NoteId(Data(bytes: nevent.event_id, count: 32)) + self = .mention(.any(.note(note_id))) + case NOSTR_BECH32_NPUB: + let npub = b.bech32.data.npub + let pubkey = Pubkey(Data(bytes: npub.pubkey, count: 32)) + self = .mention(.any(.pubkey(pubkey))) + case NOSTR_BECH32_NSEC: + let nsec = b.bech32.data.nsec + let privkey = Privkey(Data(bytes: nsec.nsec, count: 32)) + guard let pubkey = privkey_to_pubkey(privkey: privkey) else { return nil } + self = .mention(.any(.pubkey(pubkey))) + case NOSTR_BECH32_NPROFILE: + let nprofile = b.bech32.data.nprofile + let pubkey = Pubkey(Data(bytes: nprofile.pubkey, count: 32)) + self = .mention(.any(.pubkey(pubkey))) + case NOSTR_BECH32_NRELAY: + let nrelay = b.bech32.data.nrelay + guard let relay_str = String(nrelay.relay) else { + return nil + } + self = .relay(relay_str) + case NOSTR_BECH32_NADDR: + // TODO: wtf do I do with this + guard let naddr = String(b.str) else { + return nil + } + self = .text("nostr:" + naddr) + default: + return nil + } + } +} +extension Block { + var asString: String { + switch self { + case .mention(let m): + if let idx = m.index { + return "#[\(idx)]" + } + + switch m.ref { + case .pubkey(let pk): return "nostr:\(pk.npub)" + case .note(let note_id): return "nostr:\(note_id.bech32)" + } + case .relay(let relay): + return relay + case .text(let txt): + return txt + case .hashtag(let htag): + return "#" + htag + case .url(let url): + return url.absoluteString + case .invoice(let inv): + return inv.string + } + } +} diff --git a/damus/Util/Zap.swift b/damus/Util/Zap.swift index 7749de723d..931810a99e 100644 --- a/damus/Util/Zap.swift +++ b/damus/Util/Zap.swift @@ -393,7 +393,7 @@ func decode_bolt11(_ s: String) -> Invoice? { let block = bs.blocks[0] - guard let converted = convert_block(block, tags: nil) else { + guard let converted = Block(block) else { blocks_free(&bs) return nil } diff --git a/damus/Views/DMChatView.swift b/damus/Views/DMChatView.swift index a905c68a32..18bf288fc1 100644 --- a/damus/Views/DMChatView.swift +++ b/damus/Views/DMChatView.swift @@ -130,7 +130,9 @@ struct DMChatView: View, KeyboardReadable { func send_message() { let tags = [["p", pubkey.hex()]] let post_blocks = parse_post_blocks(content: dms.draft) - let content = render_blocks(blocks: post_blocks) + let content = post_blocks + .map(\.asString) + .joined(separator: "") guard let dm = create_dm(content, to_pk: pubkey, tags: tags, keypair: damus_state.keypair) else { print("error creating dm") diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index 82069515ae..9e4539ec1a 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -445,7 +445,15 @@ func render_blocks(blocks bs: Blocks, profiles: Profiles) -> NoteArtifactsSepara let blocks = bs.blocks let one_note_ref = blocks - .filter({ $0.is_note_mention }) + .filter({ + if case .mention(let mention) = $0, + case .note = mention.ref { + return true + } + else { + return false + } + }) .count == 1 var ind: Int = -1 diff --git a/nostrdb/NdbNote.swift b/nostrdb/NdbNote.swift index 4b54253cff..efb4e25605 100644 --- a/nostrdb/NdbNote.swift +++ b/nostrdb/NdbNote.swift @@ -415,7 +415,15 @@ extension NdbNote { // Rely on Apple's NLLanguageRecognizer to tell us which language it thinks the note is in // and filter on only the text portions of the content as URLs and hashtags confuse the language recognizer. let originalBlocks = self.blocks(privkey).blocks - let originalOnlyText = originalBlocks.compactMap { $0.is_text }.joined(separator: " ") + let originalOnlyText = originalBlocks.compactMap { + if case .text(let txt) = $0 { + return txt + } + else { + return nil + } + } + .joined(separator: " ") // Only accept language recognition hypothesis if there's at least a 50% probability that it's accurate. let languageRecognizer = NLLanguageRecognizer() From 8b600a977400519ad7f01788f3bb36b66788e122 Mon Sep 17 00:00:00 2001 From: "tappu75e@duck.com" Date: Sun, 20 Aug 2023 23:25:54 +0300 Subject: [PATCH 015/111] Avoid notification for zap from mute profiles Changelog-Fixed: Avoid notification for zaps from muted profiles Closes: https://github.com/damus-io/damus/pull/1494 Signed-off-by: William Casarin --- damus/Models/HomeModel.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index b06669711f..a0210a75cd 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -228,6 +228,10 @@ class HomeModel { guard zap.target.pubkey == self.damus_state.keypair.pubkey else { return } + + guard should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: zap.request.ev) else { + return + } if !self.notifications.insert_zap(.zap(zap)) { return From c5d8e4a4a14e71113002e04f60f7ceb1a0550945 Mon Sep 17 00:00:00 2001 From: Grimless Date: Tue, 22 Aug 2023 00:00:46 -0400 Subject: [PATCH 016/111] Simplify and inline Report event logic. Closes: https://github.com/damus-io/damus/pull/1498 Signed-off-by: William Casarin --- damus/Models/Report.swift | 26 -------------------------- damus/Views/Events/EventMenu.swift | 3 +-- damus/Views/ReportView.swift | 25 +++++++++++++++---------- 3 files changed, 16 insertions(+), 38 deletions(-) diff --git a/damus/Models/Report.swift b/damus/Models/Report.swift index 3414125a52..c293e600d3 100644 --- a/damus/Models/Report.swift +++ b/damus/Models/Report.swift @@ -38,31 +38,5 @@ struct ReportNoteTarget { enum ReportTarget { case user(Pubkey) case note(ReportNoteTarget) - - static func note(pubkey: Pubkey, note_id: NoteId) -> ReportTarget { - return .note(ReportNoteTarget(pubkey: pubkey, note_id: note_id)) - } -} - -struct Report { - let type: ReportType - let target: ReportTarget - let message: String - } -func create_report_tags(target: ReportTarget, type: ReportType) -> [[String]] { - switch target { - case .user(let pubkey): - return [["p", pubkey.hex(), type.rawValue]] - case .note(let notet): - return [["e", notet.note_id.hex(), type.rawValue], - ["p", notet.pubkey.hex()]] - } -} - -func create_report_event(keypair: FullKeypair, report: Report) -> NostrEvent? { - let kind: UInt32 = 1984 - let tags = create_report_tags(target: report.target, type: report.type) - return NostrEvent(content: report.message, keypair: keypair.to_keypair(), kind: kind, tags: tags) -} diff --git a/damus/Views/Events/EventMenu.swift b/damus/Views/Events/EventMenu.swift index 5c194f791e..6b6c8b39ac 100644 --- a/damus/Views/Events/EventMenu.swift +++ b/damus/Views/Events/EventMenu.swift @@ -66,7 +66,6 @@ struct MenuItems: View { } var body: some View { - Group { Button { UIPasteboard.general.string = event.get_content(keypair.privkey) @@ -126,7 +125,7 @@ struct MenuItems: View { // Only allow reporting if logged in with private key and the currently viewed profile is not the logged in profile. if keypair.pubkey != target_pubkey && keypair.privkey != nil { Button(role: .destructive) { - notify(.report(.note(pubkey: target_pubkey, note_id: event.id))) + notify(.report(.note(ReportNoteTarget(pubkey: target_pubkey, note_id: event.id)))) } label: { Label(NSLocalizedString("Report", comment: "Context menu option for reporting content."), image: "raising-hand") } diff --git a/damus/Views/ReportView.swift b/damus/Views/ReportView.swift index 517161918a..1f7e990bfc 100644 --- a/damus/Views/ReportView.swift +++ b/damus/Views/ReportView.swift @@ -7,6 +7,18 @@ import SwiftUI +fileprivate extension ReportTarget { + func reportTags(type: ReportType) -> [[String]] { + switch self { + case .user(let pubkey): + return [["p", pubkey.hex(), type.rawValue]] + case .note(let notet): + return [["e", notet.note_id.hex(), type.rawValue], + ["p", notet.pubkey.hex()]] + } + } +} + struct ReportView: View { let postbox: PostBox let target: ReportTarget @@ -47,10 +59,12 @@ struct ReportView: View { func do_send_report() { guard let selected_report_type, - let ev = send_report(keypair: keypair, postbox: postbox, target: target, type: selected_report_type, message: report_message) else { + let ev = NostrEvent(content: report_message, keypair: keypair.to_keypair(), kind: 1984, tags: target.reportTags(type: selected_report_type)) else { return } + postbox.send(ev) + report_sent = true report_id = bech32_note_id(ev.id) } @@ -113,15 +127,6 @@ struct ReportView: View { } } -func send_report(keypair: FullKeypair, postbox: PostBox, target: ReportTarget, type: ReportType, message: String) -> NostrEvent? { - let report = Report(type: type, target: target, message: message) - guard let ev = create_report_event(keypair: keypair, report: report) else { - return nil - } - postbox.send(ev) - return ev -} - struct ReportView_Previews: PreviewProvider { static var previews: some View { let ds = test_damus_state() From 5caa4a6e975bb05141cdefcb761fa3ea282db85f Mon Sep 17 00:00:00 2001 From: gladiusKatana Date: Wed, 21 Jun 2023 03:11:50 -0400 Subject: [PATCH 017/111] videos: improve precision & sensitivity of auto-pause mechanism Closes: https://github.com/damus-io/damus/pull/1308 Signed-off-by: William Casarin --- damus/Views/Video/DamusVideoPlayer.swift | 28 +++++++++++++++++------- damus/damusApp.swift | 17 ++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/damus/Views/Video/DamusVideoPlayer.swift b/damus/Views/Video/DamusVideoPlayer.swift index 2ab8d3d124..ca64e42edd 100644 --- a/damus/Views/Video/DamusVideoPlayer.swift +++ b/damus/Views/Video/DamusVideoPlayer.swift @@ -7,10 +7,20 @@ import SwiftUI +/// get coordinates in Global reference frame given a Local point & geometry +func globalCoordinate(localX x: CGFloat, localY y: CGFloat, + localGeometry geo: GeometryProxy) -> CGPoint { + let localPoint = CGPoint(x: x, y: y) + return geo.frame(in: .global).origin.applying( + .init(translationX: localPoint.x, y: localPoint.y) + ) +} + struct DamusVideoPlayer: View { var url: URL @ObservedObject var model: VideoPlayerModel @Binding var video_size: CGSize? + @EnvironmentObject private var orientationTracker: OrientationTracker var mute_icon: String { if model.has_audio == false || model.muted { @@ -45,10 +55,8 @@ struct DamusVideoPlayer: View { var body: some View { GeometryReader { geo in let localFrame = geo.frame(in: .local) - let localCenter = CGPoint(x: localFrame.midX, y: localFrame.midY) - let globalCenter = geo.frame(in: .global).origin.applying(.init(translationX: localCenter.x, y: localCenter.y)) - let centerY = globalCenter.y - + let centerY = globalCoordinate(localX: 0, localY: localFrame.midY, localGeometry: geo).y + let delta = localFrame.height / 2 ZStack(alignment: .bottomTrailing) { VideoPlayer(url: url, model: model) if model.has_audio == true { @@ -66,10 +74,14 @@ struct DamusVideoPlayer: View { video_size = size } .onChange(of: centerY) { _ in - let screenHeight = UIScreen.main.bounds.height - let screenMidY = screenHeight / 2 - let tol = 0.20 * screenHeight /// tolerance - can vary to taste ie., % of screen height of a centered box in which video plays - model.play = centerY > screenMidY - tol && centerY < screenMidY + tol /// video plays when inside tolerance box + /// pause video when it is scrolled beyond visible range + let isBelowTop = centerY + delta > 100, /// 100 =~ approx. bottom (y) of ContentView's TabView + isAboveBottom = centerY - delta < orientationTracker.deviceMajorAxis + if isBelowTop && isAboveBottom { + model.start() + } else { + model.stop() + } } } } diff --git a/damus/damusApp.swift b/damus/damusApp.swift index 08c5352162..7300b1b0e3 100644 --- a/damus/damusApp.swift +++ b/damus/damusApp.swift @@ -20,11 +20,13 @@ struct damusApp: App { struct MainView: View { @State var needs_setup = false; @State var keypair: Keypair? = nil; + @StateObject private var orientationTracker = OrientationTracker() var body: some View { Group { if let kp = keypair, !needs_setup { ContentView(keypair: kp) + .environmentObject(orientationTracker) } else { SetupView() .onReceive(handle_notify(.login)) { notif in @@ -38,7 +40,11 @@ struct MainView: View { try? clear_keypair() keypair = nil } + .onReceive(NotificationCenter.default.publisher(for: UIDevice.orientationDidChangeNotification)) { _ in + orientationTracker.setDeviceMajorAxis() + } .onAppear { + orientationTracker.setDeviceMajorAxis() keypair = get_saved_keypair() } } @@ -65,3 +71,14 @@ class AppDelegate: NSObject, UIApplicationDelegate, UNUserNotificationCenterDele completionHandler() } } + +class OrientationTracker: ObservableObject { + var deviceMajorAxis: CGFloat = 0 + func setDeviceMajorAxis() { + let bounds = UIScreen.main.bounds + let height = max(bounds.height, bounds.width) /// device's longest dimension + let width = min(bounds.height, bounds.width) /// device's shortest dimension + let orientation = UIDevice.current.orientation + deviceMajorAxis = (orientation == .portrait || orientation == .unknown) ? height : width + } +} From b665a40a11ced8e249a9647cebf4aa3874ae7332 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 09:25:47 -0700 Subject: [PATCH 018/111] fix build --- damus/Models/HomeModel.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index a0210a75cd..41bd95cec9 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -229,7 +229,7 @@ class HomeModel { return } - guard should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: zap.request.ev) else { + guard should_show_event(privkey: self.damus_state.keypair.privkey, hellthreads: self.damus_state.muted_threads, contacts: self.damus_state.contacts, ev: zap.request.ev) else { return } From d34d417fcc3a82693fb41ee09a36e3b346d0971f Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 09:27:09 -0700 Subject: [PATCH 019/111] home: collapse guard statement small nit refactor --- damus/Models/HomeModel.swift | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index 41bd95cec9..b6d33919d4 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -223,13 +223,9 @@ class HomeModel { func handle_zap_event(_ ev: NostrEvent) { process_zap_event(damus_state: damus_state, ev: ev) { zapres in - guard case .done(let zap) = zapres else { return } - - guard zap.target.pubkey == self.damus_state.keypair.pubkey else { - return - } - - guard should_show_event(privkey: self.damus_state.keypair.privkey, hellthreads: self.damus_state.muted_threads, contacts: self.damus_state.contacts, ev: zap.request.ev) else { + guard case .done(let zap) = zapres, + zap.target.pubkey == self.damus_state.keypair.pubkey, + should_show_event(privkey: self.damus_state.keypair.privkey, hellthreads: self.damus_state.muted_threads, contacts: self.damus_state.contacts, ev: zap.request.ev) else { return } From 59cf8056bd1db41da05abefcc93dd19f5df0b40f Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 09:52:28 -0700 Subject: [PATCH 020/111] sidemenu: split out profile section We will be adding to this and it is getting messy --- damus/Views/SideMenuView.swift | 43 +++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/damus/Views/SideMenuView.swift b/damus/Views/SideMenuView.swift index 0851c56679..38a0717d23 100644 --- a/damus/Views/SideMenuView.swift +++ b/damus/Views/SideMenuView.swift @@ -81,32 +81,37 @@ struct SideMenuView: View { } } + var TopProfile: some View { + let profile = damus_state.profiles.lookup(id: damus_state.pubkey) + return HStack { + ProfilePicView(pubkey: damus_state.pubkey, size: 60, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) + + VStack(alignment: .leading) { + if let display_name = profile?.display_name { + Text(display_name) + .foregroundColor(textColor()) + .font(.title) + .lineLimit(1) + } + if let name = profile?.name { + Text("@" + name) + .foregroundColor(DamusColors.mediumGrey) + .font(.body) + .lineLimit(1) + } + } + } + } + var MainSidemenu: some View { VStack(alignment: .leading, spacing: 0) { - let profile = damus_state.profiles.lookup(id: damus_state.pubkey) let followers = FollowersModel(damus_state: damus_state, target: damus_state.pubkey) let profile_model = ProfileModel(pubkey: damus_state.pubkey, damus: damus_state) NavigationLink(value: Route.Profile(profile: profile_model, followers: followers), label: { - HStack { - ProfilePicView(pubkey: damus_state.pubkey, size: 60, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) - VStack(alignment: .leading) { - if let display_name = profile?.display_name { - Text(display_name) - .foregroundColor(textColor()) - .font(.title) - .lineLimit(1) - } - if let name = profile?.name { - Text("@" + name) - .foregroundColor(DamusColors.mediumGrey) - .font(.body) - .lineLimit(1) - } - } - } - .padding(.bottom, verticalSpacing) + TopProfile + .padding(.bottom, verticalSpacing) }) Divider() From 0338297bfe33066137ee3ba2512d0a9c628138d5 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 21 Aug 2023 22:12:01 -0700 Subject: [PATCH 021/111] Live Music & Generic Statuses Changelog-Added: Added live music statuses Changelog-Added: Added generic user statuses --- damus.xcodeproj/project.pbxproj | 32 ++++ .../Status/Music/MusicController.swift | 48 ++++++ damus/Components/Status/UserStatus.swift | 141 ++++++++++++++++++ damus/Components/Status/UserStatusSheet.swift | 116 ++++++++++++++ damus/Components/Status/UserStatusView.swift | 37 +++++ damus/ContentView.swift | 28 +++- damus/Info.plist | 2 + damus/Models/DamusState.swift | 7 +- damus/Models/HomeModel.swift | 16 +- damus/Nostr/NostrKind.swift | 1 + damus/Nostr/Profiles.swift | 67 ++++++--- damus/Views/Events/Components/EventTop.swift | 1 - damus/Views/Events/EventProfile.swift | 7 +- damus/Views/Events/EventShell.swift | 3 +- damus/Views/Events/SelectedEventView.swift | 2 +- damus/Views/Profile/EventProfileName.swift | 2 +- damus/Views/Profile/ProfilePopup.swift | 20 +++ damus/Views/SideMenuView.swift | 64 ++++---- 18 files changed, 538 insertions(+), 56 deletions(-) create mode 100644 damus/Components/Status/Music/MusicController.swift create mode 100644 damus/Components/Status/UserStatus.swift create mode 100644 damus/Components/Status/UserStatusSheet.swift create mode 100644 damus/Components/Status/UserStatusView.swift create mode 100644 damus/Views/Profile/ProfilePopup.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 3e9d71bd91..f0f2ddbf79 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -163,11 +163,14 @@ 4C5D5C992A6AF8F80024563C /* NdbNote.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C90548A2A6AEDEE00811EEC /* NdbNote.swift */; }; 4C5D5C9A2A6AF8F80024563C /* NdbTagIterator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C9054882A6AED4700811EEC /* NdbTagIterator.swift */; }; 4C5D5C9D2A6B2CB40024563C /* AsciiCharacter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C5D5C9C2A6B2CB40024563C /* AsciiCharacter.swift */; }; + 4C5E54032A9522F600FF6E60 /* UserStatus.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C5E54022A9522F600FF6E60 /* UserStatus.swift */; }; + 4C5E54062A9671F800FF6E60 /* UserStatusSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C5E54052A9671F800FF6E60 /* UserStatusSheet.swift */; }; 4C5F9114283D694D0052CD1C /* FollowTarget.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C5F9113283D694D0052CD1C /* FollowTarget.swift */; }; 4C5F9116283D855D0052CD1C /* EventsModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C5F9115283D855D0052CD1C /* EventsModel.swift */; }; 4C5F9118283D88E40052CD1C /* FollowingModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C5F9117283D88E40052CD1C /* FollowingModel.swift */; }; 4C633350283D40E500B1C9C3 /* HomeModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C63334F283D40E500B1C9C3 /* HomeModel.swift */; }; 4C633352283D419F00B1C9C3 /* SignalModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C633351283D419F00B1C9C3 /* SignalModel.swift */; }; + 4C64305C2A945AFF00B0C0E9 /* MusicController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C64305B2A945AFF00B0C0E9 /* MusicController.swift */; }; 4C64987C286D03E000EAE2B3 /* DirectMessagesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C64987B286D03E000EAE2B3 /* DirectMessagesView.swift */; }; 4C64987E286D082C00EAE2B3 /* DirectMessagesModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C64987D286D082C00EAE2B3 /* DirectMessagesModel.swift */; }; 4C649881286E0EE300EAE2B3 /* secp256k1 in Frameworks */ = {isa = PBXBuildFile; productRef = 4C649880286E0EE300EAE2B3 /* secp256k1 */; }; @@ -339,6 +342,7 @@ 4CF0ABEE29844B5500D66079 /* AnyEncodable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABED29844B5500D66079 /* AnyEncodable.swift */; }; 4CF0ABF029857E9200D66079 /* Bech32Object.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABEF29857E9200D66079 /* Bech32Object.swift */; }; 4CF0ABF62985CD5500D66079 /* UserSearch.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABF52985CD5500D66079 /* UserSearch.swift */; }; + 4CF38C882A9442DC00BE01B6 /* UserStatusView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF38C872A9442DC00BE01B6 /* UserStatusView.swift */; }; 4CFD502F2A2DA45800A229DB /* MediaView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CFD502E2A2DA45800A229DB /* MediaView.swift */; }; 4CFF8F6329CC9AD7008DB934 /* ImageContextMenuModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CFF8F6229CC9AD7008DB934 /* ImageContextMenuModifier.swift */; }; 4CFF8F6729CC9E3A008DB934 /* ImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CFF8F6629CC9E3A008DB934 /* ImageView.swift */; }; @@ -697,11 +701,14 @@ 4C5C7E67284ED36500A22DF5 /* SearchHomeModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchHomeModel.swift; sourceTree = ""; }; 4C5C7E69284EDE2E00A22DF5 /* SearchResultsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchResultsView.swift; sourceTree = ""; }; 4C5D5C9C2A6B2CB40024563C /* AsciiCharacter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsciiCharacter.swift; sourceTree = ""; }; + 4C5E54022A9522F600FF6E60 /* UserStatus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserStatus.swift; sourceTree = ""; }; + 4C5E54052A9671F800FF6E60 /* UserStatusSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserStatusSheet.swift; sourceTree = ""; }; 4C5F9113283D694D0052CD1C /* FollowTarget.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FollowTarget.swift; sourceTree = ""; }; 4C5F9115283D855D0052CD1C /* EventsModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventsModel.swift; sourceTree = ""; }; 4C5F9117283D88E40052CD1C /* FollowingModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FollowingModel.swift; sourceTree = ""; }; 4C63334F283D40E500B1C9C3 /* HomeModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeModel.swift; sourceTree = ""; }; 4C633351283D419F00B1C9C3 /* SignalModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SignalModel.swift; sourceTree = ""; }; + 4C64305B2A945AFF00B0C0E9 /* MusicController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MusicController.swift; sourceTree = ""; }; 4C64987B286D03E000EAE2B3 /* DirectMessagesView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectMessagesView.swift; sourceTree = ""; }; 4C64987D286D082C00EAE2B3 /* DirectMessagesModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectMessagesModel.swift; sourceTree = ""; }; 4C684A542A7E91FE005E6031 /* LongPostTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LongPostTests.swift; sourceTree = ""; }; @@ -894,6 +901,7 @@ 4CF0ABED29844B5500D66079 /* AnyEncodable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AnyEncodable.swift; sourceTree = ""; }; 4CF0ABEF29857E9200D66079 /* Bech32Object.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Bech32Object.swift; sourceTree = ""; }; 4CF0ABF52985CD5500D66079 /* UserSearch.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserSearch.swift; sourceTree = ""; }; + 4CF38C872A9442DC00BE01B6 /* UserStatusView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserStatusView.swift; sourceTree = ""; }; 4CFD502E2A2DA45800A229DB /* MediaView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaView.swift; sourceTree = ""; }; 4CFF8F6229CC9AD7008DB934 /* ImageContextMenuModifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageContextMenuModifier.swift; sourceTree = ""; }; 4CFF8F6629CC9E3A008DB934 /* ImageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageView.swift; sourceTree = ""; }; @@ -1231,6 +1239,25 @@ path = Notifications; sourceTree = ""; }; + 4C5E54042A95232A00FF6E60 /* Status */ = { + isa = PBXGroup; + children = ( + 4C64305A2A945AF200B0C0E9 /* Music */, + 4CF38C872A9442DC00BE01B6 /* UserStatusView.swift */, + 4C5E54022A9522F600FF6E60 /* UserStatus.swift */, + 4C5E54052A9671F800FF6E60 /* UserStatusSheet.swift */, + ); + path = Status; + sourceTree = ""; + }; + 4C64305A2A945AF200B0C0E9 /* Music */ = { + isa = PBXGroup; + children = ( + 4C64305B2A945AFF00B0C0E9 /* MusicController.swift */, + ); + path = Music; + sourceTree = ""; + }; 4C687C2A2A6058450092C550 /* Search */ = { isa = PBXGroup; children = ( @@ -1635,6 +1662,7 @@ 4CE4F9DF285287A000C00DD9 /* Components */ = { isa = PBXGroup; children = ( + 4C5E54042A95232A00FF6E60 /* Status */, 4C687C2A2A6058450092C550 /* Search */, 4C7D09702A0AEF4C00943473 /* Gradients */, 31D2E846295218AF006D67F8 /* Shimmer.swift */, @@ -2293,6 +2321,7 @@ 3A23838E2A297DD200E5AA2E /* ZapButtonModel.swift in Sources */, F71694F82A6983AF001F4053 /* GrayGradient.swift in Sources */, 4C1D4FB12A7958E60024F453 /* VersionInfo.swift in Sources */, + 4C64305C2A945AFF00B0C0E9 /* MusicController.swift in Sources */, 5053ACA72A56DF3B00851AE3 /* DeveloperSettingsView.swift in Sources */, F79C7FAD29D5E9620000F946 /* EditPictureControl.swift in Sources */, 4C9F18E229AA9B6C008C55EC /* CustomizeZapView.swift in Sources */, @@ -2307,6 +2336,7 @@ 4CE1399229F0666100AC6A0B /* ShareActionButton.swift in Sources */, 4C42812C298C848200DBF26F /* TranslateView.swift in Sources */, 4C363A9C282838B9006E126D /* EventRef.swift in Sources */, + 4C5E54032A9522F600FF6E60 /* UserStatus.swift in Sources */, 4C7D095F2A098C5D00943473 /* ConnectWalletView.swift in Sources */, 3AA24802297E3DC20090C62D /* RepostView.swift in Sources */, 5C6E1DAF2A194075008FC15A /* PinkGradient.swift in Sources */, @@ -2368,6 +2398,7 @@ 4C3EA66028FF5E7700C48A62 /* node_id.c in Sources */, 4C687C212A5F7ED00092C550 /* DamusBackground.swift in Sources */, 4CA352A02A76AE80003BB08B /* Notify.swift in Sources */, + 4CF38C882A9442DC00BE01B6 /* UserStatusView.swift in Sources */, 4CE6DEE727F7A08100C66700 /* damusApp.swift in Sources */, 4C1253582A76C9060004F4B8 /* PresentSheetNotify.swift in Sources */, 4C363A962827096D006E126D /* PostBlock.swift in Sources */, @@ -2376,6 +2407,7 @@ 4CEE2AED2805B22500AB5EEF /* NostrRequest.swift in Sources */, 4C06670E28FDEAA000038D2A /* utf8.c in Sources */, 4C3EA66D28FF782800C48A62 /* amount.c in Sources */, + 4C5E54062A9671F800FF6E60 /* UserStatusSheet.swift in Sources */, F71694F42A6732B7001F4053 /* GradientFollowButton.swift in Sources */, 4C3AC7A728369BA200E1F516 /* SearchHomeView.swift in Sources */, 4CB883B0297705DD00DC99E7 /* ZapButton.swift in Sources */, diff --git a/damus/Components/Status/Music/MusicController.swift b/damus/Components/Status/Music/MusicController.swift new file mode 100644 index 0000000000..5e16d59d57 --- /dev/null +++ b/damus/Components/Status/Music/MusicController.swift @@ -0,0 +1,48 @@ +// +// MusicController.swift +// damus +// +// Created by William Casarin on 2023-08-21. +// +import SwiftUI +import MediaPlayer + +enum MusicState { + case playback_state(MPMusicPlaybackState) + case song(MPMediaItem?) +} + +class MusicController { + let player: MPMusicPlayerController + + let onChange: (MusicState) -> () + + init(onChange: @escaping (MusicState) -> ()) { + player = .systemMusicPlayer + + player.beginGeneratingPlaybackNotifications() + + self.onChange = onChange + + print("Playback State: \(player.playbackState)") + print("Now Playing Item: \(player.nowPlayingItem?.title ?? "None")") + + NotificationCenter.default.addObserver(self, selector: #selector(self.songChanged(notification:)), name: .MPMusicPlayerControllerNowPlayingItemDidChange, object: player) + + NotificationCenter.default.addObserver(self, selector: #selector(self.playbackStatusChanged(notification:)), name: .MPMusicPlayerControllerPlaybackStateDidChange, object: player) + } + + deinit { + print("deinit musiccontroller") + } + + @objc + func songChanged(notification: Notification) { + onChange(.song(player.nowPlayingItem)) + } + + @objc + func playbackStatusChanged(notification: Notification) { + onChange(.playback_state(player.playbackState)) + } +} diff --git a/damus/Components/Status/UserStatus.swift b/damus/Components/Status/UserStatus.swift new file mode 100644 index 0000000000..6af8d85dd9 --- /dev/null +++ b/damus/Components/Status/UserStatus.swift @@ -0,0 +1,141 @@ +// +// UserStatus.swift +// damus +// +// Created by William Casarin on 2023-08-22. +// + +import Foundation +import MediaPlayer + +struct Song { + let started_playing: Date + let content: String + + +} + +struct UserStatus { + let type: UserStatusType + let expires_at: Date? + let content: String + + func to_note(keypair: FullKeypair) -> NostrEvent? { + return make_user_status_note(status: self, keypair: keypair) + } + + init(type: UserStatusType, expires_at: Date?, content: String) { + self.type = type + self.expires_at = expires_at + self.content = content + } + + init?(ev: NostrEvent) { + guard let tag = ev.referenced_params.just_one() else { + return nil + } + + let str = tag.param.string() + if str == "general" { + self.type = .general + } else if str == "music" { + self.type = .music + } else { + return nil + } + + if let tag = ev.tags.first(where: { t in t.count >= 2 && t[0].matches_str("expiration") }), + tag.count == 2, + let expires = UInt32(tag[1].string()) + { + self.expires_at = Date(timeIntervalSince1970: TimeInterval(expires)) + } else { + self.expires_at = nil + } + + self.content = ev.content + } + +} + +enum UserStatusType: String { + case music + case general + +} + +class UserStatusModel: ObservableObject { + @Published var general: UserStatus? + @Published var music: UserStatus? + + func update_status(_ s: UserStatus) { + switch s.type { + case .music: + self.music = s + case .general: + self.general = s + } + } + + var _playing_enabled: Bool + var playing_enabled: Bool { + set { + var new_val = newValue + + if newValue { + MPMediaLibrary.requestAuthorization { astatus in + switch astatus { + case .notDetermined: new_val = false + case .denied: new_val = false + case .restricted: new_val = false + case .authorized: new_val = true + @unknown default: + new_val = false + } + + } + } + + if new_val != playing_enabled { + _playing_enabled = new_val + self.objectWillChange.send() + } + } + + get { + return _playing_enabled + } + } + + init(playing: UserStatus? = nil, status: UserStatus? = nil) { + self.general = status + self.music = playing + self._playing_enabled = false + self.playing_enabled = false + } + + static var current_track: String? { + let player = MPMusicPlayerController.systemMusicPlayer + guard let nowPlayingItem = player.nowPlayingItem else { return nil } + return nowPlayingItem.title + } +} + +func make_user_status_note(status: UserStatus, keypair: FullKeypair, expiry: Date? = nil) -> NostrEvent? +{ + var tags: [[String]] = [ ["d", status.type.rawValue] ] + + if let expiry { + tags.append(["expiration", String(UInt32(expiry.timeIntervalSince1970))]) + } else if let expiry = status.expires_at { + tags.append(["expiration", String(UInt32(expiry.timeIntervalSince1970))]) + } + + let kind = NostrKind.status.rawValue + guard let ev = NostrEvent(content: status.content, keypair: keypair.to_keypair(), kind: kind, tags: tags) else { + return nil + } + + return ev +} + diff --git a/damus/Components/Status/UserStatusSheet.swift b/damus/Components/Status/UserStatusSheet.swift new file mode 100644 index 0000000000..52eb10ccbc --- /dev/null +++ b/damus/Components/Status/UserStatusSheet.swift @@ -0,0 +1,116 @@ +// +// UserStatusSheet.swift +// damus +// +// Created by William Casarin on 2023-08-23. +// + +import SwiftUI + +enum StatusDuration: String, CaseIterable { + case never = "Never" + case thirty_mins = "30 Minutes" + case hour = "1 Hour" + case four_hours = "4 Hours" + case day = "1 Day" + case week = "1 Week" + + var expiration: Date? { + switch self { + case .never: + return nil + case .thirty_mins: + return Date.now.addingTimeInterval(60 * 30) + case .hour: + return Date.now.addingTimeInterval(60 * 60) + case .four_hours: + return Date.now.addingTimeInterval(60 * 60 * 4) + case .day: + return Date.now.addingTimeInterval(60 * 60 * 24) + case .week: + return Date.now.addingTimeInterval(60 * 60 * 24 * 7) + } + } +} + +struct UserStatusSheet: View { + let postbox: PostBox + let keypair: Keypair + + @State var duration: StatusDuration = .never + @ObservedObject var status: UserStatusModel + @Environment(\.dismiss) var dismiss + + var status_binding: Binding { + Binding(get: { + status.general?.content ?? "" + }, set: { v in + status.general = UserStatus(type: .general, expires_at: duration.expiration, content: v) + }) + } + + var body: some View { + VStack(alignment: .leading, spacing: 20) { + Text("Set Status") + .font(.largeTitle) + + TextField(text: status_binding, label: { + Text("📋 Working") + }) + + HStack { + Text("Clear status") + + Spacer() + + Picker("Duration", selection: $duration) { + ForEach(StatusDuration.allCases, id: \.self) { d in + Text("\(d.rawValue)") + .tag(d) + } + } + } + + Toggle(isOn: $status.playing_enabled, label: { + Text("Broadcast music playing on Apple Music") + }) + + HStack(alignment: .center) { + Button(action: { + dismiss() + }, label: { + Text("Cancel") + }) + + Spacer() + + Button(action: { + guard let status = self.status.general, + let kp = keypair.to_full(), + let ev = make_user_status_note(status: status, keypair: kp, expiry: duration.expiration) + else { + return + } + + postbox.send(ev) + + dismiss() + }, label: { + Text("Save") + }) + .buttonStyle(GradientButtonStyle()) + } + .padding([.top], 30) + + Spacer() + } + .padding(30) + } +} + + +struct UserStatusSheet_Previews: PreviewProvider { + static var previews: some View { + UserStatusSheet(postbox: PostBox(pool: RelayPool()), keypair: Keypair(pubkey: .empty, privkey: nil), status: .init()) + } +} diff --git a/damus/Components/Status/UserStatusView.swift b/damus/Components/Status/UserStatusView.swift new file mode 100644 index 0000000000..8f765dae2f --- /dev/null +++ b/damus/Components/Status/UserStatusView.swift @@ -0,0 +1,37 @@ +// +// UserStatus.swift +// damus +// +// Created by William Casarin on 2023-08-21. +// + +import SwiftUI +import MediaPlayer + + +struct UserStatusView: View { + @ObservedObject var status: UserStatusModel + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + if let general = status.general { + Text(verbatim: "\(general.content)") + .foregroundColor(.gray) + .font(.callout.italic()) + } + + if let playing = status.music { + Text(verbatim: "🎵\(playing.content)") + .foregroundColor(.gray) + .font(.callout.italic()) + } + } + + } +} + +struct UserStatusView_Previews: PreviewProvider { + static var previews: some View { + UserStatusView(status: .init()) + } +} diff --git a/damus/ContentView.swift b/damus/ContentView.swift index d5a6044074..832303b503 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -7,6 +7,7 @@ import SwiftUI import AVKit +import MediaPlayer struct TimestampedProfile { let profile: Profile @@ -30,6 +31,7 @@ enum Sheets: Identifiable { case zap(ZapSheet) case select_wallet(SelectWallet) case filter + case user_status case suggestedUsers static func zap(target: ZapTarget, lnurl: String) -> Sheets { @@ -43,6 +45,7 @@ enum Sheets: Identifiable { var id: String { switch self { case .report: return "report" + case .user_status: return "user_status" case .post(let action): return "post-" + (action.ev?.id.hex() ?? "") case .event(let ev): return "event-" + ev.id.hex() case .zap(let sheet): return "zap-" + hex_encode(sheet.target.id) @@ -315,6 +318,8 @@ struct ContentView: View { MaybeReportView(target: target) case .post(let action): PostView(action: action, damus_state: damus_state!) + case .user_status: + UserStatusSheet(postbox: damus_state!.postbox, keypair: damus_state!.keypair, status: damus_state!.profiles.profile_data(damus_state!.pubkey).status) case .event: EventDetailView() case .zap(let zapsheet): @@ -647,14 +652,32 @@ struct ContentView: View { muted_threads: MutedThreadsManager(keypair: keypair), wallet: WalletModel(settings: settings), nav: self.navigationCoordinator, - user_search_cache: user_search_cache + user_search_cache: user_search_cache, + music: MusicController(onChange: music_changed) ) home.damus_state = self.damus_state! pool.connect() } - + func music_changed(_ state: MusicState) { + guard let damus_state else { return } + switch state { + case .playback_state: + break + case .song(let song): + guard let song, let kp = damus_state.keypair.to_full() else { return } + + let pdata = damus_state.profiles.profile_data(damus_state.pubkey) + + let music = UserStatus(type: .music, expires_at: Date.now.addingTimeInterval(song.playbackDuration), content: "\(song.title ?? "Unknown") - \(song.artist ?? "Unknown")") + pdata.status.music = music + + guard let ev = music.to_note(keypair: kp) else { return } + damus_state.postbox.send(ev) + } + } + } struct ContentView_Previews: PreviewProvider { @@ -744,7 +767,6 @@ func update_filters_with_since(last_of_kind: [UInt32: NostrEvent], filters: [Nos func setup_notifications() { - UIApplication.shared.registerForRemoteNotifications() let center = UNUserNotificationCenter.current() diff --git a/damus/Info.plist b/damus/Info.plist index 69de9d887a..b4e50bf497 100644 --- a/damus/Info.plist +++ b/damus/Info.plist @@ -68,6 +68,8 @@ NSCameraUsageDescription Damus needs access to your camera if you want to upload photos from it + NSAppleMusicUsageDescription + Damus needs access to your media library for playback statuses NSMicrophoneUsageDescription Damus needs access to your microphone if you want to upload recorded videos from it diff --git a/damus/Models/DamusState.swift b/damus/Models/DamusState.swift index 0b1a4f77d2..c26d43b82d 100644 --- a/damus/Models/DamusState.swift +++ b/damus/Models/DamusState.swift @@ -32,7 +32,8 @@ struct DamusState { let wallet: WalletModel let nav: NavigationCoordinator let user_search_cache: UserSearchCache - + let music: MusicController? + @discardableResult func add_zap(zap: Zapping) -> Bool { // store generic zap mapping @@ -87,6 +88,8 @@ struct DamusState { muted_threads: MutedThreadsManager(keypair: kp), wallet: WalletModel(settings: UserSettingsStore()), nav: NavigationCoordinator(), - user_search_cache: user_search_cache) + user_search_cache: user_search_cache, + music: nil + ) } } diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index b6d33919d4..2027cc2d89 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -190,9 +190,19 @@ class HomeModel { handle_nwc_response(ev, relay: relay_id) case .http_auth: break + case .status: + handle_status_event(ev) } } - + + func handle_status_event(_ ev: NostrEvent) { + guard let st = UserStatus(ev: ev) else { + return + } + + damus_state.profiles.profile_data(ev.pubkey).status.update_status(st) + } + func handle_nwc_response(_ ev: NostrEvent, relay: String) { Task { @MainActor in // TODO: Adapt KeychainStorage to StringCodable and instead of parsing to WalletConnectURL every time @@ -502,7 +512,7 @@ class HomeModel { func subscribe_to_home_filters(friends fs: [Pubkey]? = nil, relay_id: String? = nil) { // TODO: separate likes? var home_filter_kinds: [NostrKind] = [ - .text, .longform, .boost + .text, .longform, .boost, .status ] if !damus_state.settings.onlyzaps_mode { home_filter_kinds.append(.like) @@ -1401,7 +1411,7 @@ func process_zap_event(damus_state: DamusState, ev: NostrEvent, completion: @esc } DispatchQueue.main.async { - damus_state.profiles.zappers[ptag] = zapper + damus_state.profiles.profile_data(ptag).zapper = zapper guard let zap = process_zap_event_with_zapper(damus_state: damus_state, ev: ev, zapper: zapper) else { completion(.failed) return diff --git a/damus/Nostr/NostrKind.swift b/damus/Nostr/NostrKind.swift index 217d0978e1..dd64594630 100644 --- a/damus/Nostr/NostrKind.swift +++ b/damus/Nostr/NostrKind.swift @@ -24,4 +24,5 @@ enum NostrKind: UInt32, Codable { case nwc_request = 23194 case nwc_response = 23195 case http_auth = 27235 + case status = 30315 } diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index 75db29245b..9c24d612e2 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -7,6 +7,36 @@ import Foundation +class ValidationModel: ObservableObject { + @Published var validated: NIP05? + + init() { + self.validated = nil + } +} + +class ProfileDataModel: ObservableObject { + @Published var profile: TimestampedProfile? + + init() { + self.profile = nil + } +} + +class ProfileData { + var status: UserStatusModel + var profile_model: ProfileDataModel + var validation_model: ValidationModel + var zapper: Pubkey? + + init() { + status = .init() + profile_model = .init() + validation_model = .init() + zapper = nil + } +} + class Profiles { static let db_freshness_threshold: TimeInterval = 24 * 60 * 60 @@ -21,10 +51,9 @@ class Profiles { qos: .userInteractive, attributes: .concurrent) - private var profiles: [Pubkey: TimestampedProfile] = [:] - private var validated: [Pubkey: NIP05] = [:] + private var profiles: [Pubkey: ProfileData] = [:] + var nip05_pubkey: [String: Pubkey] = [:] - var zappers: [Pubkey: Pubkey] = [:] private let database = ProfileDatabase() @@ -36,36 +65,40 @@ class Profiles { func is_validated(_ pk: Pubkey) -> NIP05? { validated_queue.sync { - validated[pk] + self.profile_data(pk).validation_model.validated } } func invalidate_nip05(_ pk: Pubkey) { validated_queue.async(flags: .barrier) { - self.validated.removeValue(forKey: pk) + self.profile_data(pk).validation_model.validated = nil } } func set_validated(_ pk: Pubkey, nip05: NIP05?) { validated_queue.async(flags: .barrier) { - self.validated[pk] = nip05 + self.profile_data(pk).validation_model.validated = nip05 } } - func enumerated() -> EnumeratedSequence<[Pubkey: TimestampedProfile]> { - return profiles_queue.sync { - return profiles.enumerated() + func profile_data(_ pubkey: Pubkey) -> ProfileData { + guard let data = profiles[pubkey] else { + let data = ProfileData() + profiles[pubkey] = data + return data } + + return data } - + func lookup_zapper(pubkey: Pubkey) -> Pubkey? { - zappers[pubkey] + profile_data(pubkey).zapper } func add(id: Pubkey, profile: TimestampedProfile) { profiles_queue.async(flags: .barrier) { - let old_timestamped_profile = self.profiles[id] - self.profiles[id] = profile + let old_timestamped_profile = self.profile_data(id).profile_model.profile + self.profile_data(id).profile_model.profile = profile self.user_search_cache.updateProfile(id: id, profiles: self, oldProfile: old_timestamped_profile?.profile, newProfile: profile.profile) } @@ -81,21 +114,21 @@ class Profiles { func lookup(id: Pubkey) -> Profile? { var profile: Profile? profiles_queue.sync { - profile = profiles[id]?.profile + profile = self.profile_data(id).profile_model.profile?.profile } return profile ?? database.get(id: id) } func lookup_with_timestamp(id: Pubkey) -> TimestampedProfile? { profiles_queue.sync { - return profiles[id] + return self.profile_data(id).profile_model.profile } } func has_fresh_profile(id: Pubkey) -> Bool { var profile: Profile? profiles_queue.sync { - profile = profiles[id]?.profile + profile = self.profile_data(id).profile_model.profile?.profile } if profile != nil { return true @@ -113,6 +146,6 @@ class Profiles { func invalidate_zapper_cache(pubkey: Pubkey, profiles: Profiles, lnurl: LNUrls) { - profiles.zappers.removeValue(forKey: pubkey) + profiles.profile_data(pubkey).zapper = nil lnurl.endpoints.removeValue(forKey: pubkey) } diff --git a/damus/Views/Events/Components/EventTop.swift b/damus/Views/Events/Components/EventTop.swift index ffd8cd0b42..ac2e762fb0 100644 --- a/damus/Views/Events/Components/EventTop.swift +++ b/damus/Views/Events/Components/EventTop.swift @@ -34,7 +34,6 @@ struct EventTop: View { Spacer() EventMenuContext(damus: state, event: event) } - .lineLimit(1) } } diff --git a/damus/Views/Events/EventProfile.swift b/damus/Views/Events/EventProfile.swift index 89d7273702..75a05a1238 100644 --- a/damus/Views/Events/EventProfile.swift +++ b/damus/Views/Events/EventProfile.swift @@ -43,8 +43,11 @@ struct EventProfile: View { ProfilePicView(pubkey: pubkey, size: pfp_size, highlight: .none, profiles: damus_state.profiles, disable_animation: disable_animation) } } - - EventProfileName(pubkey: pubkey, profile: profile, damus: damus_state, size: size) + + VStack(alignment: .leading) { + EventProfileName(pubkey: pubkey, profile: profile, damus: damus_state, size: size) + UserStatusView(status: damus_state.profiles.profile_data(pubkey).status) + } } } } diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift index b9cc53bf16..6b1685ce33 100644 --- a/damus/Views/Events/EventShell.swift +++ b/damus/Views/Events/EventShell.swift @@ -93,8 +93,9 @@ struct EventShell: View { HStack(spacing: 10) { Pfp(is_anon: is_anon) - VStack { + VStack(alignment: .leading, spacing: 2) { EventTop(state: state, event: event, pubkey: pubkey, is_anon: is_anon) + UserStatusView(status: state.profiles.profile_data(pubkey).status) ReplyPart(events: state.events, event: event, privkey: state.keypair.privkey, profiles: state.profiles) } } diff --git a/damus/Views/Events/SelectedEventView.swift b/damus/Views/Events/SelectedEventView.swift index 4b29e05eea..bfa46aad06 100644 --- a/damus/Views/Events/SelectedEventView.swift +++ b/damus/Views/Events/SelectedEventView.swift @@ -49,7 +49,7 @@ struct SelectedEventView: View { .padding(.horizontal) .minimumScaleFactor(0.75) .lineLimit(1) - + if event_is_reply(event.event_refs(damus.keypair.privkey)) { ReplyDescription(event: event, replying_to: replying_to, profiles: damus.profiles) .padding(.horizontal) diff --git a/damus/Views/Profile/EventProfileName.swift b/damus/Views/Profile/EventProfileName.swift index 01f474279c..9e604b4e5d 100644 --- a/damus/Views/Profile/EventProfileName.swift +++ b/damus/Views/Profile/EventProfileName.swift @@ -55,7 +55,7 @@ struct EventProfileName: View { return donation } - + var body: some View { HStack(spacing: 2) { switch current_display_name { diff --git a/damus/Views/Profile/ProfilePopup.swift b/damus/Views/Profile/ProfilePopup.swift new file mode 100644 index 0000000000..f3edfb2ddb --- /dev/null +++ b/damus/Views/Profile/ProfilePopup.swift @@ -0,0 +1,20 @@ +// +// ProfilePopup.swift +// damus +// +// Created by William Casarin on 2023-08-21. +// + +import SwiftUI + +struct ProfilePopup: View { + var body: some View { + Text(/*@START_MENU_TOKEN@*/"Hello, World!"/*@END_MENU_TOKEN@*/) + } +} + +struct ProfilePopup_Previews: PreviewProvider { + static var previews: some View { + ProfilePopup() + } +} diff --git a/damus/Views/SideMenuView.swift b/damus/Views/SideMenuView.swift index 38a0717d23..a3fc07de13 100644 --- a/damus/Views/SideMenuView.swift +++ b/damus/Views/SideMenuView.swift @@ -83,23 +83,37 @@ struct SideMenuView: View { var TopProfile: some View { let profile = damus_state.profiles.lookup(id: damus_state.pubkey) - return HStack { - ProfilePicView(pubkey: damus_state.pubkey, size: 60, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) - - VStack(alignment: .leading) { - if let display_name = profile?.display_name { - Text(display_name) - .foregroundColor(textColor()) - .font(.title) - .lineLimit(1) - } - if let name = profile?.name { - Text("@" + name) - .foregroundColor(DamusColors.mediumGrey) - .font(.body) - .lineLimit(1) + return VStack(alignment: .leading, spacing: verticalSpacing) { + HStack { + ProfilePicView(pubkey: damus_state.pubkey, size: 60, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) + + VStack(alignment: .leading) { + if let display_name = profile?.display_name { + Text(display_name) + .foregroundColor(textColor()) + .font(.title) + .lineLimit(1) + } + if let name = profile?.name { + Text("@" + name) + .foregroundColor(DamusColors.mediumGrey) + .font(.body) + .lineLimit(1) + } } } + + navLabel(title: NSLocalizedString("Set Status", comment: "Sidebar menu label to set user status"), img: "add-reaction") + .font(.title2) + .foregroundColor(textColor()) + .frame(maxWidth: .infinity, alignment: .leading) + .dynamicTypeSize(.xSmall) + .onTapGesture { + present_sheet(.user_status) + } + + UserStatusView(status: damus_state.profiles.profile_data(damus_state.pubkey).status) + .dynamicTypeSize(.xSmall) } } @@ -190,17 +204,17 @@ struct SideMenuView: View { } } - - @ViewBuilder func navLabel(title: String, img: String) -> some View { - Image(img) - .tint(DamusColors.adaptableBlack) - - Text(title) - .font(.title2) - .foregroundColor(textColor()) - .frame(maxWidth: .infinity, alignment: .leading) - .dynamicTypeSize(.xSmall) + HStack { + Image(img) + .tint(DamusColors.adaptableBlack) + + Text(title) + .font(.title2) + .foregroundColor(textColor()) + .frame(maxWidth: .infinity, alignment: .leading) + .dynamicTypeSize(.xSmall) + } } struct SideMenuLabelStyle: LabelStyle { From 1d11bb40b501980395e03fabdd0b798e9dee9dd6 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 13:30:38 -0700 Subject: [PATCH 022/111] v1.6 (16) --- damus.xcodeproj/project.pbxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index f0f2ddbf79..be39b401eb 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -2790,7 +2790,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 15; + CURRENT_PROJECT_VERSION = 16; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; @@ -2839,7 +2839,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 15; + CURRENT_PROJECT_VERSION = 16; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; From e62ba5826bb48f929b6574f9f5d9534f8b4e1740 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 13:31:41 -0700 Subject: [PATCH 023/111] v1.6-16 changelog --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5d28c988c..3540be4f82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## [1.6-16] - 2023-08-23 + +### Added + +- Added live music statuses (William Casarin) +- Added generic user statuses (William Casarin) + +### Fixed + +- Avoid notification for zaps from muted profiles (tappu75e@duck.com) +- Fix text editing issues on characters added right after mention link (Daniel D’Aquino) +- Mute hellthreads everywhere (William Casarin) + + +[1.6-16]: https://github.com/damus-io/damus/releases/tag/v1.6-16 + ## [1.6-13] - 2023-08-18 ### Fixed From 042b7da31593c15595c7b35f5cc005f85b93e4e6 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 15:56:25 -0700 Subject: [PATCH 024/111] status: ignore processing expired events --- damus/Models/HomeModel.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index 2027cc2d89..fa512c2ff4 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -200,6 +200,10 @@ class HomeModel { return } + if let expires = st.expires_at, Date.now >= expires { + return + } + damus_state.profiles.profile_data(ev.pubkey).status.update_status(st) } From 23a8d6fb6b84503a86a1f13d785c9aa2cb1e30c9 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 16:11:48 -0700 Subject: [PATCH 025/111] status: fix status events not expiring locally Changelog-Fixed: Fix status events not expiring locally --- damus/Components/Status/UserStatus.swift | 20 ++++++++++++++++++- damus/Components/Status/UserStatusSheet.swift | 2 +- damus/ContentView.swift | 4 +++- damus/Models/HomeModel.swift | 16 ++++++++++++++- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/damus/Components/Status/UserStatus.swift b/damus/Components/Status/UserStatus.swift index 6af8d85dd9..39fde3ae2c 100644 --- a/damus/Components/Status/UserStatus.swift +++ b/damus/Components/Status/UserStatus.swift @@ -19,15 +19,22 @@ struct UserStatus { let type: UserStatusType let expires_at: Date? let content: String + let created_at: UInt32 func to_note(keypair: FullKeypair) -> NostrEvent? { return make_user_status_note(status: self, keypair: keypair) } - init(type: UserStatusType, expires_at: Date?, content: String) { + init(type: UserStatusType, expires_at: Date?, content: String, created_at: UInt32) { self.type = type self.expires_at = expires_at self.content = content + self.created_at = created_at + } + + func expired() -> Bool { + guard let expires_at else { return false } + return Date.now >= expires_at } init?(ev: NostrEvent) { @@ -54,6 +61,7 @@ struct UserStatus { } self.content = ev.content + self.created_at = ev.created_at } } @@ -77,6 +85,16 @@ class UserStatusModel: ObservableObject { } } + func try_expire() { + if let general, general.expired() { + self.general = nil + } + + if let music, music.expired() { + self.music = nil + } + } + var _playing_enabled: Bool var playing_enabled: Bool { set { diff --git a/damus/Components/Status/UserStatusSheet.swift b/damus/Components/Status/UserStatusSheet.swift index 52eb10ccbc..cc0c7db4be 100644 --- a/damus/Components/Status/UserStatusSheet.swift +++ b/damus/Components/Status/UserStatusSheet.swift @@ -45,7 +45,7 @@ struct UserStatusSheet: View { Binding(get: { status.general?.content ?? "" }, set: { v in - status.general = UserStatus(type: .general, expires_at: duration.expiration, content: v) + status.general = UserStatus(type: .general, expires_at: duration.expiration, content: v, created_at: UInt32(Date.now.timeIntervalSince1970)) }) } diff --git a/damus/ContentView.swift b/damus/ContentView.swift index 832303b503..d9096ffe00 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -359,6 +359,7 @@ struct ContentView: View { } .onReceive(timer) { n in self.damus_state?.postbox.try_flushing_events() + self.damus_state!.profiles.profile_data(self.damus_state!.pubkey).status.try_expire() } .onReceive(handle_notify(.report)) { target in self.active_sheet = .report(target) @@ -670,7 +671,8 @@ struct ContentView: View { let pdata = damus_state.profiles.profile_data(damus_state.pubkey) - let music = UserStatus(type: .music, expires_at: Date.now.addingTimeInterval(song.playbackDuration), content: "\(song.title ?? "Unknown") - \(song.artist ?? "Unknown")") + let music = UserStatus(type: .music, expires_at: Date.now.addingTimeInterval(song.playbackDuration), content: "\(song.title ?? "Unknown") - \(song.artist ?? "Unknown")", created_at: UInt32(Date.now.timeIntervalSince1970)) + pdata.status.music = music guard let ev = music.to_note(keypair: kp) else { return } diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index fa512c2ff4..1040518b84 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -200,11 +200,25 @@ class HomeModel { return } + // don't process expired events if let expires = st.expires_at, Date.now >= expires { return } - damus_state.profiles.profile_data(ev.pubkey).status.update_status(st) + let pdata = damus_state.profiles.profile_data(ev.pubkey) + + // don't use old events + if st.type == .music, + let music = pdata.status.music, + ev.created_at < music.created_at { + return + } else if st.type == .general, + let general = pdata.status.general, + ev.created_at < general.created_at { + return + } + + pdata.status.update_status(st) } func handle_nwc_response(_ ev: NostrEvent, relay: String) { From 53c2b3a48ddcd9ec06f602c8287efb6040d01354 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 16:19:19 -0700 Subject: [PATCH 026/111] status: clear statuses if they only contain whitespace Changelog-Changed: clear statuses if they only contain whitespace --- damus/Components/Status/UserStatus.swift | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/damus/Components/Status/UserStatus.swift b/damus/Components/Status/UserStatus.swift index 39fde3ae2c..bc2b69d1e4 100644 --- a/damus/Components/Status/UserStatus.swift +++ b/damus/Components/Status/UserStatus.swift @@ -77,11 +77,22 @@ class UserStatusModel: ObservableObject { @Published var music: UserStatus? func update_status(_ s: UserStatus) { + // whitespace = delete + let del = s.content.allSatisfy({ c in c.isWhitespace }) + switch s.type { case .music: - self.music = s + if del { + self.music = nil + } else { + self.music = s + } case .general: - self.general = s + if del { + self.general = nil + } else { + self.general = s + } } } From bf3ca4a1869adc15d614c3976a62b8db3276377b Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 16:23:18 -0700 Subject: [PATCH 027/111] status: truncate statuses to a single line Changelog-Fixed: Fix long status lines --- damus/Components/Status/UserStatusView.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/damus/Components/Status/UserStatusView.swift b/damus/Components/Status/UserStatusView.swift index 8f765dae2f..d3d1d367f1 100644 --- a/damus/Components/Status/UserStatusView.swift +++ b/damus/Components/Status/UserStatusView.swift @@ -13,15 +13,17 @@ struct UserStatusView: View { @ObservedObject var status: UserStatusModel var body: some View { - VStack(alignment: .leading, spacing: 0) { + VStack(alignment: .leading, spacing: 2) { if let general = status.general { Text(verbatim: "\(general.content)") + .lineLimit(1) .foregroundColor(.gray) .font(.callout.italic()) } if let playing = status.music { Text(verbatim: "🎵\(playing.content)") + .lineLimit(1) .foregroundColor(.gray) .font(.callout.italic()) } From db59f7497005c9db0aa8ff25140e2baa60fdea98 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 16:31:10 -0700 Subject: [PATCH 028/111] status: add missing status to some thread event views --- damus/Views/Events/EventShell.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift index 6b1685ce33..930a8da9ba 100644 --- a/damus/Views/Events/EventShell.swift +++ b/damus/Views/Events/EventShell.swift @@ -68,6 +68,8 @@ struct EventShell: View { VStack(alignment: .leading) { EventTop(state: state, event: event, pubkey: pubkey, is_anon: is_anon) + UserStatusView(status: state.profiles.profile_data(pubkey).status) + if !options.contains(.no_replying_to) { ReplyPart(events: state.events, event: event, privkey: state.keypair.privkey, profiles: state.profiles) } From d02fc9142dcf07a5c329f8bea27dd9d91c55feb1 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 16:42:38 -0700 Subject: [PATCH 029/111] status: add settings for disabling statuses in the UI Suggested-by: Tanel Changelog-Added: Add settings for disabling user statuses --- damus/Components/Status/UserStatusView.swift | 9 ++++++--- damus/Models/HomeModel.swift | 7 ++++++- damus/Models/UserSettingsStore.swift | 6 ++++++ damus/Views/Events/EventProfile.swift | 3 ++- damus/Views/Events/EventShell.swift | 4 ++-- damus/Views/Settings/AppearanceSettingsView.swift | 10 +++++++++- damus/Views/SideMenuView.swift | 2 +- 7 files changed, 32 insertions(+), 9 deletions(-) diff --git a/damus/Components/Status/UserStatusView.swift b/damus/Components/Status/UserStatusView.swift index d3d1d367f1..3f475f18fe 100644 --- a/damus/Components/Status/UserStatusView.swift +++ b/damus/Components/Status/UserStatusView.swift @@ -12,16 +12,19 @@ import MediaPlayer struct UserStatusView: View { @ObservedObject var status: UserStatusModel + var show_general: Bool + var show_music: Bool + var body: some View { VStack(alignment: .leading, spacing: 2) { - if let general = status.general { + if show_general, let general = status.general { Text(verbatim: "\(general.content)") .lineLimit(1) .foregroundColor(.gray) .font(.callout.italic()) } - if let playing = status.music { + if show_music, let playing = status.music { Text(verbatim: "🎵\(playing.content)") .lineLimit(1) .foregroundColor(.gray) @@ -34,6 +37,6 @@ struct UserStatusView: View { struct UserStatusView_Previews: PreviewProvider { static var previews: some View { - UserStatusView(status: .init()) + UserStatusView(status: .init(), show_general: true, show_music: true) } } diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index 1040518b84..7ced11899d 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -530,12 +530,17 @@ class HomeModel { func subscribe_to_home_filters(friends fs: [Pubkey]? = nil, relay_id: String? = nil) { // TODO: separate likes? var home_filter_kinds: [NostrKind] = [ - .text, .longform, .boost, .status + .text, .longform, .boost ] if !damus_state.settings.onlyzaps_mode { home_filter_kinds.append(.like) } + // only pull status data if we care for it + if damus_state.settings.show_music_statuses || damus_state.settings.show_general_statuses { + home_filter_kinds.append(.status) + } + let friends = fs ?? get_friends() var home_filter = NostrFilter(kinds: home_filter_kinds) // include our pubkey as well even if we're not technically a friend diff --git a/damus/Models/UserSettingsStore.swift b/damus/Models/UserSettingsStore.swift index 526945b7c7..862c71a5cc 100644 --- a/damus/Models/UserSettingsStore.swift +++ b/damus/Models/UserSettingsStore.swift @@ -159,6 +159,12 @@ class UserSettingsStore: ObservableObject { @Setting(key: "auto_translate", default_value: true) var auto_translate: Bool + @Setting(key: "show_general_statuses", default_value: true) + var show_general_statuses: Bool + + @Setting(key: "show_music_statuses", default_value: true) + var show_music_statuses: Bool + @Setting(key: "show_only_preferred_languages", default_value: false) var show_only_preferred_languages: Bool diff --git a/damus/Views/Events/EventProfile.swift b/damus/Views/Events/EventProfile.swift index 75a05a1238..7b2fe4ef68 100644 --- a/damus/Views/Events/EventProfile.swift +++ b/damus/Views/Events/EventProfile.swift @@ -46,7 +46,8 @@ struct EventProfile: View { VStack(alignment: .leading) { EventProfileName(pubkey: pubkey, profile: profile, damus: damus_state, size: size) - UserStatusView(status: damus_state.profiles.profile_data(pubkey).status) + + UserStatusView(status: damus_state.profiles.profile_data(pubkey).status, show_general: damus_state.settings.show_general_statuses, show_music: damus_state.settings.show_music_statuses) } } } diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift index 930a8da9ba..ac8acb0908 100644 --- a/damus/Views/Events/EventShell.swift +++ b/damus/Views/Events/EventShell.swift @@ -68,7 +68,7 @@ struct EventShell: View { VStack(alignment: .leading) { EventTop(state: state, event: event, pubkey: pubkey, is_anon: is_anon) - UserStatusView(status: state.profiles.profile_data(pubkey).status) + UserStatusView(status: state.profiles.profile_data(pubkey).status, show_general: state.settings.show_general_statuses, show_music: state.settings.show_music_statuses) if !options.contains(.no_replying_to) { ReplyPart(events: state.events, event: event, privkey: state.keypair.privkey, profiles: state.profiles) @@ -97,7 +97,7 @@ struct EventShell: View { VStack(alignment: .leading, spacing: 2) { EventTop(state: state, event: event, pubkey: pubkey, is_anon: is_anon) - UserStatusView(status: state.profiles.profile_data(pubkey).status) + UserStatusView(status: state.profiles.profile_data(pubkey).status, show_general: state.settings.show_general_statuses, show_music: state.settings.show_music_statuses) ReplyPart(events: state.events, event: event, privkey: state.keypair.privkey, profiles: state.profiles) } } diff --git a/damus/Views/Settings/AppearanceSettingsView.swift b/damus/Views/Settings/AppearanceSettingsView.swift index f4397dca15..899347acbb 100644 --- a/damus/Views/Settings/AppearanceSettingsView.swift +++ b/damus/Views/Settings/AppearanceSettingsView.swift @@ -48,7 +48,15 @@ struct AppearanceSettingsView: View { Toggle(NSLocalizedString("Truncate notification mention text", comment: "Setting to truncate text in mention notifications"), isOn: $settings.truncate_mention_text) .toggleStyle(.switch) } - + + Section(header: Text("User Statuses")) { + Toggle(NSLocalizedString("Show general statuses", comment: "Settings toggle for enabling general user statuses"), isOn: $settings.show_general_statuses) + .toggleStyle(.switch) + + Toggle(NSLocalizedString("Show music statuses", comment: "Settings toggle for enabling now playing music statuses"), isOn: $settings.show_music_statuses) + .toggleStyle(.switch) + } + // MARK: - Accessibility Section(header: Text(NSLocalizedString("Accessibility", comment: "Section header for accessibility settings"))) { Toggle(NSLocalizedString("Left Handed", comment: "Moves the post button to the left side of the screen"), isOn: $settings.left_handed) diff --git a/damus/Views/SideMenuView.swift b/damus/Views/SideMenuView.swift index a3fc07de13..b1c420c7bb 100644 --- a/damus/Views/SideMenuView.swift +++ b/damus/Views/SideMenuView.swift @@ -112,7 +112,7 @@ struct SideMenuView: View { present_sheet(.user_status) } - UserStatusView(status: damus_state.profiles.profile_data(damus_state.pubkey).status) + UserStatusView(status: damus_state.profiles.profile_data(damus_state.pubkey).status, show_general: true, show_music: true) .dynamicTypeSize(.xSmall) } } From 981d500c25a1f6ab01af0ff40e32bbdb842c23b0 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 17:17:53 -0700 Subject: [PATCH 030/111] status: click music urls to display in spotify Changelog-Added: Click music statuses to display in spotify --- damus/Components/Status/UserStatus.swift | 17 ++++++++++++++++- damus/Components/Status/UserStatusView.swift | 12 ++++++++++++ damus/ContentView.swift | 7 ++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/damus/Components/Status/UserStatus.swift b/damus/Components/Status/UserStatus.swift index bc2b69d1e4..1d15dc65fb 100644 --- a/damus/Components/Status/UserStatus.swift +++ b/damus/Components/Status/UserStatus.swift @@ -20,16 +20,18 @@ struct UserStatus { let expires_at: Date? let content: String let created_at: UInt32 + let url: URL? func to_note(keypair: FullKeypair) -> NostrEvent? { return make_user_status_note(status: self, keypair: keypair) } - init(type: UserStatusType, expires_at: Date?, content: String, created_at: UInt32) { + init(type: UserStatusType, expires_at: Date?, content: String, created_at: UInt32, url: URL? = nil) { self.type = type self.expires_at = expires_at self.content = content self.created_at = created_at + self.url = url } func expired() -> Bool { @@ -51,6 +53,15 @@ struct UserStatus { return nil } + if let tag = ev.tags.first(where: { t in t.count >= 2 && t[0].matches_char("r") }), + tag.count >= 2, + let url = URL(string: tag[1].string()) + { + self.url = url + } else { + self.url = nil + } + if let tag = ev.tags.first(where: { t in t.count >= 2 && t[0].matches_str("expiration") }), tag.count == 2, let expires = UInt32(tag[1].string()) @@ -160,6 +171,10 @@ func make_user_status_note(status: UserStatus, keypair: FullKeypair, expiry: Dat tags.append(["expiration", String(UInt32(expiry.timeIntervalSince1970))]) } + if let url = status.url { + tags.append(["r", url.absoluteString]) + } + let kind = NostrKind.status.rawValue guard let ev = NostrEvent(content: status.content, keypair: keypair.to_keypair(), kind: kind, tags: tags) else { return nil diff --git a/damus/Components/Status/UserStatusView.swift b/damus/Components/Status/UserStatusView.swift index 3f475f18fe..d06b977896 100644 --- a/damus/Components/Status/UserStatusView.swift +++ b/damus/Components/Status/UserStatusView.swift @@ -15,6 +15,8 @@ struct UserStatusView: View { var show_general: Bool var show_music: Bool + @Environment(\.openURL) var openURL + var body: some View { VStack(alignment: .leading, spacing: 2) { if show_general, let general = status.general { @@ -22,6 +24,11 @@ struct UserStatusView: View { .lineLimit(1) .foregroundColor(.gray) .font(.callout.italic()) + .onTapGesture { + if let url = general.url { + openURL(url) + } + } } if show_music, let playing = status.music { @@ -29,6 +36,11 @@ struct UserStatusView: View { .lineLimit(1) .foregroundColor(.gray) .font(.callout.italic()) + .onTapGesture { + if let url = playing.url { + openURL(url) + } + } } } diff --git a/damus/ContentView.swift b/damus/ContentView.swift index d9096ffe00..a3075dfd7b 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -671,7 +671,12 @@ struct ContentView: View { let pdata = damus_state.profiles.profile_data(damus_state.pubkey) - let music = UserStatus(type: .music, expires_at: Date.now.addingTimeInterval(song.playbackDuration), content: "\(song.title ?? "Unknown") - \(song.artist ?? "Unknown")", created_at: UInt32(Date.now.timeIntervalSince1970)) + let desc = "\(song.title ?? "Unknown") - \(song.artist ?? "Unknown")" + let encodedDesc = desc.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) + let url = encodedDesc.flatMap { enc in + URL(string: "spotify:search:\(enc)") + } + let music = UserStatus(type: .music, expires_at: Date.now.addingTimeInterval(song.playbackDuration), content: desc, created_at: UInt32(Date.now.timeIntervalSince1970), url: url) pdata.status.music = music From 2c6999e15c868386a0147a2a5e973de15d1c7789 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 17:46:15 -0700 Subject: [PATCH 031/111] status: support clickable status urls Changelog-Added: Add support for status URLs --- damus/Components/Status/UserStatus.swift | 6 +-- damus/Components/Status/UserStatusSheet.swift | 27 ++++++++++- damus/Components/Status/UserStatusView.swift | 45 +++++++++++-------- 3 files changed, 54 insertions(+), 24 deletions(-) diff --git a/damus/Components/Status/UserStatus.swift b/damus/Components/Status/UserStatus.swift index 1d15dc65fb..be21d702b3 100644 --- a/damus/Components/Status/UserStatus.swift +++ b/damus/Components/Status/UserStatus.swift @@ -11,16 +11,14 @@ import MediaPlayer struct Song { let started_playing: Date let content: String - - } struct UserStatus { let type: UserStatusType let expires_at: Date? - let content: String + var content: String let created_at: UInt32 - let url: URL? + var url: URL? func to_note(keypair: FullKeypair) -> NostrEvent? { return make_user_status_note(status: self, keypair: keypair) diff --git a/damus/Components/Status/UserStatusSheet.swift b/damus/Components/Status/UserStatusSheet.swift index cc0c7db4be..9a76d5c313 100644 --- a/damus/Components/Status/UserStatusSheet.swift +++ b/damus/Components/Status/UserStatusSheet.swift @@ -38,6 +38,7 @@ struct UserStatusSheet: View { let keypair: Keypair @State var duration: StatusDuration = .never + @ObservedObject var status: UserStatusModel @Environment(\.dismiss) var dismiss @@ -45,7 +46,23 @@ struct UserStatusSheet: View { Binding(get: { status.general?.content ?? "" }, set: { v in - status.general = UserStatus(type: .general, expires_at: duration.expiration, content: v, created_at: UInt32(Date.now.timeIntervalSince1970)) + if let general = status.general { + status.general = UserStatus(type: .general, expires_at: duration.expiration, content: v, created_at: UInt32(Date.now.timeIntervalSince1970), url: general.url) + } else { + status.general = UserStatus(type: .general, expires_at: duration.expiration, content: v, created_at: UInt32(Date.now.timeIntervalSince1970), url: nil) + } + }) + } + + var url_binding: Binding { + Binding(get: { + status.general?.url?.absoluteString ?? "" + }, set: { v in + if let general = status.general { + status.general = UserStatus(type: .general, expires_at: duration.expiration, content: general.content, created_at: UInt32(Date.now.timeIntervalSince1970), url: URL(string: v)) + } else { + status.general = UserStatus(type: .general, expires_at: duration.expiration, content: "", created_at: UInt32(Date.now.timeIntervalSince1970), url: URL(string: v)) + } }) } @@ -58,6 +75,14 @@ struct UserStatusSheet: View { Text("📋 Working") }) + HStack { + Image("link") + + TextField(text: url_binding, label: { + Text("https://example.com") + }) + } + HStack { Text("Clear status") diff --git a/damus/Components/Status/UserStatusView.swift b/damus/Components/Status/UserStatusView.swift index d06b977896..a1444d4770 100644 --- a/damus/Components/Status/UserStatusView.swift +++ b/damus/Components/Status/UserStatusView.swift @@ -17,38 +17,45 @@ struct UserStatusView: View { @Environment(\.openURL) var openURL + func Status(st: UserStatus, prefix: String = "") -> some View { + HStack { + Text(verbatim: "\(prefix)\(st.content)") + .lineLimit(1) + .foregroundColor(.gray) + .font(.callout.italic()) + if st.url != nil { + Image("link") + .resizable() + .frame(width: 16, height: 16) + .foregroundColor(.gray) + } + } + .onTapGesture { + if let url = st.url { + openURL(url) + } + } + } + var body: some View { VStack(alignment: .leading, spacing: 2) { if show_general, let general = status.general { - Text(verbatim: "\(general.content)") - .lineLimit(1) - .foregroundColor(.gray) - .font(.callout.italic()) - .onTapGesture { - if let url = general.url { - openURL(url) - } - } + Status(st: general) } if show_music, let playing = status.music { - Text(verbatim: "🎵\(playing.content)") - .lineLimit(1) - .foregroundColor(.gray) - .font(.callout.italic()) - .onTapGesture { - if let url = playing.url { - openURL(url) - } - } + Status(st: playing, prefix: "🎵") } } } } +/* struct UserStatusView_Previews: PreviewProvider { static var previews: some View { - UserStatusView(status: .init(), show_general: true, show_music: true) + UserStatusView(status: UserStatus(type: .music, expires_at: nil, content: "Track - Artist", created_at: 0, url: URL(string: "spotify:search:abc")), show_general: true, show_music: true) } } + +*/ From 16fa7015094173e2595187f5fa74d97ba171434b Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 17:48:32 -0700 Subject: [PATCH 032/111] v1.6 (17) --- damus.xcodeproj/project.pbxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index be39b401eb..6c59a46eaf 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -2790,7 +2790,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 16; + CURRENT_PROJECT_VERSION = 17; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; @@ -2839,7 +2839,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 16; + CURRENT_PROJECT_VERSION = 17; DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\""; DEVELOPMENT_TEAM = XK7H4JAB3D; ENABLE_PREVIEWS = YES; From 725548170567a9911fb6cc8059994e633806d2d5 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 23 Aug 2023 17:49:30 -0700 Subject: [PATCH 033/111] v1.6 (17) changelog --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3540be4f82..81a6d83924 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## [1.6-17] - 2023-08-23 + +### Added + +- Add support for status URLs (William Casarin) +- Click music statuses to display in spotify (William Casarin) +- Add settings for disabling user statuses (William Casarin) + +### Changed + +- clear statuses if they only contain whitespace (William Casarin) + +### Fixed + +- Fix long status lines (William Casarin) +- Fix status events not expiring locally (William Casarin) + +[1.6-17]: https://github.com/damus-io/damus/releases/tag/v1.6-17 + ## [1.6-16] - 2023-08-23 ### Added @@ -1507,3 +1526,4 @@ [0.1.2]: https://github.com/damus-io/damus/releases/tag/v0.1.2 + From f30f93f65cb78a508731a682100485dc62908f97 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Fri, 25 Aug 2023 19:03:19 -0700 Subject: [PATCH 034/111] Revert "Move the Block helper type to its own file" This fixes the broken tests This reverts commit 286ae68fd63754fbf5a9f4b271c207e9d2df1728. --- damus.xcodeproj/project.pbxproj | 4 - damus/ContentParsing.swift | 2 +- damus/Models/Mentions.swift | 227 +++++++++++++++++++++++++++++- damus/Types/Block.swift | 214 ---------------------------- damus/Util/Zap.swift | 2 +- damus/Views/DMChatView.swift | 4 +- damus/Views/NoteContentView.swift | 10 +- damusTests/LongPostTests.swift | 2 +- nostrdb/NdbNote.swift | 10 +- 9 files changed, 230 insertions(+), 245 deletions(-) delete mode 100644 damus/Types/Block.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 6c59a46eaf..5e6db4a9d6 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -375,7 +375,6 @@ 643EA5C8296B764E005081BB /* RelayFilterView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 643EA5C7296B764E005081BB /* RelayFilterView.swift */; }; 647D9A8D2968520300A295DE /* SideMenuView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 647D9A8C2968520300A295DE /* SideMenuView.swift */; }; 64FBD06F296255C400D9D3B2 /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64FBD06E296255C400D9D3B2 /* Theme.swift */; }; - 7527271E2A93FF0100214108 /* Block.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7527271D2A93FF0100214108 /* Block.swift */; }; 7C60CAEF298471A1009C80D6 /* CoreSVG.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */; }; 7C902AE32981D55B002AB16E /* ZoomableScrollView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */; }; 7C95CAEE299DCEF1009DCB67 /* KFOptionSetter+.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */; }; @@ -934,7 +933,6 @@ 643EA5C7296B764E005081BB /* RelayFilterView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterView.swift; sourceTree = ""; }; 647D9A8C2968520300A295DE /* SideMenuView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SideMenuView.swift; sourceTree = ""; }; 64FBD06E296255C400D9D3B2 /* Theme.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Theme.swift; sourceTree = ""; }; - 7527271D2A93FF0100214108 /* Block.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Block.swift; sourceTree = ""; }; 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreSVG.swift; sourceTree = ""; }; 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ZoomableScrollView.swift; sourceTree = ""; }; 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KFOptionSetter+.swift"; sourceTree = ""; }; @@ -1607,7 +1605,6 @@ isa = PBXGroup; children = ( 4CC14FED2A73FCBB007AEB17 /* Ids */, - 7527271D2A93FF0100214108 /* Block.swift */, ); path = Types; sourceTree = ""; @@ -2423,7 +2420,6 @@ 7CFF6317299FEFE5005D382A /* SelectableText.swift in Sources */, 4CA352A82A76B37E003BB08B /* NewMutesNotify.swift in Sources */, 4CFF8F6929CC9ED1008DB934 /* ImageContainerView.swift in Sources */, - 7527271E2A93FF0100214108 /* Block.swift in Sources */, 4C54AA0729A540BA003E4487 /* NotificationsModel.swift in Sources */, 4C12536C2A76D4B00004F4B8 /* RepostedNotify.swift in Sources */, 4CB55EF3295E5D59007FD187 /* RecommendedRelayView.swift in Sources */, diff --git a/damus/ContentParsing.swift b/damus/ContentParsing.swift index 1362f39df4..6f2f389ddf 100644 --- a/damus/ContentParsing.swift +++ b/damus/ContentParsing.swift @@ -27,7 +27,7 @@ func parsed_blocks_finish(bs: inout note_blocks, tags: TagsSequence?) -> Blocks while (i < bs.num_blocks) { let block = bs.blocks[i] - if let converted = Block(block, tags: tags) { + if let converted = convert_block(block, tags: tags) { out.append(converted) } diff --git a/damus/Models/Mentions.swift b/damus/Models/Mentions.swift index 2132ca8ed4..d6cc781add 100644 --- a/damus/Models/Mentions.swift +++ b/damus/Models/Mentions.swift @@ -123,11 +123,147 @@ struct LightningInvoice { } } +enum Block: Equatable { + static func == (lhs: Block, rhs: Block) -> Bool { + switch (lhs, rhs) { + case (.text(let a), .text(let b)): + return a == b + case (.mention(let a), .mention(let b)): + return a == b + case (.hashtag(let a), .hashtag(let b)): + return a == b + case (.url(let a), .url(let b)): + return a == b + case (.invoice(let a), .invoice(let b)): + return a.string == b.string + case (_, _): + return false + } + } + + case text(String) + case mention(Mention) + case hashtag(String) + case url(URL) + case invoice(Invoice) + case relay(String) + + var is_invoice: Invoice? { + if case .invoice(let invoice) = self { + return invoice + } + return nil + } + + var is_hashtag: String? { + if case .hashtag(let htag) = self { + return htag + } + return nil + } + + var is_url: URL? { + if case .url(let url) = self { + return url + } + + return nil + } + + var is_text: String? { + if case .text(let txt) = self { + return txt + } + return nil + } + + var is_note_mention: Bool { + if case .mention(let mention) = self, + case .note = mention.ref { + return true + } + return false + } + + var is_mention: Mention? { + if case .mention(let m) = self { + return m + } + return nil + } +} + +func render_blocks(blocks: [Block]) -> String { + return blocks.reduce("") { str, block in + switch block { + case .mention(let m): + if let idx = m.index { + return str + "#[\(idx)]" + } + + switch m.ref { + case .pubkey(let pk): return str + "nostr:\(pk.npub)" + case .note(let note_id): return str + "nostr:\(note_id.bech32)" + } + case .relay(let relay): + return str + relay + case .text(let txt): + return str + txt + case .hashtag(let htag): + return str + "#" + htag + case .url(let url): + return str + url.absoluteString + case .invoice(let inv): + return str + inv.string + } + } +} + struct Blocks: Equatable { let words: Int let blocks: [Block] } +func strblock_to_string(_ s: str_block_t) -> String? { + let len = s.end - s.start + let bytes = Data(bytes: s.start, count: len) + return String(bytes: bytes, encoding: .utf8) +} + +func convert_block(_ b: block_t, tags: TagsSequence?) -> Block? { + if b.type == BLOCK_HASHTAG { + guard let str = strblock_to_string(b.block.str) else { + return nil + } + return .hashtag(str) + } else if b.type == BLOCK_TEXT { + guard let str = strblock_to_string(b.block.str) else { + return nil + } + return .text(str) + } else if b.type == BLOCK_MENTION_INDEX { + return convert_mention_index_block(ind: Int(b.block.mention_index), tags: tags) + } else if b.type == BLOCK_URL { + return convert_url_block(b.block.str) + } else if b.type == BLOCK_INVOICE { + return convert_invoice_block(b.block.invoice) + } else if b.type == BLOCK_MENTION_BECH32 { + return convert_mention_bech32_block(b.block.mention_bech32) + } + + return nil +} + +func convert_url_block(_ b: str_block) -> Block? { + guard let str = strblock_to_string(b) else { + return nil + } + guard let url = URL(string: str) else { + return .text(str) + } + return .url(url) +} + func maybe_pointee(_ p: UnsafeMutablePointer!) -> T? { guard p != nil else { return nil @@ -190,6 +326,75 @@ func format_msats(_ msat: Int64, locale: Locale = Locale.current) -> String { return String(format: format, locale: locale, sats.decimalValue as NSDecimalNumber, formattedSats) } +func convert_invoice_block(_ b: invoice_block) -> Block? { + guard let invstr = strblock_to_string(b.invstr) else { + return nil + } + + guard var b11 = maybe_pointee(b.bolt11) else { + return nil + } + + guard let description = convert_invoice_description(b11: b11) else { + return nil + } + + let amount: Amount = maybe_pointee(b11.msat).map { .specific(Int64($0.millisatoshis)) } ?? .any + let payment_hash = Data(bytes: &b11.payment_hash, count: 32) + let created_at = b11.timestamp + + tal_free(b.bolt11) + return .invoice(Invoice(description: description, amount: amount, string: invstr, expiry: b11.expiry, payment_hash: payment_hash, created_at: created_at)) +} + +func convert_mention_bech32_block(_ b: mention_bech32_block) -> Block? +{ + switch b.bech32.type { + case NOSTR_BECH32_NOTE: + let note = b.bech32.data.note; + let note_id = NoteId(Data(bytes: note.event_id, count: 32)) + return .mention(.any(.note(note_id))) + + case NOSTR_BECH32_NEVENT: + let nevent = b.bech32.data.nevent; + let note_id = NoteId(Data(bytes: nevent.event_id, count: 32)) + return .mention(.any(.note(note_id))) + + case NOSTR_BECH32_NPUB: + let npub = b.bech32.data.npub + let pubkey = Pubkey(Data(bytes: npub.pubkey, count: 32)) + return .mention(.any(.pubkey(pubkey))) + + case NOSTR_BECH32_NSEC: + let nsec = b.bech32.data.nsec + let privkey = Privkey(Data(bytes: nsec.nsec, count: 32)) + guard let pubkey = privkey_to_pubkey(privkey: privkey) else { return nil } + return .mention(.any(.pubkey(pubkey))) + + case NOSTR_BECH32_NPROFILE: + let nprofile = b.bech32.data.nprofile + let pubkey = Pubkey(Data(bytes: nprofile.pubkey, count: 32)) + return .mention(.any(.pubkey(pubkey))) + + case NOSTR_BECH32_NRELAY: + let nrelay = b.bech32.data.nrelay + guard let relay_str = strblock_to_string(nrelay.relay) else { + return nil + } + return .relay(relay_str) + + case NOSTR_BECH32_NADDR: + // TODO: wtf do I do with this + guard let naddr = strblock_to_string(b.str) else { + return nil + } + return .text("nostr:" + naddr) + + default: + return nil + } +} + func convert_invoice_description(b11: bolt11) -> InvoiceDescription? { if let desc = b11.description { return .description(String(cString: desc)) @@ -202,6 +407,24 @@ func convert_invoice_description(b11: bolt11) -> InvoiceDescription? { return nil } +func convert_mention_index_block(ind: Int, tags: TagsSequence?) -> Block? +{ + guard let tags, + ind >= 0, + ind + 1 <= tags.count + else { + return .text("#[\(ind)]") + } + + let tag = tags[ind] + + guard let mention = MentionRef.from_tag(tag: tag) else { + return .text("#[\(ind)]") + } + + return .mention(.any(mention, index: ind)) +} + func find_tag_ref(type: String, id: String, tags: [[String]]) -> Int? { var i: Int = 0 for tag in tags { @@ -251,9 +474,7 @@ func post_to_event(post: NostrPost, keypair: FullKeypair) -> NostrEvent? { let tags = post.references.map({ r in r.tag }) + post.tags let post_blocks = parse_post_blocks(content: post.content) let post_tags = make_post_tags(post_blocks: post_blocks, tags: tags) - let content = post_tags.blocks - .map(\.asString) - .joined(separator: "") + let content = render_blocks(blocks: post_tags.blocks) return NostrEvent(content: content, keypair: keypair.to_keypair(), kind: post.kind.rawValue, tags: post_tags.tags) } diff --git a/damus/Types/Block.swift b/damus/Types/Block.swift deleted file mode 100644 index e478f6007b..0000000000 --- a/damus/Types/Block.swift +++ /dev/null @@ -1,214 +0,0 @@ -// -// Block.swift -// damus -// -// Created by Kyle Roucis on 2023-08-21. -// - -import Foundation - - -fileprivate extension String { - /// Failable initializer to build a Swift.String from a C-backed `str_block_t`. - init?(_ s: str_block_t) { - let len = s.end - s.start - let bytes = Data(bytes: s.start, count: len) - self.init(bytes: bytes, encoding: .utf8) - } -} - -/// Represents a block of data stored by the NOSTR protocol. This can be -/// simple text, a hashtag, a url, a relay reference, a mention ref and -/// potentially more in the future. -enum Block: Equatable { - static func == (lhs: Block, rhs: Block) -> Bool { - switch (lhs, rhs) { - case (.text(let a), .text(let b)): - return a == b - case (.mention(let a), .mention(let b)): - return a == b - case (.hashtag(let a), .hashtag(let b)): - return a == b - case (.url(let a), .url(let b)): - return a == b - case (.invoice(let a), .invoice(let b)): - return a.string == b.string - case (_, _): - return false - } - } - - case text(String) - case mention(Mention) - case hashtag(String) - case url(URL) - case invoice(Invoice) - case relay(String) -} -extension Block { - /// Failable initializer for the C-backed type `block_t`. This initializer will inspect - /// the underlying block type and build the appropriate enum value as needed. - init?(_ block: block_t, tags: TagsSequence? = nil) { - switch block.type { - case BLOCK_HASHTAG: - guard let str = String(block.block.str) else { - return nil - } - self = .hashtag(str) - case BLOCK_TEXT: - guard let str = String(block.block.str) else { - return nil - } - self = .text(str) - case BLOCK_MENTION_INDEX: - guard let b = Block(index: Int(block.block.mention_index), tags: tags) else { - return nil - } - self = b - case BLOCK_URL: - guard let b = Block(block.block.str) else { - return nil - } - self = b - case BLOCK_INVOICE: - guard let b = Block(invoice: block.block.invoice) else { - return nil - } - self = b - case BLOCK_MENTION_BECH32: - guard let b = Block(bech32: block.block.mention_bech32) else { - return nil - } - self = b - default: - return nil - } - } -} -fileprivate extension Block { - /// Failable initializer for the C-backed type `str_block_t`. - init?(_ b: str_block_t) { - guard let str = String(b) else { - return nil - } - - if let url = URL(string: str) { - self = .url(url) - } - else { - self = .text(str) - } - } -} -fileprivate extension Block { - /// Failable initializer for a block index and a tag sequence. - init?(index: Int, tags: TagsSequence? = nil) { - guard let tags, - index >= 0, - index + 1 <= tags.count - else { - self = .text("#[\(index)]") - return - } - - let tag = tags[index] - - if let mention = MentionRef.from_tag(tag: tag) { - self = .mention(.any(mention, index: index)) - } - else { - self = .text("#[\(index)]") - } - } -} -fileprivate extension Block { - /// Failable initializer for the C-backed type `invoice_block_t`. - init?(invoice: invoice_block_t) { - guard let invstr = String(invoice.invstr) else { - return nil - } - - guard var b11 = maybe_pointee(invoice.bolt11) else { - return nil - } - - guard let description = convert_invoice_description(b11: b11) else { - return nil - } - - let amount: Amount = maybe_pointee(b11.msat).map { .specific(Int64($0.millisatoshis)) } ?? .any - let payment_hash = Data(bytes: &b11.payment_hash, count: 32) - let created_at = b11.timestamp - - tal_free(invoice.bolt11) - self = .invoice(Invoice(description: description, amount: amount, string: invstr, expiry: b11.expiry, payment_hash: payment_hash, created_at: created_at)) - } -} -fileprivate extension Block { - /// Failable initializer for the C-backed type `mention_bech32_block_t`. This initializer will inspect the - /// bech32 type code and build the appropriate enum type. - init?(bech32 b: mention_bech32_block_t) { - switch b.bech32.type { - case NOSTR_BECH32_NOTE: - let note = b.bech32.data.note; - let note_id = NoteId(Data(bytes: note.event_id, count: 32)) - self = .mention(.any(.note(note_id))) - case NOSTR_BECH32_NEVENT: - let nevent = b.bech32.data.nevent; - let note_id = NoteId(Data(bytes: nevent.event_id, count: 32)) - self = .mention(.any(.note(note_id))) - case NOSTR_BECH32_NPUB: - let npub = b.bech32.data.npub - let pubkey = Pubkey(Data(bytes: npub.pubkey, count: 32)) - self = .mention(.any(.pubkey(pubkey))) - case NOSTR_BECH32_NSEC: - let nsec = b.bech32.data.nsec - let privkey = Privkey(Data(bytes: nsec.nsec, count: 32)) - guard let pubkey = privkey_to_pubkey(privkey: privkey) else { return nil } - self = .mention(.any(.pubkey(pubkey))) - case NOSTR_BECH32_NPROFILE: - let nprofile = b.bech32.data.nprofile - let pubkey = Pubkey(Data(bytes: nprofile.pubkey, count: 32)) - self = .mention(.any(.pubkey(pubkey))) - case NOSTR_BECH32_NRELAY: - let nrelay = b.bech32.data.nrelay - guard let relay_str = String(nrelay.relay) else { - return nil - } - self = .relay(relay_str) - case NOSTR_BECH32_NADDR: - // TODO: wtf do I do with this - guard let naddr = String(b.str) else { - return nil - } - self = .text("nostr:" + naddr) - default: - return nil - } - } -} -extension Block { - var asString: String { - switch self { - case .mention(let m): - if let idx = m.index { - return "#[\(idx)]" - } - - switch m.ref { - case .pubkey(let pk): return "nostr:\(pk.npub)" - case .note(let note_id): return "nostr:\(note_id.bech32)" - } - case .relay(let relay): - return relay - case .text(let txt): - return txt - case .hashtag(let htag): - return "#" + htag - case .url(let url): - return url.absoluteString - case .invoice(let inv): - return inv.string - } - } -} diff --git a/damus/Util/Zap.swift b/damus/Util/Zap.swift index 931810a99e..7749de723d 100644 --- a/damus/Util/Zap.swift +++ b/damus/Util/Zap.swift @@ -393,7 +393,7 @@ func decode_bolt11(_ s: String) -> Invoice? { let block = bs.blocks[0] - guard let converted = Block(block) else { + guard let converted = convert_block(block, tags: nil) else { blocks_free(&bs) return nil } diff --git a/damus/Views/DMChatView.swift b/damus/Views/DMChatView.swift index 18bf288fc1..a905c68a32 100644 --- a/damus/Views/DMChatView.swift +++ b/damus/Views/DMChatView.swift @@ -130,9 +130,7 @@ struct DMChatView: View, KeyboardReadable { func send_message() { let tags = [["p", pubkey.hex()]] let post_blocks = parse_post_blocks(content: dms.draft) - let content = post_blocks - .map(\.asString) - .joined(separator: "") + let content = render_blocks(blocks: post_blocks) guard let dm = create_dm(content, to_pk: pubkey, tags: tags, keypair: damus_state.keypair) else { print("error creating dm") diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index 9e4539ec1a..82069515ae 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -445,15 +445,7 @@ func render_blocks(blocks bs: Blocks, profiles: Profiles) -> NoteArtifactsSepara let blocks = bs.blocks let one_note_ref = blocks - .filter({ - if case .mention(let mention) = $0, - case .note = mention.ref { - return true - } - else { - return false - } - }) + .filter({ $0.is_note_mention }) .count == 1 var ind: Int = -1 diff --git a/damusTests/LongPostTests.swift b/damusTests/LongPostTests.swift index 70637607a0..6cc34a1f8a 100644 --- a/damusTests/LongPostTests.swift +++ b/damusTests/LongPostTests.swift @@ -34,7 +34,7 @@ final class LongPostTests: XCTestCase { XCTAssertEqual(subid, "subid") XCTAssertTrue(ev.should_show_event) XCTAssertTrue(!ev.too_big) - XCTAssertTrue(should_show_event(contacts: contacts, ev: ev)) + XCTAssertTrue(should_show_event(privkey: test_keypair.privkey, hellthreads: test_damus_state().muted_threads, contacts: contacts, ev: ev)) XCTAssertTrue(validate_event(ev: ev) == .ok ) } diff --git a/nostrdb/NdbNote.swift b/nostrdb/NdbNote.swift index efb4e25605..4b54253cff 100644 --- a/nostrdb/NdbNote.swift +++ b/nostrdb/NdbNote.swift @@ -415,15 +415,7 @@ extension NdbNote { // Rely on Apple's NLLanguageRecognizer to tell us which language it thinks the note is in // and filter on only the text portions of the content as URLs and hashtags confuse the language recognizer. let originalBlocks = self.blocks(privkey).blocks - let originalOnlyText = originalBlocks.compactMap { - if case .text(let txt) = $0 { - return txt - } - else { - return nil - } - } - .joined(separator: " ") + let originalOnlyText = originalBlocks.compactMap { $0.is_text }.joined(separator: " ") // Only accept language recognition hypothesis if there's at least a 50% probability that it's accurate. let languageRecognizer = NLLanguageRecognizer() From 1f5f1e28a4fff3fa8dd69593b17149b5ef4c7911 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Fri, 25 Aug 2023 12:32:30 -0700 Subject: [PATCH 035/111] nostrdb: pull latest, adding flatcc and lmdb --- damus-c/damus-Bridging-Header.h | 1 + damus.xcodeproj/project.pbxproj | 282 + damus/Nostr/NostrResponse.swift | 2 +- nostrdb/NdbNote.swift | 2 +- nostrdb/bindings/c/.dir | 0 .../bindings/c/flatbuffers_common_builder.h | 685 + .../bindings/c/flatbuffers_common_reader.h | 578 + nostrdb/bindings/c/meta_builder.h | 52 + nostrdb/bindings/c/meta_json_parser.h | 116 + nostrdb/bindings/c/meta_reader.h | 53 + nostrdb/bindings/c/meta_verifier.h | 42 + nostrdb/bindings/c/profile_builder.h | 88 + nostrdb/bindings/c/profile_json_parser.h | 284 + nostrdb/bindings/c/profile_reader.h | 63 + nostrdb/bindings/c/profile_verifier.h | 52 + nostrdb/bindings/swift/NdbProfile.swift | 110 + nostrdb/copy-ndb | 11 +- nostrdb/flatcc.patch | 289 + nostrdb/flatcc/CMakeLists.txt | 16 + nostrdb/flatcc/builder.c | 2035 +++ nostrdb/flatcc/emitter.c | 269 + nostrdb/flatcc/flatcc.h | 268 + nostrdb/flatcc/flatcc_accessors.h | 101 + nostrdb/flatcc/flatcc_alloc.h | 127 + nostrdb/flatcc/flatcc_assert.h | 45 + nostrdb/flatcc/flatcc_builder.h | 1908 +++ nostrdb/flatcc/flatcc_emitter.h | 215 + nostrdb/flatcc/flatcc_endian.h | 125 + nostrdb/flatcc/flatcc_epilogue.h | 8 + nostrdb/flatcc/flatcc_flatbuffers.h | 55 + nostrdb/flatcc/flatcc_identifier.h | 148 + nostrdb/flatcc/flatcc_iov.h | 31 + nostrdb/flatcc/flatcc_json_parser.h | 895 ++ nostrdb/flatcc/flatcc_json_printer.h | 789 ++ nostrdb/flatcc/flatcc_portable.h | 14 + nostrdb/flatcc/flatcc_prologue.h | 8 + nostrdb/flatcc/flatcc_refmap.h | 144 + nostrdb/flatcc/flatcc_rtconfig.h | 162 + nostrdb/flatcc/flatcc_types.h | 97 + nostrdb/flatcc/flatcc_unaligned.h | 16 + nostrdb/flatcc/flatcc_verifier.h | 239 + nostrdb/flatcc/flatcc_version.h | 14 + nostrdb/flatcc/json_parser.c | 1298 ++ nostrdb/flatcc/json_printer.c | 1486 +++ nostrdb/flatcc/portable/LICENSE | 14 + nostrdb/flatcc/portable/README.md | 57 + nostrdb/flatcc/portable/grisu3_math.h | 329 + nostrdb/flatcc/portable/grisu3_parse.h | 582 + nostrdb/flatcc/portable/grisu3_print.h | 265 + nostrdb/flatcc/portable/include/README | 4 + .../flatcc/portable/include/linux/endian.h | 1 + .../flatcc/portable/include/std/inttypes.h | 1 + .../flatcc/portable/include/std/stdalign.h | 1 + nostrdb/flatcc/portable/include/std/stdbool.h | 1 + nostrdb/flatcc/portable/include/std/stdint.h | 1 + nostrdb/flatcc/portable/paligned_alloc.h | 210 + nostrdb/flatcc/portable/pattributes.h | 84 + nostrdb/flatcc/portable/pbase64.h | 448 + nostrdb/flatcc/portable/pcrt.h | 48 + nostrdb/flatcc/portable/pdiagnostic.h | 85 + nostrdb/flatcc/portable/pdiagnostic_pop.h | 20 + nostrdb/flatcc/portable/pdiagnostic_push.h | 51 + nostrdb/flatcc/portable/pendian.h | 206 + nostrdb/flatcc/portable/pendian_detect.h | 118 + nostrdb/flatcc/portable/pinline.h | 19 + nostrdb/flatcc/portable/pinttypes.h | 52 + nostrdb/flatcc/portable/portable.h | 2 + nostrdb/flatcc/portable/portable_basic.h | 25 + nostrdb/flatcc/portable/pparsefp.h | 140 + nostrdb/flatcc/portable/pparseint.h | 374 + nostrdb/flatcc/portable/pprintfp.h | 39 + nostrdb/flatcc/portable/pprintint.h | 628 + nostrdb/flatcc/portable/pstatic_assert.h | 67 + .../flatcc/portable/pstatic_assert_scope.h | 280 + nostrdb/flatcc/portable/pstdalign.h | 162 + nostrdb/flatcc/portable/pstdbool.h | 37 + nostrdb/flatcc/portable/pstdint.h | 898 ++ nostrdb/flatcc/portable/punaligned.h | 190 + nostrdb/flatcc/portable/pversion.h | 6 + nostrdb/flatcc/portable/pwarnings.h | 52 + nostrdb/flatcc/reflection/README | 19 + .../reflection/flatbuffers_common_builder.h | 685 + .../reflection/flatbuffers_common_reader.h | 578 + .../flatcc/reflection/reflection_builder.h | 457 + nostrdb/flatcc/reflection/reflection_reader.h | 411 + .../flatcc/reflection/reflection_verifier.h | 308 + nostrdb/flatcc/refmap.c | 248 + nostrdb/flatcc/support/README | 1 + nostrdb/flatcc/support/cdump.h | 38 + nostrdb/flatcc/support/elapsed.h | 73 + nostrdb/flatcc/support/hexdump.h | 47 + nostrdb/flatcc/support/readfile.h | 66 + nostrdb/flatcc/verifier.c | 617 + nostrdb/jsmn.h | 25 +- nostrdb/lmdb.h | 1608 +++ nostrdb/mdb.c | 10354 ++++++++++++++++ nostrdb/memchr.h | 72 + nostrdb/midl.c | 359 + nostrdb/midl.h | 186 + nostrdb/nostrdb.c | 1013 +- nostrdb/nostrdb.h | 41 +- nostrdb/protected_queue.h | 235 + nostrdb/threadpool.h | 103 + nostrdb/util.h | 33 + 104 files changed, 36269 insertions(+), 28 deletions(-) create mode 100644 nostrdb/bindings/c/.dir create mode 100644 nostrdb/bindings/c/flatbuffers_common_builder.h create mode 100644 nostrdb/bindings/c/flatbuffers_common_reader.h create mode 100644 nostrdb/bindings/c/meta_builder.h create mode 100644 nostrdb/bindings/c/meta_json_parser.h create mode 100644 nostrdb/bindings/c/meta_reader.h create mode 100644 nostrdb/bindings/c/meta_verifier.h create mode 100644 nostrdb/bindings/c/profile_builder.h create mode 100644 nostrdb/bindings/c/profile_json_parser.h create mode 100644 nostrdb/bindings/c/profile_reader.h create mode 100644 nostrdb/bindings/c/profile_verifier.h create mode 100644 nostrdb/bindings/swift/NdbProfile.swift create mode 100644 nostrdb/flatcc.patch create mode 100644 nostrdb/flatcc/CMakeLists.txt create mode 100644 nostrdb/flatcc/builder.c create mode 100644 nostrdb/flatcc/emitter.c create mode 100644 nostrdb/flatcc/flatcc.h create mode 100644 nostrdb/flatcc/flatcc_accessors.h create mode 100644 nostrdb/flatcc/flatcc_alloc.h create mode 100644 nostrdb/flatcc/flatcc_assert.h create mode 100644 nostrdb/flatcc/flatcc_builder.h create mode 100644 nostrdb/flatcc/flatcc_emitter.h create mode 100644 nostrdb/flatcc/flatcc_endian.h create mode 100644 nostrdb/flatcc/flatcc_epilogue.h create mode 100644 nostrdb/flatcc/flatcc_flatbuffers.h create mode 100644 nostrdb/flatcc/flatcc_identifier.h create mode 100644 nostrdb/flatcc/flatcc_iov.h create mode 100644 nostrdb/flatcc/flatcc_json_parser.h create mode 100644 nostrdb/flatcc/flatcc_json_printer.h create mode 100644 nostrdb/flatcc/flatcc_portable.h create mode 100644 nostrdb/flatcc/flatcc_prologue.h create mode 100644 nostrdb/flatcc/flatcc_refmap.h create mode 100644 nostrdb/flatcc/flatcc_rtconfig.h create mode 100644 nostrdb/flatcc/flatcc_types.h create mode 100644 nostrdb/flatcc/flatcc_unaligned.h create mode 100644 nostrdb/flatcc/flatcc_verifier.h create mode 100644 nostrdb/flatcc/flatcc_version.h create mode 100644 nostrdb/flatcc/json_parser.c create mode 100644 nostrdb/flatcc/json_printer.c create mode 100644 nostrdb/flatcc/portable/LICENSE create mode 100644 nostrdb/flatcc/portable/README.md create mode 100644 nostrdb/flatcc/portable/grisu3_math.h create mode 100644 nostrdb/flatcc/portable/grisu3_parse.h create mode 100644 nostrdb/flatcc/portable/grisu3_print.h create mode 100644 nostrdb/flatcc/portable/include/README create mode 100644 nostrdb/flatcc/portable/include/linux/endian.h create mode 100644 nostrdb/flatcc/portable/include/std/inttypes.h create mode 100644 nostrdb/flatcc/portable/include/std/stdalign.h create mode 100644 nostrdb/flatcc/portable/include/std/stdbool.h create mode 100644 nostrdb/flatcc/portable/include/std/stdint.h create mode 100644 nostrdb/flatcc/portable/paligned_alloc.h create mode 100644 nostrdb/flatcc/portable/pattributes.h create mode 100644 nostrdb/flatcc/portable/pbase64.h create mode 100644 nostrdb/flatcc/portable/pcrt.h create mode 100644 nostrdb/flatcc/portable/pdiagnostic.h create mode 100644 nostrdb/flatcc/portable/pdiagnostic_pop.h create mode 100644 nostrdb/flatcc/portable/pdiagnostic_push.h create mode 100644 nostrdb/flatcc/portable/pendian.h create mode 100644 nostrdb/flatcc/portable/pendian_detect.h create mode 100644 nostrdb/flatcc/portable/pinline.h create mode 100644 nostrdb/flatcc/portable/pinttypes.h create mode 100644 nostrdb/flatcc/portable/portable.h create mode 100644 nostrdb/flatcc/portable/portable_basic.h create mode 100644 nostrdb/flatcc/portable/pparsefp.h create mode 100644 nostrdb/flatcc/portable/pparseint.h create mode 100644 nostrdb/flatcc/portable/pprintfp.h create mode 100644 nostrdb/flatcc/portable/pprintint.h create mode 100644 nostrdb/flatcc/portable/pstatic_assert.h create mode 100644 nostrdb/flatcc/portable/pstatic_assert_scope.h create mode 100644 nostrdb/flatcc/portable/pstdalign.h create mode 100644 nostrdb/flatcc/portable/pstdbool.h create mode 100644 nostrdb/flatcc/portable/pstdint.h create mode 100644 nostrdb/flatcc/portable/punaligned.h create mode 100644 nostrdb/flatcc/portable/pversion.h create mode 100644 nostrdb/flatcc/portable/pwarnings.h create mode 100644 nostrdb/flatcc/reflection/README create mode 100644 nostrdb/flatcc/reflection/flatbuffers_common_builder.h create mode 100644 nostrdb/flatcc/reflection/flatbuffers_common_reader.h create mode 100644 nostrdb/flatcc/reflection/reflection_builder.h create mode 100644 nostrdb/flatcc/reflection/reflection_reader.h create mode 100644 nostrdb/flatcc/reflection/reflection_verifier.h create mode 100644 nostrdb/flatcc/refmap.c create mode 100644 nostrdb/flatcc/support/README create mode 100644 nostrdb/flatcc/support/cdump.h create mode 100644 nostrdb/flatcc/support/elapsed.h create mode 100644 nostrdb/flatcc/support/hexdump.h create mode 100644 nostrdb/flatcc/support/readfile.h create mode 100644 nostrdb/flatcc/verifier.c create mode 100644 nostrdb/lmdb.h create mode 100644 nostrdb/mdb.c create mode 100644 nostrdb/memchr.h create mode 100644 nostrdb/midl.c create mode 100644 nostrdb/midl.h create mode 100644 nostrdb/protected_queue.h create mode 100644 nostrdb/threadpool.h create mode 100644 nostrdb/util.h diff --git a/damus-c/damus-Bridging-Header.h b/damus-c/damus-Bridging-Header.h index 3b66becc61..f0d593f01e 100644 --- a/damus-c/damus-Bridging-Header.h +++ b/damus-c/damus-Bridging-Header.h @@ -9,4 +9,5 @@ #include "wasm.h" #include "nostrscript.h" #include "nostrdb.h" +#include "lmdb.h" diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 5e6db4a9d6..4c1bb4d328 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -150,6 +150,12 @@ 4C3EA67D28FFBBA300C48A62 /* InvoicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3EA67C28FFBBA200C48A62 /* InvoicesView.swift */; }; 4C3EA67F28FFC01D00C48A62 /* InvoiceView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3EA67E28FFC01D00C48A62 /* InvoiceView.swift */; }; 4C42812C298C848200DBF26F /* TranslateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C42812B298C848200DBF26F /* TranslateView.swift */; }; + 4C4793012A993CDA00489948 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4793002A993B9A00489948 /* mdb.c */; }; + 4C4793042A993DC000489948 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4793032A993DB900489948 /* midl.c */; }; + 4C4793052A993E3200489948 /* builder.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792942A9939BD00489948 /* builder.c */; }; + 4C4793062A993E5300489948 /* json_parser.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792C82A9939BD00489948 /* json_parser.c */; }; + 4C4793072A993E6200489948 /* emitter.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792CF2A9939BD00489948 /* emitter.c */; }; + 4C4793082A993E8900489948 /* refmap.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792D12A9939BD00489948 /* refmap.c */; }; 4C4DD3DB2A6CA7E8005B4E85 /* ContentParsing.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C4DD3DA2A6CA7E8005B4E85 /* ContentParsing.swift */; }; 4C4E137B2A76D5FB00BDD832 /* MuteThreadNotify.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C4E137A2A76D5FB00BDD832 /* MuteThreadNotify.swift */; }; 4C4E137D2A76D63600BDD832 /* UnmuteThreadNotify.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C4E137C2A76D63600BDD832 /* UnmuteThreadNotify.swift */; }; @@ -686,6 +692,101 @@ 4C3EA67C28FFBBA200C48A62 /* InvoicesView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InvoicesView.swift; sourceTree = ""; }; 4C3EA67E28FFC01D00C48A62 /* InvoiceView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InvoiceView.swift; sourceTree = ""; }; 4C42812B298C848200DBF26F /* TranslateView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranslateView.swift; sourceTree = ""; }; + 4C478E242A9932C100489948 /* Ndb.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Ndb.swift; sourceTree = ""; }; + 4C478E262A99353500489948 /* threadpool.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = threadpool.h; sourceTree = ""; }; + 4C478E272A99354E00489948 /* protected_queue.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = protected_queue.h; sourceTree = ""; }; + 4C478E282A99357400489948 /* memchr.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = memchr.h; sourceTree = ""; }; + 4C478E292A99359900489948 /* util.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = util.h; sourceTree = ""; }; + 4C478E2C2A9935D300489948 /* NdbProfile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NdbProfile.swift; sourceTree = ""; }; + 4C478E2E2A9935D300489948 /* profile_json_parser.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = profile_json_parser.h; sourceTree = ""; }; + 4C478E2F2A9935D300489948 /* profile_reader.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = profile_reader.h; sourceTree = ""; }; + 4C478E302A9935D300489948 /* meta_json_parser.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = meta_json_parser.h; sourceTree = ""; }; + 4C478E312A9935D300489948 /* profile_builder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = profile_builder.h; sourceTree = ""; }; + 4C478E322A9935D300489948 /* meta_builder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = meta_builder.h; sourceTree = ""; }; + 4C478E332A9935D300489948 /* profile_verifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = profile_verifier.h; sourceTree = ""; }; + 4C478E352A9935D300489948 /* meta_reader.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = meta_reader.h; sourceTree = ""; }; + 4C478E362A9935D300489948 /* flatbuffers_common_reader.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatbuffers_common_reader.h; sourceTree = ""; }; + 4C478E372A9935D300489948 /* meta_verifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = meta_verifier.h; sourceTree = ""; }; + 4C478E382A9935D300489948 /* flatbuffers_common_builder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatbuffers_common_builder.h; sourceTree = ""; }; + 4C47928E2A9939BD00489948 /* flatcc.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc.h; sourceTree = ""; }; + 4C47928F2A9939BD00489948 /* flatcc_version.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_version.h; sourceTree = ""; }; + 4C4792902A9939BD00489948 /* flatcc_emitter.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_emitter.h; sourceTree = ""; }; + 4C4792912A9939BD00489948 /* flatcc_alloc.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_alloc.h; sourceTree = ""; }; + 4C4792922A9939BD00489948 /* flatcc_json_printer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_json_printer.h; sourceTree = ""; }; + 4C4792932A9939BD00489948 /* CMakeLists.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = CMakeLists.txt; sourceTree = ""; }; + 4C4792942A9939BD00489948 /* builder.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = builder.c; sourceTree = ""; }; + 4C4792952A9939BD00489948 /* flatcc_verifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_verifier.h; sourceTree = ""; }; + 4C4792962A9939BD00489948 /* flatcc_refmap.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_refmap.h; sourceTree = ""; }; + 4C4792972A9939BD00489948 /* flatcc_unaligned.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_unaligned.h; sourceTree = ""; }; + 4C4792992A9939BD00489948 /* grisu3_print.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = grisu3_print.h; sourceTree = ""; }; + 4C47929A2A9939BD00489948 /* pprintfp.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pprintfp.h; sourceTree = ""; }; + 4C47929B2A9939BD00489948 /* pbase64.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pbase64.h; sourceTree = ""; }; + 4C47929C2A9939BD00489948 /* punaligned.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = punaligned.h; sourceTree = ""; }; + 4C47929D2A9939BD00489948 /* LICENSE */ = {isa = PBXFileReference; lastKnownFileType = text; path = LICENSE; sourceTree = ""; }; + 4C47929E2A9939BD00489948 /* pdiagnostic.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pdiagnostic.h; sourceTree = ""; }; + 4C47929F2A9939BD00489948 /* pinttypes.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pinttypes.h; sourceTree = ""; }; + 4C4792A02A9939BD00489948 /* pinline.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pinline.h; sourceTree = ""; }; + 4C4792A12A9939BD00489948 /* pprintint.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pprintint.h; sourceTree = ""; }; + 4C4792A22A9939BD00489948 /* pdiagnostic_pop.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pdiagnostic_pop.h; sourceTree = ""; }; + 4C4792A52A9939BD00489948 /* stdalign.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = stdalign.h; sourceTree = ""; }; + 4C4792A62A9939BD00489948 /* inttypes.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = inttypes.h; sourceTree = ""; }; + 4C4792A72A9939BD00489948 /* stdbool.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = stdbool.h; sourceTree = ""; }; + 4C4792A82A9939BD00489948 /* stdint.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = stdint.h; sourceTree = ""; }; + 4C4792A92A9939BD00489948 /* README */ = {isa = PBXFileReference; lastKnownFileType = text; path = README; sourceTree = ""; }; + 4C4792AB2A9939BD00489948 /* endian.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = endian.h; sourceTree = ""; }; + 4C4792AC2A9939BD00489948 /* pversion.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pversion.h; sourceTree = ""; }; + 4C4792AD2A9939BD00489948 /* pstdalign.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pstdalign.h; sourceTree = ""; }; + 4C4792AE2A9939BD00489948 /* pdiagnostic_push.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pdiagnostic_push.h; sourceTree = ""; }; + 4C4792AF2A9939BD00489948 /* pendian_detect.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pendian_detect.h; sourceTree = ""; }; + 4C4792B02A9939BD00489948 /* paligned_alloc.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = paligned_alloc.h; sourceTree = ""; }; + 4C4792B12A9939BD00489948 /* pendian.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pendian.h; sourceTree = ""; }; + 4C4792B22A9939BD00489948 /* pstatic_assert.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pstatic_assert.h; sourceTree = ""; }; + 4C4792B32A9939BD00489948 /* pwarnings.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pwarnings.h; sourceTree = ""; }; + 4C4792B42A9939BD00489948 /* pparsefp.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pparsefp.h; sourceTree = ""; }; + 4C4792B52A9939BD00489948 /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; + 4C4792B62A9939BD00489948 /* portable_basic.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = portable_basic.h; sourceTree = ""; }; + 4C4792B72A9939BD00489948 /* portable.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = portable.h; sourceTree = ""; }; + 4C4792B82A9939BD00489948 /* grisu3_math.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = grisu3_math.h; sourceTree = ""; }; + 4C4792B92A9939BD00489948 /* pattributes.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pattributes.h; sourceTree = ""; }; + 4C4792BA2A9939BD00489948 /* pstdint.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pstdint.h; sourceTree = ""; }; + 4C4792BB2A9939BD00489948 /* pstdbool.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pstdbool.h; sourceTree = ""; }; + 4C4792BC2A9939BD00489948 /* pcrt.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pcrt.h; sourceTree = ""; }; + 4C4792BD2A9939BD00489948 /* pstatic_assert_scope.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pstatic_assert_scope.h; sourceTree = ""; }; + 4C4792BE2A9939BD00489948 /* grisu3_parse.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = grisu3_parse.h; sourceTree = ""; }; + 4C4792BF2A9939BD00489948 /* pparseint.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pparseint.h; sourceTree = ""; }; + 4C4792C02A9939BD00489948 /* flatcc_endian.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_endian.h; sourceTree = ""; }; + 4C4792C12A9939BD00489948 /* flatcc_iov.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_iov.h; sourceTree = ""; }; + 4C4792C22A9939BD00489948 /* flatcc_rtconfig.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_rtconfig.h; sourceTree = ""; }; + 4C4792C32A9939BD00489948 /* flatcc_accessors.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_accessors.h; sourceTree = ""; }; + 4C4792C42A9939BD00489948 /* flatcc_epilogue.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_epilogue.h; sourceTree = ""; }; + 4C4792C52A9939BD00489948 /* flatcc_identifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_identifier.h; sourceTree = ""; }; + 4C4792C62A9939BD00489948 /* flatcc_prologue.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_prologue.h; sourceTree = ""; }; + 4C4792C72A9939BD00489948 /* flatcc_builder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_builder.h; sourceTree = ""; }; + 4C4792C82A9939BD00489948 /* json_parser.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = json_parser.c; sourceTree = ""; }; + 4C4792CA2A9939BD00489948 /* README */ = {isa = PBXFileReference; lastKnownFileType = text; path = README; sourceTree = ""; }; + 4C4792CB2A9939BD00489948 /* readfile.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = readfile.h; sourceTree = ""; }; + 4C4792CC2A9939BD00489948 /* cdump.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = cdump.h; sourceTree = ""; }; + 4C4792CD2A9939BD00489948 /* elapsed.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = elapsed.h; sourceTree = ""; }; + 4C4792CE2A9939BD00489948 /* hexdump.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = hexdump.h; sourceTree = ""; }; + 4C4792CF2A9939BD00489948 /* emitter.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = emitter.c; sourceTree = ""; }; + 4C4792D02A9939BD00489948 /* flatcc_json_parser.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_json_parser.h; sourceTree = ""; }; + 4C4792D12A9939BD00489948 /* refmap.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = refmap.c; sourceTree = ""; }; + 4C4792D22A9939BD00489948 /* flatcc_flatbuffers.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_flatbuffers.h; sourceTree = ""; }; + 4C4792D32A9939BD00489948 /* flatcc_portable.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_portable.h; sourceTree = ""; }; + 4C4792D42A9939BD00489948 /* verifier.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = verifier.c; sourceTree = ""; }; + 4C4792D52A9939BD00489948 /* flatcc_types.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_types.h; sourceTree = ""; }; + 4C4792D62A9939BD00489948 /* json_printer.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = json_printer.c; sourceTree = ""; }; + 4C4792D72A9939BD00489948 /* flatcc_assert.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatcc_assert.h; sourceTree = ""; }; + 4C4792D92A9939BD00489948 /* README */ = {isa = PBXFileReference; lastKnownFileType = text; path = README; sourceTree = ""; }; + 4C4792DA2A9939BD00489948 /* reflection_reader.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = reflection_reader.h; sourceTree = ""; }; + 4C4792DB2A9939BD00489948 /* flatbuffers_common_reader.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatbuffers_common_reader.h; sourceTree = ""; }; + 4C4792DC2A9939BD00489948 /* reflection_builder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = reflection_builder.h; sourceTree = ""; }; + 4C4792DD2A9939BD00489948 /* reflection_verifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = reflection_verifier.h; sourceTree = ""; }; + 4C4792DE2A9939BD00489948 /* flatbuffers_common_builder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = flatbuffers_common_builder.h; sourceTree = ""; }; + 4C4792FF2A993B9A00489948 /* lmdb.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = lmdb.h; sourceTree = ""; }; + 4C4793002A993B9A00489948 /* mdb.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = mdb.c; sourceTree = ""; }; + 4C4793022A993D9300489948 /* midl.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = midl.h; sourceTree = ""; }; + 4C4793032A993DB900489948 /* midl.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = midl.c; sourceTree = ""; }; 4C4A3A5A288A1B2200453788 /* damus.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = damus.entitlements; sourceTree = ""; }; 4C4DD3DA2A6CA7E8005B4E85 /* ContentParsing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentParsing.swift; sourceTree = ""; }; 4C4E137A2A76D5FB00BDD832 /* MuteThreadNotify.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MuteThreadNotify.swift; sourceTree = ""; }; @@ -1227,6 +1328,170 @@ path = Notifications; sourceTree = ""; }; + 4C478E2A2A9935D300489948 /* bindings */ = { + isa = PBXGroup; + children = ( + 4C478E2B2A9935D300489948 /* swift */, + 4C478E2D2A9935D300489948 /* c */, + ); + path = bindings; + sourceTree = ""; + }; + 4C478E2B2A9935D300489948 /* swift */ = { + isa = PBXGroup; + children = ( + 4C478E2C2A9935D300489948 /* NdbProfile.swift */, + ); + path = swift; + sourceTree = ""; + }; + 4C478E2D2A9935D300489948 /* c */ = { + isa = PBXGroup; + children = ( + 4C478E2E2A9935D300489948 /* profile_json_parser.h */, + 4C478E2F2A9935D300489948 /* profile_reader.h */, + 4C478E302A9935D300489948 /* meta_json_parser.h */, + 4C478E312A9935D300489948 /* profile_builder.h */, + 4C478E322A9935D300489948 /* meta_builder.h */, + 4C478E332A9935D300489948 /* profile_verifier.h */, + 4C478E352A9935D300489948 /* meta_reader.h */, + 4C478E362A9935D300489948 /* flatbuffers_common_reader.h */, + 4C478E372A9935D300489948 /* meta_verifier.h */, + 4C478E382A9935D300489948 /* flatbuffers_common_builder.h */, + ); + path = c; + sourceTree = ""; + }; + 4C47928D2A9939BD00489948 /* flatcc */ = { + isa = PBXGroup; + children = ( + 4C47928E2A9939BD00489948 /* flatcc.h */, + 4C47928F2A9939BD00489948 /* flatcc_version.h */, + 4C4792902A9939BD00489948 /* flatcc_emitter.h */, + 4C4792912A9939BD00489948 /* flatcc_alloc.h */, + 4C4792922A9939BD00489948 /* flatcc_json_printer.h */, + 4C4792932A9939BD00489948 /* CMakeLists.txt */, + 4C4792942A9939BD00489948 /* builder.c */, + 4C4792952A9939BD00489948 /* flatcc_verifier.h */, + 4C4792962A9939BD00489948 /* flatcc_refmap.h */, + 4C4792972A9939BD00489948 /* flatcc_unaligned.h */, + 4C4792982A9939BD00489948 /* portable */, + 4C4792C02A9939BD00489948 /* flatcc_endian.h */, + 4C4792C12A9939BD00489948 /* flatcc_iov.h */, + 4C4792C22A9939BD00489948 /* flatcc_rtconfig.h */, + 4C4792C32A9939BD00489948 /* flatcc_accessors.h */, + 4C4792C42A9939BD00489948 /* flatcc_epilogue.h */, + 4C4792C52A9939BD00489948 /* flatcc_identifier.h */, + 4C4792C62A9939BD00489948 /* flatcc_prologue.h */, + 4C4792C72A9939BD00489948 /* flatcc_builder.h */, + 4C4792C82A9939BD00489948 /* json_parser.c */, + 4C4792C92A9939BD00489948 /* support */, + 4C4792CF2A9939BD00489948 /* emitter.c */, + 4C4792D02A9939BD00489948 /* flatcc_json_parser.h */, + 4C4792D12A9939BD00489948 /* refmap.c */, + 4C4792D22A9939BD00489948 /* flatcc_flatbuffers.h */, + 4C4792D32A9939BD00489948 /* flatcc_portable.h */, + 4C4792D42A9939BD00489948 /* verifier.c */, + 4C4792D52A9939BD00489948 /* flatcc_types.h */, + 4C4792D62A9939BD00489948 /* json_printer.c */, + 4C4792D72A9939BD00489948 /* flatcc_assert.h */, + 4C4792D82A9939BD00489948 /* reflection */, + ); + path = flatcc; + sourceTree = ""; + }; + 4C4792982A9939BD00489948 /* portable */ = { + isa = PBXGroup; + children = ( + 4C4792992A9939BD00489948 /* grisu3_print.h */, + 4C47929A2A9939BD00489948 /* pprintfp.h */, + 4C47929B2A9939BD00489948 /* pbase64.h */, + 4C47929C2A9939BD00489948 /* punaligned.h */, + 4C47929D2A9939BD00489948 /* LICENSE */, + 4C47929E2A9939BD00489948 /* pdiagnostic.h */, + 4C47929F2A9939BD00489948 /* pinttypes.h */, + 4C4792A02A9939BD00489948 /* pinline.h */, + 4C4792A12A9939BD00489948 /* pprintint.h */, + 4C4792A22A9939BD00489948 /* pdiagnostic_pop.h */, + 4C4792A32A9939BD00489948 /* include */, + 4C4792AC2A9939BD00489948 /* pversion.h */, + 4C4792AD2A9939BD00489948 /* pstdalign.h */, + 4C4792AE2A9939BD00489948 /* pdiagnostic_push.h */, + 4C4792AF2A9939BD00489948 /* pendian_detect.h */, + 4C4792B02A9939BD00489948 /* paligned_alloc.h */, + 4C4792B12A9939BD00489948 /* pendian.h */, + 4C4792B22A9939BD00489948 /* pstatic_assert.h */, + 4C4792B32A9939BD00489948 /* pwarnings.h */, + 4C4792B42A9939BD00489948 /* pparsefp.h */, + 4C4792B52A9939BD00489948 /* README.md */, + 4C4792B62A9939BD00489948 /* portable_basic.h */, + 4C4792B72A9939BD00489948 /* portable.h */, + 4C4792B82A9939BD00489948 /* grisu3_math.h */, + 4C4792B92A9939BD00489948 /* pattributes.h */, + 4C4792BA2A9939BD00489948 /* pstdint.h */, + 4C4792BB2A9939BD00489948 /* pstdbool.h */, + 4C4792BC2A9939BD00489948 /* pcrt.h */, + 4C4792BD2A9939BD00489948 /* pstatic_assert_scope.h */, + 4C4792BE2A9939BD00489948 /* grisu3_parse.h */, + 4C4792BF2A9939BD00489948 /* pparseint.h */, + ); + path = portable; + sourceTree = ""; + }; + 4C4792A32A9939BD00489948 /* include */ = { + isa = PBXGroup; + children = ( + 4C4792A42A9939BD00489948 /* std */, + 4C4792A92A9939BD00489948 /* README */, + 4C4792AA2A9939BD00489948 /* linux */, + ); + path = include; + sourceTree = ""; + }; + 4C4792A42A9939BD00489948 /* std */ = { + isa = PBXGroup; + children = ( + 4C4792A52A9939BD00489948 /* stdalign.h */, + 4C4792A62A9939BD00489948 /* inttypes.h */, + 4C4792A72A9939BD00489948 /* stdbool.h */, + 4C4792A82A9939BD00489948 /* stdint.h */, + ); + path = std; + sourceTree = ""; + }; + 4C4792AA2A9939BD00489948 /* linux */ = { + isa = PBXGroup; + children = ( + 4C4792AB2A9939BD00489948 /* endian.h */, + ); + path = linux; + sourceTree = ""; + }; + 4C4792C92A9939BD00489948 /* support */ = { + isa = PBXGroup; + children = ( + 4C4792CA2A9939BD00489948 /* README */, + 4C4792CB2A9939BD00489948 /* readfile.h */, + 4C4792CC2A9939BD00489948 /* cdump.h */, + 4C4792CD2A9939BD00489948 /* elapsed.h */, + 4C4792CE2A9939BD00489948 /* hexdump.h */, + ); + path = support; + sourceTree = ""; + }; + 4C4792D82A9939BD00489948 /* reflection */ = { + isa = PBXGroup; + children = ( + 4C4792D92A9939BD00489948 /* README */, + 4C4792DA2A9939BD00489948 /* reflection_reader.h */, + 4C4792DB2A9939BD00489948 /* flatbuffers_common_reader.h */, + 4C4792DC2A9939BD00489948 /* reflection_builder.h */, + 4C4792DD2A9939BD00489948 /* reflection_verifier.h */, + 4C4792DE2A9939BD00489948 /* flatbuffers_common_builder.h */, + ); + path = reflection; + sourceTree = ""; + }; 4C54AA0829A55416003E4487 /* Notifications */ = { isa = PBXGroup; children = ( @@ -1460,6 +1725,8 @@ 4C9054862A6AEB4500811EEC /* nostrdb */ = { isa = PBXGroup; children = ( + 4C47928D2A9939BD00489948 /* flatcc */, + 4C478E2A2A9935D300489948 /* bindings */, 4CE9FBBB2A6B3D9C007E485C /* Test */, 4C9054882A6AED4700811EEC /* NdbTagIterator.swift */, 4C90548A2A6AEDEE00811EEC /* NdbNote.swift */, @@ -1467,13 +1734,22 @@ 4CDD1ADF2A6B305F001CD4DF /* NdbTagElem.swift */, 4CDD1AE12A6B3074001CD4DF /* NdbTagsIterator.swift */, 4CE9FBB82A6B3B26007E485C /* nostrdb.c */, + 4C4793032A993DB900489948 /* midl.c */, + 4C4793002A993B9A00489948 /* mdb.c */, + 4C4793022A993D9300489948 /* midl.h */, + 4C4792FF2A993B9A00489948 /* lmdb.h */, 4CE9FBB92A6B3B26007E485C /* nostrdb.h */, 4C78EFD62A7078C5007E8197 /* random.h */, 4CDD1AE72A6B3611001CD4DF /* jsmn.h */, + 4C478E292A99359900489948 /* util.h */, + 4C478E282A99357400489948 /* memchr.h */, + 4C478E272A99354E00489948 /* protected_queue.h */, + 4C478E262A99353500489948 /* threadpool.h */, 4C78EFD82A707C4D007E8197 /* secp256k1_ecdh.h */, 4C78EFD72A707C4D007E8197 /* secp256k1_schnorrsig.h */, 4C78EFDA2A707C67007E8197 /* secp256k1_extrakeys.h */, 4C78EFD92A707C4D007E8197 /* secp256k1.h */, + 4C478E242A9932C100489948 /* Ndb.swift */, ); path = nostrdb; sourceTree = ""; @@ -2127,6 +2403,12 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 4C4793082A993E8900489948 /* refmap.c in Sources */, + 4C4793072A993E6200489948 /* emitter.c in Sources */, + 4C4793062A993E5300489948 /* json_parser.c in Sources */, + 4C4793052A993E3200489948 /* builder.c in Sources */, + 4C4793042A993DC000489948 /* midl.c in Sources */, + 4C4793012A993CDA00489948 /* mdb.c in Sources */, 4CE9FBBA2A6B3C63007E485C /* nostrdb.c in Sources */, 4C3AC79D2833036D00E1F516 /* FollowingView.swift in Sources */, 5CF72FC229B9142F00124A13 /* ShareAction.swift in Sources */, diff --git a/damus/Nostr/NostrResponse.swift b/damus/Nostr/NostrResponse.swift index 5b1ec08b63..218293428a 100644 --- a/damus/Nostr/NostrResponse.swift +++ b/damus/Nostr/NostrResponse.swift @@ -51,7 +51,7 @@ enum NostrResponse { //json_cs var tce = ndb_tce() - let len = ndb_ws_event_from_json(cstr, Int32(json.utf8.count), &tce, data, Int32(bufsize)) + let len = ndb_ws_event_from_json(cstr, Int32(json.utf8.count), &tce, data, Int32(bufsize), nil) if len <= 0 { free(data) return nil diff --git a/nostrdb/NdbNote.swift b/nostrdb/NdbNote.swift index 4b54253cff..d8b9fa59a4 100644 --- a/nostrdb/NdbNote.swift +++ b/nostrdb/NdbNote.swift @@ -156,7 +156,7 @@ class NdbNote: Encodable, Equatable, Hashable { ndb_builder_set_pubkey(&builder, &pk_raw) ndb_builder_set_kind(&builder, UInt32(kind)) - ndb_builder_set_created_at(&builder, createdAt) + ndb_builder_set_created_at(&builder, UInt64(createdAt)) var ok = true for tag in tags { diff --git a/nostrdb/bindings/c/.dir b/nostrdb/bindings/c/.dir new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nostrdb/bindings/c/flatbuffers_common_builder.h b/nostrdb/bindings/c/flatbuffers_common_builder.h new file mode 100644 index 0000000000..a4be1ce6e8 --- /dev/null +++ b/nostrdb/bindings/c/flatbuffers_common_builder.h @@ -0,0 +1,685 @@ +#ifndef FLATBUFFERS_COMMON_BUILDER_H +#define FLATBUFFERS_COMMON_BUILDER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +/* Common FlatBuffers build functionality for C. */ + +#include "flatcc/flatcc_prologue.h" +#ifndef FLATBUILDER_H +#include "flatcc/flatcc_builder.h" +#endif +typedef flatcc_builder_t flatbuffers_builder_t; +typedef flatcc_builder_ref_t flatbuffers_ref_t; +typedef flatcc_builder_ref_t flatbuffers_vec_ref_t; +typedef flatcc_builder_union_ref_t flatbuffers_union_ref_t; +typedef flatcc_builder_union_vec_ref_t flatbuffers_union_vec_ref_t; +/* integer return code (ref and ptr always fail on 0) */ +#define flatbuffers_failed(x) ((x) < 0) +typedef flatbuffers_ref_t flatbuffers_root_t; +#define flatbuffers_root(ref) ((flatbuffers_root_t)(ref)) + +#define __flatbuffers_memoize_begin(B, src)\ +do { flatcc_builder_ref_t _ref; if ((_ref = flatcc_builder_refmap_find((B), (src)))) return _ref; } while (0) +#define __flatbuffers_memoize_end(B, src, op) do { return flatcc_builder_refmap_insert((B), (src), (op)); } while (0) +#define __flatbuffers_memoize(B, src, op) do { __flatbuffers_memoize_begin(B, src); __flatbuffers_memoize_end(B, src, op); } while (0) + +#define __flatbuffers_build_buffer(NS)\ +typedef NS ## ref_t NS ## buffer_ref_t;\ +static inline int NS ## buffer_start(NS ## builder_t *B, const NS ##fid_t fid)\ +{ return flatcc_builder_start_buffer(B, fid, 0, 0); }\ +static inline int NS ## buffer_start_with_size(NS ## builder_t *B, const NS ##fid_t fid)\ +{ return flatcc_builder_start_buffer(B, fid, 0, flatcc_builder_with_size); }\ +static inline int NS ## buffer_start_aligned(NS ## builder_t *B, NS ##fid_t fid, uint16_t block_align)\ +{ return flatcc_builder_start_buffer(B, fid, block_align, 0); }\ +static inline int NS ## buffer_start_aligned_with_size(NS ## builder_t *B, NS ##fid_t fid, uint16_t block_align)\ +{ return flatcc_builder_start_buffer(B, fid, block_align, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t NS ## buffer_end(NS ## builder_t *B, NS ## ref_t root)\ +{ return flatcc_builder_end_buffer(B, root); } + +#define __flatbuffers_build_table_root(NS, N, FID, TFID)\ +static inline int N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? -1 : N ## _start(B); }\ +static inline int N ## _start_as_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, FID) ? -1 : N ## _start(B); }\ +static inline int N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, TFID) ? -1 : N ## _start(B); }\ +static inline int N ## _start_as_typed_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, TFID) ? -1 : N ## _start(B); }\ +static inline NS ## buffer_ref_t N ## _end_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _create_as_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start(B, FID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _create_as_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start_with_size(B, FID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start(B, TFID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start_with_size(B, TFID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start(B, FID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root_with_size(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start_with_size(B, FID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start(B, TFID)) return 0;return NS ## buffer_end(B, N ## _clone(B, t)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root_with_size(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start_with_size(B, TFID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); } + +#define __flatbuffers_build_table_prolog(NS, N, FID, TFID)\ +__flatbuffers_build_table_vector_ops(NS, N ## _vec, N)\ +__flatbuffers_build_table_root(NS, N, FID, TFID) + +#define __flatbuffers_build_struct_root(NS, N, A, FID, TFID)\ +static inline N ## _t *N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? 0 : N ## _start(B); }\ +static inline N ## _t *N ## _start_as_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, FID) ? 0 : N ## _start(B); }\ +static inline N ## _t *N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, TFID) ? 0 : N ## _start(B); }\ +static inline N ## _t *N ## _start_as_typed_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, TFID) ? 0 : N ## _start(B); }\ +static inline NS ## buffer_ref_t N ## _end_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _end_pe_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end_pe(B)); }\ +static inline NS ## buffer_ref_t N ## _end_pe_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end_pe(B)); }\ +static inline NS ## buffer_ref_t N ## _create_as_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, FID, 0,\ + N ## _create(B __ ## N ## _call_args), A, 0); }\ +static inline NS ## buffer_ref_t N ## _create_as_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, FID, 0,\ + N ## _create(B __ ## N ## _call_args), A, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, TFID, 0,\ + N ## _create(B __ ## N ## _call_args), A, 0); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, TFID, 0,\ + N ## _create(B __ ## N ## _call_args), A, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, FID, 0, N ## _clone(B, p), A, 0); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root_with_size(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, FID, 0, N ## _clone(B, p), A, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, TFID, 0, N ## _clone(B, p), A, 0); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root_with_size(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, TFID, 0, N ## _clone(B, p), A, flatcc_builder_with_size); } + +#define __flatbuffers_build_nested_table_root(NS, N, TN, FID, TFID)\ +static inline int N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? -1 : TN ## _start(B); }\ +static inline int N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, TFID) ? -1 : TN ## _start(B); }\ +static inline int N ## _end_as_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align ? align : 8, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _typed_nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align ? align : 8, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _clone_as_root(NS ## builder_t *B, TN ## _table_t t)\ +{ return N ## _add(B, TN ## _clone_as_root(B, t)); }\ +static inline int N ## _clone_as_typed_root(NS ## builder_t *B, TN ## _table_t t)\ +{ return N ## _add(B, TN ## _clone_as_typed_root(B, t)); } + +#define __flatbuffers_build_nested_struct_root(NS, N, TN, A, FID, TFID)\ +static inline TN ## _t *N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? 0 : TN ## _start(B); }\ +static inline TN ## _t *N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? 0 : TN ## _start(B); }\ +static inline int N ## _end_as_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _end_pe_as_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end_pe(B))); }\ +static inline int N ## _create_as_root(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ return N ## _add(B, flatcc_builder_create_buffer(B, FID, 0,\ + TN ## _create(B __ ## TN ## _call_args), A, flatcc_builder_is_nested)); }\ +static inline int N ## _create_as_typed_root(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ return N ## _add(B, flatcc_builder_create_buffer(B, TFID, 0,\ + TN ## _create(B __ ## TN ## _call_args), A, flatcc_builder_is_nested)); }\ +static inline int N ## _nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align < A ? A : align, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _typed_nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align < A ? A : align, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _clone_as_root(NS ## builder_t *B, TN ## _struct_t p)\ +{ return N ## _add(B, TN ## _clone_as_root(B, p)); }\ +static inline int N ## _clone_as_typed_root(NS ## builder_t *B, TN ## _struct_t p)\ +{ return N ## _add(B, TN ## _clone_as_typed_root(B, p)); } + +#define __flatbuffers_build_vector_ops(NS, V, N, TN, T)\ +static inline T *V ## _extend(NS ## builder_t *B, size_t len)\ +{ return (T *)flatcc_builder_extend_vector(B, len); }\ +static inline T *V ## _append(NS ## builder_t *B, const T *data, size_t len)\ +{ return (T *)flatcc_builder_append_vector(B, data, len); }\ +static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_vector(B, len); }\ +static inline T *V ## _edit(NS ## builder_t *B)\ +{ return (T *)flatcc_builder_vector_edit(B); }\ +static inline size_t V ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_vector_count(B); }\ +static inline T *V ## _push(NS ## builder_t *B, const T *p)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? (memcpy(_p, p, TN ## __size()), _p) : 0; }\ +static inline T *V ## _push_copy(NS ## builder_t *B, const T *p)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _copy(_p, p) : 0; }\ +static inline T *V ## _push_clone(NS ## builder_t *B, const T *p)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _copy(_p, p) : 0; }\ +static inline T *V ## _push_create(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _assign(_p __ ## TN ## _call_args) : 0; } + +#define __flatbuffers_build_vector(NS, N, T, S, A)\ +typedef NS ## ref_t N ## _vec_ref_t;\ +static inline int N ## _vec_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_vector(B, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +static inline N ## _vec_ref_t N ## _vec_end_pe(NS ## builder_t *B)\ +{ return flatcc_builder_end_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_end(NS ## builder_t *B)\ +{ if (!NS ## is_native_pe()) { size_t i, n; T *p = (T *)flatcc_builder_vector_edit(B);\ + for (i = 0, n = flatcc_builder_vector_count(B); i < n; ++i)\ + { N ## _to_pe(N ## __ptr_add(p, i)); }} return flatcc_builder_end_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_create_pe(NS ## builder_t *B, const T *data, size_t len)\ +{ return flatcc_builder_create_vector(B, data, len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +static inline N ## _vec_ref_t N ## _vec_create(NS ## builder_t *B, const T *data, size_t len)\ +{ if (!NS ## is_native_pe()) { size_t i; T *p; int ret = flatcc_builder_start_vector(B, S, A, FLATBUFFERS_COUNT_MAX(S)); if (ret) { return ret; }\ + p = (T *)flatcc_builder_extend_vector(B, len); if (!p) return 0;\ + for (i = 0; i < len; ++i) { N ## _copy_to_pe(N ## __ptr_add(p, i), N ## __const_ptr_add(data, i)); }\ + return flatcc_builder_end_vector(B); } else return flatcc_builder_create_vector(B, data, len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +static inline N ## _vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_vec_t vec)\ +{ __flatbuffers_memoize(B, vec, flatcc_builder_create_vector(B, vec, N ## _vec_len(vec), S, A, FLATBUFFERS_COUNT_MAX(S))); }\ +static inline N ## _vec_ref_t N ## _vec_slice(NS ## builder_t *B, N ##_vec_t vec, size_t index, size_t len)\ +{ size_t n = N ## _vec_len(vec); if (index >= n) index = n; n -= index; if (len > n) len = n;\ + return flatcc_builder_create_vector(B, N ## __const_ptr_add(vec, index), len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +__flatbuffers_build_vector_ops(NS, N ## _vec, N, N, T) + +#define __flatbuffers_build_union_vector_ops(NS, V, N, TN)\ +static inline TN ## _union_ref_t *V ## _extend(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_extend_union_vector(B, len); }\ +static inline TN ## _union_ref_t *V ## _append(NS ## builder_t *B, const TN ## _union_ref_t *data, size_t len)\ +{ return flatcc_builder_append_union_vector(B, data, len); }\ +static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_union_vector(B, len); }\ +static inline TN ## _union_ref_t *V ## _edit(NS ## builder_t *B)\ +{ return (TN ## _union_ref_t *) flatcc_builder_union_vector_edit(B); }\ +static inline size_t V ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_union_vector_count(B); }\ +static inline TN ## _union_ref_t *V ## _push(NS ## builder_t *B, const TN ## _union_ref_t ref)\ +{ return flatcc_builder_union_vector_push(B, ref); }\ +static inline TN ## _union_ref_t *V ## _push_clone(NS ## builder_t *B, TN ## _union_t u)\ +{ return TN ## _vec_push(B, TN ## _clone(B, u)); } + +#define __flatbuffers_build_union_vector(NS, N)\ +static inline int N ## _vec_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_union_vector(B); }\ +static inline N ## _union_vec_ref_t N ## _vec_end(NS ## builder_t *B)\ +{ return flatcc_builder_end_union_vector(B); }\ +static inline N ## _union_vec_ref_t N ## _vec_create(NS ## builder_t *B, const N ## _union_ref_t *data, size_t len)\ +{ return flatcc_builder_create_union_vector(B, data, len); }\ +__flatbuffers_build_union_vector_ops(NS, N ## _vec, N, N)\ +/* Preserves DAG structure separately for type and value vector, so a type vector could be shared for many value vectors. */\ +static inline N ## _union_vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_union_vec_t vec)\ +{ N ## _union_vec_ref_t _uvref, _ret = { 0, 0 }; NS ## union_ref_t _uref; size_t _i, _len;\ + if (vec.type == 0) return _ret;\ + _uvref.type = flatcc_builder_refmap_find(B, vec.type); _uvref.value = flatcc_builder_refmap_find(B, vec.value);\ + _len = N ## _union_vec_len(vec); if (_uvref.type == 0) {\ + _uvref.type = flatcc_builder_refmap_insert(B, vec.type, (flatcc_builder_create_type_vector(B, vec.type, _len))); }\ + if (_uvref.type == 0) return _ret; if (_uvref.value == 0) {\ + if (flatcc_builder_start_offset_vector(B)) return _ret;\ + for (_i = 0; _i < _len; ++_i) { _uref = N ## _clone(B, N ## _union_vec_at(vec, _i));\ + if (!_uref.value || !(flatcc_builder_offset_vector_push(B, _uref.value))) return _ret; }\ + _uvref.value = flatcc_builder_refmap_insert(B, vec.value, flatcc_builder_end_offset_vector(B));\ + if (_uvref.value == 0) return _ret; } return _uvref; } + +#define __flatbuffers_build_string_vector_ops(NS, N)\ +static inline int N ## _push_start(NS ## builder_t *B)\ +{ return NS ## string_start(B); }\ +static inline NS ## string_ref_t *N ## _push_end(NS ## builder_t *B)\ +{ return NS ## string_vec_push(B, NS ## string_end(B)); }\ +static inline NS ## string_ref_t *N ## _push_create(NS ## builder_t *B, const char *s, size_t len)\ +{ return NS ## string_vec_push(B, NS ## string_create(B, s, len)); }\ +static inline NS ## string_ref_t *N ## _push_create_str(NS ## builder_t *B, const char *s)\ +{ return NS ## string_vec_push(B, NS ## string_create_str(B, s)); }\ +static inline NS ## string_ref_t *N ## _push_create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ +{ return NS ## string_vec_push(B, NS ## string_create_strn(B, s, max_len)); }\ +static inline NS ## string_ref_t *N ## _push_clone(NS ## builder_t *B, NS ## string_t string)\ +{ return NS ## string_vec_push(B, NS ## string_clone(B, string)); }\ +static inline NS ## string_ref_t *N ## _push_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ return NS ## string_vec_push(B, NS ## string_slice(B, string, index, len)); } + +#define __flatbuffers_build_table_vector_ops(NS, N, TN)\ +static inline int N ## _push_start(NS ## builder_t *B)\ +{ return TN ## _start(B); }\ +static inline TN ## _ref_t *N ## _push_end(NS ## builder_t *B)\ +{ return N ## _push(B, TN ## _end(B)); }\ +static inline TN ## _ref_t *N ## _push_create(NS ## builder_t *B __ ## TN ##_formal_args)\ +{ return N ## _push(B, TN ## _create(B __ ## TN ## _call_args)); } + +#define __flatbuffers_build_offset_vector_ops(NS, V, N, TN)\ +static inline TN ## _ref_t *V ## _extend(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_extend_offset_vector(B, len); }\ +static inline TN ## _ref_t *V ## _append(NS ## builder_t *B, const TN ## _ref_t *data, size_t len)\ +{ return flatcc_builder_append_offset_vector(B, data, len); }\ +static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_offset_vector(B, len); }\ +static inline TN ## _ref_t *V ## _edit(NS ## builder_t *B)\ +{ return (TN ## _ref_t *)flatcc_builder_offset_vector_edit(B); }\ +static inline size_t V ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_offset_vector_count(B); }\ +static inline TN ## _ref_t *V ## _push(NS ## builder_t *B, const TN ## _ref_t ref)\ +{ return ref ? flatcc_builder_offset_vector_push(B, ref) : 0; } + +#define __flatbuffers_build_offset_vector(NS, N)\ +typedef NS ## ref_t N ## _vec_ref_t;\ +static inline int N ## _vec_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_offset_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_end(NS ## builder_t *B)\ +{ return flatcc_builder_end_offset_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_create(NS ## builder_t *B, const N ## _ref_t *data, size_t len)\ +{ return flatcc_builder_create_offset_vector(B, data, len); }\ +__flatbuffers_build_offset_vector_ops(NS, N ## _vec, N, N)\ +static inline N ## _vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_vec_t vec)\ +{ int _ret; N ## _ref_t _e; size_t _i, _len; __flatbuffers_memoize_begin(B, vec);\ + _len = N ## _vec_len(vec); if (flatcc_builder_start_offset_vector(B)) return 0;\ + for (_i = 0; _i < _len; ++_i) { if (!(_e = N ## _clone(B, N ## _vec_at(vec, _i)))) return 0;\ + if (!flatcc_builder_offset_vector_push(B, _e)) return 0; }\ + __flatbuffers_memoize_end(B, vec, flatcc_builder_end_offset_vector(B)); }\ + +#define __flatbuffers_build_string_ops(NS, N)\ +static inline char *N ## _append(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_append_string(B, s, len); }\ +static inline char *N ## _append_str(NS ## builder_t *B, const char *s)\ +{ return flatcc_builder_append_string_str(B, s); }\ +static inline char *N ## _append_strn(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_append_string_strn(B, s, len); }\ +static inline size_t N ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_string_len(B); }\ +static inline char *N ## _extend(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_extend_string(B, len); }\ +static inline char *N ## _edit(NS ## builder_t *B)\ +{ return flatcc_builder_string_edit(B); }\ +static inline int N ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_string(B, len); } + +#define __flatbuffers_build_string(NS)\ +typedef NS ## ref_t NS ## string_ref_t;\ +static inline int NS ## string_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_string(B); }\ +static inline NS ## string_ref_t NS ## string_end(NS ## builder_t *B)\ +{ return flatcc_builder_end_string(B); }\ +static inline NS ## ref_t NS ## string_create(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_create_string(B, s, len); }\ +static inline NS ## ref_t NS ## string_create_str(NS ## builder_t *B, const char *s)\ +{ return flatcc_builder_create_string_str(B, s); }\ +static inline NS ## ref_t NS ## string_create_strn(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_create_string_strn(B, s, len); }\ +static inline NS ## string_ref_t NS ## string_clone(NS ## builder_t *B, NS ## string_t string)\ +{ __flatbuffers_memoize(B, string, flatcc_builder_create_string(B, string, NS ## string_len(string))); }\ +static inline NS ## string_ref_t NS ## string_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ size_t n = NS ## string_len(string); if (index >= n) index = n; n -= index; if (len > n) len = n;\ + return flatcc_builder_create_string(B, string + index, len); }\ +__flatbuffers_build_string_ops(NS, NS ## string)\ +__flatbuffers_build_offset_vector(NS, NS ## string) + +#define __flatbuffers_copy_from_pe(P, P2, N) (*(P) = N ## _read_from_pe(P2), (P)) +#define __flatbuffers_from_pe(P, N) (*(P) = N ## _read_from_pe(P), (P)) +#define __flatbuffers_copy_to_pe(P, P2, N) (N ## _write_to_pe((P), *(P2)), (P)) +#define __flatbuffers_to_pe(P, N) (N ## _write_to_pe((P), *(P)), (P)) +#define __flatbuffers_define_fixed_array_primitives(NS, N, T)\ +static inline T *N ## _array_copy(T *p, const T *p2, size_t n)\ +{ memcpy(p, p2, n * sizeof(T)); return p; }\ +static inline T *N ## _array_copy_from_pe(T *p, const T *p2, size_t n)\ +{ size_t i; if (NS ## is_native_pe()) memcpy(p, p2, n * sizeof(T)); else\ + for (i = 0; i < n; ++i) N ## _copy_from_pe(&p[i], &p2[i]); return p; }\ +static inline T *N ## _array_copy_to_pe(T *p, const T *p2, size_t n)\ +{ size_t i; if (NS ## is_native_pe()) memcpy(p, p2, n * sizeof(T)); else\ + for (i = 0; i < n; ++i) N ## _copy_to_pe(&p[i], &p2[i]); return p; } +#define __flatbuffers_define_scalar_primitives(NS, N, T)\ +static inline T *N ## _from_pe(T *p) { return __ ## NS ## from_pe(p, N); }\ +static inline T *N ## _to_pe(T *p) { return __ ## NS ## to_pe(p, N); }\ +static inline T *N ## _copy(T *p, const T *p2) { *p = *p2; return p; }\ +static inline T *N ## _copy_from_pe(T *p, const T *p2)\ +{ return __ ## NS ## copy_from_pe(p, p2, N); }\ +static inline T *N ## _copy_to_pe(T *p, const T *p2) \ +{ return __ ## NS ## copy_to_pe(p, p2, N); }\ +static inline T *N ## _assign(T *p, const T v0) { *p = v0; return p; }\ +static inline T *N ## _assign_from_pe(T *p, T v0)\ +{ *p = N ## _read_from_pe(&v0); return p; }\ +static inline T *N ## _assign_to_pe(T *p, T v0)\ +{ N ## _write_to_pe(p, v0); return p; } +#define __flatbuffers_build_scalar(NS, N, T)\ +__ ## NS ## define_scalar_primitives(NS, N, T)\ +__ ## NS ## define_fixed_array_primitives(NS, N, T)\ +__ ## NS ## build_vector(NS, N, T, sizeof(T), sizeof(T)) +/* Depends on generated copy_to/from_pe functions, and the type. */ +#define __flatbuffers_define_struct_primitives(NS, N)\ +static inline N ## _t *N ##_to_pe(N ## _t *p)\ +{ if (!NS ## is_native_pe()) { N ## _copy_to_pe(p, p); }; return p; }\ +static inline N ## _t *N ##_from_pe(N ## _t *p)\ +{ if (!NS ## is_native_pe()) { N ## _copy_from_pe(p, p); }; return p; }\ +static inline N ## _t *N ## _clear(N ## _t *p) { return (N ## _t *)memset(p, 0, N ## __size()); } + +/* Depends on generated copy/assign_to/from_pe functions, and the type. */ +#define __flatbuffers_build_struct(NS, N, S, A, FID, TFID)\ +__ ## NS ## define_struct_primitives(NS, N)\ +typedef NS ## ref_t N ## _ref_t;\ +static inline N ## _t *N ## _start(NS ## builder_t *B)\ +{ return (N ## _t *)flatcc_builder_start_struct(B, S, A); }\ +static inline N ## _ref_t N ## _end(NS ## builder_t *B)\ +{ if (!NS ## is_native_pe()) { N ## _to_pe((N ## _t *)flatcc_builder_struct_edit(B)); }\ + return flatcc_builder_end_struct(B); }\ +static inline N ## _ref_t N ## _end_pe(NS ## builder_t *B)\ +{ return flatcc_builder_end_struct(B); }\ +static inline N ## _ref_t N ## _create(NS ## builder_t *B __ ## N ## _formal_args)\ +{ N ## _t *_p = N ## _start(B); if (!_p) return 0; N ##_assign_to_pe(_p __ ## N ## _call_args);\ + return N ## _end_pe(B); }\ +static inline N ## _ref_t N ## _clone(NS ## builder_t *B, N ## _struct_t p)\ +{ N ## _t *_p; __flatbuffers_memoize_begin(B, p); _p = N ## _start(B); if (!_p) return 0;\ + N ## _copy(_p, p); __flatbuffers_memoize_end(B, p, N ##_end_pe(B)); }\ +__flatbuffers_build_vector(NS, N, N ## _t, S, A)\ +__flatbuffers_build_struct_root(NS, N, A, FID, TFID)\ + +#define __flatbuffers_struct_clear_field(p) memset((p), 0, sizeof(*(p))) +#define __flatbuffers_build_table(NS, N, K)\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_table(B, K); }\ +static inline N ## _ref_t N ## _end(NS ## builder_t *B)\ +{ FLATCC_ASSERT(flatcc_builder_check_required(B, __ ## N ## _required,\ + sizeof(__ ## N ## _required) / sizeof(__ ## N ## _required[0]) - 1));\ + return flatcc_builder_end_table(B); }\ +__flatbuffers_build_offset_vector(NS, N) + +#define __flatbuffers_build_table_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _ref_t ref)\ +{ TN ## _ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ?\ + ((*_p = ref), 0) : -1; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return TN ## _start(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, TN ## _end(B)); }\ +static inline TN ## _ref_t N ## _create(NS ## builder_t *B __ ## TN ##_formal_args)\ +{ return N ## _add(B, TN ## _create(B __ ## TN ## _call_args)); }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _table_t p)\ +{ return N ## _add(B, TN ## _clone(B, p)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _table_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_union_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _union_ref_t uref)\ +{ NS ## ref_t *_p; TN ## _union_type_t *_pt; if (uref.type == TN ## _NONE) return 0; if (uref.value == 0) return -1;\ + if (!(_pt = (TN ## _union_type_t *)flatcc_builder_table_add(B, ID - 1, sizeof(*_pt), sizeof(*_pt)))) return -1;\ + *_pt = uref.type; if (!(_p = flatcc_builder_table_add_offset(B, ID))) return -1; *_p = uref.value; return 0; }\ +static inline int N ## _add_type(NS ## builder_t *B, TN ## _union_type_t type)\ +{ TN ## _union_type_t *_pt; if (type == TN ## _NONE) return 0; return (_pt = (TN ## _union_type_t *)flatcc_builder_table_add(B, ID - 1,\ + sizeof(*_pt), sizeof(*_pt))) ? ((*_pt = type), 0) : -1; }\ +static inline int N ## _add_value(NS ## builder_t *B, TN ## _union_ref_t uref)\ +{ NS ## ref_t *p; if (uref.type == TN ## _NONE) return 0; return (p = flatcc_builder_table_add_offset(B, ID)) ?\ + ((*p = uref.value), 0) : -1; }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _union_t p)\ +{ return N ## _add(B, TN ## _clone(B, p)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _union_t _p = N ## _union(t); return _p.type ? N ## _clone(B, _p) : 0; } + +/* M is the union value name and T is its type, i.e. the qualified name. */ +#define __flatbuffers_build_union_table_value_field(NS, N, NU, M, T)\ +static inline int N ## _ ## M ## _add(NS ## builder_t *B, T ## _ref_t ref)\ +{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ +static inline int N ## _ ## M ## _start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline int N ## _ ## M ## _end(NS ## builder_t *B)\ +{ T ## _ref_t ref = T ## _end(B);\ + return ref ? N ## _ ## M ## _add(B, ref) : -1; }\ +static inline int N ## _ ## M ## _create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ T ## _ref_t ref = T ## _create(B __ ## T ## _call_args);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ +static inline int N ## _ ## M ## _clone(NS ## builder_t *B, T ## _table_t t)\ +{ T ## _ref_t ref = T ## _clone(B, t);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; } + +/* M is the union value name and T is its type, i.e. the qualified name. */ +#define __flatbuffers_build_union_struct_value_field(NS, N, NU, M, T)\ +static inline int N ## _ ## M ## _add(NS ## builder_t *B, T ## _ref_t ref)\ +{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ +static inline T ## _t *N ## _ ## M ## _start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline int N ## _ ## M ## _end(NS ## builder_t *B)\ +{ T ## _ref_t ref = T ## _end(B);\ + return ref ? N ## _ ## M ## _add(B, ref) : -1; }\ +static inline int N ## _ ## M ## _create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ T ## _ref_t ref = T ## _create(B __ ## T ## _call_args);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ +static inline int N ## _ ## M ## _end_pe(NS ## builder_t *B)\ +{ T ## _ref_t ref = T ## _end_pe(B);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ +static inline int N ## _ ## M ## _clone(NS ## builder_t *B, T ## _struct_t p)\ +{ T ## _ref_t ref = T ## _clone(B, p);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; } +#define __flatbuffers_build_union_string_value_field(NS, N, NU, M)\ +static inline int N ## _ ## M ## _add(NS ## builder_t *B, NS ## string_ref_t ref)\ +{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ +__flatbuffers_build_string_field_ops(NS, N ## _ ## M) + +/* NS: common namespace, ID: table field id (not offset), TN: name of type T, TT: name of table type + * S: sizeof of scalar type, A: alignment of type T, default value V of type T. */ +#define __flatbuffers_build_scalar_field(ID, NS, N, TN, T, S, A, V, TT)\ +static inline int N ## _add(NS ## builder_t *B, const T v)\ +{ T *_p; if (v == V) return 0; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ + TN ## _assign_to_pe(_p, v); return 0; }\ +static inline int N ## _force_add(NS ## builder_t *B, const T v)\ +{ T *_p; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ + TN ## _assign_to_pe(_p, v); return 0; }\ +/* Clone does not skip default values and expects pe endian content. */\ +static inline int N ## _clone(NS ## builder_t *B, const T *p)\ +{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ +/* Transferring a missing field is a nop success with 0 as result. */\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ const T *_p = N ## _get_ptr(t); return _p ? N ## _clone(B, _p) : 0; } + +/* NS: common namespace, ID: table field id (not offset), TN: name of type T, TT: name of table type + * S: sizeof of scalar type, A: alignment of type T. */ +#define __flatbuffers_build_scalar_optional_field(ID, NS, N, TN, T, S, A, TT)\ +static inline int N ## _add(NS ## builder_t *B, const T v)\ +{ T *_p; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ + TN ## _assign_to_pe(_p, v); return 0; }\ +/* Clone does not skip default values and expects pe endian content. */\ +static inline int N ## _clone(NS ## builder_t *B, const T *p)\ +{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ +/* Transferring a missing field is a nop success with 0 as result. */\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ const T *_p = N ## _get_ptr(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_struct_field(ID, NS, N, TN, S, A, TT)\ +static inline TN ## _t *N ## _start(NS ## builder_t *B)\ +{ return (TN ## _t *)flatcc_builder_table_add(B, ID, S, A); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ if (!NS ## is_native_pe()) { TN ## _to_pe((TN ## _t *)flatcc_builder_table_edit(B, S)); } return 0; }\ +static inline int N ## _end_pe(NS ## builder_t *B) { return 0; }\ +static inline int N ## _create(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ TN ## _t *_p = N ## _start(B); if (!_p) return -1; TN ##_assign_to_pe(_p __ ## TN ## _call_args);\ + return 0; }\ +static inline int N ## _add(NS ## builder_t *B, const TN ## _t *p)\ +{ TN ## _t *_p = N ## _start(B); if (!_p) return -1; TN ##_copy_to_pe(_p, p); return 0; }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _struct_t p)\ +{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _struct_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_vector_field(ID, NS, N, TN, T, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _vec_ref_t ref)\ +{ TN ## _vec_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return TN ## _vec_start(B); }\ +static inline int N ## _end_pe(NS ## builder_t *B)\ +{ return N ## _add(B, TN ## _vec_end_pe(B)); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, TN ## _vec_end(B)); }\ +static inline int N ## _create_pe(NS ## builder_t *B, const T *data, size_t len)\ +{ return N ## _add(B, TN ## _vec_create_pe(B, data, len)); }\ +static inline int N ## _create(NS ## builder_t *B, const T *data, size_t len)\ +{ return N ## _add(B, TN ## _vec_create(B, data, len)); }\ +static inline int N ## _slice(NS ## builder_t *B, TN ## _vec_t vec, size_t index, size_t len)\ +{ return N ## _add(B, TN ## _vec_slice(B, vec, index, len)); }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _vec_t vec)\ +{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _vec_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; }\ +__flatbuffers_build_vector_ops(NS, N, N, TN, T)\ + +#define __flatbuffers_build_offset_vector_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _vec_ref_t ref)\ +{ TN ## _vec_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_offset_vector(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, flatcc_builder_end_offset_vector(B)); }\ +static inline int N ## _create(NS ## builder_t *B, const TN ## _ref_t *data, size_t len)\ +{ return N ## _add(B, flatcc_builder_create_offset_vector(B, data, len)); }\ +__flatbuffers_build_offset_vector_ops(NS, N, N, TN)\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _vec_t vec)\ +{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _vec_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +/* depends on N ## _add which differs for union member fields and ordinary fields */\ +#define __flatbuffers_build_string_field_ops(NS, N)\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_string(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, flatcc_builder_end_string(B)); }\ +static inline int N ## _create(NS ## builder_t *B, const char *s, size_t len)\ +{ return N ## _add(B, flatcc_builder_create_string(B, s, len)); }\ +static inline int N ## _create_str(NS ## builder_t *B, const char *s)\ +{ return N ## _add(B, flatcc_builder_create_string_str(B, s)); }\ +static inline int N ## _create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ +{ return N ## _add(B, flatcc_builder_create_string_strn(B, s, max_len)); }\ +static inline int N ## _clone(NS ## builder_t *B, NS ## string_t string)\ +{ return N ## _add(B, NS ## string_clone(B, string)); }\ +static inline int N ## _slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ return N ## _add(B, NS ## string_slice(B, string, index, len)); }\ +__flatbuffers_build_string_ops(NS, N) + +#define __flatbuffers_build_string_field(ID, NS, N, TT)\ +static inline int N ## _add(NS ## builder_t *B, NS ## string_ref_t ref)\ +{ NS ## string_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ +__flatbuffers_build_string_field_ops(NS, N)\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ NS ## string_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_table_vector_field(ID, NS, N, TN, TT)\ +__flatbuffers_build_offset_vector_field(ID, NS, N, TN, TT)\ +__flatbuffers_build_table_vector_ops(NS, N, TN) + +#define __flatbuffers_build_union_vector_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _union_vec_ref_t uvref)\ +{ NS ## vec_ref_t *_p; if (!uvref.type || !uvref.value) return uvref.type == uvref.value ? 0 : -1;\ + if (!(_p = flatcc_builder_table_add_offset(B, ID - 1))) return -1; *_p = uvref.type;\ + if (!(_p = flatcc_builder_table_add_offset(B, ID))) return -1; *_p = uvref.value; return 0; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_union_vector(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, flatcc_builder_end_union_vector(B)); }\ +static inline int N ## _create(NS ## builder_t *B, const TN ## _union_ref_t *data, size_t len)\ +{ return N ## _add(B, flatcc_builder_create_union_vector(B, data, len)); }\ +__flatbuffers_build_union_vector_ops(NS, N, N, TN)\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _union_vec_t vec)\ +{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _union_vec_t _p = N ## _union(t); return _p.type ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_union_table_vector_value_field(NS, N, NU, M, T)\ +static inline int N ## _ ## M ## _push_start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (T ## _end(B))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, T ## _ref_t ref)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _create(B __ ## T ## _call_args))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, T ## _table_t t)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _clone(B, t))); } + +#define __flatbuffers_build_union_struct_vector_value_field(NS, N, NU, M, T)\ +static inline T ## _t *N ## _ ## M ## _push_start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (T ## _end(B))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, T ## _ref_t ref)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _create(B __ ## T ## _call_args))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, T ## _struct_t p)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _clone(B, p))); } + +#define __flatbuffers_build_union_string_vector_value_field(NS, N, NU, M)\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, NS ## string_ref_t ref)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ +static inline int N ## _ ## M ## _push_start(NS ## builder_t *B)\ +{ return NS ## string_start(B); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_end(B))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B, const char *s, size_t len)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create(B, s, len))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create_str(NS ## builder_t *B, const char *s)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create_str(B, s))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create_strn(B, s, max_len))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, NS ## string_t string)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_clone(B, string))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_slice(B, string, index, len))); } + +#define __flatbuffers_build_string_vector_field(ID, NS, N, TT)\ +__flatbuffers_build_offset_vector_field(ID, NS, N, NS ## string, TT)\ +__flatbuffers_build_string_vector_ops(NS, N) + +#define __flatbuffers_char_formal_args , char v0 +#define __flatbuffers_char_call_args , v0 +#define __flatbuffers_uint8_formal_args , uint8_t v0 +#define __flatbuffers_uint8_call_args , v0 +#define __flatbuffers_int8_formal_args , int8_t v0 +#define __flatbuffers_int8_call_args , v0 +#define __flatbuffers_bool_formal_args , flatbuffers_bool_t v0 +#define __flatbuffers_bool_call_args , v0 +#define __flatbuffers_uint16_formal_args , uint16_t v0 +#define __flatbuffers_uint16_call_args , v0 +#define __flatbuffers_uint32_formal_args , uint32_t v0 +#define __flatbuffers_uint32_call_args , v0 +#define __flatbuffers_uint64_formal_args , uint64_t v0 +#define __flatbuffers_uint64_call_args , v0 +#define __flatbuffers_int16_formal_args , int16_t v0 +#define __flatbuffers_int16_call_args , v0 +#define __flatbuffers_int32_formal_args , int32_t v0 +#define __flatbuffers_int32_call_args , v0 +#define __flatbuffers_int64_formal_args , int64_t v0 +#define __flatbuffers_int64_call_args , v0 +#define __flatbuffers_float_formal_args , float v0 +#define __flatbuffers_float_call_args , v0 +#define __flatbuffers_double_formal_args , double v0 +#define __flatbuffers_double_call_args , v0 + +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_char, char) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint8, uint8_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int8, int8_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_bool, flatbuffers_bool_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint16, uint16_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint32, uint32_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint64, uint64_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int16, int16_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int32, int32_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int64, int64_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_float, float) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_double, double) + +__flatbuffers_build_string(flatbuffers_) + +__flatbuffers_build_buffer(flatbuffers_) +#include "flatcc/flatcc_epilogue.h" +#endif /* FLATBUFFERS_COMMON_BUILDER_H */ diff --git a/nostrdb/bindings/c/flatbuffers_common_reader.h b/nostrdb/bindings/c/flatbuffers_common_reader.h new file mode 100644 index 0000000000..49e479e299 --- /dev/null +++ b/nostrdb/bindings/c/flatbuffers_common_reader.h @@ -0,0 +1,578 @@ +#ifndef FLATBUFFERS_COMMON_READER_H +#define FLATBUFFERS_COMMON_READER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +/* Common FlatBuffers read functionality for C. */ + +#include "flatcc_prologue.h" +#include "flatcc_flatbuffers.h" + + +#define __flatbuffers_read_scalar_at_byteoffset(N, p, o) N ## _read_from_pe((uint8_t *)(p) + (o)) +#define __flatbuffers_read_scalar(N, p) N ## _read_from_pe(p) +#define __flatbuffers_read_vt(ID, offset, t)\ +flatbuffers_voffset_t offset = 0;\ +{ flatbuffers_voffset_t id__tmp, *vt__tmp;\ + FLATCC_ASSERT(t != 0 && "null pointer table access");\ + id__tmp = ID;\ + vt__tmp = (flatbuffers_voffset_t *)((uint8_t *)(t) -\ + __flatbuffers_soffset_read_from_pe(t));\ + if (__flatbuffers_voffset_read_from_pe(vt__tmp) >= sizeof(vt__tmp[0]) * (id__tmp + 3u)) {\ + offset = __flatbuffers_voffset_read_from_pe(vt__tmp + id__tmp + 2);\ + }\ +} +#define __flatbuffers_field_present(ID, t) { __flatbuffers_read_vt(ID, offset__tmp, t) return offset__tmp != 0; } +#define __flatbuffers_scalar_field(T, ID, t)\ +{\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + if (offset__tmp) {\ + return (const T *)((uint8_t *)(t) + offset__tmp);\ + }\ + return 0;\ +} +#define __flatbuffers_define_scalar_field(ID, N, NK, TK, T, V)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +{ __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ + return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ +}\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +{ __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ + return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ +}\ +static inline const T *N ## _ ## NK ## _get_ptr(N ## _table_t t__tmp)\ +__flatbuffers_scalar_field(T, ID, t__tmp)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp)\ +__flatbuffers_define_scan_by_scalar_field(N, NK, T) +#define __flatbuffers_define_scalar_optional_field(ID, N, NK, TK, T, V)\ +__flatbuffers_define_scalar_field(ID, N, NK, TK, T, V)\ +static inline TK ## _option_t N ## _ ## NK ## _option(N ## _table_t t__tmp)\ +{ TK ## _option_t ret; __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ + ret.is_null = offset__tmp == 0; ret.value = offset__tmp ?\ + __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ + return ret; } +#define __flatbuffers_struct_field(T, ID, t, r)\ +{\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + if (offset__tmp) {\ + return (T)((uint8_t *)(t) + offset__tmp);\ + }\ + FLATCC_ASSERT(!(r) && "required field missing");\ + return 0;\ +} +#define __flatbuffers_offset_field(T, ID, t, r, adjust)\ +{\ + flatbuffers_uoffset_t *elem__tmp;\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + if (offset__tmp) {\ + elem__tmp = (flatbuffers_uoffset_t *)((uint8_t *)(t) + offset__tmp);\ + /* Add sizeof so C api can have raw access past header field. */\ + return (T)((uint8_t *)(elem__tmp) + adjust +\ + __flatbuffers_uoffset_read_from_pe(elem__tmp));\ + }\ + FLATCC_ASSERT(!(r) && "required field missing");\ + return 0;\ +} +#define __flatbuffers_vector_field(T, ID, t, r) __flatbuffers_offset_field(T, ID, t, r, sizeof(flatbuffers_uoffset_t)) +#define __flatbuffers_table_field(T, ID, t, r) __flatbuffers_offset_field(T, ID, t, r, 0) +#define __flatbuffers_define_struct_field(ID, N, NK, T, r)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_struct_field(T, ID, t__tmp, r)\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_struct_field(T, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp) +#define __flatbuffers_define_vector_field(ID, N, NK, T, r)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(T, ID, t__tmp, r)\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(T, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp) +#define __flatbuffers_define_table_field(ID, N, NK, T, r)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_table_field(T, ID, t__tmp, r)\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_table_field(T, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp) +#define __flatbuffers_define_string_field(ID, N, NK, r)\ +static inline flatbuffers_string_t N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(flatbuffers_string_t, ID, t__tmp, r)\ +static inline flatbuffers_string_t N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(flatbuffers_string_t, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp)\ +__flatbuffers_define_scan_by_string_field(N, NK) +#define __flatbuffers_vec_len(vec)\ +{ return (vec) ? (size_t)__flatbuffers_uoffset_read_from_pe((flatbuffers_uoffset_t *)vec - 1) : 0; } +#define __flatbuffers_string_len(s) __flatbuffers_vec_len(s) +static inline size_t flatbuffers_vec_len(const void *vec) +__flatbuffers_vec_len(vec) +#define __flatbuffers_scalar_vec_at(N, vec, i)\ +{ FLATCC_ASSERT(flatbuffers_vec_len(vec) > (i) && "index out of range");\ + return __flatbuffers_read_scalar(N, &(vec)[i]); } +#define __flatbuffers_struct_vec_at(vec, i)\ +{ FLATCC_ASSERT(flatbuffers_vec_len(vec) > (i) && "index out of range"); return (vec) + (i); } +/* `adjust` skips past the header for string vectors. */ +#define __flatbuffers_offset_vec_at(T, vec, i, adjust)\ +{ const flatbuffers_uoffset_t *elem__tmp = (vec) + (i);\ + FLATCC_ASSERT(flatbuffers_vec_len(vec) > (i) && "index out of range");\ + return (T)((uint8_t *)(elem__tmp) + (size_t)__flatbuffers_uoffset_read_from_pe(elem__tmp) + (adjust)); } +#define __flatbuffers_define_scalar_vec_len(N)\ +static inline size_t N ## _vec_len(N ##_vec_t vec__tmp)\ +{ return flatbuffers_vec_len(vec__tmp); } +#define __flatbuffers_define_scalar_vec_at(N, T) \ +static inline T N ## _vec_at(N ## _vec_t vec__tmp, size_t i__tmp)\ +__flatbuffers_scalar_vec_at(N, vec__tmp, i__tmp) +typedef const char *flatbuffers_string_t; +static inline size_t flatbuffers_string_len(flatbuffers_string_t s) +__flatbuffers_string_len(s) +typedef const flatbuffers_uoffset_t *flatbuffers_string_vec_t; +typedef flatbuffers_uoffset_t *flatbuffers_string_mutable_vec_t; +static inline size_t flatbuffers_string_vec_len(flatbuffers_string_vec_t vec) +__flatbuffers_vec_len(vec) +static inline flatbuffers_string_t flatbuffers_string_vec_at(flatbuffers_string_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(flatbuffers_string_t, vec, i, sizeof(vec[0])) +typedef const void *flatbuffers_generic_t; +typedef void *flatbuffers_mutable_generic_t; +static inline flatbuffers_string_t flatbuffers_string_cast_from_generic(const flatbuffers_generic_t p) +{ return p ? ((const char *)p) + __flatbuffers_uoffset__size() : 0; } +typedef const flatbuffers_uoffset_t *flatbuffers_generic_vec_t; +typedef flatbuffers_uoffset_t *flatbuffers_generic_table_mutable_vec_t; +static inline size_t flatbuffers_generic_vec_len(flatbuffers_generic_vec_t vec) +__flatbuffers_vec_len(vec) +static inline flatbuffers_generic_t flatbuffers_generic_vec_at(flatbuffers_generic_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(flatbuffers_generic_t, vec, i, 0) +static inline flatbuffers_generic_t flatbuffers_generic_vec_at_as_string(flatbuffers_generic_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(flatbuffers_generic_t, vec, i, sizeof(vec[0])) +typedef struct flatbuffers_union { + flatbuffers_union_type_t type; + flatbuffers_generic_t value; +} flatbuffers_union_t; +typedef struct flatbuffers_union_vec { + const flatbuffers_union_type_t *type; + const flatbuffers_uoffset_t *value; +} flatbuffers_union_vec_t; +typedef struct flatbuffers_mutable_union { + flatbuffers_union_type_t type; + flatbuffers_mutable_generic_t value; +} flatbuffers_mutable_union_t; +typedef struct flatbuffers_mutable_union_vec { + flatbuffers_union_type_t *type; + flatbuffers_uoffset_t *value; +} flatbuffers_mutable_union_vec_t; +static inline flatbuffers_mutable_union_t flatbuffers_mutable_union_cast(flatbuffers_union_t u__tmp)\ +{ flatbuffers_mutable_union_t mu = { u__tmp.type, (flatbuffers_mutable_generic_t)u__tmp.value };\ + return mu; } +static inline flatbuffers_mutable_union_vec_t flatbuffers_mutable_union_vec_cast(flatbuffers_union_vec_t uv__tmp)\ +{ flatbuffers_mutable_union_vec_t muv =\ + { (flatbuffers_union_type_t *)uv__tmp.type, (flatbuffers_uoffset_t *)uv__tmp.value }; return muv; } +#define __flatbuffers_union_type_field(ID, t)\ +{\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(__flatbuffers_utype, t, offset__tmp) : 0;\ +} +static inline flatbuffers_string_t flatbuffers_string_cast_from_union(const flatbuffers_union_t u__tmp)\ +{ return flatbuffers_string_cast_from_generic(u__tmp.value); } +#define __flatbuffers_define_union_field(NS, ID, N, NK, T, r)\ +static inline T ## _union_type_t N ## _ ## NK ## _type_get(N ## _table_t t__tmp)\ +__## NS ## union_type_field(((ID) - 1), t__tmp)\ +static inline NS ## generic_t N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__## NS ## table_field(NS ## generic_t, ID, t__tmp, r)\ +static inline T ## _union_type_t N ## _ ## NK ## _type(N ## _table_t t__tmp)\ +__## NS ## union_type_field(((ID) - 1), t__tmp)\ +static inline NS ## generic_t N ## _ ## NK(N ## _table_t t__tmp)\ +__## NS ## table_field(NS ## generic_t, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__## NS ## field_present(ID, t__tmp)\ +static inline T ## _union_t N ## _ ## NK ## _union(N ## _table_t t__tmp)\ +{ T ## _union_t u__tmp = { 0, 0 }; u__tmp.type = N ## _ ## NK ## _type_get(t__tmp);\ + if (u__tmp.type == 0) return u__tmp; u__tmp.value = N ## _ ## NK ## _get(t__tmp); return u__tmp; }\ +static inline NS ## string_t N ## _ ## NK ## _as_string(N ## _table_t t__tmp)\ +{ return NS ## string_cast_from_generic(N ## _ ## NK ## _get(t__tmp)); }\ + +#define __flatbuffers_define_union_vector_ops(NS, T)\ +static inline size_t T ## _union_vec_len(T ## _union_vec_t uv__tmp)\ +{ return NS ## vec_len(uv__tmp.type); }\ +static inline T ## _union_t T ## _union_vec_at(T ## _union_vec_t uv__tmp, size_t i__tmp)\ +{ T ## _union_t u__tmp = { 0, 0 }; size_t n__tmp = NS ## vec_len(uv__tmp.type);\ + FLATCC_ASSERT(n__tmp > (i__tmp) && "index out of range"); u__tmp.type = uv__tmp.type[i__tmp];\ + /* Unknown type is treated as NONE for schema evolution. */\ + if (u__tmp.type == 0) return u__tmp;\ + u__tmp.value = NS ## generic_vec_at(uv__tmp.value, i__tmp); return u__tmp; }\ +static inline NS ## string_t T ## _union_vec_at_as_string(T ## _union_vec_t uv__tmp, size_t i__tmp)\ +{ return (NS ## string_t) NS ## generic_vec_at_as_string(uv__tmp.value, i__tmp); }\ + +#define __flatbuffers_define_union_vector(NS, T)\ +typedef NS ## union_vec_t T ## _union_vec_t;\ +typedef NS ## mutable_union_vec_t T ## _mutable_union_vec_t;\ +static inline T ## _mutable_union_vec_t T ## _mutable_union_vec_cast(T ## _union_vec_t u__tmp)\ +{ return NS ## mutable_union_vec_cast(u__tmp); }\ +__## NS ## define_union_vector_ops(NS, T) +#define __flatbuffers_define_union(NS, T)\ +typedef NS ## union_t T ## _union_t;\ +typedef NS ## mutable_union_t T ## _mutable_union_t;\ +static inline T ## _mutable_union_t T ## _mutable_union_cast(T ## _union_t u__tmp)\ +{ return NS ## mutable_union_cast(u__tmp); }\ +__## NS ## define_union_vector(NS, T) +#define __flatbuffers_define_union_vector_field(NS, ID, N, NK, T, r)\ +__## NS ## define_vector_field(ID - 1, N, NK ## _type, T ## _vec_t, r)\ +__## NS ## define_vector_field(ID, N, NK, flatbuffers_generic_vec_t, r)\ +static inline T ## _union_vec_t N ## _ ## NK ## _union(N ## _table_t t__tmp)\ +{ T ## _union_vec_t uv__tmp; uv__tmp.type = N ## _ ## NK ## _type_get(t__tmp);\ + uv__tmp.value = N ## _ ## NK(t__tmp);\ + FLATCC_ASSERT(NS ## vec_len(uv__tmp.type) == NS ## vec_len(uv__tmp.value)\ + && "union vector type length mismatch"); return uv__tmp; } +#include +static const size_t flatbuffers_not_found = (size_t)-1; +static const size_t flatbuffers_end = (size_t)-1; +#define __flatbuffers_identity(n) (n) +#define __flatbuffers_min(a, b) ((a) < (b) ? (a) : (b)) +/* Subtraction doesn't work for unsigned types. */ +#define __flatbuffers_scalar_cmp(x, y, n) ((x) < (y) ? -1 : (x) > (y)) +static inline int __flatbuffers_string_n_cmp(flatbuffers_string_t v, const char *s, size_t n) +{ size_t nv = flatbuffers_string_len(v); int x = strncmp(v, s, nv < n ? nv : n); + return x != 0 ? x : nv < n ? -1 : nv > n; } +/* `n` arg unused, but needed by string find macro expansion. */ +static inline int __flatbuffers_string_cmp(flatbuffers_string_t v, const char *s, size_t n) { (void)n; return strcmp(v, s); } +/* A = identity if searching scalar vectors rather than key fields. */ +/* Returns lowest matching index or not_found. */ +#define __flatbuffers_find_by_field(A, V, E, L, K, Kn, T, D)\ +{ T v__tmp; size_t a__tmp = 0, b__tmp, m__tmp; if (!(b__tmp = L(V))) { return flatbuffers_not_found; }\ + --b__tmp;\ + while (a__tmp < b__tmp) {\ + m__tmp = a__tmp + ((b__tmp - a__tmp) >> 1);\ + v__tmp = A(E(V, m__tmp));\ + if ((D(v__tmp, (K), (Kn))) < 0) {\ + a__tmp = m__tmp + 1;\ + } else {\ + b__tmp = m__tmp;\ + }\ + }\ + if (a__tmp == b__tmp) {\ + v__tmp = A(E(V, a__tmp));\ + if (D(v__tmp, (K), (Kn)) == 0) {\ + return a__tmp;\ + }\ + }\ + return flatbuffers_not_found;\ +} +#define __flatbuffers_find_by_scalar_field(A, V, E, L, K, T)\ +__flatbuffers_find_by_field(A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) +#define __flatbuffers_find_by_string_field(A, V, E, L, K)\ +__flatbuffers_find_by_field(A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) +#define __flatbuffers_find_by_string_n_field(A, V, E, L, K, Kn)\ +__flatbuffers_find_by_field(A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) +#define __flatbuffers_define_find_by_scalar_field(N, NK, TK)\ +static inline size_t N ## _vec_find_by_ ## NK(N ## _vec_t vec__tmp, TK key__tmp)\ +__flatbuffers_find_by_scalar_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, TK) +#define __flatbuffers_define_scalar_find(N, T)\ +static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_find_by_scalar_field(__flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) +#define __flatbuffers_define_find_by_string_field(N, NK) \ +/* Note: find only works on vectors sorted by this field. */\ +static inline size_t N ## _vec_find_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ +__flatbuffers_find_by_string_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_find_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_find_by_string_n_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp) +#define __flatbuffers_define_default_find_by_scalar_field(N, NK, TK)\ +static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, TK key__tmp)\ +{ return N ## _vec_find_by_ ## NK(vec__tmp, key__tmp); } +#define __flatbuffers_define_default_find_by_string_field(N, NK) \ +static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, const char *s__tmp)\ +{ return N ## _vec_find_by_ ## NK(vec__tmp, s__tmp); }\ +static inline size_t N ## _vec_find_n(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_find_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); } +/* A = identity if searching scalar vectors rather than key fields. */ +/* Returns lowest matching index or not_found. */ +#define __flatbuffers_scan_by_field(b, e, A, V, E, L, K, Kn, T, D)\ +{ T v__tmp; size_t i__tmp;\ + for (i__tmp = b; i__tmp < e; ++i__tmp) {\ + v__tmp = A(E(V, i__tmp));\ + if (D(v__tmp, (K), (Kn)) == 0) {\ + return i__tmp;\ + }\ + }\ + return flatbuffers_not_found;\ +} +#define __flatbuffers_rscan_by_field(b, e, A, V, E, L, K, Kn, T, D)\ +{ T v__tmp; size_t i__tmp = e;\ + while (i__tmp-- > b) {\ + v__tmp = A(E(V, i__tmp));\ + if (D(v__tmp, (K), (Kn)) == 0) {\ + return i__tmp;\ + }\ + }\ + return flatbuffers_not_found;\ +} +#define __flatbuffers_scan_by_scalar_field(b, e, A, V, E, L, K, T)\ +__flatbuffers_scan_by_field(b, e, A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) +#define __flatbuffers_scan_by_string_field(b, e, A, V, E, L, K)\ +__flatbuffers_scan_by_field(b, e, A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) +#define __flatbuffers_scan_by_string_n_field(b, e, A, V, E, L, K, Kn)\ +__flatbuffers_scan_by_field(b, e, A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) +#define __flatbuffers_rscan_by_scalar_field(b, e, A, V, E, L, K, T)\ +__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) +#define __flatbuffers_rscan_by_string_field(b, e, A, V, E, L, K)\ +__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) +#define __flatbuffers_rscan_by_string_n_field(b, e, A, V, E, L, K, Kn)\ +__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) +#define __flatbuffers_define_scan_by_scalar_field(N, NK, T)\ +static inline size_t N ## _vec_scan_by_ ## NK(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_scan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan_by_ ## NK(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) +#define __flatbuffers_define_scalar_scan(N, T)\ +static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(0, N ## _vec_len(vec__tmp), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(0, N ## _vec_len(vec__tmp), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) +#define __flatbuffers_define_scan_by_string_field(N, NK) \ +static inline size_t N ## _vec_scan_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ +__flatbuffers_scan_by_string_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_scan_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_scan_by_string_n_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ +static inline size_t N ## _vec_scan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +__flatbuffers_scan_by_string_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_scan_ex_n_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_scan_by_string_n_field(begin__tmp, __flatbuffers_min( end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ +static inline size_t N ## _vec_rscan_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ +__flatbuffers_rscan_by_string_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_rscan_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_rscan_by_string_n_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ +static inline size_t N ## _vec_rscan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +__flatbuffers_rscan_by_string_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_rscan_ex_n_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_rscan_by_string_n_field(begin__tmp, __flatbuffers_min( end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp) +#define __flatbuffers_define_default_scan_by_scalar_field(N, NK, TK)\ +static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, TK key__tmp)\ +{ return N ## _vec_scan_by_ ## NK(vec__tmp, key__tmp); }\ +static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, TK key__tmp)\ +{ return N ## _vec_scan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, key__tmp); }\ +static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, TK key__tmp)\ +{ return N ## _vec_rscan_by_ ## NK(vec__tmp, key__tmp); }\ +static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, TK key__tmp)\ +{ return N ## _vec_rscan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, key__tmp); } +#define __flatbuffers_define_default_scan_by_string_field(N, NK) \ +static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, const char *s__tmp)\ +{ return N ## _vec_scan_by_ ## NK(vec__tmp, s__tmp); }\ +static inline size_t N ## _vec_scan_n(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_scan_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); }\ +static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +{ return N ## _vec_scan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp); }\ +static inline size_t N ## _vec_scan_ex_n(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_scan_ex_n_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp, n__tmp); }\ +static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, const char *s__tmp)\ +{ return N ## _vec_rscan_by_ ## NK(vec__tmp, s__tmp); }\ +static inline size_t N ## _vec_rscan_n(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_rscan_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); }\ +static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +{ return N ## _vec_rscan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp); }\ +static inline size_t N ## _vec_rscan_ex_n(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_rscan_ex_n_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp, n__tmp); } +#define __flatbuffers_heap_sort(N, X, A, E, L, TK, TE, D, S)\ +static inline void __ ## N ## X ## __heap_sift_down(\ + N ## _mutable_vec_t vec__tmp, size_t start__tmp, size_t end__tmp)\ +{ size_t child__tmp, root__tmp; TK v1__tmp, v2__tmp, vroot__tmp;\ + root__tmp = start__tmp;\ + while ((root__tmp << 1) <= end__tmp) {\ + child__tmp = root__tmp << 1;\ + if (child__tmp < end__tmp) {\ + v1__tmp = A(E(vec__tmp, child__tmp));\ + v2__tmp = A(E(vec__tmp, child__tmp + 1));\ + if (D(v1__tmp, v2__tmp) < 0) {\ + child__tmp++;\ + }\ + }\ + vroot__tmp = A(E(vec__tmp, root__tmp));\ + v1__tmp = A(E(vec__tmp, child__tmp));\ + if (D(vroot__tmp, v1__tmp) < 0) {\ + S(vec__tmp, root__tmp, child__tmp, TE);\ + root__tmp = child__tmp;\ + } else {\ + return;\ + }\ + }\ +}\ +static inline void __ ## N ## X ## __heap_sort(N ## _mutable_vec_t vec__tmp)\ +{ size_t start__tmp, end__tmp, size__tmp;\ + size__tmp = L(vec__tmp); if (size__tmp == 0) return; end__tmp = size__tmp - 1; start__tmp = size__tmp >> 1;\ + do { __ ## N ## X ## __heap_sift_down(vec__tmp, start__tmp, end__tmp); } while (start__tmp--);\ + while (end__tmp > 0) { \ + S(vec__tmp, 0, end__tmp, TE);\ + __ ## N ## X ## __heap_sift_down(vec__tmp, 0, --end__tmp); } } +#define __flatbuffers_define_sort_by_field(N, NK, TK, TE, D, S)\ + __flatbuffers_heap_sort(N, _sort_by_ ## NK, N ## _ ## NK ## _get, N ## _vec_at, N ## _vec_len, TK, TE, D, S)\ +static inline void N ## _vec_sort_by_ ## NK(N ## _mutable_vec_t vec__tmp)\ +{ __ ## N ## _sort_by_ ## NK ## __heap_sort(vec__tmp); } +#define __flatbuffers_define_sort(N, TK, TE, D, S)\ +__flatbuffers_heap_sort(N, , __flatbuffers_identity, N ## _vec_at, N ## _vec_len, TK, TE, D, S)\ +static inline void N ## _vec_sort(N ## _mutable_vec_t vec__tmp) { __ ## N ## __heap_sort(vec__tmp); } +#define __flatbuffers_scalar_diff(x, y) ((x) < (y) ? -1 : (x) > (y)) +#define __flatbuffers_string_diff(x, y) __flatbuffers_string_n_cmp((x), (const char *)(y), flatbuffers_string_len(y)) +#define __flatbuffers_value_swap(vec, a, b, TE) { TE x__tmp = vec[b]; vec[b] = vec[a]; vec[a] = x__tmp; } +#define __flatbuffers_uoffset_swap(vec, a, b, TE)\ +{ TE ta__tmp, tb__tmp, d__tmp;\ + d__tmp = (TE)((a - b) * sizeof(vec[0]));\ + ta__tmp = __flatbuffers_uoffset_read_from_pe(vec + b) - d__tmp;\ + tb__tmp = __flatbuffers_uoffset_read_from_pe(vec + a) + d__tmp;\ + __flatbuffers_uoffset_write_to_pe(vec + a, ta__tmp);\ + __flatbuffers_uoffset_write_to_pe(vec + b, tb__tmp); } +#define __flatbuffers_scalar_swap(vec, a, b, TE) __flatbuffers_value_swap(vec, a, b, TE) +#define __flatbuffers_string_swap(vec, a, b, TE) __flatbuffers_uoffset_swap(vec, a, b, TE) +#define __flatbuffers_struct_swap(vec, a, b, TE) __flatbuffers_value_swap(vec, a, b, TE) +#define __flatbuffers_table_swap(vec, a, b, TE) __flatbuffers_uoffset_swap(vec, a, b, TE) +#define __flatbuffers_define_struct_sort_by_scalar_field(N, NK, TK, TE)\ + __flatbuffers_define_sort_by_field(N, NK, TK, TE, __flatbuffers_scalar_diff, __flatbuffers_struct_swap) +#define __flatbuffers_define_table_sort_by_scalar_field(N, NK, TK)\ + __flatbuffers_define_sort_by_field(N, NK, TK, flatbuffers_uoffset_t, __flatbuffers_scalar_diff, __flatbuffers_table_swap) +#define __flatbuffers_define_table_sort_by_string_field(N, NK)\ + __flatbuffers_define_sort_by_field(N, NK, flatbuffers_string_t, flatbuffers_uoffset_t, __flatbuffers_string_diff, __flatbuffers_table_swap) +#define __flatbuffers_define_scalar_sort(N, T) __flatbuffers_define_sort(N, T, T, __flatbuffers_scalar_diff, __flatbuffers_scalar_swap) +#define __flatbuffers_define_string_sort() __flatbuffers_define_sort(flatbuffers_string, flatbuffers_string_t, flatbuffers_uoffset_t, __flatbuffers_string_diff, __flatbuffers_string_swap) +#define __flatbuffers_sort_vector_field(N, NK, T, t)\ +{ T ## _mutable_vec_t v__tmp = (T ## _mutable_vec_t) N ## _ ## NK ## _get(t);\ + if (v__tmp) T ## _vec_sort(v__tmp); } +#define __flatbuffers_sort_table_field(N, NK, T, t)\ +{ T ## _sort((T ## _mutable_table_t)N ## _ ## NK ## _get(t)); } +#define __flatbuffers_sort_union_field(N, NK, T, t)\ +{ T ## _sort(T ## _mutable_union_cast(N ## _ ## NK ## _union(t))); } +#define __flatbuffers_sort_table_vector_field_elements(N, NK, T, t)\ +{ T ## _vec_t v__tmp = N ## _ ## NK ## _get(t); size_t i__tmp, n__tmp;\ + n__tmp = T ## _vec_len(v__tmp); for (i__tmp = 0; i__tmp < n__tmp; ++i__tmp) {\ + T ## _sort((T ## _mutable_table_t)T ## _vec_at(v__tmp, i__tmp)); }} +#define __flatbuffers_sort_union_vector_field_elements(N, NK, T, t)\ +{ T ## _union_vec_t v__tmp = N ## _ ## NK ## _union(t); size_t i__tmp, n__tmp;\ + n__tmp = T ## _union_vec_len(v__tmp); for (i__tmp = 0; i__tmp < n__tmp; ++i__tmp) {\ + T ## _sort(T ## _mutable_union_cast(T ## _union_vec_at(v__tmp, i__tmp))); }} +#define __flatbuffers_define_scalar_vector(N, T)\ +typedef const T *N ## _vec_t;\ +typedef T *N ## _mutable_vec_t;\ +__flatbuffers_define_scalar_vec_len(N)\ +__flatbuffers_define_scalar_vec_at(N, T)\ +__flatbuffers_define_scalar_find(N, T)\ +__flatbuffers_define_scalar_scan(N, T)\ +__flatbuffers_define_scalar_sort(N, T) + +#define __flatbuffers_define_integer_type(N, T, W)\ +__flatcc_define_integer_accessors(N, T, W, flatbuffers_endian)\ +__flatbuffers_define_scalar_vector(N, T) +__flatbuffers_define_scalar_vector(flatbuffers_bool, flatbuffers_bool_t) +__flatbuffers_define_scalar_vector(flatbuffers_char, char) +__flatbuffers_define_scalar_vector(flatbuffers_uint8, uint8_t) +__flatbuffers_define_scalar_vector(flatbuffers_int8, int8_t) +__flatbuffers_define_scalar_vector(flatbuffers_uint16, uint16_t) +__flatbuffers_define_scalar_vector(flatbuffers_int16, int16_t) +__flatbuffers_define_scalar_vector(flatbuffers_uint32, uint32_t) +__flatbuffers_define_scalar_vector(flatbuffers_int32, int32_t) +__flatbuffers_define_scalar_vector(flatbuffers_uint64, uint64_t) +__flatbuffers_define_scalar_vector(flatbuffers_int64, int64_t) +__flatbuffers_define_scalar_vector(flatbuffers_float, float) +__flatbuffers_define_scalar_vector(flatbuffers_double, double) +__flatbuffers_define_scalar_vector(flatbuffers_union_type, flatbuffers_union_type_t) +static inline size_t flatbuffers_string_vec_find(flatbuffers_string_vec_t vec, const char *s) +__flatbuffers_find_by_string_field(__flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_find_n(flatbuffers_string_vec_t vec, const char *s, size_t n) +__flatbuffers_find_by_string_n_field(__flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_scan(flatbuffers_string_vec_t vec, const char *s) +__flatbuffers_scan_by_string_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_scan_n(flatbuffers_string_vec_t vec, const char *s, size_t n) +__flatbuffers_scan_by_string_n_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_scan_ex(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s) +__flatbuffers_scan_by_string_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_scan_ex_n(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s, size_t n) +__flatbuffers_scan_by_string_n_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_rscan(flatbuffers_string_vec_t vec, const char *s) +__flatbuffers_rscan_by_string_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_rscan_n(flatbuffers_string_vec_t vec, const char *s, size_t n) +__flatbuffers_rscan_by_string_n_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_rscan_ex(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s) +__flatbuffers_rscan_by_string_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_rscan_ex_n(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s, size_t n) +__flatbuffers_rscan_by_string_n_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +__flatbuffers_define_string_sort() +#define __flatbuffers_define_struct_scalar_fixed_array_field(N, NK, TK, T, L)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp, size_t i__tmp)\ +{ if (!t__tmp || i__tmp >= L) return 0;\ + return __flatbuffers_read_scalar(TK, &(t__tmp->NK[i__tmp])); }\ +static inline const T *N ## _ ## NK ## _get_ptr(N ## _struct_t t__tmp)\ +{ return t__tmp ? t__tmp->NK : 0; }\ +static inline size_t N ## _ ## NK ## _get_len(void) { return L; }\ +static inline T N ## _ ## NK (N ## _struct_t t__tmp, size_t i__tmp)\ +{ return N ## _ ## NK ## _get(t__tmp, i__tmp); } +#define __flatbuffers_define_struct_struct_fixed_array_field(N, NK, T, L)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp, size_t i__tmp)\ +{ if (!t__tmp || i__tmp >= L) return 0; return t__tmp->NK + i__tmp; }static inline T N ## _ ## NK ## _get_ptr(N ## _struct_t t__tmp)\ +{ return t__tmp ? t__tmp->NK : 0; }\ +static inline size_t N ## _ ## NK ## _get_len(void) { return L; }\ +static inline T N ## _ ## NK(N ## _struct_t t__tmp, size_t i__tmp)\ +{ if (!t__tmp || i__tmp >= L) return 0; return t__tmp->NK + i__tmp; } +#define __flatbuffers_define_struct_scalar_field(N, NK, TK, T)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp)\ +{ return t__tmp ? __flatbuffers_read_scalar(TK, &(t__tmp->NK)) : 0; }\ +static inline const T *N ## _ ## NK ## _get_ptr(N ## _struct_t t__tmp)\ +{ return t__tmp ? &(t__tmp->NK) : 0; }\ +static inline T N ## _ ## NK (N ## _struct_t t__tmp)\ +{ return t__tmp ? __flatbuffers_read_scalar(TK, &(t__tmp->NK)) : 0; }\ +__flatbuffers_define_scan_by_scalar_field(N, NK, T) +#define __flatbuffers_define_struct_struct_field(N, NK, T)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; }\ +static inline T N ## _ ## NK (N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; } +/* If fid is null, the function returns true without testing as buffer is not expected to have any id. */ +static inline int flatbuffers_has_identifier(const void *buffer, const char *fid) +{ flatbuffers_thash_t id, id2 = 0; if (fid == 0) { return 1; }; + id2 = flatbuffers_type_hash_from_string(fid); + id = __flatbuffers_thash_read_from_pe(((flatbuffers_uoffset_t *)buffer) + 1); + return id2 == 0 || id == id2; } +static inline int flatbuffers_has_type_hash(const void *buffer, flatbuffers_thash_t thash) +{ return thash == 0 || (__flatbuffers_thash_read_from_pe((flatbuffers_uoffset_t *)buffer + 1) == thash); } + +static inline flatbuffers_thash_t flatbuffers_get_type_hash(const void *buffer) +{ return __flatbuffers_thash_read_from_pe((flatbuffers_uoffset_t *)buffer + 1); } + +#define flatbuffers_verify_endian() flatbuffers_has_identifier("\x00\x00\x00\x00" "1234", "1234") +static inline void *flatbuffers_read_size_prefix(void *b, size_t *size_out) +{ if (size_out) { *size_out = (size_t)__flatbuffers_uoffset_read_from_pe(b); } + return (uint8_t *)b + sizeof(flatbuffers_uoffset_t); } +/* Null file identifier accepts anything, otherwise fid should be 4 characters. */ +#define __flatbuffers_read_root(T, K, buffer, fid)\ + ((!buffer || !flatbuffers_has_identifier(buffer, fid)) ? 0 :\ + ((T ## _ ## K ## t)(((uint8_t *)buffer) +\ + __flatbuffers_uoffset_read_from_pe(buffer)))) +#define __flatbuffers_read_typed_root(T, K, buffer, thash)\ + ((!buffer || !flatbuffers_has_type_hash(buffer, thash)) ? 0 :\ + ((T ## _ ## K ## t)(((uint8_t *)buffer) +\ + __flatbuffers_uoffset_read_from_pe(buffer)))) +#define __flatbuffers_nested_buffer_as_root(C, N, T, K)\ +static inline T ## _ ## K ## t C ## _ ## N ## _as_root_with_identifier(C ## _ ## table_t t__tmp, const char *fid__tmp)\ +{ const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, fid__tmp); }\ +static inline T ## _ ## K ## t C ## _ ## N ## _as_typed_root(C ## _ ## table_t t__tmp)\ +{ const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, C ## _ ## type_identifier); }\ +static inline T ## _ ## K ## t C ## _ ## N ## _as_root(C ## _ ## table_t t__tmp)\ +{ const char *fid__tmp = T ## _file_identifier;\ + const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, fid__tmp); } +#define __flatbuffers_buffer_as_root(N, K)\ +static inline N ## _ ## K ## t N ## _as_root_with_identifier(const void *buffer__tmp, const char *fid__tmp)\ +{ return __flatbuffers_read_root(N, K, buffer__tmp, fid__tmp); }\ +static inline N ## _ ## K ## t N ## _as_root_with_type_hash(const void *buffer__tmp, flatbuffers_thash_t thash__tmp)\ +{ return __flatbuffers_read_typed_root(N, K, buffer__tmp, thash__tmp); }\ +static inline N ## _ ## K ## t N ## _as_root(const void *buffer__tmp)\ +{ const char *fid__tmp = N ## _file_identifier;\ + return __flatbuffers_read_root(N, K, buffer__tmp, fid__tmp); }\ +static inline N ## _ ## K ## t N ## _as_typed_root(const void *buffer__tmp)\ +{ return __flatbuffers_read_typed_root(N, K, buffer__tmp, N ## _type_hash); } +#define __flatbuffers_struct_as_root(N) __flatbuffers_buffer_as_root(N, struct_) +#define __flatbuffers_table_as_root(N) __flatbuffers_buffer_as_root(N, table_) + +#include "flatcc_epilogue.h" +#endif /* FLATBUFFERS_COMMON_H */ diff --git a/nostrdb/bindings/c/meta_builder.h b/nostrdb/bindings/c/meta_builder.h new file mode 100644 index 0000000000..19521e17eb --- /dev/null +++ b/nostrdb/bindings/c/meta_builder.h @@ -0,0 +1,52 @@ +#ifndef META_BUILDER_H +#define META_BUILDER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef META_READER_H +#include "meta_reader.h" +#endif +#ifndef FLATBUFFERS_COMMON_BUILDER_H +#include "flatbuffers_common_builder.h" +#endif +#include "flatcc/flatcc_prologue.h" +#ifndef flatbuffers_identifier +#define flatbuffers_identifier 0 +#endif +#ifndef flatbuffers_extension +#define flatbuffers_extension "bin" +#endif + +static const flatbuffers_voffset_t __NdbEventMeta_required[] = { 0 }; +typedef flatbuffers_ref_t NdbEventMeta_ref_t; +static NdbEventMeta_ref_t NdbEventMeta_clone(flatbuffers_builder_t *B, NdbEventMeta_table_t t); +__flatbuffers_build_table(flatbuffers_, NdbEventMeta, 1) + +#define __NdbEventMeta_formal_args , int32_t v0 +#define __NdbEventMeta_call_args , v0 +static inline NdbEventMeta_ref_t NdbEventMeta_create(flatbuffers_builder_t *B __NdbEventMeta_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, NdbEventMeta, NdbEventMeta_file_identifier, NdbEventMeta_type_identifier) + +__flatbuffers_build_scalar_field(0, flatbuffers_, NdbEventMeta_received_at, flatbuffers_int32, int32_t, 4, 4, INT32_C(0), NdbEventMeta) + +static inline NdbEventMeta_ref_t NdbEventMeta_create(flatbuffers_builder_t *B __NdbEventMeta_formal_args) +{ + if (NdbEventMeta_start(B) + || NdbEventMeta_received_at_add(B, v0)) { + return 0; + } + return NdbEventMeta_end(B); +} + +static NdbEventMeta_ref_t NdbEventMeta_clone(flatbuffers_builder_t *B, NdbEventMeta_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (NdbEventMeta_start(B) + || NdbEventMeta_received_at_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, NdbEventMeta_end(B)); +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* META_BUILDER_H */ diff --git a/nostrdb/bindings/c/meta_json_parser.h b/nostrdb/bindings/c/meta_json_parser.h new file mode 100644 index 0000000000..1c18728fa1 --- /dev/null +++ b/nostrdb/bindings/c/meta_json_parser.h @@ -0,0 +1,116 @@ +#ifndef META_JSON_PARSER_H +#define META_JSON_PARSER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#include "flatcc/flatcc_json_parser.h" +#include "flatcc/flatcc_prologue.h" + +/* + * Parses the default root table or struct of the schema and constructs a FlatBuffer. + * + * Builder `B` must be initialized. `ctx` can be null but will hold + * hold detailed error info on return when available. + * Returns 0 on success, or error code. + * `flags` : 0 by default, `flatcc_json_parser_f_skip_unknown` silently + * ignores unknown table and structs fields, and union types. + */ +static int meta_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags); + +static const char *NdbEventMeta_parse_json_table(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_builder_ref_t *result); +static const char *meta_local_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, +int *value_type, uint64_t *value, int *aggregate); +static const char *meta_global_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_type, uint64_t *value, int *aggregate); + +static const char *NdbEventMeta_parse_json_table(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_builder_ref_t *result) +{ + int more; + void *pval; + flatcc_builder_ref_t ref, *pref; + const char *mark; + uint64_t w; + + *result = 0; + if (flatcc_builder_start_table(ctx->ctx, 1)) goto failed; + buf = flatcc_json_parser_object_start(ctx, buf, end, &more); + while (more) { + buf = flatcc_json_parser_symbol_start(ctx, buf, end); + w = flatcc_json_parser_symbol_part(buf, end); + if (w == 0x7265636569766564) { /* descend "received" */ + buf += 8; + w = flatcc_json_parser_symbol_part(buf, end); + if ((w & 0xffffff0000000000) == 0x5f61740000000000) { /* "_at" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 3); + if (mark != buf) { + int32_t val = 0; + static flatcc_json_parser_integral_symbol_f *symbolic_parsers[] = { + meta_local_json_parser_enum, + meta_global_json_parser_enum, 0 }; + buf = flatcc_json_parser_int32(ctx, (mark = buf), end, &val); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_int32(ctx, (mark = buf), end, symbolic_parsers, &val); + if (buf == mark || buf == end) goto failed; + } + if (val != INT32_C(0) || (ctx->flags & flatcc_json_parser_f_force_add)) { + if (!(pval = flatcc_builder_table_add(ctx->ctx, 0, 4, 4))) goto failed; + flatbuffers_int32_write_to_pe(pval, val); + } + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "_at" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "_at" */ + } else { /* descend "received" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* descend "received" */ + buf = flatcc_json_parser_object_end(ctx, buf, end, &more); + } + if (ctx->error) goto failed; + if (!(*result = flatcc_builder_end_table(ctx->ctx))) goto failed; + return buf; +failed: + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_runtime); +} + +static inline int NdbEventMeta_parse_json_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx, const char *buf, size_t bufsiz, int flags, const char *fid) +{ + return flatcc_json_parser_table_as_root(B, ctx, buf, bufsiz, flags, fid, NdbEventMeta_parse_json_table); +} + +static const char *meta_local_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_type, uint64_t *value, int *aggregate) +{ + /* Scope has no enum / union types to look up. */ + return buf; /* unmatched; */ +} + +static const char *meta_global_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_type, uint64_t *value, int *aggregate) +{ + /* Global scope has no enum / union types to look up. */ + return buf; /* unmatched; */ +} + +static int meta_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags) +{ + flatcc_json_parser_t parser; + flatcc_builder_ref_t root; + + ctx = ctx ? ctx : &parser; + flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags); + if (flatcc_builder_start_buffer(B, 0, 0, 0)) return -1; + NdbEventMeta_parse_json_table(ctx, buf, buf + bufsiz, &root); + if (ctx->error) { + return ctx->error; + } + if (!flatcc_builder_end_buffer(B, root)) return -1; + ctx->end_loc = buf; + return 0; +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* META_JSON_PARSER_H */ diff --git a/nostrdb/bindings/c/meta_reader.h b/nostrdb/bindings/c/meta_reader.h new file mode 100644 index 0000000000..1e72408e62 --- /dev/null +++ b/nostrdb/bindings/c/meta_reader.h @@ -0,0 +1,53 @@ +#ifndef META_READER_H +#define META_READER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef FLATBUFFERS_COMMON_READER_H +#include "flatbuffers_common_reader.h" +#endif +#include "flatcc/flatcc_flatbuffers.h" +#ifndef __alignas_is_defined +#include +#endif +#include "flatcc/flatcc_prologue.h" +#ifndef flatbuffers_identifier +#define flatbuffers_identifier 0 +#endif +#ifndef flatbuffers_extension +#define flatbuffers_extension "bin" +#endif + + +typedef const struct NdbEventMeta_table *NdbEventMeta_table_t; +typedef struct NdbEventMeta_table *NdbEventMeta_mutable_table_t; +typedef const flatbuffers_uoffset_t *NdbEventMeta_vec_t; +typedef flatbuffers_uoffset_t *NdbEventMeta_mutable_vec_t; +#ifndef NdbEventMeta_file_identifier +#define NdbEventMeta_file_identifier 0 +#endif +/* deprecated, use NdbEventMeta_file_identifier */ +#ifndef NdbEventMeta_identifier +#define NdbEventMeta_identifier 0 +#endif +#define NdbEventMeta_type_hash ((flatbuffers_thash_t)0xa8c23be8) +#define NdbEventMeta_type_identifier "\xe8\x3b\xc2\xa8" +#ifndef NdbEventMeta_file_extension +#define NdbEventMeta_file_extension "bin" +#endif + + + +struct NdbEventMeta_table { uint8_t unused__; }; + +static inline size_t NdbEventMeta_vec_len(NdbEventMeta_vec_t vec) +__flatbuffers_vec_len(vec) +static inline NdbEventMeta_table_t NdbEventMeta_vec_at(NdbEventMeta_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(NdbEventMeta_table_t, vec, i, 0) +__flatbuffers_table_as_root(NdbEventMeta) + +__flatbuffers_define_scalar_field(0, NdbEventMeta, received_at, flatbuffers_int32, int32_t, INT32_C(0)) + + +#include "flatcc/flatcc_epilogue.h" +#endif /* META_READER_H */ diff --git a/nostrdb/bindings/c/meta_verifier.h b/nostrdb/bindings/c/meta_verifier.h new file mode 100644 index 0000000000..713ba38b37 --- /dev/null +++ b/nostrdb/bindings/c/meta_verifier.h @@ -0,0 +1,42 @@ +#ifndef META_VERIFIER_H +#define META_VERIFIER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef META_READER_H +#include "meta_reader.h" +#endif +#include "flatcc/flatcc_verifier.h" +#include "flatcc/flatcc_prologue.h" + +static int NdbEventMeta_verify_table(flatcc_table_verifier_descriptor_t *td); + +static int NdbEventMeta_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_field(td, 0, 4, 4) /* received_at */)) return ret; + return flatcc_verify_ok; +} + +static inline int NdbEventMeta_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, NdbEventMeta_identifier, &NdbEventMeta_verify_table); +} + +static inline int NdbEventMeta_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, NdbEventMeta_type_identifier, &NdbEventMeta_verify_table); +} + +static inline int NdbEventMeta_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &NdbEventMeta_verify_table); +} + +static inline int NdbEventMeta_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &NdbEventMeta_verify_table); +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* META_VERIFIER_H */ diff --git a/nostrdb/bindings/c/profile_builder.h b/nostrdb/bindings/c/profile_builder.h new file mode 100644 index 0000000000..185fba73e3 --- /dev/null +++ b/nostrdb/bindings/c/profile_builder.h @@ -0,0 +1,88 @@ +#ifndef PROFILE_BUILDER_H +#define PROFILE_BUILDER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef PROFILE_READER_H +#include "profile_reader.h" +#endif +#ifndef FLATBUFFERS_COMMON_BUILDER_H +#include "flatbuffers_common_builder.h" +#endif +#include "flatcc/flatcc_prologue.h" +#ifndef flatbuffers_identifier +#define flatbuffers_identifier 0 +#endif +#ifndef flatbuffers_extension +#define flatbuffers_extension "bin" +#endif + +static const flatbuffers_voffset_t __NdbProfile_required[] = { 0 }; +typedef flatbuffers_ref_t NdbProfile_ref_t; +static NdbProfile_ref_t NdbProfile_clone(flatbuffers_builder_t *B, NdbProfile_table_t t); +__flatbuffers_build_table(flatbuffers_, NdbProfile, 11) + +#define __NdbProfile_formal_args ,\ + flatbuffers_string_ref_t v0, flatbuffers_string_ref_t v1, flatbuffers_string_ref_t v2, flatbuffers_string_ref_t v3,\ + flatbuffers_string_ref_t v4, flatbuffers_string_ref_t v5, flatbuffers_bool_t v6, flatbuffers_string_ref_t v7,\ + flatbuffers_string_ref_t v8, int32_t v9, int32_t v10 +#define __NdbProfile_call_args ,\ + v0, v1, v2, v3,\ + v4, v5, v6, v7,\ + v8, v9, v10 +static inline NdbProfile_ref_t NdbProfile_create(flatbuffers_builder_t *B __NdbProfile_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, NdbProfile, NdbProfile_file_identifier, NdbProfile_type_identifier) + +__flatbuffers_build_string_field(0, flatbuffers_, NdbProfile_name, NdbProfile) +__flatbuffers_build_string_field(1, flatbuffers_, NdbProfile_website, NdbProfile) +__flatbuffers_build_string_field(2, flatbuffers_, NdbProfile_about, NdbProfile) +__flatbuffers_build_string_field(3, flatbuffers_, NdbProfile_lud16, NdbProfile) +__flatbuffers_build_string_field(4, flatbuffers_, NdbProfile_banner, NdbProfile) +__flatbuffers_build_string_field(5, flatbuffers_, NdbProfile_display_name, NdbProfile) +__flatbuffers_build_scalar_field(6, flatbuffers_, NdbProfile_reactions, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(1), NdbProfile) +__flatbuffers_build_string_field(7, flatbuffers_, NdbProfile_picture, NdbProfile) +__flatbuffers_build_string_field(8, flatbuffers_, NdbProfile_nip05, NdbProfile) +__flatbuffers_build_scalar_field(9, flatbuffers_, NdbProfile_damus_donation, flatbuffers_int32, int32_t, 4, 4, INT32_C(0), NdbProfile) +__flatbuffers_build_scalar_field(10, flatbuffers_, NdbProfile_damus_donation_v2, flatbuffers_int32, int32_t, 4, 4, INT32_C(0), NdbProfile) + +static inline NdbProfile_ref_t NdbProfile_create(flatbuffers_builder_t *B __NdbProfile_formal_args) +{ + if (NdbProfile_start(B) + || NdbProfile_name_add(B, v0) + || NdbProfile_website_add(B, v1) + || NdbProfile_about_add(B, v2) + || NdbProfile_lud16_add(B, v3) + || NdbProfile_banner_add(B, v4) + || NdbProfile_display_name_add(B, v5) + || NdbProfile_picture_add(B, v7) + || NdbProfile_nip05_add(B, v8) + || NdbProfile_damus_donation_add(B, v9) + || NdbProfile_damus_donation_v2_add(B, v10) + || NdbProfile_reactions_add(B, v6)) { + return 0; + } + return NdbProfile_end(B); +} + +static NdbProfile_ref_t NdbProfile_clone(flatbuffers_builder_t *B, NdbProfile_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (NdbProfile_start(B) + || NdbProfile_name_pick(B, t) + || NdbProfile_website_pick(B, t) + || NdbProfile_about_pick(B, t) + || NdbProfile_lud16_pick(B, t) + || NdbProfile_banner_pick(B, t) + || NdbProfile_display_name_pick(B, t) + || NdbProfile_picture_pick(B, t) + || NdbProfile_nip05_pick(B, t) + || NdbProfile_damus_donation_pick(B, t) + || NdbProfile_damus_donation_v2_pick(B, t) + || NdbProfile_reactions_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, NdbProfile_end(B)); +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* PROFILE_BUILDER_H */ diff --git a/nostrdb/bindings/c/profile_json_parser.h b/nostrdb/bindings/c/profile_json_parser.h new file mode 100644 index 0000000000..f404b6679f --- /dev/null +++ b/nostrdb/bindings/c/profile_json_parser.h @@ -0,0 +1,284 @@ +#ifndef PROFILE_JSON_PARSER_H +#define PROFILE_JSON_PARSER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#include "flatcc_json_parser.h" +#include "flatcc_prologue.h" + +/* + * Parses the default root table or struct of the schema and constructs a FlatBuffer. + * + * Builder `B` must be initialized. `ctx` can be null but will hold + * hold detailed error info on return when available. + * Returns 0 on success, or error code. + * `flags` : 0 by default, `flatcc_json_parser_f_skip_unknown` silently + * ignores unknown table and structs fields, and union types. + */ +static int profile_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags); + +static const char *NdbProfile_parse_json_table(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_builder_ref_t *result); +static const char *profile_local_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, +int *value_type, uint64_t *value, int *aggregate); +static const char *profile_global_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_type, uint64_t *value, int *aggregate); + +static const char *NdbProfile_parse_json_table(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_builder_ref_t *result) +{ + int more; + void *pval; + flatcc_builder_ref_t ref, *pref; + const char *mark; + uint64_t w; + + *result = 0; + if (flatcc_builder_start_table(ctx->ctx, 11)) goto failed; + buf = flatcc_json_parser_object_start(ctx, buf, end, &more); + while (more) { + buf = flatcc_json_parser_symbol_start(ctx, buf, end); + w = flatcc_json_parser_symbol_part(buf, end); + if (w < 0x6c75643136000000) { /* branch "lud16" */ + if (w < 0x64616d75735f646f) { /* branch "damus_do" */ + if ((w & 0xffffffffffff0000) == 0x62616e6e65720000) { /* "banner" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 6); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 4))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "banner" */ + if ((w & 0xffffffffff000000) == 0x61626f7574000000) { /* "about" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 5); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 2))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "about" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "about" */ + } /* "banner" */ + } else { /* branch "damus_do" */ + if (w == 0x64616d75735f646f) { /* descend "damus_do" */ + buf += 8; + w = flatcc_json_parser_symbol_part(buf, end); + if (w == 0x6e6174696f6e5f76) { /* descend "nation_v" */ + buf += 8; + w = flatcc_json_parser_symbol_part(buf, end); + if ((w & 0xff00000000000000) == 0x3200000000000000) { /* "2" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 1); + if (mark != buf) { + int32_t val = 0; + static flatcc_json_parser_integral_symbol_f *symbolic_parsers[] = { + profile_local_json_parser_enum, + profile_global_json_parser_enum, 0 }; + buf = flatcc_json_parser_int32(ctx, (mark = buf), end, &val); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_int32(ctx, (mark = buf), end, symbolic_parsers, &val); + if (buf == mark || buf == end) goto failed; + } + if (val != INT32_C(0) || (ctx->flags & flatcc_json_parser_f_force_add)) { + if (!(pval = flatcc_builder_table_add(ctx->ctx, 10, 4, 4))) goto failed; + flatbuffers_int32_write_to_pe(pval, val); + } + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "2" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "2" */ + } else { /* descend "nation_v" */ + if ((w & 0xffffffffffff0000) == 0x6e6174696f6e0000) { /* "nation" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 6); + if (mark != buf) { + int32_t val = 0; + static flatcc_json_parser_integral_symbol_f *symbolic_parsers[] = { + profile_local_json_parser_enum, + profile_global_json_parser_enum, 0 }; + buf = flatcc_json_parser_int32(ctx, (mark = buf), end, &val); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_int32(ctx, (mark = buf), end, symbolic_parsers, &val); + if (buf == mark || buf == end) goto failed; + } + if (val != INT32_C(0) || (ctx->flags & flatcc_json_parser_f_force_add)) { + if (!(pval = flatcc_builder_table_add(ctx->ctx, 9, 4, 4))) goto failed; + flatbuffers_int32_write_to_pe(pval, val); + } + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "nation" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "nation" */ + } /* descend "nation_v" */ + } else { /* descend "damus_do" */ + if (w == 0x646973706c61795f) { /* descend "display_" */ + buf += 8; + w = flatcc_json_parser_symbol_part(buf, end); + if ((w & 0xffffffff00000000) == 0x6e616d6500000000) { /* "name" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 4); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 5))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "name" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "name" */ + } else { /* descend "display_" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* descend "display_" */ + } /* descend "damus_do" */ + } /* branch "damus_do" */ + } else { /* branch "lud16" */ + if (w < 0x6e69703035000000) { /* branch "nip05" */ + if ((w & 0xffffffff00000000) == 0x6e616d6500000000) { /* "name" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 4); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 0))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "name" */ + if ((w & 0xffffffffff000000) == 0x6c75643136000000) { /* "lud16" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 5); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 3))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "lud16" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "lud16" */ + } /* "name" */ + } else { /* branch "nip05" */ + if (w < 0x7069637475726500) { /* branch "picture" */ + if ((w & 0xffffffffff000000) == 0x6e69703035000000) { /* "nip05" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 5); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 8))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "nip05" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "nip05" */ + } else { /* branch "picture" */ + if (w < 0x7265616374696f6e) { /* branch "reaction" */ + if ((w & 0xffffffffffffff00) == 0x7069637475726500) { /* "picture" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 7); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 7))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "picture" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "picture" */ + } else { /* branch "reaction" */ + if (w == 0x7265616374696f6e) { /* descend "reaction" */ + buf += 8; + w = flatcc_json_parser_symbol_part(buf, end); + if ((w & 0xff00000000000000) == 0x7300000000000000) { /* "s" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 1); + if (mark != buf) { + uint8_t val = 0; + static flatcc_json_parser_integral_symbol_f *symbolic_parsers[] = { + profile_local_json_parser_enum, + profile_global_json_parser_enum, 0 }; + buf = flatcc_json_parser_bool(ctx, (mark = buf), end, &val); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_bool(ctx, (mark = buf), end, symbolic_parsers, &val); + if (buf == mark || buf == end) goto failed; + } + if (val != UINT8_C(1) || (ctx->flags & flatcc_json_parser_f_force_add)) { + if (!(pval = flatcc_builder_table_add(ctx->ctx, 6, 1, 1))) goto failed; + flatbuffers_bool_write_to_pe(pval, val); + } + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "s" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "s" */ + } else { /* descend "reaction" */ + if ((w & 0xffffffffffffff00) == 0x7765627369746500) { /* "website" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 7); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 1))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "website" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "website" */ + } /* descend "reaction" */ + } /* branch "reaction" */ + } /* branch "picture" */ + } /* branch "nip05" */ + } /* branch "lud16" */ + buf = flatcc_json_parser_object_end(ctx, buf, end, &more); + } + if (ctx->error) goto failed; + if (!(*result = flatcc_builder_end_table(ctx->ctx))) goto failed; + return buf; +failed: + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_runtime); +} + +static inline int NdbProfile_parse_json_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx, const char *buf, size_t bufsiz, int flags, const char *fid) +{ + return flatcc_json_parser_table_as_root(B, ctx, buf, bufsiz, flags, fid, NdbProfile_parse_json_table); +} + +static const char *profile_local_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_type, uint64_t *value, int *aggregate) +{ + /* Scope has no enum / union types to look up. */ + return buf; /* unmatched; */ +} + +static const char *profile_global_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_type, uint64_t *value, int *aggregate) +{ + /* Global scope has no enum / union types to look up. */ + return buf; /* unmatched; */ +} + +static int profile_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags) +{ + flatcc_json_parser_t parser; + flatcc_builder_ref_t root; + + ctx = ctx ? ctx : &parser; + flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags); + if (flatcc_builder_start_buffer(B, 0, 0, 0)) return -1; + NdbProfile_parse_json_table(ctx, buf, buf + bufsiz, &root); + if (ctx->error) { + return ctx->error; + } + if (!flatcc_builder_end_buffer(B, root)) return -1; + ctx->end_loc = buf; + return 0; +} + +#include "flatcc_epilogue.h" +#endif /* PROFILE_JSON_PARSER_H */ diff --git a/nostrdb/bindings/c/profile_reader.h b/nostrdb/bindings/c/profile_reader.h new file mode 100644 index 0000000000..ddc8fad446 --- /dev/null +++ b/nostrdb/bindings/c/profile_reader.h @@ -0,0 +1,63 @@ +#ifndef PROFILE_READER_H +#define PROFILE_READER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef FLATBUFFERS_COMMON_READER_H +#include "flatbuffers_common_reader.h" +#endif +#include "flatcc/flatcc_flatbuffers.h" +#ifndef __alignas_is_defined +#include +#endif +#include "flatcc/flatcc_prologue.h" +#ifndef flatbuffers_identifier +#define flatbuffers_identifier 0 +#endif +#ifndef flatbuffers_extension +#define flatbuffers_extension "bin" +#endif + + +typedef const struct NdbProfile_table *NdbProfile_table_t; +typedef struct NdbProfile_table *NdbProfile_mutable_table_t; +typedef const flatbuffers_uoffset_t *NdbProfile_vec_t; +typedef flatbuffers_uoffset_t *NdbProfile_mutable_vec_t; +#ifndef NdbProfile_file_identifier +#define NdbProfile_file_identifier 0 +#endif +/* deprecated, use NdbProfile_file_identifier */ +#ifndef NdbProfile_identifier +#define NdbProfile_identifier 0 +#endif +#define NdbProfile_type_hash ((flatbuffers_thash_t)0xba639e28) +#define NdbProfile_type_identifier "\x28\x9e\x63\xba" +#ifndef NdbProfile_file_extension +#define NdbProfile_file_extension "bin" +#endif + + + +struct NdbProfile_table { uint8_t unused__; }; + +static inline size_t NdbProfile_vec_len(NdbProfile_vec_t vec) +__flatbuffers_vec_len(vec) +static inline NdbProfile_table_t NdbProfile_vec_at(NdbProfile_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(NdbProfile_table_t, vec, i, 0) +__flatbuffers_table_as_root(NdbProfile) + +__flatbuffers_define_string_field(0, NdbProfile, name, 0) +__flatbuffers_define_string_field(1, NdbProfile, website, 0) +__flatbuffers_define_string_field(2, NdbProfile, about, 0) +__flatbuffers_define_string_field(3, NdbProfile, lud16, 0) +__flatbuffers_define_string_field(4, NdbProfile, banner, 0) +__flatbuffers_define_string_field(5, NdbProfile, display_name, 0) +__flatbuffers_define_scalar_field(6, NdbProfile, reactions, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(1)) +__flatbuffers_define_string_field(7, NdbProfile, picture, 0) +__flatbuffers_define_string_field(8, NdbProfile, nip05, 0) +__flatbuffers_define_scalar_field(9, NdbProfile, damus_donation, flatbuffers_int32, int32_t, INT32_C(0)) +__flatbuffers_define_scalar_field(10, NdbProfile, damus_donation_v2, flatbuffers_int32, int32_t, INT32_C(0)) + + +#include "flatcc/flatcc_epilogue.h" +#endif /* PROFILE_READER_H */ diff --git a/nostrdb/bindings/c/profile_verifier.h b/nostrdb/bindings/c/profile_verifier.h new file mode 100644 index 0000000000..3fc2ae960f --- /dev/null +++ b/nostrdb/bindings/c/profile_verifier.h @@ -0,0 +1,52 @@ +#ifndef PROFILE_VERIFIER_H +#define PROFILE_VERIFIER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef PROFILE_READER_H +#include "profile_reader.h" +#endif +#include "flatcc/flatcc_verifier.h" +#include "flatcc/flatcc_prologue.h" + +static int NdbProfile_verify_table(flatcc_table_verifier_descriptor_t *td); + +static int NdbProfile_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 0) /* name */)) return ret; + if ((ret = flatcc_verify_string_field(td, 1, 0) /* website */)) return ret; + if ((ret = flatcc_verify_string_field(td, 2, 0) /* about */)) return ret; + if ((ret = flatcc_verify_string_field(td, 3, 0) /* lud16 */)) return ret; + if ((ret = flatcc_verify_string_field(td, 4, 0) /* banner */)) return ret; + if ((ret = flatcc_verify_string_field(td, 5, 0) /* display_name */)) return ret; + if ((ret = flatcc_verify_field(td, 6, 1, 1) /* reactions */)) return ret; + if ((ret = flatcc_verify_string_field(td, 7, 0) /* picture */)) return ret; + if ((ret = flatcc_verify_string_field(td, 8, 0) /* nip05 */)) return ret; + if ((ret = flatcc_verify_field(td, 9, 4, 4) /* damus_donation */)) return ret; + if ((ret = flatcc_verify_field(td, 10, 4, 4) /* damus_donation_v2 */)) return ret; + return flatcc_verify_ok; +} + +static inline int NdbProfile_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, NdbProfile_identifier, &NdbProfile_verify_table); +} + +static inline int NdbProfile_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, NdbProfile_type_identifier, &NdbProfile_verify_table); +} + +static inline int NdbProfile_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &NdbProfile_verify_table); +} + +static inline int NdbProfile_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &NdbProfile_verify_table); +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* PROFILE_VERIFIER_H */ diff --git a/nostrdb/bindings/swift/NdbProfile.swift b/nostrdb/bindings/swift/NdbProfile.swift new file mode 100644 index 0000000000..847cafdcb0 --- /dev/null +++ b/nostrdb/bindings/swift/NdbProfile.swift @@ -0,0 +1,110 @@ +// automatically generated by the FlatBuffers compiler, do not modify +// swiftlint:disable all +// swiftformat:disable all + +import FlatBuffers + +public struct NdbProfile: FlatBufferObject, Verifiable { + + static func validateVersion() { FlatBuffersVersion_23_5_26() } + public var __buffer: ByteBuffer! { return _accessor.bb } + private var _accessor: Table + + private init(_ t: Table) { _accessor = t } + public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + + private enum VTOFFSET: VOffset { + case name = 4 + case website = 6 + case about = 8 + case lud16 = 10 + case banner = 12 + case displayName = 14 + case reactions = 16 + case picture = 18 + case nip05 = 20 + case damusDonation = 22 + case damusDonationV2 = 24 + var v: Int32 { Int32(self.rawValue) } + var p: VOffset { self.rawValue } + } + + public var name: String? { let o = _accessor.offset(VTOFFSET.name.v); return o == 0 ? nil : _accessor.string(at: o) } + public var nameSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.name.v) } + public var website: String? { let o = _accessor.offset(VTOFFSET.website.v); return o == 0 ? nil : _accessor.string(at: o) } + public var websiteSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.website.v) } + public var about: String? { let o = _accessor.offset(VTOFFSET.about.v); return o == 0 ? nil : _accessor.string(at: o) } + public var aboutSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.about.v) } + public var lud16: String? { let o = _accessor.offset(VTOFFSET.lud16.v); return o == 0 ? nil : _accessor.string(at: o) } + public var lud16SegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.lud16.v) } + public var banner: String? { let o = _accessor.offset(VTOFFSET.banner.v); return o == 0 ? nil : _accessor.string(at: o) } + public var bannerSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.banner.v) } + public var displayName: String? { let o = _accessor.offset(VTOFFSET.displayName.v); return o == 0 ? nil : _accessor.string(at: o) } + public var displayNameSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.displayName.v) } + public var reactions: Bool { let o = _accessor.offset(VTOFFSET.reactions.v); return o == 0 ? true : _accessor.readBuffer(of: Bool.self, at: o) } + public var picture: String? { let o = _accessor.offset(VTOFFSET.picture.v); return o == 0 ? nil : _accessor.string(at: o) } + public var pictureSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.picture.v) } + public var nip05: String? { let o = _accessor.offset(VTOFFSET.nip05.v); return o == 0 ? nil : _accessor.string(at: o) } + public var nip05SegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.nip05.v) } + public var damusDonation: Int32 { let o = _accessor.offset(VTOFFSET.damusDonation.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } + public var damusDonationV2: Int32 { let o = _accessor.offset(VTOFFSET.damusDonationV2.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } + public static func startNdbProfile(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 11) } + public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: name, at: VTOFFSET.name.p) } + public static func add(website: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: website, at: VTOFFSET.website.p) } + public static func add(about: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: about, at: VTOFFSET.about.p) } + public static func add(lud16: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: lud16, at: VTOFFSET.lud16.p) } + public static func add(banner: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: banner, at: VTOFFSET.banner.p) } + public static func add(displayName: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: displayName, at: VTOFFSET.displayName.p) } + public static func add(reactions: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: reactions, def: true, + at: VTOFFSET.reactions.p) } + public static func add(picture: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: picture, at: VTOFFSET.picture.p) } + public static func add(nip05: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: nip05, at: VTOFFSET.nip05.p) } + public static func add(damusDonation: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: damusDonation, def: 0, at: VTOFFSET.damusDonation.p) } + public static func add(damusDonationV2: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: damusDonationV2, def: 0, at: VTOFFSET.damusDonationV2.p) } + public static func endNdbProfile(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } + public static func createNdbProfile( + _ fbb: inout FlatBufferBuilder, + nameOffset name: Offset = Offset(), + websiteOffset website: Offset = Offset(), + aboutOffset about: Offset = Offset(), + lud16Offset lud16: Offset = Offset(), + bannerOffset banner: Offset = Offset(), + displayNameOffset displayName: Offset = Offset(), + reactions: Bool = true, + pictureOffset picture: Offset = Offset(), + nip05Offset nip05: Offset = Offset(), + damusDonation: Int32 = 0, + damusDonationV2: Int32 = 0 + ) -> Offset { + let __start = NdbProfile.startNdbProfile(&fbb) + NdbProfile.add(name: name, &fbb) + NdbProfile.add(website: website, &fbb) + NdbProfile.add(about: about, &fbb) + NdbProfile.add(lud16: lud16, &fbb) + NdbProfile.add(banner: banner, &fbb) + NdbProfile.add(displayName: displayName, &fbb) + NdbProfile.add(reactions: reactions, &fbb) + NdbProfile.add(picture: picture, &fbb) + NdbProfile.add(nip05: nip05, &fbb) + NdbProfile.add(damusDonation: damusDonation, &fbb) + NdbProfile.add(damusDonationV2: damusDonationV2, &fbb) + return NdbProfile.endNdbProfile(&fbb, start: __start) + } + + public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + var _v = try verifier.visitTable(at: position) + try _v.visit(field: VTOFFSET.name.p, fieldName: "name", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.website.p, fieldName: "website", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.about.p, fieldName: "about", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.lud16.p, fieldName: "lud16", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.banner.p, fieldName: "banner", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.displayName.p, fieldName: "displayName", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.reactions.p, fieldName: "reactions", required: false, type: Bool.self) + try _v.visit(field: VTOFFSET.picture.p, fieldName: "picture", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.nip05.p, fieldName: "nip05", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.damusDonation.p, fieldName: "damusDonation", required: false, type: Int32.self) + try _v.visit(field: VTOFFSET.damusDonationV2.p, fieldName: "damusDonationV2", required: false, type: Int32.self) + _v.finish() + } +} + diff --git a/nostrdb/copy-ndb b/nostrdb/copy-ndb index 51eaaa9750..2e2f1eb60e 100755 --- a/nostrdb/copy-ndb +++ b/nostrdb/copy-ndb @@ -1,4 +1,13 @@ #!/usr/bin/env bash +rm -rf flatcc bindings +mkdir -p flatcc cp ~/src/c/nostrdb/nostrdb.{c,h} . -cp ~/src/c/nostrdb/jsmn.h . +cp ~/src/c/nostrdb/{jsmn,threadpool,protected_queue,memchr,util}.h . +cp ~/src/c/nostrdb/deps/lmdb/{lmdb,midl}.h . +cp ~/src/c/nostrdb/deps/lmdb/mdb.c . +cp ~/src/c/nostrdb/deps/lmdb/midl.c . +cp -r ~/src/c/nostrdb/deps/flatcc/include/flatcc/* flatcc +cp ~/src/c/nostrdb/deps/flatcc/src/runtime/* flatcc +cp -r ~/src/c/nostrdb/bindings . +patch -p2 < flatcc.patch diff --git a/nostrdb/flatcc.patch b/nostrdb/flatcc.patch new file mode 100644 index 0000000000..fda1c3c4c3 --- /dev/null +++ b/nostrdb/flatcc.patch @@ -0,0 +1,289 @@ +diff --git b/nostrdb/bindings/c/flatbuffers_common_reader.h a/nostrdb/bindings/c/flatbuffers_common_reader.h +index c575308689b9..49e479e29980 100644 +--- b/nostrdb/bindings/c/flatbuffers_common_reader.h ++++ a/nostrdb/bindings/c/flatbuffers_common_reader.h +@@ -5,8 +5,8 @@ + + /* Common FlatBuffers read functionality for C. */ + +-#include "flatcc/flatcc_prologue.h" +-#include "flatcc/flatcc_flatbuffers.h" ++#include "flatcc_prologue.h" ++#include "flatcc_flatbuffers.h" + + + #define __flatbuffers_read_scalar_at_byteoffset(N, p, o) N ## _read_from_pe((uint8_t *)(p) + (o)) +@@ -574,5 +574,5 @@ static inline N ## _ ## K ## t N ## _as_typed_root(const void *buffer__tmp)\ + #define __flatbuffers_struct_as_root(N) __flatbuffers_buffer_as_root(N, struct_) + #define __flatbuffers_table_as_root(N) __flatbuffers_buffer_as_root(N, table_) + +-#include "flatcc/flatcc_epilogue.h" ++#include "flatcc_epilogue.h" + #endif /* FLATBUFFERS_COMMON_H */ +diff --git b/nostrdb/bindings/c/profile_json_parser.h a/nostrdb/bindings/c/profile_json_parser.h +index a7caaaec6d37..f404b6679fe0 100644 +--- b/nostrdb/bindings/c/profile_json_parser.h ++++ a/nostrdb/bindings/c/profile_json_parser.h +@@ -3,8 +3,8 @@ + + /* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +-#include "flatcc/flatcc_json_parser.h" +-#include "flatcc/flatcc_prologue.h" ++#include "flatcc_json_parser.h" ++#include "flatcc_prologue.h" + + /* + * Parses the default root table or struct of the schema and constructs a FlatBuffer. +@@ -280,5 +280,5 @@ static int profile_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + return 0; + } + +-#include "flatcc/flatcc_epilogue.h" ++#include "flatcc_epilogue.h" + #endif /* PROFILE_JSON_PARSER_H */ +diff --git b/nostrdb/flatcc/builder.c a/nostrdb/flatcc/builder.c +index 9f54d884ff53..c5155a85e407 100644 +--- b/nostrdb/flatcc/builder.c ++++ a/nostrdb/flatcc/builder.c +@@ -16,8 +16,8 @@ + #include + #include + +-#include "flatcc/flatcc_builder.h" +-#include "flatcc/flatcc_emitter.h" ++#include "flatcc_builder.h" ++#include "flatcc_emitter.h" + + /* + * `check` is designed to handle incorrect use errors that can be +diff --git b/nostrdb/flatcc/emitter.c a/nostrdb/flatcc/emitter.c +index 089ea00b2060..dbeffacd97ed 100644 +--- b/nostrdb/flatcc/emitter.c ++++ a/nostrdb/flatcc/emitter.c +@@ -1,7 +1,7 @@ + #include + +-#include "flatcc/flatcc_rtconfig.h" +-#include "flatcc/flatcc_emitter.h" ++#include "flatcc_rtconfig.h" ++#include "flatcc_emitter.h" + + static int advance_front(flatcc_emitter_t *E) + { +diff --git b/nostrdb/flatcc/flatcc_alloc.h a/nostrdb/flatcc/flatcc_alloc.h +index 155364c1e2ba..c07462d57754 100644 +--- b/nostrdb/flatcc/flatcc_alloc.h ++++ a/nostrdb/flatcc/flatcc_alloc.h +@@ -69,7 +69,7 @@ extern "C" { + #ifndef FLATCC_USE_GENERIC_ALIGNED_ALLOC + + #ifndef FLATCC_NO_PALIGNED_ALLOC +-#include "flatcc/portable/paligned_alloc.h" ++#include "paligned_alloc.h" + #else + #if !defined(__aligned_free_is_defined) || !__aligned_free_is_defined + #define aligned_free free +diff --git b/nostrdb/flatcc/flatcc_emitter.h a/nostrdb/flatcc/flatcc_emitter.h +index b8c83b94d3de..11756f23f4d0 100644 +--- b/nostrdb/flatcc/flatcc_emitter.h ++++ a/nostrdb/flatcc/flatcc_emitter.h +@@ -16,9 +16,9 @@ extern "C" { + #include + #include + +-#include "flatcc/flatcc_types.h" +-#include "flatcc/flatcc_iov.h" +-#include "flatcc/flatcc_alloc.h" ++#include "flatcc_types.h" ++#include "flatcc_iov.h" ++#include "flatcc_alloc.h" + + /* + * The buffer steadily grows during emission but the design allows for +diff --git b/nostrdb/flatcc/flatcc_endian.h a/nostrdb/flatcc/flatcc_endian.h +index 0592f3132380..d16f72c89a11 100644 +--- b/nostrdb/flatcc/flatcc_endian.h ++++ a/nostrdb/flatcc/flatcc_endian.h +@@ -66,7 +66,7 @@ extern "C" { + #define htobe8(n) (n) + #endif + +-#include "flatcc/flatcc_accessors.h" ++#include "flatcc_accessors.h" + + /* This is the binary encoding endianness, usually LE for flatbuffers. */ + #if FLATBUFFERS_PROTOCOL_IS_LE +diff --git b/nostrdb/flatcc/flatcc_epilogue.h a/nostrdb/flatcc/flatcc_epilogue.h +index 496857ba1251..dc724f6c98ed 100644 +--- b/nostrdb/flatcc/flatcc_epilogue.h ++++ a/nostrdb/flatcc/flatcc_epilogue.h +@@ -4,5 +4,5 @@ + } + #endif + +-#include "flatcc/portable/pdiagnostic_pop.h" ++#include "pdiagnostic_pop.h" + +diff --git b/nostrdb/flatcc/flatcc_flatbuffers.h a/nostrdb/flatcc/flatcc_flatbuffers.h +index 4bfc7435251a..210c9f2a420d 100644 +--- b/nostrdb/flatcc/flatcc_flatbuffers.h ++++ a/nostrdb/flatcc/flatcc_flatbuffers.h +@@ -4,7 +4,7 @@ + * + * Outside include guard to handle scope counter. + */ +-#include "flatcc/portable/pstatic_assert.h" ++#include "pstatic_assert.h" + + #ifndef FLATCC_FLATBUFFERS_H + #define FLATCC_FLATBUFFERS_H +@@ -19,15 +19,15 @@ extern "C" { + #ifdef FLATCC_PORTABLE + #include "flatcc/flatcc_portable.h" + #endif +-#include "flatcc/portable/pwarnings.h" ++#include "pwarnings.h" + /* Needed by C99 compilers without FLATCC_PORTABLE. */ +-#include "flatcc/portable/pstdalign.h" ++#include "pstdalign.h" + + /* Handle fallthrough attribute in switch statements. */ +-#include "flatcc/portable/pattributes.h" ++#include "pattributes.h" + +-#include "flatcc/flatcc_alloc.h" +-#include "flatcc/flatcc_assert.h" ++#include "flatcc_alloc.h" ++#include "flatcc_assert.h" + + #define __FLATBUFFERS_PASTE2(a, b) a ## b + #define __FLATBUFFERS_PASTE3(a, b, c) a ## b ## c +@@ -37,10 +37,10 @@ extern "C" { + * "flatcc_endian.h" requires the preceeding include files, + * or compatible definitions. + */ +-#include "flatcc/portable/pendian.h" +-#include "flatcc/flatcc_types.h" +-#include "flatcc/flatcc_endian.h" +-#include "flatcc/flatcc_identifier.h" ++#include "pendian.h" ++#include "flatcc_types.h" ++#include "flatcc_endian.h" ++#include "flatcc_identifier.h" + + #ifndef FLATBUFFERS_WRAP_NAMESPACE + #define FLATBUFFERS_WRAP_NAMESPACE(ns, x) ns ## _ ## x +diff --git b/nostrdb/flatcc/flatcc_json_parser.h a/nostrdb/flatcc/flatcc_json_parser.h +index 1907fc7fc635..ed7151c2fd6b 100644 +--- b/nostrdb/flatcc/flatcc_json_parser.h ++++ a/nostrdb/flatcc/flatcc_json_parser.h +@@ -15,12 +15,12 @@ extern "C" { + #include + #include + +-#include "flatcc/flatcc_rtconfig.h" +-#include "flatcc/flatcc_builder.h" +-#include "flatcc/flatcc_unaligned.h" ++#include "flatcc_rtconfig.h" ++#include "flatcc_builder.h" ++#include "flatcc_unaligned.h" + + #define PDIAGNOSTIC_IGNORE_UNUSED +-#include "flatcc/portable/pdiagnostic_push.h" ++#include "pdiagnostic_push.h" + + enum flatcc_json_parser_flags { + flatcc_json_parser_f_skip_unknown = 1, +@@ -886,7 +886,7 @@ int flatcc_json_parser_struct_as_root(flatcc_builder_t *B, flatcc_json_parser_t + const char *buf, size_t bufsiz, int flags, const char *fid, + flatcc_json_parser_struct_f *parser); + +-#include "flatcc/portable/pdiagnostic_pop.h" ++#include "pdiagnostic_pop.h" + + #ifdef __cplusplus + } +diff --git b/nostrdb/flatcc/flatcc_prologue.h a/nostrdb/flatcc/flatcc_prologue.h +index 3a74ed6040db..36344c4c071f 100644 +--- b/nostrdb/flatcc/flatcc_prologue.h ++++ a/nostrdb/flatcc/flatcc_prologue.h +@@ -1,7 +1,7 @@ + /* Include guard intentionally left out. */ + + #define PDIAGNOSTIC_IGNORE_UNUSED +-#include "flatcc/portable/pdiagnostic_push.h" ++#include "pdiagnostic_push.h" + + #ifdef __cplusplus + extern "C" { +diff --git b/nostrdb/flatcc/flatcc_refmap.h a/nostrdb/flatcc/flatcc_refmap.h +index 062d94f5d35d..beafa301d042 100644 +--- b/nostrdb/flatcc/flatcc_refmap.h ++++ a/nostrdb/flatcc/flatcc_refmap.h +@@ -50,7 +50,7 @@ + extern "C" { + #endif + +-#include "flatcc/flatcc_types.h" ++#include "flatcc_types.h" + + #ifndef FLATCC_REFMAP_MIN_BUCKETS + /* 8 buckets gives us 5 useful initial entries with a load factor of 0.7 */ +diff --git b/nostrdb/flatcc/flatcc_unaligned.h a/nostrdb/flatcc/flatcc_unaligned.h +index a7dc546111cd..5ea26cede6ee 100644 +--- b/nostrdb/flatcc/flatcc_unaligned.h ++++ a/nostrdb/flatcc/flatcc_unaligned.h +@@ -5,7 +5,7 @@ + extern "C" { + #endif + +-#include "flatcc/portable/punaligned.h" ++#include "punaligned.h" + + #define FLATCC_ALLOW_UNALIGNED_ACCESS PORTABLE_UNALIGNED_ACCESS + +diff --git b/nostrdb/flatcc/json_parser.c a/nostrdb/flatcc/json_parser.c +index 0e3aeea9834c..06f778da33f3 100644 +--- b/nostrdb/flatcc/json_parser.c ++++ a/nostrdb/flatcc/json_parser.c +@@ -1,6 +1,6 @@ +-#include "flatcc/flatcc_rtconfig.h" +-#include "flatcc/flatcc_json_parser.h" +-#include "flatcc/flatcc_assert.h" ++#include "flatcc_rtconfig.h" ++#include "flatcc_json_parser.h" ++#include "flatcc_assert.h" + + #define uoffset_t flatbuffers_uoffset_t + #define soffset_t flatbuffers_soffset_t +@@ -16,8 +16,8 @@ + #if FLATCC_USE_GRISU3 && !defined(PORTABLE_USE_GRISU3) + #define PORTABLE_USE_GRISU3 1 + #endif +-#include "flatcc/portable/pparsefp.h" +-#include "flatcc/portable/pbase64.h" ++#include "portable/pparsefp.h" ++#include "portable/pbase64.h" + + #if FLATCC_USE_SSE4_2 + #ifdef __SSE4_2__ +diff --git b/nostrdb/flatcc/refmap.c a/nostrdb/flatcc/refmap.c +index a2497f02247b..d8c6034fbb12 100644 +--- b/nostrdb/flatcc/refmap.c ++++ a/nostrdb/flatcc/refmap.c +@@ -13,10 +13,10 @@ + #include + #include + +-#include "flatcc/flatcc_rtconfig.h" +-#include "flatcc/flatcc_refmap.h" +-#include "flatcc/flatcc_alloc.h" +-#include "flatcc/flatcc_assert.h" ++#include "flatcc_rtconfig.h" ++#include "flatcc_refmap.h" ++#include "flatcc_alloc.h" ++#include "flatcc_assert.h" + + #define _flatcc_refmap_calloc FLATCC_CALLOC + #define _flatcc_refmap_free FLATCC_FREE diff --git a/nostrdb/flatcc/CMakeLists.txt b/nostrdb/flatcc/CMakeLists.txt new file mode 100644 index 0000000000..127e2a47ff --- /dev/null +++ b/nostrdb/flatcc/CMakeLists.txt @@ -0,0 +1,16 @@ +include_directories ( + "${PROJECT_SOURCE_DIR}/include" +) + +add_library(flatccrt + builder.c + emitter.c + refmap.c + verifier.c + json_parser.c + json_printer.c +) + +if (FLATCC_INSTALL) + install(TARGETS flatccrt DESTINATION ${lib_dir}) +endif() diff --git a/nostrdb/flatcc/builder.c b/nostrdb/flatcc/builder.c new file mode 100644 index 0000000000..c5155a85e4 --- /dev/null +++ b/nostrdb/flatcc/builder.c @@ -0,0 +1,2035 @@ +/* + * Codegenerator for C, building FlatBuffers. + * + * There are several approaches, some light, some requiring a library, + * some with vectored I/O etc. + * + * Here we focus on a reasonable balance of light code and efficiency. + * + * Builder code is generated to a separate file that includes the + * generated read-only code. + * + * Mutable buffers are not supported in this version. + * + */ + +#include +#include + +#include "flatcc_builder.h" +#include "flatcc_emitter.h" + +/* + * `check` is designed to handle incorrect use errors that can be + * ignored in production of a tested product. + * + * `check_error` fails if condition is false and is designed to return an + * error code in production. + */ + +#if FLATCC_BUILDER_ASSERT_ON_ERROR +#define check(cond, reason) FLATCC_BUILDER_ASSERT(cond, reason) +#else +#define check(cond, reason) ((void)0) +#endif + +#if FLATCC_BUILDER_SKIP_CHECKS +#define check_error(cond, err, reason) ((void)0) +#else +#define check_error(cond, err, reason) if (!(cond)) { check(cond, reason); return err; } +#endif + +/* `strnlen` not widely supported. */ +static inline size_t pstrnlen(const char *s, size_t max_len) +{ + const char *end = memchr(s, 0, max_len); + return end ? (size_t)(end - s) : max_len; +} +#undef strnlen +#define strnlen pstrnlen + +/* Padding can be up to 255 zeroes, and 1 zero string termination byte. + * When two paddings are combined at nested buffers, we need twice that. + * Visible to emitter so it can test for zero padding in iov. */ +const uint8_t flatcc_builder_padding_base[512] = { 0 }; +#define _pad flatcc_builder_padding_base + +#define uoffset_t flatbuffers_uoffset_t +#define soffset_t flatbuffers_soffset_t +#define voffset_t flatbuffers_voffset_t +#define utype_t flatbuffers_utype_t + +#define write_uoffset __flatbuffers_uoffset_write_to_pe +#define write_voffset __flatbuffers_voffset_write_to_pe +#define write_identifier __flatbuffers_uoffset_write_to_pe +#define write_utype __flatbuffers_utype_write_to_pe + +#define field_size sizeof(uoffset_t) +#define max_offset_count FLATBUFFERS_COUNT_MAX(field_size) +#define union_size sizeof(flatcc_builder_union_ref_t) +#define max_union_count FLATBUFFERS_COUNT_MAX(union_size) +#define utype_size sizeof(utype_t) +#define max_utype_count FLATBUFFERS_COUNT_MAX(utype_size) + +#define max_string_len FLATBUFFERS_COUNT_MAX(1) +#define identifier_size FLATBUFFERS_IDENTIFIER_SIZE + + +#define iovec_t flatcc_iovec_t +#define frame_size sizeof(__flatcc_builder_frame_t) +#define frame(x) (B->frame[0].x) + + +/* `align` must be a power of 2. */ +static inline uoffset_t alignup_uoffset(uoffset_t x, size_t align) +{ + return (x + (uoffset_t)align - 1u) & ~((uoffset_t)align - 1u); +} + +static inline size_t alignup_size(size_t x, size_t align) +{ + return (x + align - 1u) & ~(align - 1u); +} + + +typedef struct vtable_descriptor vtable_descriptor_t; +struct vtable_descriptor { + /* Where the vtable is emitted. */ + flatcc_builder_ref_t vt_ref; + /* Which buffer it was emitted to. */ + uoffset_t nest_id; + /* Where the vtable is cached. */ + uoffset_t vb_start; + /* Hash table collision chain. */ + uoffset_t next; +}; + +typedef struct flatcc_iov_state flatcc_iov_state_t; +struct flatcc_iov_state { + size_t len; + int count; + flatcc_iovec_t iov[FLATCC_IOV_COUNT_MAX]; +}; + +#define iov_state_t flatcc_iov_state_t + +/* This assumes `iov_state_t iov;` has been declared in scope */ +#define push_iov_cond(base, size, cond) if ((size) > 0 && (cond)) { iov.len += size;\ + iov.iov[iov.count].iov_base = (void *)(base); iov.iov[iov.count].iov_len = (size); ++iov.count; } +#define push_iov(base, size) push_iov_cond(base, size, 1) +#define init_iov() { iov.len = 0; iov.count = 0; } + + +int flatcc_builder_default_alloc(void *alloc_context, iovec_t *b, size_t request, int zero_fill, int hint) +{ + void *p; + size_t n; + + (void)alloc_context; + + if (request == 0) { + if (b->iov_base) { + FLATCC_BUILDER_FREE(b->iov_base); + b->iov_base = 0; + b->iov_len = 0; + } + return 0; + } + switch (hint) { + case flatcc_builder_alloc_ds: + n = 256; + break; + case flatcc_builder_alloc_ht: + /* Should be exact size, or space size is just wasted. */ + n = request; + break; + case flatcc_builder_alloc_fs: + n = sizeof(__flatcc_builder_frame_t) * 8; + break; + case flatcc_builder_alloc_us: + n = 64; + break; + default: + /* + * We have many small structures - vs stack for tables with few + * elements, and few offset fields in patch log. No need to + * overallocate in case of busy small messages. + */ + n = 32; + break; + } + while (n < request) { + n *= 2; + } + if (request <= b->iov_len && b->iov_len / 2 >= n) { + /* Add hysteresis to shrink. */ + return 0; + } + if (!(p = FLATCC_BUILDER_REALLOC(b->iov_base, n))) { + return -1; + } + /* Realloc might also shrink. */ + if (zero_fill && b->iov_len < n) { + memset((uint8_t *)p + b->iov_len, 0, n - b->iov_len); + } + b->iov_base = p; + b->iov_len = n; + return 0; +} + +#define T_ptr(base, pos) ((void *)((uint8_t *)(base) + (uoffset_t)(pos))) +#define ds_ptr(pos) (T_ptr(B->buffers[flatcc_builder_alloc_ds].iov_base, (pos))) +#define vs_ptr(pos) (T_ptr(B->buffers[flatcc_builder_alloc_vs].iov_base, (pos))) +#define pl_ptr(pos) (T_ptr(B->buffers[flatcc_builder_alloc_pl].iov_base, (pos))) +#define us_ptr(pos) (T_ptr(B->buffers[flatcc_builder_alloc_us].iov_base, (pos))) +#define vd_ptr(pos) (T_ptr(B->buffers[flatcc_builder_alloc_vd].iov_base, (pos))) +#define vb_ptr(pos) (T_ptr(B->buffers[flatcc_builder_alloc_vb].iov_base, (pos))) +#define vs_offset(ptr) ((uoffset_t)((size_t)(ptr) - (size_t)B->buffers[flatcc_builder_alloc_vs].iov_base)) +#define pl_offset(ptr) ((uoffset_t)((size_t)(ptr) - (size_t)B->buffers[flatcc_builder_alloc_pl].iov_base)) +#define us_offset(ptr) ((uoffset_t)((size_t)(ptr) - (size_t)B->buffers[flatcc_builder_alloc_us].iov_base)) + +#define table_limit (FLATBUFFERS_VOFFSET_MAX - field_size + 1) +#define data_limit (FLATBUFFERS_UOFFSET_MAX - field_size + 1) + +#define set_identifier(id) memcpy(&B->identifier, (id) ? (void *)(id) : (void *)_pad, identifier_size) + +/* Must also return true when no buffer has been started. */ +#define is_top_buffer(B) (B->nest_id == 0) + +/* + * Tables use a stack represention better suited for quickly adding + * fields to tables, but it must occasionally be refreshed following + * reallocation or reentry from child frame. + */ +static inline void refresh_ds(flatcc_builder_t *B, uoffset_t type_limit) +{ + iovec_t *buf = B->buffers + flatcc_builder_alloc_ds; + + B->ds = ds_ptr(B->ds_first); + B->ds_limit = (uoffset_t)buf->iov_len - B->ds_first; + /* + * So we don't allocate outside tables representation size, nor our + * current buffer size. + */ + if (B->ds_limit > type_limit) { + B->ds_limit = type_limit; + } + /* So exit frame can refresh fast. */ + frame(type_limit) = type_limit; +} + +static int reserve_ds(flatcc_builder_t *B, size_t need, uoffset_t limit) +{ + iovec_t *buf = B->buffers + flatcc_builder_alloc_ds; + + if (B->alloc(B->alloc_context, buf, B->ds_first + need, 1, flatcc_builder_alloc_ds)) { + return -1; + } + refresh_ds(B, limit); + return 0; +} + +/* + * Make sure there is always an extra zero termination on stack + * even if it isn't emitted such that string updates may count + * on zero termination being present always. + */ +static inline void *push_ds(flatcc_builder_t *B, uoffset_t size) +{ + size_t offset; + + offset = B->ds_offset; + if ((B->ds_offset += size) >= B->ds_limit) { + if (reserve_ds(B, B->ds_offset + 1, data_limit)) { + return 0; + } + } + return B->ds + offset; +} + +static inline void unpush_ds(flatcc_builder_t *B, uoffset_t size) +{ + B->ds_offset -= size; + memset(B->ds + B->ds_offset, 0, size); +} + +static inline void *push_ds_copy(flatcc_builder_t *B, const void *data, uoffset_t size) +{ + void *p; + + if (!(p = push_ds(B, size))) { + return 0; + } + memcpy(p, data, size); + return p; +} + +static inline void *push_ds_field(flatcc_builder_t *B, uoffset_t size, uint16_t align, voffset_t id) +{ + uoffset_t offset; + + /* + * We calculate table field alignment relative to first entry, not + * header field with vtable offset. + * + * Note: >= comparison handles special case where B->ds is not + * allocated yet and size is 0 so the return value would be mistaken + * for an error. + */ + offset = alignup_uoffset(B->ds_offset, align); + if ((B->ds_offset = offset + size) >= B->ds_limit) { + if (reserve_ds(B, B->ds_offset + 1, table_limit)) { + return 0; + } + } + B->vs[id] = (voffset_t)(offset + field_size); + if (id >= B->id_end) { + B->id_end = id + 1u; + } + return B->ds + offset; +} + +static inline void *push_ds_offset_field(flatcc_builder_t *B, voffset_t id) +{ + uoffset_t offset; + + offset = alignup_uoffset(B->ds_offset, field_size); + if ((B->ds_offset = offset + field_size) > B->ds_limit) { + if (reserve_ds(B, B->ds_offset, table_limit)) { + return 0; + } + } + B->vs[id] = (voffset_t)(offset + field_size); + if (id >= B->id_end) { + B->id_end = id + 1u; + } + *B->pl++ = (flatbuffers_voffset_t)offset; + return B->ds + offset; +} + +static inline void *reserve_buffer(flatcc_builder_t *B, int alloc_type, size_t used, size_t need, int zero_init) +{ + iovec_t *buf = B->buffers + alloc_type; + + if (used + need > buf->iov_len) { + if (B->alloc(B->alloc_context, buf, used + need, zero_init, alloc_type)) { + check(0, "memory allocation failed"); + return 0; + } + } + return (void *)((size_t)buf->iov_base + used); +} + +static inline int reserve_fields(flatcc_builder_t *B, int count) +{ + size_t used, need; + + /* Provide faster stack operations for common table operations. */ + used = frame(container.table.vs_end) + frame(container.table.id_end) * sizeof(voffset_t); + need = (size_t)(count + 2) * sizeof(voffset_t); + if (!(B->vs = reserve_buffer(B, flatcc_builder_alloc_vs, used, need, 1))) { + return -1; + } + /* Move past header for convenience. */ + B->vs += 2; + used = frame(container.table.pl_end); + /* Add one to handle special case of first table being empty. */ + need = (size_t)count * sizeof(*(B->pl)) + 1; + if (!(B->pl = reserve_buffer(B, flatcc_builder_alloc_pl, used, need, 0))) { + return -1; + } + return 0; +} + +static int alloc_ht(flatcc_builder_t *B) +{ + iovec_t *buf = B->buffers + flatcc_builder_alloc_ht; + + size_t size, k; + /* Allocate null entry so we can check for return errors. */ + FLATCC_ASSERT(B->vd_end == 0); + if (!reserve_buffer(B, flatcc_builder_alloc_vd, B->vd_end, sizeof(vtable_descriptor_t), 0)) { + return -1; + } + B->vd_end = sizeof(vtable_descriptor_t); + size = field_size * FLATCC_BUILDER_MIN_HASH_COUNT; + if (B->alloc(B->alloc_context, buf, size, 1, flatcc_builder_alloc_ht)) { + return -1; + } + while (size * 2 <= buf->iov_len) { + size *= 2; + } + size /= field_size; + for (k = 0; (((size_t)1) << k) < size; ++k) { + } + B->ht_width = k; + return 0; +} + +static inline uoffset_t *lookup_ht(flatcc_builder_t *B, uint32_t hash) +{ + uoffset_t *T; + + if (B->ht_width == 0) { + if (alloc_ht(B)) { + return 0; + } + } + T = B->buffers[flatcc_builder_alloc_ht].iov_base; + + return &T[FLATCC_BUILDER_BUCKET_VT_HASH(hash, B->ht_width)]; +} + +void flatcc_builder_flush_vtable_cache(flatcc_builder_t *B) +{ + iovec_t *buf = B->buffers + flatcc_builder_alloc_ht; + + if (B->ht_width == 0) { + return; + } + memset(buf->iov_base, 0, buf->iov_len); + /* Reserve the null entry. */ + B->vd_end = sizeof(vtable_descriptor_t); + B->vb_end = 0; +} + +int flatcc_builder_custom_init(flatcc_builder_t *B, + flatcc_builder_emit_fun *emit, void *emit_context, + flatcc_builder_alloc_fun *alloc, void *alloc_context) +{ + /* + * Do not allocate anything here. Only the required buffers will be + * allocated. For simple struct buffers, no allocation is required + * at all. + */ + memset(B, 0, sizeof(*B)); + + if (emit == 0) { + B->is_default_emitter = 1; + emit = flatcc_emitter; + emit_context = &B->default_emit_context; + } + if (alloc == 0) { + alloc = flatcc_builder_default_alloc; + } + B->alloc_context = alloc_context; + B->alloc = alloc; + B->emit_context = emit_context; + B->emit = emit; + return 0; +} + +int flatcc_builder_init(flatcc_builder_t *B) +{ + return flatcc_builder_custom_init(B, 0, 0, 0, 0); +} + +int flatcc_builder_custom_reset(flatcc_builder_t *B, int set_defaults, int reduce_buffers) +{ + iovec_t *buf; + int i; + + for (i = 0; i < FLATCC_BUILDER_ALLOC_BUFFER_COUNT; ++i) { + buf = B->buffers + i; + if (buf->iov_base) { + /* Don't try to reduce the hash table. */ + if (i != flatcc_builder_alloc_ht && + reduce_buffers && B->alloc(B->alloc_context, buf, 1, 1, i)) { + return -1; + } + memset(buf->iov_base, 0, buf->iov_len); + } else { + FLATCC_ASSERT(buf->iov_len == 0); + } + } + B->vb_end = 0; + if (B->vd_end > 0) { + /* Reset past null entry. */ + B->vd_end = sizeof(vtable_descriptor_t); + } + B->min_align = 0; + B->emit_start = 0; + B->emit_end = 0; + B->level = 0; + B->limit_level = 0; + B->ds_offset = 0; + B->ds_limit = 0; + B->nest_count = 0; + B->nest_id = 0; + /* Needed for correct offset calculation. */ + B->ds = B->buffers[flatcc_builder_alloc_ds].iov_base; + B->pl = B->buffers[flatcc_builder_alloc_pl].iov_base; + B->vs = B->buffers[flatcc_builder_alloc_vs].iov_base; + B->frame = 0; + if (set_defaults) { + B->vb_flush_limit = 0; + B->max_level = 0; + B->disable_vt_clustering = 0; + } + if (B->is_default_emitter) { + flatcc_emitter_reset(&B->default_emit_context); + } + if (B->refmap) { + flatcc_refmap_reset(B->refmap); + } + return 0; +} + +int flatcc_builder_reset(flatcc_builder_t *B) +{ + return flatcc_builder_custom_reset(B, 0, 0); +} + +void flatcc_builder_clear(flatcc_builder_t *B) +{ + iovec_t *buf; + int i; + + for (i = 0; i < FLATCC_BUILDER_ALLOC_BUFFER_COUNT; ++i) { + buf = B->buffers + i; + B->alloc(B->alloc_context, buf, 0, 0, i); + } + if (B->is_default_emitter) { + flatcc_emitter_clear(&B->default_emit_context); + } + if (B->refmap) { + flatcc_refmap_clear(B->refmap); + } + memset(B, 0, sizeof(*B)); +} + +static inline void set_min_align(flatcc_builder_t *B, uint16_t align) +{ + if (B->min_align < align) { + B->min_align = align; + } +} + +/* + * It's a max, but the minimum viable alignment is the largest observed + * alignment requirement, but no larger. + */ +static inline void get_min_align(uint16_t *align, uint16_t b) +{ + if (*align < b) { + *align = b; + } +} + +void *flatcc_builder_enter_user_frame_ptr(flatcc_builder_t *B, size_t size) +{ + size_t *frame; + + size = alignup_size(size, sizeof(size_t)) + sizeof(size_t); + + if (!(frame = reserve_buffer(B, flatcc_builder_alloc_us, B->user_frame_end, size, 0))) { + return 0; + } + memset(frame, 0, size); + *frame++ = B->user_frame_offset; + B->user_frame_offset = B->user_frame_end + sizeof(size_t); + B->user_frame_end += size; + return frame; +} + +size_t flatcc_builder_enter_user_frame(flatcc_builder_t *B, size_t size) +{ + size_t *frame; + + size = alignup_size(size, sizeof(size_t)) + sizeof(size_t); + + if (!(frame = reserve_buffer(B, flatcc_builder_alloc_us, B->user_frame_end, size, 0))) { + return 0; + } + memset(frame, 0, size); + *frame++ = B->user_frame_offset; + B->user_frame_offset = B->user_frame_end + sizeof(size_t); + B->user_frame_end += size; + return B->user_frame_offset; +} + + +size_t flatcc_builder_exit_user_frame(flatcc_builder_t *B) +{ + size_t *hdr; + + FLATCC_ASSERT(B->user_frame_offset > 0); + + hdr = us_ptr(B->user_frame_offset); + B->user_frame_end = B->user_frame_offset - sizeof(size_t); + return B->user_frame_offset = hdr[-1]; +} + +size_t flatcc_builder_exit_user_frame_at(flatcc_builder_t *B, size_t handle) +{ + FLATCC_ASSERT(B->user_frame_offset >= handle); + + B->user_frame_offset = handle; + return flatcc_builder_exit_user_frame(B); +} + +size_t flatcc_builder_get_current_user_frame(flatcc_builder_t *B) +{ + return B->user_frame_offset; +} + +void *flatcc_builder_get_user_frame_ptr(flatcc_builder_t *B, size_t handle) +{ + return us_ptr(handle); +} + +static int enter_frame(flatcc_builder_t *B, uint16_t align) +{ + if (++B->level > B->limit_level) { + if (B->max_level > 0 && B->level > B->max_level) { + return -1; + } + if (!(B->frame = reserve_buffer(B, flatcc_builder_alloc_fs, + (size_t)(B->level - 1) * frame_size, frame_size, 0))) { + return -1; + } + B->limit_level = (int)(B->buffers[flatcc_builder_alloc_fs].iov_len / frame_size); + if (B->max_level > 0 && B->max_level < B->limit_level) { + B->limit_level = B->max_level; + } + } else { + ++B->frame; + } + frame(ds_offset) = B->ds_offset; + frame(align) = B->align; + B->align = align; + /* Note: do not assume padding before first has been allocated! */ + frame(ds_first) = B->ds_first; + frame(type_limit) = data_limit; + B->ds_first = alignup_uoffset(B->ds_first + B->ds_offset, 8); + B->ds_offset = 0; + return 0; +} + +static inline void exit_frame(flatcc_builder_t *B) +{ + memset(B->ds, 0, B->ds_offset); + B->ds_offset = frame(ds_offset); + B->ds_first = frame(ds_first); + refresh_ds(B, frame(type_limit)); + + /* + * Restore local alignment: e.g. a table should not change alignment + * because a child table was just created elsewhere in the buffer, + * but the overall alignment (min align), should be aware of it. + * Each buffer has its own min align that then migrates up without + * being affected by sibling or child buffers. + */ + set_min_align(B, B->align); + B->align = frame(align); + + --B->frame; + --B->level; +} + +static inline uoffset_t front_pad(flatcc_builder_t *B, uoffset_t size, uint16_t align) +{ + return (uoffset_t)(B->emit_start - (flatcc_builder_ref_t)size) & (align - 1u); +} + +static inline uoffset_t back_pad(flatcc_builder_t *B, uint16_t align) +{ + return (uoffset_t)(B->emit_end) & (align - 1u); +} + +static inline flatcc_builder_ref_t emit_front(flatcc_builder_t *B, iov_state_t *iov) +{ + flatcc_builder_ref_t ref; + + /* + * We might have overflow when including headers, but without + * headers we should have checks to prevent overflow in the + * uoffset_t range, hence we subtract 16 to be safe. With that + * guarantee we can also make a safe check on the soffset_t range. + * + * We only allow buffers half the theoritical size of + * FLATBUFFERS_UOFFSET_MAX so we can safely use signed references. + * + * NOTE: vtables vt_offset field is signed, and the check in create + * table only ensures the signed limit. The check would fail if the + * total buffer size could grow beyond UOFFSET_MAX, and we prevent + * that by limiting the lower end to SOFFSET_MIN, and the upper end + * at emit_back to SOFFSET_MAX. + */ + ref = B->emit_start - (flatcc_builder_ref_t)iov->len; + if ((iov->len > 16 && iov->len - 16 > FLATBUFFERS_UOFFSET_MAX) || ref >= B->emit_start) { + check(0, "buffer too large to represent"); + return 0; + } + if (B->emit(B->emit_context, iov->iov, iov->count, ref, iov->len)) { + check(0, "emitter rejected buffer content"); + return 0; + } + return B->emit_start = ref; +} + +static inline flatcc_builder_ref_t emit_back(flatcc_builder_t *B, iov_state_t *iov) +{ + flatcc_builder_ref_t ref; + + ref = B->emit_end; + B->emit_end = ref + (flatcc_builder_ref_t)iov->len; + /* + * Similar to emit_front check, but since we only emit vtables and + * padding at the back, we are not concerned with iov->len overflow, + * only total buffer overflow. + * + * With this check, vtable soffset references at table header can + * still overflow in extreme cases, so this must be checked + * separately. + */ + if (B->emit_end < ref) { + check(0, "buffer too large to represent"); + return 0; + } + if (B->emit(B->emit_context, iov->iov, iov->count, ref, iov->len)) { + check(0, "emitter rejected buffer content"); + return 0; + } + /* + * Back references always return ref + 1 because ref == 0 is valid and + * should not be mistaken for error. vtables understand this. + */ + return ref + 1; +} + +static int align_to_block(flatcc_builder_t *B, uint16_t *align, uint16_t block_align, int is_nested) +{ + size_t end_pad; + iov_state_t iov; + + block_align = block_align ? block_align : B->block_align ? B->block_align : 1; + get_min_align(align, field_size); + get_min_align(align, block_align); + /* Pad end of buffer to multiple. */ + if (!is_nested) { + end_pad = back_pad(B, block_align); + if (end_pad) { + init_iov(); + push_iov(_pad, end_pad); + if (0 == emit_back(B, &iov)) { + check(0, "emitter rejected buffer content"); + return -1; + } + } + } + return 0; +} + +flatcc_builder_ref_t flatcc_builder_embed_buffer(flatcc_builder_t *B, + uint16_t block_align, + const void *data, size_t size, uint16_t align, int flags) +{ + uoffset_t size_field, pad; + iov_state_t iov; + int with_size = flags & flatcc_builder_with_size; + + if (align_to_block(B, &align, block_align, !is_top_buffer(B))) { + return 0; + } + pad = front_pad(B, (uoffset_t)(size + (with_size ? field_size : 0)), align); + write_uoffset(&size_field, (uoffset_t)size + pad); + init_iov(); + /* Add ubyte vector size header if nested buffer. */ + push_iov_cond(&size_field, field_size, !is_top_buffer(B)); + push_iov(data, size); + push_iov(_pad, pad); + return emit_front(B, &iov); +} + +flatcc_builder_ref_t flatcc_builder_create_buffer(flatcc_builder_t *B, + const char identifier[identifier_size], uint16_t block_align, + flatcc_builder_ref_t object_ref, uint16_t align, int flags) +{ + flatcc_builder_ref_t buffer_ref; + uoffset_t header_pad, id_size = 0; + uoffset_t object_offset, buffer_size, buffer_base; + iov_state_t iov; + flatcc_builder_identifier_t id_out = 0; + int is_nested = (flags & flatcc_builder_is_nested) != 0; + int with_size = (flags & flatcc_builder_with_size) != 0; + + if (align_to_block(B, &align, block_align, is_nested)) { + return 0; + } + set_min_align(B, align); + if (identifier) { + FLATCC_ASSERT(sizeof(flatcc_builder_identifier_t) == identifier_size); + FLATCC_ASSERT(sizeof(flatcc_builder_identifier_t) == field_size); + memcpy(&id_out, identifier, identifier_size); + id_out = __flatbuffers_thash_read_from_le(&id_out); + write_identifier(&id_out, id_out); + } + id_size = id_out ? identifier_size : 0; + header_pad = front_pad(B, field_size + id_size + (uoffset_t)(with_size ? field_size : 0), align); + init_iov(); + /* ubyte vectors size field wrapping nested buffer. */ + push_iov_cond(&buffer_size, field_size, is_nested || with_size); + push_iov(&object_offset, field_size); + /* Identifiers are not always present in buffer. */ + push_iov(&id_out, id_size); + push_iov(_pad, header_pad); + buffer_base = (uoffset_t)B->emit_start - (uoffset_t)iov.len + (uoffset_t)((is_nested || with_size) ? field_size : 0); + if (is_nested) { + write_uoffset(&buffer_size, (uoffset_t)B->buffer_mark - buffer_base); + } else { + /* Also include clustered vtables. */ + write_uoffset(&buffer_size, (uoffset_t)B->emit_end - buffer_base); + } + write_uoffset(&object_offset, (uoffset_t)object_ref - buffer_base); + if (0 == (buffer_ref = emit_front(B, &iov))) { + check(0, "emitter rejected buffer content"); + return 0; + } + return buffer_ref; +} + +flatcc_builder_ref_t flatcc_builder_create_struct(flatcc_builder_t *B, const void *data, size_t size, uint16_t align) +{ + size_t pad; + iov_state_t iov; + + check(align >= 1, "align cannot be 0"); + set_min_align(B, align); + pad = front_pad(B, (uoffset_t)size, align); + init_iov(); + push_iov(data, size); + /* + * Normally structs will already be a multiple of their alignment, + * so this padding will not likely be emitted. + */ + push_iov(_pad, pad); + return emit_front(B, &iov); +} + +int flatcc_builder_start_buffer(flatcc_builder_t *B, + const char identifier[identifier_size], uint16_t block_align, int flags) +{ + /* + * This saves the parent `min_align` in the align field since we + * shouldn't use that for the current buffer. `exit_frame` + * automatically aggregates align up, so it is updated when the + * buffer frame exits. + */ + if (enter_frame(B, B->min_align)) { + return -1; + } + /* B->align now has parent min_align, and child frames will save it. */ + B->min_align = 1; + /* Save the parent block align, and set proper defaults for this buffer. */ + frame(container.buffer.block_align) = B->block_align; + B->block_align = block_align; + frame(container.buffer.flags = B->buffer_flags); + B->buffer_flags = (uint16_t)flags; + frame(container.buffer.mark) = B->buffer_mark; + frame(container.buffer.nest_id) = B->nest_id; + /* + * End of buffer when nested. Not defined for top-level because we + * here (on only here) permit strings etc. to be created before buffer start and + * because top-level buffer vtables can be clustered. + */ + B->buffer_mark = B->emit_start; + /* Must be 0 before and after entering top-level buffer, and unique otherwise. */ + B->nest_id = B->nest_count++; + frame(container.buffer.identifier) = B->identifier; + set_identifier(identifier); + frame(type) = flatcc_builder_buffer; + return 0; +} + +flatcc_builder_ref_t flatcc_builder_end_buffer(flatcc_builder_t *B, flatcc_builder_ref_t root) +{ + flatcc_builder_ref_t buffer_ref; + int flags; + + flags = B->buffer_flags & flatcc_builder_with_size; + flags |= is_top_buffer(B) ? 0 : flatcc_builder_is_nested; + check(frame(type) == flatcc_builder_buffer, "expected buffer frame"); + set_min_align(B, B->block_align); + if (0 == (buffer_ref = flatcc_builder_create_buffer(B, (void *)&B->identifier, + B->block_align, root, B->min_align, flags))) { + return 0; + } + B->buffer_mark = frame(container.buffer.mark); + B->nest_id = frame(container.buffer.nest_id); + B->identifier = frame(container.buffer.identifier); + B->buffer_flags = frame(container.buffer.flags); + exit_frame(B); + return buffer_ref; +} + +void *flatcc_builder_start_struct(flatcc_builder_t *B, size_t size, uint16_t align) +{ + /* Allocate space for the struct on the ds stack. */ + if (enter_frame(B, align)) { + return 0; + } + frame(type) = flatcc_builder_struct; + refresh_ds(B, data_limit); + return push_ds(B, (uoffset_t)size); +} + +void *flatcc_builder_struct_edit(flatcc_builder_t *B) +{ + return B->ds; +} + +flatcc_builder_ref_t flatcc_builder_end_struct(flatcc_builder_t *B) +{ + flatcc_builder_ref_t object_ref; + + check(frame(type) == flatcc_builder_struct, "expected struct frame"); + if (0 == (object_ref = flatcc_builder_create_struct(B, B->ds, B->ds_offset, B->align))) { + return 0; + } + exit_frame(B); + return object_ref; +} + +static inline int vector_count_add(flatcc_builder_t *B, uoffset_t count, uoffset_t max_count) +{ + uoffset_t n, n1; + n = frame(container.vector.count); + n1 = n + count; + /* + * This prevents elem_size * count from overflowing iff max_vector + * has been set sensible. Without this check we might allocate to + * little on the ds stack and return a buffer the user thinks is + * much larger which of course is bad even though the buffer eventually + * would fail anyway. + */ + check_error(n <= n1 && n1 <= max_count, -1, "vector too large to represent"); + frame(container.vector.count) = n1; + return 0; +} + +void *flatcc_builder_extend_vector(flatcc_builder_t *B, size_t count) +{ + if (vector_count_add(B, (uoffset_t)count, frame(container.vector.max_count))) { + return 0; + } + return push_ds(B, frame(container.vector.elem_size) * (uoffset_t)count); +} + +void *flatcc_builder_vector_push(flatcc_builder_t *B, const void *data) +{ + check(frame(type) == flatcc_builder_vector, "expected vector frame"); + check_error(frame(container.vector.count) <= frame(container.vector.max_count), 0, "vector max count exceeded"); + frame(container.vector.count) += 1; + return push_ds_copy(B, data, frame(container.vector.elem_size)); +} + +void *flatcc_builder_append_vector(flatcc_builder_t *B, const void *data, size_t count) +{ + check(frame(type) == flatcc_builder_vector, "expected vector frame"); + if (vector_count_add(B, (uoffset_t)count, frame(container.vector.max_count))) { + return 0; + } + return push_ds_copy(B, data, frame(container.vector.elem_size) * (uoffset_t)count); +} + +flatcc_builder_ref_t *flatcc_builder_extend_offset_vector(flatcc_builder_t *B, size_t count) +{ + if (vector_count_add(B, (uoffset_t)count, max_offset_count)) { + return 0; + } + return push_ds(B, (uoffset_t)(field_size * count)); +} + +flatcc_builder_ref_t *flatcc_builder_offset_vector_push(flatcc_builder_t *B, flatcc_builder_ref_t ref) +{ + flatcc_builder_ref_t *p; + + check(frame(type) == flatcc_builder_offset_vector, "expected offset vector frame"); + if (frame(container.vector.count) == max_offset_count) { + return 0; + } + frame(container.vector.count) += 1; + if (0 == (p = push_ds(B, field_size))) { + return 0; + } + *p = ref; + return p; +} + +flatcc_builder_ref_t *flatcc_builder_append_offset_vector(flatcc_builder_t *B, const flatcc_builder_ref_t *refs, size_t count) +{ + check(frame(type) == flatcc_builder_offset_vector, "expected offset vector frame"); + if (vector_count_add(B, (uoffset_t)count, max_offset_count)) { + return 0; + } + return push_ds_copy(B, refs, (uoffset_t)(field_size * count)); +} + +char *flatcc_builder_extend_string(flatcc_builder_t *B, size_t len) +{ + check(frame(type) == flatcc_builder_string, "expected string frame"); + if (vector_count_add(B, (uoffset_t)len, max_string_len)) { + return 0; + } + return push_ds(B, (uoffset_t)len); +} + +char *flatcc_builder_append_string(flatcc_builder_t *B, const char *s, size_t len) +{ + check(frame(type) == flatcc_builder_string, "expected string frame"); + if (vector_count_add(B, (uoffset_t)len, max_string_len)) { + return 0; + } + return push_ds_copy(B, s, (uoffset_t)len); +} + +char *flatcc_builder_append_string_str(flatcc_builder_t *B, const char *s) +{ + return flatcc_builder_append_string(B, s, strlen(s)); +} + +char *flatcc_builder_append_string_strn(flatcc_builder_t *B, const char *s, size_t max_len) +{ + return flatcc_builder_append_string(B, s, strnlen(s, max_len)); +} + +int flatcc_builder_truncate_vector(flatcc_builder_t *B, size_t count) +{ + check(frame(type) == flatcc_builder_vector, "expected vector frame"); + check_error(frame(container.vector.count) >= count, -1, "cannot truncate vector past empty"); + frame(container.vector.count) -= (uoffset_t)count; + unpush_ds(B, frame(container.vector.elem_size) * (uoffset_t)count); + return 0; +} + +int flatcc_builder_truncate_offset_vector(flatcc_builder_t *B, size_t count) +{ + check(frame(type) == flatcc_builder_offset_vector, "expected offset vector frame"); + check_error(frame(container.vector.count) >= (uoffset_t)count, -1, "cannot truncate vector past empty"); + frame(container.vector.count) -= (uoffset_t)count; + unpush_ds(B, frame(container.vector.elem_size) * (uoffset_t)count); + return 0; +} + +int flatcc_builder_truncate_string(flatcc_builder_t *B, size_t len) +{ + check(frame(type) == flatcc_builder_string, "expected string frame"); + check_error(frame(container.vector.count) >= len, -1, "cannot truncate string past empty"); + frame(container.vector.count) -= (uoffset_t)len; + unpush_ds(B, (uoffset_t)len); + return 0; +} + +int flatcc_builder_start_vector(flatcc_builder_t *B, size_t elem_size, uint16_t align, size_t max_count) +{ + get_min_align(&align, field_size); + if (enter_frame(B, align)) { + return -1; + } + frame(container.vector.elem_size) = (uoffset_t)elem_size; + frame(container.vector.count) = 0; + frame(container.vector.max_count) = (uoffset_t)max_count; + frame(type) = flatcc_builder_vector; + refresh_ds(B, data_limit); + return 0; +} + +int flatcc_builder_start_offset_vector(flatcc_builder_t *B) +{ + if (enter_frame(B, field_size)) { + return -1; + } + frame(container.vector.elem_size) = field_size; + frame(container.vector.count) = 0; + frame(type) = flatcc_builder_offset_vector; + refresh_ds(B, data_limit); + return 0; +} + +flatcc_builder_ref_t flatcc_builder_create_offset_vector(flatcc_builder_t *B, + const flatcc_builder_ref_t *vec, size_t count) +{ + flatcc_builder_ref_t *_vec; + + if (flatcc_builder_start_offset_vector(B)) { + return 0; + } + if (!(_vec = flatcc_builder_extend_offset_vector(B, count))) { + return 0; + } + memcpy(_vec, vec, count * field_size); + return flatcc_builder_end_offset_vector(B); +} + +int flatcc_builder_start_string(flatcc_builder_t *B) +{ + if (enter_frame(B, 1)) { + return -1; + } + frame(container.vector.elem_size) = 1; + frame(container.vector.count) = 0; + frame(type) = flatcc_builder_string; + refresh_ds(B, data_limit); + return 0; +} + +int flatcc_builder_reserve_table(flatcc_builder_t *B, int count) +{ + check(count >= 0, "cannot reserve negative count"); + return reserve_fields(B, count); +} + +int flatcc_builder_start_table(flatcc_builder_t *B, int count) +{ + if (enter_frame(B, field_size)) { + return -1; + } + frame(container.table.vs_end) = vs_offset(B->vs); + frame(container.table.pl_end) = pl_offset(B->pl); + frame(container.table.vt_hash) = B->vt_hash; + frame(container.table.id_end) = B->id_end; + B->vt_hash = 0; + FLATCC_BUILDER_INIT_VT_HASH(B->vt_hash); + B->id_end = 0; + frame(type) = flatcc_builder_table; + if (reserve_fields(B, count)) { + return -1; + } + refresh_ds(B, table_limit); + return 0; +} + +flatcc_builder_vt_ref_t flatcc_builder_create_vtable(flatcc_builder_t *B, + const voffset_t *vt, voffset_t vt_size) +{ + flatcc_builder_vt_ref_t vt_ref; + iov_state_t iov; + voffset_t *vt_; + size_t i; + + /* + * Only top-level buffer can cluster vtables because only it can + * extend beyond the end. + * + * We write the vtable after the referencing table to maintain + * the construction invariant that any offset reference has + * valid emitted data at a higher address, and also that any + * issued negative emit address represents an offset reference + * to some flatbuffer object or vector (or possibly a root + * struct). + * + * The vt_ref is stored as the reference + 1 to avoid having 0 as a + * valid reference (which usally means error). It also idententifies + * vtable references as the only uneven references, and the only + * references that can be used multiple times in the same buffer. + * + * We do the vtable conversion here so cached vtables can be built + * hashed and compared more efficiently, and so end users with + * direct vtable construction don't have to worry about endianness. + * This also ensures the hash function works the same wrt. + * collision frequency. + */ + + if (!flatbuffers_is_native_pe()) { + /* Make space in vtable cache for temporary endian conversion. */ + if (!(vt_ = reserve_buffer(B, flatcc_builder_alloc_vb, B->vb_end, vt_size, 0))) { + return 0; + } + for (i = 0; i < vt_size / sizeof(voffset_t); ++i) { + write_voffset(&vt_[i], vt[i]); + } + vt = vt_; + /* We don't need to free the reservation since we don't advance any base pointer. */ + } + + init_iov(); + push_iov(vt, vt_size); + if (is_top_buffer(B) && !B->disable_vt_clustering) { + /* Note that `emit_back` already returns ref + 1 as we require for vtables. */ + if (0 == (vt_ref = emit_back(B, &iov))) { + return 0; + } + } else { + if (0 == (vt_ref = emit_front(B, &iov))) { + return 0; + } + /* + * We don't have a valid 0 ref here, but to be consistent with + * clustered vtables we offset by one. This cannot be zero + * either. + */ + vt_ref += 1; + } + return vt_ref; +} + +flatcc_builder_vt_ref_t flatcc_builder_create_cached_vtable(flatcc_builder_t *B, + const voffset_t *vt, voffset_t vt_size, uint32_t vt_hash) +{ + vtable_descriptor_t *vd, *vd2; + uoffset_t *pvd, *pvd_head; + uoffset_t next; + voffset_t *vt_; + + /* This just gets the hash table slot, we still have to inspect it. */ + if (!(pvd_head = lookup_ht(B, vt_hash))) { + return 0; + } + pvd = pvd_head; + next = *pvd; + /* Tracks if there already is a cached copy. */ + vd2 = 0; + while (next) { + vd = vd_ptr(next); + vt_ = vb_ptr(vd->vb_start); + if (vt_[0] != vt_size || 0 != memcmp(vt, vt_, vt_size)) { + pvd = &vd->next; + next = vd->next; + continue; + } + /* Can't share emitted vtables between buffers, */ + if (vd->nest_id != B->nest_id) { + /* but we don't have to resubmit to cache. */ + vd2 = vd; + /* See if there is a better match. */ + pvd = &vd->next; + next = vd->next; + continue; + } + /* Move to front hash strategy. */ + if (pvd != pvd_head) { + *pvd = vd->next; + vd->next = *pvd_head; + *pvd_head = next; + } + /* vtable exists and has been emitted within current buffer. */ + return vd->vt_ref; + } + /* Allocate new descriptor. */ + if (!(vd = reserve_buffer(B, flatcc_builder_alloc_vd, B->vd_end, sizeof(vtable_descriptor_t), 0))) { + return 0; + } + next = B->vd_end; + B->vd_end += (uoffset_t)sizeof(vtable_descriptor_t); + + /* Identify the buffer this vtable descriptor belongs to. */ + vd->nest_id = B->nest_id; + + /* Move to front hash strategy. */ + vd->next = *pvd_head; + *pvd_head = next; + if (0 == (vd->vt_ref = flatcc_builder_create_vtable(B, vt, vt_size))) { + return 0; + } + if (vd2) { + /* Reuse cached copy. */ + vd->vb_start = vd2->vb_start; + } else { + if (B->vb_flush_limit && B->vb_flush_limit < B->vb_end + vt_size) { + flatcc_builder_flush_vtable_cache(B); + } else { + /* Make space in vtable cache. */ + if (!(vt_ = reserve_buffer(B, flatcc_builder_alloc_vb, B->vb_end, vt_size, 0))) { + return -1; + } + vd->vb_start = B->vb_end; + B->vb_end += vt_size; + memcpy(vt_, vt, vt_size); + } + } + return vd->vt_ref; +} + +flatcc_builder_ref_t flatcc_builder_create_table(flatcc_builder_t *B, const void *data, size_t size, uint16_t align, + flatbuffers_voffset_t *offsets, int offset_count, flatcc_builder_vt_ref_t vt_ref) +{ + int i; + uoffset_t pad, vt_offset, vt_offset_field, vt_base, base, offset, *offset_field; + iov_state_t iov; + + check(offset_count >= 0, "expected non-negative offset_count"); + /* + * vtable references are offset by 1 to avoid confusion with + * 0 as an error reference. It also uniquely identifies them + * as vtables being the only uneven reference type. + */ + check(vt_ref & 1, "invalid vtable referenc"); + get_min_align(&align, field_size); + set_min_align(B, align); + /* Alignment is calculated for the first element, not the header. */ + pad = front_pad(B, (uoffset_t)size, align); + base = (uoffset_t)B->emit_start - (uoffset_t)(pad + size + field_size); + /* Adjust by 1 to get unencoded vtable reference. */ + vt_base = (uoffset_t)(vt_ref - 1); + vt_offset = base - vt_base; + /* Avoid overflow. */ + if (base - vt_offset != vt_base) { + return -1; + } + /* Protocol endian encoding. */ + write_uoffset(&vt_offset_field, vt_offset); + for (i = 0; i < offset_count; ++i) { + offset_field = (uoffset_t *)((size_t)data + offsets[i]); + offset = *offset_field - base - offsets[i] - (uoffset_t)field_size; + write_uoffset(offset_field, offset); + } + init_iov(); + push_iov(&vt_offset_field, field_size); + push_iov(data, size); + push_iov(_pad, pad); + return emit_front(B, &iov); +} + +int flatcc_builder_check_required_field(flatcc_builder_t *B, flatbuffers_voffset_t id) +{ + check(frame(type) == flatcc_builder_table, "expected table frame"); + + return id < B->id_end && B->vs[id] != 0; +} + +int flatcc_builder_check_union_field(flatcc_builder_t *B, flatbuffers_voffset_t id) +{ + check(frame(type) == flatcc_builder_table, "expected table frame"); + + if (id == 0 || id >= B->id_end) { + return 0; + } + if (B->vs[id - 1] == 0) { + return B->vs[id] == 0; + } + if (*(uint8_t *)(B->ds + B->vs[id - 1])) { + return B->vs[id] != 0; + } + return B->vs[id] == 0; +} + +int flatcc_builder_check_required(flatcc_builder_t *B, const flatbuffers_voffset_t *required, int count) +{ + int i; + + check(frame(type) == flatcc_builder_table, "expected table frame"); + + if (B->id_end < count) { + return 0; + } + for (i = 0; i < count; ++i) { + if (B->vs[required[i]] == 0) { + return 0; + } + } + return 1; +} + +flatcc_builder_ref_t flatcc_builder_end_table(flatcc_builder_t *B) +{ + voffset_t *vt, vt_size; + flatcc_builder_ref_t table_ref, vt_ref; + int pl_count; + voffset_t *pl; + + check(frame(type) == flatcc_builder_table, "expected table frame"); + + /* We have `ds_limit`, so we should not have to check for overflow here. */ + + vt = B->vs - 2; + vt_size = (voffset_t)(sizeof(voffset_t) * (B->id_end + 2u)); + /* Update vtable header fields, first vtable size, then object table size. */ + vt[0] = vt_size; + /* + * The `ds` buffer is always at least `field_size` aligned but excludes the + * initial vtable offset field. Therefore `field_size` is added here + * to the total table size in the vtable. + */ + vt[1] = (voffset_t)(B->ds_offset + field_size); + FLATCC_BUILDER_UPDATE_VT_HASH(B->vt_hash, (uint32_t)vt[0], (uint32_t)vt[1]); + /* Find already emitted vtable, or emit a new one. */ + if (!(vt_ref = flatcc_builder_create_cached_vtable(B, vt, vt_size, B->vt_hash))) { + return 0; + } + /* Clear vs stack so it is ready for the next vtable (ds stack is cleared by exit frame). */ + memset(vt, 0, vt_size); + + pl = pl_ptr(frame(container.table.pl_end)); + pl_count = (int)(B->pl - pl); + if (0 == (table_ref = flatcc_builder_create_table(B, B->ds, B->ds_offset, B->align, pl, pl_count, vt_ref))) { + return 0; + } + B->vt_hash = frame(container.table.vt_hash); + B->id_end = frame(container.table.id_end); + B->vs = vs_ptr(frame(container.table.vs_end)); + B->pl = pl_ptr(frame(container.table.pl_end)); + exit_frame(B); + return table_ref; +} + +flatcc_builder_ref_t flatcc_builder_create_vector(flatcc_builder_t *B, + const void *data, size_t count, size_t elem_size, uint16_t align, size_t max_count) +{ + /* + * Note: it is important that vec_size is uoffset not size_t + * in case sizeof(uoffset_t) > sizeof(size_t) because max_count is + * defined in terms of uoffset_t representation size, and also + * because we risk accepting too large a vector even if max_count is + * not violated. + */ + uoffset_t vec_size, vec_pad, length_prefix; + iov_state_t iov; + + check_error(count <= max_count, 0, "vector max_count violated"); + get_min_align(&align, field_size); + set_min_align(B, align); + vec_size = (uoffset_t)count * (uoffset_t)elem_size; + /* + * That can happen on 32 bit systems when uoffset_t is defined as 64-bit. + * `emit_front/back` captures overflow, but not if our size type wraps first. + */ +#if FLATBUFFERS_UOFFSET_MAX > SIZE_MAX + check_error(vec_size < SIZE_MAX, 0, "vector larger than address space"); +#endif + write_uoffset(&length_prefix, (uoffset_t)count); + /* Alignment is calculated for the first element, not the header. */ + vec_pad = front_pad(B, vec_size, align); + init_iov(); + push_iov(&length_prefix, field_size); + push_iov(data, vec_size); + push_iov(_pad, vec_pad); + return emit_front(B, &iov); +} + +/* + * Note: FlatBuffers official documentation states that the size field of a + * vector is a 32-bit element count. It is not quite clear if the + * intention is to have the size field be of type uoffset_t since tables + * also have a uoffset_t sized header, or if the vector size should + * remain unchanged if uoffset is changed to 16- or 64-bits + * respectively. Since it makes most sense to have a vector compatible + * with the addressable space, we choose to use uoffset_t as size field, + * which remains compatible with the default 32-bit version of uoffset_t. + */ +flatcc_builder_ref_t flatcc_builder_end_vector(flatcc_builder_t *B) +{ + flatcc_builder_ref_t vector_ref; + + check(frame(type) == flatcc_builder_vector, "expected vector frame"); + + if (0 == (vector_ref = flatcc_builder_create_vector(B, B->ds, + frame(container.vector.count), frame(container.vector.elem_size), + B->align, frame(container.vector.max_count)))) { + return 0; + } + exit_frame(B); + return vector_ref; +} + +size_t flatcc_builder_vector_count(flatcc_builder_t *B) +{ + return frame(container.vector.count); +} + +void *flatcc_builder_vector_edit(flatcc_builder_t *B) +{ + return B->ds; +} + +/* This function destroys the source content but avoids stack allocation. */ +static flatcc_builder_ref_t _create_offset_vector_direct(flatcc_builder_t *B, + flatcc_builder_ref_t *vec, size_t count, const utype_t *types) +{ + uoffset_t vec_size, vec_pad; + uoffset_t length_prefix, offset; + uoffset_t i; + soffset_t base; + iov_state_t iov; + + if ((uoffset_t)count > max_offset_count) { + return 0; + } + set_min_align(B, field_size); + vec_size = (uoffset_t)(count * field_size); + write_uoffset(&length_prefix, (uoffset_t)count); + /* Alignment is calculated for the first element, not the header. */ + vec_pad = front_pad(B, vec_size, field_size); + init_iov(); + push_iov(&length_prefix, field_size); + push_iov(vec, vec_size); + push_iov(_pad, vec_pad); + base = B->emit_start - (soffset_t)iov.len; + for (i = 0; i < (uoffset_t)count; ++i) { + /* + * 0 is either end of buffer, start of vtables, or start of + * buffer depending on the direction in which the buffer is + * built. None of these can create a valid 0 reference but it + * is easy to create by mistake when manually building offset + * vectors. + * + * Unions do permit nulls, but only when the type is NONE. + */ + if (vec[i] != 0) { + offset = (uoffset_t) + (vec[i] - base - (soffset_t)(i * field_size) - (soffset_t)field_size); + write_uoffset(&vec[i], offset); + if (types) { + check(types[i] != 0, "union vector cannot have non-null element with type NONE"); + } + } else { + if (types) { + check(types[i] == 0, "union vector cannot have null element without type NONE"); + } else { + check(0, "offset vector cannot have null element"); + } + } + } + return emit_front(B, &iov); +} + +flatcc_builder_ref_t flatcc_builder_create_offset_vector_direct(flatcc_builder_t *B, + flatcc_builder_ref_t *vec, size_t count) +{ + return _create_offset_vector_direct(B, vec, count, 0); +} + +flatcc_builder_ref_t flatcc_builder_end_offset_vector(flatcc_builder_t *B) +{ + flatcc_builder_ref_t vector_ref; + + check(frame(type) == flatcc_builder_offset_vector, "expected offset vector frame"); + if (0 == (vector_ref = flatcc_builder_create_offset_vector_direct(B, + (flatcc_builder_ref_t *)B->ds, frame(container.vector.count)))) { + return 0; + } + exit_frame(B); + return vector_ref; +} + +flatcc_builder_ref_t flatcc_builder_end_offset_vector_for_unions(flatcc_builder_t *B, const utype_t *types) +{ + flatcc_builder_ref_t vector_ref; + + check(frame(type) == flatcc_builder_offset_vector, "expected offset vector frame"); + if (0 == (vector_ref = _create_offset_vector_direct(B, + (flatcc_builder_ref_t *)B->ds, frame(container.vector.count), types))) { + return 0; + } + exit_frame(B); + return vector_ref; +} + +void *flatcc_builder_offset_vector_edit(flatcc_builder_t *B) +{ + return B->ds; +} + +size_t flatcc_builder_offset_vector_count(flatcc_builder_t *B) +{ + return frame(container.vector.count); +} + +int flatcc_builder_table_add_union(flatcc_builder_t *B, int id, + flatcc_builder_union_ref_t uref) +{ + flatcc_builder_ref_t *pref; + flatcc_builder_utype_t *putype; + + check(frame(type) == flatcc_builder_table, "expected table frame"); + check_error(uref.type != 0 || uref.value == 0, -1, "expected null value for type NONE"); + if (uref.value != 0) { + pref = flatcc_builder_table_add_offset(B, id); + check_error(pref != 0, -1, "unable to add union value"); + *pref = uref.value; + } + putype = flatcc_builder_table_add(B, id - 1, utype_size, utype_size); + check_error(putype != 0, -1, "unable to add union type"); + write_utype(putype, uref.type); + return 0; +} + +int flatcc_builder_table_add_union_vector(flatcc_builder_t *B, int id, + flatcc_builder_union_vec_ref_t uvref) +{ + flatcc_builder_ref_t *pref; + + check(frame(type) == flatcc_builder_table, "expected table frame"); + check_error((uvref.type == 0) == (uvref.value == 0), -1, "expected both type and value vector, or neither"); + if (uvref.type != 0) { + pref = flatcc_builder_table_add_offset(B, id - 1); + check_error(pref != 0, -1, "unable to add union member"); + *pref = uvref.type; + + pref = flatcc_builder_table_add_offset(B, id); + check_error(pref != 0, -1, "unable to add union member"); + *pref = uvref.value; + } + return 0; +} + +flatcc_builder_union_vec_ref_t flatcc_builder_create_union_vector(flatcc_builder_t *B, + const flatcc_builder_union_ref_t *urefs, size_t count) +{ + flatcc_builder_union_vec_ref_t uvref = { 0, 0 }; + flatcc_builder_utype_t *types; + flatcc_builder_ref_t *refs; + size_t i; + + if (flatcc_builder_start_offset_vector(B)) { + return uvref; + } + if (0 == flatcc_builder_extend_offset_vector(B, count)) { + return uvref; + } + if (0 == (types = push_ds(B, (uoffset_t)(utype_size * count)))) { + return uvref; + } + + /* Safe even if push_ds caused stack reallocation. */ + refs = flatcc_builder_offset_vector_edit(B); + + for (i = 0; i < count; ++i) { + types[i] = urefs[i].type; + refs[i] = urefs[i].value; + } + uvref = flatcc_builder_create_union_vector_direct(B, + types, refs, count); + /* No need to clean up after out temporary types vector. */ + exit_frame(B); + return uvref; +} + +flatcc_builder_union_vec_ref_t flatcc_builder_create_union_vector_direct(flatcc_builder_t *B, + const flatcc_builder_utype_t *types, flatcc_builder_ref_t *data, size_t count) +{ + flatcc_builder_union_vec_ref_t uvref = { 0, 0 }; + + if (0 == (uvref.value = _create_offset_vector_direct(B, data, count, types))) { + return uvref; + } + if (0 == (uvref.type = flatcc_builder_create_type_vector(B, types, count))) { + return uvref; + } + return uvref; +} + +flatcc_builder_ref_t flatcc_builder_create_type_vector(flatcc_builder_t *B, + const flatcc_builder_utype_t *types, size_t count) +{ + return flatcc_builder_create_vector(B, types, count, + utype_size, utype_size, max_utype_count); +} + +int flatcc_builder_start_union_vector(flatcc_builder_t *B) +{ + if (enter_frame(B, field_size)) { + return -1; + } + frame(container.vector.elem_size) = union_size; + frame(container.vector.count) = 0; + frame(type) = flatcc_builder_union_vector; + refresh_ds(B, data_limit); + return 0; +} + +flatcc_builder_union_vec_ref_t flatcc_builder_end_union_vector(flatcc_builder_t *B) +{ + flatcc_builder_union_vec_ref_t uvref = { 0, 0 }; + flatcc_builder_utype_t *types; + flatcc_builder_union_ref_t *urefs; + flatcc_builder_ref_t *refs; + size_t i, count; + + check(frame(type) == flatcc_builder_union_vector, "expected union vector frame"); + + /* + * We could split the union vector in-place, but then we would have + * to deal with strict pointer aliasing rules which is not worthwhile + * so we create a new offset and type vector on the stack. + * + * We assume the stack is sufficiently aligned as is. + */ + count = flatcc_builder_union_vector_count(B); + if (0 == (refs = push_ds(B, (uoffset_t)(count * (utype_size + field_size))))) { + return uvref; + } + types = (flatcc_builder_utype_t *)(refs + count); + + /* Safe even if push_ds caused stack reallocation. */ + urefs = flatcc_builder_union_vector_edit(B); + + for (i = 0; i < count; ++i) { + types[i] = urefs[i].type; + refs[i] = urefs[i].value; + } + uvref = flatcc_builder_create_union_vector_direct(B, types, refs, count); + /* No need to clean up after out temporary types vector. */ + exit_frame(B); + return uvref; +} + +void *flatcc_builder_union_vector_edit(flatcc_builder_t *B) +{ + return B->ds; +} + +size_t flatcc_builder_union_vector_count(flatcc_builder_t *B) +{ + return frame(container.vector.count); +} + +flatcc_builder_union_ref_t *flatcc_builder_extend_union_vector(flatcc_builder_t *B, size_t count) +{ + if (vector_count_add(B, (uoffset_t)count, max_union_count)) { + return 0; + } + return push_ds(B, (uoffset_t)(union_size * count)); +} + +int flatcc_builder_truncate_union_vector(flatcc_builder_t *B, size_t count) +{ + check(frame(type) == flatcc_builder_union_vector, "expected union vector frame"); + check_error(frame(container.vector.count) >= (uoffset_t)count, -1, "cannot truncate vector past empty"); + frame(container.vector.count) -= (uoffset_t)count; + unpush_ds(B, frame(container.vector.elem_size) * (uoffset_t)count); + return 0; +} + +flatcc_builder_union_ref_t *flatcc_builder_union_vector_push(flatcc_builder_t *B, + flatcc_builder_union_ref_t uref) +{ + flatcc_builder_union_ref_t *p; + + check(frame(type) == flatcc_builder_union_vector, "expected union vector frame"); + if (frame(container.vector.count) == max_union_count) { + return 0; + } + frame(container.vector.count) += 1; + if (0 == (p = push_ds(B, union_size))) { + return 0; + } + *p = uref; + return p; +} + +flatcc_builder_union_ref_t *flatcc_builder_append_union_vector(flatcc_builder_t *B, + const flatcc_builder_union_ref_t *urefs, size_t count) +{ + check(frame(type) == flatcc_builder_union_vector, "expected union vector frame"); + if (vector_count_add(B, (uoffset_t)count, max_union_count)) { + return 0; + } + return push_ds_copy(B, urefs, (uoffset_t)(union_size * count)); +} + +flatcc_builder_ref_t flatcc_builder_create_string(flatcc_builder_t *B, const char *s, size_t len) +{ + uoffset_t s_pad; + uoffset_t length_prefix; + iov_state_t iov; + + if (len > max_string_len) { + return 0; + } + write_uoffset(&length_prefix, (uoffset_t)len); + /* Add 1 for zero termination. */ + s_pad = front_pad(B, (uoffset_t)len + 1, field_size) + 1; + init_iov(); + push_iov(&length_prefix, field_size); + push_iov(s, len); + push_iov(_pad, s_pad); + return emit_front(B, &iov); +} + +flatcc_builder_ref_t flatcc_builder_create_string_str(flatcc_builder_t *B, const char *s) +{ + return flatcc_builder_create_string(B, s, strlen(s)); +} + +flatcc_builder_ref_t flatcc_builder_create_string_strn(flatcc_builder_t *B, const char *s, size_t max_len) +{ + return flatcc_builder_create_string(B, s, strnlen(s, max_len)); +} + +flatcc_builder_ref_t flatcc_builder_end_string(flatcc_builder_t *B) +{ + flatcc_builder_ref_t string_ref; + + check(frame(type) == flatcc_builder_string, "expected string frame"); + FLATCC_ASSERT(frame(container.vector.count) == B->ds_offset); + if (0 == (string_ref = flatcc_builder_create_string(B, + (const char *)B->ds, B->ds_offset))) { + return 0; + } + exit_frame(B); + return string_ref; +} + +char *flatcc_builder_string_edit(flatcc_builder_t *B) +{ + return (char *)B->ds; +} + +size_t flatcc_builder_string_len(flatcc_builder_t *B) +{ + return frame(container.vector.count); +} + +void *flatcc_builder_table_add(flatcc_builder_t *B, int id, size_t size, uint16_t align) +{ + /* + * We align the offset relative to the first table field, excluding + * the header holding the vtable reference. On the stack, `ds_first` + * is aligned to 8 bytes thanks to the `enter_frame` logic, and this + * provides a safe way to update the fields on the stack, but here + * we are concerned with the target buffer alignment. + * + * We could also have aligned relative to the end of the table which + * would allow us to emit each field immediately, but it would be a + * confusing user experience wrt. field ordering, and it would add + * more variability to vtable layouts, thus reducing reuse, and + * frequent emissions to external emitter interface would be + * sub-optimal. Also, with that appoach, the vtable offsets would + * have to be adjusted at table end. + * + * As we have it, each emit occur at table end, vector end, string + * end, or buffer end, which might be helpful to various backend + * processors. + */ + check(frame(type) == flatcc_builder_table, "expected table frame"); + check(id >= 0 && id <= (int)FLATBUFFERS_ID_MAX, "table id out of range"); + if (align > B->align) { + B->align = align; + } +#if FLATCC_BUILDER_ALLOW_REPEAT_TABLE_ADD + if (B->vs[id] != 0) { + return B->ds + B->vs[id] - field_size; + } +#else + if (B->vs[id] != 0) { + check(0, "table field already set"); + return 0; + } +#endif + FLATCC_BUILDER_UPDATE_VT_HASH(B->vt_hash, (uint32_t)id, (uint32_t)size); + return push_ds_field(B, (uoffset_t)size, align, (voffset_t)id); +} + +void *flatcc_builder_table_edit(flatcc_builder_t *B, size_t size) +{ + check(frame(type) == flatcc_builder_table, "expected table frame"); + + return B->ds + B->ds_offset - size; +} + +void *flatcc_builder_table_add_copy(flatcc_builder_t *B, int id, const void *data, size_t size, uint16_t align) +{ + void *p; + + if ((p = flatcc_builder_table_add(B, id, size, align))) { + memcpy(p, data, size); + } + return p; +} + +flatcc_builder_ref_t *flatcc_builder_table_add_offset(flatcc_builder_t *B, int id) +{ + check(frame(type) == flatcc_builder_table, "expected table frame"); + check(id >= 0 && id <= (int)FLATBUFFERS_ID_MAX, "table id out of range"); +#if FLATCC_BUILDER_ALLOW_REPEAT_TABLE_ADD + if (B->vs[id] != 0) { + return B->ds + B->vs[id] - field_size; + } +#else + if (B->vs[id] != 0) { + check(0, "table field already set"); + return 0; + } +#endif + FLATCC_BUILDER_UPDATE_VT_HASH(B->vt_hash, (uint32_t)id, (uint32_t)field_size); + return push_ds_offset_field(B, (voffset_t)id); +} + +uint16_t flatcc_builder_push_buffer_alignment(flatcc_builder_t *B) +{ + uint16_t old_min_align = B->min_align; + + B->min_align = field_size; + return old_min_align; +} + +void flatcc_builder_pop_buffer_alignment(flatcc_builder_t *B, uint16_t pushed_align) +{ + set_min_align(B, pushed_align); +} + +uint16_t flatcc_builder_get_buffer_alignment(flatcc_builder_t *B) +{ + return B->min_align; +} + +void flatcc_builder_set_vtable_clustering(flatcc_builder_t *B, int enable) +{ + /* Inverted because we zero all memory in B on init. */ + B->disable_vt_clustering = !enable; +} + +void flatcc_builder_set_block_align(flatcc_builder_t *B, uint16_t align) +{ + B->block_align = align; +} + +int flatcc_builder_get_level(flatcc_builder_t *B) +{ + return B->level; +} + +void flatcc_builder_set_max_level(flatcc_builder_t *B, int max_level) +{ + B->max_level = max_level; + if (B->limit_level < B->max_level) { + B->limit_level = B->max_level; + } +} + +size_t flatcc_builder_get_buffer_size(flatcc_builder_t *B) +{ + return (size_t)(B->emit_end - B->emit_start); +} + +flatcc_builder_ref_t flatcc_builder_get_buffer_start(flatcc_builder_t *B) +{ + return B->emit_start; +} + +flatcc_builder_ref_t flatcc_builder_get_buffer_end(flatcc_builder_t *B) +{ + return B->emit_end; +} + +void flatcc_builder_set_vtable_cache_limit(flatcc_builder_t *B, size_t size) +{ + B->vb_flush_limit = size; +} + +void flatcc_builder_set_identifier(flatcc_builder_t *B, const char identifier[identifier_size]) +{ + set_identifier(identifier); +} + +enum flatcc_builder_type flatcc_builder_get_type(flatcc_builder_t *B) +{ + return B->frame ? frame(type) : flatcc_builder_empty; +} + +enum flatcc_builder_type flatcc_builder_get_type_at(flatcc_builder_t *B, int level) +{ + if (level < 1 || level > B->level) { + return flatcc_builder_empty; + } + return B->frame[level - B->level].type; +} + +void *flatcc_builder_get_direct_buffer(flatcc_builder_t *B, size_t *size_out) +{ + if (B->is_default_emitter) { + return flatcc_emitter_get_direct_buffer(&B->default_emit_context, size_out); + } else { + if (size_out) { + *size_out = 0; + } + } + return 0; +} + +void *flatcc_builder_copy_buffer(flatcc_builder_t *B, void *buffer, size_t size) +{ + /* User is allowed to call tentatively to see if there is support. */ + if (!B->is_default_emitter) { + return 0; + } + buffer = flatcc_emitter_copy_buffer(&B->default_emit_context, buffer, size); + check(buffer, "default emitter declined to copy buffer"); + return buffer; +} + +void *flatcc_builder_finalize_buffer(flatcc_builder_t *B, size_t *size_out) +{ + void * buffer; + size_t size; + + size = flatcc_builder_get_buffer_size(B); + + if (size_out) { + *size_out = size; + } + + buffer = FLATCC_BUILDER_ALLOC(size); + + if (!buffer) { + check(0, "failed to allocated memory for finalized buffer"); + goto done; + } + if (!flatcc_builder_copy_buffer(B, buffer, size)) { + check(0, "default emitter declined to copy buffer"); + FLATCC_BUILDER_FREE(buffer); + buffer = 0; + } +done: + if (!buffer && size_out) { + *size_out = 0; + } + return buffer; +} + +void *flatcc_builder_finalize_aligned_buffer(flatcc_builder_t *B, size_t *size_out) +{ + void * buffer; + size_t align; + size_t size; + + size = flatcc_builder_get_buffer_size(B); + + if (size_out) { + *size_out = size; + } + align = flatcc_builder_get_buffer_alignment(B); + + size = (size + align - 1) & ~(align - 1); + buffer = FLATCC_BUILDER_ALIGNED_ALLOC(align, size); + + if (!buffer) { + goto done; + } + if (!flatcc_builder_copy_buffer(B, buffer, size)) { + FLATCC_BUILDER_ALIGNED_FREE(buffer); + buffer = 0; + goto done; + } +done: + if (!buffer && size_out) { + *size_out = 0; + } + return buffer; +} + +void *flatcc_builder_aligned_alloc(size_t alignment, size_t size) +{ + return FLATCC_BUILDER_ALIGNED_ALLOC(alignment, size); +} + +void flatcc_builder_aligned_free(void *p) +{ + FLATCC_BUILDER_ALIGNED_FREE(p); +} + +void *flatcc_builder_alloc(size_t size) +{ + return FLATCC_BUILDER_ALLOC(size); +} + +void flatcc_builder_free(void *p) +{ + FLATCC_BUILDER_FREE(p); +} + +void *flatcc_builder_get_emit_context(flatcc_builder_t *B) +{ + return B->emit_context; +} diff --git a/nostrdb/flatcc/emitter.c b/nostrdb/flatcc/emitter.c new file mode 100644 index 0000000000..dbeffacd97 --- /dev/null +++ b/nostrdb/flatcc/emitter.c @@ -0,0 +1,269 @@ +#include + +#include "flatcc_rtconfig.h" +#include "flatcc_emitter.h" + +static int advance_front(flatcc_emitter_t *E) +{ + flatcc_emitter_page_t *p = 0; + + if (E->front && E->front->prev != E->back) { + E->front->prev->page_offset = E->front->page_offset - FLATCC_EMITTER_PAGE_SIZE; + E->front = E->front->prev; + goto done; + } + if (!(p = FLATCC_EMITTER_ALLOC(sizeof(flatcc_emitter_page_t)))) { + return -1; + } + E->capacity += FLATCC_EMITTER_PAGE_SIZE; + if (E->front) { + p->prev = E->back; + p->next = E->front; + E->front->prev = p; + E->back->next = p; + E->front = p; + goto done; + } + /* + * The first page is shared between front and back to avoid + * double unecessary extra allocation. + */ + E->front = p; + E->back = p; + p->next = p; + p->prev = p; + E->front_cursor = E->front->page + FLATCC_EMITTER_PAGE_SIZE / 2; + E->back_cursor = E->front_cursor; + E->front_left = FLATCC_EMITTER_PAGE_SIZE / 2; + E->back_left = FLATCC_EMITTER_PAGE_SIZE - E->front_left; + p->page_offset = -(flatbuffers_soffset_t)E->front_left; + return 0; +done: + E->front_cursor = E->front->page + FLATCC_EMITTER_PAGE_SIZE; + E->front_left = FLATCC_EMITTER_PAGE_SIZE; + E->front->page_offset = E->front->next->page_offset - FLATCC_EMITTER_PAGE_SIZE; + return 0; +} + +static int advance_back(flatcc_emitter_t *E) +{ + flatcc_emitter_page_t *p = 0; + + if (E->back && E->back->next != E->front) { + E->back = E->back->next; + goto done; + } + if (!(p = FLATCC_EMITTER_ALLOC(sizeof(flatcc_emitter_page_t)))) { + return -1; + } + E->capacity += FLATCC_EMITTER_PAGE_SIZE; + if (E->back) { + p->prev = E->back; + p->next = E->front; + E->front->prev = p; + E->back->next = p; + E->back = p; + goto done; + } + /* + * The first page is shared between front and back to avoid + * double unecessary extra allocation. + */ + E->front = p; + E->back = p; + p->next = p; + p->prev = p; + E->front_cursor = E->front->page + FLATCC_EMITTER_PAGE_SIZE / 2; + E->back_cursor = E->front_cursor; + E->front_left = FLATCC_EMITTER_PAGE_SIZE / 2; + E->back_left = FLATCC_EMITTER_PAGE_SIZE - E->front_left; + p->page_offset = -(flatbuffers_soffset_t)E->front_left; + return 0; +done: + E->back_cursor = E->back->page; + E->back_left = FLATCC_EMITTER_PAGE_SIZE; + E->back->page_offset = E->back->prev->page_offset + FLATCC_EMITTER_PAGE_SIZE; + return 0; +} + +static int copy_front(flatcc_emitter_t *E, uint8_t *data, size_t size) +{ + size_t k; + + data += size; + while (size) { + k = size; + if (k > E->front_left) { + k = E->front_left; + if (k == 0) { + if (advance_front(E)) { + return -1; + } + continue; + } + } + E->front_cursor -= k; + E->front_left -= k; + data -= k; + size -= k; + memcpy(E->front_cursor, data, k); + }; + return 0; +} + +static int copy_back(flatcc_emitter_t *E, uint8_t *data, size_t size) +{ + size_t k; + + while (size) { + k = size; + if (k > E->back_left) { + k = E->back_left; + if (k == 0) { + if (advance_back(E)) { + return -1; + } + continue; + } + } + memcpy(E->back_cursor, data, k); + size -= k; + data += k; + E->back_cursor += k; + E->back_left -= k; + } + return 0; +} + +int flatcc_emitter_recycle_page(flatcc_emitter_t *E, flatcc_emitter_page_t *p) +{ + if (p == E->front || p == E->back) { + return -1; + } + p->next->prev = p->prev; + p->prev->next = p->next; + p->prev = E->front->prev; + p->next = E->front; + p->prev->next = p; + p->next->prev = p; + return 0; +} + +void flatcc_emitter_reset(flatcc_emitter_t *E) +{ + flatcc_emitter_page_t *p = E->front; + + if (!E->front) { + return; + } + E->back = E->front; + E->front_cursor = E->front->page + FLATCC_EMITTER_PAGE_SIZE / 2; + E->back_cursor = E->front_cursor; + E->front_left = FLATCC_EMITTER_PAGE_SIZE / 2; + E->back_left = FLATCC_EMITTER_PAGE_SIZE - FLATCC_EMITTER_PAGE_SIZE / 2; + E->front->page_offset = -(flatbuffers_soffset_t)E->front_left; + /* Heuristic to reduce peak allocation over time. */ + if (E->used_average == 0) { + E->used_average = E->used; + } + E->used_average = E->used_average * 3 / 4 + E->used / 4; + E->used = 0; + while (E->used_average * 2 < E->capacity && E->back->next != E->front) { + /* We deallocate the page after back since it is less likely to be hot in cache. */ + p = E->back->next; + E->back->next = p->next; + p->next->prev = E->back; + FLATCC_EMITTER_FREE(p); + E->capacity -= FLATCC_EMITTER_PAGE_SIZE; + } +} + +void flatcc_emitter_clear(flatcc_emitter_t *E) +{ + flatcc_emitter_page_t *p = E->front; + + if (!p) { + return; + } + p->prev->next = 0; + while (p->next) { + p = p->next; + FLATCC_EMITTER_FREE(p->prev); + } + FLATCC_EMITTER_FREE(p); + memset(E, 0, sizeof(*E)); +} + +int flatcc_emitter(void *emit_context, + const flatcc_iovec_t *iov, int iov_count, + flatbuffers_soffset_t offset, size_t len) +{ + flatcc_emitter_t *E = emit_context; + uint8_t *p; + + E->used += len; + if (offset < 0) { + if (len <= E->front_left) { + E->front_cursor -= len; + E->front_left -= len; + p = E->front_cursor; + goto copy; + } + iov += iov_count; + while (iov_count--) { + --iov; + if (copy_front(E, iov->iov_base, iov->iov_len)) { + return -1; + } + } + } else { + if (len <= E->back_left) { + p = E->back_cursor; + E->back_cursor += len; + E->back_left -= len; + goto copy; + } + while (iov_count--) { + if (copy_back(E, iov->iov_base, iov->iov_len)) { + return -1; + } + ++iov; + } + } + return 0; +copy: + while (iov_count--) { + memcpy(p, iov->iov_base, iov->iov_len); + p += iov->iov_len; + ++iov; + } + return 0; +} + +void *flatcc_emitter_copy_buffer(flatcc_emitter_t *E, void *buf, size_t size) +{ + flatcc_emitter_page_t *p; + size_t len; + + if (size < E->used) { + return 0; + } + if (!E->front) { + return 0; + } + if (E->front == E->back) { + memcpy(buf, E->front_cursor, E->used); + return buf; + } + len = FLATCC_EMITTER_PAGE_SIZE - E->front_left; + memcpy(buf, E->front_cursor, len); + buf = (uint8_t *)buf + len; + p = E->front->next; + while (p != E->back) { + memcpy(buf, p->page, FLATCC_EMITTER_PAGE_SIZE); + buf = (uint8_t *)buf + FLATCC_EMITTER_PAGE_SIZE; + p = p->next; + } + memcpy(buf, p->page, FLATCC_EMITTER_PAGE_SIZE - E->back_left); + return buf; +} diff --git a/nostrdb/flatcc/flatcc.h b/nostrdb/flatcc/flatcc.h new file mode 100644 index 0000000000..04eb187f47 --- /dev/null +++ b/nostrdb/flatcc/flatcc.h @@ -0,0 +1,268 @@ +#ifndef FLATCC_H +#define FLATCC_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This is the primary `flatcc` interface when compiling `flatcc` as a + * library. Functions and types in the this interface will be kept + * stable to the extend possible or reasonable, but do not rely on other + * interfaces except "config.h" used to set default options for this + * interface. + * + * This interface is unrelated to the standalone flatbuilder library + * which has a life of its own. + */ + +#include + +#ifndef UINT8_MAX +#include +#endif + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4820) /* x bytes padding added in struct */ +#endif + +typedef struct flatcc_options flatcc_options_t; +typedef void (*flatcc_error_fun) (void *err_ctx, const char *buf, size_t len); + +struct flatcc_options { + size_t max_schema_size; + int max_include_depth; + int max_include_count; + int disable_includes; + int allow_boolean_conversion; + int allow_enum_key; + int allow_enum_struct_field; + int allow_multiple_key_fields; + int allow_primary_key; + int allow_scan_for_all_fields; + int allow_string_key; + int allow_struct_field_deprecate; + int allow_struct_field_key; + int allow_struct_root; + int ascending_enum; + int hide_later_enum; + int hide_later_struct; + int offset_size; + int voffset_size; + int utype_size; + int bool_size; + int require_root_type; + int strict_enum_init; + uint64_t vt_max_count; + + const char *default_schema_ext; + const char *default_bin_schema_ext; + const char *default_bin_ext; + + /* Code Generator specific options. */ + int gen_stdout; + int gen_dep; + + const char *gen_depfile; + const char *gen_deptarget; + const char *gen_outfile; + + int gen_append; + + int cgen_pad; + int cgen_sort; + int cgen_pragmas; + + int cgen_common_reader; + int cgen_common_builder; + int cgen_reader; + int cgen_builder; + int cgen_verifier; + int cgen_json_parser; + int cgen_json_printer; + int cgen_recursive; + int cgen_spacing; + int cgen_no_conflicts; + + + int bgen_bfbs; + int bgen_qualify_names; + int bgen_length_prefix; + + /* Namespace args - these can override defaults so are null by default. */ + const char *ns; + const char *nsc; + + const char **inpaths; + const char **srcpaths; + int inpath_count; + int srcpath_count; + const char *outpath; +}; + +/* Runtime configurable optoins. */ +void flatcc_init_options(flatcc_options_t *opts); + +typedef void *flatcc_context_t; + +/* + * Call functions below in order listed one at a time. + * Each parse requires a new context. + * + * A reader file is named after the source base name, e.g. + * `monster.fbs` becomes `monster.h`. Builders are optional and created + * as `monster_builder.h`. A reader require a common header + * `flatbuffers_commoner.h` and a builder requires + * `flatbuffers_common_builder.h` in addition to the reader filers. A + * reader need no other source, but builders must link with the + * `flatbuilder` library and include files in `include/flatbuffers`. + * + * All the files may also be concatenated into one single file and then + * files will not be attempted included externally. This can be used + * with stdout output. The common builder can follow the common + * reader immediately, or at any later point before the first builder. + * The common files should only be included once, but not harm is done + * if duplication occurs. + * + * The outpath is prefixed every output filename. The containing + * directory must exist, but the prefix may have text following + * the directory, for example the namespace. If outpath = "stdout", + * files are generated to stdout. + * + * Note that const char * options must remain valid for the lifetime + * of the context since they are not copied. The options object itself + * is not used after initialization and may be reused. +*/ + +/* + * `name` is the name of the schema file or buffer. If it is path, the + * basename is extracted (leading path stripped), and the default schema + * extension is stripped if present. The resulting name is used + * internally when generating output files. Typically the `name` + * argument will be the same as a schema file path given to + * `flatcc_parse_file`, but it does not have to be. + * + * `name` may be null if only common files are generated. + * + * `error_out` is an optional error handler. If null output is truncated + * to a reasonable size and sent to stderr. `error_ctx` is provided as + * first argument to `error_out` if `error_out` is non-zero, otherwise + * it is ignored. + * + * Returns context or null on error. + */ +flatcc_context_t flatcc_create_context(flatcc_options_t *options, const char *name, + flatcc_error_fun error_out, void *error_ctx); + +/* Like `flatcc_create_context`, but with length argument for name. */ +/* + * Parse is optional - not needed for common files. If the input buffer version + * is called, the buffer must be zero terminated, otherwise an input + * path can be specified. The output path can be null. + * + * Only one parse can be called per context. + * + * The buffer size is limited to the max_schema_size option unless it is + * 0. The default is reasonable size like 64K depending on config flags. + * + * The buffer must remain valid for the duration of the context. + * + * The schema cannot contain include statements when parsed as a buffer. + * + * Returns 0 on success. + */ +int flatcc_parse_buffer(flatcc_context_t ctx, const char *buf, size_t buflen); + +/* + * If options contain a non-zero `inpath` option, the resulting filename is + * prefixed with that path unless the filename is an absolute path. + * + * Errors are sent to the error handler given during initialization, + * or to stderr. + * + * The file size is limited to the max_schema_size option unless it is + * 0. The default is reasonable size like 64K depending on config flags. + * + * Returns 0 on success. + */ +int flatcc_parse_file(flatcc_context_t ctx, const char *filename); + +/* + * Generate output files. The basename derived when the context was + * created is used used to name the output files with respective + * extensions. If the outpath option is not null it is prefixed the + * output files. The `cgen_common_reader, cgen_common_builder, + * cgen_reader, and cgen_builder` must be set or reset depending on what + * is to be generated. The common files do not require a parse, and the + * non-common files require a successfull parse or the result is + * undefined. + * + * Unlinke the parser, the code generator produce errors to stderr + * always. These errors are rare, such as using too long namespace + * names. + * + * If the `gen_stdout` option is set, all files are generated to stdout. + * In this case it is unwise to mix C and binary schema output options. + * + * If `bgen_bfbs` is set, a binary schema is generated to a file with + * the `.bfbs` extension. See also `flatcc_generate_binary_schema` for + * further details. Only `flatcc_generate_files` is called via the + * `flatcc` cli command. + * + * The option `bgen_length_prefix` option will cause a length prefix to be + * written to the each output binary schema. This option is only + * understood when writing to files. + * + * Returns 0 on success. + */ +int flatcc_generate_files(flatcc_context_t ctx); + +/* + * Returns a buffer with a binary schema for a previous parse. + * The user is responsible for calling `free` on the returned buffer + * unless it returns 0 on error. + * + * Can be called instead of generate files, before, or after, but a + * schema must be parsed first. + * + * Returns a binary schema in `reflection.fbs` format. Any included + * files will be contained in the schema and there are no separate + * schema files for included schema. + * + * All type names are scoped, mening that they are refixed their + * namespace using `.` as the namespace separator, for example: + * "MyGame.Example.Monster". Note that the this differs from the current + * `flatc` compiler which does not prefix names. Enum names are not + * scoped, but the scope is implied by the containing enum type. + * The option `bgen_qualify_names=0` changes this behavior. + * + * If the default option `ascending_enum` is disabled, the `flatcc` will + * accept duplicate values and overlapping ranges like the C programming + * language. In this case enum values in the binary schema will not be + * searchable. At any rate enum names are not searchable in the current + * schema format. + * + */ +void *flatcc_generate_binary_schema(flatcc_context_t ctx, size_t *size); + +/* + * Similar to `flatcc_generate_binary_schema` but copies the binary + * schema into a user supplied buffer. If the buffer is too small + * the return value will be negative and the buffer content undefined. + */ +int flatcc_generate_binary_schema_to_buffer(flatcc_context_t ctx, void *buf, size_t bufsiz); + +/* Must be called to deallocate resources eventually - it valid but + * without effect to call with a null context. */ +void flatcc_destroy_context(flatcc_context_t ctx); + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_H */ diff --git a/nostrdb/flatcc/flatcc_accessors.h b/nostrdb/flatcc/flatcc_accessors.h new file mode 100644 index 0000000000..084ecb1b17 --- /dev/null +++ b/nostrdb/flatcc/flatcc_accessors.h @@ -0,0 +1,101 @@ +#ifndef FLATCC_ACCESSORS +#define FLATCC_ACCESSORS + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef UINT8_MAX +#include +#endif + +#define __flatcc_basic_scalar_accessors_impl(N, T, W, E) \ +static inline size_t N ## __size(void) \ +{ return sizeof(T); } \ +static inline T *N ## __ptr_add(T *p, size_t i) \ +{ return p + i; } \ +static inline const T *N ## __const_ptr_add(const T *p, size_t i) \ +{ return p + i; } \ +static inline T N ## _read_from_pe(const void *p) \ +{ return N ## _cast_from_pe(*(T *)p); } \ +static inline T N ## _read_to_pe(const void *p) \ +{ return N ## _cast_to_pe(*(T *)p); } \ +static inline T N ## _read(const void *p) \ +{ return *(T *)p; } \ +static inline void N ## _write_from_pe(void *p, T v) \ +{ *(T *)p = N ## _cast_from_pe(v); } \ +static inline void N ## _write_to_pe(void *p, T v) \ +{ *(T *)p = N ## _cast_to_pe(v); } \ +static inline void N ## _write(void *p, T v) \ +{ *(T *)p = v; } \ +static inline T N ## _read_from_le(const void *p) \ +{ return N ## _cast_from_le(*(T *)p); } \ +typedef struct { int is_null; T value; } N ## _option_t; + +#define __flatcc_define_integer_accessors_impl(N, T, W, E) \ +static inline T N ## _cast_from_pe(T v) \ +{ return (T) E ## W ## toh((uint ## W ## _t)v); } \ +static inline T N ## _cast_to_pe(T v) \ +{ return (T) hto ## E ## W((uint ## W ## _t)v); } \ +static inline T N ## _cast_from_le(T v) \ +{ return (T) le ## W ## toh((uint ## W ## _t)v); } \ +static inline T N ## _cast_to_le(T v) \ +{ return (T) htole ## W((uint ## W ## _t)v); } \ +static inline T N ## _cast_from_be(T v) \ +{ return (T) be ## W ## toh((uint ## W ## _t)v); } \ +static inline T N ## _cast_to_be(T v) \ +{ return (T) htobe ## W((uint ## W ## _t)v); } \ +__flatcc_basic_scalar_accessors_impl(N, T, W, E) + +#define __flatcc_define_real_accessors_impl(N, T, W, E) \ +union __ ## N ## _cast { T v; uint ## W ## _t u; }; \ +static inline T N ## _cast_from_pe(T v) \ +{ union __ ## N ## _cast x; \ + x.v = v; x.u = E ## W ## toh(x.u); return x.v; } \ +static inline T N ## _cast_to_pe(T v) \ +{ union __ ## N ## _cast x; \ + x.v = v; x.u = hto ## E ## W(x.u); return x.v; } \ +static inline T N ## _cast_from_le(T v) \ +{ union __ ## N ## _cast x; \ + x.v = v; x.u = le ## W ## toh(x.u); return x.v; } \ +static inline T N ## _cast_to_le(T v) \ +{ union __ ## N ## _cast x; \ + x.v = v; x.u = htole ## W(x.u); return x.v; } \ +static inline T N ## _cast_from_be(T v) \ +{ union __ ## N ## _cast x; \ + x.v = v; x.u = be ## W ## toh(x.u); return x.v; } \ +static inline T N ## _cast_to_be(T v) \ +{ union __ ## N ## _cast x; \ + x.v = v; x.u = htobe ## W(x.u); return x.v; } \ +__flatcc_basic_scalar_accessors_impl(N, T, W, E) + +#define __flatcc_define_integer_accessors(N, T, W, E) \ +__flatcc_define_integer_accessors_impl(N, T, W, E) + +#define __flatcc_define_real_accessors(N, T, W, E) \ +__flatcc_define_real_accessors_impl(N, T, W, E) + +#define __flatcc_define_basic_integer_accessors(NS, TN, T, W, E) \ +__flatcc_define_integer_accessors(NS ## TN, T, W, E) + +#define __flatcc_define_basic_real_accessors(NS, TN, T, W, E) \ +__flatcc_define_real_accessors(NS ## TN, T, W, E) + +#define __flatcc_define_basic_scalar_accessors(NS, E) \ +__flatcc_define_basic_integer_accessors(NS, char, char, 8, E) \ +__flatcc_define_basic_integer_accessors(NS, uint8, uint8_t, 8, E) \ +__flatcc_define_basic_integer_accessors(NS, uint16, uint16_t, 16, E) \ +__flatcc_define_basic_integer_accessors(NS, uint32, uint32_t, 32, E) \ +__flatcc_define_basic_integer_accessors(NS, uint64, uint64_t, 64, E) \ +__flatcc_define_basic_integer_accessors(NS, int8, int8_t, 8, E) \ +__flatcc_define_basic_integer_accessors(NS, int16, int16_t, 16, E) \ +__flatcc_define_basic_integer_accessors(NS, int32, int32_t, 32, E) \ +__flatcc_define_basic_integer_accessors(NS, int64, int64_t, 64, E) \ +__flatcc_define_basic_real_accessors(NS, float, float, 32, E) \ +__flatcc_define_basic_real_accessors(NS, double, double, 64, E) + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_ACCESSORS */ diff --git a/nostrdb/flatcc/flatcc_alloc.h b/nostrdb/flatcc/flatcc_alloc.h new file mode 100644 index 0000000000..c07462d577 --- /dev/null +++ b/nostrdb/flatcc/flatcc_alloc.h @@ -0,0 +1,127 @@ +#ifndef FLATCC_ALLOC_H +#define FLATCC_ALLOC_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * These allocation abstractions are __only__ for runtime libraries. + * + * The flatcc compiler uses Posix allocation routines regardless + * of how this file is configured. + * + * This header makes it possible to use systems where malloc is not + * valid to use. In this case the portable library will not help + * because it implements Posix / C11 abstractions. + * + * Systems like FreeRTOS do not work with Posix memory calls and here it + * can be helpful to override runtime allocation primitives. + * + * In general, it is better to customize the allocator and emitter via + * flatcc_builder_custom_init and to avoid using the default emitter + * specific high level calls the copy out a buffer that must later be + * deallocated. This provides full control of allocation withou the need + * for this file. + * + * + * IMPORTANT + * + * If you override malloc, free, etc., make sure your applications + * use the same allocation methods. For example, samples/monster.c + * and several test cases are no longer guaranteed to work out of the + * box. + * + * The changes must only affect target runtime compilation including the + * the runtime library libflatccrt. + * + * The host system flatcc compiler and the compiler library libflatcc + * should NOT be compiled with non-Posix allocation since the compiler + * has a dependency on the runtime library and the wrong free operation + * might be callled. The safest way to avoid this problem this is to + * compile flatcc with the CMake script and the runtime files with a + * dedicated build system for the target system. + */ + +#include + +#ifndef FLATCC_ALLOC +#define FLATCC_ALLOC(n) malloc(n) +#endif + +#ifndef FLATCC_FREE +#define FLATCC_FREE(p) free(p) +#endif + +#ifndef FLATCC_REALLOC +#define FLATCC_REALLOC(p, n) realloc(p, n) +#endif + +#ifndef FLATCC_CALLOC +#define FLATCC_CALLOC(nm, n) calloc(nm, n) +#endif + +/* + * Implements `aligned_alloc` and `aligned_free`. + * Even with C11, this implements non-standard aligned_free needed for portable + * aligned_alloc implementations. + */ +#ifndef FLATCC_USE_GENERIC_ALIGNED_ALLOC + +#ifndef FLATCC_NO_PALIGNED_ALLOC +#include "paligned_alloc.h" +#else +#if !defined(__aligned_free_is_defined) || !__aligned_free_is_defined +#define aligned_free free +#endif +#endif + +#else /* FLATCC_USE_GENERIC_ALIGNED_ALLOC */ + +#ifndef FLATCC_ALIGNED_ALLOC +static inline void *__flatcc_aligned_alloc(size_t alignment, size_t size) +{ + char *raw; + void *buf; + size_t total_size = (size + alignment - 1 + sizeof(void *)); + + if (alignment < sizeof(void *)) { + alignment = sizeof(void *); + } + raw = (char *)(size_t)FLATCC_ALLOC(total_size); + buf = raw + alignment - 1 + sizeof(void *); + buf = (void *)(((size_t)buf) & ~(alignment - 1)); + ((void **)buf)[-1] = raw; + return buf; +} +#define FLATCC_ALIGNED_ALLOC(alignment, size) __flatcc_aligned_alloc(alignment, size) +#endif /* FLATCC_USE_GENERIC_ALIGNED_ALLOC */ + +#ifndef FLATCC_ALIGNED_FREE +static inline void __flatcc_aligned_free(void *p) +{ + char *raw; + + if (!p) return; + raw = ((void **)p)[-1]; + + FLATCC_FREE(raw); +} +#define FLATCC_ALIGNED_FREE(p) __flatcc_aligned_free(p) +#endif + +#endif /* FLATCC_USE_GENERIC_ALIGNED_ALLOC */ + +#ifndef FLATCC_ALIGNED_ALLOC +#define FLATCC_ALIGNED_ALLOC(a, n) aligned_alloc(a, n) +#endif + +#ifndef FLATCC_ALIGNED_FREE +#define FLATCC_ALIGNED_FREE(p) aligned_free(p) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_ALLOC_H */ diff --git a/nostrdb/flatcc/flatcc_assert.h b/nostrdb/flatcc/flatcc_assert.h new file mode 100644 index 0000000000..3db3e7b02b --- /dev/null +++ b/nostrdb/flatcc/flatcc_assert.h @@ -0,0 +1,45 @@ +#ifndef FLATCC_ASSERT_H +#define FLATCC_ASSERT_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* +* This assert abstraction is only used for the flatcc runtime library. +* The flatcc compiler uses Posix assert routines regardless of how this +* file is configured. +* +* This header makes it possible to use systems where assert is not +* valid to use. Note that `` may remain a dependency for static +* assertions. +* +* `FLATCC_ASSERT` is designed to handle errors which cannot be ignored +* and could lead to crash. The portable library may use assertions that +* are not affected by this macro. +* +* `FLATCC_ASSERT` defaults to POSIX assert but can be overrided by a +* preprocessor definition. +* +* Runtime assertions can be entirely disabled by defining +* `FLATCC_NO_ASSERT`. +*/ + +#ifdef FLATCC_NO_ASSERT +/* NOTE: This will not affect inclusion of for static assertions. */ +#undef FLATCC_ASSERT +#define FLATCC_ASSERT(x) ((void)0) +/* Grisu3 is used for floating point conversion in JSON processing. */ +#define GRISU3_NO_ASSERT +#endif + +#ifndef FLATCC_ASSERT +#include +#define FLATCC_ASSERT assert +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_ASSERT_H */ diff --git a/nostrdb/flatcc/flatcc_builder.h b/nostrdb/flatcc/flatcc_builder.h new file mode 100644 index 0000000000..3871b1d075 --- /dev/null +++ b/nostrdb/flatcc/flatcc_builder.h @@ -0,0 +1,1908 @@ +#ifndef FLATCC_BUILDER_H +#define FLATCC_BUILDER_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Library for building untyped FlatBuffers. Intended as a support + * library for generated C code to produce typed builders, but might + * also be useful in runtime environments and as support for scripting + * languages. + * + * The builder has two API layers: a stack based `start/end` approach, + * and a direct `create`, and they may be mixed freely. The direct + * approach may be used as part of more specialized optimizations such + * as rewriting buffers while the stack approach is convenient for state + * machine driven parsers without a stack, or with a very simple stack + * without extra allocations. + * + * The builder emits partial buffer sequences to a user provided emitter + * function and does not require a full buffer reprensenation in memory. + * For this reason it also does not support sorting or other operations + * that requires representing the buffer, but post-processors can easily + * do this, and the generated schema specific code and provide functions + * to handle this. + * + * A custom allocator with a default realloc implementation can place + * restraints on resource consumption and provide initial allocation + * sizes for various buffers and stacks in use. + * + * A buffer under construction uses a virtual address space for the + * completed part of the buffer, starting at 0 and growing in both + * directions, or just down depending on whether vtables should be + * clustered at the end or not. Clustering may help caching and + * preshipping that part of the buffer. + * + * Because an offset cannot be known before its reference location is + * defined, every completed table, vector, etc. returns a reference into + * the virtual address range. If the final buffer keeps the 0 offset, + * these references remain stable an may be used for external references + * into the buffer. + * + * The maximum buffer that can be constructed is in praxis limited to + * half the UOFFSET_MAX size, typically 2^31 bytes, not counting + * clustered vtables that may consume and additional 2^31 bytes + * (positive address range), but in praxis cannot because vtable + * references are signed and thus limited to 2^31 bytes (or equivalent + * depending on the flatbuffer types chosen). + * + * CORRECTION: in various places rules are mentioned about nesting and using + * a reference at most once. In fact, DAG's are also valid flatbuffers. + * This means a reference may be reused as long as each individual use + * obeys the rules and, for example, circular references are not + * constructed (circular types are ok, but objects graphs with cycles + * are not permitted). Be especially aware of the offset vector create + * call which translates the references into offsets - this can be + * reverted by noting the reference in vector and calculate the base + * used for the offset to restore the original references after the + * vector has been emitted. + */ + +#include +#ifndef UINT8_MAX +#include +#endif + +#include "flatcc_flatbuffers.h" +#include "flatcc_emitter.h" +#include "flatcc_refmap.h" + +/* It is possible to enable logging here. */ +#ifndef FLATCC_BUILDER_ASSERT +#define FLATCC_BUILDER_ASSERT(cond, reason) FLATCC_ASSERT(cond) +#endif + +/* + * Eror handling is not convenient and correct use should not cause + * errors beyond possibly memory allocation, but assertions are a + * good way to trace problems. + * + * Note: some internal assertion will remain if disabled. + */ +#ifndef FLATCC_BUILDER_ASSERT_ON_ERROR +#define FLATCC_BUILDER_ASSERT_ON_ERROR 1 +#endif + +/* + * If set, checks user input agains state and returns error, + * otherwise errors are ignored (assuming they won't happen). + * Errors will be asserted if enabled and checks are not skipped. + */ +#ifndef FLATCC_BUILDER_SKIP_CHECKS +#define FLATCC_BUILDER_SKIP_CHECKS 0 +#endif + + +/* + * When adding the same field to a table twice this is either an error + * or the existing field is returned, potentially introducing garbage + * if the type is a vector, table, or string. When implementing parsers + * it may be convenient to not treat this as an error. + */ +#ifndef FLATCC_BUILDER_ALLOW_REPEAT_TABLE_ADD +#define FLATCC_BUILDER_ALLOW_REPEAT_TABLE_ADD 0 +#endif + +/** + * This type must have same size as `flatbuffers_uoffset_t` + * and must be a signed type. + */ +typedef flatbuffers_soffset_t flatcc_builder_ref_t; +typedef flatbuffers_utype_t flatcc_builder_utype_t; + +/** + * This type must be compatible with code generation that + * creates union specific ref types. + */ +typedef struct flatcc_builder_union_ref { + flatcc_builder_utype_t type; + flatcc_builder_ref_t value; +} flatcc_builder_union_ref_t; + +typedef struct flatcc_builder_union_vec_ref { + flatcc_builder_ref_t type; + flatcc_builder_ref_t value; +} flatcc_builder_union_vec_ref_t; + +/** + * Virtual tables are off by one to avoid being mistaken for error at + * position 0, and it makes them detectable as such because no other + * reference is uneven. Vtables are emitted at their actual location + * which is one less than the reference value. + */ +typedef flatbuffers_soffset_t flatcc_builder_vt_ref_t; + +typedef flatbuffers_uoffset_t flatcc_builder_identifier_t; + +/** + * Hints to custom allocators so they can provide initial alloc sizes + * etc. There will be at most one buffer for each allocation type per + * flatcc_builder instance. Buffers containing only structs may avoid + * allocation altogether using a `create` call. The vs stack must hold + * vtable entries for all open tables up to their requested max id, but + * unused max id overlap on the stack. The final vtables only store the + * largest id actually added. The fs stack must hold stack frames for + * the nesting levels expected in the buffer, each about 50-100 bytes. + * The ds stack holds open vectors, table data, and nested buffer state. + * `create` calls bypass the `ds` and `fs` stack and are thus faster. + * The vb buffer holds a copy of all vtables seen and emitted since last + * vtable flush. The patch log holds a uoffset for every table field + * added to currently open tables. The hash table holds a uoffset entry + * for each hash slot where the allocator decides how many to provide + * above a certain minimum. The vd buffer allocates vtable descriptors + * which is a reference to an emitted vtable, an offset to a cached + * vtable, and a link to next descriptor with same hash. Calling `reset` + * after build can either keep the allocation levels for the next + * buffer, or reduce the buffers already allocated by requesting 1 byte + * allocations (meaning provide a default). + * + * The user stack is not automatically allocated, but when entered + * explicitly, the boundary is rembered in the current live + * frame. + */ +enum flatcc_builder_alloc_type { + /* The stack where vtables are build. */ + flatcc_builder_alloc_vs, + /* The stack where data structures are build. */ + flatcc_builder_alloc_ds, + /* The virtual table buffer cache, holds a copy of each vt seen. */ + flatcc_builder_alloc_vb, + /* The patch log, remembers table fields with outstanding offset refs. */ + flatcc_builder_alloc_pl, + /* The stack of frames for nested types. */ + flatcc_builder_alloc_fs, + /* The hash table part of the virtual table cache. */ + flatcc_builder_alloc_ht, + /* The vtable descriptor buffer, i.e. list elements for emitted vtables. */ + flatcc_builder_alloc_vd, + /* User stack frame for custom data. */ + flatcc_builder_alloc_us, + + /* Number of allocation buffers. */ + flatcc_builder_alloc_buffer_count +}; + +/** Must reflect the `flatcc_builder_alloc_type` enum. */ +#define FLATCC_BUILDER_ALLOC_BUFFER_COUNT flatcc_builder_alloc_buffer_count + +#ifndef FLATCC_BUILDER_ALLOC +#define FLATCC_BUILDER_ALLOC(n) FLATCC_ALLOC(n) +#endif + +#ifndef FLATCC_BUILDER_FREE +#define FLATCC_BUILDER_FREE(p) FLATCC_FREE(p) +#endif + +#ifndef FLATCC_BUILDER_REALLOC +#define FLATCC_BUILDER_REALLOC(p, n) FLATCC_REALLOC(p, n) +#endif + +#ifndef FLATCC_BUILDER_ALIGNED_ALLOC +#define FLATCC_BUILDER_ALIGNED_ALLOC(a, n) FLATCC_ALIGNED_ALLOC(a, n) +#endif + +#ifndef FLATCC_BUILDER_ALIGNED_FREE +#define FLATCC_BUILDER_ALIGNED_FREE(p) FLATCC_ALIGNED_FREE(p) +#endif + +/** + * Emits data to a conceptual deque by appending to either front or + * back, starting from offset 0. + * + * Each emit call appends a strictly later or earlier sequence than the + * last emit with same offset sign. Thus a buffer is gradually grown at + * both ends. `len` is the combined length of all iov entries such that + * `offset + len` yields the former offset for negative offsets and + * `offset + len` yields the next offset for non-negative offsets. + * The bulk of the data will be in the negative range, possibly all of + * it. The first emitted emitted range will either start or end at + * offset 0. If offset 0 is emitted, it indicates the start of clustered + * vtables. The last positive (non-zero) offset may be zero padding to + * place the buffer in a full multiple of `block_align`, if set. + * + * No iov entry is empty, 0 < iov_count <= FLATCC_IOV_COUNT_MAX. + * + * The source data are in general ephemeral and should be consumed + * immediately, as opposed to caching iov. + * + * For high performance applications: + * + * The `create` calls may reference longer living data, but header + * fields etc. will still be short lived. If an emitter wants to + * reference data in another buffer rather than copying, it should + * inspect the memory range. The length of an iov entry may also be used + * since headers are never very long (anything starting at 16 bytes can + * safely be assumed to be user provided, or static zero padding). It is + * guaranteed that data pointers in `create` calls receive a unique slot + * separate from temporary headers, in the iov table which may be used + * for range checking or hashing (`create_table` is the only call that + * mutates the data buffer). It is also guaranteed (with the exception + * of `create_table` and `create_cached_vtable`) that data provided to + * create calls are not referenced at all by the builder, and these data + * may therefore de-facto be handles rather than direct pointers when + * the emitter and data provider can agree on such a protocol. This does + * NOT apply to any start/end/add/etc. calls which do copy to stack. + * `flatcc_builder_padding_base` may be used to test if an iov entry is + * zero padding which always begins at that address. + * + * Future: the emit interface could be extended with a type code + * and return an existing object insted of the emitted if, for + * example, they are identical. Outside this api level, generated + * code could provide a table comparison function to help such + * deduplication. It would be optional because two equal objects + * are not necessarily identical. The emitter already receives + * one object at time. + * + * Returns 0 on success and otherwise causes the flatcc_builder + * to fail. + */ +typedef int flatcc_builder_emit_fun(void *emit_context, + const flatcc_iovec_t *iov, int iov_count, flatbuffers_soffset_t offset, size_t len); + +/* + * Returns a pointer to static padding used in emitter calls. May + * sometimes also be used for empty defaults such as identifier. + */ +extern const uint8_t flatcc_builder_padding_base[]; + +/** + * `request` is a minimum size to be returned, but allocation is + * expected to grow exponentially or in reasonable chunks. Notably, + * `alloc_type = flatcc_builder_alloc_ht` will only use highest available + * power of 2. The allocator may shrink if `request` is well below + * current size but should avoid repeated resizing on small changes in + * request sizes. If `zero_fill` is non-zero, allocated data beyond + * the current size must be zeroed. The buffer `b` may be null with 0 + * length initially. `alloc_context` is completely implementation + * dependendent, and not needed when just relying on realloc. The + * resulting buffer may be the same or different with moved data, like + * realloc. Returns -1 with unmodified buffer on failure or 0 on + * success. The `alloc_type` identifies the buffer type. This may be + * used to cache buffers between instances of builders, or to decide a + * default allocation size larger than requested. If `need` is zero the + * buffer should be deallocate if non-zero, and return success (0) + * regardless. + */ +typedef int flatcc_builder_alloc_fun(void *alloc_context, + flatcc_iovec_t *b, size_t request, int zero_fill, int alloc_type); + +/* + * The number of hash slots there will be allocated space for. The + * allocator may provide more. The size returned should be + * `sizeof(flatbuffers_uoffset_t) * count`, where the size is a power of + * 2 (or the rest is wasted). The hash table can store many more entries + * than slots using linear search. The table does not resize. + */ +#ifndef FLATCC_BUILDER_MIN_HASH_COUNT +#define FLATCC_BUILDER_MIN_HASH_COUNT 64 +#endif + +typedef struct __flatcc_builder_buffer_frame __flatcc_builder_buffer_frame_t; +struct __flatcc_builder_buffer_frame { + flatcc_builder_identifier_t identifier; + flatcc_builder_ref_t mark; + flatbuffers_uoffset_t vs_end; + flatbuffers_uoffset_t nest_id; + uint16_t flags; + uint16_t block_align; +}; + +typedef struct __flatcc_builder_vector_frame __flatcc_builder_vector_frame_t; +struct __flatcc_builder_vector_frame { + flatbuffers_uoffset_t elem_size; + flatbuffers_uoffset_t count; + flatbuffers_uoffset_t max_count; +}; + +typedef struct __flatcc_builder_table_frame __flatcc_builder_table_frame_t; +struct __flatcc_builder_table_frame { + flatbuffers_uoffset_t vs_end; + flatbuffers_uoffset_t pl_end; + uint32_t vt_hash; + flatbuffers_voffset_t id_end; +}; + +/* + * Store state for nested structures such as buffers, tables and vectors. + * + * For less busy data and data where access to a previous state is + * irrelevant, the frame may store the current state directly. Otherwise + * the current state is maintained in the flatcc_builder_t structure in a + * possibly derived form (e.g. ds pointer instead of ds_end offset) and + * the frame is used to store the previous state when the frame is + * entered. + * + * Most operations have a start/update/end cycle the decides the + * liftetime of a frame, but these generally also have a direct form + * (create) that does not use a frame at all. These still do some + * state updates notably passing min_align to parent which may also be + * an operation without a frame following the child level operation + * (e.g. create struct, create buffer). Ending a frame results in the + * same kind of updates. + */ +typedef struct __flatcc_builder_frame __flatcc_builder_frame_t; +struct __flatcc_builder_frame { + flatbuffers_uoffset_t ds_first; + flatbuffers_uoffset_t type_limit; + flatbuffers_uoffset_t ds_offset; + uint16_t align; + uint16_t type; + union { + __flatcc_builder_table_frame_t table; + __flatcc_builder_vector_frame_t vector; + __flatcc_builder_buffer_frame_t buffer; + } container; +}; + +/** + * The main flatcc_builder structure. Can be stack allocated and must + * be initialized with `flatcc_builder_init` and cleared with + * `flatcc_builder_clear` to reclaim memory. Between buffer builds, + * `flatcc_builder_reset` may be used. + */ +typedef struct flatcc_builder flatcc_builder_t; + +struct flatcc_builder { + /* Next entry on reserved stack in `alloc_pl` buffer. */ + flatbuffers_voffset_t *pl; + /* Next entry on reserved stack in `alloc_vs` buffer. */ + flatbuffers_voffset_t *vs; + /* One above the highest entry in vs, used to track vt_size. */ + flatbuffers_voffset_t id_end; + /* The evolving vtable hash updated with every new field. */ + uint32_t vt_hash; + + /* Pointer to ds_first. */ + uint8_t *ds; + /* Offset from `ds` on current frame. */ + flatbuffers_uoffset_t ds_offset; + /* ds buffer size relative to ds_first, clamped to max size of current type. */ + flatbuffers_uoffset_t ds_limit; + + /* ds_first, ds_first + ds_offset is current ds stack range. */ + flatbuffers_uoffset_t ds_first; + /* Points to currently open frame in `alloc_fs` buffer. */ + __flatcc_builder_frame_t *frame; + + /* Only significant to emitter function, if at all. */ + void *emit_context; + /* Only significant to allocator function, if at all. */ + void *alloc_context; + /* Customizable write function that both appends and prepends data. */ + flatcc_builder_emit_fun *emit; + /* Customizable allocator that also deallocates. */ + flatcc_builder_alloc_fun *alloc; + /* Buffers indexed by `alloc_type` */ + flatcc_iovec_t buffers[FLATCC_BUILDER_ALLOC_BUFFER_COUNT]; + /* Number of slots in ht given as 1 << ht_width. */ + size_t ht_width; + + /* The location in vb to add next cached vtable. */ + flatbuffers_uoffset_t vb_end; + /* Where to allocate next vtable descriptor for hash table. */ + flatbuffers_uoffset_t vd_end; + /* Ensure final buffer is aligned to at least this. Nested buffers get their own `min_align`. */ + uint16_t min_align; + /* The current active objects alignment isolated from nested activity. */ + uint16_t align; + /* The current buffers block alignment used when emitting buffer. */ + uint16_t block_align; + /* Signed virtual address range used for `flatcc_builder_ref_t` and emitter. */ + flatcc_builder_ref_t emit_start; + flatcc_builder_ref_t emit_end; + /* 0 for top level, and end of buffer ref for nested buffers (can also be 0). */ + flatcc_builder_ref_t buffer_mark; + /* Next nest_id. */ + flatbuffers_uoffset_t nest_count; + /* Unique id to prevent sharing of vtables across buffers. */ + flatbuffers_uoffset_t nest_id; + /* Current nesting level. Helpful to state-machines with explicit stack and to check `max_level`. */ + int level; + /* Aggregate check for allocated frame and max_level. */ + int limit_level; + /* Track size prefixed buffer. */ + uint16_t buffer_flags; + + /* Settings that may happen with no frame allocated. */ + + flatcc_builder_identifier_t identifier; + + /* Settings that survive reset (emitter, alloc, and contexts also survive): */ + + /* If non-zero, vtable cache gets flushed periodically. */ + size_t vb_flush_limit; + /* If non-zero, fails on deep nesting to help drivers with a stack, such as recursive parsers etc. */ + int max_level; + /* If non-zero, do not cluster vtables at end, only emit negative offsets (0 by default). */ + int disable_vt_clustering; + + /* Set if the default emitter is being used. */ + int is_default_emitter; + /* Only used with default emitter. */ + flatcc_emitter_t default_emit_context; + + /* Offset to the last entered user frame on the user frame stack, after frame header, or 0. */ + size_t user_frame_offset; + + /* The offset to the end of the most recent user frame. */ + size_t user_frame_end; + + /* The optional user supplied refmap for cloning DAG's - not shared with nested buffers. */ + flatcc_refmap_t *refmap; +}; + +/** + * Call this before any other API call. + * + * The emitter handles the completed chunks of the buffer that will no + * longer be required by the builder. It is largely a `write` function + * that can append to both positive and negative offsets. + * + * No memory is allocated during init. Buffers will be allocated as + * needed. The `emit_context` is only used by the emitter, if at all. + * + * `flatcc_builder_reset/clear` calls are automtically forwarded to the + * default emitter. + * + * Returns -1 on failure, 0 on success. + */ +int flatcc_builder_init(flatcc_builder_t *B); + +/** + * Use instead of `flatcc_builder_init` when providing a custom allocator + * or emitter. Leave emitter or allocator null to use default. + * Cleanup of emit and alloc context must be handled manually after + * the builder is cleared or reset, except if emitter is null the + * default will be automatically cleared and reset. + * + * Returns -1 on failure, 0 on success. + */ +int flatcc_builder_custom_init(flatcc_builder_t *B, + flatcc_builder_emit_fun *emit, void *emit_context, + flatcc_builder_alloc_fun *alloc, void *alloc_context); + +/* + * Returns (flatcc_emitter_t *) if the default context is used. + * Other emitter might have null contexts. + */ +void *flatcc_builder_get_emit_context(flatcc_builder_t *B); + +/** + * Prepares builder for a new build. The emitter is not told when a + * buffer is finished or when a new begins, and must be told so + * separately. Allocated buffers will be zeroed, but may optionally be + * reduced to their defaults (signalled by reallocating each non-empty + * buffer to a single byte). General settings are cleared optionally, + * such as cache flushing. Buffer specific settings such as buffer + * identifier are always cleared. + * + * Returns -1 if allocator complains during buffer reduction, 0 on + * success. + */ +int flatcc_builder_custom_reset(flatcc_builder_t *B, + int reduce_buffers, int set_defaults); + +/* + * Same as `flatcc_builder_custom_reset` with default arguments + * where buffers are not reduced and default settings are not reset. + */ +int flatcc_builder_reset(flatcc_builder_t *B); + +/** + * Deallocates all memory by calling allocate with a zero size request + * on each buffer, then zeroing the builder structure itself. + */ +void flatcc_builder_clear(flatcc_builder_t *B); + +/** + * Allocates to next higher power of 2 using system realloc and ignores + * `alloc_context`. Only reduces size if a small subsequent increase in + * size would not trigger a reallocation. `alloc_type` is used to + * set minimum sizes. Hash tables are allocated to the exact requested + * size. See also `alloc_fun`. + */ +int flatcc_builder_default_alloc(void *alloc_context, + flatcc_iovec_t *b, size_t request, int zero_fill, int alloc_type); + +/** + * If non-zero, the vtable cache will get flushed whenever it reaches + * the given limit at a point in time where more space is needed. The + * limit is not exact as it is only tested when reallocation is + * required. + */ +void flatcc_builder_set_vtable_cache_limit(flatcc_builder_t *B, size_t size); + +/** + * Manual flushing of vtable for long running tasks. Mostly used + * internally to deal with nested buffers. + */ +void flatcc_builder_flush_vtable_cache(flatcc_builder_t *B); + +/** + * Low-level support function to aid in constructing nested buffers without + * allocation. Not for regular use. + * + * Call where `start_buffer` would have been placed when using + * `create_buffer` in a nested context. Save the return value on a stack + * as argument to `pop_buffer_alignment`. + * + * The call resets the current derived buffer alignment so the nested + * buffer will not be aligned to more than required. + * + * Often it will not be necessary to be so careful with alignment since + * the alignment cannot be invalid by failing to use push and pop, but + * for code generation it will ensure the correct result every time. + */ +uint16_t flatcc_builder_push_buffer_alignment(flatcc_builder_t *B); + +/** + * Low-level call. + * + * Call with the return value from push_buffer_alignment after a nested + * `create_buffer_call`. The alignments merge back up in the buffer + * hierarchy so the top level buffer gets the largest of all aligments. + */ +void flatcc_builder_pop_buffer_alignment(flatcc_builder_t *B, uint16_t buffer_align); + +/** + * This value may be of interest when the buffer has been ended, for + * example when subsequently allocating memory for the buffer to ensure + * that memory is properly aligned. + */ +uint16_t flatcc_builder_get_buffer_alignment(flatcc_builder_t *B); + +/** + * Level 0 means no buffer is started, otherwise it increments with + * start calls and decrements with end calls (approximately for + * optimized operations such as table vectors). + * + * If `max_level` has been set, `get_level` always returns a value <= + * `max_level` provided no start call has failed. + * + * Level continues to increment inside nested buffers. + */ +int flatcc_builder_get_level(flatcc_builder_t *B); + +/** + * Setting the max level triggers a failure on start of new nestings + * when the level is reached. May be used to protect recursive descend + * parsers etc. or later buffer readers. + * + * The builder itself is not sensitive to depth, and the allocator is a + * better way to protect resource abuse. + * + * `max_level` is not reset inside nested buffers. + */ +void flatcc_builder_set_max_level(flatcc_builder_t *B, int level); + +/** + * By default ordinary data such as tables are placed in front of + * earlier produced content and vtables are placed at the very end thus + * clustering vtables together. This can be disabled so all content is + * placed in front. Nested buffers ignores this setting because they can + * only place content in front because they cannot blend with the + * containing buffers content. Clustering could be more cache friendly + * and also enables pre-shipping of the vtables during transmission. + */ +void flatcc_builder_set_vtable_clustering(flatcc_builder_t *B, int enable); + +/** + * Sets a new user supplied refmap which maps source pointers to + * references and returns the old refmap, or null. It is also + * possible to disable an existing refmap by setting a null + * refmap. + * + * A clone or pick operation may use this map when present, + * depending on the data type. If a hit is found, the stored + * reference will be used instead of performing a new clone or + * pick operation. It is also possible to manually populate the + * refmap. Note that the builder does not have a concept of + * clone or pick - these are higher level recursive operations + * to add data from one buffer to another - but such code may + * rely on the builder to provide the current refmap during + * recursive operations. For this reason, the builder makes no + * calls to the refmap interface on its own - it just stores the + * current refmap such that recursive operations can find it. + * + * Refmaps MUST be reset, replaced or disabled if a source + * pointer may be reused for different purposes - for example if + * repeatedly reading FlatBuffers into the same memory buffer + * and performing a clone into a buffer under construction. + * Refmaps may also be replaced if the same object is to be + * cloned several times keeping the internal DAG structure + * intact with every new clone being an independent object. + * + * Refmaps must also be replaced or disabled prior to starting a + * nested buffer and after stopping it, or when cloning a object + * as a nested root. THIS IS VERY EASY TO GET WRONG! The + * builder does a lot of bookkeeping for nested buffers but not + * in this case. Shared references may happen and they WILL fail + * verification and they WILL break when copying out a nested + * buffer to somewhere else. The user_frame stack may be used + * for pushing refmaps, but often user codes recursive stack + * will work just as well. + * + * It is entirely optional to use refmaps when cloning - they + * preserve DAG structure and may speed up operations or slow + * them down, depending on the source material. + * + * Refmaps may consume a lot of space when large offset vectors + * are cloned when these do not have significant shared + * references. They may also be very cheap to use without any + * dynamic allocation when objects are small and have at most a + * few references. + * + * Refmaps only support init, insert, find, reset, clear but not + * delete. There is a standard implementation in the runtime + * source tree but it can easily be replaced compile time and it + * may also be left out if unused. The builder wraps reset, insert, + * and find so the user does not have to check if a refmap is + * present but other operations must be done direcly on the + * refmap. + * + * The builder wrapped refmap operations are valid on a null + * refmap which will find nothing and insert nothing. + * + * The builder will reset the refmap during a builder reset and + * clear the refmap during a builder clear operation. If the + * refmap goes out of scope before that happens it is important + * to call set_refmap with null and manually clear the refmap. + */ +static inline flatcc_refmap_t *flatcc_builder_set_refmap(flatcc_builder_t *B, flatcc_refmap_t *refmap) +{ + flatcc_refmap_t *refmap_old; + + refmap_old = B->refmap; + B->refmap = refmap; + return refmap_old; +} + +/* Retrieves the current refmap, or null. */ +static inline flatcc_refmap_t *flatcc_builder_get_refmap(flatcc_builder_t *B) +{ + return B->refmap; +} + +/* Finds a reference, or a null reference if no refmap is active. * */ +static inline flatcc_builder_ref_t flatcc_builder_refmap_find(flatcc_builder_t *B, const void *src) +{ + return B->refmap ? flatcc_refmap_find(B->refmap, src) : flatcc_refmap_not_found; +} + +/* + * Inserts into the current refmap with the inseted ref upon + * upon success, or not_found on failure (default 0), or just + * returns ref if refmap is absent. + * + * Note that if an existing item exists, the ref is replaced + * and the new, not the old, ref is returned. + */ +static inline flatcc_builder_ref_t flatcc_builder_refmap_insert(flatcc_builder_t *B, const void *src, flatcc_builder_ref_t ref) +{ + return B->refmap ? flatcc_refmap_insert(B->refmap, src, ref) : ref; +} + +static inline void flatcc_builder_refmap_reset(flatcc_builder_t *B) +{ + if (B->refmap) flatcc_refmap_reset(B->refmap); +} + + +enum flatcc_builder_buffer_flags { + flatcc_builder_is_nested = 1, + flatcc_builder_with_size = 2, +}; + +/** + * An alternative to start buffer, start struct/table ... end buffer. + * + * This call is mostly of interest as a means to quicly create a zero + * allocation top-level buffer header following a call to create_struct, + * or to create_vtable/create_table. For that, it is quite simple to + * use. For general buffer construction without allocation, more care is + * needed, as discussed below. + * + * If the content is created with `start/end_table` calls, or similar, + * it is better to use `start/end_buffer` since stack allocation is used + * anyway. + * + * The buffer alignment must be provided manually as it is not derived + * from constructed content, unlike `start/end_buffer`. Typically + * `align` would be same argument as provided to `create_struct`. + * `get_buffer_alignment` may also used (note: `get_buffer_alignment` + * may return different after the call because it will be updated with + * the `block_align` argument to `create_buffer` but that is ok). + * + * The buffer may be constructed as a nested buffer with the `is_nested + * = 1` flag. As a nested buffer a ubyte vector header is placed before + * the aligned buffer header. A top-level buffer will normally have + * flags set to 0. + * + * A top-level buffer may also be constructed with the `with_size = 2` + * flag for top level buffers. It adds a size prefix similar to + * `is_nested` but the size is part of the aligned buffer. A size + * prefixed top level buffer must be accessed with a size prefix aware + * reader, or the buffer given to a standard reader must point to after + * the size field while keeping the buffer aligned to the size field + * (this will depend on the readers API which may be an arbitrary other + * language). + * + * If the `with_size` is used with the `is_nested` flag, the size is + * added as usual and all fields remain aligned as before, but padding + * is adjusted to ensure the buffer is aligned to the size field so + * that, for example, the nested buffer with size can safely be copied + * to a new memory buffer for consumption. + * + * Generally, references may only be used within the same buffer + * context. With `create_buffer` this becomes less precise. The rule + * here is that anything that would be valid with start/end_buffer + * nestings is also valid when removing the `start_buffer` call and + * replacing `end_buffer` with `create_buffer`. + * + * Note the additional burden of tracking buffer alignment manually - + * To help with this use `push_buffer_alignment` where `start_buffer` + * would have been placed, and `pop_buffer_alignment after the + * `create_buffer` call, and use `get_buffer_alignemnt` as described + * above. + * + * `create_buffer` is not suitable as a container for buffers created + * with `start/end_buffer` as these make assumptions about context that + * create buffer does not provide. Also, there is no point in doing so, + * since the idea of `create_buffer` is to avoid allocation in the first + * place. + */ +flatcc_builder_ref_t flatcc_builder_create_buffer(flatcc_builder_t *B, + const char identifier[FLATBUFFERS_IDENTIFIER_SIZE], + uint16_t block_align, + flatcc_builder_ref_t ref, uint16_t align, int flags); + +/** + * Creates a struct within the current buffer without using any + * allocation. + * + * The struct should be used as a root in the `end_buffer` call or as a + * union value as there are no other ways to use struct while conforming + * to the FlatBuffer format - noting that tables embed structs in their + * own data area except in union fields. + * + * The struct should be in little endian format and follow the usual + * FlatBuffers alignment rules, although this API won't care about what + * is being stored. + * + * May also be used to simply emit a struct through the emitter + * interface without being in a buffer and without being a valid + * FlatBuffer. + */ +flatcc_builder_ref_t flatcc_builder_create_struct(flatcc_builder_t *B, + const void *data, size_t size, uint16_t align); + +/** + * Starts a struct and returns a pointer that should be used immediately + * to fill in the struct in protocol endian format, and when done, + * `end_struct` should be called. The returned reference should be used + * as argument to `end_buffer` or as a union value. See also + * `create_struct`. + */ +void *flatcc_builder_start_struct(flatcc_builder_t *B, + size_t size, uint16_t align); + +/** + * Return a pointer also returned at start struct, e.g. for endian + * conversion. + */ +void *flatcc_builder_struct_edit(flatcc_builder_t *B); + +/** + * Emits the struct started by `start_struct` and returns a reference to + * be used as root in an enclosing `end_buffer` call or as a union + * value. As mentioned in `create_struct`, these can also be used more + * freely, but not while being conformant FlatBuffers. + */ +flatcc_builder_ref_t flatcc_builder_end_struct(flatcc_builder_t *B); + +/** + * The buffer always aligns to at least the offset size (typically 4) + * and the internal alignment requirements of the buffer content which + * is derived as content is added. + * + * In addition, block_align can be specified. This ensures the resulting + * buffer is at least aligned to the block size and that the total size + * is zero padded to fill a block multiple if necessary. Because the + * emitter operates on a virtual address range before the full buffer is + * aligned, it may have to make assumptions based on that: For example, + * it may be processing encryption blocks in the fly, and the resulting + * buffer should be aligned to the encryption block size, even if the + * content is just a byte aligned struct. Block align helps ensure this. + * If the block align as 1 there will be no attempt to zero pad at the + * end, but the content may still warrant padding after the header. End + * padding is only needed with clustered vtables (which is the default). + * + * `block_align` is allowed to be 0 meaning it will inherit from parent if + * present, and otherwise it defaults to 1. + * + * The identifier may be null, and it may optionally be set later with + * `set_identifier` before the `end_buffer` call. + * + * General note: + * + * Only references returned with this buffer as current (i.e. last + * unended buffer) can be stored in other objects (tables, offset + * vectors) also belonging to this buffer, or used as the root argument + * to `end_buffer`. A reference may be stored at most once, and unused + * references will result in buffer garbage. All calls must be balanced + * around the respective start / end operations, but may otherwise nest + * freely, including nested buffers. Nested buffers are supposed to be + * stored in a table offset field to comply with FlatBuffers, but the + * API does not place any restrictions on where references are stored, + * as long as they are indicated as offset fields. + * + * All alignment in all API calls must be between 1 and 256 and must be a + * power of 2. This is not checked. Only if explicitly documented can it + * also be 0 for a default value. + * + * `flags` can be `with_size` but `is_nested` is derived from context + * see also `create_buffer`. + */ +int flatcc_builder_start_buffer(flatcc_builder_t *B, + const char identifier[FLATBUFFERS_IDENTIFIER_SIZE], + uint16_t block_align, int flags); + +/** + * The root object should be a struct or a table to conform to the + * FlatBuffers format, but technically it can also be a vector or a + * string, or even a child buffer (which is also vector as seen by the + * buffer). The object must be created within the current buffer + * context, that is, while the current buffer is the deepest nested + * buffer on the stack. + */ +flatcc_builder_ref_t flatcc_builder_end_buffer(flatcc_builder_t *B, flatcc_builder_ref_t root); + +/** + * The embed buffer is mostly intended to add an existing buffer as a + * nested buffer. The buffer will be wrapped in a ubyte vector such that + * the buffer is aligned at vector start, after the size field. + * + * If `align` is 0 it will default to 8 so that all FlatBuffer numeric + * types will be readable. NOTE: generally do not count on align 0 being + * valid or even checked by the API, but in this case it may be + * difficult to know the internal buffer alignment, and 1 would be the wrong + * choice. + * + * If `block_align` is set (non-zero), the buffer is placed in an isolated + * block multiple. This may cost up to almost 2 block sizes in padding. + * If the `block_align` argument is 0, it inherits from the parent + * buffer block_size, or defaults to 1. + * + * The `align` argument must be set to respect the buffers internal + * alignment requirements, but if the buffer is smaller it will not be + * padded to isolate the buffer. For example a buffer of with + * `align = 64` and `size = 65` may share its last 64 byte block with + * other content, but not if `block_align = 64`. + * + * Because the ubyte size field is not, by default, part of the aligned + * buffer, significant space can be wasted if multiple blocks are added + * in sequence with a large block size. + * + * In most cases the distinction between the two alignments is not + * important, but it allows separate configuration of block internal + * alignment and block size, which can be important for auto-generated + * code that may know the alignment of the buffer, but not the users + * operational requirements. + * + * If the buffer is embedded without a parent buffer, it will simply + * emit the buffer through the emit interface, but may also add padding + * up to block alignment. At top-level there will be no size field + * header. + * + * If `with_size` flag is set, the buffer is aligned to size field and + * the above note about padding space no longer applies. The size field + * is added regardless. The `is_nested` flag has no effect since it is + * impplied. + */ +flatcc_builder_ref_t flatcc_builder_embed_buffer(flatcc_builder_t *B, + uint16_t block_align, + const void *data, size_t size, uint16_t align, int flags); + +/** + * Applies to the innermost open buffer. The identifier may be null or + * contain all zero. Overrides any identifier given to the start buffer + * call. + */ +void flatcc_builder_set_identifier(flatcc_builder_t *B, + const char identifier[FLATBUFFERS_IDENTIFIER_SIZE]); + +enum flatcc_builder_type { + flatcc_builder_empty = 0, + flatcc_builder_buffer, + flatcc_builder_struct, + flatcc_builder_table, + flatcc_builder_vector, + flatcc_builder_offset_vector, + flatcc_builder_string, + flatcc_builder_union_vector +}; + +/** + * Returns the object type currently on the stack, for example if + * needing to decide how to close a buffer. Because a table is + * automatically added when starting a table buffer, + * `flatcc_builder_table_buffer` should not normally be seen and the level + * should be 2 before when closing a top-level table buffer, and 0 + * after. A `flatcc_builder_struct_buffer` will be visible at level 1. + * + */ +enum flatcc_builder_type flatcc_builder_get_type(flatcc_builder_t *B); + +/** + * Similar to `get_type` but for a specific level. `get_type_at(B, 1)` + * will return `flatcc_builder_table_buffer` if this is the root buffer + * type. get_type_at(B, 0) is always `flatcc_builder_empty` and so are any + * level above `get_level`. + */ +enum flatcc_builder_type flatcc_builder_get_type_at(flatcc_builder_t *B, int level); + +/** + * The user stack is available for custom data. It may be used as + * a simple stack by extending or reducing the inner-most frame. + * + * A frame has a size and a location on the user stack. Entering + * a frame ensures the start is aligned to sizeof(size_t) and + * ensures the requested space is available without reallocation. + * When exiting a frame, the previous frame is restored. + * + * A user frame works completely independently of the builders + * frame stack for tracking tables vectors etc. and does not have + * to be completely at exit, but obviously it is not valid to + * exit more often the entered. + * + * The frame is zeroed when entered. + * + * Returns a non-zero handle to the user frame upon success or + * 0 on allocation failure. + */ +size_t flatcc_builder_enter_user_frame(flatcc_builder_t *B, size_t size); + +/** + * Makes the parent user frame current, if any. It is not valid to call + * if there isn't any current frame. Returns handle to parent frame if + * any, or 0. + */ +size_t flatcc_builder_exit_user_frame(flatcc_builder_t *B); + +/** + * Exits the frame represented by the given handle. All more + * recently entered frames will also be exited. Returns the parent + * frame handle if any, or 0. + */ +size_t flatcc_builder_exit_user_frame_at(flatcc_builder_t *B, size_t handle); + +/** + * Returns a non-zero handle to the current inner-most user frame if + * any, or 0. + */ +size_t flatcc_builder_get_current_user_frame(flatcc_builder_t *B); + +/* + * Returns a pointer to the user frame at the given handle. Any active + * frame can be accessed in this manner but the pointer is invalidated + * by user frame enter and exit operations. + */ +void *flatcc_builder_get_user_frame_ptr(flatcc_builder_t *B, size_t handle); + +/** + * Returns the size of the buffer and the logical start and end address + * of with respect to the emitters address range. `end` - `start` also + * yields the size. During construction `size` is the emitted number of + * bytes and after buffer close it is the actual buffer size - by then + * the start is also the return value of close buffer. End marks the end + * of the virtual table cluster block. + * + * NOTE: there is no guarantee that all vtables end up in the cluster + * block if there is placed a limit on the vtable size, or if nested + * buffers are being used. On the other hand, if these conditions are + * met, it is guaranteed that all vtables are present if the vtable + * block is available (this depends on external transmission - the + * vtables are always emitted before tables using them). In all cases + * the vtables will behave as valid vtables in a flatbuffer. + */ +size_t flatcc_builder_get_buffer_size(flatcc_builder_t *B); + +/** + * Returns the reference to the start of the emitter buffer so far, or + * in total after buffer end, in the virtual address range used + * by the emitter. Start is also returned by buffer end. + */ +flatcc_builder_ref_t flatcc_builder_get_buffer_start(flatcc_builder_t *B); + +/** + * Returns the reference to the end of buffer emitted so far. When + * clustering vtables, this is the end of tables, or after buffer end, + * also zero padding if block aligned. If clustering is disabled, this + * method will return 0 as the buffer only grows down then. + */ +flatcc_builder_ref_t flatcc_builder_get_buffer_mark(flatcc_builder_t *B); + +/** + * Creates the vtable in the current buffer context, somewhat similar to + * how create_vector operates. Each call results in a new table even if + * an identical has already been emitted. + * + * Also consider `create_cached_vtable` which will reuse existing + * vtables. + * + * This is low-low-level function intended to support + * `create_cached_vtable` or equivalent, and `create_table`, both of + * which are normally used indirectly via `start_table`, `table_add`, + * `table_add_offset`..., `table_end`. + * + * Creates a vtable as a verbatim copy. This means the vtable must + * include the header fields containing the vtable size and the table + * size in little endian voffset_t encoding followed by the vtable + * entries in same encoding. + * + * The function may be used to copy vtables from other other buffers + * since they are directly transferable. + * + * The returned reference is actually the emitted location + 1. This + * ensures the vtable is not mistaken for error because 0 is a valid + * vtable reference. `create_table` is aware of this and substracts one + * before computing the final offset relative to the table. This also + * means vtable references are uniquely identifiable by having the + * lowest bit set. + * + * vtable references may be reused within the same buffer, not any + * parent or other related buffer (technically this is possible though, + * as long as it is within same builder context, but it will not construct + * valid FlatBuffers because the buffer cannot be extracted in isolation). + */ +flatcc_builder_vt_ref_t flatcc_builder_create_vtable(flatcc_builder_t *B, + const flatbuffers_voffset_t *vt, + flatbuffers_voffset_t vt_size); + +/** + * Support function to `create_vtable`. See also the uncached version + * `create_vtable`. + * + * Looks up the constructed vtable on the vs stack too see if it matches + * a cached entry. If not, it emits a new vtable either at the end if + * top-level and clustering is enabled, or at the front (always for + * nested buffers). + * + * If the same vtable was already emitted in a different buffer, but not + * in the current buffer, the cache entry will be reused, but a new + * table will be emitted the first it happens in the same table. + * + * The returned reference is + 1 relative to the emitted address range + * to identify it as a vtable and to avoid mistaking the valid 0 + * reference for an error (clustered vtables tend to start at the end at + * the virtual address 0, and up). + * + * The hash function can be chosen arbitrarily but may result in + * duplicate emitted vtables if different hash functions are being used + * concurrently, such as mixing the default used by `start/end table` + * with a custom function (this is not incorrect, it only increases the + * buffer size and cache pressure). + * + * If a vtable has a unique ID by other means than hashing the content, + * such as an integer id, and offset into another buffer, or a pointer, + * a good hash may be multiplication by a 32-bit prime number. The hash + * table is not very sensitive to collissions as it uses externally + * chained hashing with move to front semantics. + */ +flatcc_builder_vt_ref_t flatcc_builder_create_cached_vtable(flatcc_builder_t *B, + const flatbuffers_voffset_t *vt, + flatbuffers_voffset_t vt_size, uint32_t vt_hash); + +/* + * Based on Knuth's prime multiplier. + * + * This is an incremental hash that is called with id and size of each + * non-empty field, and finally with the two vtable header fields + * when vtables are constructed via `table_add/table_add_offset`. + * + */ +#ifndef FLATCC_SLOW_MUL +#ifndef FLATCC_BUILDER_INIT_VT_HASH +#define FLATCC_BUILDER_INIT_VT_HASH(hash) { (hash) = (uint32_t)0x2f693b52UL; } +#endif +#ifndef FLATCC_BUILDER_UPDATE_VT_HASH +#define FLATCC_BUILDER_UPDATE_VT_HASH(hash, id, offset) \ + { (hash) = (((((uint32_t)id ^ (hash)) * (uint32_t)2654435761UL)\ + ^ (uint32_t)(offset)) * (uint32_t)2654435761UL); } +#endif +#ifndef FLATCC_BUILDER_BUCKET_VT_HASH +#define FLATCC_BUILDER_BUCKET_VT_HASH(hash, width) (((uint32_t)(hash)) >> (32 - (width))) +#endif +#endif + +/* + * By default we use Bernsteins hash as fallback if multiplication is slow. + * + * This just have to be simple, fast, and work on devices without fast + * multiplication. We are not too sensitive to collisions. Feel free to + * experiment and replace. + */ +#ifndef FLATCC_BUILDER_INIT_VT_HASH +#define FLATCC_BUILDER_INIT_VT_HASH(hash) { (hash) = 5381; } +#endif +#ifndef FLATCC_BUILDER_UPDATE_VT_HASH +#define FLATCC_BUILDER_UPDATE_VT_HASH(hash, id, offset) \ + { (hash) = ((((hash) << 5) ^ (id)) << 5) ^ (offset); } +#endif +#ifndef FLATCC_BUILDER_BUCKET_VT_HASH +#define FLATCC_BUILDER_BUCKET_VT_HASH(hash, width) (((1 << (width)) - 1) & (hash)) +#endif + + + +/** + * Normally use `start_table` instead of this call. + * + * This is a low-level call only intended for high-performance + * applications that repeatedly churn about similar tables of known + * layout, or as a support layer for other builders that maintain their + * own allocation rather than using the stack of this builder. + * + * Creates a table from an already emitted vtable, actual data that is + * properly aligned relative to data start and in little endian + * encoding. Unlike structs, tables can have offset fields. These must + * be stored as flatcc_builder_ref_t types (which have uoffset_t size) as + * returned by the api in native encoding. The `offsets` table contain + * voffsets relative to `data` start (this is different from how vtables + * store offsets because they are relative to a table header). The + * `offsets` table is only used temporarily to translate the stored + * references and is not part of final buffer content. `offsets` may be + * null if `offset_count` is 0. `align` should be the highest aligned + * field in the table, but `size` need not be a multiple of `align`. + * Aside from endian encoding, the vtable must record a table size equal + * to `size + sizeof(flatbuffers_uoffset_t)` because it includes the + * table header field size. The vtable is not accessed by this call (nor + * is it available). Unlike other references, the vtable reference may + * be shared between tables in the same buffer (not with any related + * buffer such as a parent buffer). + * + * The operation will not use any allocation, but will update the + * alignment of the containing buffer if any. + * + * Note: unlike other create calls, except `create_offset_vector`, + * the source data is modified in order to translate references intok + * offsets before emitting the table. + */ +flatcc_builder_ref_t flatcc_builder_create_table(flatcc_builder_t *B, + const void *data, size_t size, uint16_t align, + flatbuffers_voffset_t *offsets, int offset_count, + flatcc_builder_vt_ref_t vt_ref); + +/** + * Starts a table, typically following a start_buffer call as an + * alternative to starting a struct, or to create table fields to be + * stored in a parent table, or in an offset vector. + * A number of `table_add` and table_add_offset` call may be placed + * before the `end_table` call. Struct fields should NOT use `struct` + * related call (because table structs are in-place), rather they should + * use the `table_add` call with the appropriate size and alignment. + * + * A table, like other reference returning calls, may also be started + * outside a buffer if the buffer header and alignment is of no + * interest to the application, for example as part of an externally + * built buffer. + * + * `count` must be larger than the largest id used for this table + * instance. Normally it is set to the number of fields defined in the + * schema, but it may be less if memory is constrained and only few + * fields with low valued id's are in use. The count can extended later + * with `reserve_table` if necessary. `count` may be also be set to a + * large enough value such as FLATBUFFERS_ID_MAX + 1 if memory is not a + * concern (reserves about twice the maximum vtable size to track the + * current vtable and voffsets where references must be translated to + * offsets at table end). `count` may be zero if for example + * `reserve_table` is being used. + * + * Returns -1 on error, 0 on success. + */ +int flatcc_builder_start_table(flatcc_builder_t *B, int count); + +/** + * Call before adding a field with an id that is not below the count set + * at table start. Not needed in most cases. For performance reasons + * the builder does not check all bounds all the the time, but the user + * can do so if memory constraints prevent start_table from using a + * conservative value. See also `table_start`. + * + * Note: this call has absolutely no effect on the table layout, it just + * prevents internal buffer overruns. + * + * Returns -1 on error, 0 on success. + */ +int flatcc_builder_reserve_table(flatcc_builder_t *B, int count); + +/** + * Completes the table constructed on the internal stack including + * emitting a vtable, or finding a matching vtable that has already been + * emitted to the same buffer. (Vtables cannot be shared between + * buffers, but they can between tables of the same buffer). + * + * Note: there is a considerable, but necessary, amount of bookkeeping + * involved in constructing tables. The `create_table` call is much + * faster, but it also expects a lot of work to be done already. + * + * Tables can be created with no fields added. This will result in an + * empty vtable and a table with just a vtable reference. If a table is + * used as a sub-table, such a table might also not be stored at all, + * but we do not return a special reference for that, nor do we provide + * and option to not create the table in this case. This may be + * interpreted as the difference between a null table (not stored in + * parent), and an empty table with a unique offset (and thus identity) + * different from other empty tables. + */ +flatcc_builder_ref_t flatcc_builder_end_table(flatcc_builder_t *B); + +/** + * Optionally this method can be called just before `flatcc_builder_end_table` + * to verify that all required fields have been set. + * Each entry is a table field id. + * + * Union fields should use the type field when checking for presence and + * may also want to check the soundness of the union field overall using + * `check_union_field` with the id one higher than the type field id. + * + * This funcion is typically called by an assertion in generated builder + * interfaces while release builds may want to avoid this performance + * overhead. + * + * Returns 1 if all fields are matched, 0 otherwise. + */ +int flatcc_builder_check_required(flatcc_builder_t *B, const flatbuffers_voffset_t *required, int count); + +/** + * Same as `check_required` when called with a single element. + * + * Typically used when direct calls are more convenient than building an + * array first. Useful when dealing with untrusted intput such as parsed + * text from an external source. + */ +int flatcc_builder_check_required_field(flatcc_builder_t *B, flatbuffers_voffset_t id); + +/** + * Checks that a union field is valid. + * + * The criteria is: + * + * If the type field is not present (at id - 1), or it holds a zero value, + * then the table field (at id) must be present. + * + * Generated builder code may be able to enforce valid unions without + * this check by setting both type and table together, but e.g. parsers + * may receive the type and the table independently and then it makes + * sense to validate the union fields before table completion. + * + * Note that an absent union field is perfectly valid. If a union is + * required, the type field (id - 1), should be checked separately + * while the table field should only be checked here because it can + * (and must) be absent when the type is NONE (= 0). + */ +int flatcc_builder_check_union_field(flatcc_builder_t *B, flatbuffers_voffset_t id); + +/** + * A struct, enum or scalar added should be stored in little endian in + * the return pointer location. The pointer is short lived and will + * not necessarily survive other builder calls. + * + * A union type field can also be set using this call. In fact, this is + * the only way to deal with unions via this API. Consequently, it is + * the users repsonsibility to ensure the appropriate type is added + * at the next higher id. + * + * Null and default values: + * + * FlatBuffers does not officially provide an option for null values + * because it does not distinguish between default values and values + * that are not present. At this api level, we do not deal with defaults + * at all. Callee should test the stored value against the default value + * and only add the field if it does not match the default. This only + * applies to scalar and enum values. Structs cannot have defaults so + * their absence means null, and strings, vectors and subtables do have + * natural null values different from the empty object and empty objects + * with different identity is also possible. + * + * To handle Null for scalars, the following approach is recommended: + * + * Provide a schema-specific `add` operation that only calls this + * low-level add method if the default does not match, and also provide + * another `set` operation that always stores the value, regardless of + * default. For most readers this will be transparent, except for extra + * space used, but for Null aware readers, these can support operations + * to test for Null/default/other value while still supporting the + * normal read operation that returns default when a value is absent + * (i.e. Null). + * + * It is valid to call with a size of 0 - the effect being adding the + * vtable entry. The call may also be dropped in this case to reduce + * the vtable size - the difference will be in null detection. + */ +void *flatcc_builder_table_add(flatcc_builder_t *B, int id, size_t size, uint16_t align); + +/** + * Returns a pointer to the buffer holding the last field added. The + * size argument must match the field size added. May, for example, be + * used to perform endian conversion after initially updating field + * as a native struct. Must be called before the table is ended. + */ +void *flatcc_builder_table_edit(flatcc_builder_t *B, size_t size); + +/** + * Similar to `table_add` but copies source data into the buffer before + * it is returned. Useful when adding a larger struct already encoded in + * little endian. + */ +void *flatcc_builder_table_add_copy(flatcc_builder_t *B, int id, const void *data, size_t size, uint16_t align); + +/** + * Add a string, vector, or sub-table depending on the type if the + * field identifier. The offset ref obtained when the field object was + * closed should be stored as is in the given pointer. The pointer + * is only valid short term, so create the object before calling + * add to table, but the owner table can be started earlier. Never mix + * refs from nested buffers with parent buffers. + * + * Also uses this method to add nested buffers. A nested buffer is + * simple a buffer created while another buffer is open. The buffer + * close operation provides the necessary reference. + * + * When the table closes, all references get converted into offsets. + * Before that point, it is not required that the offset is written + * to. + */ +flatcc_builder_ref_t *flatcc_builder_table_add_offset(flatcc_builder_t *B, int id); + +/* + * Adds a union type and reference in a single operation and returns 0 + * on success. Stores the type field at `id - 1` and the value at + * `id`. The `value` is a reference to a table, to a string, or to a + * standalone `struct` outside the table. + * + * If the type is 0, the value field must also be 0. + * + * Unions can also be added as separate calls to the type and the offset + * separately which can lead to better packing when the type is placed + * together will other small fields. + */ +int flatcc_builder_table_add_union(flatcc_builder_t *B, int id, + flatcc_builder_union_ref_t uref); + +/* + * Adds a union type vector and value vector in a single operations + * and returns 0 on success. + * + * If both the type and value vector is null, nothing is added. + * Otherwise both must be present and have the same length. + * + * Any 0 entry in the type vector must also have a 0 entry in + * the value vector. + */ +int flatcc_builder_table_add_union_vector(flatcc_builder_t *B, int id, + flatcc_builder_union_vec_ref_t uvref); +/** + * Creates a vector in a single operation using an externally supplied + * buffer. This completely bypasses the stack, but the size must be + * known and the content must be little endian. Do not use for strings + * and offset vectors. Other flatbuffer vectors could be used as a + * source, but the length prefix is not required. + * + * Set `max_count` to `FLATBUFFERS_COUNT_MAX(elem_size)` before a call + * to any string or vector operation to the get maximum safe vector + * size, or use (size_t)-1 if overflow is not a concern. + * + * The max count property is a global property that remains until + * explicitly changed. + * + * `max_count` is to prevent malicous or accidental overflow which is + * difficult to detect by multiplication alone, depending on the type + * sizes being used and having `max_count` thus avoids a division for + * every vector created. `max_count` does not guarantee a vector will + * fit in an empty buffer, it just ensures the internal size checks do + * not overflow. A safe, sane limit woud be max_count / 4 because that + * is half the maximum buffer size that can realistically be + * constructed, corresponding to a vector size of `UOFFSET_MAX / 4` + * which can always hold the vector in 1GB excluding the size field when + * sizeof(uoffset_t) = 4. + */ +flatcc_builder_ref_t flatcc_builder_create_vector(flatcc_builder_t *B, + const void *data, size_t count, size_t elem_size, uint16_t align, size_t max_count); + +/** + * Starts a vector on the stack. + * + * Do not use these calls for string or offset vectors, but do store + * scalars, enums and structs, always in little endian encoding. + * + * Use `extend_vector` subsequently to add zero, one or more elements + * at time. + * + * See `create_vector` for `max_count` argument (strings and offset + * vectors have a fixed element size and does not need this argument). + * + * Returns 0 on success. + */ +int flatcc_builder_start_vector(flatcc_builder_t *B, size_t elem_size, + uint16_t align, size_t max_count); + +/** + * Emits the vector constructed on the stack by start_vector. + * + * The vector may be accessed in the emitted stream using the returned + * reference, even if the containing buffer is still under construction. + * This may be useful for sorting. This api does not support sorting + * because offset vectors cannot read their references after emission, + * and while plain vectors could be sorted, it has been chosen that this + * task is better left as a separate processing step. Generated code can + * provide sorting functions that work on final in-memory buffers. + */ +flatcc_builder_ref_t flatcc_builder_end_vector(flatcc_builder_t *B); + +/** Returns the number of elements currently on the stack. */ +size_t flatcc_builder_vector_count(flatcc_builder_t *B); + +/** + * Returns a pointer ot the first vector element on stack, + * accessible up to the number of elements currently on stack. + */ +void *flatcc_builder_vector_edit(flatcc_builder_t *B); + +/** + * Returns a zero initialized buffer to a new region of the vector which + * is extended at the end. The buffer must be consumed before other api + * calls that may affect the stack, including `extend_vector`. + * + * Do not use for strings, offset or union vectors. May be used for nested + * buffers, but these have dedicated calls to provide better alignment. + */ +void *flatcc_builder_extend_vector(flatcc_builder_t *B, size_t count); + +/** + * A specialized `vector_extend` that pushes a single element. + * + * Returns the buffer holding a modifiable copy of the added content, + * or null on error. Note: for structs, care must be taken to ensure + * the source has been zero padded. For this reason it may be better to + * use extend(B, 1) and assign specific fields instead. + */ +void *flatcc_builder_vector_push(flatcc_builder_t *B, const void *data); + +/** + * Pushes multiple elements at a time. + * + * Returns the buffer holding a modifiable copy of the added content, + * or null on error. + */ +void *flatcc_builder_append_vector(flatcc_builder_t *B, const void *data, size_t count); + +/** + * Removes elements already added to vector that has not been ended. + * For example, a vector of parsed list may remove the trailing comma, + * or the vector may simply overallocate to get some temporary working + * space. The total vector size must never become negative. + * + * Returns -1 if the count as larger than current count, or 0 on success. + */ +int flatcc_builder_truncate_vector(flatcc_builder_t *B, size_t count); + +/* + * Similar to `create_vector` but with references that get translated + * into offsets. The references must, as usual, belong to the current + * buffer. Strings, scalar and struct vectors can emit directly without + * stack allocation, but offset vectors must translate the offsets + * and therefore need the temporary space. Thus, this function is + * roughly equivalent to to start, append, end offset vector. + * + * See also `flatcc_builder_create_offset_vector_direct`. + */ +flatcc_builder_ref_t flatcc_builder_create_offset_vector(flatcc_builder_t *B, + const flatcc_builder_ref_t *data, size_t count); + +/* + * NOTE: this call takes non-const source array of references + * and destroys the content. + * + * This is a faster version of `create_offset_vector` where the + * source references are destroyed. In return the vector can be + * emitted directly without passing over the stack. + */ +flatcc_builder_ref_t flatcc_builder_create_offset_vector_direct(flatcc_builder_t *B, + flatcc_builder_ref_t *data, size_t count); + + +/** + * Starts a vector holding offsets to tables or strings. Before + * completion it will hold `flatcc_builder_ref_t` references because the + * offset is not known until the vector start location is known, which + * depends to the final size, which for parsers is generally unknown. + */ +int flatcc_builder_start_offset_vector(flatcc_builder_t *B); + +/** + * Similar to `end_vector` but updates all stored references so they + * become offsets to the vector start. + */ +flatcc_builder_ref_t flatcc_builder_end_offset_vector(flatcc_builder_t *B); + +/** + * Same as `flatcc_builder_end_offset_vector` except null references are + * permitted when the corresponding `type` entry is 0 (the 'NONE' type). + * This makes it possible to build union vectors with less overhead when + * the `type` vector is already known. Use standand offset vector calls + * prior to this call. + */ +flatcc_builder_ref_t flatcc_builder_end_offset_vector_for_unions(flatcc_builder_t *B, + const flatcc_builder_utype_t *type); + +/** Returns the number of elements currently on the stack. */ +size_t flatcc_builder_offset_vector_count(flatcc_builder_t *B); + +/** + * Returns a pointer ot the first vector element on stack, + * accessible up to the number of elements currently on stack. + */ +void *flatcc_builder_offset_vector_edit(flatcc_builder_t *B); + +/** + * Similar to `extend_vector` but returns a buffer indexable as + * `flatcc_builder_ref_t` array. All elements must be set to a valid + * unique non-null reference, but truncate and extend may be used to + * perform edits. Unused references will leave garbage in the buffer. + * References should not originate from any other buffer than the + * current, including parents and nested buffers. It is valid to reuse + * references in DAG form when contained in the sammer, excluding any + * nested, sibling or parent buffers. + */ +flatcc_builder_ref_t *flatcc_builder_extend_offset_vector(flatcc_builder_t *B, size_t count); + +/** Similar to truncate_vector. */ +int flatcc_builder_truncate_offset_vector(flatcc_builder_t *B, size_t count); + +/** + * A specialized extend that pushes a single element. + * + * Returns the buffer holding a modifiable copy of the added content, + * or null on error. + */ +flatcc_builder_ref_t *flatcc_builder_offset_vector_push(flatcc_builder_t *B, + flatcc_builder_ref_t ref); + +/** + * Takes an array of refs as argument to do a multi push operation. + * + * Returns the buffer holding a modifiable copy of the added content, + * or null on error. + */ +flatcc_builder_ref_t *flatcc_builder_append_offset_vector(flatcc_builder_t *B, + const flatcc_builder_ref_t *refs, size_t count); + +/** + * All union vector operations are like offset vector operations, + * except they take a struct with a type and a reference rather than + * just a reference. The finished union vector is returned as a struct + * of two references, one for the type vector and one for the table offset + * vector. Each reference goes to a separate table field where the type + * offset vector id must be one larger than the type vector. + */ + +/** + * Creates a union vector which is in reality two vectors, a type vector + * and an offset vector. Both vectors references are returned. + */ +flatcc_builder_union_vec_ref_t flatcc_builder_create_union_vector(flatcc_builder_t *B, + const flatcc_builder_union_ref_t *urefs, size_t count); + +/* + * NOTE: this call takes non-const source array of references + * and destroys the content. The type array remains intact. + * + * This is a faster version of `create_union_vector` where the source + * references are destroyed and where the types are given in a separate + * array. In return the vector can be emitted directly without passing + * over the stack. + * + * Unlike `create_offset_vector` we do allow null references but only if + * the union type is NONE (0). + */ +flatcc_builder_union_vec_ref_t flatcc_builder_create_union_vector_direct(flatcc_builder_t *B, + const flatcc_builder_utype_t *types, flatcc_builder_ref_t *data, size_t count); + +/* + * Creates just the type vector part of a union vector. This is + * similar to a normal `create_vector` call except that the size + * and alignment are given implicitly. Can be used during + * cloning or similar operations where the types are all given + * but the values must be handled one by one as prescribed by + * the type. The values can be added separately as an offset vector. + */ +flatcc_builder_ref_t flatcc_builder_create_type_vector(flatcc_builder_t *B, + const flatcc_builder_utype_t *types, size_t count); + +/** + * Starts a vector holding types and offsets to tables or strings. Before + * completion it will hold `flatcc_builder_union_ref_t` references because the + * offset is not known until the vector start location is known, which + * depends to the final size, which for parsers is generally unknown, + * and also because the union type must be separated out into a separate + * vector. It would not be practicaly to push on two different vectors + * during construction. + */ +int flatcc_builder_start_union_vector(flatcc_builder_t *B); + +/** + * Similar to `end_vector` but updates all stored references so they + * become offsets to the vector start and splits the union references + * into a type vector and an offset vector. + */ +flatcc_builder_union_vec_ref_t flatcc_builder_end_union_vector(flatcc_builder_t *B); + +/** Returns the number of elements currently on the stack. */ +size_t flatcc_builder_union_vector_count(flatcc_builder_t *B); + +/** + * Returns a pointer ot the first vector element on stack, + * accessible up to the number of elements currently on stack. + */ +void *flatcc_builder_union_vector_edit(flatcc_builder_t *B); + +/** + * Similar to `extend_offset_vector` but returns a buffer indexable as a + * `flatcc_builder_union_ref_t` array. All elements must be set to a valid + * unique non-null reference with a valid union type to match, or it + * must be null with a zero union type. + */ +flatcc_builder_union_ref_t *flatcc_builder_extend_union_vector(flatcc_builder_t *B, size_t count); + +/** Similar to truncate_vector. */ +int flatcc_builder_truncate_union_vector(flatcc_builder_t *B, size_t count); + +/** + * A specialized extend that pushes a single element. + * + * Returns the buffer holding a modifiable copy of the added content, + * or null on error. + */ +flatcc_builder_union_ref_t *flatcc_builder_union_vector_push(flatcc_builder_t *B, + flatcc_builder_union_ref_t uref); + +/** + * Takes an array of union_refs as argument to do a multi push operation. + * + * Returns the buffer holding a modifiable copy of the added content, + * or null on error. + */ +flatcc_builder_union_ref_t *flatcc_builder_append_union_vector(flatcc_builder_t *B, + const flatcc_builder_union_ref_t *urefs, size_t count); + +/** + * Faster string operation that avoids temporary stack storage. The + * string is not required to be zero-terminated, but is expected + * (unchecked) to be utf-8. Embedded zeroes would be allowed but + * ubyte vectors should be used for that. The resulting string will + * have a zero termination added, not included in length. + */ +flatcc_builder_ref_t flatcc_builder_create_string(flatcc_builder_t *B, + const char *s, size_t len); + +/** `create_string` up to zero termination of source. */ +flatcc_builder_ref_t flatcc_builder_create_string_str(flatcc_builder_t *B, + const char *s); + +/** + * `create_string` up to zero termination or at most max_len of source. + * + * Note that like `strncpy` it will include `max_len` characters if + * the source is longer than `max_len`, but unlike `strncpy` it will + * always add zero termination. + */ +flatcc_builder_ref_t flatcc_builder_create_string_strn(flatcc_builder_t *B, const char *s, size_t max_len); + +/** + * Starts an empty string that can be extended subsequently. + * + * While the string is being created, it is guaranteed that there is + * always a null character after the end of the current string length. + * This also holds after `extend` and `append` operations. It is not + * allowed to modify the null character. + * + * Returns 0 on success. + */ +int flatcc_builder_start_string(flatcc_builder_t *B); + +/** + * Similar to `extend_vector` except for the buffer return type and a + * slight speed advantage. Strings are expected to contain utf-8 content + * but this isn't verified, and null characters would be accepted. The + * length is given in bytes. + * + * Appending too much, then truncating can be used to trim string + * escapes during parsing, or convert between unicode formats etc. + */ +char *flatcc_builder_extend_string(flatcc_builder_t *B, size_t len); + +/** + * Concatenes a length of string. If the string contains zeroes (which + * it formally shouldn't), they will be copied in. + * + * Returns the buffer holding a modifiable copy of the added content, + * or null on error. + */ +char *flatcc_builder_append_string(flatcc_builder_t *B, const char *s, size_t len); + +/** `append_string` up to zero termination of source. */ +char *flatcc_builder_append_string_str(flatcc_builder_t *B, const char *s); + +/** `append_string` up zero termination or at most max_len of source. */ +char *flatcc_builder_append_string_strn(flatcc_builder_t *B, const char *s, size_t max_len); + +/** + * Similar to `truncate_vector` available for consistency and a slight + * speed advantage. Reduces string by `len` bytes - it does not set + * the length. The resulting length must not become negative. Zero + * termination is not counted. + * + * Returns -1 of the length becomes negative, 0 on success. + */ +int flatcc_builder_truncate_string(flatcc_builder_t *B, size_t len); + +/** + * Similar to `end_vector` but adds a trailing zero not included + * in the length. The trailing zero is added regardless of whatever + * zero content may exist in the provided string (although it + * formally should not contain any). + */ +flatcc_builder_ref_t flatcc_builder_end_string(flatcc_builder_t *B); + +/** Returns the length of string currently on the stack. */ +size_t flatcc_builder_string_len(flatcc_builder_t *B); + +/** + * Returns a ponter to the start of the string + * accessible up the length of string currently on the stack. + */ +char *flatcc_builder_string_edit(flatcc_builder_t *B); + + +/* + * Only for use with the default emitter. + * + * Fast acces to small buffers from default emitter. + * + * Only valid for default emitters before `flatcc_builder_clear`. The + * return buffer is not valid after a call to `flatcc_builder_reset` or + * `flatcc_builder_clear`. + * + * Returns null if the buffer size is too large to a have a linear + * memory representation or if the emitter is not the default. A valid + * size is between half and a full emitter page size depending on vtable + * content. + * + * Non-default emitters must be accessed by means specific to the + * particular emitter. + * + * If `size_out` is not null, it is set to the buffer size, or 0 if + * operation failed. + * + * The returned buffer should NOT be deallocated explicitly. + * + * The buffer size is the size reported by `flatcc_builder_get_buffer_size`. + */ +void *flatcc_builder_get_direct_buffer(flatcc_builder_t *B, size_t *size_out); + +/* + * Only for use with the default emitter. + * + * Default finalizer that allocates a buffer from the default emitter. + * + * Returns null if memory could not be allocated or if the emitter is + * not the default. This is just a convenience method - there are many + * other possible ways to extract the result of the emitter depending on + * use case. + * + * If `size_out` is not null, it is set to the buffer size, or 0 if + * operation failed. + * + * The allocated buffer is aligned according to malloc which may not be + * sufficient in advanced cases - for that purpose + * `flatcc_builder_finalize_aligned_buffer` may be used. + * + * It may be worth calling `flatcc_builder_get_direct_buffer` first to see + * if the buffer is small enough to avoid copying. + * + * The returned buffer must be deallocated using `free`. + */ +void *flatcc_builder_finalize_buffer(flatcc_builder_t *B, size_t *size_out); + +/* + * Only for use with the default emitter. + * + * Similar to `flatcc_builder_finalize_buffer` but ensures the returned + * memory is aligned to the overall alignment required for the buffer. + * Often it is not necessary unless special operations rely on larger + * alignments than the stored scalars. + * + * If `size_out` is not null, it is set to the buffer size, or 0 if + * operation failed. + * + * The returned buffer must be deallocated using `aligned_free` which is + * implemented via `flatcc_flatbuffers.h`. `free` will usually work but + * is not portable to platforms without posix_memalign or C11 + * aligned_alloc support. + * + * NOTE: if a library might be compiled with a version of aligned_free + * that differs from the application using it, use + * `flatcc_builder_aligned_free` to make sure the correct deallocation + * function is used. + */ +void *flatcc_builder_finalize_aligned_buffer(flatcc_builder_t *B, size_t *size_out); + +/* + * A stable implementation of `aligned_alloc` that is not sensitive + * to the applications compile time flags. + */ +void *flatcc_builder_aligned_alloc(size_t alignment, size_t size); + +/* + * A stable implementation of `aligned_free` that is not sensitive + * to the applications compile time flags. + */ +void flatcc_builder_aligned_free(void *p); + +/* + * Same allocation as `flatcc_builder_finalize_buffer` returnes. Usually + * same as `malloc` but can redefined via macros. + */ +void *flatcc_builder_alloc(size_t size); + +/* + * A stable implementation of `free` when the default allocation + * methods have been redefined. + * + * Deallocates memory returned from `flatcc_builder_finalize_buffer`. + */ +void flatcc_builder_free(void *p); + +/* + * Only for use with the default emitter. + * + * Convenience method to copy buffer from default emitter. Forwards + * call to default emitter and returns input pointer, or null if + * the emitter is not default or of the given size is smaller than + * the buffer size. + * + * Note: the `size` argument is the target buffers capacity, not the + * flatcc_builders buffer size. + * + * Other emitters have custom interfaces for reaching their content. + */ +void *flatcc_builder_copy_buffer(flatcc_builder_t *B, void *buffer, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_BUILDER_H */ diff --git a/nostrdb/flatcc/flatcc_emitter.h b/nostrdb/flatcc/flatcc_emitter.h new file mode 100644 index 0000000000..11756f23f4 --- /dev/null +++ b/nostrdb/flatcc/flatcc_emitter.h @@ -0,0 +1,215 @@ +#ifndef FLATCC_EMITTER_H +#define FLATCC_EMITTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Default implementation of a flatbuilder emitter. + * + * This may be used as a starting point for more advanced emitters, + * for example writing completed pages to disk or network and + * the recycling those pages. + */ + +#include +#include + +#include "flatcc_types.h" +#include "flatcc_iov.h" +#include "flatcc_alloc.h" + +/* + * The buffer steadily grows during emission but the design allows for + * an extension where individual pages can recycled before the buffer + * is complete, for example because they have been transmitted. + * + * When done, the buffer can be cleared to free all memory, or reset to + * maintain an adaptive page pool for next buffer construction. + * + * Unlike an exponentially growing buffer, each buffer page remains + * stable in memory until reset, clear or recycle is called. + * + * Design notes for possible extensions: + * + * The buffer is a ring buffer marked by a front and a back page. The + * front and back may be the same page and may initially be absent. + * Anything outside these pages are unallocated pages for recycling. + * Any page between (but excluding) the front and back pages may be + * recycled by unlinking and relinking outside the front and back pages + * but then copy operations no longer makes sense. Each page stores the + * logical offset within the buffer but isn't otherwise used by the + * implemention - it might be used for network transmission. The buffer + * is not explicitly designed for multithreaded access but any page + * strictly between front and back is not touched unless recycled and in + * this case aligned allocation is useful to prevent cache line sharing. + */ + +/* + * Memory is allocated in fixed length page units - the first page is + * split between front and back so each get half the page size. If the + * size is a multiple of 128 then each page offset will be a multiple of + * 64, which may be useful for sequencing etc. + */ +#ifndef FLATCC_EMITTER_PAGE_SIZE +#define FLATCC_EMITTER_MAX_PAGE_SIZE 3000 +#define FLATCC_EMITTER_PAGE_MULTIPLE 64 +#define FLATCC_EMITTER_PAGE_SIZE ((FLATCC_EMITTER_MAX_PAGE_SIZE) &\ + ~(2 * (FLATCC_EMITTER_PAGE_MULTIPLE) - 1)) +#endif + +#ifndef FLATCC_EMITTER_ALLOC +#ifdef FLATCC_EMITTER_USE_ALIGNED_ALLOC +/* + * does not always provide aligned_alloc, so include whatever + * is required when enabling this feature. + */ +#define FLATCC_EMITTER_ALLOC(n) aligned_alloc(FLATCC_EMITTER_PAGE_MULTIPLE,\ + (((n) + FLATCC_EMITTER_PAGE_MULTIPLE - 1) & ~(FLATCC_EMITTER_PAGE_MULTIPLE - 1))) +#ifndef FLATCC_EMITTER_FREE +#define FLATCC_EMITTER_FREE(p) aligned_free(p) +#endif +#endif +#endif + +#ifndef FLATCC_EMITTER_ALLOC +#define FLATCC_EMITTER_ALLOC(n) FLATCC_ALLOC(n) +#endif +#ifndef FLATCC_EMITTER_FREE +#define FLATCC_EMITTER_FREE(p) FLATCC_FREE(p) +#endif + +typedef struct flatcc_emitter_page flatcc_emitter_page_t; +typedef struct flatcc_emitter flatcc_emitter_t; + +struct flatcc_emitter_page { + uint8_t page[FLATCC_EMITTER_PAGE_SIZE]; + flatcc_emitter_page_t *next; + flatcc_emitter_page_t *prev; + /* + * The offset is relative to page start, but not necessarily + * to any present content if part of front or back page, + * and undefined for unused pages. + */ + flatbuffers_soffset_t page_offset; +}; + +/* + * Must be allocated and zeroed externally, e.g. on the stack + * then provided as emit_context to the flatbuilder along + * with the `flatcc_emitter` function. + */ +struct flatcc_emitter { + flatcc_emitter_page_t *front, *back; + uint8_t *front_cursor; + size_t front_left; + uint8_t *back_cursor; + size_t back_left; + size_t used; + size_t capacity; + size_t used_average; +}; + +/* Optional helper to ensure emitter is zeroed initially. */ +static inline void flatcc_emitter_init(flatcc_emitter_t *E) +{ + memset(E, 0, sizeof(*E)); +} + +/* Deallocates all buffer memory making the emitter ready for next use. */ +void flatcc_emitter_clear(flatcc_emitter_t *E); + +/* + * Similar to `clear_flatcc_emitter` but heuristacally keeps some allocated + * memory between uses while gradually reducing peak allocations. + * For small buffers, a single page will remain available with no + * additional allocations or deallocations after first use. + */ +void flatcc_emitter_reset(flatcc_emitter_t *E); + +/* + * Helper function that allows a page between front and back to be + * recycled while the buffer is still being constructed - most likely as part + * of partial copy or transmission. Attempting to recycle front or back + * pages will result in an error. Recycling pages outside the + * front and back will be valid but pointless. After recycling and copy + * operations are no longer well-defined and should be replaced with + * whatever logic is recycling the pages. The reset operation + * automatically recycles all (remaining) pages when emission is + * complete. After recycling, the `flatcc_emitter_size` function will + * return as if recycle was not called, but will only represent the + * logical size, not the size of the active buffer. Because a recycled + * page is fully utilized, it is fairly easy to compensate for this if + * required. + * + * Returns 0 on success. + */ +int flatcc_emitter_recycle_page(flatcc_emitter_t *E, flatcc_emitter_page_t *p); + +/* + * The amount of data copied with `flatcc_emitter_copy_buffer` and related + * functions. Normally called at end of buffer construction but is + * always valid, as is the copy functions. The size is a direct + * function of the amount emitted data so the flatbuilder itself can + * also provide this information. + */ +static inline size_t flatcc_emitter_get_buffer_size(flatcc_emitter_t *E) +{ + return E->used; +} + +/* + * Returns buffer start iff the buffer fits on a single internal page. + * Only useful for fairly small buffers - about half the page size since + * one half of first page goes to vtables that likely use little space. + * Returns null if request could not be served. + * + * If `size_out` is not null, it is set to the buffer size, or 0 if + * operation failed. + */ +static inline void *flatcc_emitter_get_direct_buffer(flatcc_emitter_t *E, size_t *size_out) +{ + if (E->front == E->back) { + if (size_out) { + *size_out = E->used; + } + return E->front_cursor; + } + if (size_out) { + *size_out = 0; + } + return 0; +} + +/* + * Copies the internal flatcc_emitter representation to an externally + * provided linear buffer that must have size `flatcc_emitter_get_size`. + * + * If pages have been recycled, only the remaining pages will be copied + * and thus less data than what `flatcc_emitter_get_size` would suggest. It + * makes more sense to provide a customized copy operation when + * recycling pages. + * + * If the buffer is too small, nothing is copied, otherwise the + * full buffer is copied and the input buffer is returned. + */ +void *flatcc_emitter_copy_buffer(flatcc_emitter_t *E, void *buf, size_t size); + +/* + * The emitter interface function to the flatbuilder API. + * `emit_context` should be of type `flatcc_emitter_t` for this + * particular implementation. + * + * This function is compatible with the `flatbuilder_emit_fun` + * type defined in "flatbuilder.h". + */ +int flatcc_emitter(void *emit_context, + const flatcc_iovec_t *iov, int iov_count, + flatbuffers_soffset_t offset, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_EMITTER_H */ diff --git a/nostrdb/flatcc/flatcc_endian.h b/nostrdb/flatcc/flatcc_endian.h new file mode 100644 index 0000000000..d16f72c89a --- /dev/null +++ b/nostrdb/flatcc/flatcc_endian.h @@ -0,0 +1,125 @@ +#ifndef FLATCC_ENDIAN_H +#define FLATCC_ENDIAN_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This file provides helper macros to define type-specific macros and + * inline functions that convert between stored data and native data + * indedpently of both native (host) endianness and protocol endianness + * (i.e. the serialized endian format). + * + * To detect endianness correctly ensure one of the following is defined. + * + * __LITTLE_ENDIAN__ + * __BIG_ENDIAN__ + * FLATBUFFERS_LITTLEENDIAN=1 + * FLATBUFFERS_LITTLEENDIAN=0 + * + * Note: the Clang compiler likely already does this, but other + * compilers may have their own way, if at all. + * + * It is also necessary to include or a compatible + * implementation in order to provide: + * + * le16toh, le32to, le64toh, be16toh, be32toh, be64toh, + * htole16, htole32, htole64, htobe16, htobe32, htobe64. + * + * A simple way to ensure all of the above for most platforms is + * to include the portable endian support file: + * + * #include "flatcc/portable/pendian.h" + * + * It is also necessary to include + * + * #include "flatcc/flatcc_types.h" + * + * or an equivalent file. This makes it possible to change the + * endianness of the serialized data and the sizes of flatbuffer + * specific types such as `uoffset_t`. + * + * Note: the mentioned include files are likely already included + * by the file including this file, at least for the default + * configuration. + */ + +#ifndef UINT8_t +#include +#endif + +/* These are needed to simplify accessor macros and are not found in . */ +#ifndef le8toh +#define le8toh(n) (n) +#endif + +#ifndef be8toh +#define be8toh(n) (n) +#endif + +#ifndef htole8 +#define htole8(n) (n) +#endif + +#ifndef htobe8 +#define htobe8(n) (n) +#endif + +#include "flatcc_accessors.h" + +/* This is the binary encoding endianness, usually LE for flatbuffers. */ +#if FLATBUFFERS_PROTOCOL_IS_LE +#define flatbuffers_endian le +#elif FLATBUFFERS_PROTOCOL_IS_BE +#define flatbuffers_endian be +#else +#error "flatbuffers has no defined endiannesss" +#endif + + __flatcc_define_basic_scalar_accessors(flatbuffers_, flatbuffers_endian) + + __flatcc_define_integer_accessors(flatbuffers_bool, flatbuffers_bool_t, + FLATBUFFERS_BOOL_WIDTH, flatbuffers_endian) + __flatcc_define_integer_accessors(flatbuffers_union_type, flatbuffers_union_type_t, + FLATBUFFERS_UTYPE_WIDTH, flatbuffers_endian) + + __flatcc_define_integer_accessors(__flatbuffers_uoffset, flatbuffers_uoffset_t, + FLATBUFFERS_UOFFSET_WIDTH, flatbuffers_endian) + __flatcc_define_integer_accessors(__flatbuffers_soffset, flatbuffers_soffset_t, + FLATBUFFERS_SOFFSET_WIDTH, flatbuffers_endian) + __flatcc_define_integer_accessors(__flatbuffers_voffset, flatbuffers_voffset_t, + FLATBUFFERS_VOFFSET_WIDTH, flatbuffers_endian) + __flatcc_define_integer_accessors(__flatbuffers_utype, flatbuffers_utype_t, + FLATBUFFERS_UTYPE_WIDTH, flatbuffers_endian) + __flatcc_define_integer_accessors(__flatbuffers_thash, flatbuffers_thash_t, + FLATBUFFERS_THASH_WIDTH, flatbuffers_endian) + +/* flatcc/portable/pendian.h sets LITTLE/BIG flags if possible, and always defines le16toh. */ +#ifndef flatbuffers_is_native_pe +#if defined(__LITTLE_ENDIAN__) || FLATBUFFERS_LITTLEENDIAN +#undef FLATBUFFERS_LITTLEENDIAN +#define FLATBUFFERS_LITTLEENDIAN 1 +#define flatbuffers_is_native_pe() (FLATBUFFERS_PROTOCOL_IS_LE) +#elif defined(__BIG_ENDIAN__) || (defined(FLATBUFFERS_LITTLEENDIAN) && !FLATBUFFERS_LITTLEENDIAN) +#undef FLATBUFFERS_LITTLEENDIAN +#define FLATBUFFERS_LITTLEENDIAN 0 +#define flatbuffers_is_native_pe() (FLATBUFFERS_PROTOCOL_IS_BE) +#else +#define flatbuffers_is_native_pe() (__FLATBUFFERS_CONCAT(flatbuffers_endian, 16toh)(1) == 1) +#endif +#endif + +#ifndef flatbuffers_is_native_le +#define flatbuffers_is_native_le() flatbuffers_is_native_pe() +#endif + +#ifndef flatbuffers_is_native_be +#define flatbuffers_is_native_be() (!flatbuffers_is_native_pe()) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_ENDIAN_H */ diff --git a/nostrdb/flatcc/flatcc_epilogue.h b/nostrdb/flatcc/flatcc_epilogue.h new file mode 100644 index 0000000000..dc724f6c98 --- /dev/null +++ b/nostrdb/flatcc/flatcc_epilogue.h @@ -0,0 +1,8 @@ +/* Include guard intentionally left out. */ + +#ifdef __cplusplus +} +#endif + +#include "pdiagnostic_pop.h" + diff --git a/nostrdb/flatcc/flatcc_flatbuffers.h b/nostrdb/flatcc/flatcc_flatbuffers.h new file mode 100644 index 0000000000..210c9f2a42 --- /dev/null +++ b/nostrdb/flatcc/flatcc_flatbuffers.h @@ -0,0 +1,55 @@ +/* + * Even C11 compilers depend on clib support for `static_assert` which + * isn't always present, so we deal with this here for all compilers. + * + * Outside include guard to handle scope counter. + */ +#include "pstatic_assert.h" + +#ifndef FLATCC_FLATBUFFERS_H +#define FLATCC_FLATBUFFERS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef flatcc_flatbuffers_defined +#define flatcc_flatbuffers_defined + +#ifdef FLATCC_PORTABLE +#include "flatcc/flatcc_portable.h" +#endif +#include "pwarnings.h" +/* Needed by C99 compilers without FLATCC_PORTABLE. */ +#include "pstdalign.h" + +/* Handle fallthrough attribute in switch statements. */ +#include "pattributes.h" + +#include "flatcc_alloc.h" +#include "flatcc_assert.h" + +#define __FLATBUFFERS_PASTE2(a, b) a ## b +#define __FLATBUFFERS_PASTE3(a, b, c) a ## b ## c +#define __FLATBUFFERS_CONCAT(a, b) __FLATBUFFERS_PASTE2(a, b) + +/* + * "flatcc_endian.h" requires the preceeding include files, + * or compatible definitions. + */ +#include "pendian.h" +#include "flatcc_types.h" +#include "flatcc_endian.h" +#include "flatcc_identifier.h" + +#ifndef FLATBUFFERS_WRAP_NAMESPACE +#define FLATBUFFERS_WRAP_NAMESPACE(ns, x) ns ## _ ## x +#endif + +#endif /* flatcc_flatbuffers_defined */ + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_FLATBUFFERS_H */ diff --git a/nostrdb/flatcc/flatcc_identifier.h b/nostrdb/flatcc/flatcc_identifier.h new file mode 100644 index 0000000000..825f0fdaf8 --- /dev/null +++ b/nostrdb/flatcc/flatcc_identifier.h @@ -0,0 +1,148 @@ +#ifndef FLATCC_IDENTIFIER_H +#define FLATCC_IDENTIFIER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef FLATCC_FLATBUFFERS_H +#error "include via flatcc/flatcc_flatbuffers.h" +#endif + +#ifndef UINT8_MAX +#include +#endif + +/* + * FlatBuffers identifiers are normally specified by "file_identifer" in + * the schema, but a standard hash of the fully qualified type name can + * also be used. This file implements such a mapping, but the generated + * headers also contain the necessary information for known types. + */ + + +/* + * Returns the type hash of a given name in native endian format. + * Generated code already provides these, but if a name was changed + * in the schema it may be relevant to recompute the hash manually. + * + * The wire-format of this value should always be little endian. + * + * Note: this must be the fully qualified name, e.g. in the namespace + * "MyGame.Example": + * + * flatbuffers_type_hash_from_name("MyGame.Example.Monster"); + * + * or, in the global namespace just: + * + * flatbuffers_type_hash_from_name("MyTable"); + * + * This assumes 32 bit hash type. For other sizes, other FNV-1a + * constants would be required. + * + * Note that we reserve hash value 0 for missing or ignored value. + */ +static inline flatbuffers_thash_t flatbuffers_type_hash_from_name(const char *name) +{ + uint32_t hash = UINT32_C(2166136261); + while (*name) { + hash ^= (unsigned char)*name; + hash = hash * UINT32_C(16777619); + ++name; + } + if (hash == 0) { + hash = UINT32_C(2166136261); + } + return hash; +} + +/* + * Type hash encoded as little endian file identifier string. + * Note: if type hash is 0, the identifier should be null which + * we cannot return in this interface. + */ +static inline void flatbuffers_identifier_from_type_hash(flatbuffers_thash_t type_hash, flatbuffers_fid_t out_identifier) +{ + out_identifier[0] = (char)(type_hash & 0xff); + type_hash >>= 8; + out_identifier[1] = (char)(type_hash & 0xff); + type_hash >>= 8; + out_identifier[2] = (char)(type_hash & 0xff); + type_hash >>= 8; + out_identifier[3] = (char)(type_hash & 0xff); +} + +/* Native integer encoding of file identifier. */ +static inline flatbuffers_thash_t flatbuffers_type_hash_from_identifier(const flatbuffers_fid_t identifier) +{ + uint8_t *p = (uint8_t *)identifier; + + return identifier ? + (uint32_t)p[0] + (((uint32_t)p[1]) << 8) + (((uint32_t)p[2]) << 16) + (((uint32_t)p[3]) << 24) : 0; +} + +/* + * Convert a null terminated string identifier like "MONS" or "X" into a + * native type hash identifier, usually for comparison. This will not + * work with type hash strings because they can contain null bytes. + */ +static inline flatbuffers_thash_t flatbuffers_type_hash_from_string(const char *identifier) +{ + flatbuffers_thash_t h = 0; + const uint8_t *p = (const uint8_t *)identifier; + + if (!p[0]) return h; + h += ((flatbuffers_thash_t)p[0]); + if (!p[1]) return h; + h += ((flatbuffers_thash_t)p[1]) << 8; + if (!p[2]) return h; + h += ((flatbuffers_thash_t)p[2]) << 16; + /* No need to test for termination here. */ + h += ((flatbuffers_thash_t)p[3]) << 24; + return h; +} + +/* + * Computes the little endian wire format of the type hash. It can be + * used as a file identifer argument to various flatcc buffer calls. + * + * `flatbuffers_fid_t` is just `char [4]` for the default flatbuffers + * type system defined in `flatcc/flatcc_types.h`. + */ +static inline void flatbuffers_identifier_from_name(const char *name, flatbuffers_fid_t out_identifier) +{ + flatbuffers_identifier_from_type_hash(flatbuffers_type_hash_from_name(name), out_identifier); +} + +/* + * This is a collision free hash (a permutation) of the type hash to + * provide better distribution for use in hash tables. It is likely not + * necessary in praxis, and for uniqueness of identifiers it provides no + * advantage over just using the FNV-1a type hash, except when truncating + * the identifier to less than 32-bits. + * + * Note: the output should not be used in transmission. It provides no + * additional information and just complicates matters. Furthermore, the + * unmodified type hash has the benefit that it can seed a child namespace. + */ +static inline uint32_t flatbuffers_disperse_type_hash(flatbuffers_thash_t type_hash) +{ + /* http://stackoverflow.com/a/12996028 */ + uint32_t x = type_hash; + + x = ((x >> 16) ^ x) * UINT32_C(0x45d9f3b); + x = ((x >> 16) ^ x) * UINT32_C(0x45d9f3b); + x = ((x >> 16) ^ x); + return x; +} + + +/* We have hardcoded assumptions about identifier size. */ +static_assert(sizeof(flatbuffers_fid_t) == 4, "unexpected file identifier size"); +static_assert(sizeof(flatbuffers_thash_t) == 4, "unexpected type hash size"); + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_IDENTIFIER_H */ diff --git a/nostrdb/flatcc/flatcc_iov.h b/nostrdb/flatcc/flatcc_iov.h new file mode 100644 index 0000000000..a6d27f8802 --- /dev/null +++ b/nostrdb/flatcc/flatcc_iov.h @@ -0,0 +1,31 @@ +#ifndef FLATCC_IOV_H +#define FLATCC_IOV_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* + * The emitter receives one, or a few buffers at a time via + * this type. compatible iovec structure used for + * allocation and emitter interface. + */ +typedef struct flatcc_iovec flatcc_iovec_t; +struct flatcc_iovec { + void *iov_base; + size_t iov_len; +}; + +/* + * The largest iovec vector the builder will issue. It will + * always be a relatively small number. + */ +#define FLATCC_IOV_COUNT_MAX 8 + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_IOV_H */ diff --git a/nostrdb/flatcc/flatcc_json_parser.h b/nostrdb/flatcc/flatcc_json_parser.h new file mode 100644 index 0000000000..ed7151c2fd --- /dev/null +++ b/nostrdb/flatcc/flatcc_json_parser.h @@ -0,0 +1,895 @@ +#ifndef FLATCC_JSON_PARSE_H +#define FLATCC_JSON_PARSE_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * JSON RFC: + * http://www.ietf.org/rfc/rfc4627.txt?number=4627 + * + * With several flatbuffers specific extensions. + */ + +#include +#include + +#include "flatcc_rtconfig.h" +#include "flatcc_builder.h" +#include "flatcc_unaligned.h" + +#define PDIAGNOSTIC_IGNORE_UNUSED +#include "pdiagnostic_push.h" + +enum flatcc_json_parser_flags { + flatcc_json_parser_f_skip_unknown = 1, + flatcc_json_parser_f_force_add = 2, + flatcc_json_parser_f_with_size = 4, + flatcc_json_parser_f_skip_array_overflow = 8, + flatcc_json_parser_f_reject_array_underflow = 16 +}; + +#define FLATCC_JSON_PARSE_ERROR_MAP(XX) \ + XX(ok, "ok") \ + XX(eof, "eof") \ + XX(deep_nesting, "deep nesting") \ + XX(trailing_comma, "trailing comma") \ + XX(expected_colon, "expected colon") \ + XX(unexpected_character, "unexpected character") \ + XX(invalid_numeric, "invalid numeric") \ + XX(overflow, "overflow") \ + XX(underflow, "underflow") \ + XX(unbalanced_array, "unbalanced array") \ + XX(unbalanced_object, "unbalanced object") \ + XX(precision_loss, "precision loss") \ + XX(float_unexpected, "float unexpected") \ + XX(unknown_symbol, "unknown symbol") \ + XX(unquoted_symbolic_list, "unquoted list of symbols") \ + XX(unknown_union, "unknown union type") \ + XX(expected_string, "expected string") \ + XX(invalid_character, "invalid character") \ + XX(invalid_escape, "invalid escape") \ + XX(invalid_type, "invalid type") \ + XX(unterminated_string, "unterminated string") \ + XX(expected_object, "expected object") \ + XX(expected_array, "expected array") \ + XX(expected_scalar, "expected literal or symbolic scalar") \ + XX(expected_union_type, "expected union type") \ + XX(union_none_present, "union present with type NONE") \ + XX(union_none_not_null, "union of type NONE is not null") \ + XX(union_incomplete, "table has incomplete union") \ + XX(duplicate, "table has duplicate field") \ + XX(required, "required field missing") \ + XX(union_vector_length, "union vector length mismatch") \ + XX(base64, "invalid base64 content") \ + XX(base64url, "invalid base64url content") \ + XX(array_underflow, "fixed length array underflow") \ + XX(array_overflow, "fixed length array overflow") \ + XX(runtime, "runtime error") \ + XX(not_supported, "not supported") + +enum flatcc_json_parser_error_no { +#define XX(no, str) flatcc_json_parser_error_##no, + FLATCC_JSON_PARSE_ERROR_MAP(XX) +#undef XX +}; + +const char *flatcc_json_parser_error_string(int err); + +#define flatcc_json_parser_ok flatcc_json_parser_error_ok +#define flatcc_json_parser_eof flatcc_json_parser_error_eof + +/* + * The struct may be zero initialized in which case the line count will + * start at line zero, or the line may be set to 1 initially. The ctx + * is only used for error reporting and tracking non-standard unquoted + * ctx. + * + * `ctx` may for example hold a flatcc_builder_t pointer. + */ +typedef struct flatcc_json_parser_ctx flatcc_json_parser_t; +struct flatcc_json_parser_ctx { + flatcc_builder_t *ctx; + const char *line_start; + int flags; +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + int unquoted; +#endif + + int line, pos; + int error; + const char *start; + const char *end; + const char *error_loc; + /* Set at end of successful parse. */ + const char *end_loc; +}; + +static inline int flatcc_json_parser_get_error(flatcc_json_parser_t *ctx) +{ + return ctx->error; +} + +static inline void flatcc_json_parser_init(flatcc_json_parser_t *ctx, flatcc_builder_t *B, const char *buf, const char *end, int flags) +{ + memset(ctx, 0, sizeof(*ctx)); + ctx->ctx = B; + ctx->line_start = buf; + ctx->line = 1; + ctx->flags = flags; + /* These are not needed for parsing, but may be helpful in reporting etc. */ + ctx->start = buf; + ctx->end = end; + ctx->error_loc = buf; +} + +const char *flatcc_json_parser_set_error(flatcc_json_parser_t *ctx, const char *loc, const char *end, int reason); + +/* + * Wide space is not necessarily beneficial in the typical space, but it + * also isn't expensive so it may be added when there are applications + * that can benefit. + */ +const char *flatcc_json_parser_space_ext(flatcc_json_parser_t *ctx, const char *buf, const char *end); + +static inline const char *flatcc_json_parser_space(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + if (end - buf > 1) { + if (buf[0] > 0x20) { + return buf; + } + if (buf[0] == 0x20 && buf[1] > 0x20) { + return buf + 1; + } + } + return flatcc_json_parser_space_ext(ctx, buf, end); +} + + +static inline const char *flatcc_json_parser_string_start(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + if (buf == end || *buf != '\"') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_expected_string); + } + return ++buf; +} + +static inline const char *flatcc_json_parser_string_end(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + if (buf == end || *buf != '\"') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unterminated_string); + } + return ++buf; +} + +/* + * Parse a string as a fixed length char array as `s` with length `n`. + * and raise errors according to overflow/underflow runtime flags. Zero + * and truncate as needed. A trailing zero is not inserted if the input + * is at least the same length as the char array. + * + * Runtime flags: `skip_array_overflow`, `pad_array_underflow`. + */ +const char *flatcc_json_parser_char_array(flatcc_json_parser_t *ctx, + const char *buf, const char *end, char *s, size_t n); + +/* + * Creates a string. Returns *ref == 0 on unrecoverable error or + * sets *ref to a valid new string reference. + */ +const char *flatcc_json_parser_build_string(flatcc_json_parser_t *ctx, + const char *buf, const char *end, flatcc_builder_ref_t *ref); + +typedef char flatcc_json_parser_escape_buffer_t[5]; +/* + * If the buffer does not hold a valid escape sequence, an error is + * returned with code[0] = 0/ + * + * Otherwise code[0] the length (1-4) of the remaining + * characters in the code, transcoded from the escape sequence + * where a length of 4 only happens with escapaped surrogate pairs. + * + * The JSON extension `\xXX` is supported and may produced invalid UTF-8 + * characters such as 0xff. The standard JSON escape `\uXXXX` is not + * checked for invalid code points and may produce invalid UTF-8. + * + * Regular characters are expected to valid UTF-8 but they are not checked + * and may therefore produce invalid UTF-8. + * + * Control characters within a string are rejected except in the + * standard JSON escpaped form for `\n \r \t \b \f`. + * + * Additional escape codes as per standard JSON: `\\ \/ \"`. + */ +const char *flatcc_json_parser_string_escape(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_json_parser_escape_buffer_t code); + +/* + * Parses the longest unescaped run of string content followed by either + * an escape encoding, string termination, or error. + */ +const char *flatcc_json_parser_string_part(flatcc_json_parser_t *ctx, const char *buf, const char *end); + +static inline const char *flatcc_json_parser_symbol_start(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + if (buf == end) { + return buf; + } + if (*buf == '\"') { + ++buf; +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + ctx->unquoted = 0; +#endif + } else { +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + if (*buf == '.') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); + } + ctx->unquoted = 1; +#else + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); +#endif + } + return buf; +} + +static inline uint64_t flatcc_json_parser_symbol_part_ext(const char *buf, const char *end) +{ + uint64_t w = 0; + size_t n = (size_t)(end - buf); + + if (n > 8) { + n = 8; + } + /* This can bloat inlining for a rarely executed case. */ +#if 1 + /* Fall through comments needed to silence gcc 7 warnings. */ + switch (n) { + case 8: w |= ((uint64_t)buf[7]) << (0 * 8); + fallthrough; + case 7: w |= ((uint64_t)buf[6]) << (1 * 8); + fallthrough; + case 6: w |= ((uint64_t)buf[5]) << (2 * 8); + fallthrough; + case 5: w |= ((uint64_t)buf[4]) << (3 * 8); + fallthrough; + case 4: w |= ((uint64_t)buf[3]) << (4 * 8); + fallthrough; + case 3: w |= ((uint64_t)buf[2]) << (5 * 8); + fallthrough; + case 2: w |= ((uint64_t)buf[1]) << (6 * 8); + fallthrough; + case 1: w |= ((uint64_t)buf[0]) << (7 * 8); + fallthrough; + case 0: + break; + } +#else + /* But this is hardly much of an improvement. */ + { + size_t i; + for (i = 0; i < n; ++i) { + w <<= 8; + if (i < n) { + w = buf[i]; + } + } + } +#endif + return w; +} + +/* + * Read out string as a big endian word. This allows for trie lookup, + * also when trailing characters are beyond keyword. This assumes the + * external words tested against are valid and therefore there need be + * no checks here. If a match is not made, the symbol_end function will + * consume and check any unmatched content - from _before_ this function + * was called - i.e. the returned buffer is tentative for use only if we + * accept the part returned here. + * + * Used for both symbols and symbolic constants. + */ +static inline uint64_t flatcc_json_parser_symbol_part(const char *buf, const char *end) +{ + size_t n = (size_t)(end - buf); + +#if FLATCC_ALLOW_UNALIGNED_ACCESS + if (n >= 8) { + return be64toh(*(uint64_t *)buf); + } +#endif + return flatcc_json_parser_symbol_part_ext(buf, end); +} + +/* Don't allow space in dot notation neither inside nor outside strings. */ +static inline const char *flatcc_json_parser_match_scope(flatcc_json_parser_t *ctx, const char *buf, const char *end, int pos) +{ + const char *mark = buf; + + (void)ctx; + + if (end - buf <= pos) { + return mark; + } + if (buf[pos] != '.') { + return mark; + } + return buf + pos + 1; +} + +const char *flatcc_json_parser_match_constant(flatcc_json_parser_t *ctx, const char *buf, const char *end, int pos, int *more); + +/* We allow '.' in unquoted symbols, but not at the start or end. */ +static inline const char *flatcc_json_parser_symbol_end(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + char c, clast = 0; + + +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + if (ctx->unquoted) { + while (buf != end && *buf > 0x20) { + clast = c = *buf; + if (c == '_' || c == '.' || (c & 0x80) || (c >= '0' && c <= '9')) { + ++buf; + continue; + } + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'z') { + ++buf; + continue; + } + break; + } + if (clast == '.') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); + } + } else { +#else + { +#endif + while (buf != end && *buf != '\"') { + if (*buf == '\\') { + if (end - buf < 2) { + break; + } + ++buf; + } + ++buf; + } + if (buf == end || *buf != '\"') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unterminated_string); + } + ++buf; + } + return buf; +} + +static inline const char *flatcc_json_parser_constant_start(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + buf = flatcc_json_parser_symbol_start(ctx, buf, end); +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + if (!ctx->unquoted) { +#else + { +#endif + buf = flatcc_json_parser_space(ctx, buf, end); + } + return buf; +} + +static inline const char *flatcc_json_parser_object_start(flatcc_json_parser_t *ctx, const char *buf, const char *end, int *more) +{ + if (buf == end || *buf != '{') { + *more = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_expected_object); + } + buf = flatcc_json_parser_space(ctx, buf + 1, end); + if (buf != end && *buf == '}') { + *more = 0; + return flatcc_json_parser_space(ctx, buf + 1, end); + } + *more = 1; + return buf; +} + +static inline const char *flatcc_json_parser_object_end(flatcc_json_parser_t *ctx, const char *buf, + const char *end, int *more) +{ + buf = flatcc_json_parser_space(ctx, buf, end); + if (buf == end) { + *more = 0; + return buf; + } + if (*buf != ',') { + *more = 0; + if (*buf != '}') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unbalanced_object); + } else { + return flatcc_json_parser_space(ctx, buf + 1, end); + } + } + buf = flatcc_json_parser_space(ctx, buf + 1, end); + if (buf == end) { + *more = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unbalanced_object); + } +#if FLATCC_JSON_PARSE_ALLOW_TRAILING_COMMA + if (*buf == '}') { + *more = 0; + return flatcc_json_parser_space(ctx, buf + 1, end); + } +#endif + *more = 1; + return buf; +} + +static inline const char *flatcc_json_parser_array_start(flatcc_json_parser_t *ctx, const char *buf, const char *end, int *more) +{ + if (buf == end || *buf != '[') { + *more = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_expected_array); + } + buf = flatcc_json_parser_space(ctx, buf + 1, end); + if (buf != end && *buf == ']') { + *more = 0; + return flatcc_json_parser_space(ctx, buf + 1, end); + } + *more = 1; + return buf; +} + +static inline const char *flatcc_json_parser_array_end(flatcc_json_parser_t *ctx, const char *buf, + const char *end, int *more) +{ + buf = flatcc_json_parser_space(ctx, buf, end); + if (buf == end) { + *more = 0; + return buf; + } + if (*buf != ',') { + *more = 0; + if (*buf != ']') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unbalanced_array); + } else { + return flatcc_json_parser_space(ctx, buf + 1, end); + } + } + buf = flatcc_json_parser_space(ctx, buf + 1, end); + if (buf == end) { + *more = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unbalanced_array); + } +#if FLATCC_JSON_PARSE_ALLOW_TRAILING_COMMA + if (*buf == ']') { + *more = 0; + return flatcc_json_parser_space(ctx, buf + 1, end); + } +#endif + *more = 1; + return buf; +} + +/* + * Detects if a symbol terminates at a given `pos` relative to the + * buffer pointer, or return fast. + * + * Failure to match is not an error but a recommendation to try + * alternative longer suffixes - only if such do not exist will + * there be an error. If a match was not eventually found, + * the `flatcc_json_parser_unmatched_symbol` should be called to consume + * the symbol and generate error messages. + * + * If a match was detected, ':' and surrounding space is consumed, + * or an error is generated. + */ +static inline const char *flatcc_json_parser_match_symbol(flatcc_json_parser_t *ctx, const char *buf, + const char *end, int pos) +{ + const char *mark = buf; + + if (end - buf <= pos) { + return mark; + } +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + if (ctx->unquoted) { + if (buf[pos] > 0x20 && buf[pos] != ':') { + return mark; + } + buf += pos; + ctx->unquoted = 0; + } else { +#else + { +#endif + if (buf[pos] != '\"') { + return mark; + } + buf += pos + 1; + } + buf = flatcc_json_parser_space(ctx, buf, end); + if (buf != end && *buf == ':') { + ++buf; + return flatcc_json_parser_space(ctx, buf, end); + } + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_expected_colon); +} + +static inline const char *flatcc_json_parser_match_type_suffix(flatcc_json_parser_t *ctx, const char *buf, const char *end, int pos) +{ + if (end - buf <= pos + 5) { + return buf; + } + if (memcmp(buf + pos, "_type", 5)) { + return buf; + } + return flatcc_json_parser_match_symbol(ctx, buf, end, pos + 5); +} + +const char *flatcc_json_parser_unmatched_symbol(flatcc_json_parser_t *ctx, const char *buf, const char *end); + +static inline const char *flatcc_json_parser_coerce_uint64( + flatcc_json_parser_t *ctx, const char *buf, + const char *end, int value_sign, uint64_t value, uint64_t *v) +{ + if (value_sign) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_underflow); + } + *v = value; + return buf; +} + +static inline const char *flatcc_json_parser_coerce_bool(flatcc_json_parser_t *ctx, const char *buf, + const char *end, int value_sign, uint64_t value, uint8_t *v) +{ + if (value_sign) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_underflow); + } + *v = (uint8_t)!!value; + return buf; +} + +#define __flatcc_json_parser_define_coerce_unsigned(type, basetype, uctype) \ +static inline const char *flatcc_json_parser_coerce_ ## type( \ + flatcc_json_parser_t *ctx, const char *buf, \ + const char *end, int value_sign, uint64_t value, basetype *v) \ +{ \ + if (value_sign) { \ + return flatcc_json_parser_set_error(ctx, buf, end, \ + flatcc_json_parser_error_underflow); \ + } \ + if (value > uctype ## _MAX) { \ + return flatcc_json_parser_set_error(ctx, buf, end, \ + flatcc_json_parser_error_overflow); \ + } \ + *v = (basetype)value; \ + return buf; \ +} + +__flatcc_json_parser_define_coerce_unsigned(uint32, uint32_t, UINT32) +__flatcc_json_parser_define_coerce_unsigned(uint16, uint16_t, UINT16) +__flatcc_json_parser_define_coerce_unsigned(uint8, uint8_t, UINT8) + +#define __flatcc_json_parser_define_coerce_signed(type, basetype, uctype) \ +static inline const char *flatcc_json_parser_coerce_ ## type( \ + flatcc_json_parser_t *ctx, const char *buf, \ + const char *end, int value_sign, uint64_t value, basetype *v) \ +{ \ + if (value_sign) { \ + if (value > (uint64_t)(uctype ## _MAX) + 1) { \ + return flatcc_json_parser_set_error(ctx, buf, end, \ + flatcc_json_parser_error_underflow); \ + } \ + *v = (basetype)-(int64_t)value; \ + } else { \ + if (value > uctype ## _MAX) { \ + return flatcc_json_parser_set_error(ctx, buf, end, \ + flatcc_json_parser_error_overflow); \ + } \ + *v = (basetype)value; \ + } \ + return buf; \ +} + +__flatcc_json_parser_define_coerce_signed(int64, int64_t, INT64) +__flatcc_json_parser_define_coerce_signed(int32, int32_t, INT32) +__flatcc_json_parser_define_coerce_signed(int16, int16_t, INT16) +__flatcc_json_parser_define_coerce_signed(int8, int8_t, INT8) + +static inline const char *flatcc_json_parser_coerce_float( + flatcc_json_parser_t *ctx, const char *buf, + const char *end, int value_sign, uint64_t value, float *v) +{ + (void)ctx; + (void)end; + + *v = value_sign ? -(float)value : (float)value; + return buf; +} + +static inline const char *flatcc_json_parser_coerce_double( + flatcc_json_parser_t *ctx, const char *buf, + const char *end, int value_sign, uint64_t value, double *v) +{ + (void)ctx; + (void)end; + + *v = value_sign ? -(double)value : (double)value; + return buf; +} + +const char *flatcc_json_parser_double(flatcc_json_parser_t *ctx, const char *buf, const char *end, double *v); + +const char *flatcc_json_parser_float(flatcc_json_parser_t *ctx, const char *buf, const char *end, float *v); + +/* + * If the buffer does not contain a valid start character for a numeric + * value, the function will return the the input buffer without failure. + * This makes is possible to try a symbolic parse. + */ +const char *flatcc_json_parser_integer(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_sign, uint64_t *value); + +/* Returns unchanged buffer without error if `null` is not matched. */ +static inline const char *flatcc_json_parser_null(const char *buf, const char *end) +{ + if (end - buf >= 4 && memcmp(buf, "null", 4) == 0) { + return buf + 4; + } + return buf; +} + +static inline const char *flatcc_json_parser_none(flatcc_json_parser_t *ctx, + const char *buf, const char *end) +{ + if (end - buf >= 4 && memcmp(buf, "null", 4) == 0) { + return buf + 4; + } + return flatcc_json_parser_set_error(ctx, buf, end, + flatcc_json_parser_error_union_none_not_null); +} + +/* + * `parsers` is a null terminated array of parsers with at least one + * valid parser. A numeric literal parser may also be included. + */ +#define __flatcc_json_parser_define_integral_parser(type, basetype) \ +static inline const char *flatcc_json_parser_ ## type( \ + flatcc_json_parser_t *ctx, \ + const char *buf, const char *end, basetype *v) \ +{ \ + uint64_t value = 0; \ + int value_sign = 0; \ + const char *mark = buf; \ + \ + *v = 0; \ + if (buf == end) { \ + return buf; \ + } \ + buf = flatcc_json_parser_integer(ctx, buf, end, &value_sign, &value); \ + if (buf != mark) { \ + return flatcc_json_parser_coerce_ ## type(ctx, \ + buf, end, value_sign, value, v); \ + } \ + return buf; \ +} + +__flatcc_json_parser_define_integral_parser(uint64, uint64_t) +__flatcc_json_parser_define_integral_parser(uint32, uint32_t) +__flatcc_json_parser_define_integral_parser(uint16, uint16_t) +__flatcc_json_parser_define_integral_parser(uint8, uint8_t) +__flatcc_json_parser_define_integral_parser(int64, int64_t) +__flatcc_json_parser_define_integral_parser(int32, int32_t) +__flatcc_json_parser_define_integral_parser(int16, int16_t) +__flatcc_json_parser_define_integral_parser(int8, int8_t) + +static inline const char *flatcc_json_parser_bool(flatcc_json_parser_t *ctx, const char *buf, const char *end, uint8_t *v) +{ + const char *k; + uint8_t tmp; + + k = buf; + if (end - buf >= 4 && memcmp(buf, "true", 4) == 0) { + *v = 1; + return k + 4; + } else if (end - buf >= 5 && memcmp(buf, "false", 5) == 0) { + *v = 0; + return k + 5; + } + buf = flatcc_json_parser_uint8(ctx, buf, end, &tmp); + *v = !!tmp; + return buf; +} + +/* + * The `parsers` argument is a zero terminated array of parser + * functions with increasingly general scopes. + * + * Symbols can be be or'ed together by listing multiple space separated + * flags in source being parsed, like `{ x : "Red Blue" }`. + * Intended for flags, but generally available. + * + * `aggregate` means there are more symbols to follow. + * + * This function does not return input `buf` value if match was + * unsuccessful. It will either match or error. + */ +typedef const char *flatcc_json_parser_integral_symbol_f(flatcc_json_parser_t *ctx, + const char *buf, const char *end, int *value_sign, uint64_t *value, int *aggregate); + +/* + * Raise an error if a syntax like `color: Red Green` is seen unless + * explicitly permitted. `color: "Red Green"` or `"color": "Red Green" + * or `color: Red` is permitted if unquoted is permitted but not + * unquoted list. Googles flatc JSON parser does not allow multiple + * symbolic values unless quoted, so this is the default. + */ +#if !FLATCC_JSON_PARSE_ALLOW_UNQUOTED || FLATCC_JSON_PARSE_ALLOW_UNQUOTED_LIST +#define __flatcc_json_parser_init_check_unquoted_list() +#define __flatcc_json_parser_check_unquoted_list() +#else +#define __flatcc_json_parser_init_check_unquoted_list() int list_count = 0; +#define __flatcc_json_parser_check_unquoted_list() \ + if (list_count++ && ctx->unquoted) { \ + return flatcc_json_parser_set_error(ctx, buf, end, \ + flatcc_json_parser_error_unquoted_symbolic_list); \ + } +#endif + +#define __flatcc_json_parser_define_symbolic_integral_parser(type, basetype)\ +static const char *flatcc_json_parser_symbolic_ ## type( \ + flatcc_json_parser_t *ctx, \ + const char *buf, const char *end, \ + flatcc_json_parser_integral_symbol_f *parsers[], \ + basetype *v) \ +{ \ + flatcc_json_parser_integral_symbol_f **p; \ + const char *mark; \ + basetype tmp = 0; \ + uint64_t value; \ + int value_sign, aggregate; \ + __flatcc_json_parser_init_check_unquoted_list() \ + \ + *v = 0; \ + buf = flatcc_json_parser_constant_start(ctx, buf, end); \ + if (buf == end) { \ + return buf; \ + } \ + do { \ + p = parsers; \ + do { \ + /* call parser function */ \ + buf = (*p)(ctx, (mark = buf), end, \ + &value_sign, &value, &aggregate); \ + if (buf == end) { \ + return buf; \ + } \ + } while (buf == mark && *++p); \ + if (mark == buf) { \ + return flatcc_json_parser_set_error(ctx, buf, end, \ + flatcc_json_parser_error_expected_scalar); \ + } \ + __flatcc_json_parser_check_unquoted_list() \ + if (end == flatcc_json_parser_coerce_ ## type(ctx, \ + buf, end, value_sign, value, &tmp)) { \ + return end; \ + } \ + /* \ + * `+=`, not `|=` because we also coerce to float and double, \ + * and because we need to handle signed values. This may give \ + * unexpected results with duplicate flags. \ + */ \ + *v += tmp; \ + } while (aggregate); \ + return buf; \ +} + +__flatcc_json_parser_define_symbolic_integral_parser(uint64, uint64_t) +__flatcc_json_parser_define_symbolic_integral_parser(uint32, uint32_t) +__flatcc_json_parser_define_symbolic_integral_parser(uint16, uint16_t) +__flatcc_json_parser_define_symbolic_integral_parser(uint8, uint8_t) +__flatcc_json_parser_define_symbolic_integral_parser(int64, int64_t) +__flatcc_json_parser_define_symbolic_integral_parser(int32, int32_t) +__flatcc_json_parser_define_symbolic_integral_parser(int16, int16_t) +__flatcc_json_parser_define_symbolic_integral_parser(int8, int8_t) + +__flatcc_json_parser_define_symbolic_integral_parser(bool, uint8_t) + +/* We still parse integral values, but coerce to float or double. */ +__flatcc_json_parser_define_symbolic_integral_parser(float, float) +__flatcc_json_parser_define_symbolic_integral_parser(double, double) + +/* Parse vector as a base64 or base64url encoded string with no spaces permitted. */ +const char *flatcc_json_parser_build_uint8_vector_base64(flatcc_json_parser_t *ctx, + const char *buf, const char *end, flatcc_builder_ref_t *ref, int urlsafe); + +/* + * This doesn't do anything other than validate and advance past + * a JSON value which may use unquoted symbols. + * + * Upon call it is assumed that leading space has been stripped and that + * a JSON value is expected (i.e. root, or just after ':' in a + * container object, or less likely as an array member). Any trailing + * comma is assumed to belong to the parent context. Returns a parse + * location stripped from space so container should post call expect + * ',', '}', or ']', or EOF if the JSON is valid. + */ +const char *flatcc_json_parser_generic_json(flatcc_json_parser_t *ctx, const char *buf, const char *end); + +/* Parse a JSON table. */ +typedef const char *flatcc_json_parser_table_f(flatcc_json_parser_t *ctx, + const char *buf, const char *end, flatcc_builder_ref_t *pref); + +/* Parses a JSON struct. */ +typedef const char *flatcc_json_parser_struct_f(flatcc_json_parser_t *ctx, + const char *buf, const char *end, flatcc_builder_ref_t *pref); + +/* Constructs a table, struct, or string object unless the type is 0 or unknown. */ +typedef const char *flatcc_json_parser_union_f(flatcc_json_parser_t *ctx, + const char *buf, const char *end, uint8_t type, flatcc_builder_ref_t *pref); + +typedef int flatcc_json_parser_is_known_type_f(uint8_t type); + +/* Called at start by table parsers with at least 1 union. */ +const char *flatcc_json_parser_prepare_unions(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_total, size_t *handle); + +const char *flatcc_json_parser_finalize_unions(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t handle); + +const char *flatcc_json_parser_union(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, + flatbuffers_voffset_t id, size_t handle, + flatcc_json_parser_union_f *union_parser); + +const char *flatcc_json_parser_union_type(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, + flatbuffers_voffset_t id, size_t handle, + flatcc_json_parser_integral_symbol_f *type_parsers[], + flatcc_json_parser_union_f *union_parser); + +const char *flatcc_json_parser_union_vector(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, + flatbuffers_voffset_t id, size_t handle, + flatcc_json_parser_union_f *union_parser); + +const char *flatcc_json_parser_union_type_vector(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, + flatbuffers_voffset_t id, size_t handle, + flatcc_json_parser_integral_symbol_f *type_parsers[], + flatcc_json_parser_union_f *union_parser, + flatcc_json_parser_is_known_type_f accept_type); + +/* + * Parses a table as root. + * + * Use the flag `flatcc_json_parser_f_with_size` to create a buffer with + * size prefix. + * + * `ctx` may be null or an uninitialized json parser to receive parse results. + * `builder` must a newly initialized or reset builder object. + * `buf`, `bufsiz` may be larger than the parsed json if trailing + * space or zeroes are expected, but they must represent a valid memory buffer. + * `fid` must be null, or a valid file identifier. + * `flags` default to 0. See also `flatcc_json_parser_flags`. + */ +int flatcc_json_parser_table_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags, const char *fid, + flatcc_json_parser_table_f *parser); + +/* + * Similar to `flatcc_json_parser_table_as_root` but parses a struct as + * root. + */ +int flatcc_json_parser_struct_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags, const char *fid, + flatcc_json_parser_struct_f *parser); + +#include "pdiagnostic_pop.h" + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_JSON_PARSE_H */ diff --git a/nostrdb/flatcc/flatcc_json_printer.h b/nostrdb/flatcc/flatcc_json_printer.h new file mode 100644 index 0000000000..0ce49c1461 --- /dev/null +++ b/nostrdb/flatcc/flatcc_json_printer.h @@ -0,0 +1,789 @@ +#ifndef FLATCC_JSON_PRINTER_H +#define FLATCC_JSON_PRINTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Definitions for default implementation, do not assume these are + * always valid. + */ +#define FLATCC_JSON_PRINT_FLUSH_SIZE (1024 * 16) +#define FLATCC_JSON_PRINT_RESERVE 64 +#define FLATCC_JSON_PRINT_BUFFER_SIZE (FLATCC_JSON_PRINT_FLUSH_SIZE + FLATCC_JSON_PRINT_RESERVE) + +#ifndef FLATCC_JSON_PRINTER_ALLOC +#define FLATCC_JSON_PRINTER_ALLOC(n) FLATCC_ALLOC(n) +#endif + +#ifndef FLATCC_JSON_PRINTER_FREE +#define FLATCC_JSON_PRINTER_FREE(p) FLATCC_FREE(p) +#endif + +#ifndef FLATCC_JSON_PRINTER_REALLOC +#define FLATCC_JSON_PRINTER_REALLOC(p, n) FLATCC_REALLOC(p, n) +#endif + +/* Initial size that grows exponentially. */ +#define FLATCC_JSON_PRINT_DYN_BUFFER_SIZE 4096 + + +#include +#include + +#include "flatcc/flatcc_rtconfig.h" +#include "flatcc/flatcc_flatbuffers.h" + +/* -DFLATCC_PORTABLE may help if inttypes.h is missing. */ +#ifndef PRId64 +#include +#endif + +#define FLATCC_JSON_PRINT_ERROR_MAP(XX) \ + XX(ok, "ok") \ + /* \ + * When the flatbuffer is null, has too small a header, or has \ + * mismatching identifier when a match was requested. \ + */ \ + XX(bad_input, "bad input") \ + XX(deep_recursion, "deep recursion") \ + /* \ + * When the output was larger than the available fixed length buffer, \ + * or dynamic allocation could not grow the buffer sufficiently. \ + */ \ + XX(overflow, "overflow") + +enum flatcc_json_printer_error_no { +#define XX(no, str) flatcc_json_printer_error_##no, + FLATCC_JSON_PRINT_ERROR_MAP(XX) +#undef XX +}; + +#define flatcc_json_printer_ok flatcc_json_printer_error_ok + +typedef struct flatcc_json_printer_ctx flatcc_json_printer_t; + +typedef void flatcc_json_printer_flush_f(flatcc_json_printer_t *ctx, int all); + +struct flatcc_json_printer_ctx { + char *buf; + size_t size; + size_t flush_size; + size_t total; + const char *pflush; + char *p; + uint8_t own_buffer; + uint8_t indent; + uint8_t unquote; + uint8_t noenum; + uint8_t skip_default; + uint8_t force_default; + int level; + int error; + + void *fp; + flatcc_json_printer_flush_f *flush; +}; + +static inline void flatcc_json_printer_set_error(flatcc_json_printer_t *ctx, int err) +{ + if (!ctx->error) { + ctx->error = err; + } +} + +const char *flatcc_json_printer_error_string(int err); + +static inline int flatcc_json_printer_get_error(flatcc_json_printer_t *ctx) +{ + return ctx->error; +} + +/* + * Call to reuse context between parses without without + * returning buffer. If a file pointer is being used, + * it will remain open. + * + * Reset does not affect the formatting settings indentation, and + * operational flags, but does zero the indentation level. + */ +static inline void flatcc_json_printer_reset(flatcc_json_printer_t *ctx) +{ + ctx->p = ctx->buf; + ctx->level = 0; + ctx->total = 0; + ctx->error = 0; +} + +/* + * A custom init function can be implemented with a custom flush + * function can be custom implemented. A few have been provided: + * init with external fixed length buffer, and init with dynamically + * growing buffer. + * + * Because there are a lot of small print functions, it is essentially + * always faster to print to local buffer than moving to io directly + * such as using fprintf or fwrite. The flush callback is used to + * move data when enough has been collected. + * + * `fp` should be of type `FILE *` but we do not enforce it here + * because it allows the header to be independent of + * when not required. If `fp` is null, it defaults to stdout. + * + * Returns -1 on alloc error (no cleanup needed), or 0 on success. + * Eventually the clear method must be called to return memory. + * + * The file pointer may be stdout or a custom file. The file pointer + * is not affected by reset or clear and should be closed manually. + * + * `set_flags` and related may be called subsequently to modify + * behavior. + */ +int flatcc_json_printer_init(flatcc_json_printer_t *ctx, void *fp); + +/* + * Prints to external buffer and sets overflow error if buffer is too + * small. Earlier content is then overwritten. A custom version of this + * function could flush the content to elsewhere before allowing the + * buffer content to be overwritten. The `buffers_size` must be large + * enough to hold `FLATCC_JSON_PRINT_RESERVED_SIZE` which is small but + * large enough value to hold entire numbers and the like. + * + * It is not strictly necessary to call clear because the buffer is + * external, but still good form and case the context type is changed + * later. + * + * Returns -1 on buffer size error (no cleanup needed), or 0 on success. + * + * `set_flags` and related may be called subsequently to modify + * behavior. + */ +int flatcc_json_printer_init_buffer(flatcc_json_printer_t *ctx, char *buffer, size_t buffer_size); + +/* + * Returns the current buffer pointer and also the content size in + * `buffer_size` if it is null. The operation is not very useful for + * file oriented printers (created with `init`) and will then only + * return the unflushed buffer content. For fixed length buffers + * (`init_buffer`), only the last content is available if the buffer + * overflowed. Works well with (`init_buffer`) when the dynamic buffer + * is be reused, otherwise `finalize_dynamic_buffer` could be more + * appropriate. + * + * The returned buffer is zero terminated. + * + * The returned pointer is only valid until next operation and should + * not deallocated manually. + */ +void *flatcc_json_printer_get_buffer(flatcc_json_printer_t *ctx, size_t *buffer_size); + +/* + * Set to non-zero if names and enum symbols can be unquoted thus + * diverging from standard JSON while remaining compatible with `flatc` + * JSON flavor. + */ +static inline void flatcc_json_printer_set_unquoted(flatcc_json_printer_t *ctx, int x) +{ + ctx->unquote = !!x; +} + +/* + * Set to non-zero if enums should always be printed as numbers. + * Otherwise enums are printed as a symbol for member values, and as + * numbers for other values. + * + * NOTE: this setting will not affect code generated with enum mapping + * disabled - statically disabling enum mapping is signficantly faster + * for enums, less so for for union types. + */ +static inline void flatcc_json_printer_set_noenum(flatcc_json_printer_t *ctx, int x) +{ + ctx->noenum = !!x; +} + +/* + * Override priting an existing scalar field if it equals the default value. + * Note that this setting is not mutually exclusive to `set_force_default`. + */ +static inline void flatcc_json_printer_set_skip_default(flatcc_json_printer_t *ctx, int x) +{ + ctx->skip_default = !!x; +} + +/* + * Override skipping absent scalar fields and print the default value. + * Note that this setting is not mutually exclusive to `set_skip_default`. + */ +static inline void flatcc_json_printer_set_force_default(flatcc_json_printer_t *ctx, int x) +{ + ctx->force_default = !!x; +} + + +/* + * Set pretty-print indentation in number of spaces. 0 (default) is + * compact with no spaces or linebreaks (default), anything above + * triggers pretty print. + */ +static inline void flatcc_json_printer_set_indent(flatcc_json_printer_t *ctx, uint8_t x) +{ + ctx->indent = x; +} + +/* + * Override the default compact valid JSON format with a + * pretty printed non-strict version. Enums are translated + * to names, which is also the default. + */ +static inline void flatcc_json_printer_set_nonstrict(flatcc_json_printer_t *ctx) +{ + flatcc_json_printer_set_indent(ctx, 2); + flatcc_json_printer_set_unquoted(ctx, 1); + flatcc_json_printer_set_noenum(ctx, 0); +} + +enum flatcc_json_printer_flags { + flatcc_json_printer_f_unquote = 1, + flatcc_json_printer_f_noenum = 2, + flatcc_json_printer_f_skip_default = 4, + flatcc_json_printer_f_force_default = 8, + flatcc_json_printer_f_pretty = 16, + flatcc_json_printer_f_nonstrict = 32, +}; + +/* + * May be called instead of setting operational modes individually. + * Formatting is strict quoted json witout pretty printing by default. + * + * flags are: + * + * `unquote`, + * `noenum`, + * `skip_default`, + * `force_default`, + * `pretty`, + * `nonstrict` + * + * `pretty` flag sets indentation to 2. + * `nonstrict` implies: `noenum`, `unquote`, `pretty`. + */ +static inline void flatcc_json_printer_set_flags(flatcc_json_printer_t *ctx, int flags) +{ + ctx->unquote = !!(flags & flatcc_json_printer_f_unquote); + ctx->noenum = !!(flags & flatcc_json_printer_f_noenum); + ctx->skip_default = !!(flags & flatcc_json_printer_f_skip_default); + ctx->force_default = !!(flags & flatcc_json_printer_f_force_default); + if (flags & flatcc_json_printer_f_pretty) { + flatcc_json_printer_set_indent(ctx, 2); + } + if (flags & flatcc_json_printer_f_nonstrict) { + flatcc_json_printer_set_nonstrict(ctx); + } +} + + +/* + * Detects if the conctext type uses dynamically allocated memory + * using malloc and realloc and frees any such memory. + * + * Not all context types needs to be cleared. + */ +void flatcc_json_printer_clear(flatcc_json_printer_t *ctx); + +/* + * Ensures that there ia always buffer capacity for priting the next + * primitive with delimiters. + * + * Only flushes complete flush units and is inexpensive to call. + * The content buffer has an extra reserve which ensures basic + * data types and delimiters can always be printed after a partial + * flush. At the end, a `flush` is required to flush the + * remaining incomplete buffer data. + * + * Numbers do not call partial flush but will always fit into the reserve + * capacity after a partial flush, also surrounded by delimiters. + * + * Variable length operations generally submit a partial flush so it is + * safe to print a number after a name without flushing, but vectors of + * numbers must (and do) issue a partial flush between elements. This is + * handled automatically but must be considered if using the primitives + * for special purposes. Because repeated partial flushes are very cheap + * this is only a concern for high performance applications. + * + * When identiation is enabled, partial flush is also automatically + * issued . + */ +static inline void flatcc_json_printer_flush_partial(flatcc_json_printer_t *ctx) +{ + if (ctx->p >= ctx->pflush) { + ctx->flush(ctx, 0); + } +} + +/* Returns the total printed size but flushed and in buffer. */ +static inline size_t flatcc_json_printer_total(flatcc_json_printer_t *ctx) +{ + return ctx->total + (size_t)(ctx->p - ctx->buf); +} + +/* + * Flush the remaining data not flushed by partial flush. It is valid to + * call at any point if it is acceptable to have unaligned flush units, + * but this is not desireable if, for example, compression or encryption + * is added to the flush pipeline. + * + * Not called automatically at the end of printing a flatbuffer object + * in case more data needs to be appended without submitting incomplete + * flush units prematurely - for example adding a newline at the end. + * + * The flush behavior depeends on the underlying `ctx` object, for + * example dynamic buffers have no distinction between partial and full + * flushes - here it is merely ensured that the buffer always has a + * reserve capacity left. + * + * Returns the total printed size. + */ +static inline size_t flatcc_json_printer_flush(flatcc_json_printer_t *ctx) +{ + ctx->flush(ctx, 1); + return flatcc_json_printer_total(ctx); +} + +/* + * Helper functions to print anything into the json buffer. + * Strings are escaped. + * + * When pretty printing (indent > 0), level 0 has special significance - + * so if wrapping printed json in a manually printed container json + * object, these functions can help manage this. + */ + +/* Escaped and quoted string. */ +void flatcc_json_printer_string(flatcc_json_printer_t *ctx, const char *s, size_t n); +/* Unescaped and unquoted string. */ +void flatcc_json_printer_write(flatcc_json_printer_t *ctx, const char *s, size_t n); +/* Print a newline and issues a partial flush. */ +void flatcc_json_printer_nl(flatcc_json_printer_t *ctx); +/* Like numbers, a partial flush is not issued. */ +void flatcc_json_printer_char(flatcc_json_printer_t *ctx, char c); +/* Indents and issues a partial flush. */ +void flatcc_json_printer_indent(flatcc_json_printer_t *ctx); +/* Adjust identation level, usually +/-1. */ +void flatcc_json_printer_add_level(flatcc_json_printer_t *ctx, int n); +/* Returns current identation level (0 is top level). */ +int flatcc_json_printer_get_level(flatcc_json_printer_t *ctx); + +/* + * If called explicitly be aware that repeated calls to numeric + * printers may cause buffer overflow without flush in-between. + */ +void flatcc_json_printer_uint8(flatcc_json_printer_t *ctx, uint8_t v); +void flatcc_json_printer_uint16(flatcc_json_printer_t *ctx, uint16_t v); +void flatcc_json_printer_uint32(flatcc_json_printer_t *ctx, uint32_t v); +void flatcc_json_printer_uint64(flatcc_json_printer_t *ctx, uint64_t v); +void flatcc_json_printer_int8(flatcc_json_printer_t *ctx, int8_t v); +void flatcc_json_printer_int16(flatcc_json_printer_t *ctx, int16_t v); +void flatcc_json_printer_int32(flatcc_json_printer_t *ctx, int32_t v); +void flatcc_json_printer_int64(flatcc_json_printer_t *ctx, int64_t v); +void flatcc_json_printer_bool(flatcc_json_printer_t *ctx, int v); +void flatcc_json_printer_float(flatcc_json_printer_t *ctx, float v); +void flatcc_json_printer_double(flatcc_json_printer_t *ctx, double v); + +void flatcc_json_printer_enum(flatcc_json_printer_t *ctx, + const char *symbol, size_t len); + +/* + * Convenience function to add a trailing newline, flush the buffer, + * test for error and reset the context for reuse. + * + * Returns total size printed or < 0 on error. + * + * This function makes most sense for file oriented output. + * See also `finalize_dynamic_buffer`. + */ +static inline int flatcc_json_printer_finalize(flatcc_json_printer_t *ctx) +{ + int ret; + flatcc_json_printer_nl(ctx); + ret = (int)flatcc_json_printer_flush(ctx); + if (ctx->error) { + ret = -1; + } + flatcc_json_printer_reset(ctx); + return ret; +} + +/* + * Allocates a small buffer and grows it dynamically. + * Buffer survives past reset. To reduce size between uses, call clear + * followed by init call. To reuse buffer just call reset between uses. + * If `buffer_size` is 0 a sensible default is being used. The size is + * automatically rounded up to reserved size if too small. + * + * Returns -1 on alloc error (no cleanup needed), or 0 on success. + * Eventually the clear method must be called to return memory. + * + * `set_flags` and related may be called subsequently to modify + * behavior. + */ +int flatcc_json_printer_init_dynamic_buffer(flatcc_json_printer_t *ctx, size_t buffer_size); + +/* + * Similar to calling `finalize` but returns the buffer and does NOT + * reset, but rather clears printer object and the returned buffer must + * be deallocated with `free`. + * + * The returned buffer is zero terminated. + * + * NOTE: it is entirely optional to use this method. For repeated used + * of dynamic buffers, `newline` (or not) followed by `get_buffer` + * and `reset` will be an alternative. + * + * Stores the printed buffer size in `buffer_size` if it is not null. + * + * See also `get_dynamic_buffer`. + */ +void *flatcc_json_printer_finalize_dynamic_buffer(flatcc_json_printer_t *ctx, size_t *buffer_size); + + +/************************************************************* + * The following is normally only used by generated code. + *************************************************************/ + +typedef struct flatcc_json_printer_table_descriptor flatcc_json_printer_table_descriptor_t; + +struct flatcc_json_printer_table_descriptor { + const void *table; + const void *vtable; + int vsize; + int ttl; + int count; +}; + +typedef struct flatcc_json_printer_union_descriptor flatcc_json_printer_union_descriptor_t; + +struct flatcc_json_printer_union_descriptor { + const void *member; + int ttl; + uint8_t type; +}; + +typedef void flatcc_json_printer_table_f(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td); + +typedef void flatcc_json_printer_struct_f(flatcc_json_printer_t *ctx, + const void *p); + +typedef void flatcc_json_printer_union_f(flatcc_json_printer_t *ctx, + flatcc_json_printer_union_descriptor_t *ud); + +/* Generated value to name map callbacks. */ +typedef void flatcc_json_printer_union_type_f(flatcc_json_printer_t *ctx, flatbuffers_utype_t type); +typedef void flatcc_json_printer_uint8_enum_f(flatcc_json_printer_t *ctx, uint8_t v); +typedef void flatcc_json_printer_uint16_enum_f(flatcc_json_printer_t *ctx, uint16_t v); +typedef void flatcc_json_printer_uint32_enum_f(flatcc_json_printer_t *ctx, uint32_t v); +typedef void flatcc_json_printer_uint64_enum_f(flatcc_json_printer_t *ctx, uint64_t v); +typedef void flatcc_json_printer_int8_enum_f(flatcc_json_printer_t *ctx, int8_t v); +typedef void flatcc_json_printer_int16_enum_f(flatcc_json_printer_t *ctx, int16_t v); +typedef void flatcc_json_printer_int32_enum_f(flatcc_json_printer_t *ctx, int32_t v); +typedef void flatcc_json_printer_int64_enum_f(flatcc_json_printer_t *ctx, int64_t v); +typedef void flatcc_json_printer_bool_enum_f(flatcc_json_printer_t *ctx, flatbuffers_bool_t v); + +#define __define_print_scalar_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _field(flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, T v); + +#define __define_print_scalar_optional_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _optional_field( \ + flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len); + +#define __define_print_scalar_struct_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _struct_field(flatcc_json_printer_t *ctx,\ + int index, const void *p, size_t offset, \ + const char *name, size_t len); + +#define __define_print_scalar_array_struct_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _array_struct_field( \ + flatcc_json_printer_t *ctx, \ + int index, const void *p, size_t offset, \ + const char *name, size_t len, size_t count); + +#define __define_print_enum_array_struct_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_array_struct_field( \ + flatcc_json_printer_t *ctx, \ + int index, const void *p, size_t offset, \ + const char *name, size_t len, size_t count, \ + flatcc_json_printer_ ## TN ##_enum_f *pf); + +#define __define_print_enum_struct_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_struct_field( \ + flatcc_json_printer_t *ctx, \ + int index, const void *p, size_t offset, \ + const char *name, size_t len, \ + flatcc_json_printer_ ## TN ##_enum_f *pf); + +#define __define_print_enum_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_field(flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, T v, \ + flatcc_json_printer_ ## TN ##_enum_f *pf); + +#define __define_print_enum_optional_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_optional_field( \ + flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, \ + flatcc_json_printer_ ## TN ##_enum_f *pf); + +#define __define_print_scalar_vector_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _vector_field(flatcc_json_printer_t *ctx,\ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len); + +#define __define_print_enum_vector_field_proto(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_vector_field( \ + flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, \ + flatcc_json_printer_ ## TN ##_enum_f *pf); + +__define_print_scalar_field_proto(uint8, uint8_t) +__define_print_scalar_field_proto(uint16, uint16_t) +__define_print_scalar_field_proto(uint32, uint32_t) +__define_print_scalar_field_proto(uint64, uint64_t) +__define_print_scalar_field_proto(int8, int8_t) +__define_print_scalar_field_proto(int16, int16_t) +__define_print_scalar_field_proto(int32, int32_t) +__define_print_scalar_field_proto(int64, int64_t) +__define_print_scalar_field_proto(bool, flatbuffers_bool_t) +__define_print_scalar_field_proto(float, float) +__define_print_scalar_field_proto(double, double) + +__define_print_enum_field_proto(uint8, uint8_t) +__define_print_enum_field_proto(uint16, uint16_t) +__define_print_enum_field_proto(uint32, uint32_t) +__define_print_enum_field_proto(uint64, uint64_t) +__define_print_enum_field_proto(int8, int8_t) +__define_print_enum_field_proto(int16, int16_t) +__define_print_enum_field_proto(int32, int32_t) +__define_print_enum_field_proto(int64, int64_t) +__define_print_enum_field_proto(bool, flatbuffers_bool_t) + +__define_print_scalar_optional_field_proto(uint8, uint8_t) +__define_print_scalar_optional_field_proto(uint16, uint16_t) +__define_print_scalar_optional_field_proto(uint32, uint32_t) +__define_print_scalar_optional_field_proto(uint64, uint64_t) +__define_print_scalar_optional_field_proto(int8, int8_t) +__define_print_scalar_optional_field_proto(int16, int16_t) +__define_print_scalar_optional_field_proto(int32, int32_t) +__define_print_scalar_optional_field_proto(int64, int64_t) +__define_print_scalar_optional_field_proto(bool, flatbuffers_bool_t) +__define_print_scalar_optional_field_proto(float, float) +__define_print_scalar_optional_field_proto(double, double) + +__define_print_enum_optional_field_proto(uint8, uint8_t) +__define_print_enum_optional_field_proto(uint16, uint16_t) +__define_print_enum_optional_field_proto(uint32, uint32_t) +__define_print_enum_optional_field_proto(uint64, uint64_t) +__define_print_enum_optional_field_proto(int8, int8_t) +__define_print_enum_optional_field_proto(int16, int16_t) +__define_print_enum_optional_field_proto(int32, int32_t) +__define_print_enum_optional_field_proto(int64, int64_t) +__define_print_enum_optional_field_proto(bool, flatbuffers_bool_t) + +__define_print_scalar_struct_field_proto(uint8, uint8_t) +__define_print_scalar_struct_field_proto(uint16, uint16_t) +__define_print_scalar_struct_field_proto(uint32, uint32_t) +__define_print_scalar_struct_field_proto(uint64, uint64_t) +__define_print_scalar_struct_field_proto(int8, int8_t) +__define_print_scalar_struct_field_proto(int16, int16_t) +__define_print_scalar_struct_field_proto(int32, int32_t) +__define_print_scalar_struct_field_proto(int64, int64_t) +__define_print_scalar_struct_field_proto(bool, flatbuffers_bool_t) +__define_print_scalar_struct_field_proto(float, float) +__define_print_scalar_struct_field_proto(double, double) + +/* + * char arrays are special as there are no char fields + * without arrays and because they are printed as strings. + */ +__define_print_scalar_array_struct_field_proto(char, char) + +__define_print_scalar_array_struct_field_proto(uint8, uint8_t) +__define_print_scalar_array_struct_field_proto(uint16, uint16_t) +__define_print_scalar_array_struct_field_proto(uint32, uint32_t) +__define_print_scalar_array_struct_field_proto(uint64, uint64_t) +__define_print_scalar_array_struct_field_proto(int8, int8_t) +__define_print_scalar_array_struct_field_proto(int16, int16_t) +__define_print_scalar_array_struct_field_proto(int32, int32_t) +__define_print_scalar_array_struct_field_proto(int64, int64_t) +__define_print_scalar_array_struct_field_proto(bool, flatbuffers_bool_t) +__define_print_scalar_array_struct_field_proto(float, float) +__define_print_scalar_array_struct_field_proto(double, double) + +__define_print_enum_array_struct_field_proto(uint8, uint8_t) +__define_print_enum_array_struct_field_proto(uint16, uint16_t) +__define_print_enum_array_struct_field_proto(uint32, uint32_t) +__define_print_enum_array_struct_field_proto(uint64, uint64_t) +__define_print_enum_array_struct_field_proto(int8, int8_t) +__define_print_enum_array_struct_field_proto(int16, int16_t) +__define_print_enum_array_struct_field_proto(int32, int32_t) +__define_print_enum_array_struct_field_proto(int64, int64_t) +__define_print_enum_array_struct_field_proto(bool, flatbuffers_bool_t) + +__define_print_enum_struct_field_proto(uint8, uint8_t) +__define_print_enum_struct_field_proto(uint16, uint16_t) +__define_print_enum_struct_field_proto(uint32, uint32_t) +__define_print_enum_struct_field_proto(uint64, uint64_t) +__define_print_enum_struct_field_proto(int8, int8_t) +__define_print_enum_struct_field_proto(int16, int16_t) +__define_print_enum_struct_field_proto(int32, int32_t) +__define_print_enum_struct_field_proto(int64, int64_t) +__define_print_enum_struct_field_proto(bool, flatbuffers_bool_t) + +__define_print_scalar_vector_field_proto(uint8, uint8_t) +__define_print_scalar_vector_field_proto(uint16, uint16_t) +__define_print_scalar_vector_field_proto(uint32, uint32_t) +__define_print_scalar_vector_field_proto(uint64, uint64_t) +__define_print_scalar_vector_field_proto(int8, int8_t) +__define_print_scalar_vector_field_proto(int16, int16_t) +__define_print_scalar_vector_field_proto(int32, int32_t) +__define_print_scalar_vector_field_proto(int64, int64_t) +__define_print_scalar_vector_field_proto(bool, flatbuffers_bool_t) +__define_print_scalar_vector_field_proto(float, float) +__define_print_scalar_vector_field_proto(double, double) + +__define_print_enum_vector_field_proto(uint8, uint8_t) +__define_print_enum_vector_field_proto(uint16, uint16_t) +__define_print_enum_vector_field_proto(uint32, uint32_t) +__define_print_enum_vector_field_proto(uint64, uint64_t) +__define_print_enum_vector_field_proto(int8, int8_t) +__define_print_enum_vector_field_proto(int16, int16_t) +__define_print_enum_vector_field_proto(int32, int32_t) +__define_print_enum_vector_field_proto(int64, int64_t) +__define_print_enum_vector_field_proto(bool, flatbuffers_bool_t) + +void flatcc_json_printer_uint8_vector_base64_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, int urlsafe); + +/* + * If `fid` is null, the identifier is not checked and is allowed to be + * entirely absent. + * + * The buffer must at least be aligned to uoffset_t on systems that + * require aligned memory addresses (as always for flatbuffers). + */ +int flatcc_json_printer_table_as_root(flatcc_json_printer_t *ctx, + const void *buf, size_t bufsiz, const char *fid, + flatcc_json_printer_table_f *pf); + +int flatcc_json_printer_struct_as_root(flatcc_json_printer_t *ctx, + const void *buf, size_t bufsiz, const char *fid, + flatcc_json_printer_struct_f *pf); + +/* + * Call before and after enum flags to ensure proper quotation. Enum + * quotes may be configured runtime, but regardless of this, multiple + * flags may be forced to be quoted depending on compile time flag since + * not all parsers may be able to handle unquoted space separated values + * even if they handle non-strict unquoted json otherwise. + * + * Flags should only be called when not empty (0) and when there are no + * unknown flags in the value. Otherwise print the numeric value. The + * auto generated code deals with this. + * + * This bit twiddling hack may be useful: + * + * `multiple = 0 != (v & (v - 1);` + */ +void flatcc_json_printer_delimit_enum_flags(flatcc_json_printer_t *ctx, int multiple); + +/* The index increments from 0 to handle space. It is not the flag bit position. */ +void flatcc_json_printer_enum_flag(flatcc_json_printer_t *ctx, int index, const char *symbol, size_t len); + +/* A struct inside another struct, as opposed to inside a table or a root. */ +void flatcc_json_printer_embedded_struct_field(flatcc_json_printer_t *ctx, + int index, const void *p, size_t offset, + const char *name, size_t len, + flatcc_json_printer_struct_f pf); + +void flatcc_json_printer_embedded_struct_array_field(flatcc_json_printer_t *ctx, + int index, const void *p, size_t offset, + const char *name, size_t len, + size_t size, size_t count, + flatcc_json_printer_struct_f pf); + +void flatcc_json_printer_struct_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_struct_f *pf); + +void flatcc_json_printer_string_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len); + +void flatcc_json_printer_string_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len); + +void flatcc_json_printer_table_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_table_f pf); + +void flatcc_json_printer_struct_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + size_t size, + flatcc_json_printer_struct_f pf); + +void flatcc_json_printer_table_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_table_f pf); + +void flatcc_json_printer_union_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_union_type_f ptf, + flatcc_json_printer_union_f pf); + +void flatcc_json_printer_struct_as_nested_root(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + const char *fid, + flatcc_json_printer_struct_f *pf); + +void flatcc_json_printer_table_as_nested_root(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + const char *fid, + flatcc_json_printer_table_f pf); + +void flatcc_json_printer_union_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_union_type_f ptf, + flatcc_json_printer_union_f pf); + +void flatcc_json_printer_union_table(flatcc_json_printer_t *ctx, + flatcc_json_printer_union_descriptor_t *ud, + flatcc_json_printer_table_f pf); + +void flatcc_json_printer_union_struct(flatcc_json_printer_t *ctx, + flatcc_json_printer_union_descriptor_t *ud, + flatcc_json_printer_struct_f pf); + +void flatcc_json_printer_union_string(flatcc_json_printer_t *ctx, + flatcc_json_printer_union_descriptor_t *ud); + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_JSON_PRINTER_H */ diff --git a/nostrdb/flatcc/flatcc_portable.h b/nostrdb/flatcc/flatcc_portable.h new file mode 100644 index 0000000000..9b0eb0c3f7 --- /dev/null +++ b/nostrdb/flatcc/flatcc_portable.h @@ -0,0 +1,14 @@ +#ifndef FLATCC_PORTABLE_H +#define FLATCC_PORTABLE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "flatcc/portable/portable_basic.h" + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_PORTABLE_H */ diff --git a/nostrdb/flatcc/flatcc_prologue.h b/nostrdb/flatcc/flatcc_prologue.h new file mode 100644 index 0000000000..36344c4c07 --- /dev/null +++ b/nostrdb/flatcc/flatcc_prologue.h @@ -0,0 +1,8 @@ +/* Include guard intentionally left out. */ + +#define PDIAGNOSTIC_IGNORE_UNUSED +#include "pdiagnostic_push.h" + +#ifdef __cplusplus +extern "C" { +#endif diff --git a/nostrdb/flatcc/flatcc_refmap.h b/nostrdb/flatcc/flatcc_refmap.h new file mode 100644 index 0000000000..beafa301d0 --- /dev/null +++ b/nostrdb/flatcc/flatcc_refmap.h @@ -0,0 +1,144 @@ +/* + * The flatcc builder supports storing a pointer to a refmap + * and wraps some operations to make them work as a dummy + * even if no refmap has been set. This enables optional + * DAG preservation possible during clone operations. + * + * A refmap maps a source address to a builder reference. + * + * This is just a map, but the semantics are important: + * + * The map thus preserves identity of the source. It is not a + * cache because cache eviction would fail to properly track + * identity. + * + * The map is used for memoization during object cloning are and + * may also be used by user logic doing similar operations. + * This ensures that identity is preserved so a source object is + * not duplicated which could lead to either loss of semantic + * information, or an explosion in size, or both. In some, or + * even most, cases this concern may not be important, but when + * it is important, it is important. + * + * The source address must not be reused for different content + * for the lifetime of the map, although the content doest not + * have to be valid or event exist at that location since source + * address is just used as a key. + * + * The lifetime may be a single clone operation which then + * tracks child object references as well, or it may be the + * lifetime of the buffer builder. + * + * The map may be flushed explicitly when the source addresses + * are no longer unique, such as when reusing a memory buffer, + * and when identity preservation is no longer important. + * Flushing a map is esentially the same as ending a lifetime. + * + * Multiple maps may exist concurrently for example if cloning + * an object twice into two new objects that should have + * separate identities. This is especially true and necessary + * when creating a new nested buffer because the nested buffer + * cannot share references with the parent. Cloning and object + * that contains a nested buffer does not require multiple maps + * because the nested buffer is then opaque. + */ + +#ifndef FLATCC_REFMAP_H +#define FLATCC_REFMAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "flatcc_types.h" + +#ifndef FLATCC_REFMAP_MIN_BUCKETS +/* 8 buckets gives us 5 useful initial entries with a load factor of 0.7 */ +#define FLATCC_REFMAP_MIN_BUCKETS 8 +#endif + +#define FLATCC_REFMAP_LOAD_FACTOR 0.7f + +typedef struct flatcc_refmap flatcc_refmap_t; +typedef flatbuffers_soffset_t flatcc_refmap_ref_t; + +static const flatcc_refmap_ref_t flatcc_refmap_not_found = 0; + +struct flatcc_refmap_item { + const void *src; + flatcc_refmap_ref_t ref; +}; + +struct flatcc_refmap { + size_t count; + size_t buckets; + struct flatcc_refmap_item *table; + /* Use stack allocation for small maps. */ + struct flatcc_refmap_item min_table[FLATCC_REFMAP_MIN_BUCKETS]; +}; + +/* + * Fast zero initialization - does not allocate any memory. + * May be replaced by memset 0, but `init` avoids clearing the + * stack allocated initial hash table until it is needed. + */ +static inline int flatcc_refmap_init(flatcc_refmap_t *refmap) +{ + refmap->count = 0; + refmap->buckets = 0; + refmap->table = 0; + return 0; +} + +/* + * Removes all items and deallocates memory. + * Not required unless `insert` or `resize` took place. The map can be + * reused subsequently without calling `init`. + */ +void flatcc_refmap_clear(flatcc_refmap_t *refmap); + +/* + * Keeps allocated memory as is, but removes all items. The map + * must intialized first. + */ +void flatcc_refmap_reset(flatcc_refmap_t *refmap); + +/* + * Returns the inserted reference if the `src` pointer was found, + * without inspecting the content of the `src` pointer. + * + * Returns flatcc_refmap_not_found (default 0) if the `src` pointer was + * not found. + */ +flatcc_refmap_ref_t flatcc_refmap_find(flatcc_refmap_t *refmap, const void *src); + +/* + * Inserts a `src` source pointer and its associated `ref` reference + * into the refmap without inspecting the `src` pointer content. The + * `ref` value will be replaced if the the `src` pointer already exists. + * + * Inserting null will just return the ref without updating the map. + * + * There is no delete operation which simplifies an open + * addressing hash table, and it isn't needed for this use case. + * + * Returns the input ref or not_found on allocation error. + */ +flatcc_refmap_ref_t flatcc_refmap_insert(flatcc_refmap_t *refmap, const void *src, flatcc_refmap_ref_t ref); + +/* + * Set the hash table to accommodate at least `count` items while staying + * within the predefined load factor. + * + * Resize is primarily an internal operation, but the user may resize + * ahead of a large anticipated load, or after a large load to shrink + * the table using 0 as the `count` argument. The table never shrinks + * on its own account. + */ +int flatcc_refmap_resize(flatcc_refmap_t *refmap, size_t count); + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_REFMAP_H */ diff --git a/nostrdb/flatcc/flatcc_rtconfig.h b/nostrdb/flatcc/flatcc_rtconfig.h new file mode 100644 index 0000000000..59727b6516 --- /dev/null +++ b/nostrdb/flatcc/flatcc_rtconfig.h @@ -0,0 +1,162 @@ +#ifndef FLATCC_RTCONFIG_H +#define FLATCC_RTCONFIG_H + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Include portability layer here since all other files depend on it. */ +#ifdef FLATCC_PORTABLE +#include "flatcc/portable/portable.h" +#endif + +/* + * Fast printing and parsing of double. + * + * This requires the grisu3/grisu3_* files to be in the include path, + * otherwise strod and sprintf will be used (these needed anyway + * as fallback for cases not supported by grisu3). + */ +#ifndef FLATCC_USE_GRISU3 +#define FLATCC_USE_GRISU3 1 +#endif + +/* + * This requires compiler that has enabled marc=native or similar so + * __SSE4_2__ flag is defined. Otherwise it will have no effect. + * + * While SSE may be used for different purposes, it has (as of this + * writing) only be used to test the effect on JSON whitespace handling + * which improved, but not by a lot, assuming 64-bit unligned access is + * otherwise available: + * + * With 8 space indentation, the JSON benchmark handles 308K parse ops/sec + * while SSE ups that to 333 parse ops/sec or 336 if \r\n is also + * consumed by SSE. Disabling indentation leaves SSE spacing handling + * ineffective, and performance reaches 450K parse ops/sec and can + * improve further to 500+K parse ops/sec if inexact GRISU3 numbers are + * allowed (they are pretty accurate anyway, just not exact). This + * feature requires hacking a flag direct in the grisu3 double parsing + * lib directly and only mentioned for comparison. + * + * In conclusion SSE doesn't add a lot to JSON space handling at least. + * + * Disabled by default, but can be overriden by build system. + */ +#ifndef FLATCC_USE_SSE4_2 +#define FLATCC_USE_SSE4_2 0 +#endif + +/* + * The verifier only reports yes and no. The following setting + * enables assertions in debug builds. It must be compiled into + * the runtime library and is not normally the desired behavior. + * + * NOTE: enabling this can break test cases so use with build, not test. + */ +#if !defined(FLATCC_DEBUG_VERIFY) && !defined(NDEBUG) +#define FLATCC_DEBUG_VERIFY 0 +#endif + +#if !defined(FLATCC_TRACE_VERIFY) +#define FLATCC_TRACE_VERIFY 0 +#endif + + +/* + * Limit recursion level for tables. Actual level may be deeper + * when structs are deeply nested - but these are limited by the + * schema compiler. + */ +#ifndef FLATCC_JSON_PRINT_MAX_LEVELS +#define FLATCC_JSON_PRINT_MAX_LEVELS 100 +#endif + +/* Maximum length of names printed exluding _type suffix. */ +#ifndef FLATCC_JSON_PRINT_NAME_LEN_MAX +#define FLATCC_JSON_PRINT_NAME_LEN_MAX 100 +#endif + +/* + * Print float and double values with C99 hexadecimal floating point + * notation. This option is not valid JSON but it avoids precision + * loss, correctly handles NaN, +/-Infinity and is significantly faster + * to parse and print. Some JSON parsers rely on strtod which does + * support hexadecimal floating points when C99 compliant. + */ +#ifndef FLATCC_JSON_PRINT_HEX_FLOAT +#define FLATCC_JSON_PRINT_HEX_FLOAT 0 +#endif + +/* + * Always print multipe enum flags like `color: "Red Green"` + * even when unquote is selected as an option for single + * value like `color: Green`. Otherwise multiple values + * are printed as `color: Red Green`, but this could break + * some flatbuffer json parser. + */ +#ifndef FLATCC_JSON_PRINT_ALWAYS_QUOTE_MULTIPLE_FLAGS +#define FLATCC_JSON_PRINT_ALWAYS_QUOTE_MULTIPLE_FLAGS 1 +#endif + +/* + * The general nesting limit may be lower, but for skipping + * JSON we do not need to - we can set this high as it only + * costs a single char per level in a stack array. + */ +#ifndef FLATCC_JSON_PARSE_GENERIC_MAX_NEST +#define FLATCC_JSON_PARSE_GENERIC_MAX_NEST 512 +#endif + +/* Store value even if it is default. */ +#ifndef FLATCC_JSON_PARSE_FORCE_DEFAULTS +#define FLATCC_JSON_PARSE_FORCE_DEFAULTS 0 +#endif + +#ifndef FLATCC_JSON_PARSE_ALLOW_UNQUOTED +#define FLATCC_JSON_PARSE_ALLOW_UNQUOTED 1 +#endif + +/* + * Multiple enum values are by default not permitted unless + * quoted like `color: "Red Green" as per Googles flatc JSON + * parser while a single value like `color: Red` can be + * unquoted. Enabling this setting will allow `color: Red + * Green`, but only if FLATCC_JSON_PARSE_ALLOW_UNQUOTED is + * also enabled. + */ +#ifndef FLATCC_JSON_PARSE_ALLOW_UNQUOTED_LIST +#define FLATCC_JSON_PARSE_ALLOW_UNQUOTED_LIST 0 +#endif + +#ifndef FLATCC_JSON_PARSE_ALLOW_UNKNOWN_FIELD +#define FLATCC_JSON_PARSE_ALLOW_UNKNOWN_FIELD 1 +#endif + +#ifndef FLATCC_JSON_PARSE_ALLOW_TRAILING_COMMA +#define FLATCC_JSON_PARSE_ALLOW_TRAILING_COMMA 1 +#endif + +/* + * Just parse to the closing bracket '}' if set. + * Otherwise parse to end by consuming space and + * fail if anything but space follows. + */ +#ifndef FLATCC_PARSE_IGNORE_TRAILING_DATA +#define FLATCC_PARSE_IGNORE_TRAILING_DATA 0 +#endif + +/* + * Optimize to parse a lot of white space, but + * in most cases it probably slows parsing down. + */ +#ifndef FLATCC_JSON_PARSE_WIDE_SPACE +#define FLATCC_JSON_PARSE_WIDE_SPACE 0 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_RTCONFIG_H */ diff --git a/nostrdb/flatcc/flatcc_types.h b/nostrdb/flatcc/flatcc_types.h new file mode 100644 index 0000000000..69605d2d63 --- /dev/null +++ b/nostrdb/flatcc/flatcc_types.h @@ -0,0 +1,97 @@ +#ifndef FLATCC_TYPES_H +#define FLATCC_TYPES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifndef UINT8_MAX +#include +#endif + +/* + * This should match generated type declaratios in + * `flatbuffers_common_reader.h` (might have different name prefix). + * Read only generated code does not depend on library code, + * hence the duplication. + */ +#ifndef flatbuffers_types_defined +#define flatbuffers_types_defined + +/* + * uoffset_t and soffset_t must be same integer type, except for sign. + * They can be (u)int16_t, (u)int32_t, or (u)int64_t. + * The default is (u)int32_t. + * + * voffset_t is expected to be uint16_t, but can experimentally be + * compiled from uint8_t up to uint32_t. + * + * ID_MAX is the largest value that can index a vtable. The table size + * is given as voffset value. Each id represents a voffset value index + * from 0 to max inclusive. Space is required for two header voffset + * fields and the unaddressible highest index (due to the table size + * representation). For 16-bit voffsets this yields a max of 2^15 - 4, + * or (2^16 - 1) / 2 - 3. + */ + +#define flatbuffers_uoffset_t_defined +#define flatbuffers_soffset_t_defined +#define flatbuffers_voffset_t_defined +#define flatbuffers_utype_t_defined +#define flatbuffers_bool_t_defined +#define flatbuffers_thash_t_defined +#define flatbuffers_fid_t_defined + +/* uoffset_t is also used for vector and string headers. */ +#define FLATBUFFERS_UOFFSET_MAX UINT32_MAX +#define FLATBUFFERS_SOFFSET_MAX INT32_MAX +#define FLATBUFFERS_SOFFSET_MIN INT32_MIN +#define FLATBUFFERS_VOFFSET_MAX UINT16_MAX +#define FLATBUFFERS_UTYPE_MAX UINT8_MAX +/* Well - the max of the underlying type. */ +#define FLATBUFFERS_BOOL_MAX UINT8_MAX +#define FLATBUFFERS_THASH_MAX UINT32_MAX + +#define FLATBUFFERS_ID_MAX (FLATBUFFERS_VOFFSET_MAX / sizeof(flatbuffers_voffset_t) - 3) +/* Vectors of empty structs can yield div by zero, so we must guard against this. */ +#define FLATBUFFERS_COUNT_MAX(elem_size) (FLATBUFFERS_UOFFSET_MAX/((elem_size) == 0 ? 1 : (elem_size))) + +#define FLATBUFFERS_UOFFSET_WIDTH 32 +#define FLATBUFFERS_COUNT_WIDTH 32 +#define FLATBUFFERS_SOFFSET_WIDTH 32 +#define FLATBUFFERS_VOFFSET_WIDTH 16 +#define FLATBUFFERS_UTYPE_WIDTH 8 +#define FLATBUFFERS_BOOL_WIDTH 8 +#define FLATBUFFERS_THASH_WIDTH 32 + +#define FLATBUFFERS_TRUE 1 +#define FLATBUFFERS_FALSE 0 + +#define FLATBUFFERS_PROTOCOL_IS_LE 1 +#define FLATBUFFERS_PROTOCOL_IS_BE 0 + +typedef uint32_t flatbuffers_uoffset_t; +typedef int32_t flatbuffers_soffset_t; +typedef uint16_t flatbuffers_voffset_t; +typedef uint8_t flatbuffers_utype_t; +typedef uint8_t flatbuffers_bool_t; +typedef uint32_t flatbuffers_thash_t; +/* Public facing type operations. */ +typedef flatbuffers_utype_t flatbuffers_union_type_t; + +static const flatbuffers_bool_t flatbuffers_true = FLATBUFFERS_TRUE; +static const flatbuffers_bool_t flatbuffers_false = FLATBUFFERS_FALSE; + +#define FLATBUFFERS_IDENTIFIER_SIZE (FLATBUFFERS_THASH_WIDTH / 8) + +typedef char flatbuffers_fid_t[FLATBUFFERS_IDENTIFIER_SIZE]; + +#endif /* flatbuffers_types_defined */ + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_TYPES_H */ diff --git a/nostrdb/flatcc/flatcc_unaligned.h b/nostrdb/flatcc/flatcc_unaligned.h new file mode 100644 index 0000000000..5ea26cede6 --- /dev/null +++ b/nostrdb/flatcc/flatcc_unaligned.h @@ -0,0 +1,16 @@ +#ifndef FLATCC_UNLIGNED_H +#define FLATCC_UNLIGNED_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "punaligned.h" + +#define FLATCC_ALLOW_UNALIGNED_ACCESS PORTABLE_UNALIGNED_ACCESS + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_UNLIGNED_H */ diff --git a/nostrdb/flatcc/flatcc_verifier.h b/nostrdb/flatcc/flatcc_verifier.h new file mode 100644 index 0000000000..7e0d296651 --- /dev/null +++ b/nostrdb/flatcc/flatcc_verifier.h @@ -0,0 +1,239 @@ +#ifndef FLATCC_VERIFIER_H +#define FLATCC_VERIFIER_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Runtime support for verifying flatbuffers. + * + * Link with the verifier implementation file. + * + * Note: + * + * 1) nested buffers will NOT have their identifier verified. + * The user may do so subsequently. The reason is in part because + * the information is not readily avaible without generated reader code, + * in part because the buffer might use a different, but valid, + * identifier and the user has no chance of specifiying this in the + * verifier code. The root verifier also doesn't assume a specific id + * but accepts a user supplied input which may be null. + * + * 2) All offsets in a buffer are verified for alignment relative to the + * buffer start, but the buffer itself is only assumed to aligned to + * uoffset_t. A reader should therefore ensure buffer alignment separately + * before reading the buffer. Nested buffers are in fact checked for + * alignment, but still only relative to the root buffer. + * + * 3) The max nesting level includes nested buffer nestings, so the + * verifier might fail even if the individual buffers are otherwise ok. + * This is to prevent abuse with lots of nested buffers. + * + * + * IMPORTANT: + * + * Even if verifier passes, the buffer may be invalid to access due to + * lack of alignemnt in memory, but the verifier is safe to call. + * + * NOTE: The buffer is not safe to modify after verification because an + * attacker may craft overlapping data structures such that modification + * of one field updates another in a way that violates the buffer + * constraints. This may also be caused by a clever compression scheme. + * + * It is likely faster to rewrite the table although this is also + * dangerous because an attacker (or even normal user) can draft a DAG + * that explodes when expanded carelesslessly. A safer approach is to + * hash all object references written and reuse those that match. This + * will expand references into other objects while bounding expansion + * and it will be safe to update assuming shared objects are ok to + * update. + * + */ + +#include "flatcc/flatcc_types.h" + +#define FLATCC_VERIFY_ERROR_MAP(XX)\ + XX(ok, "ok")\ + XX(buffer_header_too_small, "buffer header too small")\ + XX(identifier_mismatch, "identifier mismatch")\ + XX(max_nesting_level_reached, "max nesting level reached")\ + XX(required_field_missing, "required field missing")\ + XX(runtime_buffer_header_not_aligned, "runtime: buffer header not aligned")\ + XX(runtime_buffer_size_too_large, "runtime: buffer size too large")\ + XX(string_not_zero_terminated, "string not zero terminated")\ + XX(string_out_of_range, "string out of range")\ + XX(struct_out_of_range, "struct out of range")\ + XX(struct_size_overflow, "struct size overflow")\ + XX(struct_unaligned, "struct unaligned")\ + XX(table_field_not_aligned, "table field not aligned")\ + XX(table_field_out_of_range, "table field out of range")\ + XX(table_field_size_overflow, "table field size overflow")\ + XX(table_header_out_of_range_or_unaligned, "table header out of range or unaligned")\ + XX(vector_header_out_of_range_or_unaligned, "vector header out of range or unaligned")\ + XX(string_header_out_of_range_or_unaligned, "string header out of range or unaligned")\ + XX(offset_out_of_range, "offset out of range")\ + XX(table_offset_out_of_range_or_unaligned, "table offset out of range or unaligned")\ + XX(table_size_out_of_range, "table size out of range")\ + XX(type_field_absent_from_required_union_field, "type field absent from required union field")\ + XX(type_field_absent_from_required_union_vector_field, "type field absent from required union vector field")\ + XX(union_cannot_have_a_table_without_a_type, "union cannot have a table without a type")\ + XX(union_type_NONE_cannot_have_a_value, "union value field present with type NONE")\ + XX(vector_count_exceeds_representable_vector_size, "vector count exceeds representable vector size")\ + XX(vector_out_of_range, "vector out of range")\ + XX(vtable_header_out_of_range, "vtable header out of range")\ + XX(vtable_header_too_small, "vtable header too small")\ + XX(vtable_offset_out_of_range_or_unaligned, "vtable offset out of range or unaligned")\ + XX(vtable_size_out_of_range_or_unaligned, "vtable size out of range or unaligned")\ + XX(vtable_size_overflow, "vtable size overflow")\ + XX(union_element_absent_without_type_NONE, "union element absent without type NONE")\ + XX(union_element_present_with_type_NONE, "union element present with type NONE")\ + XX(union_vector_length_mismatch, "union type and table vectors have different lengths")\ + XX(union_vector_verification_not_supported, "union vector verification not supported")\ + XX(not_supported, "not supported") + + +enum flatcc_verify_error_no { +#define XX(no, str) flatcc_verify_error_##no, + FLATCC_VERIFY_ERROR_MAP(XX) +#undef XX +}; + +#define flatcc_verify_ok flatcc_verify_error_ok + +const char *flatcc_verify_error_string(int err); + +/* + * Type specific table verifier function that checks each known field + * for existence in the vtable and then calls the appropriate verifier + * function in this library. + * + * The table descriptor values have been verified for bounds, overflow, + * and alignment, but vtable entries after header must be verified + * for all fields the table verifier function understands. + * + * Calls other typespecific verifier functions recursively whenever a + * table field, union or table vector is encountered. + */ +typedef struct flatcc_table_verifier_descriptor flatcc_table_verifier_descriptor_t; +struct flatcc_table_verifier_descriptor { + /* Pointer to buffer. Not assumed to be aligned beyond uoffset_t. */ + const void *buf; + /* Buffer size. */ + flatbuffers_uoffset_t end; + /* Time to live: number nesting levels left before failure. */ + int ttl; + /* Vtable of current table. */ + const void *vtable; + /* Table offset relative to buffer start */ + flatbuffers_uoffset_t table; + /* Table end relative to buffer start as per vtable[1] field. */ + flatbuffers_voffset_t tsize; + /* Size of vtable in bytes. */ + flatbuffers_voffset_t vsize; +}; + +typedef int flatcc_table_verifier_f(flatcc_table_verifier_descriptor_t *td); + +typedef struct flatcc_union_verifier_descriptor flatcc_union_verifier_descriptor_t; + +struct flatcc_union_verifier_descriptor { + /* Pointer to buffer. Not assumed to be aligned beyond uoffset_t. */ + const void *buf; + /* Buffer size. */ + flatbuffers_uoffset_t end; + /* Time to live: number nesting levels left before failure. */ + int ttl; + /* Type of union value to be verified */ + flatbuffers_utype_t type; + /* Offset relative to buffer start to where union value offset is stored. */ + flatbuffers_uoffset_t base; + /* Offset of union value relative to base. */ + flatbuffers_uoffset_t offset; +}; + +typedef int flatcc_union_verifier_f(flatcc_union_verifier_descriptor_t *ud); + +/* + * The `as_root` functions are normally the only functions called + * explicitly in this interface. + * + * If `fid` is null, the identifier is not checked and is allowed to be entirely absent. + * + * The buffer must at least be aligned to uoffset_t on systems that + * require aligned memory addresses. The buffer pointers alignment is + * not significant to internal verification of the buffer. + */ +int flatcc_verify_struct_as_root(const void *buf, size_t bufsiz, const char *fid, + size_t size, uint16_t align); + +int flatcc_verify_struct_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash, + size_t size, uint16_t align); + +int flatcc_verify_table_as_root(const void *buf, size_t bufsiz, const char *fid, + flatcc_table_verifier_f *root_tvf); + +int flatcc_verify_table_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash, + flatcc_table_verifier_f *root_tvf); +/* + * The buffer header is verified by any of the `_as_root` verifiers, but + * this function may be used as a quick sanity check. + */ +int flatcc_verify_buffer_header(const void *buf, size_t bufsiz, const char *fid); + +int flatcc_verify_typed_buffer_header(const void *buf, size_t bufsiz, flatbuffers_thash_t type_hash); + +/* + * The following functions are typically called by a generated table + * verifier function. + */ + +/* Scalar, enum or struct field. */ +int flatcc_verify_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, size_t size, uint16_t align); +/* Vector of scalars, enums or structs. */ +int flatcc_verify_vector_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, size_t elem_size, uint16_t align, size_t max_count); +int flatcc_verify_string_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required); +int flatcc_verify_string_vector_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required); +int flatcc_verify_table_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, flatcc_table_verifier_f tvf); +int flatcc_verify_table_vector_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, flatcc_table_verifier_f tvf); +/* Table verifiers pass 0 as fid. */ +int flatcc_verify_struct_as_nested_root(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, const char *fid, + size_t size, uint16_t align); +int flatcc_verify_table_as_nested_root(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, const char *fid, + uint16_t align, flatcc_table_verifier_f tvf); + +/* + * A NONE type will not accept a table being present, and a required + * union will not accept a type field being absent, and an absent type + * field will not accept a table field being present. + * + * If the above checks out and the type is not NONE, the uvf callback + * is executed. It must test each known table type and silently accept + * any unknown table type for forward compatibility. A union table + * value is verified without the required flag because an absent table + * encodes a typed NULL value while an absent type field encodes a + * missing union which fails if required. + */ +int flatcc_verify_union_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, flatcc_union_verifier_f uvf); + +int flatcc_verify_union_vector_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, flatcc_union_verifier_f uvf); + +int flatcc_verify_union_table(flatcc_union_verifier_descriptor_t *ud, flatcc_table_verifier_f *tvf); +int flatcc_verify_union_struct(flatcc_union_verifier_descriptor_t *ud, size_t size, uint16_t align); +int flatcc_verify_union_string(flatcc_union_verifier_descriptor_t *ud); + +#ifdef __cplusplus +} +#endif + +#endif /* FLATCC_VERIFIER_H */ diff --git a/nostrdb/flatcc/flatcc_version.h b/nostrdb/flatcc/flatcc_version.h new file mode 100644 index 0000000000..0e677d1c1b --- /dev/null +++ b/nostrdb/flatcc/flatcc_version.h @@ -0,0 +1,14 @@ +#ifdef __cplusplus +extern "C" { +#endif + +#define FLATCC_VERSION_TEXT "0.6.1" +#define FLATCC_VERSION_MAJOR 0 +#define FLATCC_VERSION_MINOR 6 +#define FLATCC_VERSION_PATCH 1 +/* 1 or 0 */ +#define FLATCC_VERSION_RELEASED 1 + +#ifdef __cplusplus +} +#endif diff --git a/nostrdb/flatcc/json_parser.c b/nostrdb/flatcc/json_parser.c new file mode 100644 index 0000000000..06f778da33 --- /dev/null +++ b/nostrdb/flatcc/json_parser.c @@ -0,0 +1,1298 @@ +#include "flatcc_rtconfig.h" +#include "flatcc_json_parser.h" +#include "flatcc_assert.h" + +#define uoffset_t flatbuffers_uoffset_t +#define soffset_t flatbuffers_soffset_t +#define voffset_t flatbuffers_voffset_t +#define utype_t flatbuffers_utype_t + +#define uoffset_size sizeof(uoffset_t) +#define soffset_size sizeof(soffset_t) +#define voffset_size sizeof(voffset_t) +#define utype_size sizeof(utype_t) + +#define offset_size uoffset_size +#if FLATCC_USE_GRISU3 && !defined(PORTABLE_USE_GRISU3) +#define PORTABLE_USE_GRISU3 1 +#endif +#include "portable/pparsefp.h" +#include "portable/pbase64.h" + +#if FLATCC_USE_SSE4_2 +#ifdef __SSE4_2__ +#define USE_SSE4_2 +#endif +#endif + +#ifdef USE_SSE4_2 +#include +#define cmpistri(end, haystack, needle, flags) \ + if (end - haystack >= 16) do { \ + int i; \ + __m128i a = _mm_loadu_si128((const __m128i *)(needle)); \ + do { \ + __m128i b = _mm_loadu_si128((const __m128i *)(haystack)); \ + i = _mm_cmpistri(a, b, flags); \ + haystack += i; \ + } while (i == 16 && end - haystack >= 16); \ + } while(0) +#endif + +const char *flatcc_json_parser_error_string(int err) +{ + switch (err) { +#define XX(no, str) \ + case flatcc_json_parser_error_##no: \ + return str; + FLATCC_JSON_PARSE_ERROR_MAP(XX) +#undef XX + default: + return "unknown"; + } +} + +const char *flatcc_json_parser_set_error(flatcc_json_parser_t *ctx, const char *loc, const char *end, int err) +{ + if (!ctx->error) { + ctx->error = err; + ctx->pos = (int)(loc - ctx->line_start + 1); + ctx->error_loc = loc; + } + return end; +} + +const char *flatcc_json_parser_string_part(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ +/* + * Disabled because it doesn't catch all control characters, but is + * useful for performance testing. + */ +#if 0 +//#ifdef USE_SSE4_2 + cmpistri(end, buf, "\"\\\0\r\n\t\v\f", _SIDD_POSITIVE_POLARITY); +#else + /* + * Testing for signed char >= 0x20 would also capture UTF-8 + * encodings that we could verify, and also invalid encodings like + * 0xff, but we do not wan't to enforce strict UTF-8. + */ + while (buf != end && *buf != '\"' && ((unsigned char)*buf) >= 0x20 && *buf != '\\') { + ++buf; + } +#endif + if (buf == end) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unterminated_string); + } + if (*buf == '"') { + return buf; + } + if (*buf < 0x20) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_character); + } + return buf; +} + +const char *flatcc_json_parser_space_ext(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ +again: +#ifdef USE_SSE4_2 + /* + * We can include line break, but then error reporting suffers and + * it really makes no big difference. + */ + //cmpistri(end, buf, "\x20\t\v\f\r\n", _SIDD_NEGATIVE_POLARITY); + cmpistri(end, buf, "\x20\t\v\f", _SIDD_NEGATIVE_POLARITY); +#else +#if FLATCC_ALLOW_UNALIGNED_ACCESS + while (end - buf >= 16) { + if (*buf > 0x20) { + return buf; + } +#if FLATCC_JSON_PARSE_WIDE_SPACE + if (((uint64_t *)buf)[0] != 0x2020202020202020) { +descend: + if (((uint32_t *)buf)[0] == 0x20202020) { + buf += 4; + } +#endif + if (((uint16_t *)buf)[0] == 0x2020) { + buf += 2; + } + if (*buf == 0x20) { + ++buf; + } + if (*buf > 0x20) { + return buf; + } + break; +#if FLATCC_JSON_PARSE_WIDE_SPACE + } + if (((uint64_t *)buf)[1] != 0x2020202020202020) { + buf += 8; + goto descend; + } + buf += 16; +#endif + } +#endif +#endif + while (buf != end && *buf == 0x20) { + ++buf; + } + while (buf != end && *buf <= 0x20) { + /* Fall through comments needed to silence gcc 7 warnings. */ + switch (*buf) { + case 0x0d: buf += (end - buf > 1 && buf[1] == 0x0a); + /* Consume following LF or treating CR as LF. */ + fallthrough; + case 0x0a: ++ctx->line; ctx->line_start = ++buf; continue; + case 0x09: ++buf; continue; + case 0x20: goto again; /* Don't consume here, sync with power of 2 spaces. */ + default: return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); + } + } + return buf; +} + +static int decode_hex4(const char *buf, uint32_t *result) +{ + uint32_t u, x; + char c; + + u = 0; + c = buf[0]; + if (c >= '0' && c <= '9') { + x = (uint32_t)(c - '0'); + u = x << 12; + } else { + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'f') { + x = (uint32_t)(c - 'a' + 10); + u |= x << 12; + } else { + return -1; + } + } + c = buf[1]; + if (c >= '0' && c <= '9') { + x = (uint32_t)(c - '0'); + u |= x << 8; + } else { + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'f') { + x = (uint32_t)(c - 'a' + 10); + u |= x << 8; + } else { + return -1; + } + } + c = buf[2]; + if (c >= '0' && c <= '9') { + x = (uint32_t)(c - '0'); + u |= x << 4; + } else { + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'f') { + x = (uint32_t)(c - 'a' + 10); + u |= x << 4; + } else { + return -1; + } + } + c = buf[3]; + if (c >= '0' && c <= '9') { + x = (uint32_t)(c - '0'); + u |= x; + } else { + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'f') { + x = (uint32_t)(c - 'a' + 10); + u |= x; + } else { + return -1; + } + } + *result = u; + return 0; +} + +static int decode_unicode_char(uint32_t u, char *code) +{ + if (u <= 0x7f) { + code[0] = 1; + code[1] = (char)u; + } else if (u <= 0x7ff) { + code[0] = 2; + code[1] = (char)(0xc0 | (u >> 6)); + code[2] = (char)(0x80 | (u & 0x3f)); + } else if (u <= 0xffff) { + code[0] = 3; + code[1] = (char)(0xe0 | (u >> 12)); + code[2] = (char)(0x80 | ((u >> 6) & 0x3f)); + code[3] = (char)(0x80 | (u & 0x3f)); + } else if (u <= 0x10ffff) { + code[0] = 4; + code[1] = (char)(0xf0 | (u >> 18)); + code[2] = (char)(0x80 | ((u >> 12) & 0x3f)); + code[3] = (char)(0x80 | ((u >> 6) & 0x3f)); + code[4] = (char)(0x80 | (u & 0x3f)); + } else { + code[0] = 0; + return -1; + } + return 0; +} + +static inline uint32_t combine_utf16_surrogate_pair(uint32_t high, uint32_t low) +{ + return (high - 0xd800) * 0x400 + (low - 0xdc00) + 0x10000; +} + +static inline int decode_utf16_surrogate_pair(uint32_t high, uint32_t low, char *code) +{ + return decode_unicode_char(combine_utf16_surrogate_pair(high, low), code); +} + + +/* + * UTF-8 code points can have up to 4 bytes but JSON can only + * encode up to 3 bytes via the \uXXXX syntax. + * To handle the range U+10000..U+10FFFF two UTF-16 surrogate + * pairs must be used. If this is not detected, the pairs + * survive in the output which is not valid but often tolerated. + * Emojis generally require such a pair, unless encoded + * unescaped in UTF-8. + * + * If a high surrogate pair is detected and a low surrogate pair + * follows, the combined sequence is decoded as a 4 byte + * UTF-8 sequence. Unpaired surrogate halves are decoded as is + * despite being an invalid UTF-8 value. + */ + +const char *flatcc_json_parser_string_escape(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_json_parser_escape_buffer_t code) +{ + char c, v; + uint32_t u, u2; + + if (end - buf < 2 || buf[0] != '\\') { + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + } + switch (buf[1]) { + case 'x': + v = 0; + code[0] = 1; + if (end - buf < 4) { + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + } + c = buf[2]; + if (c >= '0' && c <= '9') { + v |= (c - '0') << 4; + } else { + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'f') { + v |= (c - 'a' + 10) << 4; + } else { + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + } + } + c = buf[3]; + if (c >= '0' && c <= '9') { + v |= c - '0'; + } else { + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'f') { + v |= c - 'a' + 10; + } else { + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + } + } + code[1] = v; + return buf + 4; + case 'u': + if (end - buf < 6) { + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + } + if (decode_hex4(buf + 2, &u)) { + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + }; + /* If a high UTF-16 surrogate half pair was detected */ + if (u >= 0xd800 && u <= 0xdbff && + /* and there is space for a matching low half pair */ + end - buf >= 12 && + /* and there is a second escape following immediately */ + buf[6] == '\\' && buf[7] == 'u' && + /* and it is valid hex */ + decode_hex4(buf + 8, &u2) == 0 && + /* and it is a low UTF-16 surrogate pair */ + u2 >= 0xdc00 && u2 <= 0xdfff) { + /* then decode the pair into a single 4 byte utf-8 sequence. */ + if (decode_utf16_surrogate_pair(u, u2, code)) { + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + } + return buf + 12; + /* + * Otherwise decode unmatched surrogate pairs as is any + * other UTF-8. Some systems might depend on these surviving. + * Leave ignored errors for the next parse step. + */ + } + decode_unicode_char(u, code); + return buf + 6; + case 't': + code[0] = 1; + code[1] = '\t'; + return buf + 2; + case 'n': + code[0] = 1; + code[1] = '\n'; + return buf + 2; + case 'r': + code[0] = 1; + code[1] = '\r'; + return buf + 2; + case 'b': + code[0] = 1; + code[1] = '\b'; + return buf + 2; + case 'f': + code[0] = 1; + code[1] = '\f'; + return buf + 2; + case '\"': + code[0] = 1; + code[1] = '\"'; + return buf + 2; + case '\\': + code[0] = 1; + code[1] = '\\'; + return buf + 2; + case '/': + code[0] = 1; + code[1] = '/'; + return buf + 2; + default: + code[0] = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + } +} + +/* Only applies to unquoted constants during generic parsring, otherwise it is skipped as a string. */ +const char *flatcc_json_parser_skip_constant(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + char c; + const char *k; + + while (buf != end) { + c = *buf; + if ((c & 0x80) || (c == '_') || (c >= '0' && c <= '9') || c == '.') { + ++buf; + continue; + } + /* Upper case. */ + c |= 0x20; + if (c >= 'a' && c <= 'z') { + ++buf; + continue; + } + buf = flatcc_json_parser_space(ctx, (k = buf), end); + if (buf == k) { + return buf; + } + } + return buf; +} + +const char *flatcc_json_parser_match_constant(flatcc_json_parser_t *ctx, const char *buf, const char *end, int pos, int *more) +{ + const char *mark = buf, *k = buf + pos; + + if (end - buf <= pos) { + *more = 0; + return buf; + } +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + if (ctx->unquoted) { + buf = flatcc_json_parser_space(ctx, k, end); + if (buf == end) { + /* + * We cannot make a decision on more. + * Just return end and let parser handle sync point in + * case it is able to resume parse later on. + * For the same reason we do not lower ctx->unquoted. + */ + *more = 0; + return buf; + } + if (buf != k) { + char c = *buf; + /* + * Space was seen - and thus we have a valid match. + * If the next char is an identifier start symbol + * we raise the more flag to support syntax like: + * + * `flags: Hungry Sleepy Awake, ...` + */ + if (c == '_' || (c & 0x80)) { + *more = 1; + return buf; + } + c |= 0x20; + if (c >= 'a' && c <= 'z') { + *more = 1; + return buf; + } + } + /* + * Space was not seen, so the match is only valid if followed + * by a JSON separator symbol, and there cannot be more values + * following so `more` is lowered. + */ + *more = 0; + if (*buf == ',' || *buf == '}' || *buf == ']') { + return buf; + } + return mark; + } +#endif + buf = k; + if (*buf == 0x20) { + ++buf; + while (buf != end && *buf == 0x20) { + ++buf; + } + if (buf == end) { + *more = 0; + return buf; + } + /* We accept untrimmed space like " Green Blue ". */ + if (*buf != '\"') { + *more = 1; + return buf; + } + } + switch (*buf) { + case '\\': + *more = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_escape); + case '\"': + buf = flatcc_json_parser_space(ctx, buf + 1, end); + *more = 0; + return buf; + } + *more = 0; + return mark; +} + +const char *flatcc_json_parser_unmatched_symbol(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + if (ctx->flags & flatcc_json_parser_f_skip_unknown) { + buf = flatcc_json_parser_symbol_end(ctx, buf, end); + buf = flatcc_json_parser_space(ctx, buf, end); + if (buf != end && *buf == ':') { + ++buf; + buf = flatcc_json_parser_space(ctx, buf, end); + } else { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_expected_colon); + } + return flatcc_json_parser_generic_json(ctx, buf, end); + } else { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unknown_symbol); + } +} + +static const char *__flatcc_json_parser_number(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + if (buf == end) { + return buf; + } + if (*buf == '-') { + ++buf; + if (buf == end) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + } + if (*buf == '0') { + ++buf; + } else { + if (*buf < '1' || *buf > '9') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + ++buf; + while (buf != end && *buf >= '0' && *buf <= '9') { + ++buf; + } + } + if (buf != end) { + if (*buf == '.') { + ++buf; + if (*buf < '0' || *buf > '9') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + ++buf; + while (buf != end && *buf >= '0' && *buf <= '9') { + ++buf; + } + } + } + if (buf != end && (*buf == 'e' || *buf == 'E')) { + ++buf; + if (buf == end) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + if (*buf == '+' || *buf == '-') { + ++buf; + } + if (buf == end || *buf < '0' || *buf > '9') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + ++buf; + while (buf != end && *buf >= '0' && *buf <= '9') { + ++buf; + } + } + + /* + * For strtod termination we must ensure the tail is not valid + * including non-json exponent types. The simplest approach is + * to accept anything that could be valid json successor + * characters and reject end of buffer since we expect a closing + * '}'. + * + * The ',' is actually not safe if strtod uses a non-POSIX locale. + */ + if (buf != end) { + switch (*buf) { + case ',': + case ':': + case ']': + case '}': + case ' ': + case '\r': + case '\t': + case '\n': + case '\v': + return buf; + } + } + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); +} + +const char *flatcc_json_parser_double(flatcc_json_parser_t *ctx, const char *buf, const char *end, double *v) +{ + const char *next, *k; + + *v = 0.0; + if (buf == end) { + return buf; + } + k = buf; + if (*buf == '-') ++k; + if (end - k > 1 && (k[0] == '.' || (k[0] == '0' && k[1] == '0'))) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + next = parse_double(buf, (size_t)(end - buf), v); + if (next == 0 || next == buf) { + if (parse_double_isinf(*v)) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_overflow); + } + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + return next; +} + +const char *flatcc_json_parser_float(flatcc_json_parser_t *ctx, const char *buf, const char *end, float *v) +{ + const char *next, *k; + + *v = 0.0; + if (buf == end) { + return buf; + } + k = buf; + if (*buf == '-') ++k; + if (end - k > 1 && (k[0] == '.' || (k[0] == '0' && k[1] == '0'))) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + next = parse_float(buf, (size_t)(end - buf), v); + if (next == 0 || next == buf) { + if (parse_float_isinf(*v)) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_overflow); + } + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_invalid_numeric); + } + return next; +} + +const char *flatcc_json_parser_generic_json(flatcc_json_parser_t *ctx, const char *buf, const char *end) +{ + char stack[FLATCC_JSON_PARSE_GENERIC_MAX_NEST]; + char *sp, *spend; + const char *k; + flatcc_json_parser_escape_buffer_t code; + int more = 0; + + sp = stack; + spend = sp + FLATCC_JSON_PARSE_GENERIC_MAX_NEST; + +again: + if (buf == end) { + return buf; + } + if (sp != stack && sp[-1] == '}') { + /* Inside an object, about to read field name. */ + buf = flatcc_json_parser_symbol_start(ctx, buf, end); + buf = flatcc_json_parser_symbol_end(ctx, buf, end); + buf = flatcc_json_parser_space(ctx, buf, end); + if (buf == end) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unbalanced_object); + } + if (*buf != ':') { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_expected_colon); + } + buf = flatcc_json_parser_space(ctx, buf + 1, end); + } + switch (*buf) { + case '\"': + buf = flatcc_json_parser_string_start(ctx, buf, end); + while (buf != end && *buf != '\"') { + buf = flatcc_json_parser_string_part(ctx, buf, end); + if (buf != end && *buf == '\"') { + break; + } + buf = flatcc_json_parser_string_escape(ctx, buf, end, code); + } + buf = flatcc_json_parser_string_end(ctx, buf, end); + break; + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + buf = __flatcc_json_parser_number(ctx, buf, end); + break; +#if !FLATCC_JSON_PARSE_ALLOW_UNQUOTED + case 't': case 'f': + { + uint8_t v; + buf = flatcc_json_parser_bool(ctx, (k = buf), end, &v); + if (k == buf) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); + } + } + break; + case 'n': + buf = flatcc_json_parser_null((k = buf), end); + if (k == buf) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); + } + break; +#endif + case '[': + if (sp == spend) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_deep_nesting); + } + *sp++ = ']'; + buf = flatcc_json_parser_space(ctx, buf + 1, end); + if (buf != end && *buf == ']') { + break; + } + goto again; + case '{': + if (sp == spend) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_deep_nesting); + } + *sp++ = '}'; + buf = flatcc_json_parser_space(ctx, buf + 1, end); + if (buf != end && *buf == '}') { + break; + } + goto again; + + default: +#if FLATCC_JSON_PARSE_ALLOW_UNQUOTED + buf = flatcc_json_parser_skip_constant(ctx, (k = buf), end); + if (k == buf) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); + } + break; +#else + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unexpected_character); +#endif + } + while (buf != end && sp != stack) { + --sp; + if (*sp == ']') { + buf = flatcc_json_parser_array_end(ctx, buf, end, &more); + } else { + buf = flatcc_json_parser_object_end(ctx, buf, end, &more); + } + if (more) { + ++sp; + goto again; + } + } + if (buf == end && sp != stack) { + return flatcc_json_parser_set_error(ctx, buf, end, sp[-1] == ']' ? + flatcc_json_parser_error_unbalanced_array : + flatcc_json_parser_error_unbalanced_object); + } + /* Any ',', ']', or '}' belongs to parent context. */ + return buf; +} + +const char *flatcc_json_parser_integer(flatcc_json_parser_t *ctx, const char *buf, const char *end, + int *value_sign, uint64_t *value) +{ + uint64_t x0, x = 0; + const char *k; + + if (buf == end) { + return buf; + } + k = buf; + *value_sign = *buf == '-'; + buf += *value_sign; + while (buf != end && *buf >= '0' && *buf <= '9') { + x0 = x; + x = x * 10 + (uint64_t)(*buf - '0'); + if (x0 > x) { + return flatcc_json_parser_set_error(ctx, buf, end, value_sign ? + flatcc_json_parser_error_underflow : flatcc_json_parser_error_overflow); + } + ++buf; + } + if (buf == k) { + /* Give up, but don't fail the parse just yet, it might be a valid symbol. */ + return buf; + } + if (buf != end && (*buf == 'e' || *buf == 'E' || *buf == '.')) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_float_unexpected); + } + *value = x; + return buf; +} + +/* Array Creation - depends on flatcc builder. */ + +const char *flatcc_json_parser_build_uint8_vector_base64(flatcc_json_parser_t *ctx, + const char *buf, const char *end, flatcc_builder_ref_t *ref, int urlsafe) +{ + const char *mark; + uint8_t *pval; + size_t max_len; + size_t decoded_len, src_len; + int mode; + int ret; + + mode = urlsafe ? base64_mode_url : base64_mode_rfc4648; + buf = flatcc_json_parser_string_start(ctx, buf, end); + buf = flatcc_json_parser_string_part(ctx, (mark = buf), end); + if (buf == end || *buf != '\"') { + goto base64_failed; + } + max_len = base64_decoded_size((size_t)(buf - mark)); + if (flatcc_builder_start_vector(ctx->ctx, 1, 1, FLATBUFFERS_COUNT_MAX((utype_size)))) { + goto failed; + } + if (!(pval = flatcc_builder_extend_vector(ctx->ctx, max_len))) { + goto failed; + } + src_len = (size_t)(buf - mark); + decoded_len = max_len; + if ((ret = base64_decode(pval, (const uint8_t *)mark, &decoded_len, &src_len, mode))) { + buf = mark + src_len; + goto base64_failed; + } + if (src_len != (size_t)(buf - mark)) { + buf = mark + src_len; + goto base64_failed; + } + if (decoded_len < max_len) { + if (flatcc_builder_truncate_vector(ctx->ctx, max_len - decoded_len)) { + goto failed; + } + } + if (!(*ref = flatcc_builder_end_vector(ctx->ctx))) { + goto failed; + } + return flatcc_json_parser_string_end(ctx, buf, end); + +failed: + *ref = 0; + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_runtime); + +base64_failed: + *ref = 0; + return flatcc_json_parser_set_error(ctx, buf, end, + urlsafe ? flatcc_json_parser_error_base64url : flatcc_json_parser_error_base64); +} + +const char *flatcc_json_parser_char_array(flatcc_json_parser_t *ctx, + const char *buf, const char *end, char *s, size_t n) +{ + flatcc_json_parser_escape_buffer_t code; + const char *mark; + size_t k = 0; + + buf = flatcc_json_parser_string_start(ctx, buf, end); + if (buf != end) + while (*buf != '\"') { + buf = flatcc_json_parser_string_part(ctx, (mark = buf), end); + if (buf == end) return end; + k = (size_t)(buf - mark); + if (k > n) { + if (!(ctx->flags & flatcc_json_parser_f_skip_array_overflow)) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_array_overflow); + } + k = n; /* Might truncate UTF-8. */ + } + memcpy(s, mark, k); + s += k; + n -= k; + if (*buf == '\"') break; + buf = flatcc_json_parser_string_escape(ctx, buf, end, code); + if (buf == end) return end; + k = (size_t)code[0]; + mark = code + 1; + if (k > n) { + if (!(ctx->flags & flatcc_json_parser_f_skip_array_overflow)) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_array_overflow); + } + k = n; /* Might truncate UTF-8. */ + } + memcpy(s, mark, k); + s += k; + n -= k; + } + if (n != 0) { + if (ctx->flags & flatcc_json_parser_f_reject_array_underflow) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_array_underflow); + } + memset(s, 0, n - k); + } + return flatcc_json_parser_string_end(ctx, buf, end); +} + + +/* String Creation - depends on flatcc builder. */ + +const char *flatcc_json_parser_build_string(flatcc_json_parser_t *ctx, + const char *buf, const char *end, flatcc_builder_ref_t *ref) +{ + flatcc_json_parser_escape_buffer_t code; + const char *mark; + + buf = flatcc_json_parser_string_start(ctx, buf, end); + buf = flatcc_json_parser_string_part(ctx, (mark = buf), end); + if (buf != end && *buf == '\"') { + *ref = flatcc_builder_create_string(ctx->ctx, mark, (size_t)(buf - mark)); + } else { + if (flatcc_builder_start_string(ctx->ctx) || + 0 == flatcc_builder_append_string(ctx->ctx, mark, (size_t)(buf - mark))) goto failed; + while (buf != end && *buf != '\"') { + buf = flatcc_json_parser_string_escape(ctx, buf, end, code); + if (0 == flatcc_builder_append_string(ctx->ctx, code + 1, (size_t)code[0])) goto failed; + if (end != (buf = flatcc_json_parser_string_part(ctx, (mark = buf), end))) { + if (0 == flatcc_builder_append_string(ctx->ctx, mark, (size_t)(buf - mark))) goto failed; + } + } + *ref = flatcc_builder_end_string(ctx->ctx); + } + return flatcc_json_parser_string_end(ctx, buf, end); + +failed: + *ref = 0; + return buf; +} + +/* UNIONS */ + +/* + * Unions are difficult to parse because the type field may appear after + * the union table and because having two fields opens up for many more + * possible error scenarios. We must store each union of a table + * temporarily - this cannot be in the generated table parser function + * because there could be many unions (about 2^15 with default voffsets) + * although usually there will be only a few. We can also not store the + * data encoded in the existing table buffer in builder because we may + * have to remove it due to schema forwarding and removing it messes up + * the table layout. We also cannot naively allocate it dynamically for + * performance reasons. Instead we place the temporary union data in a + * separate frame from the table buffer, but on a similar stack. This is + * called the user stack and we manage one frame per table that is known + * to contain unions. + * + * Even the temporary structures in place we still cannot parse a union + * before we know its type. Due to JSON typically sorting fields + * alphabetically in various pretty printers, we are likely to receive + * the type late with (`_type` following ``. + * To deal with this we store a backtracking pointer and parses the + * table generically in a first pass and reparse the table once the type + * is known. This can happen recursively with nested tables containing + * unions which is why we need to have a stack frame. + * + * If the type field is stored first we just store the type in the + * custom frame and immediately parses the table with the right type + * once we see it. The parse will be much faster and we can strongly + * recommend that flatbuffer serializers do this, but we cannot require + * it. + * + * The actual overhead of dealing with the custom stack frame is fairly + * cheap once we get past the first custom stack allocation. + * + * We cannot update the builder before both the table and table type + * has been parsed because the the type might have to be ingored due + * to schema forwarding. Therefore the union type must be cached or + * reread. This happens trivially be calling the union parser with the + * type as argument, but it is important to be aware of before + * refactoring the code. + * + * The user frame is created at table start and remains valid until + * table exit, but we cannot assume the pointers to the frame remain + * valid. Specifically we cannot use frame pointers after calling + * the union parser. This means the union type must be cached or reread + * so it can be added to the table. Because the type is passed to + * the union parser this caching happens automatically but it is still + * important to be aware that it is required. + * + * The frame reserves temporary information for all unions the table + * holds, enumerated 0 <= `union_index` < `union_total` + * where the `union_total` is fixed type specific number. + * + * The `type_present` is needed because union types range from 0..255 + * and we need an extra bit do distinguish not present from union type + * `NONE = 0`. + */ + +typedef struct { + const char *backtrace; + const char *line_start; + int line; + uint8_t type_present; + uint8_t type; + /* Union vectors: */ + uoffset_t count; + size_t h_types; +} __flatcc_json_parser_union_entry_t; + +typedef struct { + size_t union_total; + size_t union_count; + __flatcc_json_parser_union_entry_t unions[1]; +} __flatcc_json_parser_union_frame_t; + +const char *flatcc_json_parser_prepare_unions(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_total, size_t *handle) +{ + __flatcc_json_parser_union_frame_t *f; + + if (!(*handle = flatcc_builder_enter_user_frame(ctx->ctx, + sizeof(__flatcc_json_parser_union_frame_t) + (union_total - 1) * + sizeof(__flatcc_json_parser_union_entry_t)))) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_runtime); + } + f = flatcc_builder_get_user_frame_ptr(ctx->ctx, *handle); + /* Frames have zeroed memory. */ + f->union_total = union_total; + return buf; +} + +const char *flatcc_json_parser_finalize_unions(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t handle) +{ + __flatcc_json_parser_union_frame_t *f = flatcc_builder_get_user_frame_ptr(ctx->ctx, handle); + + if (f->union_count) { + buf = flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_union_incomplete); + } + flatcc_builder_exit_user_frame_at(ctx->ctx, handle); + return buf; +} + +const char *flatcc_json_parser_union(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, + flatbuffers_voffset_t id, size_t handle, flatcc_json_parser_union_f *union_parser) +{ + __flatcc_json_parser_union_frame_t *f = flatcc_builder_get_user_frame_ptr(ctx->ctx, handle); + __flatcc_json_parser_union_entry_t *e = &f->unions[union_index]; + flatcc_builder_union_ref_t uref; + + if (e->backtrace) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_duplicate); + } + if (!e->type_present) { + /* If we supported table: null, we should not count it, but we don't. */ + ++f->union_count; + e->line = ctx->line; + e->line_start = ctx->line_start; + buf = flatcc_json_parser_generic_json(ctx, (e->backtrace = buf), end); + } else { + uref.type = e->type; + if (e->type == 0) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_union_none_present); + } + --f->union_count; + buf = union_parser(ctx, buf, end, e->type, &uref.value); + if (buf != end) { + if (flatcc_builder_table_add_union(ctx->ctx, id, uref)) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_duplicate); + } + } + } + return buf; +} + +const char *flatcc_json_parser_union_type(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, flatbuffers_voffset_t id, + size_t handle, + flatcc_json_parser_integral_symbol_f *type_parsers[], + flatcc_json_parser_union_f *union_parser) +{ + __flatcc_json_parser_union_frame_t *f = flatcc_builder_get_user_frame_ptr(ctx->ctx, handle); + __flatcc_json_parser_union_entry_t *e = f->unions + union_index; + + flatcc_builder_union_ref_t uref; + const char *mark; + int line; + const char *line_start; + + if (e->type_present) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_duplicate); + } + e->type_present = 1; + buf = flatcc_json_parser_uint8(ctx, (mark = buf), end, &e->type); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_uint8(ctx, buf, end, type_parsers, &e->type); + } + /* Only count the union if the type is not NONE. */ + if (e->backtrace == 0) { + f->union_count += e->type != 0; + return buf; + } + FLATCC_ASSERT(f->union_count); + --f->union_count; + /* + * IMPORTANT: we cannot access any value in the frame or entry + * pointer after calling union parse because it might cause the + * stack to reallocate. We should read the frame pointer again if + * needed - we don't but remember it if refactoring code. + * + * IMPORTANT 2: Do not assign buf here. We are backtracking. + */ + line = ctx->line; + line_start = ctx->line_start; + ctx->line = e->line; + ctx->line_start = e->line_start; + uref.type = e->type; + if (end == union_parser(ctx, e->backtrace, end, e->type, &uref.value)) { + return end; + } + if (flatcc_builder_table_add_union(ctx->ctx, id, uref)) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_duplicate); + } + ctx->line = line; + ctx->line_start = line_start; + return buf; +} + +static const char *_parse_union_vector(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t h_types, uoffset_t count, + flatbuffers_voffset_t id, flatcc_json_parser_union_f *union_parser) +{ + flatcc_builder_ref_t ref = 0, *pref; + utype_t *types; + int more; + size_t i; + + if (flatcc_builder_start_offset_vector(ctx->ctx)) goto failed; + buf = flatcc_json_parser_array_start(ctx, buf, end, &more); + i = 0; + while (more) { + if (i == count) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_union_vector_length); + } + /* Frame must be restored between calls to table parser. */ + types = flatcc_builder_get_user_frame_ptr(ctx->ctx, h_types); + buf = union_parser(ctx, buf, end, types[i], &ref); + if (buf == end) { + return buf; + } + if (!(pref = flatcc_builder_extend_offset_vector(ctx->ctx, 1))) goto failed; + *pref = ref; + buf = flatcc_json_parser_array_end(ctx, buf, end, &more); + ++i; + } + if (i != count) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_union_vector_length); + } + /* Frame must be restored between calls to table parser. */ + types = flatcc_builder_get_user_frame_ptr(ctx->ctx, h_types); + if (!(ref = flatcc_builder_end_offset_vector_for_unions(ctx->ctx, types))) goto failed; + if (!(pref = flatcc_builder_table_add_offset(ctx->ctx, id))) goto failed; + *pref = ref; + return buf; +failed: + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_runtime); +} + +const char *flatcc_json_parser_union_vector(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, + flatbuffers_voffset_t id, size_t handle, flatcc_json_parser_union_f *union_parser) +{ + __flatcc_json_parser_union_frame_t *f = flatcc_builder_get_user_frame_ptr(ctx->ctx, handle); + __flatcc_json_parser_union_entry_t *e = f->unions + union_index; + + if (e->backtrace) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_duplicate); + } + if (!e->type_present) { + ++f->union_count; + e->line = ctx->line; + e->line_start = ctx->line_start; + buf = flatcc_json_parser_generic_json(ctx, (e->backtrace = buf), end); + } else { + --f->union_count; + buf = _parse_union_vector(ctx, buf, end, e->h_types, e->count, id, union_parser); + } + return buf; +} + +const char *flatcc_json_parser_union_type_vector(flatcc_json_parser_t *ctx, + const char *buf, const char *end, size_t union_index, flatbuffers_voffset_t id, + size_t handle, + flatcc_json_parser_integral_symbol_f *type_parsers[], + flatcc_json_parser_union_f *union_parser, + flatcc_json_parser_is_known_type_f accept_type) +{ + __flatcc_json_parser_union_frame_t *f = flatcc_builder_get_user_frame_ptr(ctx->ctx, handle); + __flatcc_json_parser_union_entry_t *e = f->unions + union_index; + + const char *mark; + int line; + const char *line_start; + int more; + utype_t val; + void *pval; + flatcc_builder_ref_t ref, *pref; + utype_t *types; + size_t size; + size_t h_types; + uoffset_t count; + +#if FLATBUFFERS_UTYPE_MAX != UINT8_MAX +#error "Update union vector parser to support current union type definition." +#endif + + if (e->type_present) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_duplicate); + } + e->type_present = 1; + if (flatcc_builder_start_vector(ctx->ctx, 1, 1, FLATBUFFERS_COUNT_MAX((utype_size)))) goto failed; + buf = flatcc_json_parser_array_start(ctx, buf, end, &more); + while (more) { + if (!(pval = flatcc_builder_extend_vector(ctx->ctx, 1))) goto failed; + buf = flatcc_json_parser_uint8(ctx, (mark = buf), end, &val); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_uint8(ctx, (mark = buf), end, type_parsers, &val); + if (buf == mark || buf == end) goto failed; + } + /* Parse unknown types as NONE */ + if (!accept_type(val)) { + if (!(ctx->flags & flatcc_json_parser_f_skip_unknown)) { + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_unknown_union); + } + val = 0; + } + flatbuffers_uint8_write_to_pe(pval, val); + buf = flatcc_json_parser_array_end(ctx, buf, end, &more); + } + count = (uoffset_t)flatcc_builder_vector_count(ctx->ctx); + e->count = count; + size = count * utype_size; + /* Store type vector so it is accessible to the table vector parser. */ + h_types = flatcc_builder_enter_user_frame(ctx->ctx, size); + types = flatcc_builder_get_user_frame_ptr(ctx->ctx, h_types); + memcpy(types, flatcc_builder_vector_edit(ctx->ctx), size); + if (!((ref = flatcc_builder_end_vector(ctx->ctx)))) goto failed; + if (!(pref = flatcc_builder_table_add_offset(ctx->ctx, id - 1))) goto failed; + *pref = ref; + + /* Restore union frame after possible invalidation due to types frame allocation. */ + f = flatcc_builder_get_user_frame_ptr(ctx->ctx, handle); + e = f->unions + union_index; + + e->h_types = h_types; + if (e->backtrace == 0) { + ++f->union_count; + return buf; + } + FLATCC_ASSERT(f->union_count); + --f->union_count; + line = ctx->line; + line_start = ctx->line_start; + ctx->line = e->line; + ctx->line_start = e->line_start; + /* We must not assign buf here because we are backtracking. */ + if (end == _parse_union_vector(ctx, e->backtrace, end, h_types, count, id, union_parser)) return end; + /* + * NOTE: We do not need the user frame anymore, but if we did, it + * would have to be restored from its handle due to the above parse. + */ + ctx->line = line; + ctx->line_start = line_start; + return buf; +failed: + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_runtime); +} + +int flatcc_json_parser_table_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags, const char *fid, + flatcc_json_parser_table_f *parser) +{ + flatcc_json_parser_t _ctx; + flatcc_builder_ref_t root; + int builder_flags = flags & flatcc_json_parser_f_with_size ? flatcc_builder_with_size : 0; + + ctx = ctx ? ctx : &_ctx; + flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags); + if (flatcc_builder_start_buffer(B, fid, 0, builder_flags)) return -1; + buf = parser(ctx, buf, buf + bufsiz, &root); + if (ctx->error) { + return ctx->error; + } + if (!flatcc_builder_end_buffer(B, root)) return -1; + ctx->end_loc = buf; + return 0; +} + +int flatcc_json_parser_struct_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx, + const char *buf, size_t bufsiz, int flags, const char *fid, + flatcc_json_parser_table_f *parser) +{ + flatcc_json_parser_t _ctx; + flatcc_builder_ref_t root; + int builder_flags = flags & flatcc_json_parser_f_with_size ? flatcc_builder_with_size : 0; + + ctx = ctx ? ctx : &_ctx; + flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags); + if (flatcc_builder_start_buffer(B, fid, 0, builder_flags)) return -1; + buf = parser(ctx, buf, buf + bufsiz, &root); + if (ctx->error) { + return ctx->error; + } + if (!flatcc_builder_end_buffer(B, root)) return -1; + ctx->end_loc = buf; + return 0; +} diff --git a/nostrdb/flatcc/json_printer.c b/nostrdb/flatcc/json_printer.c new file mode 100644 index 0000000000..4ebe1c1db1 --- /dev/null +++ b/nostrdb/flatcc/json_printer.c @@ -0,0 +1,1486 @@ +/* + * Runtime support for printing flatbuffers to JSON. + */ + +#include +#include +#include + +#include "flatcc/flatcc_rtconfig.h" +#include "flatcc/flatcc_assert.h" + +/* + * Grisu significantly improves printing speed of floating point values + * and also the overall printing speed when floating point values are + * present in non-trivial amounts. (Also applies to parsing). + */ +#if FLATCC_USE_GRISU3 && !defined(PORTABLE_USE_GRISU3) +#define PORTABLE_USE_GRISU3 1 +#endif + +#include "flatcc/flatcc_flatbuffers.h" +#include "flatcc/flatcc_json_printer.h" +#include "flatcc/flatcc_identifier.h" + +#include "flatcc/portable/pprintint.h" +#include "flatcc/portable/pprintfp.h" +#include "flatcc/portable/pbase64.h" + + +#define RAISE_ERROR(err) flatcc_json_printer_set_error(ctx, flatcc_json_printer_error_##err) + +const char *flatcc_json_printer_error_string(int err) +{ + switch (err) { +#define XX(no, str) \ + case flatcc_json_printer_error_##no: \ + return str; + FLATCC_JSON_PRINT_ERROR_MAP(XX) +#undef XX + default: + return "unknown"; + } +} + +#define flatcc_json_printer_utype_enum_f flatcc_json_printer_union_type_f +#define flatbuffers_utype_read_from_pe __flatbuffers_utype_read_from_pe + +#define uoffset_t flatbuffers_uoffset_t +#define soffset_t flatbuffers_soffset_t +#define voffset_t flatbuffers_voffset_t +#define utype_t flatbuffers_utype_t + +#define uoffset_size sizeof(uoffset_t) +#define soffset_size sizeof(soffset_t) +#define voffset_size sizeof(voffset_t) +#define utype_size sizeof(utype_t) + +#define offset_size uoffset_size + +#if FLATBUFFERS_UTYPE_MAX == UINT8_MAX +#define print_utype print_uint8 +#else +#ifdef FLATBUFFERS_UTYPE_MIN +#define print_utype print_int64 +#else +#define print_utype print_uint64 +#endif +#endif + +static inline const void *read_uoffset_ptr(const void *p) +{ + return (uint8_t *)p + __flatbuffers_uoffset_read_from_pe(p); +} + +static inline voffset_t read_voffset(const void *p, uoffset_t base) +{ + return __flatbuffers_voffset_read_from_pe((uint8_t *)p + base); +} + +static inline const void *get_field_ptr(flatcc_json_printer_table_descriptor_t *td, int id) +{ + uoffset_t vo = (uoffset_t)(id + 2) * (uoffset_t)sizeof(voffset_t); + + if (vo >= (uoffset_t)td->vsize) { + return 0; + } + vo = read_voffset(td->vtable, vo); + if (vo == 0) { + return 0; + } + return (uint8_t *)td->table + vo; +} + +#define print_char(c) *ctx->p++ = (c) + +#define print_null() do { \ + print_char('n'); \ + print_char('u'); \ + print_char('l'); \ + print_char('l'); \ +} while (0) + +#define print_start(c) do { \ + ++ctx->level; \ + *ctx->p++ = c; \ +} while (0) + +#define print_end(c) do { \ + if (ctx->indent) { \ + *ctx->p++ = '\n'; \ + --ctx->level; \ + print_indent(ctx); \ + } \ + *ctx->p++ = c; \ +} while (0) + +#define print_space() do { \ + *ctx->p = ' '; \ + ctx->p += !!ctx->indent; \ +} while (0) + +#define print_nl() do { \ + if (ctx->indent) { \ + *ctx->p++ = '\n'; \ + print_indent(ctx); \ + } else { \ + flatcc_json_printer_flush_partial(ctx); \ + } \ +} while (0) + +/* Call at the end so print_end does not have to check for level. */ +#define print_last_nl() do { \ + if (ctx->indent && ctx->level == 0) { \ + *ctx->p++ = '\n'; \ + } \ + ctx->flush(ctx, 1); \ +} while (0) + +int flatcc_json_printer_fmt_float(char *buf, float n) +{ +#if FLATCC_JSON_PRINT_HEX_FLOAT + return print_hex_float(buf, n); +#else + return print_float(n, buf); +#endif +} + +int flatcc_json_printer_fmt_double(char *buf, double n) +{ +#if FLATCC_JSON_PRINT_HEX_FLOAT + return print_hex_double(buf, n); +#else + return print_double(n, buf); +#endif +} + +int flatcc_json_printer_fmt_bool(char *buf, int n) +{ + if (n) { + memcpy(buf, "true", 4); + return 4; + } + memcpy(buf, "false", 5); + return 5; +} + +static void print_ex(flatcc_json_printer_t *ctx, const char *s, size_t n) +{ + size_t k; + + if (ctx->p >= ctx->pflush) { + ctx->flush(ctx, 0); + } + k = (size_t)(ctx->pflush - ctx->p); + while (n > k) { + memcpy(ctx->p, s, k); + ctx->p += k; + s += k; + n -= k; + ctx->flush(ctx, 0); + k = (size_t)(ctx->pflush - ctx->p); + } + memcpy(ctx->p, s, n); + ctx->p += n; +} + +static inline void print(flatcc_json_printer_t *ctx, const char *s, size_t n) +{ + if (ctx->p + n >= ctx->pflush) { + print_ex(ctx, s, n); + } else { + memcpy(ctx->p, s, n); + ctx->p += n; + } +} + +static void print_escape(flatcc_json_printer_t *ctx, unsigned char c) +{ + unsigned char x; + + print_char('\\'); + switch (c) { + case '"': print_char('\"'); break; + case '\\': print_char('\\'); break; + case '\t' : print_char('t'); break; + case '\f' : print_char('f'); break; + case '\r' : print_char('r'); break; + case '\n' : print_char('n'); break; + case '\b' : print_char('b'); break; + default: + print_char('u'); + print_char('0'); + print_char('0'); + x = c >> 4; + x += x < 10 ? '0' : 'a' - 10; + print_char((char)x); + x = c & 15; + x += x < 10 ? '0' : 'a' - 10; + print_char((char)x); + break; + } +} + +/* + * Even though we know the the string length, we need to scan for escape + * characters. There may be embedded zeroes. Because FlatBuffer strings + * are always zero terminated, we assume and optimize for this. + * + * We enforce \u00xx for control characters, but not for invalid + * characters like 0xff - this makes it possible to handle some other + * codepages transparently while formally not valid. (Formally JSON + * also supports UTF-16/32 little/big endian but flatbuffers only + * support UTF-8 and we expect this in JSON input/output too). + */ +static void print_string(flatcc_json_printer_t *ctx, const char *s, size_t n) +{ + const char *p = s; + /* Unsigned is important. */ + unsigned char c; + size_t k; + + print_char('\"'); + for (;;) { + c = (unsigned char)*p; + while (c >= 0x20 && c != '\"' && c != '\\') { + c = (unsigned char)*++p; + } + k = (size_t)(p - s); + /* Even if k == 0, print ensures buffer flush. */ + print(ctx, s, k); + n -= k; + if (n == 0) break; + s += k; + print_escape(ctx, c); + ++p; + --n; + ++s; + } + print_char('\"'); +} + +/* + * Similar to print_string, but null termination is not guaranteed, and + * trailing nulls are stripped. + */ +static void print_char_array(flatcc_json_printer_t *ctx, const char *s, size_t n) +{ + const char *p = s; + /* Unsigned is important. */ + unsigned char c = 0; + size_t k; + + while (n > 0 && s[n - 1] == '\0') --n; + + print_char('\"'); + for (;;) { + while (n) { + c = (unsigned char)*p; + if (c < 0x20 || c == '\"' || c == '\\') break; + ++p; + --n; + } + k = (size_t)(p - s); + /* Even if k == 0, print ensures buffer flush. */ + print(ctx, s, k); + if (n == 0) break; + s += k; + print_escape(ctx, c); + ++p; + --n; + ++s; + } + print_char('\"'); +} + +static void print_uint8_vector_base64_object(flatcc_json_printer_t *ctx, const void *p, int mode) +{ + const int unpadded_mode = mode & ~base64_enc_modifier_padding; + size_t k, n, len; + const uint8_t *data; + size_t data_len, src_len; + + data_len = (size_t)__flatbuffers_uoffset_read_from_pe(p); + data = (const uint8_t *)p + uoffset_size; + + print_char('\"'); + + len = base64_encoded_size(data_len, mode); + if (ctx->p + len >= ctx->pflush) { + ctx->flush(ctx, 0); + } + while (ctx->p + len > ctx->pflush) { + /* Multiples of 4 output chars consumes exactly 3 bytes before final padding. */ + k = (size_t)(ctx->pflush - ctx->p) & ~(size_t)3; + n = k * 3 / 4; + FLATCC_ASSERT(n > 0); + src_len = k * 3 / 4; + base64_encode((uint8_t *)ctx->p, data, 0, &src_len, unpadded_mode); + ctx->p += k; + data += n; + data_len -= n; + ctx->flush(ctx, 0); + len = base64_encoded_size(data_len, mode); + } + base64_encode((uint8_t *)ctx->p, data, 0, &data_len, mode); + ctx->p += len; + print_char('\"'); +} + +static void print_indent_ex(flatcc_json_printer_t *ctx, size_t n) +{ + size_t k; + + if (ctx->p >= ctx->pflush) { + ctx->flush(ctx, 0); + } + k = (size_t)(ctx->pflush - ctx->p); + while (n > k) { + memset(ctx->p, ' ', k); + ctx->p += k; + n -= k; + ctx->flush(ctx, 0); + k = (size_t)(ctx->pflush - ctx->p); + } + memset(ctx->p, ' ', n); + ctx->p += n; +} + +static inline void print_indent(flatcc_json_printer_t *ctx) +{ + size_t n = (size_t)(ctx->level * ctx->indent); + + if (ctx->p + n > ctx->pflush) { + print_indent_ex(ctx, n); + } else { + memset(ctx->p, ' ', n); + ctx->p += n; + } +} + +/* + * Helpers for external use - does not do autmatic pretty printing, but + * does escape strings. + */ +void flatcc_json_printer_string(flatcc_json_printer_t *ctx, const char *s, size_t n) +{ + print_string(ctx, s, n); +} + +void flatcc_json_printer_write(flatcc_json_printer_t *ctx, const char *s, size_t n) +{ + print(ctx, s, n); +} + +void flatcc_json_printer_nl(flatcc_json_printer_t *ctx) +{ + print_char('\n'); + flatcc_json_printer_flush_partial(ctx); +} + +void flatcc_json_printer_char(flatcc_json_printer_t *ctx, char c) +{ + print_char(c); +} + +void flatcc_json_printer_indent(flatcc_json_printer_t *ctx) +{ + /* + * This is only needed when indent is 0 but helps external users + * to avoid flushing when indenting. + */ + print_indent(ctx); +} + +void flatcc_json_printer_add_level(flatcc_json_printer_t *ctx, int n) +{ + ctx->level += n; +} + +int flatcc_json_printer_get_level(flatcc_json_printer_t *ctx) +{ + return ctx->level; +} + +static inline void print_symbol(flatcc_json_printer_t *ctx, const char *name, size_t len) +{ + *ctx->p = '\"'; + ctx->p += !ctx->unquote; + if (ctx->p + len < ctx->pflush) { + memcpy(ctx->p, name, len); + ctx->p += len; + } else { + print(ctx, name, len); + } + *ctx->p = '\"'; + ctx->p += !ctx->unquote; +} + +static inline void print_name(flatcc_json_printer_t *ctx, const char *name, size_t len) +{ + print_nl(); + print_symbol(ctx, name, len); + print_char(':'); + print_space(); +} + +#define __flatcc_define_json_printer_scalar(TN, T) \ +void flatcc_json_printer_ ## TN( \ + flatcc_json_printer_t *ctx, T v) \ +{ \ + ctx->p += print_ ## TN(v, ctx->p); \ +} + +__flatcc_define_json_printer_scalar(uint8, uint8_t) +__flatcc_define_json_printer_scalar(uint16, uint16_t) +__flatcc_define_json_printer_scalar(uint32, uint32_t) +__flatcc_define_json_printer_scalar(uint64, uint64_t) +__flatcc_define_json_printer_scalar(int8, int8_t) +__flatcc_define_json_printer_scalar(int16, int16_t) +__flatcc_define_json_printer_scalar(int32, int32_t) +__flatcc_define_json_printer_scalar(int64, int64_t) +__flatcc_define_json_printer_scalar(float, float) +__flatcc_define_json_printer_scalar(double, double) + +void flatcc_json_printer_enum(flatcc_json_printer_t *ctx, const char *symbol, size_t len) +{ + print_symbol(ctx, symbol, len); +} + +void flatcc_json_printer_delimit_enum_flags(flatcc_json_printer_t *ctx, int multiple) +{ +#if FLATCC_JSON_PRINT_ALWAYS_QUOTE_MULTIPLE_FLAGS + int quote = !ctx->unquote || multiple; +#else + int quote = !ctx->unquote; +#endif + *ctx->p = '"'; + ctx->p += quote; +} + +void flatcc_json_printer_enum_flag(flatcc_json_printer_t *ctx, int count, const char *symbol, size_t len) +{ + *ctx->p = ' '; + ctx->p += count > 0; + print(ctx, symbol, len); +} + +static inline void print_string_object(flatcc_json_printer_t *ctx, const void *p) +{ + size_t len; + const char *s; + + len = (size_t)__flatbuffers_uoffset_read_from_pe(p); + s = (const char *)p + uoffset_size; + print_string(ctx, s, len); +} + +#define __define_print_scalar_struct_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _struct_field(flatcc_json_printer_t *ctx,\ + int index, const void *p, size_t offset, \ + const char *name, size_t len) \ +{ \ + T x = flatbuffers_ ## TN ## _read_from_pe((uint8_t *)p + offset); \ + \ + if (index) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + ctx->p += print_ ## TN (x, ctx->p); \ +} + +void flatcc_json_printer_char_array_struct_field( + flatcc_json_printer_t *ctx, + int index, const void *p, size_t offset, + const char *name, size_t len, size_t count) +{ + p = (void *)((size_t)p + offset); + if (index) { + print_char(','); + } + print_name(ctx, name, len); + print_char_array(ctx, p, count); +} + +#define __define_print_scalar_array_struct_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _array_struct_field( \ + flatcc_json_printer_t *ctx, \ + int index, const void *p, size_t offset, \ + const char *name, size_t len, size_t count) \ +{ \ + p = (void *)((size_t)p + offset); \ + if (index) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + print_start('['); \ + if (count) { \ + print_nl(); \ + ctx->p += print_ ## TN ( \ + flatbuffers_ ## TN ## _read_from_pe(p), \ + ctx->p); \ + p = (void *)((size_t)p + sizeof(T)); \ + --count; \ + } \ + while (count--) { \ + print_char(','); \ + print_nl(); \ + ctx->p += print_ ## TN ( \ + flatbuffers_ ## TN ## _read_from_pe(p), \ + ctx->p); \ + p = (void *)((size_t)p + sizeof(T)); \ + } \ + print_end(']'); \ +} + +#define __define_print_enum_array_struct_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_array_struct_field( \ + flatcc_json_printer_t *ctx, \ + int index, const void *p, size_t offset, \ + const char *name, size_t len, size_t count, \ + flatcc_json_printer_ ## TN ##_enum_f *pf) \ +{ \ + T x; \ + \ + p = (void *)((size_t)p + offset); \ + if (index) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + print_start('['); \ + if (count) { \ + print_nl(); \ + x = flatbuffers_ ## TN ## _read_from_pe(p); \ + if (ctx->noenum) { \ + ctx->p += print_ ## TN (x, ctx->p); \ + } else { \ + pf(ctx, x); \ + } \ + p = (void *)((size_t)p + sizeof(T)); \ + --count; \ + } \ + while (count--) { \ + print_char(','); \ + print_nl(); \ + x = flatbuffers_ ## TN ## _read_from_pe(p); \ + if (ctx->noenum) { \ + ctx->p += print_ ## TN (x, ctx->p); \ + } else { \ + pf(ctx, x); \ + } \ + p = (void *)((size_t)p + sizeof(T)); \ + } \ + print_end(']'); \ +} + +#define __define_print_enum_struct_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_struct_field( \ + flatcc_json_printer_t *ctx, \ + int index, const void *p, size_t offset, \ + const char *name, size_t len, \ + flatcc_json_printer_ ## TN ##_enum_f *pf) \ +{ \ + T x = flatbuffers_ ## TN ## _read_from_pe((uint8_t *)p + offset); \ + \ + if (index) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + if (ctx->noenum) { \ + ctx->p += print_ ## TN (x, ctx->p); \ + } else { \ + pf(ctx, x); \ + } \ +} + +#define __define_print_scalar_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _field(flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, T v) \ +{ \ + T x; \ + const void *p = get_field_ptr(td, id); \ + \ + if (p) { \ + x = flatbuffers_ ## TN ## _read_from_pe(p); \ + if (x == v && ctx->skip_default) { \ + return; \ + } \ + } else { \ + if (!ctx->force_default) { \ + return; \ + } \ + x = v; \ + } \ + if (td->count++) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + ctx->p += print_ ## TN (x, ctx->p); \ +} + +#define __define_print_scalar_optional_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _optional_field( \ + flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len) \ +{ \ + T x; \ + const void *p = get_field_ptr(td, id); \ + \ + if (!p) return; \ + x = flatbuffers_ ## TN ## _read_from_pe(p); \ + if (td->count++) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + ctx->p += print_ ## TN (x, ctx->p); \ +} + + +#define __define_print_enum_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_field(flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, T v, \ + flatcc_json_printer_ ## TN ##_enum_f *pf) \ +{ \ + T x; \ + const void *p = get_field_ptr(td, id); \ + \ + if (p) { \ + x = flatbuffers_ ## TN ## _read_from_pe(p); \ + if (x == v && ctx->skip_default) { \ + return; \ + } \ + } else { \ + if (!ctx->force_default) { \ + return; \ + } \ + x = v; \ + } \ + if (td->count++) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + if (ctx->noenum) { \ + ctx->p += print_ ## TN (x, ctx->p); \ + } else { \ + pf(ctx, x); \ + } \ +} + +#define __define_print_enum_optional_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_optional_field( \ + flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, \ + flatcc_json_printer_ ## TN ##_enum_f *pf) \ +{ \ + T x; \ + const void *p = get_field_ptr(td, id); \ + \ + if (!p) return; \ + x = flatbuffers_ ## TN ## _read_from_pe(p); \ + if (td->count++) { \ + print_char(','); \ + } \ + print_name(ctx, name, len); \ + if (ctx->noenum) { \ + ctx->p += print_ ## TN (x, ctx->p); \ + } else { \ + pf(ctx, x); \ + } \ +} + +static inline void print_table_object(flatcc_json_printer_t *ctx, + const void *p, int ttl, flatcc_json_printer_table_f pf) +{ + flatcc_json_printer_table_descriptor_t td; + + if (!--ttl) { + flatcc_json_printer_set_error(ctx, flatcc_json_printer_error_deep_recursion); + return; + } + print_start('{'); + td.count = 0; + td.ttl = ttl; + td.table = p; + td.vtable = (uint8_t *)p - __flatbuffers_soffset_read_from_pe(p); + td.vsize = __flatbuffers_voffset_read_from_pe(td.vtable); + pf(ctx, &td); + print_end('}'); +} + +void flatcc_json_printer_string_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len) +{ + const void *p = get_field_ptr(td, id); + + if (p) { + if (td->count++) { + print_char(','); + } + print_name(ctx, name, len); + print_string_object(ctx, read_uoffset_ptr(p)); + } +} + +void flatcc_json_printer_uint8_vector_base64_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, int urlsafe) +{ + const void *p = get_field_ptr(td, id); + int mode; + + mode = urlsafe ? base64_mode_url : base64_mode_rfc4648; + mode |= base64_enc_modifier_padding; + + if (p) { + if (td->count++) { + print_char(','); + } + print_name(ctx, name, len); + print_uint8_vector_base64_object(ctx, read_uoffset_ptr(p), mode); + } +} + +#define __define_print_scalar_vector_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _vector_field( \ + flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len) \ +{ \ + const void *p = get_field_ptr(td, id); \ + uoffset_t count; \ + \ + if (p) { \ + if (td->count++) { \ + print_char(','); \ + } \ + p = read_uoffset_ptr(p); \ + count = __flatbuffers_uoffset_read_from_pe(p); \ + p = (void *)((size_t)p + uoffset_size); \ + print_name(ctx, name, len); \ + print_start('['); \ + if (count) { \ + print_nl(); \ + ctx->p += print_ ## TN ( \ + flatbuffers_ ## TN ## _read_from_pe(p), \ + ctx->p); \ + p = (void *)((size_t)p + sizeof(T)); \ + --count; \ + } \ + while (count--) { \ + print_char(','); \ + print_nl(); \ + ctx->p += print_ ## TN ( \ + flatbuffers_ ## TN ## _read_from_pe(p), \ + ctx->p); \ + p = (void *)((size_t)p + sizeof(T)); \ + } \ + print_end(']'); \ + } \ +} + +#define __define_print_enum_vector_field(TN, T) \ +void flatcc_json_printer_ ## TN ## _enum_vector_field( \ + flatcc_json_printer_t *ctx, \ + flatcc_json_printer_table_descriptor_t *td, \ + int id, const char *name, size_t len, \ + flatcc_json_printer_ ## TN ##_enum_f *pf) \ +{ \ + const void *p; \ + uoffset_t count; \ + \ + if (ctx->noenum) { \ + flatcc_json_printer_ ## TN ## _vector_field(ctx, td, id, name, len);\ + return; \ + } \ + p = get_field_ptr(td, id); \ + if (p) { \ + if (td->count++) { \ + print_char(','); \ + } \ + p = read_uoffset_ptr(p); \ + count = __flatbuffers_uoffset_read_from_pe(p); \ + p = (void *)((size_t)p + uoffset_size); \ + print_name(ctx, name, len); \ + print_start('['); \ + if (count) { \ + print_nl(); \ + pf(ctx, flatbuffers_ ## TN ## _read_from_pe(p)); \ + p = (void *)((size_t)p + sizeof(T)); \ + --count; \ + } \ + while (count--) { \ + print_char(','); \ + print_nl(); \ + pf(ctx, flatbuffers_ ## TN ## _read_from_pe(p)); \ + p = (void *)((size_t)p + sizeof(T)); \ + } \ + print_end(']'); \ + } \ +} + +__define_print_scalar_field(uint8, uint8_t) +__define_print_scalar_field(uint16, uint16_t) +__define_print_scalar_field(uint32, uint32_t) +__define_print_scalar_field(uint64, uint64_t) +__define_print_scalar_field(int8, int8_t) +__define_print_scalar_field(int16, int16_t) +__define_print_scalar_field(int32, int32_t) +__define_print_scalar_field(int64, int64_t) +__define_print_scalar_field(bool, flatbuffers_bool_t) +__define_print_scalar_field(float, float) +__define_print_scalar_field(double, double) + +__define_print_enum_field(uint8, uint8_t) +__define_print_enum_field(uint16, uint16_t) +__define_print_enum_field(uint32, uint32_t) +__define_print_enum_field(uint64, uint64_t) +__define_print_enum_field(int8, int8_t) +__define_print_enum_field(int16, int16_t) +__define_print_enum_field(int32, int32_t) +__define_print_enum_field(int64, int64_t) +__define_print_enum_field(bool, flatbuffers_bool_t) + +__define_print_scalar_optional_field(uint8, uint8_t) +__define_print_scalar_optional_field(uint16, uint16_t) +__define_print_scalar_optional_field(uint32, uint32_t) +__define_print_scalar_optional_field(uint64, uint64_t) +__define_print_scalar_optional_field(int8, int8_t) +__define_print_scalar_optional_field(int16, int16_t) +__define_print_scalar_optional_field(int32, int32_t) +__define_print_scalar_optional_field(int64, int64_t) +__define_print_scalar_optional_field(bool, flatbuffers_bool_t) +__define_print_scalar_optional_field(float, float) +__define_print_scalar_optional_field(double, double) + +__define_print_enum_optional_field(uint8, uint8_t) +__define_print_enum_optional_field(uint16, uint16_t) +__define_print_enum_optional_field(uint32, uint32_t) +__define_print_enum_optional_field(uint64, uint64_t) +__define_print_enum_optional_field(int8, int8_t) +__define_print_enum_optional_field(int16, int16_t) +__define_print_enum_optional_field(int32, int32_t) +__define_print_enum_optional_field(int64, int64_t) +__define_print_enum_optional_field(bool, flatbuffers_bool_t) + +__define_print_scalar_struct_field(uint8, uint8_t) +__define_print_scalar_struct_field(uint16, uint16_t) +__define_print_scalar_struct_field(uint32, uint32_t) +__define_print_scalar_struct_field(uint64, uint64_t) +__define_print_scalar_struct_field(int8, int8_t) +__define_print_scalar_struct_field(int16, int16_t) +__define_print_scalar_struct_field(int32, int32_t) +__define_print_scalar_struct_field(int64, int64_t) +__define_print_scalar_struct_field(bool, flatbuffers_bool_t) +__define_print_scalar_struct_field(float, float) +__define_print_scalar_struct_field(double, double) + +__define_print_scalar_array_struct_field(uint8, uint8_t) +__define_print_scalar_array_struct_field(uint16, uint16_t) +__define_print_scalar_array_struct_field(uint32, uint32_t) +__define_print_scalar_array_struct_field(uint64, uint64_t) +__define_print_scalar_array_struct_field(int8, int8_t) +__define_print_scalar_array_struct_field(int16, int16_t) +__define_print_scalar_array_struct_field(int32, int32_t) +__define_print_scalar_array_struct_field(int64, int64_t) +__define_print_scalar_array_struct_field(bool, flatbuffers_bool_t) +__define_print_scalar_array_struct_field(float, float) +__define_print_scalar_array_struct_field(double, double) + +__define_print_enum_array_struct_field(uint8, uint8_t) +__define_print_enum_array_struct_field(uint16, uint16_t) +__define_print_enum_array_struct_field(uint32, uint32_t) +__define_print_enum_array_struct_field(uint64, uint64_t) +__define_print_enum_array_struct_field(int8, int8_t) +__define_print_enum_array_struct_field(int16, int16_t) +__define_print_enum_array_struct_field(int32, int32_t) +__define_print_enum_array_struct_field(int64, int64_t) +__define_print_enum_array_struct_field(bool, flatbuffers_bool_t) + +__define_print_enum_struct_field(uint8, uint8_t) +__define_print_enum_struct_field(uint16, uint16_t) +__define_print_enum_struct_field(uint32, uint32_t) +__define_print_enum_struct_field(uint64, uint64_t) +__define_print_enum_struct_field(int8, int8_t) +__define_print_enum_struct_field(int16, int16_t) +__define_print_enum_struct_field(int32, int32_t) +__define_print_enum_struct_field(int64, int64_t) +__define_print_enum_struct_field(bool, flatbuffers_bool_t) + +__define_print_scalar_vector_field(utype, flatbuffers_utype_t) +__define_print_scalar_vector_field(uint8, uint8_t) +__define_print_scalar_vector_field(uint16, uint16_t) +__define_print_scalar_vector_field(uint32, uint32_t) +__define_print_scalar_vector_field(uint64, uint64_t) +__define_print_scalar_vector_field(int8, int8_t) +__define_print_scalar_vector_field(int16, int16_t) +__define_print_scalar_vector_field(int32, int32_t) +__define_print_scalar_vector_field(int64, int64_t) +__define_print_scalar_vector_field(bool, flatbuffers_bool_t) +__define_print_scalar_vector_field(float, float) +__define_print_scalar_vector_field(double, double) + +__define_print_enum_vector_field(utype, flatbuffers_utype_t) +__define_print_enum_vector_field(uint8, uint8_t) +__define_print_enum_vector_field(uint16, uint16_t) +__define_print_enum_vector_field(uint32, uint32_t) +__define_print_enum_vector_field(uint64, uint64_t) +__define_print_enum_vector_field(int8, int8_t) +__define_print_enum_vector_field(int16, int16_t) +__define_print_enum_vector_field(int32, int32_t) +__define_print_enum_vector_field(int64, int64_t) +__define_print_enum_vector_field(bool, flatbuffers_bool_t) + +void flatcc_json_printer_struct_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + size_t size, + flatcc_json_printer_struct_f pf) +{ + const uint8_t *p = get_field_ptr(td, id); + uoffset_t count; + + if (p) { + if (td->count++) { + print_char(','); + } + p = read_uoffset_ptr(p); + count = __flatbuffers_uoffset_read_from_pe(p); + p += uoffset_size; + print_name(ctx, name, len); + print_start('['); + if (count) { + print_nl(); + print_start('{'); + pf(ctx, p); + print_end('}'); + --count; + } + while (count--) { + p += size; + print_char(','); + print_nl(); + print_start('{'); + pf(ctx, p); + print_end('}'); + } + print_end(']'); + } +} + +void flatcc_json_printer_string_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len) +{ + const uoffset_t *p = get_field_ptr(td, id); + uoffset_t count; + + if (p) { + if (td->count++) { + print_char(','); + } + p = read_uoffset_ptr(p); + count = __flatbuffers_uoffset_read_from_pe(p); + ++p; + print_name(ctx, name, len); + print_start('['); + if (count) { + print_nl(); + print_string_object(ctx, read_uoffset_ptr(p)); + --count; + } + while (count--) { + ++p; + print_char(','); + print_nl(); + print_string_object(ctx, read_uoffset_ptr(p)); + } + print_end(']'); + } +} + +void flatcc_json_printer_table_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_table_f pf) +{ + const uoffset_t *p = get_field_ptr(td, id); + uoffset_t count; + + if (p) { + if (td->count++) { + print_char(','); + } + p = read_uoffset_ptr(p); + count = __flatbuffers_uoffset_read_from_pe(p); + ++p; + print_name(ctx, name, len); + print_start('['); + if (count) { + print_table_object(ctx, read_uoffset_ptr(p), td->ttl, pf); + --count; + } + while (count--) { + ++p; + print_char(','); + print_table_object(ctx, read_uoffset_ptr(p), td->ttl, pf); + } + print_end(']'); + } +} + +void flatcc_json_printer_union_vector_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_union_type_f ptf, + flatcc_json_printer_union_f pf) +{ + const uoffset_t *pt = get_field_ptr(td, id - 1); + const uoffset_t *p = get_field_ptr(td, id); + utype_t *types, type; + uoffset_t count; + char type_name[FLATCC_JSON_PRINT_NAME_LEN_MAX + 5]; + flatcc_json_printer_union_descriptor_t ud; + + ud.ttl = td->ttl; + if (len > FLATCC_JSON_PRINT_NAME_LEN_MAX) { + RAISE_ERROR(bad_input); + FLATCC_ASSERT(0 && "identifier too long"); + return; + } + memcpy(type_name, name, len); + memcpy(type_name + len, "_type", 5); + if (p && pt) { + flatcc_json_printer_utype_enum_vector_field(ctx, td, id - 1, + type_name, len + 5, ptf); + if (td->count++) { + print_char(','); + } + p = read_uoffset_ptr(p); + pt = read_uoffset_ptr(pt); + count = __flatbuffers_uoffset_read_from_pe(p); + ++p; + ++pt; + types = (utype_t *)pt; + print_name(ctx, name, len); + print_start('['); + + if (count) { + type = __flatbuffers_utype_read_from_pe(types); + if (type != 0) { + ud.type = type; + ud.member = p; + pf(ctx, &ud); + } else { + print_null(); + } + --count; + } + while (count--) { + ++p; + ++types; + type = __flatbuffers_utype_read_from_pe(types); + print_char(','); + if (type != 0) { + ud.type = type; + ud.member = p; + pf(ctx, &ud); + } else { + print_null(); + } + } + print_end(']'); + } +} + +void flatcc_json_printer_table_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_table_f pf) +{ + const void *p = get_field_ptr(td, id); + + if (p) { + if (td->count++) { + print_char(','); + } + print_name(ctx, name, len); + print_table_object(ctx, read_uoffset_ptr(p), td->ttl, pf); + } +} + +void flatcc_json_printer_union_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_union_type_f ptf, + flatcc_json_printer_union_f pf) +{ + const void *pt = get_field_ptr(td, id - 1); + const void *p = get_field_ptr(td, id); + utype_t type; + flatcc_json_printer_union_descriptor_t ud; + + if (!p || !pt) { + return; + } + type = __flatbuffers_utype_read_from_pe(pt); + if (td->count++) { + print_char(','); + } + print_nl(); + *ctx->p = '\"'; + ctx->p += !ctx->unquote; + if (ctx->p + len < ctx->pflush) { + memcpy(ctx->p, name, len); + ctx->p += len; + } else { + print(ctx, name, len); + } + print(ctx, "_type", 5); + *ctx->p = '\"'; + ctx->p += !ctx->unquote; + print_char(':'); + print_space(); + if (ctx->noenum) { + ctx->p += print_utype(type, ctx->p); + } else { + ptf(ctx, type); + } + if (type != 0) { + print_char(','); + print_name(ctx, name, len); + ud.ttl = td->ttl; + ud.type = type; + ud.member = p; + pf(ctx, &ud); + } +} + +void flatcc_json_printer_union_table(flatcc_json_printer_t *ctx, + flatcc_json_printer_union_descriptor_t *ud, + flatcc_json_printer_table_f pf) +{ + print_table_object(ctx, read_uoffset_ptr(ud->member), ud->ttl, pf); +} + +void flatcc_json_printer_union_struct(flatcc_json_printer_t *ctx, + flatcc_json_printer_union_descriptor_t *ud, + flatcc_json_printer_struct_f pf) +{ + print_start('{'); + pf(ctx, read_uoffset_ptr(ud->member)); + print_end('}'); +} + +void flatcc_json_printer_union_string(flatcc_json_printer_t *ctx, + flatcc_json_printer_union_descriptor_t *ud) +{ + print_string_object(ctx, read_uoffset_ptr(ud->member)); +} + +void flatcc_json_printer_embedded_struct_field(flatcc_json_printer_t *ctx, + int index, const void *p, size_t offset, + const char *name, size_t len, + flatcc_json_printer_struct_f pf) +{ + if (index) { + print_char(','); + } + print_name(ctx, name, len); + print_start('{'); + pf(ctx, (uint8_t *)p + offset); + print_end('}'); +} + +void flatcc_json_printer_embedded_struct_array_field(flatcc_json_printer_t *ctx, + int index, const void *p, size_t offset, + const char *name, size_t len, + size_t size, size_t count, + flatcc_json_printer_struct_f pf) +{ + size_t i; + if (index) { + print_char(','); + } + print_name(ctx, name, len); + print_start('['); + for (i = 0; i < count; ++i) { + if (i > 0) { + print_char(','); + } + print_start('{'); \ + pf(ctx, (uint8_t *)p + offset + i * size); + print_end('}'); + } + print_end(']'); +} + +void flatcc_json_printer_struct_field(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + flatcc_json_printer_struct_f *pf) +{ + const void *p = get_field_ptr(td, id); + + if (p) { + if (td->count++) { + print_char(','); + } + print_name(ctx, name, len); + print_start('{'); + pf(ctx, p); + print_end('}'); + } +} + +/* + * Make sure the buffer identifier is valid before assuming the rest of + * the buffer is sane. + * NOTE: this won't work with type hashes because these can contain + * nulls in the fid string. In this case use null as fid to disable + * check. + */ +static int accept_header(flatcc_json_printer_t * ctx, + const void *buf, size_t bufsiz, const char *fid) +{ + flatbuffers_thash_t id, id2 = 0; + + if (buf == 0 || bufsiz < offset_size + FLATBUFFERS_IDENTIFIER_SIZE) { + RAISE_ERROR(bad_input); + FLATCC_ASSERT(0 && "buffer header too small"); + return 0; + } + if (fid != 0) { + id2 = flatbuffers_type_hash_from_string(fid); + id = __flatbuffers_thash_read_from_pe((uint8_t *)buf + offset_size); + if (!(id2 == 0 || id == id2)) { + RAISE_ERROR(bad_input); + FLATCC_ASSERT(0 && "identifier mismatch"); + return 0; + } + } + return 1; +} + +int flatcc_json_printer_struct_as_root(flatcc_json_printer_t *ctx, + const void *buf, size_t bufsiz, const char *fid, + flatcc_json_printer_struct_f *pf) +{ + if (!accept_header(ctx, buf, bufsiz, fid)) { + return -1; + } + print_start('{'); + pf(ctx, read_uoffset_ptr(buf)); + print_end('}'); + print_last_nl(); + return flatcc_json_printer_get_error(ctx) ? -1 : (int)ctx->total + (int)(ctx->p - ctx->buf); +} + +int flatcc_json_printer_table_as_root(flatcc_json_printer_t *ctx, + const void *buf, size_t bufsiz, const char *fid, flatcc_json_printer_table_f *pf) +{ + if (!accept_header(ctx, buf, bufsiz, fid)) { + return -1; + } + print_table_object(ctx, read_uoffset_ptr(buf), FLATCC_JSON_PRINT_MAX_LEVELS, pf); + print_last_nl(); + return flatcc_json_printer_get_error(ctx) ? -1 : (int)ctx->total + (int)(ctx->p - ctx->buf); +} + +void flatcc_json_printer_struct_as_nested_root(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + const char *fid, + flatcc_json_printer_struct_f *pf) +{ + const uoffset_t *buf; + uoffset_t bufsiz; + + if (0 == (buf = get_field_ptr(td, id))) { + return; + } + buf = (const uoffset_t *)((size_t)buf + __flatbuffers_uoffset_read_from_pe(buf)); + bufsiz = __flatbuffers_uoffset_read_from_pe(buf); + if (!accept_header(ctx, buf, bufsiz, fid)) { + return; + } + if (td->count++) { + print_char(','); + } + print_name(ctx, name, len); + print_start('{'); + pf(ctx, read_uoffset_ptr(buf)); + print_end('}'); +} + +void flatcc_json_printer_table_as_nested_root(flatcc_json_printer_t *ctx, + flatcc_json_printer_table_descriptor_t *td, + int id, const char *name, size_t len, + const char *fid, + flatcc_json_printer_table_f pf) +{ + const uoffset_t *buf; + uoffset_t bufsiz; + + if (0 == (buf = get_field_ptr(td, id))) { + return; + } + buf = (const uoffset_t *)((size_t)buf + __flatbuffers_uoffset_read_from_pe(buf)); + bufsiz = __flatbuffers_uoffset_read_from_pe(buf); + ++buf; + if (!accept_header(ctx, buf, bufsiz, fid)) { + return; + } + if (td->count++) { + print_char(','); + } + print_name(ctx, name, len); + print_table_object(ctx, read_uoffset_ptr(buf), td->ttl, pf); +} + +static void __flatcc_json_printer_flush(flatcc_json_printer_t *ctx, int all) +{ + if (!all && ctx->p >= ctx->pflush) { + size_t spill = (size_t)(ctx->p - ctx->pflush); + + fwrite(ctx->buf, ctx->flush_size, 1, ctx->fp); + memcpy(ctx->buf, ctx->buf + ctx->flush_size, spill); + ctx->p = ctx->buf + spill; + ctx->total += ctx->flush_size; + } else { + size_t len = (size_t)(ctx->p - ctx->buf); + + fwrite(ctx->buf, len, 1, ctx->fp); + ctx->p = ctx->buf; + ctx->total += len; + } + *ctx->p = '\0'; +} + +int flatcc_json_printer_init(flatcc_json_printer_t *ctx, void *fp) +{ + memset(ctx, 0, sizeof(*ctx)); + ctx->fp = fp ? fp : stdout; + ctx->flush = __flatcc_json_printer_flush; + if (!(ctx->buf = FLATCC_JSON_PRINTER_ALLOC(FLATCC_JSON_PRINT_BUFFER_SIZE))) { + return -1; + } + ctx->own_buffer = 1; + ctx->size = FLATCC_JSON_PRINT_BUFFER_SIZE; + ctx->flush_size = FLATCC_JSON_PRINT_FLUSH_SIZE; + ctx->p = ctx->buf; + ctx->pflush = ctx->buf + ctx->flush_size; + /* + * Make sure we have space for primitive operations such as printing numbers + * without having to flush. + */ + FLATCC_ASSERT(ctx->flush_size + FLATCC_JSON_PRINT_RESERVE <= ctx->size); + return 0; +} + +static void __flatcc_json_printer_flush_buffer(flatcc_json_printer_t *ctx, int all) +{ + (void)all; + + if (ctx->p >= ctx->pflush) { + RAISE_ERROR(overflow); + ctx->total += (size_t)(ctx->p - ctx->buf); + ctx->p = ctx->buf; + } + *ctx->p = '\0'; +} + +int flatcc_json_printer_init_buffer(flatcc_json_printer_t *ctx, char *buffer, size_t buffer_size) +{ + FLATCC_ASSERT(buffer_size >= FLATCC_JSON_PRINT_RESERVE); + if (buffer_size < FLATCC_JSON_PRINT_RESERVE) { + return -1; + } + memset(ctx, 0, sizeof(*ctx)); + ctx->buf = buffer; + ctx->size = buffer_size; + ctx->flush_size = ctx->size - FLATCC_JSON_PRINT_RESERVE; + ctx->p = ctx->buf; + ctx->pflush = ctx->buf + ctx->flush_size; + ctx->flush = __flatcc_json_printer_flush_buffer; + return 0; +} + +static void __flatcc_json_printer_flush_dynamic_buffer(flatcc_json_printer_t *ctx, int all) +{ + size_t len = (size_t)(ctx->p - ctx->buf); + char *p; + + (void)all; + + *ctx->p = '\0'; + if (ctx->p < ctx->pflush) { + return; + } + p = FLATCC_JSON_PRINTER_REALLOC(ctx->buf, ctx->size * 2); + if (!p) { + RAISE_ERROR(overflow); + ctx->total += len; + ctx->p = ctx->buf; + } else { + ctx->size *= 2; + ctx->flush_size = ctx->size - FLATCC_JSON_PRINT_RESERVE; + ctx->buf = p; + ctx->p = p + len; + ctx->pflush = p + ctx->flush_size; + } + *ctx->p = '\0'; +} + +int flatcc_json_printer_init_dynamic_buffer(flatcc_json_printer_t *ctx, size_t buffer_size) +{ + if (buffer_size == 0) { + buffer_size = FLATCC_JSON_PRINT_DYN_BUFFER_SIZE; + } + if (buffer_size < FLATCC_JSON_PRINT_RESERVE) { + buffer_size = FLATCC_JSON_PRINT_RESERVE; + } + memset(ctx, 0, sizeof(*ctx)); + ctx->buf = FLATCC_JSON_PRINTER_ALLOC(buffer_size); + ctx->own_buffer = 1; + ctx->size = buffer_size; + ctx->flush_size = ctx->size - FLATCC_JSON_PRINT_RESERVE; + ctx->p = ctx->buf; + ctx->pflush = ctx->buf + ctx->flush_size; + ctx->flush = __flatcc_json_printer_flush_dynamic_buffer; + if (!ctx->buf) { + RAISE_ERROR(overflow); + return -1; + } + return 0; +} + +void *flatcc_json_printer_get_buffer(flatcc_json_printer_t *ctx, size_t *buffer_size) +{ + ctx->flush(ctx, 0); + if (buffer_size) { + *buffer_size = (size_t)(ctx->p - ctx->buf); + } + return ctx->buf; +} + +void *flatcc_json_printer_finalize_dynamic_buffer(flatcc_json_printer_t *ctx, size_t *buffer_size) +{ + void *buffer; + + buffer = flatcc_json_printer_get_buffer(ctx, buffer_size); + memset(ctx, 0, sizeof(*ctx)); + return buffer; +} + +void flatcc_json_printer_clear(flatcc_json_printer_t *ctx) +{ + if (ctx->own_buffer && ctx->buf) { + FLATCC_JSON_PRINTER_FREE(ctx->buf); + } + memset(ctx, 0, sizeof(*ctx)); +} diff --git a/nostrdb/flatcc/portable/LICENSE b/nostrdb/flatcc/portable/LICENSE new file mode 100644 index 0000000000..bb7ca578cd --- /dev/null +++ b/nostrdb/flatcc/portable/LICENSE @@ -0,0 +1,14 @@ +Copyright (c) 2016 Mikkel F. Jørgensen, dvide.com +Some files also Copyright author of MathGeoLib (https://github.com/juj) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. http://www.apache.org/licenses/LICENSE-2.0 diff --git a/nostrdb/flatcc/portable/README.md b/nostrdb/flatcc/portable/README.md new file mode 100644 index 0000000000..512b1a830b --- /dev/null +++ b/nostrdb/flatcc/portable/README.md @@ -0,0 +1,57 @@ +A small library for adding C11 compatibility to older C compilers, but +only a small highly useful subset such as static assertions, inline +functions and alignment. + +C++ is not a primary target, but the library has been updated to be more +C++ friendly based on user feedback. + +Many compilers already have the required functionality but with slightly +different names and arguments. + +In addition, compatibility with the Linux `` system file is +provided, and "punaligned.h" is provided for unaligned memory reads +which in part depends on endian support. + +The library also provides fast integer printing and floating point +printing and parsing optionally using the grisu3 algorithm, but can fall +back to strtod and related. The `pgrisu3` folder is header only and +excludes test cases found in the main grisu3 project the files were +extracted from. Base64 conversion is also provided. + +Integer conversion is not just an optimization. It is more difficult +than it would appear to portably parse an integer of known size such as +`uint64_t` up to at most n bytes which is needed for safe parsing. At +the same time, the sometimes significant performance gains warrants +custom implementations that might as well be done once and for all. + +Files can be included individually, or portable.h may be included to get +all functionality. If the compiler is C11 compliant, portable.h will not +include anything, except: it will provide a patch for static assertions +which clang does not fully support in all versions even with C11 flagged. + +The grisu3 header files are the runtime files for the Grisu3 floating +point conversion to/from text C port. Test coverage is provided separately. +This library can be used indirectly via pparsefp.h and pprintfp.h. + +The `pstatic_assert.h` file is often needed on C11 systems because the +compiler and standard library may support `_Static_assert` without +`static_assert`. For compilers without `_Static_assert`, a unique +identifier is needed for each assertion. This is done non-standard with +the `__COUNTER__` macro, but has a fallback to `pstatic_assert_scope.h` +for systems witout the `__COUNTER__` macro. Because of this fallback, +`pstatic_assert.h` needs to be included in every file using +`static_assert` in order to increment a scope counter, otherwise there +is a risk of assert identifier conflicts when `static_assert` happen on +the same line in different files. + +The `paligned_alloc.h` file implements the non-standard `aligned_free` +to match the C11 standard `aligned_alloc` call. `aligned_free` is +normally equivalent to `free`, but not on systems where `aligned_free` +cannot be implemented using a system provived `free` call. Use of +`aligned_free` is thus optional on some systems, but using it increases +general portablity at the cost of pure C11 compatibility. + +IMPORTANT NOTE: this library has been used on various platforms and +updated with user feedback but it is impossibly to systematically test +all platforms so please test for specific uses cases and report +any issues upstream. diff --git a/nostrdb/flatcc/portable/grisu3_math.h b/nostrdb/flatcc/portable/grisu3_math.h new file mode 100644 index 0000000000..cff6e8c673 --- /dev/null +++ b/nostrdb/flatcc/portable/grisu3_math.h @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2016 Mikkel F. Jørgensen, dvide.com + * Copyright author of MathGeoLib (https://github.com/juj) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* 2016-02-02: Updated by mikkelfj + * + * Extracted from MatGeoLib grisu3.c, Apache 2.0 license, and extended. + * + * This file is usually include via grisu3_print.h or grisu3_parse.h. + * + * The original MatGeoLib dtoa_grisu3 implementation is largely + * unchanged except for the uint64 to double cast. The remaining changes + * are file structure, name changes, and new additions for parsing: + * + * - Split into header files only: + * grisu3_math.h, grisu3_print.h, (added grisu3_parse.h) + * + * - names prefixed with grisu3_, grisu3_diy_fp_, GRISU3_. + * - added static to all functions. + * - disabled clang unused function warnings. + * - guarded to allow for alternative impl. + * - added extra numeric constants needed for parsing. + * - added dec_pow, cast_double_from_diy_fp. + * - changed some function names for consistency. + * - moved printing specific grisu3 functions to grisu3_print.h. + * - changed double to uint64 cast to avoid aliasing. + * - added new grisu3_parse.h for parsing doubles. + * - grisu3_print_double (dtoa_grisu3) format .1 as 0.1 needed for valid JSON output + * and grisu3_parse_double wouldn't consume it. + * - grsu3_print_double changed formatting to prefer 0.012 over 1.2e-2. + * + * These changes make it possible to include the files as headers only + * in other software libraries without risking name conflicts, and to + * extend the implementation with a port of Googles Double Conversion + * strtod functionality for parsing doubles. + * + * Extracted from: rev. 915501a / Dec 22, 2015 + * + * MathGeoLib License: http://www.apache.org/licenses/LICENSE-2.0.html + */ + +#ifndef GRISU3_MATH_H +#define GRISU3_MATH_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Guarded to allow inclusion of pstdint.h first, if stdint.h is not supported. */ +#ifndef UINT8_MAX +#include /* uint64_t etc. */ +#endif + +#ifdef GRISU3_NO_ASSERT +#undef GRISU3_ASSERT +#define GRISU3_ASSERT(x) ((void)0) +#endif + +#ifndef GRISU3_ASSERT +#include /* assert */ +#define GRISU3_ASSERT(x) assert(x) +#endif + +#ifdef _MSC_VER +#pragma warning(disable : 4204) /* nonstandard extension used : non-constant aggregate initializer */ +#endif + +#define GRISU3_D64_SIGN 0x8000000000000000ULL +#define GRISU3_D64_EXP_MASK 0x7FF0000000000000ULL +#define GRISU3_D64_FRACT_MASK 0x000FFFFFFFFFFFFFULL +#define GRISU3_D64_IMPLICIT_ONE 0x0010000000000000ULL +#define GRISU3_D64_EXP_POS 52 +#define GRISU3_D64_EXP_BIAS 1075 +#define GRISU3_D64_DENORM_EXP (-GRISU3_D64_EXP_BIAS + 1) +#define GRISU3_DIY_FP_FRACT_SIZE 64 +#define GRISU3_D_1_LOG2_10 0.30102999566398114 /* 1 / lg(10) */ +#define GRISU3_MIN_TARGET_EXP -60 +#define GRISU3_MASK32 0xFFFFFFFFULL +#define GRISU3_MIN_CACHED_EXP -348 +#define GRISU3_MAX_CACHED_EXP 340 +#define GRISU3_CACHED_EXP_STEP 8 +#define GRISU3_D64_MAX_DEC_EXP 309 +#define GRISU3_D64_MIN_DEC_EXP -324 +#define GRISU3_D64_INF GRISU3_D64_EXP_MASK + +#define GRISU3_MIN(x,y) ((x) <= (y) ? (x) : (y)) +#define GRISU3_MAX(x,y) ((x) >= (y) ? (x) : (y)) + + +typedef struct grisu3_diy_fp +{ + uint64_t f; + int e; +} grisu3_diy_fp_t; + +typedef struct grisu3_diy_fp_power +{ + uint64_t fract; + int16_t b_exp, d_exp; +} grisu3_diy_fp_power_t; + +typedef union { + uint64_t u64; + double d64; +} grisu3_cast_double_t; + +static uint64_t grisu3_cast_uint64_from_double(double d) +{ + grisu3_cast_double_t cd; + cd.d64 = d; + return cd.u64; +} + +static double grisu3_cast_double_from_uint64(uint64_t u) +{ + grisu3_cast_double_t cd; + cd.u64 = u; + return cd.d64; +} + +#define grisu3_double_infinity grisu3_cast_double_from_uint64(GRISU3_D64_INF) +#define grisu3_double_nan grisu3_cast_double_from_uint64(GRISU3_D64_INF + 1) + +static const grisu3_diy_fp_power_t grisu3_diy_fp_pow_cache[] = +{ + { 0xfa8fd5a0081c0288ULL, -1220, -348 }, + { 0xbaaee17fa23ebf76ULL, -1193, -340 }, + { 0x8b16fb203055ac76ULL, -1166, -332 }, + { 0xcf42894a5dce35eaULL, -1140, -324 }, + { 0x9a6bb0aa55653b2dULL, -1113, -316 }, + { 0xe61acf033d1a45dfULL, -1087, -308 }, + { 0xab70fe17c79ac6caULL, -1060, -300 }, + { 0xff77b1fcbebcdc4fULL, -1034, -292 }, + { 0xbe5691ef416bd60cULL, -1007, -284 }, + { 0x8dd01fad907ffc3cULL, -980, -276 }, + { 0xd3515c2831559a83ULL, -954, -268 }, + { 0x9d71ac8fada6c9b5ULL, -927, -260 }, + { 0xea9c227723ee8bcbULL, -901, -252 }, + { 0xaecc49914078536dULL, -874, -244 }, + { 0x823c12795db6ce57ULL, -847, -236 }, + { 0xc21094364dfb5637ULL, -821, -228 }, + { 0x9096ea6f3848984fULL, -794, -220 }, + { 0xd77485cb25823ac7ULL, -768, -212 }, + { 0xa086cfcd97bf97f4ULL, -741, -204 }, + { 0xef340a98172aace5ULL, -715, -196 }, + { 0xb23867fb2a35b28eULL, -688, -188 }, + { 0x84c8d4dfd2c63f3bULL, -661, -180 }, + { 0xc5dd44271ad3cdbaULL, -635, -172 }, + { 0x936b9fcebb25c996ULL, -608, -164 }, + { 0xdbac6c247d62a584ULL, -582, -156 }, + { 0xa3ab66580d5fdaf6ULL, -555, -148 }, + { 0xf3e2f893dec3f126ULL, -529, -140 }, + { 0xb5b5ada8aaff80b8ULL, -502, -132 }, + { 0x87625f056c7c4a8bULL, -475, -124 }, + { 0xc9bcff6034c13053ULL, -449, -116 }, + { 0x964e858c91ba2655ULL, -422, -108 }, + { 0xdff9772470297ebdULL, -396, -100 }, + { 0xa6dfbd9fb8e5b88fULL, -369, -92 }, + { 0xf8a95fcf88747d94ULL, -343, -84 }, + { 0xb94470938fa89bcfULL, -316, -76 }, + { 0x8a08f0f8bf0f156bULL, -289, -68 }, + { 0xcdb02555653131b6ULL, -263, -60 }, + { 0x993fe2c6d07b7facULL, -236, -52 }, + { 0xe45c10c42a2b3b06ULL, -210, -44 }, + { 0xaa242499697392d3ULL, -183, -36 }, + { 0xfd87b5f28300ca0eULL, -157, -28 }, + { 0xbce5086492111aebULL, -130, -20 }, + { 0x8cbccc096f5088ccULL, -103, -12 }, + { 0xd1b71758e219652cULL, -77, -4 }, + { 0x9c40000000000000ULL, -50, 4 }, + { 0xe8d4a51000000000ULL, -24, 12 }, + { 0xad78ebc5ac620000ULL, 3, 20 }, + { 0x813f3978f8940984ULL, 30, 28 }, + { 0xc097ce7bc90715b3ULL, 56, 36 }, + { 0x8f7e32ce7bea5c70ULL, 83, 44 }, + { 0xd5d238a4abe98068ULL, 109, 52 }, + { 0x9f4f2726179a2245ULL, 136, 60 }, + { 0xed63a231d4c4fb27ULL, 162, 68 }, + { 0xb0de65388cc8ada8ULL, 189, 76 }, + { 0x83c7088e1aab65dbULL, 216, 84 }, + { 0xc45d1df942711d9aULL, 242, 92 }, + { 0x924d692ca61be758ULL, 269, 100 }, + { 0xda01ee641a708deaULL, 295, 108 }, + { 0xa26da3999aef774aULL, 322, 116 }, + { 0xf209787bb47d6b85ULL, 348, 124 }, + { 0xb454e4a179dd1877ULL, 375, 132 }, + { 0x865b86925b9bc5c2ULL, 402, 140 }, + { 0xc83553c5c8965d3dULL, 428, 148 }, + { 0x952ab45cfa97a0b3ULL, 455, 156 }, + { 0xde469fbd99a05fe3ULL, 481, 164 }, + { 0xa59bc234db398c25ULL, 508, 172 }, + { 0xf6c69a72a3989f5cULL, 534, 180 }, + { 0xb7dcbf5354e9beceULL, 561, 188 }, + { 0x88fcf317f22241e2ULL, 588, 196 }, + { 0xcc20ce9bd35c78a5ULL, 614, 204 }, + { 0x98165af37b2153dfULL, 641, 212 }, + { 0xe2a0b5dc971f303aULL, 667, 220 }, + { 0xa8d9d1535ce3b396ULL, 694, 228 }, + { 0xfb9b7cd9a4a7443cULL, 720, 236 }, + { 0xbb764c4ca7a44410ULL, 747, 244 }, + { 0x8bab8eefb6409c1aULL, 774, 252 }, + { 0xd01fef10a657842cULL, 800, 260 }, + { 0x9b10a4e5e9913129ULL, 827, 268 }, + { 0xe7109bfba19c0c9dULL, 853, 276 }, + { 0xac2820d9623bf429ULL, 880, 284 }, + { 0x80444b5e7aa7cf85ULL, 907, 292 }, + { 0xbf21e44003acdd2dULL, 933, 300 }, + { 0x8e679c2f5e44ff8fULL, 960, 308 }, + { 0xd433179d9c8cb841ULL, 986, 316 }, + { 0x9e19db92b4e31ba9ULL, 1013, 324 }, + { 0xeb96bf6ebadf77d9ULL, 1039, 332 }, + { 0xaf87023b9bf0ee6bULL, 1066, 340 } +}; + +/* Avoid dependence on lib math to get (int)ceil(v) */ +static int grisu3_iceil(double v) +{ + int k = (int)v; + if (v < 0) return k; + return v - k == 0 ? k : k + 1; +} + +static int grisu3_diy_fp_cached_pow(int exp, grisu3_diy_fp_t *p) +{ + int k = grisu3_iceil((exp+GRISU3_DIY_FP_FRACT_SIZE-1) * GRISU3_D_1_LOG2_10); + int i = (k-GRISU3_MIN_CACHED_EXP-1) / GRISU3_CACHED_EXP_STEP + 1; + p->f = grisu3_diy_fp_pow_cache[i].fract; + p->e = grisu3_diy_fp_pow_cache[i].b_exp; + return grisu3_diy_fp_pow_cache[i].d_exp; +} + +static grisu3_diy_fp_t grisu3_diy_fp_minus(grisu3_diy_fp_t x, grisu3_diy_fp_t y) +{ + grisu3_diy_fp_t d; d.f = x.f - y.f; d.e = x.e; + GRISU3_ASSERT(x.e == y.e && x.f >= y.f); + return d; +} + +static grisu3_diy_fp_t grisu3_diy_fp_multiply(grisu3_diy_fp_t x, grisu3_diy_fp_t y) +{ + uint64_t a, b, c, d, ac, bc, ad, bd, tmp; + grisu3_diy_fp_t r; + a = x.f >> 32; b = x.f & GRISU3_MASK32; + c = y.f >> 32; d = y.f & GRISU3_MASK32; + ac = a*c; bc = b*c; + ad = a*d; bd = b*d; + tmp = (bd >> 32) + (ad & GRISU3_MASK32) + (bc & GRISU3_MASK32); + tmp += 1U << 31; /* round */ + r.f = ac + (ad >> 32) + (bc >> 32) + (tmp >> 32); + r.e = x.e + y.e + 64; + return r; +} + +static grisu3_diy_fp_t grisu3_diy_fp_normalize(grisu3_diy_fp_t n) +{ + GRISU3_ASSERT(n.f != 0); + while(!(n.f & 0xFFC0000000000000ULL)) { n.f <<= 10; n.e -= 10; } + while(!(n.f & GRISU3_D64_SIGN)) { n.f <<= 1; --n.e; } + return n; +} + +static grisu3_diy_fp_t grisu3_cast_diy_fp_from_double(double d) +{ + grisu3_diy_fp_t fp; + uint64_t u64 = grisu3_cast_uint64_from_double(d); + if (!(u64 & GRISU3_D64_EXP_MASK)) { fp.f = u64 & GRISU3_D64_FRACT_MASK; fp.e = 1 - GRISU3_D64_EXP_BIAS; } + else { fp.f = (u64 & GRISU3_D64_FRACT_MASK) + GRISU3_D64_IMPLICIT_ONE; fp.e = (int)((u64 & GRISU3_D64_EXP_MASK) >> GRISU3_D64_EXP_POS) - GRISU3_D64_EXP_BIAS; } + return fp; +} + +static double grisu3_cast_double_from_diy_fp(grisu3_diy_fp_t n) +{ + const uint64_t hidden_bit = GRISU3_D64_IMPLICIT_ONE; + const uint64_t frac_mask = GRISU3_D64_FRACT_MASK; + const int denorm_exp = GRISU3_D64_DENORM_EXP; + const int exp_bias = GRISU3_D64_EXP_BIAS; + const int exp_pos = GRISU3_D64_EXP_POS; + + grisu3_diy_fp_t v = n; + uint64_t e_biased; + + while (v.f > hidden_bit + frac_mask) { + v.f >>= 1; + ++v.e; + } + if (v.e < denorm_exp) { + return 0.0; + } + while (v.e > denorm_exp && (v.f & hidden_bit) == 0) { + v.f <<= 1; + --v.e; + } + if (v.e == denorm_exp && (v.f & hidden_bit) == 0) { + e_biased = 0; + } else { + e_biased = (uint64_t)(v.e + exp_bias); + } + return grisu3_cast_double_from_uint64((v.f & frac_mask) | (e_biased << exp_pos)); +} + +/* pow10_cache[i] = 10^(i-1) */ +static const unsigned int grisu3_pow10_cache[] = { 0, 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + +static int grisu3_largest_pow10(uint32_t n, int n_bits, uint32_t *power) +{ + int guess = ((n_bits + 1) * 1233 >> 12) + 1/*skip first entry*/; + if (n < grisu3_pow10_cache[guess]) --guess; /* We don't have any guarantees that 2^n_bits <= n. */ + *power = grisu3_pow10_cache[guess]; + return guess; +} + +#ifdef __cplusplus +} +#endif + +#endif /* GRISU3_MATH_H */ diff --git a/nostrdb/flatcc/portable/grisu3_parse.h b/nostrdb/flatcc/portable/grisu3_parse.h new file mode 100644 index 0000000000..3d67c9ac11 --- /dev/null +++ b/nostrdb/flatcc/portable/grisu3_parse.h @@ -0,0 +1,582 @@ +/* + * Copyright (c) 2016 Mikkel F. Jørgensen, dvide.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * Port of parts of Google Double Conversion strtod functionality + * but with fallback to strtod instead of a bignum implementation. + * + * Based on grisu3 math from MathGeoLib. + * + * See also grisu3_math.h comments. + */ + +#ifndef GRISU3_PARSE_H +#define GRISU3_PARSE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef UINT8_MAX +#include +#endif + +#include +#include + +#include "grisu3_math.h" + + +/* + * The maximum number characters a valid number may contain. The parse + * fails if the input length is longer but the character after max len + * was part of the number. + * + * The length should not be set too high because it protects against + * overflow in the exponent part derived from the input length. + */ +#define GRISU3_NUM_MAX_LEN 1000 + +/* + * The lightweight "portable" C library recognizes grisu3 support if + * included first. + */ +#define grisu3_parse_double_is_defined 1 + +/* + * Disable to compare performance and to test diy_fp algorithm in + * broader range. + */ +#define GRISU3_PARSE_FAST_CASE + +/* May result in a one off error, otherwise when uncertain, fall back to strtod. */ +//#define GRISU3_PARSE_ALLOW_ERROR + + +/* + * The dec output exponent jumps in 8, so the result is offset at most + * by 7 when the input is within range. + */ +static int grisu3_diy_fp_cached_dec_pow(int d_exp, grisu3_diy_fp_t *p) +{ + const int cached_offset = -GRISU3_MIN_CACHED_EXP; + const int d_exp_dist = GRISU3_CACHED_EXP_STEP; + int i, a_exp; + + GRISU3_ASSERT(GRISU3_MIN_CACHED_EXP <= d_exp); + GRISU3_ASSERT(d_exp < GRISU3_MAX_CACHED_EXP + d_exp_dist); + + i = (d_exp + cached_offset) / d_exp_dist; + a_exp = grisu3_diy_fp_pow_cache[i].d_exp; + p->f = grisu3_diy_fp_pow_cache[i].fract; + p->e = grisu3_diy_fp_pow_cache[i].b_exp; + + GRISU3_ASSERT(a_exp <= d_exp); + GRISU3_ASSERT(d_exp < a_exp + d_exp_dist); + + return a_exp; +} + +/* + * Ported from google double conversion strtod using + * MathGeoLibs diy_fp functions for grisu3 in C. + * + * ulp_half_error is set if needed to trunacted non-zero trialing + * characters. + * + * The actual value we need to encode is: + * + * (sign ? -1 : 1) * fraction * 2 ^ (exponent - fraction_exp) + * where exponent is the base 10 exponent assuming the decimal point is + * after the first digit. fraction_exp is the base 10 magnitude of the + * fraction or number of significant digits - 1. + * + * If the exponent is between 0 and 22 and the fraction is encoded in + * the lower 53 bits (the largest bit is implicit in a double, but not + * in this fraction), then the value can be trivially converted to + * double without loss of precision. If the fraction was in fact + * multiplied by trailing zeroes that we didn't convert to exponent, + * we there are larger values the 53 bits that can also be encoded + * trivially - but then it is better to handle this during parsing + * if it is worthwhile. We do not optimize for this here, because it + * can be done in a simple check before calling, and because it might + * not be worthwile to do at all since it cery likely will fail for + * numbers printed to be convertible back to double without loss. + * + * Returns 0 if conversion was not exact. In that case the vale is + * either one smaller than the correct one, or the correct one. + * + * Exponents must be range protected before calling otherwise cached + * powers will blow up. + * + * Google Double Conversion seems to prefer the following notion: + * + * x >= 10^309 => +Inf + * x <= 10^-324 => 0, + * + * max double: HUGE_VAL = 1.7976931348623157 * 10^308 + * min double: 4.9406564584124654 * 10^-324 + * + * Values just below or above min/max representable number + * may round towards large/small non-Inf/non-neg values. + * + * but `strtod` seems to return +/-HUGE_VAL on overflow? + */ +static int grisu3_diy_fp_encode_double(uint64_t fraction, int exponent, int fraction_exp, int ulp_half_error, double *result) +{ + /* + * Error is measures in fractions of integers, so we scale up to get + * some resolution to represent error expressions. + */ + const int log2_error_one = 3; + const int error_one = 1 << log2_error_one; + const int denorm_exp = GRISU3_D64_DENORM_EXP; + const uint64_t hidden_bit = GRISU3_D64_IMPLICIT_ONE; + const int diy_size = GRISU3_DIY_FP_FRACT_SIZE; + const int max_digits = 19; + + int error = ulp_half_error ? error_one / 2 : 0; + int d_exp = (exponent - fraction_exp); + int a_exp; + int o_exp; + grisu3_diy_fp_t v = { fraction, 0 }; + grisu3_diy_fp_t cp; + grisu3_diy_fp_t rounded; + int mag; + int prec; + int prec_bits; + int half_way; + + /* When fractions in a double aren't stored with implicit msb fraction bit. */ + + /* Shift fraction to msb. */ + v = grisu3_diy_fp_normalize(v); + /* The half point error moves up while the exponent moves down. */ + error <<= -v.e; + + a_exp = grisu3_diy_fp_cached_dec_pow(d_exp, &cp); + + /* Interpolate between cached powers at distance 8. */ + if (a_exp != d_exp) { + int adj_exp = d_exp - a_exp - 1; + static grisu3_diy_fp_t cp_10_lut[] = { + { 0xa000000000000000ULL, -60 }, + { 0xc800000000000000ULL, -57 }, + { 0xfa00000000000000ULL, -54 }, + { 0x9c40000000000000ULL, -50 }, + { 0xc350000000000000ULL, -47 }, + { 0xf424000000000000ULL, -44 }, + { 0x9896800000000000ULL, -40 }, + }; + GRISU3_ASSERT(adj_exp >= 0 && adj_exp < 7); + v = grisu3_diy_fp_multiply(v, cp_10_lut[adj_exp]); + + /* 20 decimal digits won't always fit in 64 bit. + * (`fraction_exp` is one less than significant decimal + * digits in fraction, e.g. 1 * 10e0). + * If we cannot fit, introduce 1/2 ulp error + * (says double conversion reference impl.) */ + if (1 + fraction_exp + adj_exp > max_digits) { + error += error_one / 2; + } + } + + v = grisu3_diy_fp_multiply(v, cp); + /* + * Google double conversion claims that: + * + * The error introduced by a multiplication of a*b equals + * error_a + error_b + error_a*error_b/2^64 + 0.5 + * Substituting a with 'input' and b with 'cached_power' we have + * error_b = 0.5 (all cached powers have an error of less than 0.5 ulp), + * error_ab = 0 or 1 / error_oner > error_a*error_b/ 2^64 + * + * which in our encoding becomes: + * error_a = error_one/2 + * error_ab = 1 / error_one (rounds up to 1 if error != 0, or 0 * otherwise) + * fixed_error = error_one/2 + * + * error += error_a + fixed_error + (error ? 1 : 0) + * + * (this isn't entirely clear, but that is as close as we get). + */ + error += error_one + (error ? 1 : 0); + + o_exp = v.e; + v = grisu3_diy_fp_normalize(v); + /* Again, if we shift the significant bits, the error moves along. */ + error <<= o_exp - v.e; + + /* + * The value `v` is bounded by 2^mag which is 64 + v.e. because we + * just normalized it by shifting towards msb. + */ + mag = diy_size + v.e; + + /* The effective magnitude of the IEEE double representation. */ + mag = mag >= diy_size + denorm_exp ? diy_size : mag <= denorm_exp ? 0 : mag - denorm_exp; + prec = diy_size - mag; + if (prec + log2_error_one >= diy_size) { + int e_scale = prec + log2_error_one - diy_size - 1; + v.f >>= e_scale; + v.e += e_scale; + error = (error >> e_scale) + 1 + error_one; + prec -= e_scale; + } + rounded.f = v.f >> prec; + rounded.e = v.e + prec; + prec_bits = (int)(v.f & ((uint64_t)1 << (prec - 1))) * error_one; + half_way = (int)((uint64_t)1 << (prec - 1)) * error_one; + if (prec >= half_way + error) { + rounded.f++; + /* Prevent overflow. */ + if (rounded.f & (hidden_bit << 1)) { + rounded.f >>= 1; + rounded.e += 1; + } + } + *result = grisu3_cast_double_from_diy_fp(rounded); + return half_way - error >= prec_bits || prec_bits >= half_way + error; +} + +/* + * `end` is unchanged if number is handled natively, or it is the result + * of strtod parsing in case of fallback. + */ +static const char *grisu3_encode_double(const char *buf, const char *end, int sign, uint64_t fraction, int exponent, int fraction_exp, int ulp_half_error, double *result) +{ + const int max_d_exp = GRISU3_D64_MAX_DEC_EXP; + const int min_d_exp = GRISU3_D64_MIN_DEC_EXP; + + char *v_end; + + /* Both for user experience, and to protect internal power table lookups. */ + if (fraction == 0 || exponent < min_d_exp) { + *result = 0.0; + goto done; + } + if (exponent - 1 > max_d_exp) { + *result = grisu3_double_infinity; + goto done; + } + + /* + * `exponent` is the normalized value, fraction_exp is the size of + * the representation in the `fraction value`, or one less than + * number of significant digits. + * + * If the final value can be kept in 53 bits and we can avoid + * division, then we can convert to double quite fast. + * + * ulf_half_error only happens when fraction is maxed out, so + * fraction_exp > 22 by definition. + * + * fraction_exp >= 0 always. + * + * http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + */ + + +#ifdef GRISU3_PARSE_FAST_CASE + if (fraction < (1ULL << 53) && exponent >= 0 && exponent <= 22) { + double v = (double)fraction; + /* Multiplying by 1e-k instead of dividing by 1ek results in rounding error. */ + switch (exponent - fraction_exp) { + case -22: v /= 1e22; break; + case -21: v /= 1e21; break; + case -20: v /= 1e20; break; + case -19: v /= 1e19; break; + case -18: v /= 1e18; break; + case -17: v /= 1e17; break; + case -16: v /= 1e16; break; + case -15: v /= 1e15; break; + case -14: v /= 1e14; break; + case -13: v /= 1e13; break; + case -12: v /= 1e12; break; + case -11: v /= 1e11; break; + case -10: v /= 1e10; break; + case -9: v /= 1e9; break; + case -8: v /= 1e8; break; + case -7: v /= 1e7; break; + case -6: v /= 1e6; break; + case -5: v /= 1e5; break; + case -4: v /= 1e4; break; + case -3: v /= 1e3; break; + case -2: v /= 1e2; break; + case -1: v /= 1e1; break; + case 0: break; + case 1: v *= 1e1; break; + case 2: v *= 1e2; break; + case 3: v *= 1e3; break; + case 4: v *= 1e4; break; + case 5: v *= 1e5; break; + case 6: v *= 1e6; break; + case 7: v *= 1e7; break; + case 8: v *= 1e8; break; + case 9: v *= 1e9; break; + case 10: v *= 1e10; break; + case 11: v *= 1e11; break; + case 12: v *= 1e12; break; + case 13: v *= 1e13; break; + case 14: v *= 1e14; break; + case 15: v *= 1e15; break; + case 16: v *= 1e16; break; + case 17: v *= 1e17; break; + case 18: v *= 1e18; break; + case 19: v *= 1e19; break; + case 20: v *= 1e20; break; + case 21: v *= 1e21; break; + case 22: v *= 1e22; break; + } + *result = v; + goto done; + } +#endif + + if (grisu3_diy_fp_encode_double(fraction, exponent, fraction_exp, ulp_half_error, result)) { + goto done; + } +#ifdef GRISU3_PARSE_ALLOW_ERROR + goto done; +#endif + *result = strtod(buf, &v_end); + if (v_end < end) { + return v_end; + } + return end; +done: + if (sign) { + *result = -*result; + } + return end; +} + +/* + * Returns buf if number wasn't matched, or null if number starts ok + * but contains invalid content. + */ +static const char *grisu3_parse_hex_fp(const char *buf, const char *end, int sign, double *result) +{ + (void)buf; + (void)end; + (void)sign; + *result = 0.0; + /* Not currently supported. */ + return buf; +} + +/* + * Returns end pointer on success, or null, or buf if start is not a number. + * Sets result to 0.0 on error. + * Reads up to len + 1 bytes from buffer where len + 1 must not be a + * valid part of a number, but all of buf, buf + len need not be a + * number. Leading whitespace is NOT valid. + * Very small numbers are truncated to +/-0.0 and numerically very large + * numbers are returns as +/-infinity. + * + * A value must not end or begin with '.' (like JSON), but can have + * leading zeroes (unlike JSON). A single leading zero followed by + * an encoding symbol may or may not be interpreted as a non-decimal + * encoding prefix, e.g. 0x, but a leading zero followed by a digit is + * NOT interpreted as octal. + * A single leading negative sign may appear before digits, but positive + * sign is not allowed and space after the sign is not allowed. + * At most the first 1000 characters of the input is considered. + */ +static const char *grisu3_parse_double(const char *buf, size_t len, double *result) +{ + const char *mark, *k, *end; + int sign = 0, esign = 0; + uint64_t fraction = 0; + int exponent = 0; + int ee = 0; + int fraction_exp = 0; + int ulp_half_error = 0; + + *result = 0.0; + + end = buf + len + 1; + + /* Failsafe for exponent overflow. */ + if (len > GRISU3_NUM_MAX_LEN) { + end = buf + GRISU3_NUM_MAX_LEN + 1; + } + + if (buf == end) { + return buf; + } + mark = buf; + if (*buf == '-') { + ++buf; + sign = 1; + if (buf == end) { + return 0; + } + } + if (*buf == '0') { + ++buf; + /* | 0x20 is lower case ASCII. */ + if (buf != end && (*buf | 0x20) == 'x') { + k = grisu3_parse_hex_fp(buf, end, sign, result); + if (k == buf) { + return mark; + } + return k; + } + /* Not worthwhile, except for getting the scale of integer part. */ + while (buf != end && *buf == '0') { + ++buf; + } + } else { + if (*buf < '1' || *buf > '9') { + /* + * If we didn't see a sign, just don't recognize it as + * number, otherwise make it an error. + */ + return sign ? 0 : mark; + } + fraction = (uint64_t)(*buf++ - '0'); + } + k = buf; + /* + * We do not catch trailing zeroes when there is no decimal point. + * This misses an opportunity for moving the exponent down into the + * fast case. But it is unlikely to be worthwhile as it complicates + * parsing. + */ + while (buf != end && *buf >= '0' && *buf <= '9') { + if (fraction >= UINT64_MAX / 10) { + fraction += *buf >= '5'; + ulp_half_error = 1; + break; + } + fraction = fraction * 10 + (uint64_t)(*buf++ - '0'); + } + fraction_exp = (int)(buf - k); + /* Skip surplus digits. Trailing zero does not introduce error. */ + while (buf != end && *buf == '0') { + ++exponent; + ++buf; + } + if (buf != end && *buf >= '1' && *buf <= '9') { + ulp_half_error = 1; + ++exponent; + ++buf; + while (buf != end && *buf >= '0' && *buf <= '9') { + ++exponent; + ++buf; + } + } + if (buf != end && *buf == '.') { + ++buf; + k = buf; + if (*buf < '0' || *buf > '9') { + /* We don't accept numbers without leading or trailing digit. */ + return 0; + } + while (buf != end && *buf >= '0' && *buf <= '9') { + if (fraction >= UINT64_MAX / 10) { + if (!ulp_half_error) { + fraction += *buf >= '5'; + ulp_half_error = 1; + } + break; + } + fraction = fraction * 10 + (uint64_t)(*buf++ - '0'); + --exponent; + } + fraction_exp += (int)(buf - k); + while (buf != end && *buf == '0') { + ++exponent; + ++buf; + } + if (buf != end && *buf >= '1' && *buf <= '9') { + ulp_half_error = 1; + ++buf; + while (buf != end && *buf >= '0' && *buf <= '9') { + ++buf; + } + } + } + /* + * Normalized exponent e.g: 1.23434e3 with fraction = 123434, + * fraction_exp = 5, exponent = 3. + * So value = fraction * 10^(exponent - fraction_exp) + */ + exponent += fraction_exp; + if (buf != end && (*buf | 0x20) == 'e') { + if (end - buf < 2) { + return 0; + } + ++buf; + if (*buf == '+') { + ++buf; + if (buf == end) { + return 0; + } + } else if (*buf == '-') { + esign = 1; + ++buf; + if (buf == end) { + return 0; + } + } + if (*buf < '0' || *buf > '9') { + return 0; + } + ee = *buf++ - '0'; + while (buf != end && *buf >= '0' && *buf <= '9') { + /* + * This test impacts performance and we do not need an + * exact value just one large enough to dominate the fraction_exp. + * Subsequent handling maps large absolute ee to 0 or infinity. + */ + if (ee <= 0x7fff) { + ee = ee * 10 + *buf - '0'; + } + ++buf; + } + } + exponent = exponent + (esign ? -ee : ee); + + /* + * Exponent is now a base 10 normalized exponent so the absolute value + * is less the 10^(exponent + 1) for positive exponents. For + * denormalized doubles (using 11 bit exponent 0 with a fraction + * shiftet down, extra small numbers can be achieved. + * + * https://en.wikipedia.org/wiki/Double-precision_floating-point_format + * + * 10^-324 holds the smallest normalized exponent (but not value) and + * 10^308 holds the largest exponent. Internally our lookup table is + * only safe to use within a range slightly larger than this. + * Externally, a slightly larger/smaller value represents NaNs which + * are technically also possible to store as a number. + * + */ + + /* This also protects strod fallback parsing. */ + if (buf == end) { + return 0; + } + return grisu3_encode_double(mark, buf, sign, fraction, exponent, fraction_exp, ulp_half_error, result); +} + +#ifdef __cplusplus +} +#endif + +#endif /* GRISU3_PARSE_H */ diff --git a/nostrdb/flatcc/portable/grisu3_print.h b/nostrdb/flatcc/portable/grisu3_print.h new file mode 100644 index 0000000000..d748408386 --- /dev/null +++ b/nostrdb/flatcc/portable/grisu3_print.h @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2016 Mikkel F. Jørgensen, dvide.com + * Copyright author of MathGeoLib (https://github.com/juj) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * Extracted from MathGeoLib. + * + * mikkelfj: + * - Fixed final output when printing single digit negative exponent to + * have leading zero (important for JSON). + * - Changed formatting to prefer 0.012 over 1.2-e-2. + * + * Large portions of the original grisu3.c file has been moved to + * grisu3_math.h, the rest is placed here. + * + * See also comments in grisu3_math.h. + * + * MatGeoLib grisu3.c comment: + * + * This file is part of an implementation of the "grisu3" double to string + * conversion algorithm described in the research paper + * + * "Printing Floating-Point Numbers Quickly And Accurately with Integers" + * by Florian Loitsch, available at + * http://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf + */ + +#ifndef GRISU3_PRINT_H +#define GRISU3_PRINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include /* sprintf, only needed for fallback printing */ +#include /* assert */ + +#include "grisu3_math.h" + +/* + * The lightweight "portable" C library recognizes grisu3 support if + * included first. + */ +#define grisu3_print_double_is_defined 1 + +/* + * Not sure we have an exact definition, but we get up to 23 + * emperically. There is some math ensuring it does not go awol though, + * like 18 digits + exponent or so. + * This max should be safe size buffer for printing, including zero term. + */ +#define GRISU3_PRINT_MAX 30 + +static int grisu3_round_weed(char *buffer, int len, uint64_t wp_W, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t ulp) +{ + uint64_t wp_Wup = wp_W - ulp; + uint64_t wp_Wdown = wp_W + ulp; + while(rest < wp_Wup && delta - rest >= ten_kappa + && (rest + ten_kappa < wp_Wup || wp_Wup - rest >= rest + ten_kappa - wp_Wup)) + { + --buffer[len-1]; + rest += ten_kappa; + } + if (rest < wp_Wdown && delta - rest >= ten_kappa + && (rest + ten_kappa < wp_Wdown || wp_Wdown - rest > rest + ten_kappa - wp_Wdown)) + return 0; + + return 2*ulp <= rest && rest <= delta - 4*ulp; +} + +static int grisu3_digit_gen(grisu3_diy_fp_t low, grisu3_diy_fp_t w, grisu3_diy_fp_t high, char *buffer, int *length, int *kappa) +{ + uint64_t unit = 1; + grisu3_diy_fp_t too_low = { low.f - unit, low.e }; + grisu3_diy_fp_t too_high = { high.f + unit, high.e }; + grisu3_diy_fp_t unsafe_interval = grisu3_diy_fp_minus(too_high, too_low); + grisu3_diy_fp_t one = { 1ULL << -w.e, w.e }; + uint32_t p1 = (uint32_t)(too_high.f >> -one.e); + uint64_t p2 = too_high.f & (one.f - 1); + uint32_t div; + *kappa = grisu3_largest_pow10(p1, GRISU3_DIY_FP_FRACT_SIZE + one.e, &div); + *length = 0; + + while(*kappa > 0) + { + uint64_t rest; + char digit = (char)(p1 / div); + buffer[*length] = '0' + digit; + ++*length; + p1 %= div; + --*kappa; + rest = ((uint64_t)p1 << -one.e) + p2; + if (rest < unsafe_interval.f) return grisu3_round_weed(buffer, *length, grisu3_diy_fp_minus(too_high, w).f, unsafe_interval.f, rest, (uint64_t)div << -one.e, unit); + div /= 10; + } + + for(;;) + { + char digit; + p2 *= 10; + unit *= 10; + unsafe_interval.f *= 10; + /* Integer division by one. */ + digit = (char)(p2 >> -one.e); + buffer[*length] = '0' + digit; + ++*length; + p2 &= one.f - 1; /* Modulo by one. */ + --*kappa; + if (p2 < unsafe_interval.f) return grisu3_round_weed(buffer, *length, grisu3_diy_fp_minus(too_high, w).f * unit, unsafe_interval.f, p2, one.f, unit); + } +} + +static int grisu3(double v, char *buffer, int *length, int *d_exp) +{ + int mk, kappa, success; + grisu3_diy_fp_t dfp = grisu3_cast_diy_fp_from_double(v); + grisu3_diy_fp_t w = grisu3_diy_fp_normalize(dfp); + + /* normalize boundaries */ + grisu3_diy_fp_t t = { (dfp.f << 1) + 1, dfp.e - 1 }; + grisu3_diy_fp_t b_plus = grisu3_diy_fp_normalize(t); + grisu3_diy_fp_t b_minus; + grisu3_diy_fp_t c_mk; /* Cached power of ten: 10^-k */ + uint64_t u64 = grisu3_cast_uint64_from_double(v); + assert(v > 0 && v <= 1.7976931348623157e308); /* Grisu only handles strictly positive finite numbers. */ + if (!(u64 & GRISU3_D64_FRACT_MASK) && (u64 & GRISU3_D64_EXP_MASK) != 0) { b_minus.f = (dfp.f << 2) - 1; b_minus.e = dfp.e - 2;} /* lower boundary is closer? */ + else { b_minus.f = (dfp.f << 1) - 1; b_minus.e = dfp.e - 1; } + b_minus.f = b_minus.f << (b_minus.e - b_plus.e); + b_minus.e = b_plus.e; + + mk = grisu3_diy_fp_cached_pow(GRISU3_MIN_TARGET_EXP - GRISU3_DIY_FP_FRACT_SIZE - w.e, &c_mk); + + w = grisu3_diy_fp_multiply(w, c_mk); + b_minus = grisu3_diy_fp_multiply(b_minus, c_mk); + b_plus = grisu3_diy_fp_multiply(b_plus, c_mk); + + success = grisu3_digit_gen(b_minus, w, b_plus, buffer, length, &kappa); + *d_exp = kappa - mk; + return success; +} + +static int grisu3_i_to_str(int val, char *str) +{ + int len, i; + char *s; + char *begin = str; + if (val < 0) { *str++ = '-'; val = -val; } + s = str; + + for(;;) + { + int ni = val / 10; + int digit = val - ni*10; + *s++ = (char)('0' + digit); + if (ni == 0) + break; + val = ni; + } + *s = '\0'; + len = (int)(s - str); + for(i = 0; i < len/2; ++i) + { + char ch = str[i]; + str[i] = str[len-1-i]; + str[len-1-i] = ch; + } + + return (int)(s - begin); +} + +static int grisu3_print_nan(uint64_t v, char *dst) +{ + static char hexdigits[16] = "0123456789ABCDEF"; + int i = 0; + + dst[0] = 'N'; + dst[1] = 'a'; + dst[2] = 'N'; + dst[3] = '('; + dst[20] = ')'; + dst[21] = '\0'; + dst += 4; + for (i = 15; i >= 0; --i) { + dst[i] = hexdigits[v & 0x0F]; + v >>= 4; + } + return 21; +} + +static int grisu3_print_double(double v, char *dst) +{ + int d_exp, len, success, decimals, i; + uint64_t u64 = grisu3_cast_uint64_from_double(v); + char *s2 = dst; + assert(dst); + + /* Prehandle NaNs */ + if ((u64 << 1) > 0xFFE0000000000000ULL) return grisu3_print_nan(u64, dst); + /* Prehandle negative values. */ + if ((u64 & GRISU3_D64_SIGN) != 0) { *s2++ = '-'; v = -v; u64 ^= GRISU3_D64_SIGN; } + /* Prehandle zero. */ + if (!u64) { *s2++ = '0'; *s2 = '\0'; return (int)(s2 - dst); } + /* Prehandle infinity. */ + if (u64 == GRISU3_D64_EXP_MASK) { *s2++ = 'i'; *s2++ = 'n'; *s2++ = 'f'; *s2 = '\0'; return (int)(s2 - dst); } + + success = grisu3(v, s2, &len, &d_exp); + /* If grisu3 was not able to convert the number to a string, then use old sprintf (suboptimal). */ + if (!success) return sprintf(s2, "%.17g", v) + (int)(s2 - dst); + + /* We now have an integer string of form "151324135" and a base-10 exponent for that number. */ + /* Next, decide the best presentation for that string by whether to use a decimal point, or the scientific exponent notation 'e'. */ + /* We don't pick the absolute shortest representation, but pick a balance between readability and shortness, e.g. */ + /* 1.545056189557677e-308 could be represented in a shorter form */ + /* 1545056189557677e-323 but that would be somewhat unreadable. */ + decimals = GRISU3_MIN(-d_exp, GRISU3_MAX(1, len-1)); + + /* mikkelfj: + * fix zero prefix .1 => 0.1, important for JSON export. + * prefer unscientific notation at same length: + * -1.2345e-4 over -1.00012345, + * -1.0012345 over -1.2345e-3 + */ + if (d_exp < 0 && (len + d_exp) > -3 && len <= -d_exp) + { + /* mikkelfj: fix zero prefix .1 => 0.1, and short exponents 1.3e-2 => 0.013. */ + memmove(s2 + 2 - d_exp - len, s2, (size_t)len); + s2[0] = '0'; + s2[1] = '.'; + for (i = 2; i < 2-d_exp-len; ++i) s2[i] = '0'; + len += i; + } + else if (d_exp < 0 && len > 1) /* Add decimal point? */ + { + for(i = 0; i < decimals; ++i) s2[len-i] = s2[len-i-1]; + s2[len++ - decimals] = '.'; + d_exp += decimals; + /* Need scientific notation as well? */ + if (d_exp != 0) { s2[len++] = 'e'; len += grisu3_i_to_str(d_exp, s2+len); } + } + /* Add scientific notation? */ + else if (d_exp < 0 || d_exp > 2) { s2[len++] = 'e'; len += grisu3_i_to_str(d_exp, s2+len); } + /* Add zeroes instead of scientific notation? */ + else if (d_exp > 0) { while(d_exp-- > 0) s2[len++] = '0'; } + s2[len] = '\0'; /* grisu3 doesn't null terminate, so ensure termination. */ + return (int)(s2+len-dst); +} + +#ifdef __cplusplus +} +#endif + +#endif /* GRISU3_PRINT_H */ diff --git a/nostrdb/flatcc/portable/include/README b/nostrdb/flatcc/portable/include/README new file mode 100644 index 0000000000..9f991fc6e2 --- /dev/null +++ b/nostrdb/flatcc/portable/include/README @@ -0,0 +1,4 @@ +This directory holds subdirectories it can be added to the include path +such that standard and OS specific header includes like , + and can succeed without explicitly including +special headers explicitly. diff --git a/nostrdb/flatcc/portable/include/linux/endian.h b/nostrdb/flatcc/portable/include/linux/endian.h new file mode 100644 index 0000000000..38fd1fb0e2 --- /dev/null +++ b/nostrdb/flatcc/portable/include/linux/endian.h @@ -0,0 +1 @@ +#include "portable/pendian.h" diff --git a/nostrdb/flatcc/portable/include/std/inttypes.h b/nostrdb/flatcc/portable/include/std/inttypes.h new file mode 100644 index 0000000000..99b699d5c4 --- /dev/null +++ b/nostrdb/flatcc/portable/include/std/inttypes.h @@ -0,0 +1 @@ +#include "portable/inttypes.h" diff --git a/nostrdb/flatcc/portable/include/std/stdalign.h b/nostrdb/flatcc/portable/include/std/stdalign.h new file mode 100644 index 0000000000..6d51281996 --- /dev/null +++ b/nostrdb/flatcc/portable/include/std/stdalign.h @@ -0,0 +1 @@ +#include "portable/pstdalign.h" diff --git a/nostrdb/flatcc/portable/include/std/stdbool.h b/nostrdb/flatcc/portable/include/std/stdbool.h new file mode 100644 index 0000000000..12eb4c7bf1 --- /dev/null +++ b/nostrdb/flatcc/portable/include/std/stdbool.h @@ -0,0 +1 @@ +#include "portable/pstdbool.h" diff --git a/nostrdb/flatcc/portable/include/std/stdint.h b/nostrdb/flatcc/portable/include/std/stdint.h new file mode 100644 index 0000000000..036447109b --- /dev/null +++ b/nostrdb/flatcc/portable/include/std/stdint.h @@ -0,0 +1 @@ +#include "portable/pstdint.h" diff --git a/nostrdb/flatcc/portable/paligned_alloc.h b/nostrdb/flatcc/portable/paligned_alloc.h new file mode 100644 index 0000000000..3dcf4efcdb --- /dev/null +++ b/nostrdb/flatcc/portable/paligned_alloc.h @@ -0,0 +1,210 @@ +#ifndef PALIGNED_ALLOC_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * NOTE: MSVC in general has no aligned alloc function that is + * compatible with free and it is not trivial to implement a version + * which is. Therefore, to remain portable, end user code needs to + * use `aligned_free` which is not part of C11 but defined in this header. + * + * glibc only provides aligned_alloc when _ISOC11_SOURCE is defined, but + * MingW does not support aligned_alloc despite of this, it uses the + * the _aligned_malloc as MSVC. + * + * The same issue is present on some Unix systems not providing + * posix_memalign. + * + * Note that clang and gcc with -std=c11 or -std=c99 will not define + * _POSIX_C_SOURCE and thus posix_memalign cannot be detected but + * aligned_alloc is not necessarily available either. We assume + * that clang always has posix_memalign although it is not strictly + * correct. For gcc, use -std=gnu99 or -std=gnu11 or don't use -std in + * order to enable posix_memalign, or live with the fallback until using + * a system where glibc has a version that supports aligned_alloc. + * + * For C11 compliant compilers and compilers with posix_memalign, + * it is valid to use free instead of aligned_free with the above + * caveats. + */ + +#include + +/* + * Define this to see which version is used so the fallback is not + * enganged unnecessarily: + * + * #define PORTABLE_DEBUG_ALIGNED_ALLOC + */ + +#if 0 +#define PORTABLE_DEBUG_ALIGNED_ALLOC +#endif + +#if !defined(PORTABLE_C11_ALIGNED_ALLOC) + +/* + * PORTABLE_C11_ALIGNED_ALLOC = 1 + * indicates that the system has builtin aligned_alloc + * If it doesn't, the section after detection provides an implemention. + */ +#if defined (__MINGW32__) +/* MingW does not provide aligned_alloc despite defining _ISOC11_SOURCE */ +#define PORTABLE_C11_ALIGNED_ALLOC 0 +#elif defined (_ISOC11_SOURCE) +/* glibc aligned_alloc detection, but MingW is not truthful */ +#define PORTABLE_C11_ALIGNED_ALLOC 1 +#elif defined (__GLIBC__) +/* aligned_alloc is not available in glibc just because __STDC_VERSION__ >= 201112L. */ +#define PORTABLE_C11_ALIGNED_ALLOC 0 +#elif defined (__clang__) +#define PORTABLE_C11_ALIGNED_ALLOC 0 +#elif defined(__IBMC__) +#define PORTABLE_C11_ALIGNED_ALLOC 0 +#elif (defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) +#define PORTABLE_C11_ALIGNED_ALLOC 1 +#else +#define PORTABLE_C11_ALIGNED_ALLOC 0 +#endif + +#endif /* PORTABLE_C11_ALIGNED_ALLOC */ + +/* https://linux.die.net/man/3/posix_memalign */ +#if !defined(PORTABLE_POSIX_MEMALIGN) && defined(_GNU_SOURCE) +#define PORTABLE_POSIX_MEMALIGN 1 +#endif + +/* https://forum.kde.org/viewtopic.php?p=66274 */ +#if !defined(PORTABLE_POSIX_MEMALIGN) && defined(_XOPEN_SOURCE) +#if _XOPEN_SOURCE >= 600 +#define PORTABLE_POSIX_MEMALIGN 1 +#endif +#endif + +#if !defined(PORTABLE_POSIX_MEMALIGN) && defined(_POSIX_C_SOURCE) +#if _POSIX_C_SOURCE >= 200112L +#define PORTABLE_POSIX_MEMALIGN 1 +#endif +#endif + +#if !defined(PORTABLE_POSIX_MEMALIGN) && defined(__clang__) +#define PORTABLE_POSIX_MEMALIGN 1 +#endif + +#if !defined(PORTABLE_POSIX_MEMALIGN) +#define PORTABLE_POSIX_MEMALIGN 0 +#endif + +/* https://forum.kde.org/viewtopic.php?p=66274 */ +#if (defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) +/* C11 or newer */ +#include +#endif + +/* C11 or newer */ +#if !defined(aligned_alloc) && !defined(__aligned_alloc_is_defined) + +#if PORTABLE_C11_ALIGNED_ALLOC +#ifdef PORTABLE_DEBUG_ALIGNED_ALLOC +#error "DEBUG: C11_ALIGNED_ALLOC configured" +#endif +#elif defined(_MSC_VER) || defined(__MINGW32__) + +#ifdef PORTABLE_DEBUG_ALIGNED_ALLOC +#error "DEBUG: Windows _aligned_malloc configured" +#endif + +/* Aligned _aligned_malloc is not compatible with free. */ +#define aligned_alloc(alignment, size) _aligned_malloc(size, alignment) +#define aligned_free(p) _aligned_free(p) +#define __aligned_alloc_is_defined 1 +#define __aligned_free_is_defined 1 + +#elif PORTABLE_POSIX_MEMALIGN + +#if defined(__GNUC__) +#if !defined(__GNUCC__) +extern int posix_memalign (void **, size_t, size_t); +#elif __GNUCC__ < 5 +extern int posix_memalign (void **, size_t, size_t); +#endif +#endif + +static inline void *__portable_aligned_alloc(size_t alignment, size_t size) +{ + int err; + void *p = 0; + + if (alignment < sizeof(void *)) { + alignment = sizeof(void *); + } + err = posix_memalign(&p, alignment, size); + if (err && p) { + free(p); + p = 0; + } + return p; +} + +#ifdef PORTABLE_DEBUG_ALIGNED_ALLOC +#error "DEBUG: POSIX_MEMALIGN configured" +#endif + +#define aligned_alloc(alignment, size) __portable_aligned_alloc(alignment, size) +#define aligned_free(p) free(p) +#define __aligned_alloc_is_defined 1 +#define __aligned_free_is_defined 1 + +#else + +static inline void *__portable_aligned_alloc(size_t alignment, size_t size) +{ + char *raw; + void *buf; + size_t total_size = (size + alignment - 1 + sizeof(void *)); + + if (alignment < sizeof(void *)) { + alignment = sizeof(void *); + } + raw = (char *)(size_t)malloc(total_size); + buf = raw + alignment - 1 + sizeof(void *); + buf = (void *)(((size_t)buf) & ~(alignment - 1)); + ((void **)buf)[-1] = raw; + return buf; +} + +static inline void __portable_aligned_free(void *p) +{ + char *raw; + + if (p) { + raw = (char*)((void **)p)[-1]; + free(raw); + } +} + +#define aligned_alloc(alignment, size) __portable_aligned_alloc(alignment, size) +#define aligned_free(p) __portable_aligned_free(p) +#define __aligned_alloc_is_defined 1 +#define __aligned_free_is_defined 1 + +#ifdef PORTABLE_DEBUG_ALIGNED_ALLOC +#error "DEBUG: aligned_alloc malloc fallback configured" +#endif + +#endif + +#endif /* aligned_alloc */ + +#if !defined(aligned_free) && !defined(__aligned_free_is_defined) +#define aligned_free(p) free(p) +#define __aligned_free_is_defined 1 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* PALIGNED_ALLOC_H */ diff --git a/nostrdb/flatcc/portable/pattributes.h b/nostrdb/flatcc/portable/pattributes.h new file mode 100644 index 0000000000..30b3b23d32 --- /dev/null +++ b/nostrdb/flatcc/portable/pattributes.h @@ -0,0 +1,84 @@ + +/* + * C23 introduces an attribute syntax `[[]]`. Prior to that + * other non-standard syntaxes such as `__attribute__(())` + * and `__declspec()` have been supported by some compiler + * versions. + * + * See also: + * https://en.cppreference.com/w/c/language/attributes + * + * There is no portable way to use C23 attributes in older C standards + * so in order to use these portably, some macro name needs to be + * defined for each attribute that either maps to the older supported + * syntax, or ignores the attribute as appropriate. + * + * The Linux kernel defines certain attributes as macros, such as + * `fallthrough`. When adding attributes it seems reasonable to follow + * the Linux conventions in lack of any official standard. However, it + * is not the intention that this file should mirror the Linux + * attributes 1 to 1. + * + * See also: + * https://github.com/torvalds/linux/blob/master/include/linux/compiler_attributes.h + * + * There is a risk that exposed attribute names may lead to name + * conflicts. A conflicting name can be undefined and if necessary used + * using `pattribute()`. All attributes can be hidden by + * defining `PORTABLE_EXPOSE_ATTRIBUTES=0` in which case + * `pattribute()` can still be used and then if a specific + * attribute name still needs to be exposed, it can be defined manually + * like `#define fallthrough pattribute(fallthrough)`. + */ + + +#ifndef PATTRIBUTES_H +#define PATTRIBUTES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef PORTABLE_EXPOSE_ATTRIBUTES +#define PORTABLE_EXPOSE_ATTRIBUTES 1 +#endif + +#ifdef __has_c_attribute +# define PORTABLE_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define PORTABLE_HAS_C_ATTRIBUTE(x) 0 +#endif + +#ifdef __has_attribute +# define PORTABLE_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define PORTABLE_HAS_ATTRIBUTE(x) 0 +#endif + + +/* https://en.cppreference.com/w/c/language/attributes/fallthrough */ +#if PORTABLE_HAS_C_ATTRIBUTE(__fallthrough__) +# define pattribute_fallthrough [[__fallthrough__]] +#elif PORTABLE_HAS_ATTRIBUTE(__fallthrough__) +# define pattribute_fallthrough __attribute__((__fallthrough__)) +#else +# define pattribute_fallthrough ((void)0) +#endif + + +#define pattribute(x) pattribute_##x + +#if PORTABLE_EXPOSE_ATTRIBUTES + +#ifndef fallthrough +# define fallthrough pattribute(fallthrough) +#endif + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* PATTRIBUTES_H */ diff --git a/nostrdb/flatcc/portable/pbase64.h b/nostrdb/flatcc/portable/pbase64.h new file mode 100644 index 0000000000..a6812c4b18 --- /dev/null +++ b/nostrdb/flatcc/portable/pbase64.h @@ -0,0 +1,448 @@ +#ifndef PBASE64_H +#define PBASE64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Guarded to allow inclusion of pstdint.h first, if stdint.h is not supported. */ +#ifndef UINT8_MAX +#include +#endif + +#define BASE64_EOK 0 +/* 0 or mure full blocks decoded, remaining content may be parsed with fresh buffer. */ +#define BASE64_EMORE 1 +/* The `src_len` argument is required when encoding. */ +#define BASE64_EARGS 2 +/* Unsupported mode, or modifier not supported by mode when encoding. */ +#define BASE64_EMODE 3 +/* Decoding ends at invalid tail length - either by source length or by non-alphabet symbol. */ +#define BASE64_ETAIL 4 +/* Decoding ends at valid tail length but last byte has non-zero bits where it shouldn't have. */ +#define BASE64_EDIRTY 5 + +static inline const char *base64_strerror(int err); + +/* All codecs are URL safe. Only Crockford allow for non-canocical decoding. */ +enum { + /* Most common base64 codec, but not url friendly. */ + base64_mode_rfc4648 = 0, + + /* URL safe version, '+' -> '-', '/' -> '_'. */ + base64_mode_url = 1, + + /* + * Skip ' ', '\r', and '\n' - we do not allow tab because common + * uses of base64 such as PEM do not allow tab. + */ + base64_dec_modifier_skipspace = 32, + + /* Padding is excluded by default. Not allowed for zbase64. */ + base64_enc_modifier_padding = 128, + + /* For internal use or to decide codec of mode. */ + base64_modifier_mask = 32 + 64 + 128, +}; + +/* Encoded size with or without padding. */ +static inline size_t base64_encoded_size(size_t len, int mode); + +/* + * Decoded size assuming no padding. + * If `len` does include padding, the actual size may be less + * when decoding, but never more. + */ +static inline size_t base64_decoded_size(size_t len); + +/* + * `dst` must hold ceil(len * 4 / 3) bytes. + * `src_len` points to length of source and is updated with length of + * parse on both success and failure. If `dst_len` is not null + * it is used to store resulting output lengt withh length of decoded + * output on both success and failure. + * If `hyphen` is non-zero a hyphen is encoded every `hyphen` output bytes. + * `mode` selects encoding alphabet defaulting to Crockfords base64. + * Returns 0 on success. + * + * A terminal space can be added with `dst[dst_len++] = ' '` after the + * encode call. All non-alphabet can be used as terminators except the + * padding character '='. The following characters will work as + * terminator for all modes: { '\0', '\n', ' ', '\t' }. A terminator is + * optional when the source length is given to the decoder. Note that + * crockford also reserves a few extra characters for checksum but the + * checksum must be separate from the main buffer and is not supported + * by this library. + */ +static inline int base64_encode(uint8_t *dst, const uint8_t *src, size_t *dst_len, size_t *src_len, int mode); + +/* + * Decodes according to mode while ignoring encoding modifiers. + * `src_len` and `dst_len` are optional pointers. If `src_len` is set it + * must contain the length of the input, otherwise the input must be + * terminated with a non-alphabet character or valid padding (a single + * padding character is accepted) - if the src_len output is needed but + * not the input due to guaranteed termination, then set it to + * (size_t)-1. `dst_len` must contain length of output buffer if present + * and parse will fail with BASE64_EMORE after decoding a block multiple + * if dst_len is exhausted - the parse can thus be resumed after + * draining destination. `src_len` and `dst_len` are updated with parsed + * and decoded length, when present, on both success and failure. + * Returns 0 on success. Invalid characters are not considered errors - + * they simply terminate the parse, however, if the termination is not + * at a block multiple or a valid partial block length then BASE64_ETAIL + * without output holding the last full block, if any. BASE64_ETAIL is also + * returned if the a valid length holds non-zero unused tail bits. + */ +static inline int base64_decode(uint8_t *dst, const uint8_t *src, size_t *dst_len, size_t *src_len, int mode); + +static inline const char *base64_strerror(int err) +{ + switch (err) { + case BASE64_EOK: return "ok"; + case BASE64_EARGS: return "invalid argument"; + case BASE64_EMODE: return "invalid mode"; + case BASE64_EMORE: return "destination full"; + case BASE64_ETAIL: return "invalid tail length"; + case BASE64_EDIRTY: return "invalid tail content"; + default: return "unknown error"; + } +} + +static inline size_t base64_encoded_size(size_t len, int mode) +{ + size_t k = len % 3; + size_t n = (len * 4 / 3 + 3) & ~(size_t)3; + int pad = mode & base64_enc_modifier_padding; + + if (!pad) { + switch (k) { + case 2: + n -= 1; + break; + case 1: + n -= 2; + break; + default: + break; + } + } + return n; +} + +static inline size_t base64_decoded_size(size_t len) +{ + size_t k = len % 4; + size_t n = len / 4 * 3; + + switch (k) { + case 3: + return n + 2; + case 2: + return n + 1; + case 1: /* Not valid without padding. */ + case 0: + default: + return n; + } +} + +static inline int base64_encode(uint8_t *dst, const uint8_t *src, size_t *dst_len, size_t *src_len, int mode) +{ + const uint8_t *rfc4648_alphabet = (const uint8_t *) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const uint8_t *url_alphabet = (const uint8_t *) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + + const uint8_t *T; + uint8_t *dst_base = dst; + int pad = mode & base64_enc_modifier_padding; + size_t len = 0; + int ret = BASE64_EMODE; + + if (!src_len) { + ret = BASE64_EARGS; + goto done; + } + len = *src_len; + mode = mode & ~base64_modifier_mask; + switch (mode) { + case base64_mode_rfc4648: + T = rfc4648_alphabet; + break; + case base64_mode_url: + T = url_alphabet; + break; + default: + /* Invalid mode. */ + goto done; + } + + ret = BASE64_EOK; + + /* Encodes 4 destination bytes from 3 source bytes. */ + while (len >= 3) { + dst[0] = T[((src[0] >> 2))]; + dst[1] = T[((src[0] << 4) & 0x30) | (src[1] >> 4)]; + dst[2] = T[((src[1] << 2) & 0x3c) | (src[2] >> 6)]; + dst[3] = T[((src[2] & 0x3f))]; + len -= 3; + dst += 4; + src += 3; + } + /* Encodes 8 destination bytes from 1 to 4 source bytes, if any. */ + switch(len) { + case 2: + dst[0] = T[((src[0] >> 2))]; + dst[1] = T[((src[0] << 4) & 0x30) | (src[1] >> 4)]; + dst[2] = T[((src[1] << 2) & 0x3c)]; + dst += 3; + if (pad) { + *dst++ = '='; + } + break; + case 1: + dst[0] = T[((src[0] >> 2))]; + dst[1] = T[((src[0] << 4) & 0x30)]; + dst += 2; + if (pad) { + *dst++ = '='; + *dst++ = '='; + } + break; + default: + pad = 0; + break; + } + len = 0; +done: + if (dst_len) { + *dst_len = (size_t)(dst - dst_base); + } + if (src_len) { + *src_len -= len; + } + return ret; +} + +static inline int base64_decode(uint8_t *dst, const uint8_t *src, size_t *dst_len, size_t *src_len, int mode) +{ + static const uint8_t cinvalid = 64; + static const uint8_t cignore = 65; + static const uint8_t cpadding = 66; + + /* + * 0..63: 6-bit encoded value. + * 64: flags non-alphabet symbols. + * 65: codes for ignored symbols. + * 66: codes for pad symbol '='. + * All codecs consider padding an optional terminator and if present + * consumes as many pad bytes as possible up to block termination, + * but does not fail if a block is not full. + * + * We do not currently have any ignored characters but we might + * add spaces as per MIME spec, but assuming spaces only happen + * at block boundaries this is probalby better handled by repeated + * parsing. + */ + static const uint8_t base64rfc4648_decode[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 66, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + static const uint8_t base64url_decode[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 66, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 63, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + static const uint8_t base64rfc4648_decode_skipspace[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 64, 64, 65, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 66, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + static const uint8_t base64url_decode_skipspace[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 64, 64, 65, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 65, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 66, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 63, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + int ret = BASE64_EOK; + size_t i, k; + uint8_t hold[4]; + uint8_t *dst_base = dst; + size_t limit = (size_t)-1; + size_t len = (size_t)-1, mark; + const uint8_t *T = base64rfc4648_decode; + int skipspace = mode & base64_dec_modifier_skipspace; + + if (src_len) { + len = *src_len; + } + mark = len; + mode = mode & ~base64_modifier_mask; + switch (mode) { + case base64_mode_rfc4648: + T = skipspace ? base64rfc4648_decode_skipspace : base64rfc4648_decode; + break; + case base64_mode_url: + T = skipspace ? base64url_decode_skipspace : base64url_decode; + break; + default: + ret = BASE64_EMODE; + goto done; + } + + if (dst_len && *dst_len > 0) { + limit = *dst_len; + } + while(limit > 0) { + for (i = 0; i < 4; ++i) { + if (len == i) { + k = i; + len -= i; + goto tail; + } + if ((hold[i] = T[src[i]]) >= cinvalid) { + if (hold[i] == cignore) { + ++src; + --len; + --i; + continue; + } + k = i; + /* Strip padding and ignore hyphen in padding, if present. */ + if (hold[i] == cpadding) { + ++i; + while (i < len && i < 8) { + if (T[src[i]] != cpadding && T[src[i]] != cignore) { + break; + } + ++i; + } + } + len -= i; + goto tail; + } + } + if (limit < 3) { + goto more; + } + dst[0] = (uint8_t)((hold[0] << 2) | (hold[1] >> 4)); + dst[1] = (uint8_t)((hold[1] << 4) | (hold[2] >> 2)); + dst[2] = (uint8_t)((hold[2] << 6) | (hold[3])); + dst += 3; + src += 4; + limit -= 3; + len -= 4; + mark = len; + } +done: + if (dst_len) { + *dst_len = (size_t)(dst - dst_base); + } + if (src_len) { + *src_len -= mark; + } + return ret; + +tail: + switch (k) { + case 0: + break; + case 2: + if ((hold[1] << 4) & 0xff) { + goto dirty; + } + if (limit < 1) { + goto more; + } + dst[0] = (uint8_t)((hold[0] << 2) | (hold[1] >> 4)); + dst += 1; + break; + case 3: + if ((hold[2] << 6) & 0xff) { + goto dirty; + } + if (limit < 2) { + goto more; + } + dst[0] = (uint8_t)((hold[0] << 2) | (hold[1] >> 4)); + dst[1] = (uint8_t)((hold[1] << 4) | (hold[2] >> 2)); + dst += 2; + break; + default: + ret = BASE64_ETAIL; + goto done; + } + mark = len; + goto done; +dirty: + ret = BASE64_EDIRTY; + goto done; +more: + ret = BASE64_EMORE; + goto done; +} + +#ifdef __cplusplus +} +#endif + +#endif /* PBASE64_H */ diff --git a/nostrdb/flatcc/portable/pcrt.h b/nostrdb/flatcc/portable/pcrt.h new file mode 100644 index 0000000000..0226be6f49 --- /dev/null +++ b/nostrdb/flatcc/portable/pcrt.h @@ -0,0 +1,48 @@ +#ifndef PCRT_H +#define PCRT_H + +#ifdef __cplusplus +extern "C" { +#endif + + +/* + * Assertions and pointer violations in debug mode may trigger a dialog + * on Windows. When running headless this is not helpful, but + * unfortunately it cannot be disabled with a compiler option so code + * must be injected into the runtime early in the main function. + * A call to the provided `init_headless_crt()` macro does this in + * a portable manner. + * + * See also: + * https://stackoverflow.com/questions/13943665/how-can-i-disable-the-debug-assertion-dialog-on-windows + */ + +#if defined(_WIN32) + +#include +#include +#include + +static int _portable_msvc_headless_report_hook(int reportType, char *message, int *returnValue) +{ + fprintf(stderr, "CRT[%d]: %s\n", reportType, message); + *returnValue = 1; + exit(1); + return 1; +} + +#define init_headless_crt() _CrtSetReportHook(_portable_msvc_headless_report_hook) + +#else + +#define init_headless_crt() ((void)0) + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* PCRT_H */ diff --git a/nostrdb/flatcc/portable/pdiagnostic.h b/nostrdb/flatcc/portable/pdiagnostic.h new file mode 100644 index 0000000000..b5294f3a5e --- /dev/null +++ b/nostrdb/flatcc/portable/pdiagnostic.h @@ -0,0 +1,85 @@ + /* There is intentionally no include guard in this file. */ + + +/* + * Usage: optionally disable any of these before including. + * + * #define PDIAGNOSTIC_IGNORE_UNUSED_FUNCTION + * #define PDIAGNOSTIC_IGNORE_UNUSED_VARIABLE + * #define PDIAGNOSTIC_IGNORE_UNUSED_PARAMETER + * #define PDIAGNOSTIC_IGNORE_UNUSED // all of the above + * + * #include "pdiagnostic.h" + * + * Alternatively use #include "pdiagnostic_push/pop.h" + */ + +#ifdef _MSC_VER +#pragma warning(disable: 4668) /* preprocessor name not defined */ +#endif + +#if !defined(PDIAGNOSTIC_AWARE_MSVC) && defined(_MSC_VER) +#define PDIAGNOSTIC_AWARE_MSVC 1 +#elif !defined(PDIAGNOSTIC_AWARE_MSVC) +#define PDIAGNOSTIC_AWARE_MSVC 0 +#endif + +#if !defined(PDIAGNOSTIC_AWARE_CLANG) && defined(__clang__) +#define PDIAGNOSTIC_AWARE_CLANG 1 +#elif !defined(PDIAGNOSTIC_AWARE_CLANG) +#define PDIAGNOSTIC_AWARE_CLANG 0 +#endif + +#if !defined(PDIAGNOSTIC_AWARE_GCC) && defined(__GNUC__) && !defined(__clang__) +/* Can disable some warnings even if push is not available (gcc-4.2 vs gcc-4.7) */ +#if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) +#define PDIAGNOSTIC_AWARE_GCC 1 +#endif +#endif + +#if !defined(PDIAGNOSTIC_AWARE_GCC) +#define PDIAGNOSTIC_AWARE_GCC 0 +#endif + +#if defined(PDIAGNOSTIC_IGNORE_UNUSED_FUNCTION) || defined(PDIAGNOSTIC_IGNORE_UNUSED) +#if PDIAGNOSTIC_AWARE_CLANG +#pragma clang diagnostic ignored "-Wunused-function" +#elif PDIAGNOSTIC_AWARE_GCC +#pragma GCC diagnostic ignored "-Wunused-function" +#endif +#endif +#undef PDIAGNOSTIC_IGNORE_UNUSED_FUNCTION + +#if defined(PDIAGNOSTIC_IGNORE_UNUSED_VARIABLE) || defined(PDIAGNOSTIC_IGNORE_UNUSED) +#if PDIAGNOSTIC_AWARE_MSVC +#pragma warning(disable: 4101) /* unused local variable */ +#elif PDIAGNOSTIC_AWARE_CLANG +#pragma clang diagnostic ignored "-Wunused-variable" +#elif PDIAGNOSTIC_AWARE_GCC +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif +#endif +#undef PDIAGNOSTIC_IGNORE_UNUSED_VARIABLE + +#if defined(PDIAGNOSTIC_IGNORE_UNUSED_PARAMETER) || defined(PDIAGNOSTIC_IGNORE_UNUSED) +#if PDIAGNOSTIC_AWARE_CLANG +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif PDIAGNOSTIC_AWARE_GCC +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#endif +#undef PDIAGNOSTIC_IGNORE_UNUSED_PARAMETER + +#undef PDIAGNOSTIC_IGNORE_UNUSED + +#if defined (__cplusplus) && __cplusplus < 201103L +#if PDIAGNOSTIC_AWARE_CLANG +/* Needed for < C++11 clang C++ static_assert */ +#pragma clang diagnostic ignored "-Wc11-extensions" +/* Needed for empty macro arguments. */ +#pragma clang diagnostic ignored "-Wc99-extensions" +/* Needed for trailing commas. */ +#pragma clang diagnostic ignored "-Wc++11-extensions" +#endif +#endif + diff --git a/nostrdb/flatcc/portable/pdiagnostic_pop.h b/nostrdb/flatcc/portable/pdiagnostic_pop.h new file mode 100644 index 0000000000..f5e16b340e --- /dev/null +++ b/nostrdb/flatcc/portable/pdiagnostic_pop.h @@ -0,0 +1,20 @@ +#if defined(PDIAGNOSTIC_PUSHED_MSVC) +#if PDIAGNOSTIC_PUSHED_MSVC +#pragma warning( pop ) +#endif // PDIAGNOSTIC_PUSHED_MSVC +#undef PDIAGNOSTIC_PUSHED_MSVC +#endif // defined(PDIAGNOSTIC_PUSHED_MSVC) + +#if defined(PDIAGNOSTIC_PUSHED_CLANG) +#if PDIAGNOSTIC_PUSHED_CLANG +#pragma clang diagnostic pop +#endif // PDIAGNOSTIC_PUSHED_CLANG +#undef PDIAGNOSTIC_PUSHED_CLANG +#endif // defined(PDIAGNOSTIC_PUSHED_CLANG) + +#if defined(PDIAGNOSTIC_PUSHED_GCC) +#if PDIAGNOSTIC_PUSHED_GCC +#pragma GCC diagnostic pop +#endif // PDIAGNOSTIC_PUSHED_GCC +#undef PDIAGNOSTIC_PUSHED_GCC +#endif // defined(PDIAGNOSTIC_PUSHED_GCC) diff --git a/nostrdb/flatcc/portable/pdiagnostic_push.h b/nostrdb/flatcc/portable/pdiagnostic_push.h new file mode 100644 index 0000000000..66586d721b --- /dev/null +++ b/nostrdb/flatcc/portable/pdiagnostic_push.h @@ -0,0 +1,51 @@ +/* + * See also comment in "pdiagnostic.h" + * + * e.g. + * #define PDIAGNOSTIC_IGNORE_USED_FUNCTION + * #define PDIAGNOSTIC_IGNORE_USED_VARIABLE + * #include "pdiagnostic_push" + * ... + * #include "pdiagnostic_pop.h" + * + * + * or if push pop isn't desired: + * #define PDIAGNOSTIC_IGNORE_USED_FUNCTION + * #define PDIAGNOSTIC_IGNORE_USED_VARIABLE + * #include "pdiagnostic.h" + * ... + * + * + * + * Some if these warnings cannot be ignored + * at the #pragma level, but might in the future. + * Use compiler switches like -Wno-unused-function + * to work around this. + */ + +#if defined(_MSC_VER) +#pragma warning( push ) +#define PDIAGNOSTIC_PUSHED_MSVC 1 +#else +#define PDIAGNOSTIC_PUSHED_MSVC 0 +#endif + +#if defined(__clang__) +#pragma clang diagnostic push +#define PDIAGNOSTIC_PUSHED_CLANG 1 +#else +#define PDIAGNOSTIC_PUSHED_CLANG 0 +#endif + +#if defined(__GNUC__) && !defined(__clang__) +#if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic push +#define PDIAGNOSTIC_PUSHED_GCC 1 +#else +#define PDIAGNOSTIC_PUSHED_GCC 0 +#endif // GNUC >= 4.6 +#else +#define PDIAGNOSTIC_PUSHED_GCC 0 +#endif // defined(__GNUC__) && !defined(__clang__) + +#include "pdiagnostic.h" diff --git a/nostrdb/flatcc/portable/pendian.h b/nostrdb/flatcc/portable/pendian.h new file mode 100644 index 0000000000..122ba8e245 --- /dev/null +++ b/nostrdb/flatcc/portable/pendian.h @@ -0,0 +1,206 @@ +#ifndef PENDIAN_H +#define PENDIAN_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Defines platform optimized (as per linux + * + * le16toh, le32to, le64toh, be16toh, be32toh, be64toh + * htole16, htole32, htole64, htobe16, htobe32, htobe64 + * + * Falls back to auto-detect endian conversion which is also fast + * if fast byteswap operation was detected. + * + * Also defines platform optimized: + * + * bswap16, bswap32, bswap64, + * + * with fall-back to shift-or implementation. + * + * For convenience also defines: + * + * le8to, be8toh, htole8, htobe8 + * bswap8 + * + * The convience functions makes is simpler to define conversion macros + * based on type size. + * + * NOTE: this implementation expects arguments with no side-effects and + * with appropriately sized unsigned arguments. These are expected to be + * used with typesafe wrappers. + */ + +#ifndef UINT8_MAX +#include "pstdint.h" +#endif + +#if defined(__linux__) +#include +#elif defined(__OpenBSD__) || defined(__FreeBSD__) +#include +#endif + +#include "pendian_detect.h" + +#if defined(_MSC_VER) +#if _MSC_VER >= 1300 +#include +#define bswap16 _byteswap_ushort +#define bswap32 _byteswap_ulong +#define bswap64 _byteswap_uint64 +#endif +#elif defined(__clang__) +#if __has_builtin(__builtin_bswap16) +#ifndef bswap16 +#define bswap16 __builtin_bswap16 +#endif +#endif +#if __has_builtin(__builtin_bswap32) +#ifndef bswap32 +#define bswap32 __builtin_bswap32 +#endif +#endif +#if __has_builtin(__builtin_bswap64) +#ifndef bswap64 +#define bswap64 __builtin_bswap64 +#endif +#endif +#elif defined(__OpenBSD__) || defined(__FreeBSD__) +#ifndef bswap16 +#define bswap16 swap16 +#endif +#ifndef bswap32 +#define bswap32 swap32 +#endif +#ifndef bswap64 +#define bswap64 swap64 +#endif +#elif defined(__GNUC__) /* Supported since at least GCC 4.4 */ +#ifndef bswap32 +#define bswap32 __builtin_bswap32 +#endif +#ifndef bswap64 +#define bswap64 __builtin_bswap64 +#endif +#endif + +#ifndef bswap16 +#define bswap16(v) \ + (((uint16_t)(v) << 8) | ((uint16_t)(v) >> 8)) +#endif + +#ifndef bswap32 +#define bswap32(v) \ + ((((uint32_t)(v) << 24)) \ + | (((uint32_t)(v) << 8) & UINT32_C(0x00FF0000)) \ + | (((uint32_t)(v) >> 8) & UINT32_C(0x0000FF00)) \ + | (((uint32_t)(v) >> 24))) +#endif + +#ifndef bswap64 +#define bswap64(v) \ + ((((uint64_t)(v) << 56)) \ + | (((uint64_t)(v) << 40) & UINT64_C(0x00FF000000000000)) \ + | (((uint64_t)(v) << 24) & UINT64_C(0x0000FF0000000000)) \ + | (((uint64_t)(v) << 8) & UINT64_C(0x000000FF00000000)) \ + | (((uint64_t)(v) >> 8) & UINT64_C(0x00000000FF000000)) \ + | (((uint64_t)(v) >> 24) & UINT64_C(0x0000000000FF0000)) \ + | (((uint64_t)(v) >> 40) & UINT64_C(0x000000000000FF00)) \ + | (((uint64_t)(v) >> 56))) +#endif + +#ifndef bswap8 +#define bswap8(v) ((uint8_t)(v)) +#endif + +#if !defined(le16toh) && defined(letoh16) +#define le16toh letoh16 +#define le32toh letoh32 +#define le64toh letoh64 +#endif + +#if !defined(be16toh) && defined(betoh16) +#define be16toh betoh16 +#define be32toh betoh32 +#define be64toh betoh64 +#endif + +/* Assume it goes for all. */ +#if !defined(le16toh) + +#if defined(__LITTLE_ENDIAN__) + +#define le16toh(v) (v) +#define le32toh(v) (v) +#define le64toh(v) (v) + +#define htole16(v) (v) +#define htole32(v) (v) +#define htole64(v) (v) + +#define be16toh(v) bswap16(v) +#define be32toh(v) bswap32(v) +#define be64toh(v) bswap64(v) + +#define htobe16(v) bswap16(v) +#define htobe32(v) bswap32(v) +#define htobe64(v) bswap64(v) + +#elif defined(__BIG_ENDIAN__) + +#define le16toh(v) bswap16(v) +#define le32toh(v) bswap32(v) +#define le64toh(v) bswap64(v) + +#define htole16(v) bswap16(v) +#define htole32(v) bswap32(v) +#define htole64(v) bswap64(v) + +#define be16toh(v) (v) +#define be32toh(v) (v) +#define be64toh(v) (v) + +#define htobe16(v) (v) +#define htobe32(v) (v) +#define htobe64(v) (v) + +#else + +static const int __pendian_test = 1; + +#define le16toh(v) (*(char *)&__pendian_test ? (v) : bswap16(v)) +#define le32toh(v) (*(char *)&__pendian_test ? (v) : bswap32(v)) +#define le64toh(v) (*(char *)&__pendian_test ? (v) : bswap64(v)) + +#define htole16(v) (*(char *)&__pendian_test ? (v) : bswap16(v)) +#define htole32(v) (*(char *)&__pendian_test ? (v) : bswap32(v)) +#define htole64(v) (*(char *)&__pendian_test ? (v) : bswap64(v)) + +#define be16toh(v) (*(char *)&__pendian_test ? bswap16(v) : (v)) +#define be32toh(v) (*(char *)&__pendian_test ? bswap32(v) : (v)) +#define be64toh(v) (*(char *)&__pendian_test ? bswap64(v) : (v)) + +#define htobe16(v) (*(char *)&__pendian_test ? bswap16(v) : (v)) +#define htobe32(v) (*(char *)&__pendian_test ? bswap32(v) : (v)) +#define htobe64(v) (*(char *)&__pendian_test ? bswap64(v) : (v)) + +#endif + +#endif /* le16toh */ + +/* Helpers not part of Linux */ +#if !defined(le8toh) +#define le8toh(n) (n) +#define htole8(n) (n) +#define be8toh(n) (n) +#define htobe8(n) (n) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* PENDIAN_H */ diff --git a/nostrdb/flatcc/portable/pendian_detect.h b/nostrdb/flatcc/portable/pendian_detect.h new file mode 100644 index 0000000000..1dd62c046f --- /dev/null +++ b/nostrdb/flatcc/portable/pendian_detect.h @@ -0,0 +1,118 @@ +/* + * Uses various known flags to decide endianness and defines: + * + * __LITTLE_ENDIAN__ or __BIG_ENDIAN__ if not already defined + * + * and also defines + * + * __BYTE_ORDER__ to either __ORDER_LITTLE_ENDIAN__ or + * __ORDER_BIG_ENDIAN__ if not already defined + * + * If none of these could be set, __UNKNOWN_ENDIAN__ is defined, + * which is not a known flag. If __BYTE_ORDER__ is defined but + * not big or little endian, __UNKNOWN_ENDIAN__ is also defined. + * + * Note: Some systems define __BYTE_ORDER without __ at the end + * - this will be mapped to to __BYTE_ORDER__. + */ + +#ifndef PENDIAN_DETECT +#define PENDIAN_DETECT + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef __ORDER_LITTLE_ENDIAN__ +#define __ORDER_LITTLE_ENDIAN__ 1234 +#endif + +#ifndef __ORDER_BIG_ENDIAN__ +#define __ORDER_BIG_ENDIAN__ 4321 +#endif + +#ifdef __BYTE_ORDER__ + +#if defined(__LITTLE_ENDIAN__) && __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error __LITTLE_ENDIAN__ inconsistent with __BYTE_ORDER__ +#endif + +#if defined(__BIG_ENDIAN__) && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ +#error __BIG_ENDIAN__ inconsistent with __BYTE_ORDER__ +#endif + +#else /* __BYTE_ORDER__ */ + + +#if \ + defined(__LITTLE_ENDIAN__) || \ + (defined(__BYTE_ORDER) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || \ + defined(__AARCH64EL__) || \ + (defined(_MSC_VER) && defined(_M_ARM)) || \ + defined(_MIPSEL) || defined(__MIPSEL) || defined(__MIPSEL__) || \ + defined(_M_X64) || defined(_M_IX86) || defined(_M_I86) || \ + defined(__i386__) || defined(__alpha__) || \ + defined(__ia64) || defined(__ia64__) || \ + defined(_M_IA64) || defined(_M_ALPHA) || \ + defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(__bfin__) + +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#endif + +#if \ + defined (__BIG_ENDIAN__) || \ + (defined(__BYTE_ORDER) && __BYTE_ORDER == __ORDER_BIG_ENDIAN) || \ + defined(__ARMEB__) || defined(THUMBEB__) || defined (__AARCH64EB__) || \ + defined(_MIPSEB) || defined(__MIPSEB) || defined(__MIPSEB__) || \ + defined(__sparc) || defined(__sparc__) || \ + defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || \ + defined(__hpux) || defined(__hppa) || defined(__s390__) + +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#endif + +#endif /* __BYTE_ORDER__ */ + +#ifdef __BYTE_ORDER__ + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#ifndef __LITTLE_ENDIAN__ +#define __LITTLE_ENDIAN__ 1 +#endif + +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + +#ifndef __BIG_ENDIAN__ +#define __BIG_ENDIAN__ 1 +#endif + +#else + +/* + * Custom extension - we only define __BYTE_ORDER__ if known big or little. + * User code that understands __BYTE_ORDER__ may also assume unkown if + * it is not defined by now - this will allow other endian formats than + * big or little when supported by compiler. + */ +#ifndef __UNKNOWN_ENDIAN__ +#define __UNKNOWN_ENDIAN__ 1 +#endif + +#endif +#endif /* __BYTE_ORDER__ */ + +#if defined(__LITTLE_ENDIAN__) && defined(__BIG_ENDIAN__) +#error conflicting definitions of __LITTLE_ENDIAN__ and __BIG_ENDIAN__ +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* PENDIAN_DETECT */ diff --git a/nostrdb/flatcc/portable/pinline.h b/nostrdb/flatcc/portable/pinline.h new file mode 100644 index 0000000000..f4f8f27083 --- /dev/null +++ b/nostrdb/flatcc/portable/pinline.h @@ -0,0 +1,19 @@ +#ifndef PINLINE_H +#define PINLINE_H + +#ifndef __cplusplus + +#if (defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +/* C99 or newer */ +#elif _MSC_VER >= 1500 /* MSVC 9 or newer */ +#undef inline +#define inline __inline +#elif __GNUC__ >= 3 /* GCC 3 or newer */ +#define inline __inline +#else /* Unknown or ancient */ +#define inline +#endif + +#endif /* __cplusplus */ + +#endif /* PINLINE_H */ diff --git a/nostrdb/flatcc/portable/pinttypes.h b/nostrdb/flatcc/portable/pinttypes.h new file mode 100644 index 0000000000..a1be9df64c --- /dev/null +++ b/nostrdb/flatcc/portable/pinttypes.h @@ -0,0 +1,52 @@ +#ifndef PINTTYPES_H +#define PINTTYPES_H + +#ifndef PRId16 + +#if (defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +/* C99 or newer */ +#include +#else + +/* + * This is not a complete implementation of , just the most + * useful printf modifiers. + */ + +#include "pstdint.h" + +#ifndef PRINTF_INT64_MODIFIER +#error "please define PRINTF_INT64_MODIFIER" +#endif + +#ifndef PRId64 +#define PRId64 PRINTF_INT64_MODIFIER "d" +#define PRIu64 PRINTF_INT64_MODIFIER "u" +#define PRIx64 PRINTF_INT64_MODIFIER "x" +#endif + +#ifndef PRINTF_INT32_MODIFIER +#define PRINTF_INT32_MODIFIER "l" +#endif + +#ifndef PRId32 +#define PRId32 PRINTF_INT32_MODIFIER "d" +#define PRIu32 PRINTF_INT32_MODIFIER "u" +#define PRIx32 PRINTF_INT32_MODIFIER "x" +#endif + +#ifndef PRINTF_INT16_MODIFIER +#define PRINTF_INT16_MODIFIER "h" +#endif + +#ifndef PRId16 +#define PRId16 PRINTF_INT16_MODIFIER "d" +#define PRIu16 PRINTF_INT16_MODIFIER "u" +#define PRIx16 PRINTF_INT16_MODIFIER "x" +#endif + +# endif /* __STDC__ */ + +#endif /* PRId16 */ + +#endif /* PINTTYPES */ diff --git a/nostrdb/flatcc/portable/portable.h b/nostrdb/flatcc/portable/portable.h new file mode 100644 index 0000000000..7a6a484ed9 --- /dev/null +++ b/nostrdb/flatcc/portable/portable.h @@ -0,0 +1,2 @@ +/* portable.h is widely used, so we redirect to a less conflicting name. */ +#include "portable_basic.h" diff --git a/nostrdb/flatcc/portable/portable_basic.h b/nostrdb/flatcc/portable/portable_basic.h new file mode 100644 index 0000000000..0396f3d4c7 --- /dev/null +++ b/nostrdb/flatcc/portable/portable_basic.h @@ -0,0 +1,25 @@ +#ifndef PORTABLE_BASIC_H +#define PORTABLE_BASIC_H + +/* + * Basic features need to make compilers support the most common moden C + * features, and endian / unligned read support as well. + * + * It is not assumed that this file is always included. + * Other include files are independent or include what they need. + */ + +#include "pversion.h" +#include "pwarnings.h" + +/* Featutures that ought to be supported by C11, but some aren't. */ +#include "pinttypes.h" +#include "pstdalign.h" +#include "pinline.h" +#include "pstatic_assert.h" + +/* These are not supported by C11 and are general platform abstractions. */ +#include "pendian.h" +#include "punaligned.h" + +#endif /* PORTABLE_BASIC_H */ diff --git a/nostrdb/flatcc/portable/pparsefp.h b/nostrdb/flatcc/portable/pparsefp.h new file mode 100644 index 0000000000..1d2b9da26b --- /dev/null +++ b/nostrdb/flatcc/portable/pparsefp.h @@ -0,0 +1,140 @@ +#ifndef PPARSEFP_H +#define PPARSEFP_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Parses a float or double number and returns the length parsed if + * successful. The length argument is of limited value due to dependency + * on `strtod` - buf[len] must be accessible and must not be part of + * a valid number, including hex float numbers.. + * + * Unlike strtod, whitespace is not parsed. + * + * May return: + * - null on error, + * - buffer start if first character does not start a number, + * - or end of parse on success. + * + */ + +#define PDIAGNOSTIC_IGNORE_UNUSED_FUNCTION +#include "pdiagnostic_push.h" + +/* + * isinf is needed in order to stay compatible with strtod's + * over/underflow handling but isinf has some portability issues. + * + * Use the parse_double/float_is_range_error instead of isinf directly. + * This ensures optimizations can be added when not using strtod. + * + * On gcc, clang and msvc we can use isinf or equivalent directly. + * Other compilers such as xlc may require linking with -lm which may not + * be convienent so a default isinf is provided. If isinf is available + * and there is a noticable performance issue, define + * `PORTABLE_USE_ISINF`. + */ +#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) || defined(PORTABLE_USE_ISINF) +#include +#if defined(_MSC_VER) && !defined(isinf) +#include +#define isnan _isnan +#define isinf(x) (!_finite(x)) +#endif +/* + * clang-5 through clang-8 but not clang-9 issues incorrect precision + * loss warning with -Wconversion flag when cast is absent. + */ +#if defined(__clang__) +#if __clang_major__ >= 5 && __clang_major__ <= 8 +#define parse_double_isinf(x) isinf((float)x) +#endif +#endif +#if !defined(parse_double_isinf) +#define parse_double_isinf isinf +#endif +#define parse_float_isinf isinf + +#else + +#ifndef UINT8_MAX +#include +#endif + +/* Avoid linking with libmath but depends on float/double being IEEE754 */ +static inline int parse_double_isinf(double x) +{ + union { uint64_t u64; double f64; } v; + v.f64 = x; + return (v.u64 & 0x7fffffff00000000ULL) == 0x7ff0000000000000ULL; +} + +static inline int parse_float_isinf(float x) +{ + union { uint32_t u32; float f32; } v; + v.f32 = x; + return (v.u32 & 0x7fffffff) == 0x7f800000; +} +#endif + +/* Returns 0 when in range, 1 on overflow, and -1 on underflow. */ +static inline int parse_double_is_range_error(double x) +{ + return parse_double_isinf(x) ? (x < 0.0 ? -1 : 1) : 0; +} + +static inline int parse_float_is_range_error(float x) +{ + return parse_float_isinf(x) ? (x < 0.0f ? -1 : 1) : 0; +} + +#ifndef PORTABLE_USE_GRISU3 +#define PORTABLE_USE_GRISU3 1 +#endif + +#if PORTABLE_USE_GRISU3 +#include "grisu3_parse.h" +#endif + +#ifdef grisu3_parse_double_is_defined +static inline const char *parse_double(const char *buf, size_t len, double *result) +{ + return grisu3_parse_double(buf, len, result); +} +#else +#include +static inline const char *parse_double(const char *buf, size_t len, double *result) +{ + char *end; + + (void)len; + *result = strtod(buf, &end); + return end; +} +#endif + +static inline const char *parse_float(const char *buf, size_t len, float *result) +{ + const char *end; + double v; + union { uint32_t u32; float f32; } inf; + inf.u32 = 0x7f800000; + + end = parse_double(buf, len, &v); + *result = (float)v; + if (parse_float_isinf(*result)) { + *result = v < 0 ? -inf.f32 : inf.f32; + return buf; + } + return end; +} + +#include "pdiagnostic_pop.h" + +#ifdef __cplusplus +} +#endif + +#endif /* PPARSEFP_H */ diff --git a/nostrdb/flatcc/portable/pparseint.h b/nostrdb/flatcc/portable/pparseint.h new file mode 100644 index 0000000000..96cc99fedc --- /dev/null +++ b/nostrdb/flatcc/portable/pparseint.h @@ -0,0 +1,374 @@ +#ifndef PPARSEINT_H +#define PPARSEINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Type specific integer parsers: + * + * const char * + * parse_(const char *buf, size_t len, *value, int *status); + * + * parse_uint64, parse_int64 + * parse_uint32, parse_int32 + * parse_uint16, parse_int16 + * parse_uint8, parse_int8 + * parse_ushort, parse_short + * parse_uint, parse_int + * parse_ulong, parse_long + * + * Leading space must be stripped in advance. Status argument can be + * null. + * + * Returns pointer to end of match and a non-negative status code + * on succcess (0 for unsigned, 1 for signed): + * + * PARSE_INTEGER_UNSIGNED + * PARSE_INTEGER_SIGNED + * + * Returns null with a negative status code and unmodified value on + * invalid integer formats: + * + * PARSE_INTEGER_OVERFLOW + * PARSE_INTEGER_UNDERFLOW + * PARSE_INTEGER_INVALID + * + * Returns input buffer with negative status code and unmodified value + * if first character does not start an integer (not a sign or a digit). + * + * PARSE_INTEGER_UNMATCHED + * PARSE_INTEGER_END + * + * The signed parsers only works with two's complement architectures. + * + * Note: the corresponding parse_float and parse_double parsers do not + * have a status argument because +/-Inf and NaN are conventionally used + * for this. + */ + +#include "limits.h" +#ifndef UINT8_MAX +#include +#endif + +#define PARSE_INTEGER_UNSIGNED 0 +#define PARSE_INTEGER_SIGNED 1 +#define PARSE_INTEGER_OVERFLOW -1 +#define PARSE_INTEGER_UNDERFLOW -2 +#define PARSE_INTEGER_INVALID -3 +#define PARSE_INTEGER_UNMATCHED -4 +#define PARSE_INTEGER_END -5 + +/* + * Generic integer parser that holds 64-bit unsigned values and stores + * sign separately. Leading space is not valid. + * + * Note: this function differs from the type specific parsers like + * parse_int64 by not negating the value when there is a sign. It + * differs from parse_uint64 by being able to return a negative + * UINT64_MAX successfully. + * + * This parser is used by all type specific integer parsers. + * + * Status argument can be null. + */ +static const char *parse_integer(const char *buf, size_t len, uint64_t *value, int *status) +{ + uint64_t x0, x = 0; + const char *k, *end = buf + len; + int sign, status_; + + if (!status) { + status = &status_; + } + if (buf == end) { + *status = PARSE_INTEGER_END; + return buf; + } + k = buf; + sign = *buf == '-'; + buf += sign; + while (buf != end && *buf >= '0' && *buf <= '9') { + x0 = x; + x = x * 10 + (uint64_t)(*buf - '0'); + if (x0 > x) { + *status = sign ? PARSE_INTEGER_UNDERFLOW : PARSE_INTEGER_OVERFLOW; + return 0; + } + ++buf; + } + if (buf == k) { + /* No number was matched, but it isn't an invalid number either. */ + *status = PARSE_INTEGER_UNMATCHED; + return buf; + } + if (buf == k + sign) { + *status = PARSE_INTEGER_INVALID; + return 0; + } + if (buf != end) + switch (*buf) { + case 'e': case 'E': case '.': case 'p': case 'P': + *status = PARSE_INTEGER_INVALID; + return 0; + } + *value = x; + *status = sign; + return buf; +} + +/* + * Parse hex values like 0xff, -0xff, 0XdeAdBeaf42, cannot be trailed by '.', 'p', or 'P'. + * Overflows if string is more than 16 valid hex digits. Otherwise similar to parse_integer. + */ +static const char *parse_hex_integer(const char *buf, size_t len, uint64_t *value, int *status) +{ + uint64_t x = 0; + const char *k, *k2, *end = buf + len; + int sign, status_; + unsigned char c; + + if (!status) { + status = &status_; + } + if (buf == end) { + *status = PARSE_INTEGER_END; + return buf; + } + sign = *buf == '-'; + buf += sign; + if (end - buf < 2 || buf[0] != '0' || (buf[1] | 0x20) != 'x') { + *status = PARSE_INTEGER_UNMATCHED; + return buf - sign; + } + buf += 2; + k = buf; + k2 = end; + if (end - buf > 16) { + k2 = buf + 16; + } + while (buf != k2) { + c = (unsigned char)*buf; + if (c >= '0' && c <= '9') { + x = x * 16 + c - '0'; + } else { + /* Lower case. */ + c |= 0x20; + if (c >= 'a' && c <= 'f') { + x = x * 16 + c - 'a' + 10; + } else { + break; + } + } + ++buf; + } + if (buf == k) { + if (sign) { + *status = PARSE_INTEGER_INVALID; + return 0; + } else { + /* No number was matched, but it isn't an invalid number either. */ + *status = PARSE_INTEGER_UNMATCHED; + return buf; + } + } + if (buf == end) { + goto done; + } + c = (unsigned char)*buf; + if (buf == k2) { + if (c >= '0' && c <= '9') { + *status = sign ? PARSE_INTEGER_UNDERFLOW : PARSE_INTEGER_OVERFLOW; + return 0; + } + c |= 0x20; + if (c >= 'a' && c <= 'f') { + *status = sign ? PARSE_INTEGER_UNDERFLOW : PARSE_INTEGER_OVERFLOW; + return 0; + } + } + switch (c) { + case '.': case 'p': case 'P': + *status = PARSE_INTEGER_INVALID; + return 0; + } +done: + *value = x; + *status = sign; + return buf; +} + + +#define __portable_define_parse_unsigned(NAME, TYPE, LIMIT) \ +static inline const char *parse_ ## NAME \ + (const char *buf, size_t len, TYPE *value, int *status) \ +{ \ + int status_ = 0; \ + uint64_t x; \ + \ + if (!status) { \ + status = &status_; \ + } \ + buf = parse_integer(buf, len, &x, status); \ + switch (*status) { \ + case PARSE_INTEGER_UNSIGNED: \ + if (x <= LIMIT) { \ + *value = (TYPE)x; \ + return buf; \ + } \ + *status = PARSE_INTEGER_OVERFLOW; \ + return 0; \ + case PARSE_INTEGER_SIGNED: \ + *status = PARSE_INTEGER_UNDERFLOW; \ + return 0; \ + default: \ + return buf; \ + } \ +} + +#define __portable_define_parse_hex_unsigned(NAME, TYPE, LIMIT) \ +static inline const char *parse_hex_ ## NAME \ + (const char *buf, size_t len, TYPE *value, int *status) \ +{ \ + int status_ = 0; \ + uint64_t x; \ + \ + if (!status) { \ + status = &status_; \ + } \ + buf = parse_hex_integer(buf, len, &x, status); \ + switch (*status) { \ + case PARSE_INTEGER_UNSIGNED: \ + if (x <= LIMIT) { \ + *value = (TYPE)x; \ + return buf; \ + } \ + *status = PARSE_INTEGER_OVERFLOW; \ + return 0; \ + case PARSE_INTEGER_SIGNED: \ + *status = PARSE_INTEGER_UNDERFLOW; \ + return 0; \ + default: \ + return buf; \ + } \ +} + +/* This assumes two's complement. */ +#define __portable_define_parse_signed(NAME, TYPE, LIMIT) \ +static inline const char *parse_ ## NAME \ + (const char *buf, size_t len, TYPE *value, int *status) \ +{ \ + int status_ = 0; \ + uint64_t x; \ + \ + if (!status) { \ + status = &status_; \ + } \ + buf = parse_integer(buf, len, &x, status); \ + switch (*status) { \ + case PARSE_INTEGER_UNSIGNED: \ + if (x <= LIMIT) { \ + *value = (TYPE)x; \ + return buf; \ + } \ + *status = PARSE_INTEGER_OVERFLOW; \ + return 0; \ + case PARSE_INTEGER_SIGNED: \ + if (x <= (uint64_t)(LIMIT) + 1) { \ + *value = (TYPE)-(int64_t)x; \ + return buf; \ + } \ + *status = PARSE_INTEGER_UNDERFLOW; \ + return 0; \ + default: \ + return buf; \ + } \ +} + +/* This assumes two's complement. */ +#define __portable_define_parse_hex_signed(NAME, TYPE, LIMIT) \ +static inline const char *parse_hex_ ## NAME \ + (const char *buf, size_t len, TYPE *value, int *status) \ +{ \ + int status_ = 0; \ + uint64_t x; \ + \ + if (!status) { \ + status = &status_; \ + } \ + buf = parse_hex_integer(buf, len, &x, status); \ + switch (*status) { \ + case PARSE_INTEGER_UNSIGNED: \ + if (x <= LIMIT) { \ + *value = (TYPE)x; \ + return buf; \ + } \ + *status = PARSE_INTEGER_OVERFLOW; \ + return 0; \ + case PARSE_INTEGER_SIGNED: \ + if (x <= (uint64_t)(LIMIT) + 1) { \ + *value = (TYPE)-(int64_t)x; \ + return buf; \ + } \ + *status = PARSE_INTEGER_UNDERFLOW; \ + return 0; \ + default: \ + return buf; \ + } \ +} + +static inline const char *parse_uint64(const char *buf, size_t len, uint64_t *value, int *status) +{ + buf = parse_integer(buf, len, value, status); + if (*status == PARSE_INTEGER_SIGNED) { + *status = PARSE_INTEGER_UNDERFLOW; + return 0; + } + return buf; +} + +static inline const char *parse_hex_uint64(const char *buf, size_t len, uint64_t *value, int *status) +{ + buf = parse_hex_integer(buf, len, value, status); + if (*status == PARSE_INTEGER_SIGNED) { + *status = PARSE_INTEGER_UNDERFLOW; + return 0; + } + return buf; +} + +__portable_define_parse_signed(int64, int64_t, INT64_MAX) +__portable_define_parse_signed(int32, int32_t, INT32_MAX) +__portable_define_parse_unsigned(uint16, uint16_t, UINT16_MAX) +__portable_define_parse_signed(int16, int16_t, INT16_MAX) +__portable_define_parse_unsigned(uint8, uint8_t, UINT8_MAX) +__portable_define_parse_signed(int8, int8_t, INT8_MAX) + +__portable_define_parse_hex_signed(int64, int64_t, INT64_MAX) +__portable_define_parse_hex_signed(int32, int32_t, INT32_MAX) +__portable_define_parse_hex_unsigned(uint16, uint16_t, UINT16_MAX) +__portable_define_parse_hex_signed(int16, int16_t, INT16_MAX) +__portable_define_parse_hex_unsigned(uint8, uint8_t, UINT8_MAX) +__portable_define_parse_hex_signed(int8, int8_t, INT8_MAX) + +__portable_define_parse_unsigned(ushort, unsigned short, USHRT_MAX) +__portable_define_parse_signed(short, short, SHRT_MAX) +__portable_define_parse_unsigned(uint, unsigned int, UINT_MAX) +__portable_define_parse_signed(int, int, INT_MAX) +__portable_define_parse_unsigned(ulong, unsigned long, ULONG_MAX) +__portable_define_parse_signed(long, unsigned long, LONG_MAX) + +__portable_define_parse_hex_unsigned(ushort, unsigned short, USHRT_MAX) +__portable_define_parse_hex_signed(short, short, SHRT_MAX) +__portable_define_parse_hex_unsigned(uint, unsigned int, UINT_MAX) +__portable_define_parse_hex_signed(int, int, INT_MAX) +__portable_define_parse_hex_unsigned(ulong, unsigned long, ULONG_MAX) +__portable_define_parse_hex_signed(long, unsigned long, LONG_MAX) + +#ifdef __cplusplus +} +#endif + +#endif /* PPARSEINT_H */ diff --git a/nostrdb/flatcc/portable/pprintfp.h b/nostrdb/flatcc/portable/pprintfp.h new file mode 100644 index 0000000000..c2e5c0750c --- /dev/null +++ b/nostrdb/flatcc/portable/pprintfp.h @@ -0,0 +1,39 @@ +#ifndef PPRINTFP_H +#define PPRINTFP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define PDIAGNOSTIC_IGNORE_UNUSED_FUNCTION +#include "pdiagnostic_push.h" + +#ifndef PORTABLE_USE_GRISU3 +#define PORTABLE_USE_GRISU3 1 +#endif + + +#if PORTABLE_USE_GRISU3 +#include "grisu3_print.h" +#endif + +#ifdef grisu3_print_double_is_defined +/* Currently there is not special support for floats. */ +#define print_float(n, p) grisu3_print_double((float)(n), (p)) +#define print_double(n, p) grisu3_print_double((double)(n), (p)) +#else +#include +#define print_float(n, p) sprintf(p, "%.9g", (float)(n)) +#define print_double(n, p) sprintf(p, "%.17g", (double)(n)) +#endif + +#define print_hex_float(n, p) sprintf(p, "%a", (float)(n)) +#define print_hex_double(n, p) sprintf(p, "%a", (double)(n)) + +#include "pdiagnostic_pop.h" + +#ifdef __cplusplus +} +#endif + +#endif /* PPRINTFP_H */ diff --git a/nostrdb/flatcc/portable/pprintint.h b/nostrdb/flatcc/portable/pprintint.h new file mode 100644 index 0000000000..d05f376117 --- /dev/null +++ b/nostrdb/flatcc/portable/pprintint.h @@ -0,0 +1,628 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2016 Mikkel F. Jørgensen, dvide.com + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + * Fast printing of (u)int8/16/32/64_t, (u)int, (u)long. + * + * Functions take for the + * + * int print_(type value, char *buf); + * + * and returns number of characters printed, excluding trailing '\0' + * which is also printed. Prints at most 21 characters including zero- + * termination. + * + * The function `print_bool` is a bit different - it simply prints "true\0" for + * non-zero integers, and "false\0" otherwise. + * + * The general algorithm is in-place formatting using binary search log10 + * followed by duff device loop unrolling div / 100 stages. + * + * The simpler post copy algorithm also provided for fmt_(u)int uses a + * temp buffer and loops over div/100 and post copy to target buffer. + * + * + * Benchmarks on core-i7, 2.2GHz, 64-bit clang/OS-X -O2: + * + * print_int64: avg 15ns for values between INT64_MIN + (10^7/2 .. 10^7/2) + * print_int64: avg 11ns for values between 10^9 + (0..10,000,000). + * print_int32: avg 7ns for values cast from INT64_MIN + (10^7/2 .. 10^7/2) + * print_int32: avg 7ns for values between 10^9 + (0..10,000,000). + * print_int64: avg 13ns for values between 10^16 + (0..10,000,000). + * print_int64: avg 5ns for values between 0 and 10,000,000. + * print_int32: avg 5ns for values between 0 and 10,000,000. + * print_int16: avg 10ns for values cast from 0 and 10,000,000. + * print_int8: avg 4ns for values cast from 0 and 10,000,000. + * + * Post copy algorithm: + * print_int: avg 12ns for values between INT64_MIN + (10^7/2 .. 10^7/2) + * print_int: avg 14ns for values between 10^9 + (0..10,000,000). + * print_long: avg 29ns for values between INT64_MIN + (10^7/2 .. 10^7/2) + * + * The post copy algorithm is nearly half as fast as the in-place + * algorithm, but can also be faster occasionally - possibly because the + * optimizer being able to skip the copy step. + */ + +#ifndef PPRINTINT_H +#define PPRINTINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef UINT8_MAX +#include +#endif + +#include "pattributes.h" /* fallthrough */ + +#define PDIAGNOSTIC_IGNORE_UNUSED_FUNCTION +#include "pdiagnostic_push.h" + +static int print_bool(int n, char *p); + +static int print_uint8(uint8_t n, char *p); +static int print_uint16(uint16_t n, char *p); +static int print_uint32(uint32_t n, char *p); +static int print_uint64(uint64_t n, char *p); +static int print_int8(int8_t n, char *p); +static int print_int16(int16_t n, char *p); +static int print_int32(int32_t n, char *p); +static int print_int64(int64_t n, char *p); + +/* + * Uses slightly slower, but more compact alogrithm + * that is not hardcoded to implementation size. + * Other types may be defined using macros below. + */ +static int print_ulong(unsigned long n, char *p); +static int print_uint(unsigned int n, char *p); +static int print_int(int n, char *p); +static int print_long(long n, char *p); + + +#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) +#define __print_unaligned_copy_16(p, q) (*(uint16_t*)(p) = *(uint16_t*)(q)) +#else +#define __print_unaligned_copy_16(p, q) \ + ((((uint8_t*)(p))[0] = ((uint8_t*)(q))[0]), \ + (((uint8_t*)(p))[1] = ((uint8_t*)(q))[1])) +#endif + +static const char __print_digit_pairs[] = + "0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"; + +#define __print_stage() \ + p -= 2; \ + dp = __print_digit_pairs + (n % 100) * 2; \ + n /= 100; \ + __print_unaligned_copy_16(p, dp); + +#define __print_long_stage() \ + __print_stage() \ + __print_stage() + +#define __print_short_stage() \ + *--p = (n % 10) + '0'; \ + n /= 10; + +static int print_bool(int n, char *buf) +{ + if (n) { + memcpy(buf, "true\0", 5); + return 4; + } else { + memcpy(buf, "false\0", 6); + return 5; + } +} + +static int print_uint8(uint8_t n, char *p) +{ + const char *dp; + + if (n >= 100) { + p += 3; + *p = '\0'; + __print_stage(); + p[-1] = (char)n + '0'; + return 3; + } + if (n >= 10) { + p += 2; + *p = '\0'; + __print_stage(); + return 2; + } + p[1] = '\0'; + p[0] = (char)n + '0'; + return 1; +} + +static int print_uint16(uint16_t n, char *p) +{ + int k = 0; + const char *dp; + + if (n >= 1000) { + if(n >= 10000) { + k = 5; + } else { + k = 4; + } + } else { + if(n >= 100) { + k = 3; + } else if(n >= 10) { + k = 2; + } else { + k = 1; + } + } + p += k; + *p = '\0'; + if (k & 1) { + switch (k) { + case 5: + __print_stage(); + pattribute(fallthrough); + case 3: + __print_stage(); + pattribute(fallthrough); + case 1: + p[-1] = (char)n + '0'; + } + } else { + switch (k) { + case 4: + __print_stage(); + pattribute(fallthrough); + case 2: + __print_stage(); + } + } + return k; +} + +static int print_uint32(uint32_t n, char *p) +{ + int k = 0; + const char *dp; + + if(n >= 10000UL) { + if(n >= 10000000UL) { + if(n >= 1000000000UL) { + k = 10; + } else if(n >= 100000000UL) { + k = 9; + } else { + k = 8; + } + } else { + if(n >= 1000000UL) { + k = 7; + } else if(n >= 100000UL) { + k = 6; + } else { + k = 5; + } + } + } else { + if(n >= 100UL) { + if(n >= 1000UL) { + k = 4; + } else { + k = 3; + } + } else { + if(n >= 10UL) { + k = 2; + } else { + k = 1UL; + } + } + } + p += k; + *p = '\0'; + if (k & 1) { + switch (k) { + case 9: + __print_stage(); + pattribute(fallthrough); + case 7: + __print_stage(); + pattribute(fallthrough); + case 5: + __print_stage(); + pattribute(fallthrough); + case 3: + __print_stage(); + pattribute(fallthrough); + case 1: + p[-1] = (char)n + '0'; + } + } else { + switch (k) { + case 10: + __print_stage(); + pattribute(fallthrough); + case 8: + __print_stage(); + pattribute(fallthrough); + case 6: + __print_stage(); + pattribute(fallthrough); + case 4: + __print_stage(); + pattribute(fallthrough); + case 2: + __print_stage(); + } + } + return k; +} + +static int print_uint64(uint64_t n, char *p) +{ + int k = 0; + const char *dp; + const uint64_t x = 1000000000ULL; + + if (n < x) { + return print_uint32((uint32_t)n, p); + } + if(n >= 10000ULL * x) { + if(n >= 10000000ULL * x) { + if(n >= 1000000000ULL * x) { + if (n >= 10000000000ULL * x) { + k = 11 + 9; + } else { + k = 10 + 9; + } + } else if(n >= 100000000ULL * x) { + k = 9 + 9; + } else { + k = 8 + 9; + } + } else { + if(n >= 1000000ULL * x) { + k = 7 + 9; + } else if(n >= 100000ULL * x) { + k = 6 + 9; + } else { + k = 5 + 9; + } + } + } else { + if(n >= 100ULL * x) { + if(n >= 1000ULL * x) { + k = 4 + 9; + } else { + k = 3 + 9; + } + } else { + if(n >= 10ULL * x) { + k = 2 + 9; + } else { + k = 1 + 9; + } + } + } + p += k; + *p = '\0'; + if (k & 1) { + switch (k) { + case 19: + __print_stage(); + pattribute(fallthrough); + case 17: + __print_stage(); + pattribute(fallthrough); + case 15: + __print_stage(); + pattribute(fallthrough); + case 13: + __print_stage(); + pattribute(fallthrough); + case 11: + __print_stage() + __print_short_stage(); + } + } else { + switch (k) { + case 20: + __print_stage(); + pattribute(fallthrough); + case 18: + __print_stage(); + pattribute(fallthrough); + case 16: + __print_stage(); + pattribute(fallthrough); + case 14: + __print_stage(); + pattribute(fallthrough); + case 12: + __print_stage(); + pattribute(fallthrough); + case 10: + __print_stage(); + } + } + __print_long_stage() + __print_long_stage() + return k; +} + +static int print_int8(int8_t n, char *p) +{ + int sign; + + if ((sign = n < 0)) { + *p++ = '-'; + n = -n; + } + return print_uint8((uint8_t)n, p) + sign; +} + +static int print_int16(int16_t n, char *p) +{ + int sign; + + if ((sign = n < 0)) { + *p++ = '-'; + n = -n; + } + return print_uint16((uint16_t)n, p) + sign; +} + +static int print_int32(int32_t n, char *p) +{ + int sign; + + if ((sign = n < 0)) { + *p++ = '-'; + n = -n; + } + return print_uint32((uint32_t)n, p) + sign; +} + +static int print_int64(int64_t n, char *p) +{ + int sign; + + if ((sign = n < 0)) { + *p++ = '-'; + n = -n; + } + return print_uint64((uint64_t)n, p) + sign; +} + +#define __define_print_int_simple(NAME, UNAME, T, UT) \ +static int UNAME(UT n, char *buf) \ +{ \ + char tmp[20]; \ + char* p = tmp + 20; \ + char* q = p; \ + unsigned int k, m; \ + \ + while (n >= 100) { \ + p -= 2; \ + m = (unsigned int)(n % 100) * 2; \ + n /= 100; \ + __print_unaligned_copy_16(p, __print_digit_pairs + m); \ + } \ + p -= 2; \ + m = (unsigned int)n * 2; \ + __print_unaligned_copy_16(p, __print_digit_pairs + m); \ + if (n < 10) { \ + ++p; \ + } \ + k = (unsigned int)(q - p); \ + while (p != q) { \ + *buf++ = *p++; \ + } \ + *buf = '\0'; \ + return (int)k; \ +} \ + \ +static int NAME(T n, char *buf) \ +{ \ + int sign = n < 0; \ + \ + if (sign) { \ + *buf++ = '-'; \ + n = -n; \ + } \ + return UNAME((UT)n, buf) + sign; \ +} + +__define_print_int_simple(print_int, print_uint, int, unsigned int) +__define_print_int_simple(print_long, print_ulong, long, unsigned long) + +#ifdef PPRINTINT_BENCH +int main() { + int64_t count = 10000000; /* 10^7 */ +#if 0 + int64_t base = 0; + int64_t base = 10000000000000000; /* 10^16 */ + int64_t base = 1000000000; /* 10^9 */ +#endif + int64_t base = INT64_MIN - count/2; + char buf[100]; + int i, k = 0, n = 0; + for (i = 0; i < count; i++) { + k = print_int64(i + base, buf); + n += buf[0] + buf[k - 1]; + } + return n; +} +/* Call with time on executable, multiply time in seconds by 100 to get time unit in ns/number. */ +#endif /* PPRINTINT_BENCH */ + +#ifdef PPRINTINT_TEST + +#include +#include + +int main() +{ + char buf[21]; + int failed = 0; + int k; + + k = print_uint64(UINT64_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("18446744073709551615", buf)) { + printf("UINT64_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int64(INT64_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("9223372036854775807", buf)) { + printf("INT64_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int64(INT64_MIN, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("-9223372036854775808", buf)) { + printf("INT64_MIN didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_uint32(UINT32_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("4294967295", buf)) { + printf("UINT32_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int32(INT32_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("2147483647", buf)) { + printf("INT32_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int32(INT32_MIN, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("-2147483648", buf)) { + printf("INT32_MIN didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_uint16(UINT16_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("65535", buf)) { + printf("UINT16_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int16(INT16_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("32767", buf)) { + printf("INT16_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int16(INT16_MIN, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("-32768", buf)) { + printf("INT16_MIN didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_uint8(UINT8_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("255", buf)) { + printf("INT8_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int8(INT8_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("127", buf)) { + printf("INT8_MAX didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int8(INT8_MIN, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("-128", buf)) { + printf("INT8_MIN didn't print correctly, got:\n'%s'\n", buf); + ++failed; + } + k = print_int(INT32_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("2147483647", buf)) { + printf("INT32_MAX didn't print correctly with k = print_int, got:\n'%s'\n", buf); + ++failed; + } + k = print_int(INT32_MIN, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("-2147483648", buf)) { + printf("INT32_MIN didn't print correctly k = print_int, got:\n'%s'\n", buf); + ++failed; + } + k = print_long(INT32_MAX, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("2147483647", buf)) { + printf("INT32_MAX didn't print correctly with fmt_long, got:\n'%s'\n", buf); + ++failed; + } + k = print_long(INT32_MIN, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("-2147483648", buf)) { + printf("INT32_MIN didn't print correctly fmt_long, got:\n'%s'\n", buf); + ++failed; + } + k = print_bool(1, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("true", buf) { + printf("1 didn't print 'true' as expected, got:\n'%s'\n", buf); + ++failed; + } + k = print_bool(-1, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("true", buf) { + printf("-1 didn't print 'true' as expected, got:\n'%s'\n", buf); + ++failed; + } + k = print_bool(, buf); + if (strlen(buf) != k) printf("length error\n"); + if (strcmp("false", buf) { + printf("0 didn't print 'false' as expected, got:\n'%s'\n", buf); + ++failed; + } + if (failed) { + printf("FAILED\n"); + return -1; + } + printf("SUCCESS\n"); + return 0; +} +#endif /* PPRINTINT_TEST */ + +#include "pdiagnostic_pop.h" + +#ifdef __cplusplus +} +#endif + +#endif /* PPRINTINT_H */ diff --git a/nostrdb/flatcc/portable/pstatic_assert.h b/nostrdb/flatcc/portable/pstatic_assert.h new file mode 100644 index 0000000000..24d5634e61 --- /dev/null +++ b/nostrdb/flatcc/portable/pstatic_assert.h @@ -0,0 +1,67 @@ +#ifndef PSTATIC_ASSERT_H +#define PSTATIC_ASSERT_H + +#include + +/* Handle clang */ +#ifndef __has_feature + #define __has_feature(x) 0 +#endif + +#if defined(static_assert) +#ifndef __static_assert_is_defined +#define __static_assert_is_defined 1 +#endif +#endif + +/* Handle static_assert as a keyword in C++ and compiler specifics. */ +#if !defined(__static_assert_is_defined) + +#if defined(__cplusplus) + +#if __cplusplus >= 201103L +#define __static_assert_is_defined 1 +#elif __has_feature(cxx_static_assert) +#define __static_assert_is_defined 1 +#elif defined(_MSC_VER) && (_MSC_VER >= 1600) +#define __static_assert_is_defined 1 +#endif + +#else + +#if defined(_MSC_VER) && (_MSC_VER >= 1600) +#define __static_assert_is_defined 1 +#elif __has_feature(c_static_assert) +#define static_assert(pred, msg) _Static_assert(pred, msg) +#define __static_assert_is_defined 1 +#elif defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* In case the clib headers are not compliant. */ +#define static_assert(pred, msg) _Static_assert(pred, msg) +#define __static_assert_is_defined 1 +#endif + +#endif /* __cplusplus */ +#endif /* __static_assert_is_defined */ + + +#if !defined(__static_assert_is_defined) + +#define __PSTATIC_ASSERT_CONCAT_(a, b) static_assert_scope_##a##_line_##b +#define __PSTATIC_ASSERT_CONCAT(a, b) __PSTATIC_ASSERT_CONCAT_(a, b) +#ifdef __COUNTER__ +#define static_assert(e, msg) enum { __PSTATIC_ASSERT_CONCAT(__COUNTER__, __LINE__) = 1/(!!(e)) } +#else +#include "pstatic_assert_scope.h" +#define static_assert(e, msg) enum { __PSTATIC_ASSERT_CONCAT(__PSTATIC_ASSERT_COUNTER, __LINE__) = 1/(int)(!!(e)) } +#endif + +#define __static_assert_is_defined 1 + +#endif /* __static_assert_is_defined */ + +#endif /* PSTATIC_ASSERT_H */ + +/* Update scope counter outside of include guard. */ +#ifdef __PSTATIC_ASSERT_COUNTER +#include "pstatic_assert_scope.h" +#endif diff --git a/nostrdb/flatcc/portable/pstatic_assert_scope.h b/nostrdb/flatcc/portable/pstatic_assert_scope.h new file mode 100644 index 0000000000..71a0c29888 --- /dev/null +++ b/nostrdb/flatcc/portable/pstatic_assert_scope.h @@ -0,0 +1,280 @@ +/* + * january, 2017, ported to portable library by mikkelfj. + * Based on dbgtools static assert counter, but with renamed macros. + */ + +/* + dbgtools - platform independent wrapping of "nice to have" debug functions. + + version 0.1, october, 2013 + + https://github.com/wc-duck/dbgtools + + Copyright (C) 2013- Fredrik Kihlander + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Fredrik Kihlander +*/ + +/** + * Auto-generated header implementing a counter that increases by each include of the file. + * + * This header will define the macro __PSTATIC_ASSERT_COUNTER to be increased for each inclusion of the file. + * + * It has been generated with 3 amount of digits resulting in the counter wrapping around after + * 10000 inclusions. + * + * Usage: + * + * #include "this_header.h" + * int a = __PSTATIC_ASSERT_COUNTER; // 0 + * #include "this_header.h" + * int b = __PSTATIC_ASSERT_COUNTER; // 1 + * #include "this_header.h" + * int c = __PSTATIC_ASSERT_COUNTER; // 2 + * #include "this_header.h" + * int d = __PSTATIC_ASSERT_COUNTER; // 3 + */ + +#ifndef __PSTATIC_ASSERT_COUNTER +# define __PSTATIC_ASSERT_COUNTER_0 0 +# define __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_D1_0 +# define __PSTATIC_ASSERT_COUNTER_D2_0 +# define __PSTATIC_ASSERT_COUNTER_D3_0 +#endif /* __PSTATIC_ASSERT_COUNTER */ + +#if !defined( __PSTATIC_ASSERT_COUNTER_D0_0 ) +# define __PSTATIC_ASSERT_COUNTER_D0_0 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 0 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_1 ) +# define __PSTATIC_ASSERT_COUNTER_D0_1 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 1 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_2 ) +# define __PSTATIC_ASSERT_COUNTER_D0_2 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 2 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_3 ) +# define __PSTATIC_ASSERT_COUNTER_D0_3 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 3 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_4 ) +# define __PSTATIC_ASSERT_COUNTER_D0_4 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 4 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_5 ) +# define __PSTATIC_ASSERT_COUNTER_D0_5 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 5 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_6 ) +# define __PSTATIC_ASSERT_COUNTER_D0_6 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 6 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_7 ) +# define __PSTATIC_ASSERT_COUNTER_D0_7 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 7 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_8 ) +# define __PSTATIC_ASSERT_COUNTER_D0_8 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 8 +#elif !defined( __PSTATIC_ASSERT_COUNTER_D0_9 ) +# define __PSTATIC_ASSERT_COUNTER_D0_9 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 9 +#else +# undef __PSTATIC_ASSERT_COUNTER_D0_1 +# undef __PSTATIC_ASSERT_COUNTER_D0_2 +# undef __PSTATIC_ASSERT_COUNTER_D0_3 +# undef __PSTATIC_ASSERT_COUNTER_D0_4 +# undef __PSTATIC_ASSERT_COUNTER_D0_5 +# undef __PSTATIC_ASSERT_COUNTER_D0_6 +# undef __PSTATIC_ASSERT_COUNTER_D0_7 +# undef __PSTATIC_ASSERT_COUNTER_D0_8 +# undef __PSTATIC_ASSERT_COUNTER_D0_9 +# undef __PSTATIC_ASSERT_COUNTER_0 +# define __PSTATIC_ASSERT_COUNTER_0 0 +# if !defined( __PSTATIC_ASSERT_COUNTER_D1_0 ) +# define __PSTATIC_ASSERT_COUNTER_D1_0 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 0 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_1 ) +# define __PSTATIC_ASSERT_COUNTER_D1_1 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 1 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_2 ) +# define __PSTATIC_ASSERT_COUNTER_D1_2 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 2 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_3 ) +# define __PSTATIC_ASSERT_COUNTER_D1_3 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 3 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_4 ) +# define __PSTATIC_ASSERT_COUNTER_D1_4 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 4 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_5 ) +# define __PSTATIC_ASSERT_COUNTER_D1_5 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 5 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_6 ) +# define __PSTATIC_ASSERT_COUNTER_D1_6 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 6 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_7 ) +# define __PSTATIC_ASSERT_COUNTER_D1_7 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 7 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_8 ) +# define __PSTATIC_ASSERT_COUNTER_D1_8 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 8 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D1_9 ) +# define __PSTATIC_ASSERT_COUNTER_D1_9 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 9 +# else +# undef __PSTATIC_ASSERT_COUNTER_D1_1 +# undef __PSTATIC_ASSERT_COUNTER_D1_2 +# undef __PSTATIC_ASSERT_COUNTER_D1_3 +# undef __PSTATIC_ASSERT_COUNTER_D1_4 +# undef __PSTATIC_ASSERT_COUNTER_D1_5 +# undef __PSTATIC_ASSERT_COUNTER_D1_6 +# undef __PSTATIC_ASSERT_COUNTER_D1_7 +# undef __PSTATIC_ASSERT_COUNTER_D1_8 +# undef __PSTATIC_ASSERT_COUNTER_D1_9 +# undef __PSTATIC_ASSERT_COUNTER_1 +# define __PSTATIC_ASSERT_COUNTER_1 0 +# if !defined( __PSTATIC_ASSERT_COUNTER_D2_0 ) +# define __PSTATIC_ASSERT_COUNTER_D2_0 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 0 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_1 ) +# define __PSTATIC_ASSERT_COUNTER_D2_1 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 1 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_2 ) +# define __PSTATIC_ASSERT_COUNTER_D2_2 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 2 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_3 ) +# define __PSTATIC_ASSERT_COUNTER_D2_3 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 3 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_4 ) +# define __PSTATIC_ASSERT_COUNTER_D2_4 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 4 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_5 ) +# define __PSTATIC_ASSERT_COUNTER_D2_5 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 5 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_6 ) +# define __PSTATIC_ASSERT_COUNTER_D2_6 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 6 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_7 ) +# define __PSTATIC_ASSERT_COUNTER_D2_7 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 7 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_8 ) +# define __PSTATIC_ASSERT_COUNTER_D2_8 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 8 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D2_9 ) +# define __PSTATIC_ASSERT_COUNTER_D2_9 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 9 +# else +# undef __PSTATIC_ASSERT_COUNTER_D2_1 +# undef __PSTATIC_ASSERT_COUNTER_D2_2 +# undef __PSTATIC_ASSERT_COUNTER_D2_3 +# undef __PSTATIC_ASSERT_COUNTER_D2_4 +# undef __PSTATIC_ASSERT_COUNTER_D2_5 +# undef __PSTATIC_ASSERT_COUNTER_D2_6 +# undef __PSTATIC_ASSERT_COUNTER_D2_7 +# undef __PSTATIC_ASSERT_COUNTER_D2_8 +# undef __PSTATIC_ASSERT_COUNTER_D2_9 +# undef __PSTATIC_ASSERT_COUNTER_2 +# define __PSTATIC_ASSERT_COUNTER_2 0 +# if !defined( __PSTATIC_ASSERT_COUNTER_D3_0 ) +# define __PSTATIC_ASSERT_COUNTER_D3_0 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 0 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_1 ) +# define __PSTATIC_ASSERT_COUNTER_D3_1 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 1 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_2 ) +# define __PSTATIC_ASSERT_COUNTER_D3_2 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 2 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_3 ) +# define __PSTATIC_ASSERT_COUNTER_D3_3 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 3 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_4 ) +# define __PSTATIC_ASSERT_COUNTER_D3_4 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 4 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_5 ) +# define __PSTATIC_ASSERT_COUNTER_D3_5 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 5 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_6 ) +# define __PSTATIC_ASSERT_COUNTER_D3_6 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 6 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_7 ) +# define __PSTATIC_ASSERT_COUNTER_D3_7 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 7 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_8 ) +# define __PSTATIC_ASSERT_COUNTER_D3_8 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 8 +# elif !defined( __PSTATIC_ASSERT_COUNTER_D3_9 ) +# define __PSTATIC_ASSERT_COUNTER_D3_9 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 9 +# else +# undef __PSTATIC_ASSERT_COUNTER_D3_1 +# undef __PSTATIC_ASSERT_COUNTER_D3_2 +# undef __PSTATIC_ASSERT_COUNTER_D3_3 +# undef __PSTATIC_ASSERT_COUNTER_D3_4 +# undef __PSTATIC_ASSERT_COUNTER_D3_5 +# undef __PSTATIC_ASSERT_COUNTER_D3_6 +# undef __PSTATIC_ASSERT_COUNTER_D3_7 +# undef __PSTATIC_ASSERT_COUNTER_D3_8 +# undef __PSTATIC_ASSERT_COUNTER_D3_9 +# undef __PSTATIC_ASSERT_COUNTER_3 +# define __PSTATIC_ASSERT_COUNTER_3 0 +# endif +# endif +# endif +#endif + +#define __PSTATIC_ASSERT_COUNTER_JOIN_DIGITS_MACRO_(digit0,digit1,digit2,digit3) digit0##digit1##digit2##digit3 +#define __PSTATIC_ASSERT_COUNTER_JOIN_DIGITS_MACRO(digit0,digit1,digit2,digit3) __PSTATIC_ASSERT_COUNTER_JOIN_DIGITS_MACRO_(digit0,digit1,digit2,digit3) +#undef __PSTATIC_ASSERT_COUNTER +#define __PSTATIC_ASSERT_COUNTER __PSTATIC_ASSERT_COUNTER_JOIN_DIGITS_MACRO(__PSTATIC_ASSERT_COUNTER_3,__PSTATIC_ASSERT_COUNTER_2,__PSTATIC_ASSERT_COUNTER_1,__PSTATIC_ASSERT_COUNTER_0) diff --git a/nostrdb/flatcc/portable/pstdalign.h b/nostrdb/flatcc/portable/pstdalign.h new file mode 100644 index 0000000000..169fe27ed9 --- /dev/null +++ b/nostrdb/flatcc/portable/pstdalign.h @@ -0,0 +1,162 @@ +#ifndef PSTDALIGN_H +#define PSTDALIGN_H + +/* + * NOTE: aligned_alloc is defined via paligned_alloc.h + * and requires aligned_free to be fully portable although + * free also works on C11 and platforms with posix_memalign. + * + * NOTE: C++11 defines alignas as a keyword but then also defines + * __alignas_is_defined. + * + * C++14 does not define __alignas_is_defined, at least sometimes. + * + * GCC 8.3 reverts on this and makes C++11 behave the same as C++14 + * preventing a simple __cplusplus version check from working. + * + * Clang C++ without std=c++11 or std=c++14 does define alignas + * but does so incorrectly wrt. C11 and C++11 semantics because + * `alignas(4) float x;` is not recognized. + * To fix such issues, either move to a std version, or + * include a working stdalign.h for the given compiler before + * this file. + * + * newlib defines _Alignas and _Alignof in sys/cdefs but rely on + * gcc version for which can lead to conflicts if + * stdalign is not included. + * + * newlibs need for conflicts with broken C++ stdalign + * but this can be fixed be using std=C++11 or newer. + * + * MSVC does not support at least up to MSVC 2015, + * but does appear to support alignas and alignof keywords in + * recent standard C++. + * + * TCC only supports alignas with a numeric argument like + * `alignas(4)`, but not `alignas(float)`. + * + * If stdalign.h is supported but heuristics in this file are + * insufficient to detect this, try including manually + * or define HAVE_STDALIGN_H. + */ + +/* https://github.com/dvidelabs/flatcc/issues/130 */ +#ifndef __alignas_is_defined +#if defined(__cplusplus) +#if __cplusplus == 201103 && !defined(__clang__) && ((__GNUC__ > 8) || (__GNUC__ == 8 && __GNUC_MINOR__ >= 3)) +#define __alignas_is_defined 1 +#define __alignof_is_defined 1 +#include +#endif +#endif +#endif + +/* Allow for alternative solution to be included first. */ +#ifndef __alignas_is_defined + +#ifdef __cplusplus +#if defined(PORTABLE_PATCH_CPLUSPLUS_STDALIGN) +#include +#undef alignas +#define alignas(t) __attribute__((__aligned__(t))) +#endif +#endif + +#if !defined(PORTABLE_HAS_INCLUDE_STDALIGN) +#if defined(__has_include) +#if __has_include() +#define PORTABLE_HAS_INCLUDE_STDALIGN 1 +#else +#define PORTABLE_HAS_INCLUDE_STDALIGN 0 +#endif +#endif +#endif + + /* https://lists.gnu.org/archive/html/bug-gnulib/2015-08/msg00003.html */ +#if defined(__cplusplus) +#if !defined(_MSC_VER) +#include +#endif +#if __cplusplus > 201103 +#define __alignas_is_defined 1 +#define __alignof_is_defined 1 +#endif +#elif PORTABLE_HAS_INCLUDE_STDALIGN +#include +#elif !defined(__clang__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7)) +#include +#elif defined(HAVE_STDALIGN_H) +#include +#endif + +#endif /* __alignas_is_defined */ + +#ifndef __alignas_is_defined + +#ifdef __cplusplus +extern "C" { +#endif + +#if (!defined(__clang__) && defined(__GNUC__) && \ + ((__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 7))) +#undef PORTABLE_C11_STDALIGN_MISSING +#define PORTABLE_C11_STDALIGN_MISSING +#endif + +#if defined(__IBMC__) +#undef PORTABLE_C11_STDALIGN_MISSING +#define PORTABLE_C11_STDALIGN_MISSING +#endif + +#if ((defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) && \ + !defined(PORTABLE_C11_STDALIGN_MISSING)) +/* C11 or newer */ +#include +#else +#if defined(__GNUC__) || defined(__IBM_ALIGNOF__) || defined(__clang__) + +#ifndef _Alignas +#define _Alignas(t) __attribute__((__aligned__(t))) +#endif + +#ifndef _Alignof +#define _Alignof(t) __alignof__(t) +#endif + +#elif defined(_MSC_VER) + +#define _Alignas(t) __declspec (align(t)) +#define _Alignof(t) __alignof(t) + +#elif defined(__TINYC__) + +/* Supports `_Alignas(integer-expression)`, but not `_Alignas(type)`. */ +#define _Alignas(t) __attribute__(aligned(t)) +#define _Alignof(t) __alignof__(t) + +#else +#error please update pstdalign.h with support for current compiler and library +#endif + +#endif /* __STDC__ */ + +#ifndef alignas +#define alignas _Alignas +#endif + +#ifndef alignof +#define alignof _Alignof +#endif + +#define __alignas_is_defined 1 +#define __alignof_is_defined 1 + +#ifdef __cplusplus +} +#endif + +#endif /* __alignas__is_defined */ + +#include "paligned_alloc.h" + +#endif /* PSTDALIGN_H */ diff --git a/nostrdb/flatcc/portable/pstdbool.h b/nostrdb/flatcc/portable/pstdbool.h new file mode 100644 index 0000000000..28fc89c2c4 --- /dev/null +++ b/nostrdb/flatcc/portable/pstdbool.h @@ -0,0 +1,37 @@ +#ifndef PSTDBOOL_H +#define PSTDBOOL_H + +#if !defined(__cplusplus) && !__bool_true_false_are_defined && !defined(bool) && !defined(__STDBOOL_H) + +#ifdef HAVE_STDBOOL_H + +#include + +#elif (defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +/* C99 or newer */ + +#define bool _Bool +#define true 1 +#define false 0 +#define __bool_true_false_are_defined 1 + +#elif defined(__GNUC__) && !defined(__STRICT_ANSI__) + +#define bool bool +#define true true +#define false false +#define __bool_true_false_are_defined 1 + +#else + +typedef unsigned char _Portable_bool; +#define bool _Portable_bool +#define true 1 +#define false 0 +#define __bool_true_false_are_defined 1 + +#endif + +#endif + +#endif /* PSTDBOOL_H */ diff --git a/nostrdb/flatcc/portable/pstdint.h b/nostrdb/flatcc/portable/pstdint.h new file mode 100644 index 0000000000..d522fed1b2 --- /dev/null +++ b/nostrdb/flatcc/portable/pstdint.h @@ -0,0 +1,898 @@ +/* A portable stdint.h + **************************************************************************** + * BSD License: + **************************************************************************** + * + * Copyright (c) 2005-2016 Paul Hsieh + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************** + * + * Version 0.1.15.2 + * + * The ANSI C standard committee, for the C99 standard, specified the + * inclusion of a new standard include file called stdint.h. This is + * a very useful and long desired include file which contains several + * very precise definitions for integer scalar types that is + * critically important for making portable several classes of + * applications including cryptography, hashing, variable length + * integer libraries and so on. But for most developers its likely + * useful just for programming sanity. + * + * The problem is that some compiler vendors chose to ignore the C99 + * standard and some older compilers have no opportunity to be updated. + * Because of this situation, simply including stdint.h in your code + * makes it unportable. + * + * So that's what this file is all about. Its an attempt to build a + * single universal include file that works on as many platforms as + * possible to deliver what stdint.h is supposed to. Even compilers + * that already come with stdint.h can use this file instead without + * any loss of functionality. A few things that should be noted about + * this file: + * + * 1) It is not guaranteed to be portable and/or present an identical + * interface on all platforms. The extreme variability of the + * ANSI C standard makes this an impossibility right from the + * very get go. Its really only meant to be useful for the vast + * majority of platforms that possess the capability of + * implementing usefully and precisely defined, standard sized + * integer scalars. Systems which are not intrinsically 2s + * complement may produce invalid constants. + * + * 2) There is an unavoidable use of non-reserved symbols. + * + * 3) Other standard include files are invoked. + * + * 4) This file may come in conflict with future platforms that do + * include stdint.h. The hope is that one or the other can be + * used with no real difference. + * + * 5) In the current verison, if your platform can't represent + * int32_t, int16_t and int8_t, it just dumps out with a compiler + * error. + * + * 6) 64 bit integers may or may not be defined. Test for their + * presence with the test: #ifdef INT64_MAX or #ifdef UINT64_MAX. + * Note that this is different from the C99 specification which + * requires the existence of 64 bit support in the compiler. If + * this is not defined for your platform, yet it is capable of + * dealing with 64 bits then it is because this file has not yet + * been extended to cover all of your system's capabilities. + * + * 7) (u)intptr_t may or may not be defined. Test for its presence + * with the test: #ifdef PTRDIFF_MAX. If this is not defined + * for your platform, then it is because this file has not yet + * been extended to cover all of your system's capabilities, not + * because its optional. + * + * 8) The following might not been defined even if your platform is + * capable of defining it: + * + * WCHAR_MIN + * WCHAR_MAX + * (u)int64_t + * PTRDIFF_MIN + * PTRDIFF_MAX + * (u)intptr_t + * + * 9) The following have not been defined: + * + * WINT_MIN + * WINT_MAX + * + * 10) The criteria for defining (u)int_least(*)_t isn't clear, + * except for systems which don't have a type that precisely + * defined 8, 16, or 32 bit types (which this include file does + * not support anyways). Default definitions have been given. + * + * 11) The criteria for defining (u)int_fast(*)_t isn't something I + * would trust to any particular compiler vendor or the ANSI C + * committee. It is well known that "compatible systems" are + * commonly created that have very different performance + * characteristics from the systems they are compatible with, + * especially those whose vendors make both the compiler and the + * system. Default definitions have been given, but its strongly + * recommended that users never use these definitions for any + * reason (they do *NOT* deliver any serious guarantee of + * improved performance -- not in this file, nor any vendor's + * stdint.h). + * + * 12) The following macros: + * + * PRINTF_INTMAX_MODIFIER + * PRINTF_INT64_MODIFIER + * PRINTF_INT32_MODIFIER + * PRINTF_INT16_MODIFIER + * PRINTF_LEAST64_MODIFIER + * PRINTF_LEAST32_MODIFIER + * PRINTF_LEAST16_MODIFIER + * PRINTF_INTPTR_MODIFIER + * + * are strings which have been defined as the modifiers required + * for the "d", "u" and "x" printf formats to correctly output + * (u)intmax_t, (u)int64_t, (u)int32_t, (u)int16_t, (u)least64_t, + * (u)least32_t, (u)least16_t and (u)intptr_t types respectively. + * PRINTF_INTPTR_MODIFIER is not defined for some systems which + * provide their own stdint.h. PRINTF_INT64_MODIFIER is not + * defined if INT64_MAX is not defined. These are an extension + * beyond what C99 specifies must be in stdint.h. + * + * In addition, the following macros are defined: + * + * PRINTF_INTMAX_HEX_WIDTH + * PRINTF_INT64_HEX_WIDTH + * PRINTF_INT32_HEX_WIDTH + * PRINTF_INT16_HEX_WIDTH + * PRINTF_INT8_HEX_WIDTH + * PRINTF_INTMAX_DEC_WIDTH + * PRINTF_INT64_DEC_WIDTH + * PRINTF_INT32_DEC_WIDTH + * PRINTF_INT16_DEC_WIDTH + * PRINTF_UINT8_DEC_WIDTH + * PRINTF_UINTMAX_DEC_WIDTH + * PRINTF_UINT64_DEC_WIDTH + * PRINTF_UINT32_DEC_WIDTH + * PRINTF_UINT16_DEC_WIDTH + * PRINTF_UINT8_DEC_WIDTH + * + * Which specifies the maximum number of characters required to + * print the number of that type in either hexadecimal or decimal. + * These are an extension beyond what C99 specifies must be in + * stdint.h. + * + * Compilers tested (all with 0 warnings at their highest respective + * settings): Borland Turbo C 2.0, WATCOM C/C++ 11.0 (16 bits and 32 + * bits), Microsoft Visual C++ 6.0 (32 bit), Microsoft Visual Studio + * .net (VC7), Intel C++ 4.0, GNU gcc v3.3.3 + * + * This file should be considered a work in progress. Suggestions for + * improvements, especially those which increase coverage are strongly + * encouraged. + * + * Acknowledgements + * + * The following people have made significant contributions to the + * development and testing of this file: + * + * Chris Howie + * John Steele Scott + * Dave Thorup + * John Dill + * Florian Wobbe + * Christopher Sean Morrison + * Mikkel Fahnoe Jorgensen + * + */ + +#include +#include +#include + +/* + * For gcc with _STDINT_H, fill in the PRINTF_INT*_MODIFIER macros, and + * do nothing else. On the Mac OS X version of gcc this is _STDINT_H_. + */ + +#if ((defined(_MSC_VER) && _MSC_VER >= 1600) || (defined(__STDC__) && __STDC__ && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (defined (__WATCOMC__) && (defined (_STDINT_H_INCLUDED) || __WATCOMC__ >= 1250)) || (defined(__GNUC__) && (__GNUC__ > 3 || defined(_STDINT_H) || defined(_STDINT_H_) || defined (__UINT_FAST64_TYPE__)) )) && !defined (_PSTDINT_H_INCLUDED) +#include +#define _PSTDINT_H_INCLUDED +# if defined(__GNUC__) && (defined(__x86_64__) || defined(__ppc64__)) && !(defined(__APPLE__) && defined(__MACH__)) +# ifndef PRINTF_INT64_MODIFIER +# define PRINTF_INT64_MODIFIER "l" +# endif +# ifndef PRINTF_INT32_MODIFIER +# define PRINTF_INT32_MODIFIER "" +# endif +# else +# ifndef PRINTF_INT64_MODIFIER +# define PRINTF_INT64_MODIFIER "ll" +# endif +# ifndef PRINTF_INT32_MODIFIER +# if (UINT_MAX == UINT32_MAX) +# define PRINTF_INT32_MODIFIER "" +# else +# define PRINTF_INT32_MODIFIER "l" +# endif +# endif +# endif +# ifndef PRINTF_INT16_MODIFIER +# define PRINTF_INT16_MODIFIER "h" +# endif +# ifndef PRINTF_INTMAX_MODIFIER +# define PRINTF_INTMAX_MODIFIER PRINTF_INT64_MODIFIER +# endif +# ifndef PRINTF_INT64_HEX_WIDTH +# define PRINTF_INT64_HEX_WIDTH "16" +# endif +# ifndef PRINTF_UINT64_HEX_WIDTH +# define PRINTF_UINT64_HEX_WIDTH "16" +# endif +# ifndef PRINTF_INT32_HEX_WIDTH +# define PRINTF_INT32_HEX_WIDTH "8" +# endif +# ifndef PRINTF_UINT32_HEX_WIDTH +# define PRINTF_UINT32_HEX_WIDTH "8" +# endif +# ifndef PRINTF_INT16_HEX_WIDTH +# define PRINTF_INT16_HEX_WIDTH "4" +# endif +# ifndef PRINTF_UINT16_HEX_WIDTH +# define PRINTF_UINT16_HEX_WIDTH "4" +# endif +# ifndef PRINTF_INT8_HEX_WIDTH +# define PRINTF_INT8_HEX_WIDTH "2" +# endif +# ifndef PRINTF_UINT8_HEX_WIDTH +# define PRINTF_UINT8_HEX_WIDTH "2" +# endif +# ifndef PRINTF_INT64_DEC_WIDTH +# define PRINTF_INT64_DEC_WIDTH "19" +# endif +# ifndef PRINTF_UINT64_DEC_WIDTH +# define PRINTF_UINT64_DEC_WIDTH "20" +# endif +# ifndef PRINTF_INT32_DEC_WIDTH +# define PRINTF_INT32_DEC_WIDTH "10" +# endif +# ifndef PRINTF_UINT32_DEC_WIDTH +# define PRINTF_UINT32_DEC_WIDTH "10" +# endif +# ifndef PRINTF_INT16_DEC_WIDTH +# define PRINTF_INT16_DEC_WIDTH "5" +# endif +# ifndef PRINTF_UINT16_DEC_WIDTH +# define PRINTF_UINT16_DEC_WIDTH "5" +# endif +# ifndef PRINTF_INT8_DEC_WIDTH +# define PRINTF_INT8_DEC_WIDTH "3" +# endif +# ifndef PRINTF_UINT8_DEC_WIDTH +# define PRINTF_UINT8_DEC_WIDTH "3" +# endif +# ifndef PRINTF_INTMAX_HEX_WIDTH +# define PRINTF_INTMAX_HEX_WIDTH PRINTF_UINT64_HEX_WIDTH +# endif +# ifndef PRINTF_UINTMAX_HEX_WIDTH +# define PRINTF_UINTMAX_HEX_WIDTH PRINTF_UINT64_HEX_WIDTH +# endif +# ifndef PRINTF_INTMAX_DEC_WIDTH +# define PRINTF_INTMAX_DEC_WIDTH PRINTF_UINT64_DEC_WIDTH +# endif +# ifndef PRINTF_UINTMAX_DEC_WIDTH +# define PRINTF_UINTMAX_DEC_WIDTH PRINTF_UINT64_DEC_WIDTH +# endif + +/* + * Something really weird is going on with Open Watcom. Just pull some of + * these duplicated definitions from Open Watcom's stdint.h file for now. + */ + +# if defined (__WATCOMC__) && __WATCOMC__ >= 1250 +# if !defined (INT64_C) +# define INT64_C(x) (x + (INT64_MAX - INT64_MAX)) +# endif +# if !defined (UINT64_C) +# define UINT64_C(x) (x + (UINT64_MAX - UINT64_MAX)) +# endif +# if !defined (INT32_C) +# define INT32_C(x) (x + (INT32_MAX - INT32_MAX)) +# endif +# if !defined (UINT32_C) +# define UINT32_C(x) (x + (UINT32_MAX - UINT32_MAX)) +# endif +# if !defined (INT16_C) +# define INT16_C(x) (x) +# endif +# if !defined (UINT16_C) +# define UINT16_C(x) (x) +# endif +# if !defined (INT8_C) +# define INT8_C(x) (x) +# endif +# if !defined (UINT8_C) +# define UINT8_C(x) (x) +# endif +# if !defined (UINT64_MAX) +# define UINT64_MAX 18446744073709551615ULL +# endif +# if !defined (INT64_MAX) +# define INT64_MAX 9223372036854775807LL +# endif +# if !defined (UINT32_MAX) +# define UINT32_MAX 4294967295UL +# endif +# if !defined (INT32_MAX) +# define INT32_MAX 2147483647L +# endif +# if !defined (INTMAX_MAX) +# define INTMAX_MAX INT64_MAX +# endif +# if !defined (INTMAX_MIN) +# define INTMAX_MIN INT64_MIN +# endif +# endif +#endif + +#ifndef _PSTDINT_H_INCLUDED +#define _PSTDINT_H_INCLUDED + +#ifndef SIZE_MAX +# define SIZE_MAX (~(size_t)0) +#endif + +/* + * Deduce the type assignments from limits.h under the assumption that + * integer sizes in bits are powers of 2, and follow the ANSI + * definitions. + */ + +#ifndef UINT8_MAX +# define UINT8_MAX 0xff +#endif +#if !defined(uint8_t) && !defined(_UINT8_T) +# if (UCHAR_MAX == UINT8_MAX) || defined (S_SPLINT_S) + typedef unsigned char uint8_t; +# define UINT8_C(v) ((uint8_t) v) +# else +# error "Platform not supported" +# endif +#endif + +#ifndef INT8_MAX +# define INT8_MAX 0x7f +#endif +#ifndef INT8_MIN +# define INT8_MIN INT8_C(0x80) +#endif +#if !defined(int8_t) && !defined(_INT8_T) +# if (SCHAR_MAX == INT8_MAX) || defined (S_SPLINT_S) + typedef signed char int8_t; +# define INT8_C(v) ((int8_t) v) +# else +# error "Platform not supported" +# endif +#endif + +#ifndef UINT16_MAX +# define UINT16_MAX 0xffff +#endif +#if !defined(uint16_t) && !defined(_UINT16_T) +#if (UINT_MAX == UINT16_MAX) || defined (S_SPLINT_S) + typedef unsigned int uint16_t; +# ifndef PRINTF_INT16_MODIFIER +# define PRINTF_INT16_MODIFIER "" +# endif +# define UINT16_C(v) ((uint16_t) (v)) +#elif (USHRT_MAX == UINT16_MAX) + typedef unsigned short uint16_t; +# define UINT16_C(v) ((uint16_t) (v)) +# ifndef PRINTF_INT16_MODIFIER +# define PRINTF_INT16_MODIFIER "h" +# endif +#else +#error "Platform not supported" +#endif +#endif + +#ifndef INT16_MAX +# define INT16_MAX 0x7fff +#endif +#ifndef INT16_MIN +# define INT16_MIN INT16_C(0x8000) +#endif +#if !defined(int16_t) && !defined(_INT16_T) +#if (INT_MAX == INT16_MAX) || defined (S_SPLINT_S) + typedef signed int int16_t; +# define INT16_C(v) ((int16_t) (v)) +# ifndef PRINTF_INT16_MODIFIER +# define PRINTF_INT16_MODIFIER "" +# endif +#elif (SHRT_MAX == INT16_MAX) + typedef signed short int16_t; +# define INT16_C(v) ((int16_t) (v)) +# ifndef PRINTF_INT16_MODIFIER +# define PRINTF_INT16_MODIFIER "h" +# endif +#else +#error "Platform not supported" +#endif +#endif + +#ifndef UINT32_MAX +# define UINT32_MAX (0xffffffffUL) +#endif +#if !defined(uint32_t) && !defined(_UINT32_T) +#if (ULONG_MAX == UINT32_MAX) || defined (S_SPLINT_S) + typedef unsigned long uint32_t; +# define UINT32_C(v) v ## UL +# ifndef PRINTF_INT32_MODIFIER +# define PRINTF_INT32_MODIFIER "l" +# endif +#elif (UINT_MAX == UINT32_MAX) + typedef unsigned int uint32_t; +# ifndef PRINTF_INT32_MODIFIER +# define PRINTF_INT32_MODIFIER "" +# endif +# define UINT32_C(v) v ## U +#elif (USHRT_MAX == UINT32_MAX) + typedef unsigned short uint32_t; +# define UINT32_C(v) ((unsigned short) (v)) +# ifndef PRINTF_INT32_MODIFIER +# define PRINTF_INT32_MODIFIER "" +# endif +#else +#error "Platform not supported" +#endif +#endif + +#ifndef INT32_MAX +# define INT32_MAX (0x7fffffffL) +#endif +#ifndef INT32_MIN +# define INT32_MIN INT32_C(0x80000000) +#endif +#if !defined(int32_t) && !defined(_INT32_T) +#if (LONG_MAX == INT32_MAX) || defined (S_SPLINT_S) + typedef signed long int32_t; +# define INT32_C(v) v ## L +# ifndef PRINTF_INT32_MODIFIER +# define PRINTF_INT32_MODIFIER "l" +# endif +#elif (INT_MAX == INT32_MAX) + typedef signed int int32_t; +# define INT32_C(v) v +# ifndef PRINTF_INT32_MODIFIER +# define PRINTF_INT32_MODIFIER "" +# endif +#elif (SHRT_MAX == INT32_MAX) + typedef signed short int32_t; +# define INT32_C(v) ((short) (v)) +# ifndef PRINTF_INT32_MODIFIER +# define PRINTF_INT32_MODIFIER "" +# endif +#else +#error "Platform not supported" +#endif +#endif + +/* + * The macro stdint_int64_defined is temporarily used to record + * whether or not 64 integer support is available. It must be + * defined for any 64 integer extensions for new platforms that are + * added. + */ + +#undef stdint_int64_defined +#if (defined(__STDC__) && defined(__STDC_VERSION__)) || defined (S_SPLINT_S) +# if (__STDC__ && __STDC_VERSION__ >= 199901L) || defined (S_SPLINT_S) +# define stdint_int64_defined + typedef long long int64_t; + typedef unsigned long long uint64_t; +# define UINT64_C(v) v ## ULL +# define INT64_C(v) v ## LL +# ifndef PRINTF_INT64_MODIFIER +# define PRINTF_INT64_MODIFIER "ll" +# endif +# endif +#endif + +#if !defined (stdint_int64_defined) +# if defined(__GNUC__) +# define stdint_int64_defined + __extension__ typedef long long int64_t; + __extension__ typedef unsigned long long uint64_t; +# define UINT64_C(v) v ## ULL +# define INT64_C(v) v ## LL +# ifndef PRINTF_INT64_MODIFIER +# define PRINTF_INT64_MODIFIER "ll" +# endif +# elif defined(__MWERKS__) || defined (__SUNPRO_C) || defined (__SUNPRO_CC) || defined (__APPLE_CC__) || defined (_LONG_LONG) || defined (_CRAYC) || defined (S_SPLINT_S) +# define stdint_int64_defined + typedef long long int64_t; + typedef unsigned long long uint64_t; +# define UINT64_C(v) v ## ULL +# define INT64_C(v) v ## LL +# ifndef PRINTF_INT64_MODIFIER +# define PRINTF_INT64_MODIFIER "ll" +# endif +# elif (defined(__WATCOMC__) && defined(__WATCOM_INT64__)) || (defined(_MSC_VER) && _INTEGRAL_MAX_BITS >= 64) || (defined (__BORLANDC__) && __BORLANDC__ > 0x460) || defined (__alpha) || defined (__DECC) +# define stdint_int64_defined + typedef __int64 int64_t; + typedef unsigned __int64 uint64_t; +# define UINT64_C(v) v ## UI64 +# define INT64_C(v) v ## I64 +# ifndef PRINTF_INT64_MODIFIER +# define PRINTF_INT64_MODIFIER "I64" +# endif +# endif +#endif + +#if !defined (LONG_LONG_MAX) && defined (INT64_C) +# define LONG_LONG_MAX INT64_C (9223372036854775807) +#endif +#ifndef ULONG_LONG_MAX +# define ULONG_LONG_MAX UINT64_C (18446744073709551615) +#endif + +#if !defined (INT64_MAX) && defined (INT64_C) +# define INT64_MAX INT64_C (9223372036854775807) +#endif +#if !defined (INT64_MIN) && defined (INT64_C) +# define INT64_MIN INT64_C (-9223372036854775808) +#endif +#if !defined (UINT64_MAX) && defined (INT64_C) +# define UINT64_MAX UINT64_C (18446744073709551615) +#endif + +/* + * Width of hexadecimal for number field. + */ + +#ifndef PRINTF_INT64_HEX_WIDTH +# define PRINTF_INT64_HEX_WIDTH "16" +#endif +#ifndef PRINTF_INT32_HEX_WIDTH +# define PRINTF_INT32_HEX_WIDTH "8" +#endif +#ifndef PRINTF_INT16_HEX_WIDTH +# define PRINTF_INT16_HEX_WIDTH "4" +#endif +#ifndef PRINTF_INT8_HEX_WIDTH +# define PRINTF_INT8_HEX_WIDTH "2" +#endif +#ifndef PRINTF_INT64_DEC_WIDTH +# define PRINTF_INT64_DEC_WIDTH "19" +#endif +#ifndef PRINTF_INT32_DEC_WIDTH +# define PRINTF_INT32_DEC_WIDTH "10" +#endif +#ifndef PRINTF_INT16_DEC_WIDTH +# define PRINTF_INT16_DEC_WIDTH "5" +#endif +#ifndef PRINTF_INT8_DEC_WIDTH +# define PRINTF_INT8_DEC_WIDTH "3" +#endif +#ifndef PRINTF_UINT64_DEC_WIDTH +# define PRINTF_UINT64_DEC_WIDTH "20" +#endif +#ifndef PRINTF_UINT32_DEC_WIDTH +# define PRINTF_UINT32_DEC_WIDTH "10" +#endif +#ifndef PRINTF_UINT16_DEC_WIDTH +# define PRINTF_UINT16_DEC_WIDTH "5" +#endif +#ifndef PRINTF_UINT8_DEC_WIDTH +# define PRINTF_UINT8_DEC_WIDTH "3" +#endif + +/* + * Ok, lets not worry about 128 bit integers for now. Moore's law says + * we don't need to worry about that until about 2040 at which point + * we'll have bigger things to worry about. + */ + +#ifdef stdint_int64_defined + typedef int64_t intmax_t; + typedef uint64_t uintmax_t; +# define INTMAX_MAX INT64_MAX +# define INTMAX_MIN INT64_MIN +# define UINTMAX_MAX UINT64_MAX +# define UINTMAX_C(v) UINT64_C(v) +# define INTMAX_C(v) INT64_C(v) +# ifndef PRINTF_INTMAX_MODIFIER +# define PRINTF_INTMAX_MODIFIER PRINTF_INT64_MODIFIER +# endif +# ifndef PRINTF_INTMAX_HEX_WIDTH +# define PRINTF_INTMAX_HEX_WIDTH PRINTF_INT64_HEX_WIDTH +# endif +# ifndef PRINTF_INTMAX_DEC_WIDTH +# define PRINTF_INTMAX_DEC_WIDTH PRINTF_INT64_DEC_WIDTH +# endif +#else + typedef int32_t intmax_t; + typedef uint32_t uintmax_t; +# define INTMAX_MAX INT32_MAX +# define UINTMAX_MAX UINT32_MAX +# define UINTMAX_C(v) UINT32_C(v) +# define INTMAX_C(v) INT32_C(v) +# ifndef PRINTF_INTMAX_MODIFIER +# define PRINTF_INTMAX_MODIFIER PRINTF_INT32_MODIFIER +# endif +# ifndef PRINTF_INTMAX_HEX_WIDTH +# define PRINTF_INTMAX_HEX_WIDTH PRINTF_INT32_HEX_WIDTH +# endif +# ifndef PRINTF_INTMAX_DEC_WIDTH +# define PRINTF_INTMAX_DEC_WIDTH PRINTF_INT32_DEC_WIDTH +# endif +#endif + +/* + * Because this file currently only supports platforms which have + * precise powers of 2 as bit sizes for the default integers, the + * least definitions are all trivial. Its possible that a future + * version of this file could have different definitions. + */ + +#ifndef stdint_least_defined + typedef int8_t int_least8_t; + typedef uint8_t uint_least8_t; + typedef int16_t int_least16_t; + typedef uint16_t uint_least16_t; + typedef int32_t int_least32_t; + typedef uint32_t uint_least32_t; +# define PRINTF_LEAST32_MODIFIER PRINTF_INT32_MODIFIER +# define PRINTF_LEAST16_MODIFIER PRINTF_INT16_MODIFIER +# define UINT_LEAST8_MAX UINT8_MAX +# define INT_LEAST8_MAX INT8_MAX +# define UINT_LEAST16_MAX UINT16_MAX +# define INT_LEAST16_MAX INT16_MAX +# define UINT_LEAST32_MAX UINT32_MAX +# define INT_LEAST32_MAX INT32_MAX +# define INT_LEAST8_MIN INT8_MIN +# define INT_LEAST16_MIN INT16_MIN +# define INT_LEAST32_MIN INT32_MIN +# ifdef stdint_int64_defined + typedef int64_t int_least64_t; + typedef uint64_t uint_least64_t; +# define PRINTF_LEAST64_MODIFIER PRINTF_INT64_MODIFIER +# define UINT_LEAST64_MAX UINT64_MAX +# define INT_LEAST64_MAX INT64_MAX +# define INT_LEAST64_MIN INT64_MIN +# endif +#endif +#undef stdint_least_defined + +/* + * The ANSI C committee pretending to know or specify anything about + * performance is the epitome of misguided arrogance. The mandate of + * this file is to *ONLY* ever support that absolute minimum + * definition of the fast integer types, for compatibility purposes. + * No extensions, and no attempt to suggest what may or may not be a + * faster integer type will ever be made in this file. Developers are + * warned to stay away from these types when using this or any other + * stdint.h. + */ + +typedef int_least8_t int_fast8_t; +typedef uint_least8_t uint_fast8_t; +typedef int_least16_t int_fast16_t; +typedef uint_least16_t uint_fast16_t; +typedef int_least32_t int_fast32_t; +typedef uint_least32_t uint_fast32_t; +#define UINT_FAST8_MAX UINT_LEAST8_MAX +#define INT_FAST8_MAX INT_LEAST8_MAX +#define UINT_FAST16_MAX UINT_LEAST16_MAX +#define INT_FAST16_MAX INT_LEAST16_MAX +#define UINT_FAST32_MAX UINT_LEAST32_MAX +#define INT_FAST32_MAX INT_LEAST32_MAX +#define INT_FAST8_MIN INT_LEAST8_MIN +#define INT_FAST16_MIN INT_LEAST16_MIN +#define INT_FAST32_MIN INT_LEAST32_MIN +#ifdef stdint_int64_defined + typedef int_least64_t int_fast64_t; + typedef uint_least64_t uint_fast64_t; +# define UINT_FAST64_MAX UINT_LEAST64_MAX +# define INT_FAST64_MAX INT_LEAST64_MAX +# define INT_FAST64_MIN INT_LEAST64_MIN +#endif + +#undef stdint_int64_defined + +/* + * Whatever piecemeal, per compiler thing we can do about the wchar_t + * type limits. + */ + +#if defined(__WATCOMC__) || defined(_MSC_VER) || defined (__GNUC__) +# include +# ifndef WCHAR_MIN +# define WCHAR_MIN 0 +# endif +# ifndef WCHAR_MAX +# define WCHAR_MAX ((wchar_t)-1) +# endif +#endif + +/* + * Whatever piecemeal, per compiler/platform thing we can do about the + * (u)intptr_t types and limits. + */ + +#if (defined (_MSC_VER) && defined (_UINTPTR_T_DEFINED)) || defined (_UINTPTR_T) +# define STDINT_H_UINTPTR_T_DEFINED +#endif + +#ifndef STDINT_H_UINTPTR_T_DEFINED +# if defined (__alpha__) || defined (__ia64__) || defined (__x86_64__) || defined (_WIN64) || defined (__ppc64__) +# define stdint_intptr_bits 64 +# elif defined (__WATCOMC__) || defined (__TURBOC__) +# if defined(__TINY__) || defined(__SMALL__) || defined(__MEDIUM__) +# define stdint_intptr_bits 16 +# else +# define stdint_intptr_bits 32 +# endif +# elif defined (__i386__) || defined (_WIN32) || defined (WIN32) || defined (__ppc64__) +# define stdint_intptr_bits 32 +# elif defined (__INTEL_COMPILER) +/* TODO -- what did Intel do about x86-64? */ +# else +/* #error "This platform might not be supported yet" */ +# endif + +# ifdef stdint_intptr_bits +# define stdint_intptr_glue3_i(a,b,c) a##b##c +# define stdint_intptr_glue3(a,b,c) stdint_intptr_glue3_i(a,b,c) +# ifndef PRINTF_INTPTR_MODIFIER +# define PRINTF_INTPTR_MODIFIER stdint_intptr_glue3(PRINTF_INT,stdint_intptr_bits,_MODIFIER) +# endif +# ifndef PTRDIFF_MAX +# define PTRDIFF_MAX stdint_intptr_glue3(INT,stdint_intptr_bits,_MAX) +# endif +# ifndef PTRDIFF_MIN +# define PTRDIFF_MIN stdint_intptr_glue3(INT,stdint_intptr_bits,_MIN) +# endif +# ifndef UINTPTR_MAX +# define UINTPTR_MAX stdint_intptr_glue3(UINT,stdint_intptr_bits,_MAX) +# endif +# ifndef INTPTR_MAX +# define INTPTR_MAX stdint_intptr_glue3(INT,stdint_intptr_bits,_MAX) +# endif +# ifndef INTPTR_MIN +# define INTPTR_MIN stdint_intptr_glue3(INT,stdint_intptr_bits,_MIN) +# endif +# ifndef INTPTR_C +# define INTPTR_C(x) stdint_intptr_glue3(INT,stdint_intptr_bits,_C)(x) +# endif +# ifndef UINTPTR_C +# define UINTPTR_C(x) stdint_intptr_glue3(UINT,stdint_intptr_bits,_C)(x) +# endif + typedef stdint_intptr_glue3(uint,stdint_intptr_bits,_t) uintptr_t; + typedef stdint_intptr_glue3( int,stdint_intptr_bits,_t) intptr_t; +# else +/* TODO -- This following is likely wrong for some platforms, and does + nothing for the definition of uintptr_t. */ + typedef ptrdiff_t intptr_t; +# endif +# define STDINT_H_UINTPTR_T_DEFINED +#endif + +/* + * Assumes sig_atomic_t is signed and we have a 2s complement machine. + */ + +#ifndef SIG_ATOMIC_MAX +# define SIG_ATOMIC_MAX ((((sig_atomic_t) 1) << (sizeof (sig_atomic_t)*CHAR_BIT-1)) - 1) +#endif + +#endif + +#if defined (__TEST_PSTDINT_FOR_CORRECTNESS) + +/* + * Please compile with the maximum warning settings to make sure macros are + * not defined more than once. + */ + +#include +#include +#include + +#define glue3_aux(x,y,z) x ## y ## z +#define glue3(x,y,z) glue3_aux(x,y,z) + +#define DECLU(bits) glue3(uint,bits,_t) glue3(u,bits,) = glue3(UINT,bits,_C) (0); +#define DECLI(bits) glue3(int,bits,_t) glue3(i,bits,) = glue3(INT,bits,_C) (0); + +#define DECL(us,bits) glue3(DECL,us,) (bits) + +#define TESTUMAX(bits) glue3(u,bits,) = ~glue3(u,bits,); if (glue3(UINT,bits,_MAX) != glue3(u,bits,)) printf ("Something wrong with UINT%d_MAX\n", bits) + +#define REPORTERROR(msg) { err_n++; if (err_first <= 0) err_first = __LINE__; printf msg; } + +int main () { + int err_n = 0; + int err_first = 0; + DECL(I,8) + DECL(U,8) + DECL(I,16) + DECL(U,16) + DECL(I,32) + DECL(U,32) +#ifdef INT64_MAX + DECL(I,64) + DECL(U,64) +#endif + intmax_t imax = INTMAX_C(0); + uintmax_t umax = UINTMAX_C(0); + char str0[256], str1[256]; + + sprintf (str0, "%" PRINTF_INT32_MODIFIER "d", INT32_C(2147483647)); + if (0 != strcmp (str0, "2147483647")) REPORTERROR (("Something wrong with PRINTF_INT32_MODIFIER : %s\n", str0)); + if (atoi(PRINTF_INT32_DEC_WIDTH) != (int) strlen(str0)) REPORTERROR (("Something wrong with PRINTF_INT32_DEC_WIDTH : %s\n", PRINTF_INT32_DEC_WIDTH)); + sprintf (str0, "%" PRINTF_INT32_MODIFIER "u", UINT32_C(4294967295)); + if (0 != strcmp (str0, "4294967295")) REPORTERROR (("Something wrong with PRINTF_INT32_MODIFIER : %s\n", str0)); + if (atoi(PRINTF_UINT32_DEC_WIDTH) != (int) strlen(str0)) REPORTERROR (("Something wrong with PRINTF_UINT32_DEC_WIDTH : %s\n", PRINTF_UINT32_DEC_WIDTH)); +#ifdef INT64_MAX + sprintf (str1, "%" PRINTF_INT64_MODIFIER "d", INT64_C(9223372036854775807)); + if (0 != strcmp (str1, "9223372036854775807")) REPORTERROR (("Something wrong with PRINTF_INT32_MODIFIER : %s\n", str1)); + if (atoi(PRINTF_INT64_DEC_WIDTH) != (int) strlen(str1)) REPORTERROR (("Something wrong with PRINTF_INT64_DEC_WIDTH : %s, %d\n", PRINTF_INT64_DEC_WIDTH, (int) strlen(str1))); + sprintf (str1, "%" PRINTF_INT64_MODIFIER "u", UINT64_C(18446744073709550591)); + if (0 != strcmp (str1, "18446744073709550591")) REPORTERROR (("Something wrong with PRINTF_INT32_MODIFIER : %s\n", str1)); + if (atoi(PRINTF_UINT64_DEC_WIDTH) != (int) strlen(str1)) REPORTERROR (("Something wrong with PRINTF_UINT64_DEC_WIDTH : %s, %d\n", PRINTF_UINT64_DEC_WIDTH, (int) strlen(str1))); +#endif + + sprintf (str0, "%d %x\n", 0, ~0); + + sprintf (str1, "%d %x\n", i8, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with i8 : %s\n", str1)); + sprintf (str1, "%u %x\n", u8, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with u8 : %s\n", str1)); + sprintf (str1, "%d %x\n", i16, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with i16 : %s\n", str1)); + sprintf (str1, "%u %x\n", u16, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with u16 : %s\n", str1)); + sprintf (str1, "%" PRINTF_INT32_MODIFIER "d %x\n", i32, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with i32 : %s\n", str1)); + sprintf (str1, "%" PRINTF_INT32_MODIFIER "u %x\n", u32, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with u32 : %s\n", str1)); +#ifdef INT64_MAX + sprintf (str1, "%" PRINTF_INT64_MODIFIER "d %x\n", i64, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with i64 : %s\n", str1)); +#endif + sprintf (str1, "%" PRINTF_INTMAX_MODIFIER "d %x\n", imax, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with imax : %s\n", str1)); + sprintf (str1, "%" PRINTF_INTMAX_MODIFIER "u %x\n", umax, ~0); + if (0 != strcmp (str0, str1)) REPORTERROR (("Something wrong with umax : %s\n", str1)); + + TESTUMAX(8); + TESTUMAX(16); + TESTUMAX(32); +#ifdef INT64_MAX + TESTUMAX(64); +#endif + +#define STR(v) #v +#define Q(v) printf ("sizeof " STR(v) " = %u\n", (unsigned) sizeof (v)); + if (err_n) { + printf ("pstdint.h is not correct. Please use sizes below to correct it:\n"); + } + + Q(int) + Q(unsigned) + Q(long int) + Q(short int) + Q(int8_t) + Q(int16_t) + Q(int32_t) +#ifdef INT64_MAX + Q(int64_t) +#endif + + return EXIT_SUCCESS; +} + +#endif diff --git a/nostrdb/flatcc/portable/punaligned.h b/nostrdb/flatcc/portable/punaligned.h new file mode 100644 index 0000000000..a380edd9f8 --- /dev/null +++ b/nostrdb/flatcc/portable/punaligned.h @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2016 Mikkel Fahnøe Jørgensen, dvide.com + * + * (MIT License) + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * - The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * - The Software is provided "as is", without warranty of any kind, express or + * implied, including but not limited to the warranties of merchantability, + * fitness for a particular purpose and noninfringement. In no event shall the + * authors or copyright holders be liable for any claim, damages or other + * liability, whether in an action of contract, tort or otherwise, arising from, + * out of or in connection with the Software or the use or other dealings in the + * Software. + */ + +#ifndef PUNLIGNED_H +#define PUNLIGNED_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef PORTABLE_UNALIGNED_ACCESS + +#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) +#define PORTABLE_UNALIGNED_ACCESS 1 +#else +#define PORTABLE_UNALIGNED_ACCESS 0 +#endif + +#endif + +/* `unaligned_read_16` might not be defined if endianness was not determined. */ +#if !defined(unaligned_read_le16toh) + +#include "pendian.h" + +#ifndef UINT8_MAX +#include +#endif + +#if PORTABLE_UNALIGNED_ACCESS + +#define unaligned_read_16(p) (*(uint16_t*)(p)) +#define unaligned_read_32(p) (*(uint32_t*)(p)) +#define unaligned_read_64(p) (*(uint64_t*)(p)) + +#define unaligned_read_le16toh(p) le16toh(*(uint16_t*)(p)) +#define unaligned_read_le32toh(p) le32toh(*(uint32_t*)(p)) +#define unaligned_read_le64toh(p) le64toh(*(uint64_t*)(p)) + +#define unaligned_read_be16toh(p) be16toh(*(uint16_t*)(p)) +#define unaligned_read_be32toh(p) be32toh(*(uint32_t*)(p)) +#define unaligned_read_be64toh(p) be64toh(*(uint64_t*)(p)) + +#define unaligned_write_16(p, v) (*(uint16_t*)(p) = (uint16_t)(v)) +#define unaligned_write_32(p, v) (*(uint32_t*)(p) = (uint32_t)(v)) +#define unaligned_write_64(p, v) (*(uint64_t*)(p) = (uint64_t)(v)) + +#define unaligned_write_htole16(p, v) (*(uint16_t*)(p) = htole16(v)) +#define unaligned_write_htole32(p, v) (*(uint32_t*)(p) = htole32(v)) +#define unaligned_write_htole64(p, v) (*(uint64_t*)(p) = htole64(v)) + +#define unaligned_write_htobe16(p, v) (*(uint16_t*)(p) = htobe16(v)) +#define unaligned_write_htobe32(p, v) (*(uint32_t*)(p) = htobe32(v)) +#define unaligned_write_htobe64(p, v) (*(uint64_t*)(p) = htobe64(v)) + +#else + +#define unaligned_read_le16toh(p) ( \ + (((uint16_t)(((uint8_t *)(p))[0])) << 0) | \ + (((uint16_t)(((uint8_t *)(p))[1])) << 8)) + +#define unaligned_read_le32toh(p) ( \ + (((uint32_t)(((uint8_t *)(p))[0])) << 0) | \ + (((uint32_t)(((uint8_t *)(p))[1])) << 8) | \ + (((uint32_t)(((uint8_t *)(p))[2])) << 16) | \ + (((uint32_t)(((uint8_t *)(p))[3])) << 24)) + +#define unaligned_read_le64toh(p) ( \ + (((uint64_t)(((uint8_t *)(p))[0])) << 0) | \ + (((uint64_t)(((uint8_t *)(p))[1])) << 8) | \ + (((uint64_t)(((uint8_t *)(p))[2])) << 16) | \ + (((uint64_t)(((uint8_t *)(p))[3])) << 24) | \ + (((uint64_t)(((uint8_t *)(p))[4])) << 32) | \ + (((uint64_t)(((uint8_t *)(p))[5])) << 40) | \ + (((uint64_t)(((uint8_t *)(p))[6])) << 48) | \ + (((uint64_t)(((uint8_t *)(p))[7])) << 56)) + +#define unaligned_read_be16toh(p) ( \ + (((uint16_t)(((uint8_t *)(p))[0])) << 8) | \ + (((uint16_t)(((uint8_t *)(p))[1])) << 0)) + +#define unaligned_read_be32toh(p) ( \ + (((uint32_t)(((uint8_t *)(p))[0])) << 24) | \ + (((uint32_t)(((uint8_t *)(p))[1])) << 16) | \ + (((uint32_t)(((uint8_t *)(p))[2])) << 8) | \ + (((uint32_t)(((uint8_t *)(p))[3])) << 0)) + +#define unaligned_read_be64toh(p) ( \ + (((uint64_t)(((uint8_t *)(p))[0])) << 56) | \ + (((uint64_t)(((uint8_t *)(p))[1])) << 48) | \ + (((uint64_t)(((uint8_t *)(p))[2])) << 40) | \ + (((uint64_t)(((uint8_t *)(p))[3])) << 32) | \ + (((uint64_t)(((uint8_t *)(p))[4])) << 24) | \ + (((uint64_t)(((uint8_t *)(p))[5])) << 16) | \ + (((uint64_t)(((uint8_t *)(p))[6])) << 8) | \ + (((uint64_t)(((uint8_t *)(p))[7])) << 0)) + +#define unaligned_write_htole16(p, v) do { \ + ((uint8_t *)(p))[0] = (uint8_t)(((uint16_t)(v)) >> 0); \ + ((uint8_t *)(p))[1] = (uint8_t)(((uint16_t)(v)) >> 8); \ + } while (0) + +#define unaligned_write_htole32(p, v) do { \ + ((uint8_t *)(p))[0] = (uint8_t)(((uint32_t)(v)) >> 0); \ + ((uint8_t *)(p))[1] = (uint8_t)(((uint32_t)(v)) >> 8); \ + ((uint8_t *)(p))[2] = (uint8_t)(((uint32_t)(v)) >> 16); \ + ((uint8_t *)(p))[3] = (uint8_t)(((uint32_t)(v)) >> 24); \ + } while (0) + +#define unaligned_write_htole64(p) do { \ + ((uint8_t *)(p))[0] = (uint8_t)(((uint64_t)(v)) >> 0); \ + ((uint8_t *)(p))[1] = (uint8_t)(((uint64_t)(v)) >> 8); \ + ((uint8_t *)(p))[2] = (uint8_t)(((uint64_t)(v)) >> 16); \ + ((uint8_t *)(p))[3] = (uint8_t)(((uint64_t)(v)) >> 24); \ + ((uint8_t *)(p))[4] = (uint8_t)(((uint64_t)(v)) >> 32); \ + ((uint8_t *)(p))[5] = (uint8_t)(((uint64_t)(v)) >> 40); \ + ((uint8_t *)(p))[6] = (uint8_t)(((uint64_t)(v)) >> 48); \ + ((uint8_t *)(p))[7] = (uint8_t)(((uint64_t)(v)) >> 56); \ + } while (0) + +#define unaligned_write_htobe16(p, v) do { \ + ((uint8_t *)(p))[0] = (uint8_t)(((uint16_t)(v)) >> 8); \ + ((uint8_t *)(p))[1] = (uint8_t)(((uint16_t)(v)) >> 0); \ + } while (0) + +#define unaligned_write_htobe32(p, v) do { \ + ((uint8_t *)(p))[0] = (uint8_t)(((uint32_t)(v)) >> 24); \ + ((uint8_t *)(p))[1] = (uint8_t)(((uint32_t)(v)) >> 16); \ + ((uint8_t *)(p))[2] = (uint8_t)(((uint32_t)(v)) >> 8); \ + ((uint8_t *)(p))[3] = (uint8_t)(((uint32_t)(v)) >> 0); \ + } while (0) + +#define unaligned_write_htobe64(p) do { \ + ((uint8_t *)(p))[0] = (uint8_t)(((uint64_t)(v)) >> 56); \ + ((uint8_t *)(p))[1] = (uint8_t)(((uint64_t)(v)) >> 48); \ + ((uint8_t *)(p))[2] = (uint8_t)(((uint64_t)(v)) >> 40); \ + ((uint8_t *)(p))[3] = (uint8_t)(((uint64_t)(v)) >> 32); \ + ((uint8_t *)(p))[4] = (uint8_t)(((uint64_t)(v)) >> 24); \ + ((uint8_t *)(p))[5] = (uint8_t)(((uint64_t)(v)) >> 16); \ + ((uint8_t *)(p))[6] = (uint8_t)(((uint64_t)(v)) >> 8); \ + ((uint8_t *)(p))[7] = (uint8_t)(((uint64_t)(v)) >> 0); \ + } while (0) + +#if __LITTLE_ENDIAN__ +#define unaligned_read_16(p) unaligned_read_le16toh(p) +#define unaligned_read_32(p) unaligned_read_le32toh(p) +#define unaligned_read_64(p) unaligned_read_le64toh(p) + +#define unaligned_write_16(p) unaligned_write_htole16(p) +#define unaligned_write_32(p) unaligned_write_htole32(p) +#define unaligned_write_64(p) unaligned_write_htole64(p) +#endif + +#if __BIG_ENDIAN__ +#define unaligned_read_16(p) unaligned_read_be16toh(p) +#define unaligned_read_32(p) unaligned_read_be32toh(p) +#define unaligned_read_64(p) unaligned_read_be64toh(p) + +#define unaligned_write_16(p) unaligned_write_htobe16(p) +#define unaligned_write_32(p) unaligned_write_htobe32(p) +#define unaligned_write_64(p) unaligned_write_htobe64(p) +#endif + +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* PUNALIGNED_H */ diff --git a/nostrdb/flatcc/portable/pversion.h b/nostrdb/flatcc/portable/pversion.h new file mode 100644 index 0000000000..d434104674 --- /dev/null +++ b/nostrdb/flatcc/portable/pversion.h @@ -0,0 +1,6 @@ +#define PORTABLE_VERSION_TEXT "0.2.6-pre" +#define PORTABLE_VERSION_MAJOR 0 +#define PORTABLE_VERSION_MINOR 2 +#define PORTABLE_VERSION_PATCH 6 +/* 1 or 0 */ +#define PORTABLE_VERSION_RELEASED 0 diff --git a/nostrdb/flatcc/portable/pwarnings.h b/nostrdb/flatcc/portable/pwarnings.h new file mode 100644 index 0000000000..f420861249 --- /dev/null +++ b/nostrdb/flatcc/portable/pwarnings.h @@ -0,0 +1,52 @@ +#ifndef PWARNINGS_H +#define PWARNINGS_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * See also pdiagnostics.h headers for per file control of common + * warnings. + * + * This file is intended for global disabling of warnings that shouldn't + * be present in C11 or perhaps C99, or a generally just noise where + * recent clang / gcc compile cleanly with high warning levels. + */ + +#if defined(_MSC_VER) +/* Needed when flagging code in or out and more. */ +#pragma warning(disable: 4127) /* conditional expression is constant */ +/* happens also in MS's own headers. */ +#pragma warning(disable: 4668) /* preprocessor name not defined */ +/* MSVC does not respect double parenthesis for intent */ +#pragma warning(disable: 4706) /* assignment within conditional expression */ +/* `inline` only advisory anyway. */ +#pragma warning(disable: 4710) /* function not inlined */ +/* Well, we don't intend to add the padding manually. */ +#pragma warning(disable: 4820) /* x bytes padding added in struct */ + +/* + * Don't warn that fopen etc. are unsafe + * + * Define a compiler flag like `-D_CRT_SECURE_NO_WARNINGS` in the build. + * For some reason it doesn't work when defined here. + * + * #define _CRT_SECURE_NO_WARNINGS + */ + +/* + * Anonymous union in struct is valid in C11 and has been supported in + * GCC and Clang for a while, but it is not C99. MSVC also handles it, + * but warns. Truly portable code should perhaps not use this feature, + * but this is not the place to complain about it. + */ +#pragma warning(disable: 4201) /* nonstandard extension used: nameless struct/union */ + +#endif /* _MSV_VER */ + +#ifdef __cplusplus +} +#endif + +#endif /* PWARNINGS_H */ diff --git a/nostrdb/flatcc/reflection/README b/nostrdb/flatcc/reflection/README new file mode 100644 index 0000000000..3c7207a91f --- /dev/null +++ b/nostrdb/flatcc/reflection/README @@ -0,0 +1,19 @@ +Generated by flatcc + +Keep checked in - needed by flatcc to generate binary schema. + +NOTE TO CONTRIBUTORS: DO NOT EDIT THESE FILES BY HAND + +If you need to change anything here, it is done in the code generator, +possibly followed by running `reflection/generate_code.sh` from the +project root. But please only do this for testing do not include the +generated files in a pull request unless agreed otherwise, and if so, +do it in a separate commit. + +Normally new reflection code is generated during a release which also +updates the version number in comments and there is no reason to update +reflection on every commit unless it breaks something fundamentally. + +There is a build option `FLATCC_REFLECTION` to disable reflection which +is helpful while making changes that affect the content of these files +in a way that would prevent the flatcc compiler from building. diff --git a/nostrdb/flatcc/reflection/flatbuffers_common_builder.h b/nostrdb/flatcc/reflection/flatbuffers_common_builder.h new file mode 100644 index 0000000000..a4be1ce6e8 --- /dev/null +++ b/nostrdb/flatcc/reflection/flatbuffers_common_builder.h @@ -0,0 +1,685 @@ +#ifndef FLATBUFFERS_COMMON_BUILDER_H +#define FLATBUFFERS_COMMON_BUILDER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +/* Common FlatBuffers build functionality for C. */ + +#include "flatcc/flatcc_prologue.h" +#ifndef FLATBUILDER_H +#include "flatcc/flatcc_builder.h" +#endif +typedef flatcc_builder_t flatbuffers_builder_t; +typedef flatcc_builder_ref_t flatbuffers_ref_t; +typedef flatcc_builder_ref_t flatbuffers_vec_ref_t; +typedef flatcc_builder_union_ref_t flatbuffers_union_ref_t; +typedef flatcc_builder_union_vec_ref_t flatbuffers_union_vec_ref_t; +/* integer return code (ref and ptr always fail on 0) */ +#define flatbuffers_failed(x) ((x) < 0) +typedef flatbuffers_ref_t flatbuffers_root_t; +#define flatbuffers_root(ref) ((flatbuffers_root_t)(ref)) + +#define __flatbuffers_memoize_begin(B, src)\ +do { flatcc_builder_ref_t _ref; if ((_ref = flatcc_builder_refmap_find((B), (src)))) return _ref; } while (0) +#define __flatbuffers_memoize_end(B, src, op) do { return flatcc_builder_refmap_insert((B), (src), (op)); } while (0) +#define __flatbuffers_memoize(B, src, op) do { __flatbuffers_memoize_begin(B, src); __flatbuffers_memoize_end(B, src, op); } while (0) + +#define __flatbuffers_build_buffer(NS)\ +typedef NS ## ref_t NS ## buffer_ref_t;\ +static inline int NS ## buffer_start(NS ## builder_t *B, const NS ##fid_t fid)\ +{ return flatcc_builder_start_buffer(B, fid, 0, 0); }\ +static inline int NS ## buffer_start_with_size(NS ## builder_t *B, const NS ##fid_t fid)\ +{ return flatcc_builder_start_buffer(B, fid, 0, flatcc_builder_with_size); }\ +static inline int NS ## buffer_start_aligned(NS ## builder_t *B, NS ##fid_t fid, uint16_t block_align)\ +{ return flatcc_builder_start_buffer(B, fid, block_align, 0); }\ +static inline int NS ## buffer_start_aligned_with_size(NS ## builder_t *B, NS ##fid_t fid, uint16_t block_align)\ +{ return flatcc_builder_start_buffer(B, fid, block_align, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t NS ## buffer_end(NS ## builder_t *B, NS ## ref_t root)\ +{ return flatcc_builder_end_buffer(B, root); } + +#define __flatbuffers_build_table_root(NS, N, FID, TFID)\ +static inline int N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? -1 : N ## _start(B); }\ +static inline int N ## _start_as_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, FID) ? -1 : N ## _start(B); }\ +static inline int N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, TFID) ? -1 : N ## _start(B); }\ +static inline int N ## _start_as_typed_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, TFID) ? -1 : N ## _start(B); }\ +static inline NS ## buffer_ref_t N ## _end_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _create_as_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start(B, FID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _create_as_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start_with_size(B, FID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start(B, TFID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ if (NS ## buffer_start_with_size(B, TFID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start(B, FID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root_with_size(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start_with_size(B, FID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start(B, TFID)) return 0;return NS ## buffer_end(B, N ## _clone(B, t)); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root_with_size(NS ## builder_t *B, N ## _table_t t)\ +{ if (NS ## buffer_start_with_size(B, TFID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); } + +#define __flatbuffers_build_table_prolog(NS, N, FID, TFID)\ +__flatbuffers_build_table_vector_ops(NS, N ## _vec, N)\ +__flatbuffers_build_table_root(NS, N, FID, TFID) + +#define __flatbuffers_build_struct_root(NS, N, A, FID, TFID)\ +static inline N ## _t *N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? 0 : N ## _start(B); }\ +static inline N ## _t *N ## _start_as_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, FID) ? 0 : N ## _start(B); }\ +static inline N ## _t *N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, TFID) ? 0 : N ## _start(B); }\ +static inline N ## _t *N ## _start_as_typed_root_with_size(NS ## builder_t *B)\ +{ return NS ## buffer_start_with_size(B, TFID) ? 0 : N ## _start(B); }\ +static inline NS ## buffer_ref_t N ## _end_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end(B)); }\ +static inline NS ## buffer_ref_t N ## _end_pe_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end_pe(B)); }\ +static inline NS ## buffer_ref_t N ## _end_pe_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_end(B, N ## _end_pe(B)); }\ +static inline NS ## buffer_ref_t N ## _create_as_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, FID, 0,\ + N ## _create(B __ ## N ## _call_args), A, 0); }\ +static inline NS ## buffer_ref_t N ## _create_as_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, FID, 0,\ + N ## _create(B __ ## N ## _call_args), A, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, TFID, 0,\ + N ## _create(B __ ## N ## _call_args), A, 0); }\ +static inline NS ## buffer_ref_t N ## _create_as_typed_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ +{ return flatcc_builder_create_buffer(B, TFID, 0,\ + N ## _create(B __ ## N ## _call_args), A, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, FID, 0, N ## _clone(B, p), A, 0); }\ +static inline NS ## buffer_ref_t N ## _clone_as_root_with_size(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, FID, 0, N ## _clone(B, p), A, flatcc_builder_with_size); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, TFID, 0, N ## _clone(B, p), A, 0); }\ +static inline NS ## buffer_ref_t N ## _clone_as_typed_root_with_size(NS ## builder_t *B, N ## _struct_t p)\ +{ return flatcc_builder_create_buffer(B, TFID, 0, N ## _clone(B, p), A, flatcc_builder_with_size); } + +#define __flatbuffers_build_nested_table_root(NS, N, TN, FID, TFID)\ +static inline int N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? -1 : TN ## _start(B); }\ +static inline int N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, TFID) ? -1 : TN ## _start(B); }\ +static inline int N ## _end_as_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align ? align : 8, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _typed_nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align ? align : 8, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _clone_as_root(NS ## builder_t *B, TN ## _table_t t)\ +{ return N ## _add(B, TN ## _clone_as_root(B, t)); }\ +static inline int N ## _clone_as_typed_root(NS ## builder_t *B, TN ## _table_t t)\ +{ return N ## _add(B, TN ## _clone_as_typed_root(B, t)); } + +#define __flatbuffers_build_nested_struct_root(NS, N, TN, A, FID, TFID)\ +static inline TN ## _t *N ## _start_as_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? 0 : TN ## _start(B); }\ +static inline TN ## _t *N ## _start_as_typed_root(NS ## builder_t *B)\ +{ return NS ## buffer_start(B, FID) ? 0 : TN ## _start(B); }\ +static inline int N ## _end_as_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _end_as_typed_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ +static inline int N ## _end_pe_as_root(NS ## builder_t *B)\ +{ return N ## _add(B, NS ## buffer_end(B, TN ## _end_pe(B))); }\ +static inline int N ## _create_as_root(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ return N ## _add(B, flatcc_builder_create_buffer(B, FID, 0,\ + TN ## _create(B __ ## TN ## _call_args), A, flatcc_builder_is_nested)); }\ +static inline int N ## _create_as_typed_root(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ return N ## _add(B, flatcc_builder_create_buffer(B, TFID, 0,\ + TN ## _create(B __ ## TN ## _call_args), A, flatcc_builder_is_nested)); }\ +static inline int N ## _nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align < A ? A : align, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _typed_nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ +{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ + align < A ? A : align, FLATBUFFERS_COUNT_MAX(1))); }\ +static inline int N ## _clone_as_root(NS ## builder_t *B, TN ## _struct_t p)\ +{ return N ## _add(B, TN ## _clone_as_root(B, p)); }\ +static inline int N ## _clone_as_typed_root(NS ## builder_t *B, TN ## _struct_t p)\ +{ return N ## _add(B, TN ## _clone_as_typed_root(B, p)); } + +#define __flatbuffers_build_vector_ops(NS, V, N, TN, T)\ +static inline T *V ## _extend(NS ## builder_t *B, size_t len)\ +{ return (T *)flatcc_builder_extend_vector(B, len); }\ +static inline T *V ## _append(NS ## builder_t *B, const T *data, size_t len)\ +{ return (T *)flatcc_builder_append_vector(B, data, len); }\ +static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_vector(B, len); }\ +static inline T *V ## _edit(NS ## builder_t *B)\ +{ return (T *)flatcc_builder_vector_edit(B); }\ +static inline size_t V ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_vector_count(B); }\ +static inline T *V ## _push(NS ## builder_t *B, const T *p)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? (memcpy(_p, p, TN ## __size()), _p) : 0; }\ +static inline T *V ## _push_copy(NS ## builder_t *B, const T *p)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _copy(_p, p) : 0; }\ +static inline T *V ## _push_clone(NS ## builder_t *B, const T *p)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _copy(_p, p) : 0; }\ +static inline T *V ## _push_create(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _assign(_p __ ## TN ## _call_args) : 0; } + +#define __flatbuffers_build_vector(NS, N, T, S, A)\ +typedef NS ## ref_t N ## _vec_ref_t;\ +static inline int N ## _vec_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_vector(B, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +static inline N ## _vec_ref_t N ## _vec_end_pe(NS ## builder_t *B)\ +{ return flatcc_builder_end_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_end(NS ## builder_t *B)\ +{ if (!NS ## is_native_pe()) { size_t i, n; T *p = (T *)flatcc_builder_vector_edit(B);\ + for (i = 0, n = flatcc_builder_vector_count(B); i < n; ++i)\ + { N ## _to_pe(N ## __ptr_add(p, i)); }} return flatcc_builder_end_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_create_pe(NS ## builder_t *B, const T *data, size_t len)\ +{ return flatcc_builder_create_vector(B, data, len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +static inline N ## _vec_ref_t N ## _vec_create(NS ## builder_t *B, const T *data, size_t len)\ +{ if (!NS ## is_native_pe()) { size_t i; T *p; int ret = flatcc_builder_start_vector(B, S, A, FLATBUFFERS_COUNT_MAX(S)); if (ret) { return ret; }\ + p = (T *)flatcc_builder_extend_vector(B, len); if (!p) return 0;\ + for (i = 0; i < len; ++i) { N ## _copy_to_pe(N ## __ptr_add(p, i), N ## __const_ptr_add(data, i)); }\ + return flatcc_builder_end_vector(B); } else return flatcc_builder_create_vector(B, data, len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +static inline N ## _vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_vec_t vec)\ +{ __flatbuffers_memoize(B, vec, flatcc_builder_create_vector(B, vec, N ## _vec_len(vec), S, A, FLATBUFFERS_COUNT_MAX(S))); }\ +static inline N ## _vec_ref_t N ## _vec_slice(NS ## builder_t *B, N ##_vec_t vec, size_t index, size_t len)\ +{ size_t n = N ## _vec_len(vec); if (index >= n) index = n; n -= index; if (len > n) len = n;\ + return flatcc_builder_create_vector(B, N ## __const_ptr_add(vec, index), len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ +__flatbuffers_build_vector_ops(NS, N ## _vec, N, N, T) + +#define __flatbuffers_build_union_vector_ops(NS, V, N, TN)\ +static inline TN ## _union_ref_t *V ## _extend(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_extend_union_vector(B, len); }\ +static inline TN ## _union_ref_t *V ## _append(NS ## builder_t *B, const TN ## _union_ref_t *data, size_t len)\ +{ return flatcc_builder_append_union_vector(B, data, len); }\ +static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_union_vector(B, len); }\ +static inline TN ## _union_ref_t *V ## _edit(NS ## builder_t *B)\ +{ return (TN ## _union_ref_t *) flatcc_builder_union_vector_edit(B); }\ +static inline size_t V ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_union_vector_count(B); }\ +static inline TN ## _union_ref_t *V ## _push(NS ## builder_t *B, const TN ## _union_ref_t ref)\ +{ return flatcc_builder_union_vector_push(B, ref); }\ +static inline TN ## _union_ref_t *V ## _push_clone(NS ## builder_t *B, TN ## _union_t u)\ +{ return TN ## _vec_push(B, TN ## _clone(B, u)); } + +#define __flatbuffers_build_union_vector(NS, N)\ +static inline int N ## _vec_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_union_vector(B); }\ +static inline N ## _union_vec_ref_t N ## _vec_end(NS ## builder_t *B)\ +{ return flatcc_builder_end_union_vector(B); }\ +static inline N ## _union_vec_ref_t N ## _vec_create(NS ## builder_t *B, const N ## _union_ref_t *data, size_t len)\ +{ return flatcc_builder_create_union_vector(B, data, len); }\ +__flatbuffers_build_union_vector_ops(NS, N ## _vec, N, N)\ +/* Preserves DAG structure separately for type and value vector, so a type vector could be shared for many value vectors. */\ +static inline N ## _union_vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_union_vec_t vec)\ +{ N ## _union_vec_ref_t _uvref, _ret = { 0, 0 }; NS ## union_ref_t _uref; size_t _i, _len;\ + if (vec.type == 0) return _ret;\ + _uvref.type = flatcc_builder_refmap_find(B, vec.type); _uvref.value = flatcc_builder_refmap_find(B, vec.value);\ + _len = N ## _union_vec_len(vec); if (_uvref.type == 0) {\ + _uvref.type = flatcc_builder_refmap_insert(B, vec.type, (flatcc_builder_create_type_vector(B, vec.type, _len))); }\ + if (_uvref.type == 0) return _ret; if (_uvref.value == 0) {\ + if (flatcc_builder_start_offset_vector(B)) return _ret;\ + for (_i = 0; _i < _len; ++_i) { _uref = N ## _clone(B, N ## _union_vec_at(vec, _i));\ + if (!_uref.value || !(flatcc_builder_offset_vector_push(B, _uref.value))) return _ret; }\ + _uvref.value = flatcc_builder_refmap_insert(B, vec.value, flatcc_builder_end_offset_vector(B));\ + if (_uvref.value == 0) return _ret; } return _uvref; } + +#define __flatbuffers_build_string_vector_ops(NS, N)\ +static inline int N ## _push_start(NS ## builder_t *B)\ +{ return NS ## string_start(B); }\ +static inline NS ## string_ref_t *N ## _push_end(NS ## builder_t *B)\ +{ return NS ## string_vec_push(B, NS ## string_end(B)); }\ +static inline NS ## string_ref_t *N ## _push_create(NS ## builder_t *B, const char *s, size_t len)\ +{ return NS ## string_vec_push(B, NS ## string_create(B, s, len)); }\ +static inline NS ## string_ref_t *N ## _push_create_str(NS ## builder_t *B, const char *s)\ +{ return NS ## string_vec_push(B, NS ## string_create_str(B, s)); }\ +static inline NS ## string_ref_t *N ## _push_create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ +{ return NS ## string_vec_push(B, NS ## string_create_strn(B, s, max_len)); }\ +static inline NS ## string_ref_t *N ## _push_clone(NS ## builder_t *B, NS ## string_t string)\ +{ return NS ## string_vec_push(B, NS ## string_clone(B, string)); }\ +static inline NS ## string_ref_t *N ## _push_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ return NS ## string_vec_push(B, NS ## string_slice(B, string, index, len)); } + +#define __flatbuffers_build_table_vector_ops(NS, N, TN)\ +static inline int N ## _push_start(NS ## builder_t *B)\ +{ return TN ## _start(B); }\ +static inline TN ## _ref_t *N ## _push_end(NS ## builder_t *B)\ +{ return N ## _push(B, TN ## _end(B)); }\ +static inline TN ## _ref_t *N ## _push_create(NS ## builder_t *B __ ## TN ##_formal_args)\ +{ return N ## _push(B, TN ## _create(B __ ## TN ## _call_args)); } + +#define __flatbuffers_build_offset_vector_ops(NS, V, N, TN)\ +static inline TN ## _ref_t *V ## _extend(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_extend_offset_vector(B, len); }\ +static inline TN ## _ref_t *V ## _append(NS ## builder_t *B, const TN ## _ref_t *data, size_t len)\ +{ return flatcc_builder_append_offset_vector(B, data, len); }\ +static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_offset_vector(B, len); }\ +static inline TN ## _ref_t *V ## _edit(NS ## builder_t *B)\ +{ return (TN ## _ref_t *)flatcc_builder_offset_vector_edit(B); }\ +static inline size_t V ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_offset_vector_count(B); }\ +static inline TN ## _ref_t *V ## _push(NS ## builder_t *B, const TN ## _ref_t ref)\ +{ return ref ? flatcc_builder_offset_vector_push(B, ref) : 0; } + +#define __flatbuffers_build_offset_vector(NS, N)\ +typedef NS ## ref_t N ## _vec_ref_t;\ +static inline int N ## _vec_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_offset_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_end(NS ## builder_t *B)\ +{ return flatcc_builder_end_offset_vector(B); }\ +static inline N ## _vec_ref_t N ## _vec_create(NS ## builder_t *B, const N ## _ref_t *data, size_t len)\ +{ return flatcc_builder_create_offset_vector(B, data, len); }\ +__flatbuffers_build_offset_vector_ops(NS, N ## _vec, N, N)\ +static inline N ## _vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_vec_t vec)\ +{ int _ret; N ## _ref_t _e; size_t _i, _len; __flatbuffers_memoize_begin(B, vec);\ + _len = N ## _vec_len(vec); if (flatcc_builder_start_offset_vector(B)) return 0;\ + for (_i = 0; _i < _len; ++_i) { if (!(_e = N ## _clone(B, N ## _vec_at(vec, _i)))) return 0;\ + if (!flatcc_builder_offset_vector_push(B, _e)) return 0; }\ + __flatbuffers_memoize_end(B, vec, flatcc_builder_end_offset_vector(B)); }\ + +#define __flatbuffers_build_string_ops(NS, N)\ +static inline char *N ## _append(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_append_string(B, s, len); }\ +static inline char *N ## _append_str(NS ## builder_t *B, const char *s)\ +{ return flatcc_builder_append_string_str(B, s); }\ +static inline char *N ## _append_strn(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_append_string_strn(B, s, len); }\ +static inline size_t N ## _reserved_len(NS ## builder_t *B)\ +{ return flatcc_builder_string_len(B); }\ +static inline char *N ## _extend(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_extend_string(B, len); }\ +static inline char *N ## _edit(NS ## builder_t *B)\ +{ return flatcc_builder_string_edit(B); }\ +static inline int N ## _truncate(NS ## builder_t *B, size_t len)\ +{ return flatcc_builder_truncate_string(B, len); } + +#define __flatbuffers_build_string(NS)\ +typedef NS ## ref_t NS ## string_ref_t;\ +static inline int NS ## string_start(NS ## builder_t *B)\ +{ return flatcc_builder_start_string(B); }\ +static inline NS ## string_ref_t NS ## string_end(NS ## builder_t *B)\ +{ return flatcc_builder_end_string(B); }\ +static inline NS ## ref_t NS ## string_create(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_create_string(B, s, len); }\ +static inline NS ## ref_t NS ## string_create_str(NS ## builder_t *B, const char *s)\ +{ return flatcc_builder_create_string_str(B, s); }\ +static inline NS ## ref_t NS ## string_create_strn(NS ## builder_t *B, const char *s, size_t len)\ +{ return flatcc_builder_create_string_strn(B, s, len); }\ +static inline NS ## string_ref_t NS ## string_clone(NS ## builder_t *B, NS ## string_t string)\ +{ __flatbuffers_memoize(B, string, flatcc_builder_create_string(B, string, NS ## string_len(string))); }\ +static inline NS ## string_ref_t NS ## string_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ size_t n = NS ## string_len(string); if (index >= n) index = n; n -= index; if (len > n) len = n;\ + return flatcc_builder_create_string(B, string + index, len); }\ +__flatbuffers_build_string_ops(NS, NS ## string)\ +__flatbuffers_build_offset_vector(NS, NS ## string) + +#define __flatbuffers_copy_from_pe(P, P2, N) (*(P) = N ## _read_from_pe(P2), (P)) +#define __flatbuffers_from_pe(P, N) (*(P) = N ## _read_from_pe(P), (P)) +#define __flatbuffers_copy_to_pe(P, P2, N) (N ## _write_to_pe((P), *(P2)), (P)) +#define __flatbuffers_to_pe(P, N) (N ## _write_to_pe((P), *(P)), (P)) +#define __flatbuffers_define_fixed_array_primitives(NS, N, T)\ +static inline T *N ## _array_copy(T *p, const T *p2, size_t n)\ +{ memcpy(p, p2, n * sizeof(T)); return p; }\ +static inline T *N ## _array_copy_from_pe(T *p, const T *p2, size_t n)\ +{ size_t i; if (NS ## is_native_pe()) memcpy(p, p2, n * sizeof(T)); else\ + for (i = 0; i < n; ++i) N ## _copy_from_pe(&p[i], &p2[i]); return p; }\ +static inline T *N ## _array_copy_to_pe(T *p, const T *p2, size_t n)\ +{ size_t i; if (NS ## is_native_pe()) memcpy(p, p2, n * sizeof(T)); else\ + for (i = 0; i < n; ++i) N ## _copy_to_pe(&p[i], &p2[i]); return p; } +#define __flatbuffers_define_scalar_primitives(NS, N, T)\ +static inline T *N ## _from_pe(T *p) { return __ ## NS ## from_pe(p, N); }\ +static inline T *N ## _to_pe(T *p) { return __ ## NS ## to_pe(p, N); }\ +static inline T *N ## _copy(T *p, const T *p2) { *p = *p2; return p; }\ +static inline T *N ## _copy_from_pe(T *p, const T *p2)\ +{ return __ ## NS ## copy_from_pe(p, p2, N); }\ +static inline T *N ## _copy_to_pe(T *p, const T *p2) \ +{ return __ ## NS ## copy_to_pe(p, p2, N); }\ +static inline T *N ## _assign(T *p, const T v0) { *p = v0; return p; }\ +static inline T *N ## _assign_from_pe(T *p, T v0)\ +{ *p = N ## _read_from_pe(&v0); return p; }\ +static inline T *N ## _assign_to_pe(T *p, T v0)\ +{ N ## _write_to_pe(p, v0); return p; } +#define __flatbuffers_build_scalar(NS, N, T)\ +__ ## NS ## define_scalar_primitives(NS, N, T)\ +__ ## NS ## define_fixed_array_primitives(NS, N, T)\ +__ ## NS ## build_vector(NS, N, T, sizeof(T), sizeof(T)) +/* Depends on generated copy_to/from_pe functions, and the type. */ +#define __flatbuffers_define_struct_primitives(NS, N)\ +static inline N ## _t *N ##_to_pe(N ## _t *p)\ +{ if (!NS ## is_native_pe()) { N ## _copy_to_pe(p, p); }; return p; }\ +static inline N ## _t *N ##_from_pe(N ## _t *p)\ +{ if (!NS ## is_native_pe()) { N ## _copy_from_pe(p, p); }; return p; }\ +static inline N ## _t *N ## _clear(N ## _t *p) { return (N ## _t *)memset(p, 0, N ## __size()); } + +/* Depends on generated copy/assign_to/from_pe functions, and the type. */ +#define __flatbuffers_build_struct(NS, N, S, A, FID, TFID)\ +__ ## NS ## define_struct_primitives(NS, N)\ +typedef NS ## ref_t N ## _ref_t;\ +static inline N ## _t *N ## _start(NS ## builder_t *B)\ +{ return (N ## _t *)flatcc_builder_start_struct(B, S, A); }\ +static inline N ## _ref_t N ## _end(NS ## builder_t *B)\ +{ if (!NS ## is_native_pe()) { N ## _to_pe((N ## _t *)flatcc_builder_struct_edit(B)); }\ + return flatcc_builder_end_struct(B); }\ +static inline N ## _ref_t N ## _end_pe(NS ## builder_t *B)\ +{ return flatcc_builder_end_struct(B); }\ +static inline N ## _ref_t N ## _create(NS ## builder_t *B __ ## N ## _formal_args)\ +{ N ## _t *_p = N ## _start(B); if (!_p) return 0; N ##_assign_to_pe(_p __ ## N ## _call_args);\ + return N ## _end_pe(B); }\ +static inline N ## _ref_t N ## _clone(NS ## builder_t *B, N ## _struct_t p)\ +{ N ## _t *_p; __flatbuffers_memoize_begin(B, p); _p = N ## _start(B); if (!_p) return 0;\ + N ## _copy(_p, p); __flatbuffers_memoize_end(B, p, N ##_end_pe(B)); }\ +__flatbuffers_build_vector(NS, N, N ## _t, S, A)\ +__flatbuffers_build_struct_root(NS, N, A, FID, TFID)\ + +#define __flatbuffers_struct_clear_field(p) memset((p), 0, sizeof(*(p))) +#define __flatbuffers_build_table(NS, N, K)\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_table(B, K); }\ +static inline N ## _ref_t N ## _end(NS ## builder_t *B)\ +{ FLATCC_ASSERT(flatcc_builder_check_required(B, __ ## N ## _required,\ + sizeof(__ ## N ## _required) / sizeof(__ ## N ## _required[0]) - 1));\ + return flatcc_builder_end_table(B); }\ +__flatbuffers_build_offset_vector(NS, N) + +#define __flatbuffers_build_table_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _ref_t ref)\ +{ TN ## _ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ?\ + ((*_p = ref), 0) : -1; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return TN ## _start(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, TN ## _end(B)); }\ +static inline TN ## _ref_t N ## _create(NS ## builder_t *B __ ## TN ##_formal_args)\ +{ return N ## _add(B, TN ## _create(B __ ## TN ## _call_args)); }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _table_t p)\ +{ return N ## _add(B, TN ## _clone(B, p)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _table_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_union_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _union_ref_t uref)\ +{ NS ## ref_t *_p; TN ## _union_type_t *_pt; if (uref.type == TN ## _NONE) return 0; if (uref.value == 0) return -1;\ + if (!(_pt = (TN ## _union_type_t *)flatcc_builder_table_add(B, ID - 1, sizeof(*_pt), sizeof(*_pt)))) return -1;\ + *_pt = uref.type; if (!(_p = flatcc_builder_table_add_offset(B, ID))) return -1; *_p = uref.value; return 0; }\ +static inline int N ## _add_type(NS ## builder_t *B, TN ## _union_type_t type)\ +{ TN ## _union_type_t *_pt; if (type == TN ## _NONE) return 0; return (_pt = (TN ## _union_type_t *)flatcc_builder_table_add(B, ID - 1,\ + sizeof(*_pt), sizeof(*_pt))) ? ((*_pt = type), 0) : -1; }\ +static inline int N ## _add_value(NS ## builder_t *B, TN ## _union_ref_t uref)\ +{ NS ## ref_t *p; if (uref.type == TN ## _NONE) return 0; return (p = flatcc_builder_table_add_offset(B, ID)) ?\ + ((*p = uref.value), 0) : -1; }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _union_t p)\ +{ return N ## _add(B, TN ## _clone(B, p)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _union_t _p = N ## _union(t); return _p.type ? N ## _clone(B, _p) : 0; } + +/* M is the union value name and T is its type, i.e. the qualified name. */ +#define __flatbuffers_build_union_table_value_field(NS, N, NU, M, T)\ +static inline int N ## _ ## M ## _add(NS ## builder_t *B, T ## _ref_t ref)\ +{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ +static inline int N ## _ ## M ## _start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline int N ## _ ## M ## _end(NS ## builder_t *B)\ +{ T ## _ref_t ref = T ## _end(B);\ + return ref ? N ## _ ## M ## _add(B, ref) : -1; }\ +static inline int N ## _ ## M ## _create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ T ## _ref_t ref = T ## _create(B __ ## T ## _call_args);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ +static inline int N ## _ ## M ## _clone(NS ## builder_t *B, T ## _table_t t)\ +{ T ## _ref_t ref = T ## _clone(B, t);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; } + +/* M is the union value name and T is its type, i.e. the qualified name. */ +#define __flatbuffers_build_union_struct_value_field(NS, N, NU, M, T)\ +static inline int N ## _ ## M ## _add(NS ## builder_t *B, T ## _ref_t ref)\ +{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ +static inline T ## _t *N ## _ ## M ## _start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline int N ## _ ## M ## _end(NS ## builder_t *B)\ +{ T ## _ref_t ref = T ## _end(B);\ + return ref ? N ## _ ## M ## _add(B, ref) : -1; }\ +static inline int N ## _ ## M ## _create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ T ## _ref_t ref = T ## _create(B __ ## T ## _call_args);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ +static inline int N ## _ ## M ## _end_pe(NS ## builder_t *B)\ +{ T ## _ref_t ref = T ## _end_pe(B);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ +static inline int N ## _ ## M ## _clone(NS ## builder_t *B, T ## _struct_t p)\ +{ T ## _ref_t ref = T ## _clone(B, p);\ + return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; } +#define __flatbuffers_build_union_string_value_field(NS, N, NU, M)\ +static inline int N ## _ ## M ## _add(NS ## builder_t *B, NS ## string_ref_t ref)\ +{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ +__flatbuffers_build_string_field_ops(NS, N ## _ ## M) + +/* NS: common namespace, ID: table field id (not offset), TN: name of type T, TT: name of table type + * S: sizeof of scalar type, A: alignment of type T, default value V of type T. */ +#define __flatbuffers_build_scalar_field(ID, NS, N, TN, T, S, A, V, TT)\ +static inline int N ## _add(NS ## builder_t *B, const T v)\ +{ T *_p; if (v == V) return 0; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ + TN ## _assign_to_pe(_p, v); return 0; }\ +static inline int N ## _force_add(NS ## builder_t *B, const T v)\ +{ T *_p; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ + TN ## _assign_to_pe(_p, v); return 0; }\ +/* Clone does not skip default values and expects pe endian content. */\ +static inline int N ## _clone(NS ## builder_t *B, const T *p)\ +{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ +/* Transferring a missing field is a nop success with 0 as result. */\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ const T *_p = N ## _get_ptr(t); return _p ? N ## _clone(B, _p) : 0; } + +/* NS: common namespace, ID: table field id (not offset), TN: name of type T, TT: name of table type + * S: sizeof of scalar type, A: alignment of type T. */ +#define __flatbuffers_build_scalar_optional_field(ID, NS, N, TN, T, S, A, TT)\ +static inline int N ## _add(NS ## builder_t *B, const T v)\ +{ T *_p; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ + TN ## _assign_to_pe(_p, v); return 0; }\ +/* Clone does not skip default values and expects pe endian content. */\ +static inline int N ## _clone(NS ## builder_t *B, const T *p)\ +{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ +/* Transferring a missing field is a nop success with 0 as result. */\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ const T *_p = N ## _get_ptr(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_struct_field(ID, NS, N, TN, S, A, TT)\ +static inline TN ## _t *N ## _start(NS ## builder_t *B)\ +{ return (TN ## _t *)flatcc_builder_table_add(B, ID, S, A); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ if (!NS ## is_native_pe()) { TN ## _to_pe((TN ## _t *)flatcc_builder_table_edit(B, S)); } return 0; }\ +static inline int N ## _end_pe(NS ## builder_t *B) { return 0; }\ +static inline int N ## _create(NS ## builder_t *B __ ## TN ## _formal_args)\ +{ TN ## _t *_p = N ## _start(B); if (!_p) return -1; TN ##_assign_to_pe(_p __ ## TN ## _call_args);\ + return 0; }\ +static inline int N ## _add(NS ## builder_t *B, const TN ## _t *p)\ +{ TN ## _t *_p = N ## _start(B); if (!_p) return -1; TN ##_copy_to_pe(_p, p); return 0; }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _struct_t p)\ +{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _struct_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_vector_field(ID, NS, N, TN, T, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _vec_ref_t ref)\ +{ TN ## _vec_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return TN ## _vec_start(B); }\ +static inline int N ## _end_pe(NS ## builder_t *B)\ +{ return N ## _add(B, TN ## _vec_end_pe(B)); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, TN ## _vec_end(B)); }\ +static inline int N ## _create_pe(NS ## builder_t *B, const T *data, size_t len)\ +{ return N ## _add(B, TN ## _vec_create_pe(B, data, len)); }\ +static inline int N ## _create(NS ## builder_t *B, const T *data, size_t len)\ +{ return N ## _add(B, TN ## _vec_create(B, data, len)); }\ +static inline int N ## _slice(NS ## builder_t *B, TN ## _vec_t vec, size_t index, size_t len)\ +{ return N ## _add(B, TN ## _vec_slice(B, vec, index, len)); }\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _vec_t vec)\ +{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _vec_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; }\ +__flatbuffers_build_vector_ops(NS, N, N, TN, T)\ + +#define __flatbuffers_build_offset_vector_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _vec_ref_t ref)\ +{ TN ## _vec_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_offset_vector(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, flatcc_builder_end_offset_vector(B)); }\ +static inline int N ## _create(NS ## builder_t *B, const TN ## _ref_t *data, size_t len)\ +{ return N ## _add(B, flatcc_builder_create_offset_vector(B, data, len)); }\ +__flatbuffers_build_offset_vector_ops(NS, N, N, TN)\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _vec_t vec)\ +{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _vec_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +/* depends on N ## _add which differs for union member fields and ordinary fields */\ +#define __flatbuffers_build_string_field_ops(NS, N)\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_string(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, flatcc_builder_end_string(B)); }\ +static inline int N ## _create(NS ## builder_t *B, const char *s, size_t len)\ +{ return N ## _add(B, flatcc_builder_create_string(B, s, len)); }\ +static inline int N ## _create_str(NS ## builder_t *B, const char *s)\ +{ return N ## _add(B, flatcc_builder_create_string_str(B, s)); }\ +static inline int N ## _create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ +{ return N ## _add(B, flatcc_builder_create_string_strn(B, s, max_len)); }\ +static inline int N ## _clone(NS ## builder_t *B, NS ## string_t string)\ +{ return N ## _add(B, NS ## string_clone(B, string)); }\ +static inline int N ## _slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ return N ## _add(B, NS ## string_slice(B, string, index, len)); }\ +__flatbuffers_build_string_ops(NS, N) + +#define __flatbuffers_build_string_field(ID, NS, N, TT)\ +static inline int N ## _add(NS ## builder_t *B, NS ## string_ref_t ref)\ +{ NS ## string_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ +__flatbuffers_build_string_field_ops(NS, N)\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ NS ## string_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_table_vector_field(ID, NS, N, TN, TT)\ +__flatbuffers_build_offset_vector_field(ID, NS, N, TN, TT)\ +__flatbuffers_build_table_vector_ops(NS, N, TN) + +#define __flatbuffers_build_union_vector_field(ID, NS, N, TN, TT)\ +static inline int N ## _add(NS ## builder_t *B, TN ## _union_vec_ref_t uvref)\ +{ NS ## vec_ref_t *_p; if (!uvref.type || !uvref.value) return uvref.type == uvref.value ? 0 : -1;\ + if (!(_p = flatcc_builder_table_add_offset(B, ID - 1))) return -1; *_p = uvref.type;\ + if (!(_p = flatcc_builder_table_add_offset(B, ID))) return -1; *_p = uvref.value; return 0; }\ +static inline int N ## _start(NS ## builder_t *B)\ +{ return flatcc_builder_start_union_vector(B); }\ +static inline int N ## _end(NS ## builder_t *B)\ +{ return N ## _add(B, flatcc_builder_end_union_vector(B)); }\ +static inline int N ## _create(NS ## builder_t *B, const TN ## _union_ref_t *data, size_t len)\ +{ return N ## _add(B, flatcc_builder_create_union_vector(B, data, len)); }\ +__flatbuffers_build_union_vector_ops(NS, N, N, TN)\ +static inline int N ## _clone(NS ## builder_t *B, TN ## _union_vec_t vec)\ +{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ +static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ +{ TN ## _union_vec_t _p = N ## _union(t); return _p.type ? N ## _clone(B, _p) : 0; } + +#define __flatbuffers_build_union_table_vector_value_field(NS, N, NU, M, T)\ +static inline int N ## _ ## M ## _push_start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (T ## _end(B))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, T ## _ref_t ref)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _create(B __ ## T ## _call_args))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, T ## _table_t t)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _clone(B, t))); } + +#define __flatbuffers_build_union_struct_vector_value_field(NS, N, NU, M, T)\ +static inline T ## _t *N ## _ ## M ## _push_start(NS ## builder_t *B)\ +{ return T ## _start(B); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (T ## _end(B))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, T ## _ref_t ref)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B __ ## T ##_formal_args)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _create(B __ ## T ## _call_args))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, T ## _struct_t p)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _clone(B, p))); } + +#define __flatbuffers_build_union_string_vector_value_field(NS, N, NU, M)\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, NS ## string_ref_t ref)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ +static inline int N ## _ ## M ## _push_start(NS ## builder_t *B)\ +{ return NS ## string_start(B); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_end(B))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B, const char *s, size_t len)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create(B, s, len))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create_str(NS ## builder_t *B, const char *s)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create_str(B, s))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create_strn(B, s, max_len))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, NS ## string_t string)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_clone(B, string))); }\ +static inline NU ## _union_ref_t *N ## _ ## M ## _push_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ +{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_slice(B, string, index, len))); } + +#define __flatbuffers_build_string_vector_field(ID, NS, N, TT)\ +__flatbuffers_build_offset_vector_field(ID, NS, N, NS ## string, TT)\ +__flatbuffers_build_string_vector_ops(NS, N) + +#define __flatbuffers_char_formal_args , char v0 +#define __flatbuffers_char_call_args , v0 +#define __flatbuffers_uint8_formal_args , uint8_t v0 +#define __flatbuffers_uint8_call_args , v0 +#define __flatbuffers_int8_formal_args , int8_t v0 +#define __flatbuffers_int8_call_args , v0 +#define __flatbuffers_bool_formal_args , flatbuffers_bool_t v0 +#define __flatbuffers_bool_call_args , v0 +#define __flatbuffers_uint16_formal_args , uint16_t v0 +#define __flatbuffers_uint16_call_args , v0 +#define __flatbuffers_uint32_formal_args , uint32_t v0 +#define __flatbuffers_uint32_call_args , v0 +#define __flatbuffers_uint64_formal_args , uint64_t v0 +#define __flatbuffers_uint64_call_args , v0 +#define __flatbuffers_int16_formal_args , int16_t v0 +#define __flatbuffers_int16_call_args , v0 +#define __flatbuffers_int32_formal_args , int32_t v0 +#define __flatbuffers_int32_call_args , v0 +#define __flatbuffers_int64_formal_args , int64_t v0 +#define __flatbuffers_int64_call_args , v0 +#define __flatbuffers_float_formal_args , float v0 +#define __flatbuffers_float_call_args , v0 +#define __flatbuffers_double_formal_args , double v0 +#define __flatbuffers_double_call_args , v0 + +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_char, char) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint8, uint8_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int8, int8_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_bool, flatbuffers_bool_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint16, uint16_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint32, uint32_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint64, uint64_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int16, int16_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int32, int32_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int64, int64_t) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_float, float) +__flatbuffers_build_scalar(flatbuffers_, flatbuffers_double, double) + +__flatbuffers_build_string(flatbuffers_) + +__flatbuffers_build_buffer(flatbuffers_) +#include "flatcc/flatcc_epilogue.h" +#endif /* FLATBUFFERS_COMMON_BUILDER_H */ diff --git a/nostrdb/flatcc/reflection/flatbuffers_common_reader.h b/nostrdb/flatcc/reflection/flatbuffers_common_reader.h new file mode 100644 index 0000000000..c575308689 --- /dev/null +++ b/nostrdb/flatcc/reflection/flatbuffers_common_reader.h @@ -0,0 +1,578 @@ +#ifndef FLATBUFFERS_COMMON_READER_H +#define FLATBUFFERS_COMMON_READER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +/* Common FlatBuffers read functionality for C. */ + +#include "flatcc/flatcc_prologue.h" +#include "flatcc/flatcc_flatbuffers.h" + + +#define __flatbuffers_read_scalar_at_byteoffset(N, p, o) N ## _read_from_pe((uint8_t *)(p) + (o)) +#define __flatbuffers_read_scalar(N, p) N ## _read_from_pe(p) +#define __flatbuffers_read_vt(ID, offset, t)\ +flatbuffers_voffset_t offset = 0;\ +{ flatbuffers_voffset_t id__tmp, *vt__tmp;\ + FLATCC_ASSERT(t != 0 && "null pointer table access");\ + id__tmp = ID;\ + vt__tmp = (flatbuffers_voffset_t *)((uint8_t *)(t) -\ + __flatbuffers_soffset_read_from_pe(t));\ + if (__flatbuffers_voffset_read_from_pe(vt__tmp) >= sizeof(vt__tmp[0]) * (id__tmp + 3u)) {\ + offset = __flatbuffers_voffset_read_from_pe(vt__tmp + id__tmp + 2);\ + }\ +} +#define __flatbuffers_field_present(ID, t) { __flatbuffers_read_vt(ID, offset__tmp, t) return offset__tmp != 0; } +#define __flatbuffers_scalar_field(T, ID, t)\ +{\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + if (offset__tmp) {\ + return (const T *)((uint8_t *)(t) + offset__tmp);\ + }\ + return 0;\ +} +#define __flatbuffers_define_scalar_field(ID, N, NK, TK, T, V)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +{ __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ + return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ +}\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +{ __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ + return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ +}\ +static inline const T *N ## _ ## NK ## _get_ptr(N ## _table_t t__tmp)\ +__flatbuffers_scalar_field(T, ID, t__tmp)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp)\ +__flatbuffers_define_scan_by_scalar_field(N, NK, T) +#define __flatbuffers_define_scalar_optional_field(ID, N, NK, TK, T, V)\ +__flatbuffers_define_scalar_field(ID, N, NK, TK, T, V)\ +static inline TK ## _option_t N ## _ ## NK ## _option(N ## _table_t t__tmp)\ +{ TK ## _option_t ret; __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ + ret.is_null = offset__tmp == 0; ret.value = offset__tmp ?\ + __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ + return ret; } +#define __flatbuffers_struct_field(T, ID, t, r)\ +{\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + if (offset__tmp) {\ + return (T)((uint8_t *)(t) + offset__tmp);\ + }\ + FLATCC_ASSERT(!(r) && "required field missing");\ + return 0;\ +} +#define __flatbuffers_offset_field(T, ID, t, r, adjust)\ +{\ + flatbuffers_uoffset_t *elem__tmp;\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + if (offset__tmp) {\ + elem__tmp = (flatbuffers_uoffset_t *)((uint8_t *)(t) + offset__tmp);\ + /* Add sizeof so C api can have raw access past header field. */\ + return (T)((uint8_t *)(elem__tmp) + adjust +\ + __flatbuffers_uoffset_read_from_pe(elem__tmp));\ + }\ + FLATCC_ASSERT(!(r) && "required field missing");\ + return 0;\ +} +#define __flatbuffers_vector_field(T, ID, t, r) __flatbuffers_offset_field(T, ID, t, r, sizeof(flatbuffers_uoffset_t)) +#define __flatbuffers_table_field(T, ID, t, r) __flatbuffers_offset_field(T, ID, t, r, 0) +#define __flatbuffers_define_struct_field(ID, N, NK, T, r)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_struct_field(T, ID, t__tmp, r)\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_struct_field(T, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp) +#define __flatbuffers_define_vector_field(ID, N, NK, T, r)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(T, ID, t__tmp, r)\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(T, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp) +#define __flatbuffers_define_table_field(ID, N, NK, T, r)\ +static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_table_field(T, ID, t__tmp, r)\ +static inline T N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_table_field(T, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp) +#define __flatbuffers_define_string_field(ID, N, NK, r)\ +static inline flatbuffers_string_t N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(flatbuffers_string_t, ID, t__tmp, r)\ +static inline flatbuffers_string_t N ## _ ## NK(N ## _table_t t__tmp)\ +__flatbuffers_vector_field(flatbuffers_string_t, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__flatbuffers_field_present(ID, t__tmp)\ +__flatbuffers_define_scan_by_string_field(N, NK) +#define __flatbuffers_vec_len(vec)\ +{ return (vec) ? (size_t)__flatbuffers_uoffset_read_from_pe((flatbuffers_uoffset_t *)vec - 1) : 0; } +#define __flatbuffers_string_len(s) __flatbuffers_vec_len(s) +static inline size_t flatbuffers_vec_len(const void *vec) +__flatbuffers_vec_len(vec) +#define __flatbuffers_scalar_vec_at(N, vec, i)\ +{ FLATCC_ASSERT(flatbuffers_vec_len(vec) > (i) && "index out of range");\ + return __flatbuffers_read_scalar(N, &(vec)[i]); } +#define __flatbuffers_struct_vec_at(vec, i)\ +{ FLATCC_ASSERT(flatbuffers_vec_len(vec) > (i) && "index out of range"); return (vec) + (i); } +/* `adjust` skips past the header for string vectors. */ +#define __flatbuffers_offset_vec_at(T, vec, i, adjust)\ +{ const flatbuffers_uoffset_t *elem__tmp = (vec) + (i);\ + FLATCC_ASSERT(flatbuffers_vec_len(vec) > (i) && "index out of range");\ + return (T)((uint8_t *)(elem__tmp) + (size_t)__flatbuffers_uoffset_read_from_pe(elem__tmp) + (adjust)); } +#define __flatbuffers_define_scalar_vec_len(N)\ +static inline size_t N ## _vec_len(N ##_vec_t vec__tmp)\ +{ return flatbuffers_vec_len(vec__tmp); } +#define __flatbuffers_define_scalar_vec_at(N, T) \ +static inline T N ## _vec_at(N ## _vec_t vec__tmp, size_t i__tmp)\ +__flatbuffers_scalar_vec_at(N, vec__tmp, i__tmp) +typedef const char *flatbuffers_string_t; +static inline size_t flatbuffers_string_len(flatbuffers_string_t s) +__flatbuffers_string_len(s) +typedef const flatbuffers_uoffset_t *flatbuffers_string_vec_t; +typedef flatbuffers_uoffset_t *flatbuffers_string_mutable_vec_t; +static inline size_t flatbuffers_string_vec_len(flatbuffers_string_vec_t vec) +__flatbuffers_vec_len(vec) +static inline flatbuffers_string_t flatbuffers_string_vec_at(flatbuffers_string_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(flatbuffers_string_t, vec, i, sizeof(vec[0])) +typedef const void *flatbuffers_generic_t; +typedef void *flatbuffers_mutable_generic_t; +static inline flatbuffers_string_t flatbuffers_string_cast_from_generic(const flatbuffers_generic_t p) +{ return p ? ((const char *)p) + __flatbuffers_uoffset__size() : 0; } +typedef const flatbuffers_uoffset_t *flatbuffers_generic_vec_t; +typedef flatbuffers_uoffset_t *flatbuffers_generic_table_mutable_vec_t; +static inline size_t flatbuffers_generic_vec_len(flatbuffers_generic_vec_t vec) +__flatbuffers_vec_len(vec) +static inline flatbuffers_generic_t flatbuffers_generic_vec_at(flatbuffers_generic_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(flatbuffers_generic_t, vec, i, 0) +static inline flatbuffers_generic_t flatbuffers_generic_vec_at_as_string(flatbuffers_generic_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(flatbuffers_generic_t, vec, i, sizeof(vec[0])) +typedef struct flatbuffers_union { + flatbuffers_union_type_t type; + flatbuffers_generic_t value; +} flatbuffers_union_t; +typedef struct flatbuffers_union_vec { + const flatbuffers_union_type_t *type; + const flatbuffers_uoffset_t *value; +} flatbuffers_union_vec_t; +typedef struct flatbuffers_mutable_union { + flatbuffers_union_type_t type; + flatbuffers_mutable_generic_t value; +} flatbuffers_mutable_union_t; +typedef struct flatbuffers_mutable_union_vec { + flatbuffers_union_type_t *type; + flatbuffers_uoffset_t *value; +} flatbuffers_mutable_union_vec_t; +static inline flatbuffers_mutable_union_t flatbuffers_mutable_union_cast(flatbuffers_union_t u__tmp)\ +{ flatbuffers_mutable_union_t mu = { u__tmp.type, (flatbuffers_mutable_generic_t)u__tmp.value };\ + return mu; } +static inline flatbuffers_mutable_union_vec_t flatbuffers_mutable_union_vec_cast(flatbuffers_union_vec_t uv__tmp)\ +{ flatbuffers_mutable_union_vec_t muv =\ + { (flatbuffers_union_type_t *)uv__tmp.type, (flatbuffers_uoffset_t *)uv__tmp.value }; return muv; } +#define __flatbuffers_union_type_field(ID, t)\ +{\ + __flatbuffers_read_vt(ID, offset__tmp, t)\ + return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(__flatbuffers_utype, t, offset__tmp) : 0;\ +} +static inline flatbuffers_string_t flatbuffers_string_cast_from_union(const flatbuffers_union_t u__tmp)\ +{ return flatbuffers_string_cast_from_generic(u__tmp.value); } +#define __flatbuffers_define_union_field(NS, ID, N, NK, T, r)\ +static inline T ## _union_type_t N ## _ ## NK ## _type_get(N ## _table_t t__tmp)\ +__## NS ## union_type_field(((ID) - 1), t__tmp)\ +static inline NS ## generic_t N ## _ ## NK ## _get(N ## _table_t t__tmp)\ +__## NS ## table_field(NS ## generic_t, ID, t__tmp, r)\ +static inline T ## _union_type_t N ## _ ## NK ## _type(N ## _table_t t__tmp)\ +__## NS ## union_type_field(((ID) - 1), t__tmp)\ +static inline NS ## generic_t N ## _ ## NK(N ## _table_t t__tmp)\ +__## NS ## table_field(NS ## generic_t, ID, t__tmp, r)\ +static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ +__## NS ## field_present(ID, t__tmp)\ +static inline T ## _union_t N ## _ ## NK ## _union(N ## _table_t t__tmp)\ +{ T ## _union_t u__tmp = { 0, 0 }; u__tmp.type = N ## _ ## NK ## _type_get(t__tmp);\ + if (u__tmp.type == 0) return u__tmp; u__tmp.value = N ## _ ## NK ## _get(t__tmp); return u__tmp; }\ +static inline NS ## string_t N ## _ ## NK ## _as_string(N ## _table_t t__tmp)\ +{ return NS ## string_cast_from_generic(N ## _ ## NK ## _get(t__tmp)); }\ + +#define __flatbuffers_define_union_vector_ops(NS, T)\ +static inline size_t T ## _union_vec_len(T ## _union_vec_t uv__tmp)\ +{ return NS ## vec_len(uv__tmp.type); }\ +static inline T ## _union_t T ## _union_vec_at(T ## _union_vec_t uv__tmp, size_t i__tmp)\ +{ T ## _union_t u__tmp = { 0, 0 }; size_t n__tmp = NS ## vec_len(uv__tmp.type);\ + FLATCC_ASSERT(n__tmp > (i__tmp) && "index out of range"); u__tmp.type = uv__tmp.type[i__tmp];\ + /* Unknown type is treated as NONE for schema evolution. */\ + if (u__tmp.type == 0) return u__tmp;\ + u__tmp.value = NS ## generic_vec_at(uv__tmp.value, i__tmp); return u__tmp; }\ +static inline NS ## string_t T ## _union_vec_at_as_string(T ## _union_vec_t uv__tmp, size_t i__tmp)\ +{ return (NS ## string_t) NS ## generic_vec_at_as_string(uv__tmp.value, i__tmp); }\ + +#define __flatbuffers_define_union_vector(NS, T)\ +typedef NS ## union_vec_t T ## _union_vec_t;\ +typedef NS ## mutable_union_vec_t T ## _mutable_union_vec_t;\ +static inline T ## _mutable_union_vec_t T ## _mutable_union_vec_cast(T ## _union_vec_t u__tmp)\ +{ return NS ## mutable_union_vec_cast(u__tmp); }\ +__## NS ## define_union_vector_ops(NS, T) +#define __flatbuffers_define_union(NS, T)\ +typedef NS ## union_t T ## _union_t;\ +typedef NS ## mutable_union_t T ## _mutable_union_t;\ +static inline T ## _mutable_union_t T ## _mutable_union_cast(T ## _union_t u__tmp)\ +{ return NS ## mutable_union_cast(u__tmp); }\ +__## NS ## define_union_vector(NS, T) +#define __flatbuffers_define_union_vector_field(NS, ID, N, NK, T, r)\ +__## NS ## define_vector_field(ID - 1, N, NK ## _type, T ## _vec_t, r)\ +__## NS ## define_vector_field(ID, N, NK, flatbuffers_generic_vec_t, r)\ +static inline T ## _union_vec_t N ## _ ## NK ## _union(N ## _table_t t__tmp)\ +{ T ## _union_vec_t uv__tmp; uv__tmp.type = N ## _ ## NK ## _type_get(t__tmp);\ + uv__tmp.value = N ## _ ## NK(t__tmp);\ + FLATCC_ASSERT(NS ## vec_len(uv__tmp.type) == NS ## vec_len(uv__tmp.value)\ + && "union vector type length mismatch"); return uv__tmp; } +#include +static const size_t flatbuffers_not_found = (size_t)-1; +static const size_t flatbuffers_end = (size_t)-1; +#define __flatbuffers_identity(n) (n) +#define __flatbuffers_min(a, b) ((a) < (b) ? (a) : (b)) +/* Subtraction doesn't work for unsigned types. */ +#define __flatbuffers_scalar_cmp(x, y, n) ((x) < (y) ? -1 : (x) > (y)) +static inline int __flatbuffers_string_n_cmp(flatbuffers_string_t v, const char *s, size_t n) +{ size_t nv = flatbuffers_string_len(v); int x = strncmp(v, s, nv < n ? nv : n); + return x != 0 ? x : nv < n ? -1 : nv > n; } +/* `n` arg unused, but needed by string find macro expansion. */ +static inline int __flatbuffers_string_cmp(flatbuffers_string_t v, const char *s, size_t n) { (void)n; return strcmp(v, s); } +/* A = identity if searching scalar vectors rather than key fields. */ +/* Returns lowest matching index or not_found. */ +#define __flatbuffers_find_by_field(A, V, E, L, K, Kn, T, D)\ +{ T v__tmp; size_t a__tmp = 0, b__tmp, m__tmp; if (!(b__tmp = L(V))) { return flatbuffers_not_found; }\ + --b__tmp;\ + while (a__tmp < b__tmp) {\ + m__tmp = a__tmp + ((b__tmp - a__tmp) >> 1);\ + v__tmp = A(E(V, m__tmp));\ + if ((D(v__tmp, (K), (Kn))) < 0) {\ + a__tmp = m__tmp + 1;\ + } else {\ + b__tmp = m__tmp;\ + }\ + }\ + if (a__tmp == b__tmp) {\ + v__tmp = A(E(V, a__tmp));\ + if (D(v__tmp, (K), (Kn)) == 0) {\ + return a__tmp;\ + }\ + }\ + return flatbuffers_not_found;\ +} +#define __flatbuffers_find_by_scalar_field(A, V, E, L, K, T)\ +__flatbuffers_find_by_field(A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) +#define __flatbuffers_find_by_string_field(A, V, E, L, K)\ +__flatbuffers_find_by_field(A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) +#define __flatbuffers_find_by_string_n_field(A, V, E, L, K, Kn)\ +__flatbuffers_find_by_field(A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) +#define __flatbuffers_define_find_by_scalar_field(N, NK, TK)\ +static inline size_t N ## _vec_find_by_ ## NK(N ## _vec_t vec__tmp, TK key__tmp)\ +__flatbuffers_find_by_scalar_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, TK) +#define __flatbuffers_define_scalar_find(N, T)\ +static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_find_by_scalar_field(__flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) +#define __flatbuffers_define_find_by_string_field(N, NK) \ +/* Note: find only works on vectors sorted by this field. */\ +static inline size_t N ## _vec_find_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ +__flatbuffers_find_by_string_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_find_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_find_by_string_n_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp) +#define __flatbuffers_define_default_find_by_scalar_field(N, NK, TK)\ +static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, TK key__tmp)\ +{ return N ## _vec_find_by_ ## NK(vec__tmp, key__tmp); } +#define __flatbuffers_define_default_find_by_string_field(N, NK) \ +static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, const char *s__tmp)\ +{ return N ## _vec_find_by_ ## NK(vec__tmp, s__tmp); }\ +static inline size_t N ## _vec_find_n(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_find_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); } +/* A = identity if searching scalar vectors rather than key fields. */ +/* Returns lowest matching index or not_found. */ +#define __flatbuffers_scan_by_field(b, e, A, V, E, L, K, Kn, T, D)\ +{ T v__tmp; size_t i__tmp;\ + for (i__tmp = b; i__tmp < e; ++i__tmp) {\ + v__tmp = A(E(V, i__tmp));\ + if (D(v__tmp, (K), (Kn)) == 0) {\ + return i__tmp;\ + }\ + }\ + return flatbuffers_not_found;\ +} +#define __flatbuffers_rscan_by_field(b, e, A, V, E, L, K, Kn, T, D)\ +{ T v__tmp; size_t i__tmp = e;\ + while (i__tmp-- > b) {\ + v__tmp = A(E(V, i__tmp));\ + if (D(v__tmp, (K), (Kn)) == 0) {\ + return i__tmp;\ + }\ + }\ + return flatbuffers_not_found;\ +} +#define __flatbuffers_scan_by_scalar_field(b, e, A, V, E, L, K, T)\ +__flatbuffers_scan_by_field(b, e, A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) +#define __flatbuffers_scan_by_string_field(b, e, A, V, E, L, K)\ +__flatbuffers_scan_by_field(b, e, A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) +#define __flatbuffers_scan_by_string_n_field(b, e, A, V, E, L, K, Kn)\ +__flatbuffers_scan_by_field(b, e, A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) +#define __flatbuffers_rscan_by_scalar_field(b, e, A, V, E, L, K, T)\ +__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) +#define __flatbuffers_rscan_by_string_field(b, e, A, V, E, L, K)\ +__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) +#define __flatbuffers_rscan_by_string_n_field(b, e, A, V, E, L, K, Kn)\ +__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) +#define __flatbuffers_define_scan_by_scalar_field(N, NK, T)\ +static inline size_t N ## _vec_scan_by_ ## NK(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_scan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan_by_ ## NK(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) +#define __flatbuffers_define_scalar_scan(N, T)\ +static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(0, N ## _vec_len(vec__tmp), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_scan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(0, N ## _vec_len(vec__tmp), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ +static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ +__flatbuffers_rscan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) +#define __flatbuffers_define_scan_by_string_field(N, NK) \ +static inline size_t N ## _vec_scan_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ +__flatbuffers_scan_by_string_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_scan_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_scan_by_string_n_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ +static inline size_t N ## _vec_scan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +__flatbuffers_scan_by_string_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_scan_ex_n_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_scan_by_string_n_field(begin__tmp, __flatbuffers_min( end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ +static inline size_t N ## _vec_rscan_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ +__flatbuffers_rscan_by_string_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_rscan_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_rscan_by_string_n_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ +static inline size_t N ## _vec_rscan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +__flatbuffers_rscan_by_string_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ +static inline size_t N ## _vec_rscan_ex_n_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +__flatbuffers_rscan_by_string_n_field(begin__tmp, __flatbuffers_min( end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp) +#define __flatbuffers_define_default_scan_by_scalar_field(N, NK, TK)\ +static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, TK key__tmp)\ +{ return N ## _vec_scan_by_ ## NK(vec__tmp, key__tmp); }\ +static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, TK key__tmp)\ +{ return N ## _vec_scan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, key__tmp); }\ +static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, TK key__tmp)\ +{ return N ## _vec_rscan_by_ ## NK(vec__tmp, key__tmp); }\ +static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, TK key__tmp)\ +{ return N ## _vec_rscan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, key__tmp); } +#define __flatbuffers_define_default_scan_by_string_field(N, NK) \ +static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, const char *s__tmp)\ +{ return N ## _vec_scan_by_ ## NK(vec__tmp, s__tmp); }\ +static inline size_t N ## _vec_scan_n(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_scan_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); }\ +static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +{ return N ## _vec_scan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp); }\ +static inline size_t N ## _vec_scan_ex_n(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_scan_ex_n_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp, n__tmp); }\ +static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, const char *s__tmp)\ +{ return N ## _vec_rscan_by_ ## NK(vec__tmp, s__tmp); }\ +static inline size_t N ## _vec_rscan_n(N ## _vec_t vec__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_rscan_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); }\ +static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ +{ return N ## _vec_rscan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp); }\ +static inline size_t N ## _vec_rscan_ex_n(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, size_t n__tmp)\ +{ return N ## _vec_rscan_ex_n_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp, n__tmp); } +#define __flatbuffers_heap_sort(N, X, A, E, L, TK, TE, D, S)\ +static inline void __ ## N ## X ## __heap_sift_down(\ + N ## _mutable_vec_t vec__tmp, size_t start__tmp, size_t end__tmp)\ +{ size_t child__tmp, root__tmp; TK v1__tmp, v2__tmp, vroot__tmp;\ + root__tmp = start__tmp;\ + while ((root__tmp << 1) <= end__tmp) {\ + child__tmp = root__tmp << 1;\ + if (child__tmp < end__tmp) {\ + v1__tmp = A(E(vec__tmp, child__tmp));\ + v2__tmp = A(E(vec__tmp, child__tmp + 1));\ + if (D(v1__tmp, v2__tmp) < 0) {\ + child__tmp++;\ + }\ + }\ + vroot__tmp = A(E(vec__tmp, root__tmp));\ + v1__tmp = A(E(vec__tmp, child__tmp));\ + if (D(vroot__tmp, v1__tmp) < 0) {\ + S(vec__tmp, root__tmp, child__tmp, TE);\ + root__tmp = child__tmp;\ + } else {\ + return;\ + }\ + }\ +}\ +static inline void __ ## N ## X ## __heap_sort(N ## _mutable_vec_t vec__tmp)\ +{ size_t start__tmp, end__tmp, size__tmp;\ + size__tmp = L(vec__tmp); if (size__tmp == 0) return; end__tmp = size__tmp - 1; start__tmp = size__tmp >> 1;\ + do { __ ## N ## X ## __heap_sift_down(vec__tmp, start__tmp, end__tmp); } while (start__tmp--);\ + while (end__tmp > 0) { \ + S(vec__tmp, 0, end__tmp, TE);\ + __ ## N ## X ## __heap_sift_down(vec__tmp, 0, --end__tmp); } } +#define __flatbuffers_define_sort_by_field(N, NK, TK, TE, D, S)\ + __flatbuffers_heap_sort(N, _sort_by_ ## NK, N ## _ ## NK ## _get, N ## _vec_at, N ## _vec_len, TK, TE, D, S)\ +static inline void N ## _vec_sort_by_ ## NK(N ## _mutable_vec_t vec__tmp)\ +{ __ ## N ## _sort_by_ ## NK ## __heap_sort(vec__tmp); } +#define __flatbuffers_define_sort(N, TK, TE, D, S)\ +__flatbuffers_heap_sort(N, , __flatbuffers_identity, N ## _vec_at, N ## _vec_len, TK, TE, D, S)\ +static inline void N ## _vec_sort(N ## _mutable_vec_t vec__tmp) { __ ## N ## __heap_sort(vec__tmp); } +#define __flatbuffers_scalar_diff(x, y) ((x) < (y) ? -1 : (x) > (y)) +#define __flatbuffers_string_diff(x, y) __flatbuffers_string_n_cmp((x), (const char *)(y), flatbuffers_string_len(y)) +#define __flatbuffers_value_swap(vec, a, b, TE) { TE x__tmp = vec[b]; vec[b] = vec[a]; vec[a] = x__tmp; } +#define __flatbuffers_uoffset_swap(vec, a, b, TE)\ +{ TE ta__tmp, tb__tmp, d__tmp;\ + d__tmp = (TE)((a - b) * sizeof(vec[0]));\ + ta__tmp = __flatbuffers_uoffset_read_from_pe(vec + b) - d__tmp;\ + tb__tmp = __flatbuffers_uoffset_read_from_pe(vec + a) + d__tmp;\ + __flatbuffers_uoffset_write_to_pe(vec + a, ta__tmp);\ + __flatbuffers_uoffset_write_to_pe(vec + b, tb__tmp); } +#define __flatbuffers_scalar_swap(vec, a, b, TE) __flatbuffers_value_swap(vec, a, b, TE) +#define __flatbuffers_string_swap(vec, a, b, TE) __flatbuffers_uoffset_swap(vec, a, b, TE) +#define __flatbuffers_struct_swap(vec, a, b, TE) __flatbuffers_value_swap(vec, a, b, TE) +#define __flatbuffers_table_swap(vec, a, b, TE) __flatbuffers_uoffset_swap(vec, a, b, TE) +#define __flatbuffers_define_struct_sort_by_scalar_field(N, NK, TK, TE)\ + __flatbuffers_define_sort_by_field(N, NK, TK, TE, __flatbuffers_scalar_diff, __flatbuffers_struct_swap) +#define __flatbuffers_define_table_sort_by_scalar_field(N, NK, TK)\ + __flatbuffers_define_sort_by_field(N, NK, TK, flatbuffers_uoffset_t, __flatbuffers_scalar_diff, __flatbuffers_table_swap) +#define __flatbuffers_define_table_sort_by_string_field(N, NK)\ + __flatbuffers_define_sort_by_field(N, NK, flatbuffers_string_t, flatbuffers_uoffset_t, __flatbuffers_string_diff, __flatbuffers_table_swap) +#define __flatbuffers_define_scalar_sort(N, T) __flatbuffers_define_sort(N, T, T, __flatbuffers_scalar_diff, __flatbuffers_scalar_swap) +#define __flatbuffers_define_string_sort() __flatbuffers_define_sort(flatbuffers_string, flatbuffers_string_t, flatbuffers_uoffset_t, __flatbuffers_string_diff, __flatbuffers_string_swap) +#define __flatbuffers_sort_vector_field(N, NK, T, t)\ +{ T ## _mutable_vec_t v__tmp = (T ## _mutable_vec_t) N ## _ ## NK ## _get(t);\ + if (v__tmp) T ## _vec_sort(v__tmp); } +#define __flatbuffers_sort_table_field(N, NK, T, t)\ +{ T ## _sort((T ## _mutable_table_t)N ## _ ## NK ## _get(t)); } +#define __flatbuffers_sort_union_field(N, NK, T, t)\ +{ T ## _sort(T ## _mutable_union_cast(N ## _ ## NK ## _union(t))); } +#define __flatbuffers_sort_table_vector_field_elements(N, NK, T, t)\ +{ T ## _vec_t v__tmp = N ## _ ## NK ## _get(t); size_t i__tmp, n__tmp;\ + n__tmp = T ## _vec_len(v__tmp); for (i__tmp = 0; i__tmp < n__tmp; ++i__tmp) {\ + T ## _sort((T ## _mutable_table_t)T ## _vec_at(v__tmp, i__tmp)); }} +#define __flatbuffers_sort_union_vector_field_elements(N, NK, T, t)\ +{ T ## _union_vec_t v__tmp = N ## _ ## NK ## _union(t); size_t i__tmp, n__tmp;\ + n__tmp = T ## _union_vec_len(v__tmp); for (i__tmp = 0; i__tmp < n__tmp; ++i__tmp) {\ + T ## _sort(T ## _mutable_union_cast(T ## _union_vec_at(v__tmp, i__tmp))); }} +#define __flatbuffers_define_scalar_vector(N, T)\ +typedef const T *N ## _vec_t;\ +typedef T *N ## _mutable_vec_t;\ +__flatbuffers_define_scalar_vec_len(N)\ +__flatbuffers_define_scalar_vec_at(N, T)\ +__flatbuffers_define_scalar_find(N, T)\ +__flatbuffers_define_scalar_scan(N, T)\ +__flatbuffers_define_scalar_sort(N, T) + +#define __flatbuffers_define_integer_type(N, T, W)\ +__flatcc_define_integer_accessors(N, T, W, flatbuffers_endian)\ +__flatbuffers_define_scalar_vector(N, T) +__flatbuffers_define_scalar_vector(flatbuffers_bool, flatbuffers_bool_t) +__flatbuffers_define_scalar_vector(flatbuffers_char, char) +__flatbuffers_define_scalar_vector(flatbuffers_uint8, uint8_t) +__flatbuffers_define_scalar_vector(flatbuffers_int8, int8_t) +__flatbuffers_define_scalar_vector(flatbuffers_uint16, uint16_t) +__flatbuffers_define_scalar_vector(flatbuffers_int16, int16_t) +__flatbuffers_define_scalar_vector(flatbuffers_uint32, uint32_t) +__flatbuffers_define_scalar_vector(flatbuffers_int32, int32_t) +__flatbuffers_define_scalar_vector(flatbuffers_uint64, uint64_t) +__flatbuffers_define_scalar_vector(flatbuffers_int64, int64_t) +__flatbuffers_define_scalar_vector(flatbuffers_float, float) +__flatbuffers_define_scalar_vector(flatbuffers_double, double) +__flatbuffers_define_scalar_vector(flatbuffers_union_type, flatbuffers_union_type_t) +static inline size_t flatbuffers_string_vec_find(flatbuffers_string_vec_t vec, const char *s) +__flatbuffers_find_by_string_field(__flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_find_n(flatbuffers_string_vec_t vec, const char *s, size_t n) +__flatbuffers_find_by_string_n_field(__flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_scan(flatbuffers_string_vec_t vec, const char *s) +__flatbuffers_scan_by_string_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_scan_n(flatbuffers_string_vec_t vec, const char *s, size_t n) +__flatbuffers_scan_by_string_n_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_scan_ex(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s) +__flatbuffers_scan_by_string_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_scan_ex_n(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s, size_t n) +__flatbuffers_scan_by_string_n_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_rscan(flatbuffers_string_vec_t vec, const char *s) +__flatbuffers_rscan_by_string_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_rscan_n(flatbuffers_string_vec_t vec, const char *s, size_t n) +__flatbuffers_rscan_by_string_n_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +static inline size_t flatbuffers_string_vec_rscan_ex(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s) +__flatbuffers_rscan_by_string_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) +static inline size_t flatbuffers_string_vec_rscan_ex_n(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s, size_t n) +__flatbuffers_rscan_by_string_n_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) +__flatbuffers_define_string_sort() +#define __flatbuffers_define_struct_scalar_fixed_array_field(N, NK, TK, T, L)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp, size_t i__tmp)\ +{ if (!t__tmp || i__tmp >= L) return 0;\ + return __flatbuffers_read_scalar(TK, &(t__tmp->NK[i__tmp])); }\ +static inline const T *N ## _ ## NK ## _get_ptr(N ## _struct_t t__tmp)\ +{ return t__tmp ? t__tmp->NK : 0; }\ +static inline size_t N ## _ ## NK ## _get_len(void) { return L; }\ +static inline T N ## _ ## NK (N ## _struct_t t__tmp, size_t i__tmp)\ +{ return N ## _ ## NK ## _get(t__tmp, i__tmp); } +#define __flatbuffers_define_struct_struct_fixed_array_field(N, NK, T, L)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp, size_t i__tmp)\ +{ if (!t__tmp || i__tmp >= L) return 0; return t__tmp->NK + i__tmp; }static inline T N ## _ ## NK ## _get_ptr(N ## _struct_t t__tmp)\ +{ return t__tmp ? t__tmp->NK : 0; }\ +static inline size_t N ## _ ## NK ## _get_len(void) { return L; }\ +static inline T N ## _ ## NK(N ## _struct_t t__tmp, size_t i__tmp)\ +{ if (!t__tmp || i__tmp >= L) return 0; return t__tmp->NK + i__tmp; } +#define __flatbuffers_define_struct_scalar_field(N, NK, TK, T)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp)\ +{ return t__tmp ? __flatbuffers_read_scalar(TK, &(t__tmp->NK)) : 0; }\ +static inline const T *N ## _ ## NK ## _get_ptr(N ## _struct_t t__tmp)\ +{ return t__tmp ? &(t__tmp->NK) : 0; }\ +static inline T N ## _ ## NK (N ## _struct_t t__tmp)\ +{ return t__tmp ? __flatbuffers_read_scalar(TK, &(t__tmp->NK)) : 0; }\ +__flatbuffers_define_scan_by_scalar_field(N, NK, T) +#define __flatbuffers_define_struct_struct_field(N, NK, T)\ +static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; }\ +static inline T N ## _ ## NK (N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; } +/* If fid is null, the function returns true without testing as buffer is not expected to have any id. */ +static inline int flatbuffers_has_identifier(const void *buffer, const char *fid) +{ flatbuffers_thash_t id, id2 = 0; if (fid == 0) { return 1; }; + id2 = flatbuffers_type_hash_from_string(fid); + id = __flatbuffers_thash_read_from_pe(((flatbuffers_uoffset_t *)buffer) + 1); + return id2 == 0 || id == id2; } +static inline int flatbuffers_has_type_hash(const void *buffer, flatbuffers_thash_t thash) +{ return thash == 0 || (__flatbuffers_thash_read_from_pe((flatbuffers_uoffset_t *)buffer + 1) == thash); } + +static inline flatbuffers_thash_t flatbuffers_get_type_hash(const void *buffer) +{ return __flatbuffers_thash_read_from_pe((flatbuffers_uoffset_t *)buffer + 1); } + +#define flatbuffers_verify_endian() flatbuffers_has_identifier("\x00\x00\x00\x00" "1234", "1234") +static inline void *flatbuffers_read_size_prefix(void *b, size_t *size_out) +{ if (size_out) { *size_out = (size_t)__flatbuffers_uoffset_read_from_pe(b); } + return (uint8_t *)b + sizeof(flatbuffers_uoffset_t); } +/* Null file identifier accepts anything, otherwise fid should be 4 characters. */ +#define __flatbuffers_read_root(T, K, buffer, fid)\ + ((!buffer || !flatbuffers_has_identifier(buffer, fid)) ? 0 :\ + ((T ## _ ## K ## t)(((uint8_t *)buffer) +\ + __flatbuffers_uoffset_read_from_pe(buffer)))) +#define __flatbuffers_read_typed_root(T, K, buffer, thash)\ + ((!buffer || !flatbuffers_has_type_hash(buffer, thash)) ? 0 :\ + ((T ## _ ## K ## t)(((uint8_t *)buffer) +\ + __flatbuffers_uoffset_read_from_pe(buffer)))) +#define __flatbuffers_nested_buffer_as_root(C, N, T, K)\ +static inline T ## _ ## K ## t C ## _ ## N ## _as_root_with_identifier(C ## _ ## table_t t__tmp, const char *fid__tmp)\ +{ const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, fid__tmp); }\ +static inline T ## _ ## K ## t C ## _ ## N ## _as_typed_root(C ## _ ## table_t t__tmp)\ +{ const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, C ## _ ## type_identifier); }\ +static inline T ## _ ## K ## t C ## _ ## N ## _as_root(C ## _ ## table_t t__tmp)\ +{ const char *fid__tmp = T ## _file_identifier;\ + const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, fid__tmp); } +#define __flatbuffers_buffer_as_root(N, K)\ +static inline N ## _ ## K ## t N ## _as_root_with_identifier(const void *buffer__tmp, const char *fid__tmp)\ +{ return __flatbuffers_read_root(N, K, buffer__tmp, fid__tmp); }\ +static inline N ## _ ## K ## t N ## _as_root_with_type_hash(const void *buffer__tmp, flatbuffers_thash_t thash__tmp)\ +{ return __flatbuffers_read_typed_root(N, K, buffer__tmp, thash__tmp); }\ +static inline N ## _ ## K ## t N ## _as_root(const void *buffer__tmp)\ +{ const char *fid__tmp = N ## _file_identifier;\ + return __flatbuffers_read_root(N, K, buffer__tmp, fid__tmp); }\ +static inline N ## _ ## K ## t N ## _as_typed_root(const void *buffer__tmp)\ +{ return __flatbuffers_read_typed_root(N, K, buffer__tmp, N ## _type_hash); } +#define __flatbuffers_struct_as_root(N) __flatbuffers_buffer_as_root(N, struct_) +#define __flatbuffers_table_as_root(N) __flatbuffers_buffer_as_root(N, table_) + +#include "flatcc/flatcc_epilogue.h" +#endif /* FLATBUFFERS_COMMON_H */ diff --git a/nostrdb/flatcc/reflection/reflection_builder.h b/nostrdb/flatcc/reflection/reflection_builder.h new file mode 100644 index 0000000000..65aef73e55 --- /dev/null +++ b/nostrdb/flatcc/reflection/reflection_builder.h @@ -0,0 +1,457 @@ +#ifndef REFLECTION_BUILDER_H +#define REFLECTION_BUILDER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef REFLECTION_READER_H +#include "reflection_reader.h" +#endif +#ifndef FLATBUFFERS_COMMON_BUILDER_H +#include "flatbuffers_common_builder.h" +#endif +#include "flatcc/flatcc_prologue.h" +#undef flatbuffers_identifier +#define flatbuffers_identifier "BFBS" +#undef flatbuffers_extension +#define flatbuffers_extension "bfbs" + +#define __reflection_BaseType_formal_args , reflection_BaseType_enum_t v0 +#define __reflection_BaseType_call_args , v0 +__flatbuffers_build_scalar(flatbuffers_, reflection_BaseType, reflection_BaseType_enum_t) + +static const flatbuffers_voffset_t __reflection_Type_required[] = { 0 }; +typedef flatbuffers_ref_t reflection_Type_ref_t; +static reflection_Type_ref_t reflection_Type_clone(flatbuffers_builder_t *B, reflection_Type_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_Type, 4) + +static const flatbuffers_voffset_t __reflection_KeyValue_required[] = { 0, 0 }; +typedef flatbuffers_ref_t reflection_KeyValue_ref_t; +static reflection_KeyValue_ref_t reflection_KeyValue_clone(flatbuffers_builder_t *B, reflection_KeyValue_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_KeyValue, 2) + +static const flatbuffers_voffset_t __reflection_EnumVal_required[] = { 0, 0 }; +typedef flatbuffers_ref_t reflection_EnumVal_ref_t; +static reflection_EnumVal_ref_t reflection_EnumVal_clone(flatbuffers_builder_t *B, reflection_EnumVal_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_EnumVal, 5) + +static const flatbuffers_voffset_t __reflection_Enum_required[] = { 0, 1, 3, 0 }; +typedef flatbuffers_ref_t reflection_Enum_ref_t; +static reflection_Enum_ref_t reflection_Enum_clone(flatbuffers_builder_t *B, reflection_Enum_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_Enum, 6) + +static const flatbuffers_voffset_t __reflection_Field_required[] = { 0, 1, 0 }; +typedef flatbuffers_ref_t reflection_Field_ref_t; +static reflection_Field_ref_t reflection_Field_clone(flatbuffers_builder_t *B, reflection_Field_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_Field, 12) + +static const flatbuffers_voffset_t __reflection_Object_required[] = { 0, 1, 0 }; +typedef flatbuffers_ref_t reflection_Object_ref_t; +static reflection_Object_ref_t reflection_Object_clone(flatbuffers_builder_t *B, reflection_Object_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_Object, 7) + +static const flatbuffers_voffset_t __reflection_RPCCall_required[] = { 0, 1, 2, 0 }; +typedef flatbuffers_ref_t reflection_RPCCall_ref_t; +static reflection_RPCCall_ref_t reflection_RPCCall_clone(flatbuffers_builder_t *B, reflection_RPCCall_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_RPCCall, 5) + +static const flatbuffers_voffset_t __reflection_Service_required[] = { 0, 0 }; +typedef flatbuffers_ref_t reflection_Service_ref_t; +static reflection_Service_ref_t reflection_Service_clone(flatbuffers_builder_t *B, reflection_Service_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_Service, 4) + +static const flatbuffers_voffset_t __reflection_Schema_required[] = { 0, 1, 0 }; +typedef flatbuffers_ref_t reflection_Schema_ref_t; +static reflection_Schema_ref_t reflection_Schema_clone(flatbuffers_builder_t *B, reflection_Schema_table_t t); +__flatbuffers_build_table(flatbuffers_, reflection_Schema, 6) + +#define __reflection_Type_formal_args , reflection_BaseType_enum_t v0, reflection_BaseType_enum_t v1, int32_t v2, uint16_t v3 +#define __reflection_Type_call_args , v0, v1, v2, v3 +static inline reflection_Type_ref_t reflection_Type_create(flatbuffers_builder_t *B __reflection_Type_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_Type, reflection_Type_file_identifier, reflection_Type_type_identifier) + +#define __reflection_KeyValue_formal_args , flatbuffers_string_ref_t v0, flatbuffers_string_ref_t v1 +#define __reflection_KeyValue_call_args , v0, v1 +static inline reflection_KeyValue_ref_t reflection_KeyValue_create(flatbuffers_builder_t *B __reflection_KeyValue_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_KeyValue, reflection_KeyValue_file_identifier, reflection_KeyValue_type_identifier) + +#define __reflection_EnumVal_formal_args ,\ + flatbuffers_string_ref_t v0, int64_t v1, reflection_Object_ref_t v2, reflection_Type_ref_t v3, flatbuffers_string_vec_ref_t v4 +#define __reflection_EnumVal_call_args ,\ + v0, v1, v2, v3, v4 +static inline reflection_EnumVal_ref_t reflection_EnumVal_create(flatbuffers_builder_t *B __reflection_EnumVal_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_EnumVal, reflection_EnumVal_file_identifier, reflection_EnumVal_type_identifier) + +#define __reflection_Enum_formal_args ,\ + flatbuffers_string_ref_t v0, reflection_EnumVal_vec_ref_t v1, flatbuffers_bool_t v2, reflection_Type_ref_t v3, reflection_KeyValue_vec_ref_t v4, flatbuffers_string_vec_ref_t v5 +#define __reflection_Enum_call_args ,\ + v0, v1, v2, v3, v4, v5 +static inline reflection_Enum_ref_t reflection_Enum_create(flatbuffers_builder_t *B __reflection_Enum_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_Enum, reflection_Enum_file_identifier, reflection_Enum_type_identifier) + +#define __reflection_Field_formal_args ,\ + flatbuffers_string_ref_t v0, reflection_Type_ref_t v1, uint16_t v2, uint16_t v3,\ + int64_t v4, double v5, flatbuffers_bool_t v6, flatbuffers_bool_t v7,\ + flatbuffers_bool_t v8, reflection_KeyValue_vec_ref_t v9, flatbuffers_string_vec_ref_t v10, flatbuffers_bool_t v11 +#define __reflection_Field_call_args ,\ + v0, v1, v2, v3,\ + v4, v5, v6, v7,\ + v8, v9, v10, v11 +static inline reflection_Field_ref_t reflection_Field_create(flatbuffers_builder_t *B __reflection_Field_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_Field, reflection_Field_file_identifier, reflection_Field_type_identifier) + +#define __reflection_Object_formal_args ,\ + flatbuffers_string_ref_t v0, reflection_Field_vec_ref_t v1, flatbuffers_bool_t v2, int32_t v3,\ + int32_t v4, reflection_KeyValue_vec_ref_t v5, flatbuffers_string_vec_ref_t v6 +#define __reflection_Object_call_args ,\ + v0, v1, v2, v3,\ + v4, v5, v6 +static inline reflection_Object_ref_t reflection_Object_create(flatbuffers_builder_t *B __reflection_Object_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_Object, reflection_Object_file_identifier, reflection_Object_type_identifier) + +#define __reflection_RPCCall_formal_args ,\ + flatbuffers_string_ref_t v0, reflection_Object_ref_t v1, reflection_Object_ref_t v2, reflection_KeyValue_vec_ref_t v3, flatbuffers_string_vec_ref_t v4 +#define __reflection_RPCCall_call_args ,\ + v0, v1, v2, v3, v4 +static inline reflection_RPCCall_ref_t reflection_RPCCall_create(flatbuffers_builder_t *B __reflection_RPCCall_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_RPCCall, reflection_RPCCall_file_identifier, reflection_RPCCall_type_identifier) + +#define __reflection_Service_formal_args , flatbuffers_string_ref_t v0, reflection_RPCCall_vec_ref_t v1, reflection_KeyValue_vec_ref_t v2, flatbuffers_string_vec_ref_t v3 +#define __reflection_Service_call_args , v0, v1, v2, v3 +static inline reflection_Service_ref_t reflection_Service_create(flatbuffers_builder_t *B __reflection_Service_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_Service, reflection_Service_file_identifier, reflection_Service_type_identifier) + +#define __reflection_Schema_formal_args ,\ + reflection_Object_vec_ref_t v0, reflection_Enum_vec_ref_t v1, flatbuffers_string_ref_t v2, flatbuffers_string_ref_t v3, reflection_Object_ref_t v4, reflection_Service_vec_ref_t v5 +#define __reflection_Schema_call_args ,\ + v0, v1, v2, v3, v4, v5 +static inline reflection_Schema_ref_t reflection_Schema_create(flatbuffers_builder_t *B __reflection_Schema_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, reflection_Schema, reflection_Schema_file_identifier, reflection_Schema_type_identifier) + +__flatbuffers_build_scalar_field(0, flatbuffers_, reflection_Type_base_type, reflection_BaseType, reflection_BaseType_enum_t, 1, 1, INT8_C(0), reflection_Type) +__flatbuffers_build_scalar_field(1, flatbuffers_, reflection_Type_element, reflection_BaseType, reflection_BaseType_enum_t, 1, 1, INT8_C(0), reflection_Type) +__flatbuffers_build_scalar_field(2, flatbuffers_, reflection_Type_index, flatbuffers_int32, int32_t, 4, 4, INT32_C(-1), reflection_Type) +__flatbuffers_build_scalar_field(3, flatbuffers_, reflection_Type_fixed_length, flatbuffers_uint16, uint16_t, 2, 2, UINT16_C(0), reflection_Type) + +static inline reflection_Type_ref_t reflection_Type_create(flatbuffers_builder_t *B __reflection_Type_formal_args) +{ + if (reflection_Type_start(B) + || reflection_Type_index_add(B, v2) + || reflection_Type_fixed_length_add(B, v3) + || reflection_Type_base_type_add(B, v0) + || reflection_Type_element_add(B, v1)) { + return 0; + } + return reflection_Type_end(B); +} + +static reflection_Type_ref_t reflection_Type_clone(flatbuffers_builder_t *B, reflection_Type_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_Type_start(B) + || reflection_Type_index_pick(B, t) + || reflection_Type_fixed_length_pick(B, t) + || reflection_Type_base_type_pick(B, t) + || reflection_Type_element_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_Type_end(B)); +} + +__flatbuffers_build_string_field(0, flatbuffers_, reflection_KeyValue_key, reflection_KeyValue) +__flatbuffers_build_string_field(1, flatbuffers_, reflection_KeyValue_value, reflection_KeyValue) + +static inline reflection_KeyValue_ref_t reflection_KeyValue_create(flatbuffers_builder_t *B __reflection_KeyValue_formal_args) +{ + if (reflection_KeyValue_start(B) + || reflection_KeyValue_key_add(B, v0) + || reflection_KeyValue_value_add(B, v1)) { + return 0; + } + return reflection_KeyValue_end(B); +} + +static reflection_KeyValue_ref_t reflection_KeyValue_clone(flatbuffers_builder_t *B, reflection_KeyValue_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_KeyValue_start(B) + || reflection_KeyValue_key_pick(B, t) + || reflection_KeyValue_value_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_KeyValue_end(B)); +} + +__flatbuffers_build_string_field(0, flatbuffers_, reflection_EnumVal_name, reflection_EnumVal) +__flatbuffers_build_scalar_field(1, flatbuffers_, reflection_EnumVal_value, flatbuffers_int64, int64_t, 8, 8, INT64_C(0), reflection_EnumVal) +__flatbuffers_build_table_field(2, flatbuffers_, reflection_EnumVal_object, reflection_Object, reflection_EnumVal) +__flatbuffers_build_table_field(3, flatbuffers_, reflection_EnumVal_union_type, reflection_Type, reflection_EnumVal) +__flatbuffers_build_string_vector_field(4, flatbuffers_, reflection_EnumVal_documentation, reflection_EnumVal) + +static inline reflection_EnumVal_ref_t reflection_EnumVal_create(flatbuffers_builder_t *B __reflection_EnumVal_formal_args) +{ + if (reflection_EnumVal_start(B) + || reflection_EnumVal_value_add(B, v1) + || reflection_EnumVal_name_add(B, v0) + || reflection_EnumVal_object_add(B, v2) + || reflection_EnumVal_union_type_add(B, v3) + || reflection_EnumVal_documentation_add(B, v4)) { + return 0; + } + return reflection_EnumVal_end(B); +} + +static reflection_EnumVal_ref_t reflection_EnumVal_clone(flatbuffers_builder_t *B, reflection_EnumVal_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_EnumVal_start(B) + || reflection_EnumVal_value_pick(B, t) + || reflection_EnumVal_name_pick(B, t) + || reflection_EnumVal_object_pick(B, t) + || reflection_EnumVal_union_type_pick(B, t) + || reflection_EnumVal_documentation_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_EnumVal_end(B)); +} + +__flatbuffers_build_string_field(0, flatbuffers_, reflection_Enum_name, reflection_Enum) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(1, flatbuffers_, reflection_Enum_values, reflection_EnumVal, reflection_Enum) +__flatbuffers_build_scalar_field(2, flatbuffers_, reflection_Enum_is_union, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), reflection_Enum) +__flatbuffers_build_table_field(3, flatbuffers_, reflection_Enum_underlying_type, reflection_Type, reflection_Enum) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(4, flatbuffers_, reflection_Enum_attributes, reflection_KeyValue, reflection_Enum) +__flatbuffers_build_string_vector_field(5, flatbuffers_, reflection_Enum_documentation, reflection_Enum) + +static inline reflection_Enum_ref_t reflection_Enum_create(flatbuffers_builder_t *B __reflection_Enum_formal_args) +{ + if (reflection_Enum_start(B) + || reflection_Enum_name_add(B, v0) + || reflection_Enum_values_add(B, v1) + || reflection_Enum_underlying_type_add(B, v3) + || reflection_Enum_attributes_add(B, v4) + || reflection_Enum_documentation_add(B, v5) + || reflection_Enum_is_union_add(B, v2)) { + return 0; + } + return reflection_Enum_end(B); +} + +static reflection_Enum_ref_t reflection_Enum_clone(flatbuffers_builder_t *B, reflection_Enum_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_Enum_start(B) + || reflection_Enum_name_pick(B, t) + || reflection_Enum_values_pick(B, t) + || reflection_Enum_underlying_type_pick(B, t) + || reflection_Enum_attributes_pick(B, t) + || reflection_Enum_documentation_pick(B, t) + || reflection_Enum_is_union_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_Enum_end(B)); +} + +__flatbuffers_build_string_field(0, flatbuffers_, reflection_Field_name, reflection_Field) +__flatbuffers_build_table_field(1, flatbuffers_, reflection_Field_type, reflection_Type, reflection_Field) +__flatbuffers_build_scalar_field(2, flatbuffers_, reflection_Field_id, flatbuffers_uint16, uint16_t, 2, 2, UINT16_C(0), reflection_Field) +__flatbuffers_build_scalar_field(3, flatbuffers_, reflection_Field_offset, flatbuffers_uint16, uint16_t, 2, 2, UINT16_C(0), reflection_Field) +__flatbuffers_build_scalar_field(4, flatbuffers_, reflection_Field_default_integer, flatbuffers_int64, int64_t, 8, 8, INT64_C(0), reflection_Field) +__flatbuffers_build_scalar_field(5, flatbuffers_, reflection_Field_default_real, flatbuffers_double, double, 8, 8, 0.0000000000000000, reflection_Field) +__flatbuffers_build_scalar_field(6, flatbuffers_, reflection_Field_deprecated, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), reflection_Field) +__flatbuffers_build_scalar_field(7, flatbuffers_, reflection_Field_required, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), reflection_Field) +__flatbuffers_build_scalar_field(8, flatbuffers_, reflection_Field_key, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), reflection_Field) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(9, flatbuffers_, reflection_Field_attributes, reflection_KeyValue, reflection_Field) +__flatbuffers_build_string_vector_field(10, flatbuffers_, reflection_Field_documentation, reflection_Field) +__flatbuffers_build_scalar_field(11, flatbuffers_, reflection_Field_optional, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), reflection_Field) + +static inline reflection_Field_ref_t reflection_Field_create(flatbuffers_builder_t *B __reflection_Field_formal_args) +{ + if (reflection_Field_start(B) + || reflection_Field_default_integer_add(B, v4) + || reflection_Field_default_real_add(B, v5) + || reflection_Field_name_add(B, v0) + || reflection_Field_type_add(B, v1) + || reflection_Field_attributes_add(B, v9) + || reflection_Field_documentation_add(B, v10) + || reflection_Field_id_add(B, v2) + || reflection_Field_offset_add(B, v3) + || reflection_Field_deprecated_add(B, v6) + || reflection_Field_required_add(B, v7) + || reflection_Field_key_add(B, v8) + || reflection_Field_optional_add(B, v11)) { + return 0; + } + return reflection_Field_end(B); +} + +static reflection_Field_ref_t reflection_Field_clone(flatbuffers_builder_t *B, reflection_Field_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_Field_start(B) + || reflection_Field_default_integer_pick(B, t) + || reflection_Field_default_real_pick(B, t) + || reflection_Field_name_pick(B, t) + || reflection_Field_type_pick(B, t) + || reflection_Field_attributes_pick(B, t) + || reflection_Field_documentation_pick(B, t) + || reflection_Field_id_pick(B, t) + || reflection_Field_offset_pick(B, t) + || reflection_Field_deprecated_pick(B, t) + || reflection_Field_required_pick(B, t) + || reflection_Field_key_pick(B, t) + || reflection_Field_optional_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_Field_end(B)); +} + +__flatbuffers_build_string_field(0, flatbuffers_, reflection_Object_name, reflection_Object) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(1, flatbuffers_, reflection_Object_fields, reflection_Field, reflection_Object) +__flatbuffers_build_scalar_field(2, flatbuffers_, reflection_Object_is_struct, flatbuffers_bool, flatbuffers_bool_t, 1, 1, UINT8_C(0), reflection_Object) +__flatbuffers_build_scalar_field(3, flatbuffers_, reflection_Object_minalign, flatbuffers_int32, int32_t, 4, 4, INT32_C(0), reflection_Object) +__flatbuffers_build_scalar_field(4, flatbuffers_, reflection_Object_bytesize, flatbuffers_int32, int32_t, 4, 4, INT32_C(0), reflection_Object) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(5, flatbuffers_, reflection_Object_attributes, reflection_KeyValue, reflection_Object) +__flatbuffers_build_string_vector_field(6, flatbuffers_, reflection_Object_documentation, reflection_Object) + +static inline reflection_Object_ref_t reflection_Object_create(flatbuffers_builder_t *B __reflection_Object_formal_args) +{ + if (reflection_Object_start(B) + || reflection_Object_name_add(B, v0) + || reflection_Object_fields_add(B, v1) + || reflection_Object_minalign_add(B, v3) + || reflection_Object_bytesize_add(B, v4) + || reflection_Object_attributes_add(B, v5) + || reflection_Object_documentation_add(B, v6) + || reflection_Object_is_struct_add(B, v2)) { + return 0; + } + return reflection_Object_end(B); +} + +static reflection_Object_ref_t reflection_Object_clone(flatbuffers_builder_t *B, reflection_Object_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_Object_start(B) + || reflection_Object_name_pick(B, t) + || reflection_Object_fields_pick(B, t) + || reflection_Object_minalign_pick(B, t) + || reflection_Object_bytesize_pick(B, t) + || reflection_Object_attributes_pick(B, t) + || reflection_Object_documentation_pick(B, t) + || reflection_Object_is_struct_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_Object_end(B)); +} + +__flatbuffers_build_string_field(0, flatbuffers_, reflection_RPCCall_name, reflection_RPCCall) +__flatbuffers_build_table_field(1, flatbuffers_, reflection_RPCCall_request, reflection_Object, reflection_RPCCall) +__flatbuffers_build_table_field(2, flatbuffers_, reflection_RPCCall_response, reflection_Object, reflection_RPCCall) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(3, flatbuffers_, reflection_RPCCall_attributes, reflection_KeyValue, reflection_RPCCall) +__flatbuffers_build_string_vector_field(4, flatbuffers_, reflection_RPCCall_documentation, reflection_RPCCall) + +static inline reflection_RPCCall_ref_t reflection_RPCCall_create(flatbuffers_builder_t *B __reflection_RPCCall_formal_args) +{ + if (reflection_RPCCall_start(B) + || reflection_RPCCall_name_add(B, v0) + || reflection_RPCCall_request_add(B, v1) + || reflection_RPCCall_response_add(B, v2) + || reflection_RPCCall_attributes_add(B, v3) + || reflection_RPCCall_documentation_add(B, v4)) { + return 0; + } + return reflection_RPCCall_end(B); +} + +static reflection_RPCCall_ref_t reflection_RPCCall_clone(flatbuffers_builder_t *B, reflection_RPCCall_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_RPCCall_start(B) + || reflection_RPCCall_name_pick(B, t) + || reflection_RPCCall_request_pick(B, t) + || reflection_RPCCall_response_pick(B, t) + || reflection_RPCCall_attributes_pick(B, t) + || reflection_RPCCall_documentation_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_RPCCall_end(B)); +} + +__flatbuffers_build_string_field(0, flatbuffers_, reflection_Service_name, reflection_Service) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(1, flatbuffers_, reflection_Service_calls, reflection_RPCCall, reflection_Service) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(2, flatbuffers_, reflection_Service_attributes, reflection_KeyValue, reflection_Service) +__flatbuffers_build_string_vector_field(3, flatbuffers_, reflection_Service_documentation, reflection_Service) + +static inline reflection_Service_ref_t reflection_Service_create(flatbuffers_builder_t *B __reflection_Service_formal_args) +{ + if (reflection_Service_start(B) + || reflection_Service_name_add(B, v0) + || reflection_Service_calls_add(B, v1) + || reflection_Service_attributes_add(B, v2) + || reflection_Service_documentation_add(B, v3)) { + return 0; + } + return reflection_Service_end(B); +} + +static reflection_Service_ref_t reflection_Service_clone(flatbuffers_builder_t *B, reflection_Service_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_Service_start(B) + || reflection_Service_name_pick(B, t) + || reflection_Service_calls_pick(B, t) + || reflection_Service_attributes_pick(B, t) + || reflection_Service_documentation_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_Service_end(B)); +} + +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(0, flatbuffers_, reflection_Schema_objects, reflection_Object, reflection_Schema) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(1, flatbuffers_, reflection_Schema_enums, reflection_Enum, reflection_Schema) +__flatbuffers_build_string_field(2, flatbuffers_, reflection_Schema_file_ident, reflection_Schema) +__flatbuffers_build_string_field(3, flatbuffers_, reflection_Schema_file_ext, reflection_Schema) +__flatbuffers_build_table_field(4, flatbuffers_, reflection_Schema_root_table, reflection_Object, reflection_Schema) +/* vector has keyed elements */ +__flatbuffers_build_table_vector_field(5, flatbuffers_, reflection_Schema_services, reflection_Service, reflection_Schema) + +static inline reflection_Schema_ref_t reflection_Schema_create(flatbuffers_builder_t *B __reflection_Schema_formal_args) +{ + if (reflection_Schema_start(B) + || reflection_Schema_objects_add(B, v0) + || reflection_Schema_enums_add(B, v1) + || reflection_Schema_file_ident_add(B, v2) + || reflection_Schema_file_ext_add(B, v3) + || reflection_Schema_root_table_add(B, v4) + || reflection_Schema_services_add(B, v5)) { + return 0; + } + return reflection_Schema_end(B); +} + +static reflection_Schema_ref_t reflection_Schema_clone(flatbuffers_builder_t *B, reflection_Schema_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (reflection_Schema_start(B) + || reflection_Schema_objects_pick(B, t) + || reflection_Schema_enums_pick(B, t) + || reflection_Schema_file_ident_pick(B, t) + || reflection_Schema_file_ext_pick(B, t) + || reflection_Schema_root_table_pick(B, t) + || reflection_Schema_services_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, reflection_Schema_end(B)); +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* REFLECTION_BUILDER_H */ diff --git a/nostrdb/flatcc/reflection/reflection_reader.h b/nostrdb/flatcc/reflection/reflection_reader.h new file mode 100644 index 0000000000..bf6a0e9460 --- /dev/null +++ b/nostrdb/flatcc/reflection/reflection_reader.h @@ -0,0 +1,411 @@ +#ifndef REFLECTION_READER_H +#define REFLECTION_READER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef FLATBUFFERS_COMMON_READER_H +#include "flatbuffers_common_reader.h" +#endif +#include "flatcc/flatcc_flatbuffers.h" +#ifndef __alignas_is_defined +#include +#endif +#include "flatcc/flatcc_prologue.h" +#undef flatbuffers_identifier +#define flatbuffers_identifier "BFBS" +#undef flatbuffers_extension +#define flatbuffers_extension "bfbs" + + +typedef const struct reflection_Type_table *reflection_Type_table_t; +typedef struct reflection_Type_table *reflection_Type_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_Type_vec_t; +typedef flatbuffers_uoffset_t *reflection_Type_mutable_vec_t; +typedef const struct reflection_KeyValue_table *reflection_KeyValue_table_t; +typedef struct reflection_KeyValue_table *reflection_KeyValue_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_KeyValue_vec_t; +typedef flatbuffers_uoffset_t *reflection_KeyValue_mutable_vec_t; +typedef const struct reflection_EnumVal_table *reflection_EnumVal_table_t; +typedef struct reflection_EnumVal_table *reflection_EnumVal_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_EnumVal_vec_t; +typedef flatbuffers_uoffset_t *reflection_EnumVal_mutable_vec_t; +typedef const struct reflection_Enum_table *reflection_Enum_table_t; +typedef struct reflection_Enum_table *reflection_Enum_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_Enum_vec_t; +typedef flatbuffers_uoffset_t *reflection_Enum_mutable_vec_t; +typedef const struct reflection_Field_table *reflection_Field_table_t; +typedef struct reflection_Field_table *reflection_Field_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_Field_vec_t; +typedef flatbuffers_uoffset_t *reflection_Field_mutable_vec_t; +typedef const struct reflection_Object_table *reflection_Object_table_t; +typedef struct reflection_Object_table *reflection_Object_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_Object_vec_t; +typedef flatbuffers_uoffset_t *reflection_Object_mutable_vec_t; +typedef const struct reflection_RPCCall_table *reflection_RPCCall_table_t; +typedef struct reflection_RPCCall_table *reflection_RPCCall_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_RPCCall_vec_t; +typedef flatbuffers_uoffset_t *reflection_RPCCall_mutable_vec_t; +typedef const struct reflection_Service_table *reflection_Service_table_t; +typedef struct reflection_Service_table *reflection_Service_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_Service_vec_t; +typedef flatbuffers_uoffset_t *reflection_Service_mutable_vec_t; +typedef const struct reflection_Schema_table *reflection_Schema_table_t; +typedef struct reflection_Schema_table *reflection_Schema_mutable_table_t; +typedef const flatbuffers_uoffset_t *reflection_Schema_vec_t; +typedef flatbuffers_uoffset_t *reflection_Schema_mutable_vec_t; +#ifndef reflection_Type_file_identifier +#define reflection_Type_file_identifier "BFBS" +#endif +/* deprecated, use reflection_Type_file_identifier */ +#ifndef reflection_Type_identifier +#define reflection_Type_identifier "BFBS" +#endif +#define reflection_Type_type_hash ((flatbuffers_thash_t)0x44c8fe5e) +#define reflection_Type_type_identifier "\x5e\xfe\xc8\x44" +#ifndef reflection_Type_file_extension +#define reflection_Type_file_extension "bfbs" +#endif +#ifndef reflection_KeyValue_file_identifier +#define reflection_KeyValue_file_identifier "BFBS" +#endif +/* deprecated, use reflection_KeyValue_file_identifier */ +#ifndef reflection_KeyValue_identifier +#define reflection_KeyValue_identifier "BFBS" +#endif +#define reflection_KeyValue_type_hash ((flatbuffers_thash_t)0x8c761eaa) +#define reflection_KeyValue_type_identifier "\xaa\x1e\x76\x8c" +#ifndef reflection_KeyValue_file_extension +#define reflection_KeyValue_file_extension "bfbs" +#endif +#ifndef reflection_EnumVal_file_identifier +#define reflection_EnumVal_file_identifier "BFBS" +#endif +/* deprecated, use reflection_EnumVal_file_identifier */ +#ifndef reflection_EnumVal_identifier +#define reflection_EnumVal_identifier "BFBS" +#endif +#define reflection_EnumVal_type_hash ((flatbuffers_thash_t)0x9531c946) +#define reflection_EnumVal_type_identifier "\x46\xc9\x31\x95" +#ifndef reflection_EnumVal_file_extension +#define reflection_EnumVal_file_extension "bfbs" +#endif +#ifndef reflection_Enum_file_identifier +#define reflection_Enum_file_identifier "BFBS" +#endif +/* deprecated, use reflection_Enum_file_identifier */ +#ifndef reflection_Enum_identifier +#define reflection_Enum_identifier "BFBS" +#endif +#define reflection_Enum_type_hash ((flatbuffers_thash_t)0xacffa90f) +#define reflection_Enum_type_identifier "\x0f\xa9\xff\xac" +#ifndef reflection_Enum_file_extension +#define reflection_Enum_file_extension "bfbs" +#endif +#ifndef reflection_Field_file_identifier +#define reflection_Field_file_identifier "BFBS" +#endif +/* deprecated, use reflection_Field_file_identifier */ +#ifndef reflection_Field_identifier +#define reflection_Field_identifier "BFBS" +#endif +#define reflection_Field_type_hash ((flatbuffers_thash_t)0x9f7e408a) +#define reflection_Field_type_identifier "\x8a\x40\x7e\x9f" +#ifndef reflection_Field_file_extension +#define reflection_Field_file_extension "bfbs" +#endif +#ifndef reflection_Object_file_identifier +#define reflection_Object_file_identifier "BFBS" +#endif +/* deprecated, use reflection_Object_file_identifier */ +#ifndef reflection_Object_identifier +#define reflection_Object_identifier "BFBS" +#endif +#define reflection_Object_type_hash ((flatbuffers_thash_t)0xb09729bd) +#define reflection_Object_type_identifier "\xbd\x29\x97\xb0" +#ifndef reflection_Object_file_extension +#define reflection_Object_file_extension "bfbs" +#endif +#ifndef reflection_RPCCall_file_identifier +#define reflection_RPCCall_file_identifier "BFBS" +#endif +/* deprecated, use reflection_RPCCall_file_identifier */ +#ifndef reflection_RPCCall_identifier +#define reflection_RPCCall_identifier "BFBS" +#endif +#define reflection_RPCCall_type_hash ((flatbuffers_thash_t)0xe2d586f1) +#define reflection_RPCCall_type_identifier "\xf1\x86\xd5\xe2" +#ifndef reflection_RPCCall_file_extension +#define reflection_RPCCall_file_extension "bfbs" +#endif +#ifndef reflection_Service_file_identifier +#define reflection_Service_file_identifier "BFBS" +#endif +/* deprecated, use reflection_Service_file_identifier */ +#ifndef reflection_Service_identifier +#define reflection_Service_identifier "BFBS" +#endif +#define reflection_Service_type_hash ((flatbuffers_thash_t)0xf31a13b5) +#define reflection_Service_type_identifier "\xb5\x13\x1a\xf3" +#ifndef reflection_Service_file_extension +#define reflection_Service_file_extension "bfbs" +#endif +#ifndef reflection_Schema_file_identifier +#define reflection_Schema_file_identifier "BFBS" +#endif +/* deprecated, use reflection_Schema_file_identifier */ +#ifndef reflection_Schema_identifier +#define reflection_Schema_identifier "BFBS" +#endif +#define reflection_Schema_type_hash ((flatbuffers_thash_t)0xfaf93779) +#define reflection_Schema_type_identifier "\x79\x37\xf9\xfa" +#ifndef reflection_Schema_file_extension +#define reflection_Schema_file_extension "bfbs" +#endif + +typedef int8_t reflection_BaseType_enum_t; +__flatbuffers_define_integer_type(reflection_BaseType, reflection_BaseType_enum_t, 8) +#define reflection_BaseType_None ((reflection_BaseType_enum_t)INT8_C(0)) +#define reflection_BaseType_UType ((reflection_BaseType_enum_t)INT8_C(1)) +#define reflection_BaseType_Bool ((reflection_BaseType_enum_t)INT8_C(2)) +#define reflection_BaseType_Byte ((reflection_BaseType_enum_t)INT8_C(3)) +#define reflection_BaseType_UByte ((reflection_BaseType_enum_t)INT8_C(4)) +#define reflection_BaseType_Short ((reflection_BaseType_enum_t)INT8_C(5)) +#define reflection_BaseType_UShort ((reflection_BaseType_enum_t)INT8_C(6)) +#define reflection_BaseType_Int ((reflection_BaseType_enum_t)INT8_C(7)) +#define reflection_BaseType_UInt ((reflection_BaseType_enum_t)INT8_C(8)) +#define reflection_BaseType_Long ((reflection_BaseType_enum_t)INT8_C(9)) +#define reflection_BaseType_ULong ((reflection_BaseType_enum_t)INT8_C(10)) +#define reflection_BaseType_Float ((reflection_BaseType_enum_t)INT8_C(11)) +#define reflection_BaseType_Double ((reflection_BaseType_enum_t)INT8_C(12)) +#define reflection_BaseType_String ((reflection_BaseType_enum_t)INT8_C(13)) +#define reflection_BaseType_Vector ((reflection_BaseType_enum_t)INT8_C(14)) +#define reflection_BaseType_Obj ((reflection_BaseType_enum_t)INT8_C(15)) +#define reflection_BaseType_Union ((reflection_BaseType_enum_t)INT8_C(16)) +#define reflection_BaseType_Array ((reflection_BaseType_enum_t)INT8_C(17)) +#define reflection_BaseType_MaxBaseType ((reflection_BaseType_enum_t)INT8_C(18)) + +static inline const char *reflection_BaseType_name(reflection_BaseType_enum_t value) +{ + switch (value) { + case reflection_BaseType_None: return "None"; + case reflection_BaseType_UType: return "UType"; + case reflection_BaseType_Bool: return "Bool"; + case reflection_BaseType_Byte: return "Byte"; + case reflection_BaseType_UByte: return "UByte"; + case reflection_BaseType_Short: return "Short"; + case reflection_BaseType_UShort: return "UShort"; + case reflection_BaseType_Int: return "Int"; + case reflection_BaseType_UInt: return "UInt"; + case reflection_BaseType_Long: return "Long"; + case reflection_BaseType_ULong: return "ULong"; + case reflection_BaseType_Float: return "Float"; + case reflection_BaseType_Double: return "Double"; + case reflection_BaseType_String: return "String"; + case reflection_BaseType_Vector: return "Vector"; + case reflection_BaseType_Obj: return "Obj"; + case reflection_BaseType_Union: return "Union"; + case reflection_BaseType_Array: return "Array"; + case reflection_BaseType_MaxBaseType: return "MaxBaseType"; + default: return ""; + } +} + +static inline int reflection_BaseType_is_known_value(reflection_BaseType_enum_t value) +{ + switch (value) { + case reflection_BaseType_None: return 1; + case reflection_BaseType_UType: return 1; + case reflection_BaseType_Bool: return 1; + case reflection_BaseType_Byte: return 1; + case reflection_BaseType_UByte: return 1; + case reflection_BaseType_Short: return 1; + case reflection_BaseType_UShort: return 1; + case reflection_BaseType_Int: return 1; + case reflection_BaseType_UInt: return 1; + case reflection_BaseType_Long: return 1; + case reflection_BaseType_ULong: return 1; + case reflection_BaseType_Float: return 1; + case reflection_BaseType_Double: return 1; + case reflection_BaseType_String: return 1; + case reflection_BaseType_Vector: return 1; + case reflection_BaseType_Obj: return 1; + case reflection_BaseType_Union: return 1; + case reflection_BaseType_Array: return 1; + case reflection_BaseType_MaxBaseType: return 1; + default: return 0; + } +} + + + +struct reflection_Type_table { uint8_t unused__; }; + +static inline size_t reflection_Type_vec_len(reflection_Type_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_Type_table_t reflection_Type_vec_at(reflection_Type_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_Type_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_Type) + +__flatbuffers_define_scalar_field(0, reflection_Type, base_type, reflection_BaseType, reflection_BaseType_enum_t, INT8_C(0)) +__flatbuffers_define_scalar_field(1, reflection_Type, element, reflection_BaseType, reflection_BaseType_enum_t, INT8_C(0)) +__flatbuffers_define_scalar_field(2, reflection_Type, index, flatbuffers_int32, int32_t, INT32_C(-1)) +__flatbuffers_define_scalar_field(3, reflection_Type, fixed_length, flatbuffers_uint16, uint16_t, UINT16_C(0)) + +struct reflection_KeyValue_table { uint8_t unused__; }; + +static inline size_t reflection_KeyValue_vec_len(reflection_KeyValue_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_KeyValue_table_t reflection_KeyValue_vec_at(reflection_KeyValue_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_KeyValue_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_KeyValue) + +__flatbuffers_define_string_field(0, reflection_KeyValue, key, 1) +__flatbuffers_define_find_by_string_field(reflection_KeyValue, key) +__flatbuffers_define_table_sort_by_string_field(reflection_KeyValue, key) +__flatbuffers_define_default_find_by_string_field(reflection_KeyValue, key) +__flatbuffers_define_default_scan_by_string_field(reflection_KeyValue, key) +#define reflection_KeyValue_vec_sort reflection_KeyValue_vec_sort_by_key +__flatbuffers_define_string_field(1, reflection_KeyValue, value, 0) + +struct reflection_EnumVal_table { uint8_t unused__; }; + +static inline size_t reflection_EnumVal_vec_len(reflection_EnumVal_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_EnumVal_table_t reflection_EnumVal_vec_at(reflection_EnumVal_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_EnumVal_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_EnumVal) + +__flatbuffers_define_string_field(0, reflection_EnumVal, name, 1) +__flatbuffers_define_scalar_field(1, reflection_EnumVal, value, flatbuffers_int64, int64_t, INT64_C(0)) +/* Note: find only works on vectors sorted by this field. */ +__flatbuffers_define_find_by_scalar_field(reflection_EnumVal, value, int64_t) +__flatbuffers_define_table_sort_by_scalar_field(reflection_EnumVal, value, int64_t) +__flatbuffers_define_default_find_by_scalar_field(reflection_EnumVal, value, int64_t) +__flatbuffers_define_default_scan_by_scalar_field(reflection_EnumVal, value, int64_t) +#define reflection_EnumVal_vec_sort reflection_EnumVal_vec_sort_by_value +__flatbuffers_define_table_field(2, reflection_EnumVal, object, reflection_Object_table_t, 0) +__flatbuffers_define_table_field(3, reflection_EnumVal, union_type, reflection_Type_table_t, 0) +__flatbuffers_define_vector_field(4, reflection_EnumVal, documentation, flatbuffers_string_vec_t, 0) + +struct reflection_Enum_table { uint8_t unused__; }; + +static inline size_t reflection_Enum_vec_len(reflection_Enum_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_Enum_table_t reflection_Enum_vec_at(reflection_Enum_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_Enum_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_Enum) + +__flatbuffers_define_string_field(0, reflection_Enum, name, 1) +__flatbuffers_define_find_by_string_field(reflection_Enum, name) +__flatbuffers_define_table_sort_by_string_field(reflection_Enum, name) +__flatbuffers_define_default_find_by_string_field(reflection_Enum, name) +__flatbuffers_define_default_scan_by_string_field(reflection_Enum, name) +#define reflection_Enum_vec_sort reflection_Enum_vec_sort_by_name +__flatbuffers_define_vector_field(1, reflection_Enum, values, reflection_EnumVal_vec_t, 1) +__flatbuffers_define_scalar_field(2, reflection_Enum, is_union, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) +__flatbuffers_define_table_field(3, reflection_Enum, underlying_type, reflection_Type_table_t, 1) +__flatbuffers_define_vector_field(4, reflection_Enum, attributes, reflection_KeyValue_vec_t, 0) +__flatbuffers_define_vector_field(5, reflection_Enum, documentation, flatbuffers_string_vec_t, 0) + +struct reflection_Field_table { uint8_t unused__; }; + +static inline size_t reflection_Field_vec_len(reflection_Field_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_Field_table_t reflection_Field_vec_at(reflection_Field_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_Field_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_Field) + +__flatbuffers_define_string_field(0, reflection_Field, name, 1) +__flatbuffers_define_find_by_string_field(reflection_Field, name) +__flatbuffers_define_table_sort_by_string_field(reflection_Field, name) +__flatbuffers_define_default_find_by_string_field(reflection_Field, name) +__flatbuffers_define_default_scan_by_string_field(reflection_Field, name) +#define reflection_Field_vec_sort reflection_Field_vec_sort_by_name +__flatbuffers_define_table_field(1, reflection_Field, type, reflection_Type_table_t, 1) +__flatbuffers_define_scalar_field(2, reflection_Field, id, flatbuffers_uint16, uint16_t, UINT16_C(0)) +__flatbuffers_define_scalar_field(3, reflection_Field, offset, flatbuffers_uint16, uint16_t, UINT16_C(0)) +__flatbuffers_define_scalar_field(4, reflection_Field, default_integer, flatbuffers_int64, int64_t, INT64_C(0)) +__flatbuffers_define_scalar_field(5, reflection_Field, default_real, flatbuffers_double, double, 0.0000000000000000) +__flatbuffers_define_scalar_field(6, reflection_Field, deprecated, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) +__flatbuffers_define_scalar_field(7, reflection_Field, required, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) +__flatbuffers_define_scalar_field(8, reflection_Field, key, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) +__flatbuffers_define_vector_field(9, reflection_Field, attributes, reflection_KeyValue_vec_t, 0) +__flatbuffers_define_vector_field(10, reflection_Field, documentation, flatbuffers_string_vec_t, 0) +__flatbuffers_define_scalar_field(11, reflection_Field, optional, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) + +struct reflection_Object_table { uint8_t unused__; }; + +static inline size_t reflection_Object_vec_len(reflection_Object_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_Object_table_t reflection_Object_vec_at(reflection_Object_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_Object_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_Object) + +__flatbuffers_define_string_field(0, reflection_Object, name, 1) +__flatbuffers_define_find_by_string_field(reflection_Object, name) +__flatbuffers_define_table_sort_by_string_field(reflection_Object, name) +__flatbuffers_define_default_find_by_string_field(reflection_Object, name) +__flatbuffers_define_default_scan_by_string_field(reflection_Object, name) +#define reflection_Object_vec_sort reflection_Object_vec_sort_by_name +__flatbuffers_define_vector_field(1, reflection_Object, fields, reflection_Field_vec_t, 1) +__flatbuffers_define_scalar_field(2, reflection_Object, is_struct, flatbuffers_bool, flatbuffers_bool_t, UINT8_C(0)) +__flatbuffers_define_scalar_field(3, reflection_Object, minalign, flatbuffers_int32, int32_t, INT32_C(0)) +__flatbuffers_define_scalar_field(4, reflection_Object, bytesize, flatbuffers_int32, int32_t, INT32_C(0)) +__flatbuffers_define_vector_field(5, reflection_Object, attributes, reflection_KeyValue_vec_t, 0) +__flatbuffers_define_vector_field(6, reflection_Object, documentation, flatbuffers_string_vec_t, 0) + +struct reflection_RPCCall_table { uint8_t unused__; }; + +static inline size_t reflection_RPCCall_vec_len(reflection_RPCCall_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_RPCCall_table_t reflection_RPCCall_vec_at(reflection_RPCCall_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_RPCCall_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_RPCCall) + +__flatbuffers_define_string_field(0, reflection_RPCCall, name, 1) +__flatbuffers_define_find_by_string_field(reflection_RPCCall, name) +__flatbuffers_define_table_sort_by_string_field(reflection_RPCCall, name) +__flatbuffers_define_default_find_by_string_field(reflection_RPCCall, name) +__flatbuffers_define_default_scan_by_string_field(reflection_RPCCall, name) +#define reflection_RPCCall_vec_sort reflection_RPCCall_vec_sort_by_name +__flatbuffers_define_table_field(1, reflection_RPCCall, request, reflection_Object_table_t, 1) +__flatbuffers_define_table_field(2, reflection_RPCCall, response, reflection_Object_table_t, 1) +__flatbuffers_define_vector_field(3, reflection_RPCCall, attributes, reflection_KeyValue_vec_t, 0) +__flatbuffers_define_vector_field(4, reflection_RPCCall, documentation, flatbuffers_string_vec_t, 0) + +struct reflection_Service_table { uint8_t unused__; }; + +static inline size_t reflection_Service_vec_len(reflection_Service_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_Service_table_t reflection_Service_vec_at(reflection_Service_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_Service_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_Service) + +__flatbuffers_define_string_field(0, reflection_Service, name, 1) +__flatbuffers_define_find_by_string_field(reflection_Service, name) +__flatbuffers_define_table_sort_by_string_field(reflection_Service, name) +__flatbuffers_define_default_find_by_string_field(reflection_Service, name) +__flatbuffers_define_default_scan_by_string_field(reflection_Service, name) +#define reflection_Service_vec_sort reflection_Service_vec_sort_by_name +__flatbuffers_define_vector_field(1, reflection_Service, calls, reflection_RPCCall_vec_t, 0) +__flatbuffers_define_vector_field(2, reflection_Service, attributes, reflection_KeyValue_vec_t, 0) +__flatbuffers_define_vector_field(3, reflection_Service, documentation, flatbuffers_string_vec_t, 0) + +struct reflection_Schema_table { uint8_t unused__; }; + +static inline size_t reflection_Schema_vec_len(reflection_Schema_vec_t vec) +__flatbuffers_vec_len(vec) +static inline reflection_Schema_table_t reflection_Schema_vec_at(reflection_Schema_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(reflection_Schema_table_t, vec, i, 0) +__flatbuffers_table_as_root(reflection_Schema) + +__flatbuffers_define_vector_field(0, reflection_Schema, objects, reflection_Object_vec_t, 1) +__flatbuffers_define_vector_field(1, reflection_Schema, enums, reflection_Enum_vec_t, 1) +__flatbuffers_define_string_field(2, reflection_Schema, file_ident, 0) +__flatbuffers_define_string_field(3, reflection_Schema, file_ext, 0) +__flatbuffers_define_table_field(4, reflection_Schema, root_table, reflection_Object_table_t, 0) +__flatbuffers_define_vector_field(5, reflection_Schema, services, reflection_Service_vec_t, 0) + + +#include "flatcc/flatcc_epilogue.h" +#endif /* REFLECTION_READER_H */ diff --git a/nostrdb/flatcc/reflection/reflection_verifier.h b/nostrdb/flatcc/reflection/reflection_verifier.h new file mode 100644 index 0000000000..5b5bd374a9 --- /dev/null +++ b/nostrdb/flatcc/reflection/reflection_verifier.h @@ -0,0 +1,308 @@ +#ifndef REFLECTION_VERIFIER_H +#define REFLECTION_VERIFIER_H + +/* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ + +#ifndef REFLECTION_READER_H +#include "reflection_reader.h" +#endif +#include "flatcc/flatcc_verifier.h" +#include "flatcc/flatcc_prologue.h" + +static int reflection_Type_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_KeyValue_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_EnumVal_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_Enum_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_Field_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_Object_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_RPCCall_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_Service_verify_table(flatcc_table_verifier_descriptor_t *td); +static int reflection_Schema_verify_table(flatcc_table_verifier_descriptor_t *td); + +static int reflection_Type_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_field(td, 0, 1, 1) /* base_type */)) return ret; + if ((ret = flatcc_verify_field(td, 1, 1, 1) /* element */)) return ret; + if ((ret = flatcc_verify_field(td, 2, 4, 4) /* index */)) return ret; + if ((ret = flatcc_verify_field(td, 3, 2, 2) /* fixed_length */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_Type_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Type_identifier, &reflection_Type_verify_table); +} + +static inline int reflection_Type_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Type_type_identifier, &reflection_Type_verify_table); +} + +static inline int reflection_Type_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Type_verify_table); +} + +static inline int reflection_Type_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_Type_verify_table); +} + +static int reflection_KeyValue_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 1) /* key */)) return ret; + if ((ret = flatcc_verify_string_field(td, 1, 0) /* value */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_KeyValue_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_KeyValue_identifier, &reflection_KeyValue_verify_table); +} + +static inline int reflection_KeyValue_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_KeyValue_type_identifier, &reflection_KeyValue_verify_table); +} + +static inline int reflection_KeyValue_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_KeyValue_verify_table); +} + +static inline int reflection_KeyValue_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_KeyValue_verify_table); +} + +static int reflection_EnumVal_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 1) /* name */)) return ret; + if ((ret = flatcc_verify_field(td, 1, 8, 8) /* value */)) return ret; + if ((ret = flatcc_verify_table_field(td, 2, 0, &reflection_Object_verify_table) /* object */)) return ret; + if ((ret = flatcc_verify_table_field(td, 3, 0, &reflection_Type_verify_table) /* union_type */)) return ret; + if ((ret = flatcc_verify_string_vector_field(td, 4, 0) /* documentation */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_EnumVal_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_EnumVal_identifier, &reflection_EnumVal_verify_table); +} + +static inline int reflection_EnumVal_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_EnumVal_type_identifier, &reflection_EnumVal_verify_table); +} + +static inline int reflection_EnumVal_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_EnumVal_verify_table); +} + +static inline int reflection_EnumVal_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_EnumVal_verify_table); +} + +static int reflection_Enum_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 1) /* name */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 1, 1, &reflection_EnumVal_verify_table) /* values */)) return ret; + if ((ret = flatcc_verify_field(td, 2, 1, 1) /* is_union */)) return ret; + if ((ret = flatcc_verify_table_field(td, 3, 1, &reflection_Type_verify_table) /* underlying_type */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 4, 0, &reflection_KeyValue_verify_table) /* attributes */)) return ret; + if ((ret = flatcc_verify_string_vector_field(td, 5, 0) /* documentation */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_Enum_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Enum_identifier, &reflection_Enum_verify_table); +} + +static inline int reflection_Enum_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Enum_type_identifier, &reflection_Enum_verify_table); +} + +static inline int reflection_Enum_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Enum_verify_table); +} + +static inline int reflection_Enum_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_Enum_verify_table); +} + +static int reflection_Field_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 1) /* name */)) return ret; + if ((ret = flatcc_verify_table_field(td, 1, 1, &reflection_Type_verify_table) /* type */)) return ret; + if ((ret = flatcc_verify_field(td, 2, 2, 2) /* id */)) return ret; + if ((ret = flatcc_verify_field(td, 3, 2, 2) /* offset */)) return ret; + if ((ret = flatcc_verify_field(td, 4, 8, 8) /* default_integer */)) return ret; + if ((ret = flatcc_verify_field(td, 5, 8, 8) /* default_real */)) return ret; + if ((ret = flatcc_verify_field(td, 6, 1, 1) /* deprecated */)) return ret; + if ((ret = flatcc_verify_field(td, 7, 1, 1) /* required */)) return ret; + if ((ret = flatcc_verify_field(td, 8, 1, 1) /* key */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 9, 0, &reflection_KeyValue_verify_table) /* attributes */)) return ret; + if ((ret = flatcc_verify_string_vector_field(td, 10, 0) /* documentation */)) return ret; + if ((ret = flatcc_verify_field(td, 11, 1, 1) /* optional */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_Field_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Field_identifier, &reflection_Field_verify_table); +} + +static inline int reflection_Field_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Field_type_identifier, &reflection_Field_verify_table); +} + +static inline int reflection_Field_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Field_verify_table); +} + +static inline int reflection_Field_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_Field_verify_table); +} + +static int reflection_Object_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 1) /* name */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 1, 1, &reflection_Field_verify_table) /* fields */)) return ret; + if ((ret = flatcc_verify_field(td, 2, 1, 1) /* is_struct */)) return ret; + if ((ret = flatcc_verify_field(td, 3, 4, 4) /* minalign */)) return ret; + if ((ret = flatcc_verify_field(td, 4, 4, 4) /* bytesize */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 5, 0, &reflection_KeyValue_verify_table) /* attributes */)) return ret; + if ((ret = flatcc_verify_string_vector_field(td, 6, 0) /* documentation */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_Object_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Object_identifier, &reflection_Object_verify_table); +} + +static inline int reflection_Object_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Object_type_identifier, &reflection_Object_verify_table); +} + +static inline int reflection_Object_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Object_verify_table); +} + +static inline int reflection_Object_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_Object_verify_table); +} + +static int reflection_RPCCall_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 1) /* name */)) return ret; + if ((ret = flatcc_verify_table_field(td, 1, 1, &reflection_Object_verify_table) /* request */)) return ret; + if ((ret = flatcc_verify_table_field(td, 2, 1, &reflection_Object_verify_table) /* response */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 3, 0, &reflection_KeyValue_verify_table) /* attributes */)) return ret; + if ((ret = flatcc_verify_string_vector_field(td, 4, 0) /* documentation */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_RPCCall_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_RPCCall_identifier, &reflection_RPCCall_verify_table); +} + +static inline int reflection_RPCCall_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_RPCCall_type_identifier, &reflection_RPCCall_verify_table); +} + +static inline int reflection_RPCCall_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_RPCCall_verify_table); +} + +static inline int reflection_RPCCall_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_RPCCall_verify_table); +} + +static int reflection_Service_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_string_field(td, 0, 1) /* name */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 1, 0, &reflection_RPCCall_verify_table) /* calls */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 2, 0, &reflection_KeyValue_verify_table) /* attributes */)) return ret; + if ((ret = flatcc_verify_string_vector_field(td, 3, 0) /* documentation */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_Service_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Service_identifier, &reflection_Service_verify_table); +} + +static inline int reflection_Service_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Service_type_identifier, &reflection_Service_verify_table); +} + +static inline int reflection_Service_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Service_verify_table); +} + +static inline int reflection_Service_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_Service_verify_table); +} + +static int reflection_Schema_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_table_vector_field(td, 0, 1, &reflection_Object_verify_table) /* objects */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 1, 1, &reflection_Enum_verify_table) /* enums */)) return ret; + if ((ret = flatcc_verify_string_field(td, 2, 0) /* file_ident */)) return ret; + if ((ret = flatcc_verify_string_field(td, 3, 0) /* file_ext */)) return ret; + if ((ret = flatcc_verify_table_field(td, 4, 0, &reflection_Object_verify_table) /* root_table */)) return ret; + if ((ret = flatcc_verify_table_vector_field(td, 5, 0, &reflection_Service_verify_table) /* services */)) return ret; + return flatcc_verify_ok; +} + +static inline int reflection_Schema_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Schema_identifier, &reflection_Schema_verify_table); +} + +static inline int reflection_Schema_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, reflection_Schema_type_identifier, &reflection_Schema_verify_table); +} + +static inline int reflection_Schema_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Schema_verify_table); +} + +static inline int reflection_Schema_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &reflection_Schema_verify_table); +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* REFLECTION_VERIFIER_H */ diff --git a/nostrdb/flatcc/refmap.c b/nostrdb/flatcc/refmap.c new file mode 100644 index 0000000000..d8c6034fbb --- /dev/null +++ b/nostrdb/flatcc/refmap.c @@ -0,0 +1,248 @@ +/* + * Optional file that can be included in runtime library to support DAG + * cloning with the builder and may also be used for custom purposes + * standalone. See also comments in `flatcc/flatcc_builder.h`. + * + * Note that dynamic construction takes place and that large offset + * vectors might consume significant space if there are not many shared + * references. In the basic use case no allocation takes place because a + * few references can be held using only a small stack allocated hash + * table. + */ + +#include +#include + +#include "flatcc_rtconfig.h" +#include "flatcc_refmap.h" +#include "flatcc_alloc.h" +#include "flatcc_assert.h" + +#define _flatcc_refmap_calloc FLATCC_CALLOC +#define _flatcc_refmap_free FLATCC_FREE + +/* Can be used as a primitive defense against collision attacks. */ +#ifdef FLATCC_HASH_SEED +#define _flatcc_refmap_seed FLATCC_HASH_SEED +#else +#define _flatcc_refmap_seed 0x2f693b52 +#endif + +static inline size_t _flatcc_refmap_above_load_factor(size_t count, size_t buckets) +{ + static const size_t d = 256; + static const size_t n = (size_t)((FLATCC_REFMAP_LOAD_FACTOR) * 256.0f); + + return count >= buckets * n / d; +} + +#define _flatcc_refmap_probe(k, i, N) ((k + i) & N) + +void flatcc_refmap_clear(flatcc_refmap_t *refmap) +{ + if (refmap->table && refmap->table != refmap->min_table) { + _flatcc_refmap_free(refmap->table); + } + flatcc_refmap_init(refmap); +} + +static inline size_t _flatcc_refmap_hash(const void *src) +{ + /* MurmurHash3 64-bit finalizer */ + uint64_t x; + + x = (uint64_t)((size_t)src) ^ _flatcc_refmap_seed; + + x ^= x >> 33; + x *= 0xff51afd7ed558ccdULL; + x ^= x >> 33; + x *= 0xc4ceb9fe1a85ec53ULL; + x ^= x >> 33; + return (size_t)x; +} + +void flatcc_refmap_reset(flatcc_refmap_t *refmap) +{ + if (refmap->count) { + memset(refmap->table, 0, sizeof(refmap->table[0]) * refmap->buckets); + } + refmap->count = 0; +} + +/* + * Technically resize also supports shrinking which may be useful for + * adapations, but the current hash table never deletes individual items. + */ +int flatcc_refmap_resize(flatcc_refmap_t *refmap, size_t count) +{ + const size_t min_buckets = sizeof(refmap->min_table) / sizeof(refmap->min_table[0]); + + size_t i; + size_t buckets; + size_t buckets_old; + struct flatcc_refmap_item *T_old; + + if (count < refmap->count) { + count = refmap->count; + } + buckets = min_buckets; + + while (_flatcc_refmap_above_load_factor(count, buckets)) { + buckets *= 2; + } + if (refmap->buckets == buckets) { + return 0; + } + T_old = refmap->table; + buckets_old = refmap->buckets; + if (buckets == min_buckets) { + memset(refmap->min_table, 0, sizeof(refmap->min_table)); + refmap->table = refmap->min_table; + } else { + refmap->table = _flatcc_refmap_calloc(buckets, sizeof(refmap->table[0])); + if (refmap->table == 0) { + refmap->table = T_old; + FLATCC_ASSERT(0); /* out of memory */ + return -1; + } + } + refmap->buckets = buckets; + refmap->count = 0; + for (i = 0; i < buckets_old; ++i) { + if (T_old[i].src) { + flatcc_refmap_insert(refmap, T_old[i].src, T_old[i].ref); + } + } + if (T_old && T_old != refmap->min_table) { + _flatcc_refmap_free(T_old); + } + return 0; +} + +flatcc_refmap_ref_t flatcc_refmap_insert(flatcc_refmap_t *refmap, const void *src, flatcc_refmap_ref_t ref) +{ + struct flatcc_refmap_item *T; + size_t N, i, j, k; + + if (src == 0) return ref; + if (_flatcc_refmap_above_load_factor(refmap->count, refmap->buckets)) { + if (flatcc_refmap_resize(refmap, refmap->count * 2)) { + return flatcc_refmap_not_found; /* alloc failed */ + } + } + T = refmap->table; + N = refmap->buckets - 1; + k = _flatcc_refmap_hash(src); + i = 0; + j = _flatcc_refmap_probe(k, i, N); + while (T[j].src) { + if (T[j].src == src) { + return T[j].ref = ref; + } + ++i; + j = _flatcc_refmap_probe(k, i, N); + } + ++refmap->count; + T[j].src = src; + return T[j].ref = ref; +} + +flatcc_refmap_ref_t flatcc_refmap_find(flatcc_refmap_t *refmap, const void *src) +{ + struct flatcc_refmap_item *T; + size_t N, i, j, k; + + if (refmap->count == 0) { + return flatcc_refmap_not_found; + } + T = refmap->table; + N = refmap->buckets - 1; + k = _flatcc_refmap_hash(src); + i = 0; + j = _flatcc_refmap_probe(k, i, N); + while (T[j].src) { + if (T[j].src == src) return T[j].ref; + ++i; + j = _flatcc_refmap_probe(k, i, N); + } + return flatcc_refmap_not_found; +} + +/* + * To run test from project root: + * + * cc -D FLATCC_REFMAP_TEST -I include src/runtime/refmap.c -o test_refmap && ./test_refmap + * + */ +#ifdef FLATCC_REFMAP_TEST + +#include + +#ifndef FLATCC_REFMAP_H +#include "flatcc/flatcc_refmap.h" +#endif + +#define test(x) do { if (!(x)) { fprintf(stderr, "%02d: refmap test failed\n", __LINE__); exit(-1); } } while (0) +#define test_start() fprintf(stderr, "starting refmap test ...\n") +#define test_ok() fprintf(stderr, "refmap test succeeded\n") + +int main() +{ + int i; + int data[1000]; + int a = 1; + int b = 2; + int c = 3; + flatcc_refmap_t refmap; + + flatcc_refmap_init(&refmap); + + test(flatcc_refmap_find(&refmap, &a) == flatcc_refmap_not_found); + test(flatcc_refmap_find(&refmap, &b) == flatcc_refmap_not_found); + test(flatcc_refmap_find(&refmap, &c) == flatcc_refmap_not_found); + test(flatcc_refmap_find(&refmap, 0) == flatcc_refmap_not_found); + test(flatcc_refmap_find(&refmap, &a) == 0); + + test(flatcc_refmap_insert(&refmap, &a, 42) == 42); + test(flatcc_refmap_find(&refmap, &a) == 42); + test(flatcc_refmap_find(&refmap, &b) == flatcc_refmap_not_found); + test(flatcc_refmap_find(&refmap, &c) == flatcc_refmap_not_found); + test(flatcc_refmap_insert(&refmap, &a, 42) == 42); + test(flatcc_refmap_find(&refmap, &a) == 42); + test(refmap.count == 1); + test(flatcc_refmap_insert(&refmap, &a, 43) == 43); + test(flatcc_refmap_find(&refmap, &a) == 43); + test(refmap.count == 1); + test(flatcc_refmap_insert(&refmap, &b, -10) == -10); + test(flatcc_refmap_insert(&refmap, &c, 100) == 100); + test(refmap.count == 3); + test(flatcc_refmap_find(&refmap, &a) == 43); + test(flatcc_refmap_find(&refmap, &b) == -10); + test(flatcc_refmap_find(&refmap, &c) == 100); + + test(flatcc_refmap_insert(&refmap, 0, 1000) == 1000); + test(flatcc_refmap_find(&refmap, 0) == 0); + test(refmap.count == 3); + + test(flatcc_refmap_insert(&refmap, &b, 0) == 0); + test(flatcc_refmap_find(&refmap, &b) == 0); + test(refmap.count == 3); + + flatcc_refmap_reset(&refmap); + test(refmap.count == 0); + test(refmap.buckets > 0); + for (i = 0; i < 1000; ++i) { + test(flatcc_refmap_insert(&refmap, data + i, i + 42) == i + 42); + } + test(refmap.count == 1000); + for (i = 0; i < 1000; ++i) { + test(flatcc_refmap_find(&refmap, data + i) == i + 42); + } + flatcc_refmap_clear(&refmap); + test(refmap.count == 0); + test(refmap.buckets == 0); + test_ok(); + return 0; +} + +#endif /* FLATCC_REFMAP_TEST */ diff --git a/nostrdb/flatcc/support/README b/nostrdb/flatcc/support/README new file mode 100644 index 0000000000..d9f6ec09e3 --- /dev/null +++ b/nostrdb/flatcc/support/README @@ -0,0 +1 @@ +support files mainly used for testing diff --git a/nostrdb/flatcc/support/cdump.h b/nostrdb/flatcc/support/cdump.h new file mode 100644 index 0000000000..b589362964 --- /dev/null +++ b/nostrdb/flatcc/support/cdump.h @@ -0,0 +1,38 @@ +#ifndef CDUMP_H +#define CDUMP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Generates a constant a C byte array. */ +static void cdump(const char *name, void *addr, size_t len, FILE *fp) { + unsigned int i; + unsigned char *pc = (unsigned char*)addr; + + // Output description if given. + name = name ? name : "dump"; + fprintf(fp, "const unsigned char %s[] = {", name); + + // Process every byte in the data. + for (i = 0; i < (unsigned int)len; i++) { + // Multiple of 16 means new line (with line offset). + + if ((i % 16) == 0) { + fprintf(fp, "\n "); + } else if ((i % 8) == 0) { + fprintf(fp, " "); + } + + fprintf(fp, " 0x%02x,", pc[i]); + } + fprintf(fp, "\n};\n"); +} + +#ifdef __cplusplus +} +#endif + +#endif /* CDUMP_H */ diff --git a/nostrdb/flatcc/support/elapsed.h b/nostrdb/flatcc/support/elapsed.h new file mode 100644 index 0000000000..ba3bd73334 --- /dev/null +++ b/nostrdb/flatcc/support/elapsed.h @@ -0,0 +1,73 @@ +#ifndef ELAPSED_H +#define ELAPSED_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Based on http://stackoverflow.com/a/8583395 */ +#if !defined(_WIN32) +#include +static double elapsed_realtime(void) { // returns 0 seconds first time called + static struct timeval t0; + struct timeval tv; + gettimeofday(&tv, 0); + if (!t0.tv_sec) + t0 = tv; + return (double)(tv.tv_sec - t0.tv_sec) + (double)(tv.tv_usec - t0.tv_usec) / 1e6; +} +#else +#include +#ifndef FatalError +#define FatalError(s) do { perror(s); exit(-1); } while(0) +#endif +static double elapsed_realtime(void) { // granularity about 50 microsecs on my machine + static LARGE_INTEGER freq, start; + LARGE_INTEGER count; + if (!QueryPerformanceCounter(&count)) + FatalError("QueryPerformanceCounter"); + if (!freq.QuadPart) { // one time initialization + if (!QueryPerformanceFrequency(&freq)) + FatalError("QueryPerformanceFrequency"); + start = count; + } + return (double)(count.QuadPart - start.QuadPart) / freq.QuadPart; +} +#endif + +/* end Based on stackoverflow */ + +static int show_benchmark(const char *descr, double t1, double t2, size_t size, int rep, const char *reptext) +{ + double tdiff = t2 - t1; + double nstime; + + printf("operation: %s\n", descr); + printf("elapsed time: %.3f (s)\n", tdiff); + printf("iterations: %d\n", rep); + printf("size: %lu (bytes)\n", (unsigned long)size); + printf("bandwidth: %.3f (MB/s)\n", (double)rep * (double)size / 1e6 / tdiff); + printf("throughput in ops per sec: %.3f\n", rep / tdiff); + if (reptext && rep != 1) { + printf("throughput in %s ops per sec: %.3f\n", reptext, 1 / tdiff); + } + nstime = tdiff * 1e9 / rep; + if (nstime < 1000) { + printf("time per op: %.3f (ns)\n", nstime); + } else if (nstime < 1e6) { + printf("time per op: %.3f (us)\n", nstime / 1000); + } else if (nstime < 1e9) { + printf("time per op: %.3f (ms)\n", nstime / 1e6); + } else { + printf("time per op: %.3f (s)\n", nstime / 1e9); + } + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* ELAPSED_H */ diff --git a/nostrdb/flatcc/support/hexdump.h b/nostrdb/flatcc/support/hexdump.h new file mode 100644 index 0000000000..7b6f9b8852 --- /dev/null +++ b/nostrdb/flatcc/support/hexdump.h @@ -0,0 +1,47 @@ +#ifndef HEXDUMP_H +#define HEXDUMP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Based on: http://stackoverflow.com/a/7776146 */ +static void hexdump(const char *desc, const void *addr, size_t len, FILE *fp) { + unsigned int i; + unsigned char buf[17]; + const unsigned char *pc = (const unsigned char*)addr; + + /* Output description if given. */ + if (desc != NULL) fprintf(fp, "%s:\n", desc); + + for (i = 0; i < (unsigned int)len; i++) { + + if ((i % 16) == 0) { + if (i != 0) fprintf(fp, " |%s|\n", buf); + fprintf(fp, "%08x ", i); + } else if ((i % 8) == 0) { + fprintf(fp, " "); + } + fprintf(fp, " %02x", pc[i]); + if ((pc[i] < 0x20) || (pc[i] > 0x7e)) { + buf[i % 16] = '.'; + } else { + buf[i % 16] = pc[i]; + } + buf[(i % 16) + 1] = '\0'; + } + if (i % 16 <= 8 && i % 16 != 0) fprintf(fp, " "); + while ((i % 16) != 0) { + fprintf(fp, " "); + i++; + } + fprintf(fp, " |%s|\n", buf); +} + +#ifdef __cplusplus +} +#endif + +#endif /* HEXDUMP_H */ diff --git a/nostrdb/flatcc/support/readfile.h b/nostrdb/flatcc/support/readfile.h new file mode 100644 index 0000000000..209875fb15 --- /dev/null +++ b/nostrdb/flatcc/support/readfile.h @@ -0,0 +1,66 @@ +#ifndef READFILE_H +#define READFILE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +static char *readfile(const char *filename, size_t max_size, size_t *size_out) +{ + FILE *fp; + long k; + size_t size, pos, n, _out; + char *buf; + + size_out = size_out ? size_out : &_out; + + fp = fopen(filename, "rb"); + size = 0; + buf = 0; + + if (!fp) { + goto fail; + } + fseek(fp, 0L, SEEK_END); + k = ftell(fp); + if (k < 0) goto fail; + size = (size_t)k; + *size_out = size; + if (max_size > 0 && size > max_size) { + goto fail; + } + rewind(fp); + buf = (char *)malloc(size ? size : 1); + if (!buf) { + goto fail; + } + pos = 0; + while ((n = fread(buf + pos, 1, size - pos, fp))) { + pos += n; + } + if (pos != size) { + goto fail; + } + fclose(fp); + *size_out = size; + return buf; + +fail: + if (fp) { + fclose(fp); + } + if (buf) { + free(buf); + } + *size_out = size; + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* READFILE_H */ diff --git a/nostrdb/flatcc/verifier.c b/nostrdb/flatcc/verifier.c new file mode 100644 index 0000000000..9c43bf613d --- /dev/null +++ b/nostrdb/flatcc/verifier.c @@ -0,0 +1,617 @@ +/* + * Runtime support for verifying flatbuffers. + * + * Depends mutually on generated verifier functions for table types that + * call into this library. + */ +#include + +#include "flatcc/flatcc_rtconfig.h" +#include "flatcc/flatcc_flatbuffers.h" +#include "flatcc/flatcc_verifier.h" +#include "flatcc/flatcc_identifier.h" + +/* Customization for testing. */ +#if FLATCC_DEBUG_VERIFY +#define FLATCC_VERIFIER_ASSERT_ON_ERROR 1 +#include +#define FLATCC_VERIFIER_ASSERT(cond, reason) \ + if (!(cond)) { fprintf(stderr, "verifier assert: %s\n", \ + flatcc_verify_error_string(reason)); FLATCC_ASSERT(0); return reason; } +#endif + +#if FLATCC_TRACE_VERIFY +#include +#define trace_verify(s, p) \ + fprintf(stderr, "trace verify: %s: 0x%02x\n", (s), (unsigned)(size_t)(p)); +#else +#define trace_verify(s, p) ((void)0) +#endif + +/* The runtime library does not use the global config file. */ + +/* This is a guideline, not an exact measure. */ +#ifndef FLATCC_VERIFIER_MAX_LEVELS +#define FLATCC_VERIFIER_MAX_LEVELS 100 +#endif + +#ifndef FLATCC_VERIFIER_ASSERT_ON_ERROR +#define FLATCC_VERIFIER_ASSERT_ON_ERROR 0 +#endif + +/* + * Generally a check should tell if a buffer is valid or not such + * that runtime can take appropriate actions rather than crash, + * also in debug, but assertions are helpful in debugging a problem. + * + * This must be compiled into the debug runtime library to take effect. + */ +#ifndef FLATCC_VERIFIER_ASSERT_ON_ERROR +#define FLATCC_VERIFIER_ASSERT_ON_ERROR 1 +#endif + +/* May be redefined for logging purposes. */ +#ifndef FLATCC_VERIFIER_ASSERT +#define FLATCC_VERIFIER_ASSERT(cond, reason) FLATCC_ASSERT(cond) +#endif + +#if FLATCC_VERIFIER_ASSERT_ON_ERROR +#define flatcc_verify(cond, reason) if (!(cond)) { FLATCC_VERIFIER_ASSERT(cond, reason); return reason; } +#else +#define flatcc_verify(cond, reason) if (!(cond)) { return reason; } +#endif + + +#define uoffset_t flatbuffers_uoffset_t +#define soffset_t flatbuffers_soffset_t +#define voffset_t flatbuffers_voffset_t +#define utype_t flatbuffers_utype_t +#define thash_t flatbuffers_thash_t + +#define uoffset_size sizeof(uoffset_t) +#define soffset_size sizeof(soffset_t) +#define voffset_size sizeof(voffset_t) +#define utype_size sizeof(utype_t) +#define thash_size sizeof(thash_t) +#define offset_size uoffset_size + +const char *flatcc_verify_error_string(int err) +{ + switch (err) { +#define XX(no, str) \ + case flatcc_verify_error_##no: \ + return str; + FLATCC_VERIFY_ERROR_MAP(XX) +#undef XX + default: + return "unknown"; + } +} + +/* `cond` may have side effects. */ +#define verify(cond, reason) do { int c = (cond); flatcc_verify(c, reason); } while(0) + +/* + * Identify checks related to runtime conditions (buffer size and + * alignment) as seperate from those related to buffer content. + */ +#define verify_runtime(cond, reason) verify(cond, reason) + +#define check_result(x) if (x) { return (x); } + +#define check_field(td, id, required, base) do { \ + int ret = get_offset_field(td, id, required, &base); \ + if (ret || !base) { return ret; }} while (0) + +static inline uoffset_t read_uoffset(const void *p, uoffset_t base) +{ + return __flatbuffers_uoffset_read_from_pe((uint8_t *)p + base); +} + +static inline thash_t read_thash_identifier(const char *identifier) +{ + return flatbuffers_type_hash_from_string(identifier); +} + +static inline thash_t read_thash(const void *p, uoffset_t base) +{ + return __flatbuffers_thash_read_from_pe((uint8_t *)p + base); +} + +static inline voffset_t read_voffset(const void *p, uoffset_t base) +{ + return __flatbuffers_voffset_read_from_pe((uint8_t *)p + base); +} + +static inline int check_header(uoffset_t end, uoffset_t base, uoffset_t offset) +{ + uoffset_t k = base + offset; + + if (uoffset_size <= voffset_size && k + offset_size < k) { + return 0; + } + + /* The `k > base` rather than `k >= base` is to avoid null offsets. */ + return k > base && k + offset_size <= end && !(k & (offset_size - 1)); +} + +static inline int check_aligned_header(uoffset_t end, uoffset_t base, uoffset_t offset, uint16_t align) +{ + uoffset_t k = base + offset; + + if (uoffset_size <= voffset_size && k + offset_size < k) { + return 0; + } + /* Alignment refers to element 0 and header must also be aligned. */ + align = align < uoffset_size ? uoffset_size : align; + + /* Note to self: the builder can also use the mask OR trick to propagate `min_align`. */ + return k > base && k + offset_size <= end && !((k + offset_size) & ((offset_size - 1) | (align - 1u))); +} + +static inline int verify_struct(uoffset_t end, uoffset_t base, uoffset_t offset, uoffset_t size, uint16_t align) +{ + /* Structs can have zero size so `end` is a valid value. */ + if (offset == 0 || base + offset > end) { + return flatcc_verify_error_offset_out_of_range; + } + base += offset; + verify(base + size >= base, flatcc_verify_error_struct_size_overflow); + verify(base + size <= end, flatcc_verify_error_struct_out_of_range); + verify (!(base & (align - 1u)), flatcc_verify_error_struct_unaligned); + return flatcc_verify_ok; +} + +static inline voffset_t read_vt_entry(flatcc_table_verifier_descriptor_t *td, voffset_t id) +{ + voffset_t vo = (id + 2u) * sizeof(voffset_t); + + /* Assumes tsize has been verified for alignment. */ + if (vo >= td->vsize) { + return 0; + } + return read_voffset(td->vtable, vo); +} + +static inline const void *get_field_ptr(flatcc_table_verifier_descriptor_t *td, voffset_t id) +{ + voffset_t vte = read_vt_entry(td, id); + return vte ? (const uint8_t *)td->buf + td->table + vte : 0; +} + +static int verify_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required, uoffset_t size, uint16_t align) +{ + uoffset_t k, k2; + voffset_t vte; + uoffset_t base = (uoffset_t)(size_t)td->buf; + + + /* + * Otherwise range check assumptions break, and normal access code likely also. + * We don't require voffset_size < uoffset_size, but some checks are faster if true. + */ + FLATCC_ASSERT(uoffset_size >= voffset_size); + FLATCC_ASSERT(soffset_size == uoffset_size); + + vte = read_vt_entry(td, id); + if (!vte) { + verify(!required, flatcc_verify_error_required_field_missing); + return flatcc_verify_ok; + } + trace_verify("table buffer", td->buf); + trace_verify("table", td->table); + trace_verify("id", id); + trace_verify("vte", vte); + + /* + * Note that we don't add td.table to k and we test against table + * size not table end or buffer end. Otherwise it would not be safe + * to optimized out the k <= k2 check for normal uoffset and voffset + * configurations. + */ + k = vte; + k2 = k + size; + verify(k2 <= td->tsize, flatcc_verify_error_table_field_out_of_range); + /* This normally optimizes to nop. */ + verify(uoffset_size > voffset_size || k <= k2, flatcc_verify_error_table_field_size_overflow); + trace_verify("table + vte", vte + td->table); + k += td->table + base; + trace_verify("entry: buf + table + vte", k); + trace_verify("align", align); + trace_verify("align masked entry", k & (align - 1u)); + verify(!(k & (align - 1u)), flatcc_verify_error_table_field_not_aligned); + /* We assume the table size has already been verified. */ + return flatcc_verify_ok; +} + +static int get_offset_field(flatcc_table_verifier_descriptor_t *td, voffset_t id, int required, uoffset_t *out) +{ + uoffset_t k, k2; + voffset_t vte; + + vte = read_vt_entry(td, id); + if (!vte) { + *out = 0; + if (required) { + return flatcc_verify_error_required_field_missing; + } + /* Missing, but not invalid. */ + return flatcc_verify_ok; + } + /* + * Note that we don't add td.table to k and we test against table + * size not table end or buffer end. Otherwise it would not be safe + * to optimized out the k <= k2 check for normal uoffset and voffset + * configurations. + */ + k = vte; + k2 = k + offset_size; + verify(k2 <= td->tsize, flatcc_verify_error_table_field_out_of_range); + /* This normally optimizes to nop. */ + verify(uoffset_size > voffset_size || k <= k2, flatcc_verify_error_table_field_size_overflow); + k += td->table; + verify(!(k & (offset_size - 1u)), flatcc_verify_error_table_field_not_aligned); + /* We assume the table size has already been verified. */ + *out = k; + return flatcc_verify_ok; +} + +static inline int verify_string(const void *buf, uoffset_t end, uoffset_t base, uoffset_t offset) +{ + uoffset_t n; + + verify(check_header(end, base, offset), flatcc_verify_error_string_header_out_of_range_or_unaligned); + base += offset; + n = read_uoffset(buf, base); + base += offset_size; + verify(end - base > n, flatcc_verify_error_string_out_of_range); + verify(((uint8_t *)buf + base)[n] == 0, flatcc_verify_error_string_not_zero_terminated); + return flatcc_verify_ok; +} + +/* + * Keep interface somwewhat similar ot flatcc_builder_start_vector. + * `max_count` is a precomputed division to manage overflow check on vector length. + */ +static inline int verify_vector(const void *buf, uoffset_t end, uoffset_t base, uoffset_t offset, uoffset_t elem_size, uint16_t align, uoffset_t max_count) +{ + uoffset_t n; + + verify(check_aligned_header(end, base, offset, align), flatcc_verify_error_vector_header_out_of_range_or_unaligned); + base += offset; + n = read_uoffset(buf, base); + base += offset_size; + /* `n * elem_size` can overflow uncontrollably otherwise. */ + verify(n <= max_count, flatcc_verify_error_vector_count_exceeds_representable_vector_size); + verify(end - base >= n * elem_size, flatcc_verify_error_vector_out_of_range); + return flatcc_verify_ok; +} + +static inline int verify_string_vector(const void *buf, uoffset_t end, uoffset_t base, uoffset_t offset) +{ + uoffset_t i, n; + + check_result(verify_vector(buf, end, base, offset, offset_size, offset_size, FLATBUFFERS_COUNT_MAX(offset_size))); + base += offset; + n = read_uoffset(buf, base); + base += offset_size; + for (i = 0; i < n; ++i, base += offset_size) { + check_result(verify_string(buf, end, base, read_uoffset(buf, base))); + } + return flatcc_verify_ok; +} + +static inline int verify_table(const void *buf, uoffset_t end, uoffset_t base, uoffset_t offset, + int ttl, flatcc_table_verifier_f tvf) +{ + uoffset_t vbase, vend; + flatcc_table_verifier_descriptor_t td; + + verify((td.ttl = ttl - 1), flatcc_verify_error_max_nesting_level_reached); + verify(check_header(end, base, offset), flatcc_verify_error_table_header_out_of_range_or_unaligned); + td.table = base + offset; + /* Read vtable offset - it is signed, but we want it unsigned, assuming 2's complement works. */ + vbase = td.table - read_uoffset(buf, td.table); + verify((soffset_t)vbase >= 0 && !(vbase & (voffset_size - 1)), flatcc_verify_error_vtable_offset_out_of_range_or_unaligned); + verify(vbase + voffset_size <= end, flatcc_verify_error_vtable_header_out_of_range); + /* Read vtable size. */ + td.vsize = read_voffset(buf, vbase); + vend = vbase + td.vsize; + verify(vend <= end && !(td.vsize & (voffset_size - 1)), flatcc_verify_error_vtable_size_out_of_range_or_unaligned); + /* Optimizes away overflow check if uoffset_t is large enough. */ + verify(uoffset_size > voffset_size || vend >= vbase, flatcc_verify_error_vtable_size_overflow); + + verify(td.vsize >= 2 * voffset_size, flatcc_verify_error_vtable_header_too_small); + /* Read table size. */ + td.tsize = read_voffset(buf, vbase + voffset_size); + verify(end - td.table >= td.tsize, flatcc_verify_error_table_size_out_of_range); + td.vtable = (uint8_t *)buf + vbase; + td.buf = buf; + td.end = end; + return tvf(&td); +} + +static inline int verify_table_vector(const void *buf, uoffset_t end, uoffset_t base, uoffset_t offset, int ttl, flatcc_table_verifier_f tvf) +{ + uoffset_t i, n; + + verify(ttl-- > 0, flatcc_verify_error_max_nesting_level_reached); + check_result(verify_vector(buf, end, base, offset, offset_size, offset_size, FLATBUFFERS_COUNT_MAX(offset_size))); + base += offset; + n = read_uoffset(buf, base); + base += offset_size; + for (i = 0; i < n; ++i, base += offset_size) { + check_result(verify_table(buf, end, base, read_uoffset(buf, base), ttl, tvf)); + } + return flatcc_verify_ok; +} + +static inline int verify_union_vector(const void *buf, uoffset_t end, uoffset_t base, uoffset_t offset, + uoffset_t count, const utype_t *types, int ttl, flatcc_union_verifier_f uvf) +{ + uoffset_t i, n, elem; + flatcc_union_verifier_descriptor_t ud; + + verify(ttl-- > 0, flatcc_verify_error_max_nesting_level_reached); + check_result(verify_vector(buf, end, base, offset, offset_size, offset_size, FLATBUFFERS_COUNT_MAX(offset_size))); + base += offset; + n = read_uoffset(buf, base); + verify(n == count, flatcc_verify_error_union_vector_length_mismatch); + base += offset_size; + + ud.buf = buf; + ud.end = end; + ud.ttl = ttl; + + for (i = 0; i < n; ++i, base += offset_size) { + /* Table vectors can never be null, but unions can when the type is NONE. */ + elem = read_uoffset(buf, base); + if (elem == 0) { + verify(types[i] == 0, flatcc_verify_error_union_element_absent_without_type_NONE); + } else { + verify(types[i] != 0, flatcc_verify_error_union_element_present_with_type_NONE); + ud.type = types[i]; + ud.base = base; + ud.offset = elem; + check_result(uvf(&ud)); + } + } + return flatcc_verify_ok; +} + +int flatcc_verify_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, size_t size, uint16_t align) +{ + check_result(verify_field(td, id, 0, (uoffset_t)size, align)); + return flatcc_verify_ok; +} + +int flatcc_verify_string_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required) +{ + uoffset_t base; + + check_field(td, id, required, base); + return verify_string(td->buf, td->end, base, read_uoffset(td->buf, base)); +} + +int flatcc_verify_vector_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required, size_t elem_size, uint16_t align, size_t max_count) +{ + uoffset_t base; + + check_field(td, id, required, base); + return verify_vector(td->buf, td->end, base, read_uoffset(td->buf, base), + (uoffset_t)elem_size, align, (uoffset_t)max_count); +} + +int flatcc_verify_string_vector_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required) +{ + uoffset_t base; + + check_field(td, id, required, base); + return verify_string_vector(td->buf, td->end, base, read_uoffset(td->buf, base)); +} + +int flatcc_verify_table_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required, flatcc_table_verifier_f tvf) +{ + uoffset_t base; + + check_field(td, id, required, base); + return verify_table(td->buf, td->end, base, read_uoffset(td->buf, base), td->ttl, tvf); +} + +int flatcc_verify_table_vector_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required, flatcc_table_verifier_f tvf) +{ + uoffset_t base; + + check_field(td, id, required, base); + return verify_table_vector(td->buf, td->end, base, read_uoffset(td->buf, base), td->ttl, tvf); +} + +int flatcc_verify_union_table(flatcc_union_verifier_descriptor_t *ud, flatcc_table_verifier_f *tvf) +{ + return verify_table(ud->buf, ud->end, ud->base, ud->offset, ud->ttl, tvf); +} + +int flatcc_verify_union_struct(flatcc_union_verifier_descriptor_t *ud, size_t size, uint16_t align) +{ + return verify_struct(ud->end, ud->base, ud->offset, (uoffset_t)size, align); +} + +int flatcc_verify_union_string(flatcc_union_verifier_descriptor_t *ud) +{ + return verify_string(ud->buf, ud->end, ud->base, ud->offset); +} + +int flatcc_verify_buffer_header(const void *buf, size_t bufsiz, const char *fid) +{ + thash_t id, id2; + + verify_runtime(!(((size_t)buf) & (offset_size - 1)), flatcc_verify_error_runtime_buffer_header_not_aligned); + /* -8 ensures no scalar or offset field size can overflow. */ + verify_runtime(bufsiz <= FLATBUFFERS_UOFFSET_MAX - 8, flatcc_verify_error_runtime_buffer_size_too_large); + /* + * Even if we specify no fid, the user might later. Therefore + * require space for it. Not all buffer generators will take this + * into account, so it is possible to fail an otherwise valid buffer + * - but such buffers aren't safe. + */ + verify(bufsiz >= offset_size + FLATBUFFERS_IDENTIFIER_SIZE, flatcc_verify_error_buffer_header_too_small); + if (fid != 0) { + id2 = read_thash_identifier(fid); + id = read_thash(buf, offset_size); + verify(id2 == 0 || id == id2, flatcc_verify_error_identifier_mismatch); + } + return flatcc_verify_ok; +} + +int flatcc_verify_typed_buffer_header(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + thash_t id, id2; + + verify_runtime(!(((size_t)buf) & (offset_size - 1)), flatcc_verify_error_runtime_buffer_header_not_aligned); + /* -8 ensures no scalar or offset field size can overflow. */ + verify_runtime(bufsiz <= FLATBUFFERS_UOFFSET_MAX - 8, flatcc_verify_error_runtime_buffer_size_too_large); + /* + * Even if we specify no fid, the user might later. Therefore + * require space for it. Not all buffer generators will take this + * into account, so it is possible to fail an otherwise valid buffer + * - but such buffers aren't safe. + */ + verify(bufsiz >= offset_size + FLATBUFFERS_IDENTIFIER_SIZE, flatcc_verify_error_buffer_header_too_small); + if (thash != 0) { + id2 = thash; + id = read_thash(buf, offset_size); + verify(id2 == 0 || id == id2, flatcc_verify_error_identifier_mismatch); + } + return flatcc_verify_ok; +} + +int flatcc_verify_struct_as_root(const void *buf, size_t bufsiz, const char *fid, size_t size, uint16_t align) +{ + check_result(flatcc_verify_buffer_header(buf, bufsiz, fid)); + return verify_struct((uoffset_t)bufsiz, 0, read_uoffset(buf, 0), (uoffset_t)size, align); +} + +int flatcc_verify_struct_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash, size_t size, uint16_t align) +{ + check_result(flatcc_verify_typed_buffer_header(buf, bufsiz, thash)); + return verify_struct((uoffset_t)bufsiz, 0, read_uoffset(buf, 0), (uoffset_t)size, align); +} + +int flatcc_verify_table_as_root(const void *buf, size_t bufsiz, const char *fid, flatcc_table_verifier_f *tvf) +{ + check_result(flatcc_verify_buffer_header(buf, (uoffset_t)bufsiz, fid)); + return verify_table(buf, (uoffset_t)bufsiz, 0, read_uoffset(buf, 0), FLATCC_VERIFIER_MAX_LEVELS, tvf); +} + +int flatcc_verify_table_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash, flatcc_table_verifier_f *tvf) +{ + check_result(flatcc_verify_typed_buffer_header(buf, (uoffset_t)bufsiz, thash)); + return verify_table(buf, (uoffset_t)bufsiz, 0, read_uoffset(buf, 0), FLATCC_VERIFIER_MAX_LEVELS, tvf); +} + +int flatcc_verify_struct_as_nested_root(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required, const char *fid, size_t size, uint16_t align) +{ + const uoffset_t *buf; + uoffset_t bufsiz; + + check_result(flatcc_verify_vector_field(td, id, required, align, 1, FLATBUFFERS_COUNT_MAX(1))); + if (0 == (buf = get_field_ptr(td, id))) { + return flatcc_verify_ok; + } + buf = (const uoffset_t *)((size_t)buf + read_uoffset(buf, 0)); + bufsiz = read_uoffset(buf, 0); + ++buf; + return flatcc_verify_struct_as_root(buf, bufsiz, fid, size, align); +} + +int flatcc_verify_table_as_nested_root(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required, const char *fid, + uint16_t align, flatcc_table_verifier_f tvf) +{ + const uoffset_t *buf; + uoffset_t bufsiz; + + check_result(flatcc_verify_vector_field(td, id, required, align, 1, FLATBUFFERS_COUNT_MAX(1))); + if (0 == (buf = get_field_ptr(td, id))) { + return flatcc_verify_ok; + } + buf = (const uoffset_t *)((size_t)buf + read_uoffset(buf, 0)); + bufsiz = read_uoffset(buf, 0); + ++buf; + /* + * Don't verify nested buffers identifier - information is difficult to get and + * might not be what is desired anyway. User can do it later. + */ + check_result(flatcc_verify_buffer_header(buf, bufsiz, fid)); + return verify_table(buf, bufsiz, 0, read_uoffset(buf, 0), td->ttl, tvf); +} + +int flatcc_verify_union_field(flatcc_table_verifier_descriptor_t *td, + voffset_t id, int required, flatcc_union_verifier_f uvf) +{ + voffset_t vte_type, vte_table; + const uint8_t *type; + uoffset_t base; + flatcc_union_verifier_descriptor_t ud; + + if (0 == (vte_type = read_vt_entry(td, id - 1))) { + vte_table = read_vt_entry(td, id); + verify(vte_table == 0, flatcc_verify_error_union_cannot_have_a_table_without_a_type); + verify(!required, flatcc_verify_error_type_field_absent_from_required_union_field); + return flatcc_verify_ok; + } + /* No need to check required here. */ + check_result(verify_field(td, id - 1, 0, 1, 1)); + /* Only now is it safe to read the type. */ + vte_table = read_vt_entry(td, id); + type = (const uint8_t *)td->buf + td->table + vte_type; + verify(*type || vte_table == 0, flatcc_verify_error_union_type_NONE_cannot_have_a_value); + + if (*type == 0) { + return flatcc_verify_ok; + } + check_field(td, id, required, base); + ud.buf = td->buf; + ud.end = td->end; + ud.ttl = td->ttl; + ud.base = base; + ud.offset = read_uoffset(td->buf, base); + ud.type = *type; + return uvf(&ud); +} + +int flatcc_verify_union_vector_field(flatcc_table_verifier_descriptor_t *td, + flatbuffers_voffset_t id, int required, flatcc_union_verifier_f uvf) +{ + voffset_t vte_type, vte_table; + const uoffset_t *buf; + const utype_t *types; + uoffset_t count, base; + + if (0 == (vte_type = read_vt_entry(td, id - 1))) { + if (0 == (vte_table = read_vt_entry(td, id))) { + verify(!required, flatcc_verify_error_type_field_absent_from_required_union_vector_field); + } + } + check_result(flatcc_verify_vector_field(td, id - 1, required, + utype_size, utype_size, FLATBUFFERS_COUNT_MAX(utype_size))); + if (0 == (buf = get_field_ptr(td, id - 1))) { + return flatcc_verify_ok; + } + buf = (const uoffset_t *)((size_t)buf + read_uoffset(buf, 0)); + count = read_uoffset(buf, 0); + ++buf; + types = (utype_t *)buf; + + check_field(td, id, required, base); + return verify_union_vector(td->buf, td->end, base, read_uoffset(td->buf, base), + count, types, td->ttl, uvf); +} diff --git a/nostrdb/jsmn.h b/nostrdb/jsmn.h index 2d8f591ec2..d263317d20 100644 --- a/nostrdb/jsmn.h +++ b/nostrdb/jsmn.h @@ -100,7 +100,7 @@ JSMN_API void jsmn_init(jsmn_parser *parser); * a single JSON object. */ JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, - jsmntok_t *tokens, const unsigned int num_tokens); + jsmntok_t *tokens, const unsigned int num_tokens, int stop_at_id); #ifndef JSMN_HEADER /** @@ -269,12 +269,13 @@ static int jsmn_parse_string(jsmn_parser *parser, const char *js, * Parse JSON string and fill tokens. */ JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, - jsmntok_t *tokens, const unsigned int num_tokens) { - int r; - int i; + jsmntok_t *tokens, const unsigned int num_tokens, int stop_at_id) { + int r, i, idkey; jsmntok_t *token; int count = parser->toknext; + idkey = 0; + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { char c; jsmntype_t type; @@ -370,6 +371,22 @@ JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, if (parser->toksuper != -1 && tokens != NULL) { tokens[parser->toksuper].size++; } + + // big hack. resumable parsing when encountering the id field + if (stop_at_id) { + token = &tokens[parser->toknext-1]; + if (idkey == 1 && (token->end - token->start) == 64) { + //printf("jsmn: found id '%.*s'\n", token->end - token->start, js + token->start); + parser->pos++; + return -42; + } else if (idkey == 0 && (token->end - token->start) == 2 && + (js + token->start)[0] == 'i' && + (js + token->start)[1] == 'd') { + //printf("jsmn: found id key\n"); + idkey = 1; + } + } + break; case '\t': case '\r': diff --git a/nostrdb/lmdb.h b/nostrdb/lmdb.h new file mode 100644 index 0000000000..ff03c224f2 --- /dev/null +++ b/nostrdb/lmdb.h @@ -0,0 +1,1608 @@ +/** @file lmdb.h + * @brief Lightning memory-mapped database library + * + * @mainpage Lightning Memory-Mapped Database Manager (LMDB) + * + * @section intro_sec Introduction + * LMDB is a Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. The entire database is exposed + * in a memory map, and all data fetches return data directly + * from the mapped memory, so no malloc's or memcpy's occur during + * data fetches. As such, the library is extremely simple because it + * requires no page caching layer of its own, and it is extremely high + * performance and memory-efficient. It is also fully transactional with + * full ACID semantics, and when the memory map is read-only, the + * database integrity cannot be corrupted by stray pointer writes from + * application code. + * + * The library is fully thread-aware and supports concurrent read/write + * access from multiple processes and threads. Data pages use a copy-on- + * write strategy so no active data pages are ever overwritten, which + * also provides resistance to corruption and eliminates the need of any + * special recovery procedures after a system crash. Writes are fully + * serialized; only one write transaction may be active at a time, which + * guarantees that writers can never deadlock. The database structure is + * multi-versioned so readers run with no locks; writers cannot block + * readers, and readers don't block writers. + * + * Unlike other well-known database mechanisms which use either write-ahead + * transaction logs or append-only data writes, LMDB requires no maintenance + * during operation. Both write-ahead loggers and append-only databases + * require periodic checkpointing and/or compaction of their log or database + * files otherwise they grow without bound. LMDB tracks free pages within + * the database and re-uses them for new write operations, so the database + * size does not grow without bound in normal use. + * + * The memory map can be used as a read-only or read-write map. It is + * read-only by default as this provides total immunity to corruption. + * Using read-write mode offers much higher write performance, but adds + * the possibility for stray application writes thru pointers to silently + * corrupt the database. Of course if your application code is known to + * be bug-free (...) then this is not an issue. + * + * If this is your first time using a transactional embedded key/value + * store, you may find the \ref starting page to be helpful. + * + * @section caveats_sec Caveats + * Troubleshooting the lock file, plus semaphores on BSD systems: + * + * - A broken lockfile can cause sync issues. + * Stale reader transactions left behind by an aborted program + * cause further writes to grow the database quickly, and + * stale locks can block further operation. + * + * Fix: Check for stale readers periodically, using the + * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. + * Stale writers will be cleared automatically on some systems: + * - Windows - automatic + * - Linux, systems using POSIX mutexes with Robust option - automatic + * - not on BSD, systems using POSIX semaphores. + * Otherwise just make all programs using the database close it; + * the lockfile is always reset on first open of the environment. + * + * - On BSD systems or others configured with MDB_USE_POSIX_SEM, + * startup can fail due to semaphores owned by another userid. + * + * Fix: Open and close the database as the user which owns the + * semaphores (likely last user) or as root, while no other + * process is using the database. + * + * Restrictions/caveats (in addition to those listed for some functions): + * + * - Only the database owner should normally use the database on + * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. + * Multiple users can cause startup to fail later, as noted above. + * + * - There is normally no pure read-only mode, since readers need write + * access to locks and lock file. Exceptions: On read-only filesystems + * or with the #MDB_NOLOCK flag described under #mdb_env_open(). + * + * - An LMDB configuration will often reserve considerable \b unused + * memory address space and maybe file size for future growth. + * This does not use actual memory or disk space, but users may need + * to understand the difference so they won't be scared off. + * + * - By default, in versions before 0.9.10, unused portions of the data + * file might receive garbage data from memory freed by other code. + * (This does not happen when using the #MDB_WRITEMAP flag.) As of + * 0.9.10 the default behavior is to initialize such memory before + * writing to the data file. Since there may be a slight performance + * cost due to this initialization, applications may disable it using + * the #MDB_NOMEMINIT flag. Applications handling sensitive data + * which must not be written should not use this flag. This flag is + * irrelevant when using #MDB_WRITEMAP. + * + * - A thread can only use one transaction at a time, plus any child + * transactions. Each transaction belongs to one thread. See below. + * The #MDB_NOTLS flag changes this for read-only transactions. + * + * - Use an MDB_env* in the process which opened it, not after fork(). + * + * - Do not have open an LMDB database twice in the same process at + * the same time. Not even from a plain open() call - close()ing it + * breaks fcntl() advisory locking. (It is OK to reopen it after + * fork() - exec*(), since the lockfile has FD_CLOEXEC set.) + * + * - Avoid long-lived transactions. Read transactions prevent + * reuse of pages freed by newer write transactions, thus the + * database can grow quickly. Write transactions prevent + * other write transactions, since writes are serialized. + * + * - Avoid suspending a process with active transactions. These + * would then be "long-lived" as above. Also read transactions + * suspended when writers commit could sometimes see wrong data. + * + * ...when several processes can use a database concurrently: + * + * - Avoid aborting a process with an active transaction. + * The transaction becomes "long-lived" as above until a check + * for stale readers is performed or the lockfile is reset, + * since the process may not remove it from the lockfile. + * + * This does not apply to write transactions if the system clears + * stale writers, see above. + * + * - If you do that anyway, do a periodic check for stale readers. Or + * close the environment once in a while, so the lockfile can get reset. + * + * - Do not use LMDB databases on remote filesystems, even between + * processes on the same host. This breaks flock() on some OSes, + * possibly memory map sync, and certainly sync between programs + * on different hosts. + * + * - Opening a database can fail if another process is opening or + * closing it at exactly the same time. + * + * @author Howard Chu, Symas Corporation. + * + * @copyright Copyright 2011-2021 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * @par Derived From: + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _LMDB_H_ +#define _LMDB_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Unix permissions for creating files, or dummy definition for Windows */ +#ifdef _MSC_VER +typedef int mdb_mode_t; +#else +typedef mode_t mdb_mode_t; +#endif + +/** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#ifdef _WIN32 +typedef void *mdb_filehandle_t; +#else +typedef int mdb_filehandle_t; +#endif + +/** @defgroup mdb LMDB API + * @{ + * @brief OpenLDAP Lightning Memory-Mapped Database Manager + */ +/** @defgroup Version Version Macros + * @{ + */ +/** Library major version */ +#define MDB_VERSION_MAJOR 0 +/** Library minor version */ +#define MDB_VERSION_MINOR 9 +/** Library patch version */ +#define MDB_VERSION_PATCH 31 + +/** Combine args a,b,c into a single integer for easy version comparisons */ +#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) + +/** The full library version as a single integer */ +#define MDB_VERSION_FULL \ + MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) + +/** The release date of this library version */ +#define MDB_VERSION_DATE "July 10, 2023" + +/** A stringifier for the version info */ +#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" + +/** A helper for the stringifier macro */ +#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) + +/** The full library version as a C string */ +#define MDB_VERSION_STRING \ + MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) +/** @} */ + +/** @brief Opaque structure for a database environment. + * + * A DB environment supports multiple databases, all residing in the same + * shared-memory map. + */ +typedef struct MDB_env MDB_env; + +/** @brief Opaque structure for a transaction handle. + * + * All database operations require a transaction handle. Transactions may be + * read-only or read-write. + */ +typedef struct MDB_txn MDB_txn; + +/** @brief A handle for an individual database in the DB environment. */ +typedef unsigned int MDB_dbi; + +/** @brief Opaque structure for navigating through a database */ +typedef struct MDB_cursor MDB_cursor; + +/** @brief Generic structure used for passing keys and data in and out + * of the database. + * + * Values returned from the database are valid only until a subsequent + * update operation, or the end of the transaction. Do not modify or + * free them, they commonly point into the database itself. + * + * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. + * The same applies to data sizes in databases with the #MDB_DUPSORT flag. + * Other data items can in theory be from 0 to 0xffffffff bytes long. + */ +typedef struct MDB_val { + size_t mv_size; /**< size of the data item */ + void *mv_data; /**< address of the data item */ +} MDB_val; + +/** @brief A callback function used to compare two keys in a database */ +typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); + +/** @brief A callback function used to relocate a position-dependent data item + * in a fixed-address database. + * + * The \b newptr gives the item's desired address in + * the memory map, and \b oldptr gives its previous address. The item's actual + * data resides at the address in \b item. This callback is expected to walk + * through the fields of the record in \b item and modify any + * values based at the \b oldptr address to be relative to the \b newptr address. + * @param[in,out] item The item that is to be relocated. + * @param[in] oldptr The previous address. + * @param[in] newptr The new address to relocate to. + * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). + * @todo This feature is currently unimplemented. + */ +typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); + +/** @defgroup mdb_env Environment Flags + * @{ + */ + /** mmap at a fixed address (experimental) */ +#define MDB_FIXEDMAP 0x01 + /** no environment directory */ +#define MDB_NOSUBDIR 0x4000 + /** don't fsync after commit */ +#define MDB_NOSYNC 0x10000 + /** read only */ +#define MDB_RDONLY 0x20000 + /** don't fsync metapage after commit */ +#define MDB_NOMETASYNC 0x40000 + /** use writable mmap */ +#define MDB_WRITEMAP 0x80000 + /** use asynchronous msync when #MDB_WRITEMAP is used */ +#define MDB_MAPASYNC 0x100000 + /** tie reader locktable slots to #MDB_txn objects instead of to threads */ +#define MDB_NOTLS 0x200000 + /** don't do any locking, caller must manage their own locks */ +#define MDB_NOLOCK 0x400000 + /** don't do readahead (no effect on Windows) */ +#define MDB_NORDAHEAD 0x800000 + /** don't initialize malloc'd memory before writing to datafile */ +#define MDB_NOMEMINIT 0x1000000 +/** @} */ + +/** @defgroup mdb_dbi_open Database Flags + * @{ + */ + /** use reverse string keys */ +#define MDB_REVERSEKEY 0x02 + /** use sorted duplicates */ +#define MDB_DUPSORT 0x04 + /** numeric keys in native byte order: either unsigned int or size_t. + * The keys must all be of the same size. */ +#define MDB_INTEGERKEY 0x08 + /** with #MDB_DUPSORT, sorted dup items have fixed size */ +#define MDB_DUPFIXED 0x10 + /** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */ +#define MDB_INTEGERDUP 0x20 + /** with #MDB_DUPSORT, use reverse string dups */ +#define MDB_REVERSEDUP 0x40 + /** create DB if not already existing */ +#define MDB_CREATE 0x40000 +/** @} */ + +/** @defgroup mdb_put Write Flags + * @{ + */ +/** For put: Don't write if the key already exists. */ +#define MDB_NOOVERWRITE 0x10 +/** Only for #MDB_DUPSORT
+ * For put: don't write if the key and data pair already exist.
+ * For mdb_cursor_del: remove all duplicate data items. + */ +#define MDB_NODUPDATA 0x20 +/** For mdb_cursor_put: overwrite the current key/data pair */ +#define MDB_CURRENT 0x40 +/** For put: Just reserve space for data, don't copy it. Return a + * pointer to the reserved space. + */ +#define MDB_RESERVE 0x10000 +/** Data is being appended, don't split full pages. */ +#define MDB_APPEND 0x20000 +/** Duplicate data is being appended, don't split full pages. */ +#define MDB_APPENDDUP 0x40000 +/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ +#define MDB_MULTIPLE 0x80000 +/* @} */ + +/** @defgroup mdb_copy Copy Flags + * @{ + */ +/** Compacting copy: Omit free space from copy, and renumber all + * pages sequentially. + */ +#define MDB_CP_COMPACT 0x01 +/* @} */ + +/** @brief Cursor Get operations. + * + * This is the set of all operations for retrieving data + * using a cursor. + */ +typedef enum MDB_cursor_op { + MDB_FIRST, /**< Position at first key/data item */ + MDB_FIRST_DUP, /**< Position at first data item of current key. + Only for #MDB_DUPSORT */ + MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ + MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ + MDB_GET_CURRENT, /**< Return key/data at current cursor position */ + MDB_GET_MULTIPLE, /**< Return up to a page of duplicate data items + from current cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_LAST, /**< Position at last key/data item */ + MDB_LAST_DUP, /**< Position at last data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT, /**< Position at next data item */ + MDB_NEXT_DUP, /**< Position at next data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT_MULTIPLE, /**< Return up to a page of duplicate data items + from next cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_NEXT_NODUP, /**< Position at first data item of next key */ + MDB_PREV, /**< Position at previous data item */ + MDB_PREV_DUP, /**< Position at previous data item of current key. + Only for #MDB_DUPSORT */ + MDB_PREV_NODUP, /**< Position at last data item of previous key */ + MDB_SET, /**< Position at specified key */ + MDB_SET_KEY, /**< Position at specified key, return key + data */ + MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */ + MDB_PREV_MULTIPLE /**< Position at previous page and return up to + a page of duplicate data items. Only for #MDB_DUPFIXED */ +} MDB_cursor_op; + +/** @defgroup errors Return Codes + * + * BerkeleyDB uses -30800 to -30999, we'll go under them + * @{ + */ + /** Successful result */ +#define MDB_SUCCESS 0 + /** key/data pair already exists */ +#define MDB_KEYEXIST (-30799) + /** key/data pair not found (EOF) */ +#define MDB_NOTFOUND (-30798) + /** Requested page not found - this usually indicates corruption */ +#define MDB_PAGE_NOTFOUND (-30797) + /** Located page was wrong type */ +#define MDB_CORRUPTED (-30796) + /** Update of meta page failed or environment had fatal error */ +#define MDB_PANIC (-30795) + /** Environment version mismatch */ +#define MDB_VERSION_MISMATCH (-30794) + /** File is not a valid LMDB file */ +#define MDB_INVALID (-30793) + /** Environment mapsize reached */ +#define MDB_MAP_FULL (-30792) + /** Environment maxdbs reached */ +#define MDB_DBS_FULL (-30791) + /** Environment maxreaders reached */ +#define MDB_READERS_FULL (-30790) + /** Too many TLS keys in use - Windows only */ +#define MDB_TLS_FULL (-30789) + /** Txn has too many dirty pages */ +#define MDB_TXN_FULL (-30788) + /** Cursor stack too deep - internal error */ +#define MDB_CURSOR_FULL (-30787) + /** Page has not enough space - internal error */ +#define MDB_PAGE_FULL (-30786) + /** Database contents grew beyond environment mapsize */ +#define MDB_MAP_RESIZED (-30785) + /** Operation and DB incompatible, or DB type changed. This can mean: + *
    + *
  • The operation expects an #MDB_DUPSORT / #MDB_DUPFIXED database. + *
  • Opening a named DB when the unnamed DB has #MDB_DUPSORT / #MDB_INTEGERKEY. + *
  • Accessing a data record as a database, or vice versa. + *
  • The database was dropped and recreated with different flags. + *
+ */ +#define MDB_INCOMPATIBLE (-30784) + /** Invalid reuse of reader locktable slot */ +#define MDB_BAD_RSLOT (-30783) + /** Transaction must abort, has a child, or is invalid */ +#define MDB_BAD_TXN (-30782) + /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ +#define MDB_BAD_VALSIZE (-30781) + /** The specified DBI was changed unexpectedly */ +#define MDB_BAD_DBI (-30780) + /** The last defined error code */ +#define MDB_LAST_ERRCODE MDB_BAD_DBI +/** @} */ + +/** @brief Statistics for a database in the environment */ +typedef struct MDB_stat { + unsigned int ms_psize; /**< Size of a database page. + This is currently the same for all databases. */ + unsigned int ms_depth; /**< Depth (height) of the B-tree */ + size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + size_t ms_leaf_pages; /**< Number of leaf pages */ + size_t ms_overflow_pages; /**< Number of overflow pages */ + size_t ms_entries; /**< Number of data items */ +} MDB_stat; + +/** @brief Information about the environment */ +typedef struct MDB_envinfo { + void *me_mapaddr; /**< Address of map, if fixed */ + size_t me_mapsize; /**< Size of the data memory map */ + size_t me_last_pgno; /**< ID of the last used page */ + size_t me_last_txnid; /**< ID of the last committed transaction */ + unsigned int me_maxreaders; /**< max reader slots in the environment */ + unsigned int me_numreaders; /**< max reader slots used in the environment */ +} MDB_envinfo; + + /** @brief Return the LMDB library version information. + * + * @param[out] major if non-NULL, the library major version number is copied here + * @param[out] minor if non-NULL, the library minor version number is copied here + * @param[out] patch if non-NULL, the library patch version number is copied here + * @retval "version string" The library version as a string + */ +char *mdb_version(int *major, int *minor, int *patch); + + /** @brief Return a string describing a given error code. + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function strerror(3) is returned. If the error code + * is less than 0, an error string corresponding to the LMDB library error is + * returned. See @ref errors for a list of LMDB-specific error codes. + * @param[in] err The error code + * @retval "error message" The description of the error + */ +char *mdb_strerror(int err); + + /** @brief Create an LMDB environment handle. + * + * This function allocates memory for a #MDB_env structure. To release + * the allocated memory and discard the handle, call #mdb_env_close(). + * Before the handle may be used, it must be opened using #mdb_env_open(). + * Various other options may also need to be set before opening the handle, + * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), + * depending on usage requirements. + * @param[out] env The address where the new handle will be stored + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_create(MDB_env **env); + + /** @brief Open an environment handle. + * + * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] path The directory in which the database files reside. This + * directory must already exist and be writable. + * @param[in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * Flags set by mdb_env_set_flags() are also used. + *
    + *
  • #MDB_FIXEDMAP + * use a fixed address for the mmap region. This flag must be specified + * when creating the environment, and is stored persistently in the environment. + * If successful, the memory map will always reside at the same virtual address + * and pointers used to reference data items in the database will be constant + * across multiple invocations. This option may not always work, depending on + * how the operating system has allocated memory to shared libraries and other uses. + * The feature is highly experimental. + *
  • #MDB_NOSUBDIR + * By default, LMDB creates its environment in a directory whose + * pathname is given in \b path, and creates its data and lock files + * under that directory. With this option, \b path is used as-is for + * the database main data file. The database lock file is the \b path + * with "-lock" appended. + *
  • #MDB_RDONLY + * Open the environment in read-only mode. No write operations will be + * allowed. LMDB will still modify the lock file - except on read-only + * filesystems, where LMDB does not use locks. + *
  • #MDB_WRITEMAP + * Use a writeable memory map unless MDB_RDONLY is set. This uses + * fewer mallocs but loses protection from application bugs + * like wild pointer writes and other bad updates into the database. + * This may be slightly faster for DBs that fit entirely in RAM, but + * is slower for DBs larger than RAM. + * Incompatible with nested transactions. + * Do not mix processes with and without MDB_WRITEMAP on the same + * environment. This can defeat durability (#mdb_env_sync etc). + *
  • #MDB_NOMETASYNC + * Flush system buffers to disk only once per transaction, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization + * maintains database integrity, but a system crash may undo the last + * committed transaction. I.e. it preserves the ACI (atomicity, + * consistency, isolation) but not D (durability) database property. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_NOSYNC + * Don't flush system buffers to disk when committing a transaction. + * This optimization means a system crash can corrupt the database or + * lose the last transactions if buffers are not yet flushed to disk. + * The risk is governed by how often the system flushes dirty buffers + * to disk and how often #mdb_env_sync() is called. However, if the + * filesystem preserves write order and the #MDB_WRITEMAP flag is not + * used, transactions exhibit ACI (atomicity, consistency, isolation) + * properties and only lose D (durability). I.e. database integrity + * is maintained, but a system crash may undo the final transactions. + * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no + * hint for when to write transactions to disk, unless #mdb_env_sync() + * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_MAPASYNC + * When using #MDB_WRITEMAP, use asynchronous flushes to disk. + * As with #MDB_NOSYNC, a system crash can then corrupt the + * database or lose the last transactions. Calling #mdb_env_sync() + * ensures on-disk database integrity until next commit. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_NOTLS + * Don't use Thread-Local Storage. Tie reader locktable slots to + * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps + * the slot reserved for the #MDB_txn object. A thread may use parallel + * read-only transactions. A read-only transaction may span threads if + * the user synchronizes its use. Applications that multiplex many + * user threads over individual OS threads need this option. Such an + * application must also serialize the write transactions in an OS + * thread, since LMDB's write locking is unaware of the user threads. + *
  • #MDB_NOLOCK + * Don't do any locking. If concurrent access is anticipated, the + * caller must manage all concurrency itself. For proper operation + * the caller must enforce single-writer semantics, and must ensure + * that no readers are using old transactions while a writer is + * active. The simplest approach is to use an exclusive lock so that + * no readers may be active at all when a writer begins. + *
  • #MDB_NORDAHEAD + * Turn off readahead. Most operating systems perform readahead on + * read requests by default. This option turns it off if the OS + * supports it. Turning it off may help random read performance + * when the DB is larger than RAM and system RAM is full. + * The option is not implemented on Windows. + *
  • #MDB_NOMEMINIT + * Don't initialize malloc'd memory before writing to unused spaces + * in the data file. By default, memory for pages written to the data + * file is obtained using malloc. While these pages may be reused in + * subsequent transactions, freshly malloc'd pages will be initialized + * to zeroes before use. This avoids persisting leftover data from other + * code (that used the heap and subsequently freed the memory) into the + * data file. Note that many other system libraries may allocate + * and free memory from the heap for arbitrary uses. E.g., stdio may + * use the heap for file I/O buffers. This initialization step has a + * modest performance cost so some applications may want to disable + * it using this flag. This option can be a problem for applications + * which handle sensitive data like passwords, and it makes memory + * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, + * which writes directly to the mmap instead of using malloc for pages. The + * initialization is also skipped if #MDB_RESERVE is used; the + * caller is expected to overwrite all of the memory that was + * reserved in that case. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
+ * @param[in] mode The UNIX permissions to set on created files and semaphores. + * This parameter is ignored on Windows. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the + * version that created the database environment. + *
  • #MDB_INVALID - the environment file headers are corrupted. + *
  • ENOENT - the directory specified by the path parameter doesn't exist. + *
  • EACCES - the user didn't have permission to access the environment files. + *
  • EAGAIN - the environment was locked by another process. + *
+ */ +int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); + + /** @brief Copy an LMDB environment to the specified path. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy(MDB_env *env, const char *path); + + /** @brief Copy an LMDB environment to the specified file descriptor. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); + + /** @brief Copy an LMDB environment to the specified path, with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free + * pages and sequentially renumber all pages in output. This option + * consumes more CPU and runs more slowly than the default. + * Currently it fails if the environment has suffered a page leak. + *
+ * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); + + /** @brief Copy an LMDB environment to the specified file descriptor, + * with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. See + * #mdb_env_copy2() for further details. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @param[in] flags Special options for this operation. + * See #mdb_env_copy2() for options. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); + + /** @brief Return statistics about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + */ +int mdb_env_stat(MDB_env *env, MDB_stat *stat); + + /** @brief Return information about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_envinfo structure + * where the information will be copied + */ +int mdb_env_info(MDB_env *env, MDB_envinfo *stat); + + /** @brief Flush the data buffers to disk. + * + * Data is always written to disk when #mdb_txn_commit() is called, + * but the operating system may keep it buffered. LMDB always flushes + * the OS buffers upon commit as well, unless the environment was + * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is + * not valid if the environment was opened with #MDB_RDONLY. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] force If non-zero, force a synchronous flush. Otherwise + * if the environment has the #MDB_NOSYNC flag set the flushes + * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - the environment is read-only. + *
  • EINVAL - an invalid parameter was specified. + *
  • EIO - an error occurred during synchronization. + *
+ */ +int mdb_env_sync(MDB_env *env, int force); + + /** @brief Close the environment and release the memory map. + * + * Only a single thread may call this function. All transactions, databases, + * and cursors must already be closed before calling this function. Attempts to + * use any such handles after calling this function will cause a SIGSEGV. + * The environment handle will be freed and must not be used again after this call. + * @param[in] env An environment handle returned by #mdb_env_create() + */ +void mdb_env_close(MDB_env *env); + + /** @brief Set environment flags. + * + * This may be used to set some flags in addition to those from + * #mdb_env_open(), or to unset these flags. If several threads + * change the flags at the same time, the result is undefined. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] flags The flags to change, bitwise OR'ed together + * @param[in] onoff A non-zero value sets the flags, zero clears them. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); + + /** @brief Get environment flags. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] flags The address of an integer to store the flags + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_flags(MDB_env *env, unsigned int *flags); + + /** @brief Return the path that was used in #mdb_env_open(). + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] path Address of a string pointer to contain the path. This + * is the actual string in the environment, not a copy. It should not be + * altered in any way. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_path(MDB_env *env, const char **path); + + /** @brief Return the filedescriptor for the given environment. + * + * This function may be called after fork(), so the descriptor can be + * closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC. + * (Until LMDB 0.9.18, only the lockfile had that.) + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); + + /** @brief Set the size of the memory map to use for this environment. + * + * The size should be a multiple of the OS page size. The default is + * 10485760 bytes. The size of the memory map is also the maximum size + * of the database. The value should be chosen as large as possible, + * to accommodate future growth of the database. + * This function should be called after #mdb_env_create() and before #mdb_env_open(). + * It may be called at later times if no transactions are active in + * this process. Note that the library does not check for this condition, + * the caller must ensure it explicitly. + * + * The new size takes effect immediately for the current process but + * will not be persisted to any others until a write transaction has been + * committed by the current process. Also, only mapsize increases are + * persisted into the environment. + * + * If the mapsize is increased by another process, and data has grown + * beyond the range of the current mapsize, #mdb_txn_begin() will + * return #MDB_MAP_RESIZED. This function may be called with a size + * of zero to adopt the new size. + * + * Any attempt to set a size smaller than the space already consumed + * by the environment will be silently changed to the current size of the used space. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] size The size in bytes + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment has + * an active write transaction. + *
+ */ +int mdb_env_set_mapsize(MDB_env *env, size_t size); + + /** @brief Set the maximum number of threads/reader slots for the environment. + * + * This defines the number of slots in the lock table that is used to track readers in the + * the environment. The default is 126. + * Starting a read-only transaction normally ties a lock table slot to the + * current thread until the environment closes or the thread exits. If + * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the + * MDB_txn object until it or the #MDB_env object is destroyed. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] readers The maximum number of reader lock table slots + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment is already open. + *
+ */ +int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); + + /** @brief Get the maximum number of threads/reader slots for the environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] readers Address of an integer to store the number of readers + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); + + /** @brief Set the maximum number of named databases for the environment. + * + * This function is only needed if multiple databases will be used in the + * environment. Simpler applications that use the environment as a single + * unnamed database can ignore this option. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * + * Currently a moderate number of slots are cheap but a huge number gets + * expensive: 7-120 words per transaction, and every #mdb_dbi_open() + * does a linear search of the opened slots. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbs The maximum number of databases + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment is already open. + *
+ */ +int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); + + /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. + * + * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. + * See @ref MDB_val. + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The maximum size of a key we can write + */ +int mdb_env_get_maxkeysize(MDB_env *env); + + /** @brief Set application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_userctx(MDB_env *env, void *ctx); + + /** @brief Get the application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The pointer set by #mdb_env_set_userctx(). + */ +void *mdb_env_get_userctx(MDB_env *env); + + /** @brief A callback function for most LMDB assert() failures, + * called before printing the message and aborting. + * + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] msg The assertion message, not including newline. + */ +typedef void MDB_assert_func(MDB_env *env, const char *msg); + + /** Set or reset the assert() callback of the environment. + * Disabled if liblmdb is built with NDEBUG. + * @note This hack should become obsolete as lmdb's error handling matures. + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] func An #MDB_assert_func function, or 0. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); + + /** @brief Create a transaction for use with the environment. + * + * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). + * @note A transaction and its cursors must only be used by a single + * thread, and a thread may only have a single transaction at a time. + * If #MDB_NOTLS is in use, this does not apply to read-only transactions. + * @note Cursors may not span transactions. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] parent If this parameter is non-NULL, the new transaction + * will be a nested transaction, with the transaction indicated by \b parent + * as its parent. Transactions may be nested to any level. A parent + * transaction and its cursors may not issue any other operations than + * mdb_txn_commit and mdb_txn_abort while it has active child transactions. + * @param[in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_RDONLY + * This transaction will not perform any write operations. + *
+ * @param[out] txn Address where the new #MDB_txn handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
  • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's + * mapsize and this environment's map must be resized as well. + * See #mdb_env_set_mapsize(). + *
  • #MDB_READERS_FULL - a read-only transaction was requested and + * the reader lock table is full. See #mdb_env_set_maxreaders(). + *
  • ENOMEM - out of memory. + *
+ */ +int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); + + /** @brief Returns the transaction's #MDB_env + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +MDB_env *mdb_txn_env(MDB_txn *txn); + + /** @brief Return the transaction's ID. + * + * This returns the identifier associated with this transaction. For a + * read-only transaction, this corresponds to the snapshot being read; + * concurrent readers will frequently have the same transaction ID. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A transaction ID, valid if input is an active transaction. + */ +size_t mdb_txn_id(MDB_txn *txn); + + /** @brief Commit all the operations of a transaction into the database. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
  • ENOSPC - no more disk space. + *
  • EIO - a low-level I/O error occurred while writing. + *
  • ENOMEM - out of memory. + *
+ */ +int mdb_txn_commit(MDB_txn *txn); + + /** @brief Abandon all the operations of the transaction instead of saving them. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_abort(MDB_txn *txn); + + /** @brief Reset a read-only transaction. + * + * Abort the transaction like #mdb_txn_abort(), but keep the transaction + * handle. #mdb_txn_renew() may reuse the handle. This saves allocation + * overhead if the process will start a new read-only transaction soon, + * and also locking overhead if #MDB_NOTLS is in use. The reader table + * lock is released, but the table slot stays tied to its thread or + * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free + * its lock table slot if MDB_NOTLS is in use. + * Cursors opened within the transaction must not be used + * again after this call, except with #mdb_cursor_renew(). + * Reader locks generally don't interfere with writers, but they keep old + * versions of database pages allocated. Thus they prevent the old pages + * from being reused when writers commit new data, and so under heavy load + * the database size may grow much more rapidly than otherwise. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_reset(MDB_txn *txn); + + /** @brief Renew a read-only transaction. + * + * This acquires a new reader lock for a transaction handle that had been + * released by #mdb_txn_reset(). It must be called before a reset transaction + * may be used again. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_txn_renew(MDB_txn *txn); + +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) + + /** @brief Open a database in the environment. + * + * A database handle denotes the name and parameters of a database, + * independently of whether such a database exists. + * The database handle may be discarded by calling #mdb_dbi_close(). + * The old database handle is returned if the database was already open. + * The handle may only be closed once. + * + * The database handle will be private to the current transaction until + * the transaction is successfully committed. If the transaction is + * aborted the handle will be closed automatically. + * After a successful commit the handle will reside in the shared + * environment, and may be used by other transactions. + * + * This function must not be called from multiple concurrent + * transactions in the same process. A transaction that uses + * this function must finish (either commit or abort) before + * any other transaction in the process may use this function. + * + * To use named databases (with name != NULL), #mdb_env_set_maxdbs() + * must be called before opening the environment. Database names are + * keys in the unnamed database, and may be read but not written. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] name The name of the database to open. If only a single + * database is needed in the environment, this value may be NULL. + * @param[in] flags Special options for this database. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_REVERSEKEY + * Keys are strings to be compared in reverse order, from the end + * of the strings to the beginning. By default, Keys are treated as strings and + * compared from beginning to end. + *
  • #MDB_DUPSORT + * Duplicate keys may be used in the database. (Or, from another perspective, + * keys may have multiple data items, stored in sorted order.) By default + * keys must be unique and may have only a single data item. + *
  • #MDB_INTEGERKEY + * Keys are binary integers in native byte order, either unsigned int + * or size_t, and will be sorted as such. + * The keys must all be of the same size. + *
  • #MDB_DUPFIXED + * This flag may only be used in combination with #MDB_DUPSORT. This option + * tells the library that the data items for this database are all the same + * size, which allows further optimizations in storage and retrieval. When + * all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE + * and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple + * items at once. + *
  • #MDB_INTEGERDUP + * This option specifies that duplicate data items are binary integers, + * similar to #MDB_INTEGERKEY keys. + *
  • #MDB_REVERSEDUP + * This option specifies that duplicate data items should be compared as + * strings in reverse order. + *
  • #MDB_CREATE + * Create the named database if it doesn't exist. This option is not + * allowed in a read-only transaction or a read-only environment. + *
+ * @param[out] dbi Address where the new #MDB_dbi handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - the specified database doesn't exist in the environment + * and #MDB_CREATE was not specified. + *
  • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). + *
+ */ +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); + + /** @brief Retrieve statistics for a database. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); + + /** @brief Retrieve the DB flags for a database handle. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] flags Address where the flags will be returned. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); + + /** @brief Close a database handle. Normally unnecessary. Use with care: + * + * This call is not mutex protected. Handles should only be closed by + * a single thread, and only if no other threads are going to reference + * the database handle or one of its cursors any further. Do not close + * a handle if an existing transaction has modified its database. + * Doing so can cause misbehavior from database corruption to errors + * like MDB_BAD_VALSIZE (since the DB name is gone). + * + * Closing a database handle is not necessary, but lets #mdb_dbi_open() + * reuse the handle value. Usually it's better to set a bigger + * #mdb_env_set_maxdbs(), unless that value would be large. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); + + /** @brief Empty or delete+close a database. + * + * See #mdb_dbi_close() for restrictions about closing the DB handle. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] del 0 to empty the DB, 1 to delete it from the + * environment and close the DB handle. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); + + /** @brief Set a custom key comparison function for a database. + * + * The comparison function is called whenever it is necessary to compare a + * key specified by the application with a key currently stored in the database. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating + * before longer keys. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. + * + * This comparison function is called whenever it is necessary to compare a data + * item specified by the application with a data item currently stored in the database. + * This function only takes effect if the database was opened with the #MDB_DUPSORT + * flag. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating + * before longer items. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a relocation function for a #MDB_FIXEDMAP database. + * + * @todo The relocation function is called whenever it is necessary to move the data + * of an item to a different position in the database (e.g. through tree + * balancing operations, shifts as a result of adds or deletes, etc.). It is + * intended to allow address/position-dependent data items to be stored in + * a database in an environment opened with the #MDB_FIXEDMAP option. + * Currently the relocation feature is unimplemented and setting + * this function has no effect. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] rel A #MDB_rel_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); + + /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. + * + * See #mdb_set_relfunc and #MDB_rel_func for more details. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * It will be passed to the callback function set by #mdb_set_relfunc + * as its \b relctx parameter whenever the callback is invoked. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); + + /** @brief Get items from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified \b key are returned + * in the structure to which \b data refers. + * If the database supports duplicate keys (#MDB_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of #mdb_cursor_get(). + * + * @note The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * @note Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to search for in the database + * @param[out] data The data corresponding to the key + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - the key was not in the database. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Store items into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed, or adding a duplicate data item if + * duplicates are allowed (#MDB_DUPSORT). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to store in the database + * @param[in,out] data The data to store + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). The \b data + * parameter will be set to point to the existing item. + *
  • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. + * LMDB does nothing else with this memory, the caller is expected + * to modify all of the space requested. This flag must not be + * specified if the database was opened with #MDB_DUPSORT. + *
  • #MDB_APPEND - append the given key/data pair to the end of the + * database. This option allows fast bulk loading when keys are + * already known to be in the correct order. Loading unsorted keys + * with this flag will cause a #MDB_KEYEXIST error. + *
  • #MDB_APPENDDUP - as above, but for sorted dup data. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete items from a database. + * + * This function removes key/data pairs from the database. + * If the database does not support sorted duplicate data items + * (#MDB_DUPSORT) the data parameter is ignored. + * If the database supports sorted duplicates and the data parameter + * is NULL, all of the duplicate data items for the key will be + * deleted. Otherwise, if the data parameter is non-NULL + * only the matching data item will be deleted. + * This function will return #MDB_NOTFOUND if the specified key/data + * pair is not in the database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to delete from the database + * @param[in] data The data to delete + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Create a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with #mdb_cursor_renew(). + * It can be discarded with #mdb_cursor_close(). + * A cursor in a write-transaction can be closed before its transaction + * ends, and will otherwise be closed when its transaction ends. + * A cursor in a read-only transaction must be closed explicitly, before + * or after its transaction ends. It can be reused with + * #mdb_cursor_renew() before finally closing it. + * @note Earlier documentation said that cursors in every transaction + * were closed when the transaction committed or aborted. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] cursor Address where the new #MDB_cursor handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); + + /** @brief Close a cursor handle. + * + * The cursor handle will be freed and must not be used again after this call. + * Its transaction must still be live if it is a write-transaction. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +void mdb_cursor_close(MDB_cursor *cursor); + + /** @brief Renew a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * Cursors that are only used in read-only + * transactions may be re-used, to avoid unnecessary malloc/free overhead. + * The cursor may be associated with a new read-only transaction, and + * referencing the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); + + /** @brief Return the cursor's transaction handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); + + /** @brief Return the cursor's database handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); + + /** @brief Retrieve by cursor. + * + * This function retrieves key/data pairs from the database. The address and length + * of the key are returned in the object to which \b key refers (except for the + * case of the #MDB_SET option, in which the \b key object is unchanged), and + * the address and length of the data are returned in the object to which \b data + * refers. + * See #mdb_get() for restrictions on using the output values. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in,out] key The key for a retrieved item + * @param[in,out] data The data of a retrieved item + * @param[in] op A cursor operation #MDB_cursor_op + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - no matching key found. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + MDB_cursor_op op); + + /** @brief Store by cursor. + * + * This function stores key/data pairs into the database. + * The cursor is positioned at the new item, or on failure usually near it. + * @note Earlier documentation incorrectly said errors would leave the + * state of the cursor unchanged. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] key The key operated on. + * @param[in] data The data operated on. + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
    + *
  • #MDB_CURRENT - replace the item at the current cursor position. + * The \b key parameter must still be provided, and must match it. + * If using sorted duplicates (#MDB_DUPSORT) the data item must still + * sort into the same place. This is intended to be used when the + * new data is the same size as the old. Otherwise it will simply + * perform a delete of the old record followed by an insert. + *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). + *
  • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. This flag + * must not be specified if the database was opened with #MDB_DUPSORT. + *
  • #MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * a #MDB_KEYEXIST error. + *
  • #MDB_APPENDDUP - as above, but for sorted dup data. + *
  • #MDB_MULTIPLE - store multiple contiguous data elements in a + * single request. This flag may only be specified if the database + * was opened with #MDB_DUPFIXED. The \b data argument must be an + * array of two MDB_vals. The mv_size of the first MDB_val must be + * the size of a single data element. The mv_data of the first MDB_val + * must point to the beginning of the array of contiguous data elements. + * The mv_size of the second MDB_val must be the count of the number + * of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The mv_data + * of the second MDB_val is unused. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete current key/data pair + * + * This function deletes the key/data pair to which the cursor refers. + * This does not invalidate the cursor, so operations such as MDB_NEXT + * can still be used on it. + * Both MDB_NEXT and MDB_GET_CURRENT will return the same record after + * this operation. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
    + *
  • #MDB_NODUPDATA - delete all of the data items for the current key. + * This flag may only be specified if the database was opened with #MDB_DUPSORT. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); + + /** @brief Return count of duplicates for current key. + * + * This call is only valid on databases that support sorted duplicate + * data items #MDB_DUPSORT. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[out] countp Address where the count will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - cursor is not initialized, or an invalid parameter was specified. + *
+ */ +int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two data items were keys in the + * specified database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two items were data items of + * the specified database. The database must have the #MDB_DUPSORT flag. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief A callback function used to print a message from the library. + * + * @param[in] msg The string to be printed. + * @param[in] ctx An arbitrary context pointer for the callback. + * @return < 0 on failure, >= 0 on success. + */ +typedef int (MDB_msg_func)(const char *msg, void *ctx); + + /** @brief Dump the entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] func A #MDB_msg_func function + * @param[in] ctx Anything the message function needs + * @return < 0 on failure, >= 0 on success. + */ +int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); + + /** @brief Check for stale entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] dead Number of stale slots that were cleared + * @return 0 on success, non-zero on failure. + */ +int mdb_reader_check(MDB_env *env, int *dead); +/** @} */ + +#ifdef __cplusplus +} +#endif +/** @page tools LMDB Command Line Tools + The following describes the command line tools that are available for LMDB. + \li \ref mdb_copy_1 + \li \ref mdb_dump_1 + \li \ref mdb_load_1 + \li \ref mdb_stat_1 +*/ + +#endif /* _LMDB_H_ */ diff --git a/nostrdb/mdb.c b/nostrdb/mdb.c new file mode 100644 index 0000000000..deb6779640 --- /dev/null +++ b/nostrdb/mdb.c @@ -0,0 +1,10354 @@ +/** @file mdb.c + * @brief Lightning memory-mapped database library + * + * A Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. + */ +/* + * Copyright 2011-2021 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#if defined(__WIN64__) +#define _FILE_OFFSET_BITS 64 +#endif +#ifdef _WIN32 +#include +#include +#include /* get wcscpy() */ + +/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it + * as int64 which is wrong. MSVC doesn't define it at all, so just + * don't use it. + */ +#define MDB_PID_T int +#define MDB_THR_T DWORD +#include +#include +#ifdef __GNUC__ +# include +#else +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# define BYTE_ORDER LITTLE_ENDIAN +# ifndef SSIZE_MAX +# define SSIZE_MAX INT_MAX +# endif +#endif +#else +#include +#include +#define MDB_PID_T pid_t +#define MDB_THR_T pthread_t +#include +#include +#include +#ifdef HAVE_SYS_FILE_H +#include +#endif +#include +#endif + +#if defined(__mips) && defined(__linux) +/* MIPS has cache coherency issues, requires explicit cache control */ +#include +#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) +#else +#define CACHEFLUSH(addr, bytes, cache) +#endif + +#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS) +/** fdatasync is broken on ext3/ext4fs on older kernels, see + * description in #mdb_env_open2 comments. You can safely + * define MDB_FDATASYNC_WORKS if this code will only be run + * on kernels 3.6 and newer. + */ +#define BROKEN_FDATASYNC +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#include +typedef SSIZE_T ssize_t; +#else +#include +#endif + +#if defined(__sun) || defined(ANDROID) +/* Most platforms have posix_memalign, older may only have memalign */ +#define HAVE_MEMALIGN 1 +#include +/* On Solaris, we need the POSIX sigwait function */ +#if defined (__sun) +# define _POSIX_PTHREAD_SEMANTICS 1 +#endif +#endif + +#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) +#include +#include /* defines BYTE_ORDER on HPUX and Solaris */ +#endif + +#if defined(__FreeBSD__) && defined(__FreeBSD_version) && __FreeBSD_version >= 1100110 +# define MDB_USE_POSIX_MUTEX 1 +# define MDB_USE_ROBUST 1 +#elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__) +# define MDB_USE_POSIX_SEM 1 +# define MDB_FDATASYNC fsync +#elif defined(ANDROID) +# define MDB_FDATASYNC fsync +#endif + +#ifndef _WIN32 +#include +#include +#ifdef MDB_USE_POSIX_SEM +# define MDB_USE_HASH 1 +#include +#else +#define MDB_USE_POSIX_MUTEX 1 +#endif +#endif + +#if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) \ + + defined(MDB_USE_POSIX_MUTEX) != 1 +# error "Ambiguous shared-lock implementation" +#endif + +#ifdef USE_VALGRIND +#include +#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) +#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) +#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) +#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) +#else +#define VGMEMP_CREATE(h,r,z) +#define VGMEMP_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) +#define VGMEMP_DESTROY(h) +#define VGMEMP_DEFINED(a,s) +#endif + +#ifndef BYTE_ORDER +# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) +/* Solaris just defines one or the other */ +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# ifdef _LITTLE_ENDIAN +# define BYTE_ORDER LITTLE_ENDIAN +# else +# define BYTE_ORDER BIG_ENDIAN +# endif +# else +# define BYTE_ORDER __BYTE_ORDER +# endif +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif + +#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) +#define MISALIGNED_OK 1 +#endif + +#include "lmdb.h" +#include "midl.h" + +#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) +# error "Unknown or unsupported endianness (BYTE_ORDER)" +#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +# error "Two's complement, reasonably sized integer types, please" +#endif + +#if (((__clang_major__ << 8) | __clang_minor__) >= 0x0302) || (((__GNUC__ << 8) | __GNUC_MINOR__) >= 0x0403) +/** Mark infrequently used env functions as cold. This puts them in a separate + * section, and optimizes them for size */ +#define ESECT __attribute__ ((cold)) +#else +/* On older compilers, use a separate section */ +# ifdef __GNUC__ +# ifdef __APPLE__ +# define ESECT __attribute__ ((section("__TEXT,text_env"))) +# else +# define ESECT __attribute__ ((section("text_env"))) +# endif +# else +# define ESECT +# endif +#endif + +#ifdef _WIN32 +#define CALL_CONV WINAPI +#else +#define CALL_CONV +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup compat Compatibility Macros + * A bunch of macros to minimize the amount of platform-specific ifdefs + * needed throughout the rest of the code. When the features this library + * needs are similar enough to POSIX to be hidden in a one-or-two line + * replacement, this macro approach is used. + * @{ + */ + + /** Features under development */ +#ifndef MDB_DEVEL +#define MDB_DEVEL 0 +#endif + + /** Wrapper around __func__, which is a C99 feature */ +#if __STDC_VERSION__ >= 199901L +# define mdb_func_ __func__ +#elif __GNUC__ >= 2 || _MSC_VER >= 1300 +# define mdb_func_ __FUNCTION__ +#else +/* If a debug message says (), update the #if statements above */ +# define mdb_func_ "" +#endif + +/* Internal error codes, not exposed outside liblmdb */ +#define MDB_NO_ROOT (MDB_LAST_ERRCODE + 10) +#ifdef _WIN32 +#define MDB_OWNERDEAD ((int) WAIT_ABANDONED) +#elif defined(MDB_USE_POSIX_MUTEX) && defined(EOWNERDEAD) +#define MDB_OWNERDEAD EOWNERDEAD /**< #LOCK_MUTEX0() result if dead owner */ +#endif + +#ifdef __GLIBC__ +#define GLIBC_VER ((__GLIBC__ << 16 )| __GLIBC_MINOR__) +#endif +/** Some platforms define the EOWNERDEAD error code + * even though they don't support Robust Mutexes. + * Compile with -DMDB_USE_ROBUST=0, or use some other + * mechanism like -DMDB_USE_POSIX_SEM instead of + * -DMDB_USE_POSIX_MUTEX. + * (Posix semaphores are not robust.) + */ +#ifndef MDB_USE_ROBUST +/* Android currently lacks Robust Mutex support. So does glibc < 2.4. */ +# if defined(MDB_USE_POSIX_MUTEX) && (defined(ANDROID) || \ + (defined(__GLIBC__) && GLIBC_VER < 0x020004)) +# define MDB_USE_ROBUST 0 +# else +# define MDB_USE_ROBUST 1 +# endif +#endif /* !MDB_USE_ROBUST */ + +#if defined(MDB_USE_POSIX_MUTEX) && (MDB_USE_ROBUST) +/* glibc < 2.12 only provided _np API */ +# if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \ + (defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST)) +# define PTHREAD_MUTEX_ROBUST PTHREAD_MUTEX_ROBUST_NP +# define pthread_mutexattr_setrobust(attr, flag) pthread_mutexattr_setrobust_np(attr, flag) +# define pthread_mutex_consistent(mutex) pthread_mutex_consistent_np(mutex) +# endif +#endif /* MDB_USE_POSIX_MUTEX && MDB_USE_ROBUST */ + +#if defined(MDB_OWNERDEAD) && (MDB_USE_ROBUST) +#define MDB_ROBUST_SUPPORTED 1 +#endif + +#ifdef _WIN32 +#define MDB_USE_HASH 1 +#define MDB_PIDLOCK 0 +#define THREAD_RET DWORD +#define pthread_t HANDLE +#define pthread_mutex_t HANDLE +#define pthread_cond_t HANDLE +typedef HANDLE mdb_mutex_t, mdb_mutexref_t; +#define pthread_key_t DWORD +#define pthread_self() GetCurrentThreadId() +#define pthread_key_create(x,y) \ + ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) +#define pthread_key_delete(x) TlsFree(x) +#define pthread_getspecific(x) TlsGetValue(x) +#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) +#define pthread_mutex_unlock(x) ReleaseMutex(*x) +#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) +#define pthread_cond_signal(x) SetEvent(*x) +#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) +#define THREAD_CREATE(thr,start,arg) \ + (((thr) = CreateThread(NULL, 0, start, arg, 0, NULL)) ? 0 : ErrCode()) +#define THREAD_FINISH(thr) \ + (WaitForSingleObject(thr, INFINITE) ? ErrCode() : 0) +#define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) +#define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) +#define mdb_mutex_consistent(mutex) 0 +#define getpid() GetCurrentProcessId() +#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) +#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) +#define ErrCode() GetLastError() +#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} +#define close(fd) (CloseHandle(fd) ? 0 : -1) +#define munmap(ptr,len) UnmapViewOfFile(ptr) +#ifdef PROCESS_QUERY_LIMITED_INFORMATION +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION +#else +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 +#endif +#define Z "I" +#else +#define THREAD_RET void * +#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) +#define THREAD_FINISH(thr) pthread_join(thr,NULL) +#define Z "z" /**< printf format modifier for size_t */ + + /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ +#define MDB_PIDLOCK 1 + +#ifdef MDB_USE_POSIX_SEM + +typedef sem_t *mdb_mutex_t, *mdb_mutexref_t; +#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) +#define UNLOCK_MUTEX(mutex) sem_post(mutex) + +static int +mdb_sem_wait(sem_t *sem) +{ + int rc; + while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; + return rc; +} + +#else /* MDB_USE_POSIX_MUTEX: */ + /** Shared mutex/semaphore as the original is stored. + * + * Not for copies. Instead it can be assigned to an #mdb_mutexref_t. + * When mdb_mutexref_t is a pointer and mdb_mutex_t is not, then it + * is array[size 1] so it can be assigned to the pointer. + */ +typedef pthread_mutex_t mdb_mutex_t[1]; + /** Reference to an #mdb_mutex_t */ +typedef pthread_mutex_t *mdb_mutexref_t; + /** Lock the reader or writer mutex. + * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). + */ +#define LOCK_MUTEX0(mutex) pthread_mutex_lock(mutex) + /** Unlock the reader or writer mutex. + */ +#define UNLOCK_MUTEX(mutex) pthread_mutex_unlock(mutex) + /** Mark mutex-protected data as repaired, after death of previous owner. + */ +#define mdb_mutex_consistent(mutex) pthread_mutex_consistent(mutex) +#endif /* MDB_USE_POSIX_SEM */ + + /** Get the error code for the last failed system function. + */ +#define ErrCode() errno + + /** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#define HANDLE int + + /** A value for an invalid file handle. + * Mainly used to initialize file variables and signify that they are + * unused. + */ +#define INVALID_HANDLE_VALUE (-1) + + /** Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. + */ +#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) +#endif + +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) +#define MNAME_LEN 32 +#else +#define MNAME_LEN (sizeof(pthread_mutex_t)) +#endif + +/** @} */ + +#ifdef MDB_ROBUST_SUPPORTED + /** Lock mutex, handle any error, set rc = result. + * Return 0 on success, nonzero (not rc) on error. + */ +#define LOCK_MUTEX(rc, env, mutex) \ + (((rc) = LOCK_MUTEX0(mutex)) && \ + ((rc) = mdb_mutex_failed(env, mutex, rc))) +static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc); +#else +#define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex)) +#define mdb_mutex_failed(env, mutex, rc) (rc) +#endif + +#ifndef _WIN32 +/** A flag for opening a file and requesting synchronous data writes. + * This is only used when writing a meta page. It's not strictly needed; + * we could just do a normal write and then immediately perform a flush. + * But if this flag is available it saves us an extra system call. + * + * @note If O_DSYNC is undefined but exists in /usr/include, + * preferably set some compiler flag to get the definition. + */ +#ifndef MDB_DSYNC +# ifdef O_DSYNC +# define MDB_DSYNC O_DSYNC +# else +# define MDB_DSYNC O_SYNC +# endif +#endif +#endif + +/** Function for flushing the data of a file. Define this to fsync + * if fdatasync() is not supported. + */ +#ifndef MDB_FDATASYNC +# define MDB_FDATASYNC fdatasync +#endif + +#ifndef MDB_MSYNC +# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) +#endif + +#ifndef MS_SYNC +#define MS_SYNC 1 +#endif + +#ifndef MS_ASYNC +#define MS_ASYNC 0 +#endif + + /** A page number in the database. + * Note that 64 bit page numbers are overkill, since pages themselves + * already represent 12-13 bits of addressable memory, and the OS will + * always limit applications to a maximum of 63 bits of address space. + * + * @note In the #MDB_node structure, we only store 48 bits of this value, + * which thus limits us to only 60 bits of addressable data. + */ +typedef MDB_ID pgno_t; + + /** A transaction ID. + * See struct MDB_txn.mt_txnid for details. + */ +typedef MDB_ID txnid_t; + +/** @defgroup debug Debug Macros + * @{ + */ +#ifndef MDB_DEBUG + /** Enable debug output. Needs variable argument macros (a C99 feature). + * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs + * read from and written to the database (used for free space management). + */ +#define MDB_DEBUG 0 +#endif + +#if MDB_DEBUG +static int mdb_debug; +static txnid_t mdb_debug_start; + + /** Print a debug message with printf formatting. + * Requires double parenthesis around 2 or more args. + */ +# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args)) +# define DPRINTF0(fmt, ...) \ + fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) +#else +# define DPRINTF(args) ((void) 0) +#endif + /** Print a debug string. + * The string is printed literally, with no format processing. + */ +#define DPUTS(arg) DPRINTF(("%s", arg)) + /** Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +/** @} */ + + /** @brief The maximum size of a database page. + * + * It is 32k or 64k, since value-PAGEBASE must fit in + * #MDB_page.%mp_upper. + * + * LMDB will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. + */ +#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) + + /** The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. + */ +#define MDB_MINKEYS 2 + + /** A stamp that identifies a file as an LMDB file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. + */ +#define MDB_MAGIC 0xBEEFC0DE + + /** The version number for a database's datafile format. */ +#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) + /** The version number for a database's lockfile format. */ +#define MDB_LOCK_VERSION 1 + + /** @brief The max size of a key we can write, or 0 for computed max. + * + * This macro should normally be left alone or set to 0. + * Note that a database with big keys or dupsort data cannot be + * reliably modified by a liblmdb which uses a smaller max. + * The default is 511 for backwards compat, or 0 when #MDB_DEVEL. + * + * Other values are allowed, for backwards compat. However: + * A value bigger than the computed max can break if you do not + * know what you are doing, and liblmdb <= 0.9.10 can break when + * modifying a DB with keys/dupsort data bigger than its max. + * + * Data items in an #MDB_DUPSORT database are also limited to + * this size, since they're actually keys of a sub-DB. Keys and + * #MDB_DUPSORT data items must fit on a node in a regular page. + */ +#ifndef MDB_MAXKEYSIZE +#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) +#endif + + /** The maximum size of a key we can write to the environment. */ +#if MDB_MAXKEYSIZE +#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) +#else +#define ENV_MAXKEY(env) ((env)->me_maxkey) +#endif + + /** @brief The maximum size of a data item. + * + * We only store a 32 bit value for node sizes. + */ +#define MAXDATASIZE 0xffffffffUL + +#if MDB_DEBUG + /** Key size which fits in a #DKBUF. + * @ingroup debug + */ +#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) + /** A key buffer. + * @ingroup debug + * This is used for printing a hex dump of a key's contents. + */ +#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] + /** Display a key in hex. + * @ingroup debug + * Invoke a function to display a key in hex. + */ +#define DKEY(x) mdb_dkey(x, kbuf) +#else +#define DKBUF +#define DKEY(x) 0 +#endif + + /** An invalid page number. + * Mainly used to denote an empty tree. + */ +#define P_INVALID (~(pgno_t)0) + + /** Test if the flags \b f are set in a flag word \b w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + + /** Round \b n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + + /** Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. + */ +typedef uint16_t indx_t; + + /** Default size of memory map. + * This is certainly too small for any actual applications. Apps should always set + * the size explicitly using #mdb_env_set_mapsize(). + */ +#define DEFAULT_MAPSIZE 1048576 + +/** @defgroup readers Reader Lock Table + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent read + * transactions started by the same thread need no further locking to proceed. + * + * If #MDB_NOTLS is set, the slot address is not saved in thread-specific data. + * + * No reader table is used if the database is on a read-only filesystem, or + * if #MDB_NOLOCK is set. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. + * @{ + */ + /** Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. 126 readers plus a + * couple mutexes fit exactly into 8KB on my development machine. + * Applications should set the table size using #mdb_env_set_maxreaders(). + */ +#define DEFAULT_READERS 126 + + /** The size of a CPU cache line in bytes. We want our lock structures + * aligned to this size to avoid false cache line sharing in the + * lock table. + * This value works for most CPUs. For Itanium this should be 128. + */ +#ifndef CACHELINE +#define CACHELINE 64 +#endif + + /** The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * @note We currently don't check for stale records. We simply re-init + * the table when we know that we're the only process opening the + * lock file. + */ +typedef struct MDB_rxbody { + /** Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. + */ + volatile txnid_t mrb_txnid; + /** The process ID of the process owning this reader txn. */ + volatile MDB_PID_T mrb_pid; + /** The thread ID of the thread owning this txn. */ + volatile MDB_THR_T mrb_tid; +} MDB_rxbody; + + /** The actual reader record, with cacheline padding. */ +typedef struct MDB_reader { + union { + MDB_rxbody mrx; + /** shorthand for mrb_txnid */ +#define mr_txnid mru.mrx.mrb_txnid +#define mr_pid mru.mrx.mrb_pid +#define mr_tid mru.mrx.mrb_tid + /** cache line alignment */ + char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mru; +} MDB_reader; + + /** The header for the reader table. + * The table resides in a memory-mapped file. (This is a different file + * than is used for the main database.) + * + * For POSIX the actual mutexes reside in the shared memory of this + * mapped file. On Windows, mutexes are named objects allocated by the + * kernel; we store the mutex names in this mapped file so that other + * processes can grab them. This same approach is also used on + * MacOSX/Darwin (using named semaphores) since MacOSX doesn't support + * process-shared POSIX mutexes. For these cases where a named object + * is used, the object name is derived from a 64 bit FNV hash of the + * environment pathname. As such, naming collisions are extremely + * unlikely. If a collision occurs, the results are unpredictable. + */ +typedef struct MDB_txbody { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mtb_magic; + /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ + uint32_t mtb_format; +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mtb_rmname[MNAME_LEN]; +#else + /** Mutex protecting access to this table. + * This is the reader table lock used with LOCK_MUTEX(). + */ + mdb_mutex_t mtb_rmutex; +#endif + /** The ID of the last transaction committed to the database. + * This is recorded here only for convenience; the value can always + * be determined by reading the main database meta pages. + */ + volatile txnid_t mtb_txnid; + /** The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. + */ + volatile unsigned mtb_numreaders; +} MDB_txbody; + + /** The actual reader table definition. */ +typedef struct MDB_txninfo { + union { + MDB_txbody mtb; +#define mti_magic mt1.mtb.mtb_magic +#define mti_format mt1.mtb.mtb_format +#define mti_rmutex mt1.mtb.mtb_rmutex +#define mti_rmname mt1.mtb.mtb_rmname +#define mti_txnid mt1.mtb.mtb_txnid +#define mti_numreaders mt1.mtb.mtb_numreaders + char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mt1; + union { +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mt2_wmname[MNAME_LEN]; +#define mti_wmname mt2.mt2_wmname +#else + mdb_mutex_t mt2_wmutex; +#define mti_wmutex mt2.mt2_wmutex +#endif + char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; + } mt2; + MDB_reader mti_readers[1]; +} MDB_txninfo; + + /** Lockfile format signature: version, features and field layout */ +#define MDB_LOCK_FORMAT \ + ((uint32_t) \ + ((MDB_LOCK_VERSION) \ + /* Flags which describe functionality */ \ + + (((MDB_PIDLOCK) != 0) << 16))) +/** @} */ + +/** Common header for all page types. The page type depends on #mp_flags. + * + * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with + * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages + * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header. + * + * #P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of #F_BIGDATA nodes. + * + * #P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot. + * + * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a freeDB record. + */ +typedef struct MDB_page { +#define mp_pgno mp_p.p_pgno +#define mp_next mp_p.p_next + union { + pgno_t p_pgno; /**< page number */ + struct MDB_page *p_next; /**< for in-memory list of freed pages */ + } mp_p; + uint16_t mp_pad; /**< key size if this is a LEAF2 page */ +/** @defgroup mdb_page Page Flags + * @ingroup internal + * Flags for the page headers. + * @{ + */ +#define P_BRANCH 0x01 /**< branch page */ +#define P_LEAF 0x02 /**< leaf page */ +#define P_OVERFLOW 0x04 /**< overflow page */ +#define P_META 0x08 /**< meta page */ +#define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ +#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ +#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /**< leave this page alone during spill */ +/** @} */ + uint16_t mp_flags; /**< @ref mdb_page */ +#define mp_lower mp_pb.pb.pb_lower +#define mp_upper mp_pb.pb.pb_upper +#define mp_pages mp_pb.pb_pages + union { + struct { + indx_t pb_lower; /**< lower bound of free space */ + indx_t pb_upper; /**< upper bound of free space */ + } pb; + uint32_t pb_pages; /**< number of overflow pages */ + } mp_pb; + indx_t mp_ptrs[0]; /**< dynamic size */ +} MDB_page; + +/** Alternate page header, for 2-byte aligned access */ +typedef struct MDB_page2 { + uint16_t mp2_p[sizeof(pgno_t)/2]; + uint16_t mp2_pad; + uint16_t mp2_flags; + indx_t mp2_lower; + indx_t mp2_upper; + indx_t mp2_ptrs[0]; +} MDB_page2; + +#define MP_PGNO(p) (((MDB_page2 *)(void *)(p))->mp2_p) +#define MP_PAD(p) (((MDB_page2 *)(void *)(p))->mp2_pad) +#define MP_FLAGS(p) (((MDB_page2 *)(void *)(p))->mp2_flags) +#define MP_LOWER(p) (((MDB_page2 *)(void *)(p))->mp2_lower) +#define MP_UPPER(p) (((MDB_page2 *)(void *)(p))->mp2_upper) +#define MP_PTRS(p) (((MDB_page2 *)(void *)(p))->mp2_ptrs) + + /** Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) + + /** Address of first usable data byte in a page, after the header */ +#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) + + /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ +#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) + + /** Number of nodes on a page */ +#define NUMKEYS(p) ((MP_LOWER(p) - (PAGEHDRSZ-PAGEBASE)) >> 1) + + /** The amount of space remaining in the page */ +#define SIZELEFT(p) (indx_t)(MP_UPPER(p) - MP_LOWER(p)) + + /** The percentage of space used in the page, in tenths of a percent. */ +#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ + ((env)->me_psize - PAGEHDRSZ)) + /** The minimum page fill factor, in tenths of a percent. + * Pages emptier than this are candidates for merging. + */ +#define FILL_THRESHOLD 250 + + /** Test if a page is a leaf page */ +#define IS_LEAF(p) F_ISSET(MP_FLAGS(p), P_LEAF) + /** Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) F_ISSET(MP_FLAGS(p), P_LEAF2) + /** Test if a page is a branch page */ +#define IS_BRANCH(p) F_ISSET(MP_FLAGS(p), P_BRANCH) + /** Test if a page is an overflow page */ +#define IS_OVERFLOW(p) F_ISSET(MP_FLAGS(p), P_OVERFLOW) + /** Test if a page is a sub page */ +#define IS_SUBP(p) F_ISSET(MP_FLAGS(p), P_SUBP) + + /** The number of overflow pages needed to store the given size. */ +#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) + + /** Link in #MDB_txn.%mt_loose_pgs list. + * Kept outside the page header, which is needed when reusing the page. + */ +#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) + + /** Header for a single key/data pair within a page. + * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. + * We guarantee 2-byte alignment for 'MDB_node's. + * + * #mn_lo and #mn_hi are used for data size on leaf nodes, and for child + * pgno on branch nodes. On 64 bit platforms, #mn_flags is also used + * for pgno. (Branch nodes have no flags). Lo and hi are in host byte + * order in case some accesses can be optimized to 32-bit word access. + * + * Leaf node flags describe node contents. #F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just #F_SUBDATA). + */ +typedef struct MDB_node { + /** part of data size or pgno + * @{ */ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short mn_lo, mn_hi; +#else + unsigned short mn_hi, mn_lo; +#endif + /** @} */ +/** @defgroup mdb_node Node Flags + * @ingroup internal + * Flags for node headers. + * @{ + */ +#define F_BIGDATA 0x01 /**< data put on overflow page */ +#define F_SUBDATA 0x02 /**< data is a sub-database */ +#define F_DUPDATA 0x04 /**< data has duplicates */ + +/** valid flags for #mdb_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) + +/** @} */ + unsigned short mn_flags; /**< @ref mdb_node */ + unsigned short mn_ksize; /**< key size */ + char mn_data[1]; /**< key and data are appended here */ +} MDB_node; + + /** Size of the node header, excluding dynamic data at the end */ +#define NODESIZE offsetof(MDB_node, mn_data) + + /** Bit position of top word in page number, for shifting mn_flags */ +#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) + + /** Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. + */ +#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) + + /** Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. + */ +#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) + + /** Address of node \b i in page \b p */ +#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + MP_PTRS(p)[i] + PAGEBASE)) + + /** Address of the key for the node */ +#define NODEKEY(node) (void *)((node)->mn_data) + + /** Address of the data for a node */ +#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) + + /** Get the page number pointed to by a branch node */ +#define NODEPGNO(node) \ + ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ + (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) + /** Set the page number in a branch node */ +#define SETPGNO(node,pgno) do { \ + (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ + if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) + + /** Get the size of the data in a leaf node */ +#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) + /** Set the size of the data for a leaf node */ +#define SETDSZ(node,size) do { \ + (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) + /** The size of a key in a node */ +#define NODEKSZ(node) ((node)->mn_ksize) + + /** Copy a page number from src to dst */ +#ifdef MISALIGNED_OK +#define COPY_PGNO(dst,src) dst = src +#undef MP_PGNO +#define MP_PGNO(p) ((p)->mp_pgno) +#else +#if SIZE_MAX > 4294967295UL +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d++ = *s++; \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#else +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#endif +#endif + /** The address of a key in a LEAF2 page. + * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. + * There are no node headers, keys are stored contiguously. + */ +#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) + + /** Set the \b node's key into \b keyptr, if requested. */ +#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ + (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } + + /** Set the \b node's key into \b key. */ +#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } + + /** Information about a single database in the environment. */ +typedef struct MDB_db { + uint32_t md_pad; /**< also ksize for LEAF2 pages */ + uint16_t md_flags; /**< @ref mdb_dbi_open */ + uint16_t md_depth; /**< depth of this tree */ + pgno_t md_branch_pages; /**< number of internal pages */ + pgno_t md_leaf_pages; /**< number of leaf pages */ + pgno_t md_overflow_pages; /**< number of overflow pages */ + size_t md_entries; /**< number of data items */ + pgno_t md_root; /**< the root page of this tree */ +} MDB_db; + +#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ +#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) + /** #mdb_dbi_open() flags */ +#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ + MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) + + /** Handle for the DB used to track free pages. */ +#define FREE_DBI 0 + /** Handle for the default DB. */ +#define MAIN_DBI 1 + /** Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + + /** Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 2 + + /** Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). + */ +typedef struct MDB_meta { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mm_magic; + /** Version number of this file. Must be set to #MDB_DATA_VERSION. */ + uint32_t mm_version; + void *mm_address; /**< address for fixed mapping */ + size_t mm_mapsize; /**< size of mmap region */ + MDB_db mm_dbs[CORE_DBS]; /**< first is free space, 2nd is main db */ + /** The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_pad + /** Any persistent environment flags. @ref mdb_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + /** Last used page in the datafile. + * Actually the file may be shorter if the freeDB lists the final pages. + */ + pgno_t mm_last_pg; + volatile txnid_t mm_txnid; /**< txnid that committed this page */ +} MDB_meta; + + /** Buffer for a stack-allocated meta page. + * The members define size and alignment, and silence type + * aliasing warnings. They are not used directly; that could + * mean incorrectly using several union members in parallel. + */ +typedef union MDB_metabuf { + MDB_page mb_page; + struct { + char mm_pad[PAGEHDRSZ]; + MDB_meta mm_meta; + } mb_metabuf; +} MDB_metabuf; + + /** Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. + */ +typedef struct MDB_dbx { + MDB_val md_name; /**< name of the database */ + MDB_cmp_func *md_cmp; /**< function for comparing keys */ + MDB_cmp_func *md_dcmp; /**< function for comparing data items */ + MDB_rel_func *md_rel; /**< user relocate function */ + void *md_relctx; /**< user-provided context for md_rel */ +} MDB_dbx; + + /** A database transaction. + * Every operation requires a transaction handle. + */ +struct MDB_txn { + MDB_txn *mt_parent; /**< parent of a nested txn */ + /** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */ + MDB_txn *mt_child; + pgno_t mt_next_pgno; /**< next unallocated page */ + /** The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. + */ + txnid_t mt_txnid; + MDB_env *mt_env; /**< the DB environment */ + /** The list of pages that became unused during this transaction. + */ + MDB_IDL mt_free_pgs; + /** The list of loose pages that became unused and may be reused + * in this transaction, linked through #NEXT_LOOSE_PAGE(page). + */ + MDB_page *mt_loose_pgs; + /** Number of loose pages (#mt_loose_pgs) */ + int mt_loose_count; + /** The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. + */ + MDB_IDL mt_spill_pgs; + union { + /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */ + MDB_ID2L dirty_list; + /** For read txns: This thread/txn's reader table slot, or NULL. */ + MDB_reader *reader; + } mt_u; + /** Array of records for each DB known in the environment. */ + MDB_dbx *mt_dbxs; + /** Array of MDB_db records for each known DB */ + MDB_db *mt_dbs; + /** Array of sequence numbers for each DB handle */ + unsigned int *mt_dbiseqs; +/** @defgroup mt_dbflag Transaction DB Flags + * @ingroup internal + * @{ + */ +#define DB_DIRTY 0x01 /**< DB was written in this txn */ +#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ +#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ +#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ +#define DB_USRVALID 0x10 /**< As #DB_VALID, but not set for #FREE_DBI */ +#define DB_DUPDATA 0x20 /**< DB is #MDB_DUPSORT data */ +/** @} */ + /** In write txns, array of cursors for each DB */ + MDB_cursor **mt_cursors; + /** Array of flags for each DB */ + unsigned char *mt_dbflags; + /** Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. + */ + MDB_dbi mt_numdbs; + +/** @defgroup mdb_txn Transaction Flags + * @ingroup internal + * @{ + */ + /** #mdb_txn_begin() flags */ +#define MDB_TXN_BEGIN_FLAGS MDB_RDONLY +#define MDB_TXN_RDONLY MDB_RDONLY /**< read-only transaction */ + /* internal txn flags */ +#define MDB_TXN_WRITEMAP MDB_WRITEMAP /**< copy of #MDB_env flag in writers */ +#define MDB_TXN_FINISHED 0x01 /**< txn is finished or never began */ +#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ +#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ +#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ +#define MDB_TXN_HAS_CHILD 0x10 /**< txn has an #MDB_txn.%mt_child */ + /** most operations on the txn are currently illegal */ +#define MDB_TXN_BLOCKED (MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD) +/** @} */ + unsigned int mt_flags; /**< @ref mdb_txn */ + /** #dirty_list room: Array size - \#dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirty_list into mt_parent after freeing hidden mt_parent pages. + */ + unsigned int mt_dirty_room; +}; + +/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. + */ +#define CURSOR_STACK 32 + +struct MDB_xcursor; + + /** Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. #MDB_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a #P_SUBP page can be stale. + * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). + */ +struct MDB_cursor { + /** Next cursor on this DB in this txn */ + MDB_cursor *mc_next; + /** Backup of the original cursor if this cursor is a shadow */ + MDB_cursor *mc_backup; + /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ + struct MDB_xcursor *mc_xcursor; + /** The transaction that owns this cursor */ + MDB_txn *mc_txn; + /** The database handle this cursor operates on */ + MDB_dbi mc_dbi; + /** The database record for this cursor */ + MDB_db *mc_db; + /** The database auxiliary record for this cursor */ + MDB_dbx *mc_dbx; + /** The @ref mt_dbflag for this database */ + unsigned char *mc_dbflag; + unsigned short mc_snum; /**< number of pushed pages */ + unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ +/** @defgroup mdb_cursor Cursor Flags + * @ingroup internal + * Cursor state flags. + * @{ + */ +#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ +#define C_EOF 0x02 /**< No more data */ +#define C_SUB 0x04 /**< Cursor is a sub-cursor */ +#define C_DEL 0x08 /**< last op was a cursor_del */ +#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ +/** @} */ + unsigned int mc_flags; /**< @ref mdb_cursor */ + MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ +}; + + /** Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. + */ +typedef struct MDB_xcursor { + /** A sub-cursor for traversing the Dup DB */ + MDB_cursor mx_cursor; + /** The database record for this Dup DB */ + MDB_db mx_db; + /** The auxiliary DB record for this Dup DB */ + MDB_dbx mx_dbx; + /** The @ref mt_dbflag for this Dup DB */ + unsigned char mx_dbflag; +} MDB_xcursor; + + /** Check if there is an inited xcursor */ +#define XCURSOR_INITED(mc) \ + ((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + + /** Update the xcursor's sub-page pointer, if any, in \b mc. Needed + * when the node which contains the sub-page may have moved. Called + * with leaf page \b mp = mc->mc_pg[\b top]. + */ +#define XCURSOR_REFRESH(mc, top, mp) do { \ + MDB_page *xr_pg = (mp); \ + MDB_node *xr_node; \ + if (!XCURSOR_INITED(mc) || (mc)->mc_ki[top] >= NUMKEYS(xr_pg)) break; \ + xr_node = NODEPTR(xr_pg, (mc)->mc_ki[top]); \ + if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \ + (mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \ +} while (0) + + /** State of FreeDB old pages, stored in the MDB_env */ +typedef struct MDB_pgstate { + pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ + txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ +} MDB_pgstate; + + /** The database environment. */ +struct MDB_env { + HANDLE me_fd; /**< The main data file */ + HANDLE me_lfd; /**< The lock file */ + HANDLE me_mfd; /**< For writing and syncing the meta pages */ + /** Failed to update the meta page. Probably an I/O error. */ +#define MDB_FATAL_ERROR 0x80000000U + /** Some fields are initialized. */ +#define MDB_ENV_ACTIVE 0x20000000U + /** me_txkey is set */ +#define MDB_ENV_TXKEY 0x10000000U + /** fdatasync is unreliable */ +#define MDB_FSYNCONLY 0x08000000U + uint32_t me_flags; /**< @ref mdb_env */ + unsigned int me_psize; /**< DB page size, inited from me_os_psize */ + unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ + unsigned int me_maxreaders; /**< size of the reader table */ + /** Max #MDB_txninfo.%mti_numreaders of interest to #mdb_env_close() */ + volatile int me_close_readers; + MDB_dbi me_numdbs; /**< number of DBs opened */ + MDB_dbi me_maxdbs; /**< size of the DB table */ + MDB_PID_T me_pid; /**< process ID of this env */ + char *me_path; /**< path to the DB files */ + char *me_map; /**< the memory map of the data file */ + MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ + MDB_meta *me_metas[NUM_METAS]; /**< pointers to the two meta pages */ + void *me_pbuf; /**< scratch area for DUPSORT put() */ + MDB_txn *me_txn; /**< current write transaction */ + MDB_txn *me_txn0; /**< prealloc'd write transaction */ + size_t me_mapsize; /**< size of the data memory map */ + off_t me_size; /**< current file size */ + pgno_t me_maxpg; /**< me_mapsize / me_psize */ + MDB_dbx *me_dbxs; /**< array of static DB info */ + uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ + unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ + pthread_key_t me_txkey; /**< thread-key for readers */ + txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ + MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ +# define me_pglast me_pgstate.mf_pglast +# define me_pghead me_pgstate.mf_pghead + MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ + /** IDL of pages that became unused in a write txn */ + MDB_IDL me_free_pgs; + /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ + MDB_ID2L me_dirty_list; + /** Max number of freelist items that can fit in a single overflow page */ + int me_maxfree_1pg; + /** Max size of a node on a page */ + unsigned int me_nodemax; +#if !(MDB_MAXKEYSIZE) + unsigned int me_maxkey; /**< max size of a key */ +#endif + int me_live_reader; /**< have liveness lock in reader table */ +#ifdef _WIN32 + int me_pidquery; /**< Used in OpenProcess */ +#endif +#ifdef MDB_USE_POSIX_MUTEX /* Posix mutexes reside in shared mem */ +# define me_rmutex me_txns->mti_rmutex /**< Shared reader lock */ +# define me_wmutex me_txns->mti_wmutex /**< Shared writer lock */ +#else + mdb_mutex_t me_rmutex; + mdb_mutex_t me_wmutex; +#endif + void *me_userctx; /**< User-settable context */ + MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ +}; + + /** Nested transaction */ +typedef struct MDB_ntxn { + MDB_txn mnt_txn; /**< the transaction */ + MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ +} MDB_ntxn; + + /** max number of pages to commit in one writev() call */ +#define MDB_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES +#undef MDB_COMMIT_PAGES +#define MDB_COMMIT_PAGES IOV_MAX +#endif + + /** max bytes to write in one call */ +#define MAX_WRITE (0x40000000U >> (sizeof(ssize_t) == 4)) + + /** Check \b txn and \b dbi arguments to a function */ +#define TXN_DBI_EXIST(txn, dbi, validity) \ + ((txn) && (dbi)<(txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & (validity))) + + /** Check for misused \b dbi handles */ +#define TXN_DBI_CHANGED(txn, dbi) \ + ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) + +static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); +static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); +static int mdb_page_touch(MDB_cursor *mc); + +#define MDB_END_NAMES {"committed", "empty-commit", "abort", "reset", \ + "reset-tmp", "fail-begin", "fail-beginchild"} +enum { + /* mdb_txn_end operation number, for logging */ + MDB_END_COMMITTED, MDB_END_EMPTY_COMMIT, MDB_END_ABORT, MDB_END_RESET, + MDB_END_RESET_TMP, MDB_END_FAIL_BEGIN, MDB_END_FAIL_BEGINCHILD +}; +#define MDB_END_OPMASK 0x0F /**< mask for #mdb_txn_end() operation number */ +#define MDB_END_UPDATE 0x10 /**< update env state (DBIs) */ +#define MDB_END_FREE 0x20 /**< free txn unless it is #MDB_env.%me_txn0 */ +#define MDB_END_SLOT MDB_NOTLS /**< release any reader slot if #MDB_NOTLS */ +static void mdb_txn_end(MDB_txn *txn, unsigned mode); + +static int mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **mp, int *lvl); +static int mdb_page_search_root(MDB_cursor *mc, + MDB_val *key, int modify); +#define MDB_PS_MODIFY 1 +#define MDB_PS_ROOTONLY 2 +#define MDB_PS_FIRST 4 +#define MDB_PS_LAST 8 +static int mdb_page_search(MDB_cursor *mc, + MDB_val *key, int flags); +static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); + +#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ +static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, + pgno_t newpgno, unsigned int nflags); + +static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); +static MDB_meta *mdb_env_pick_meta(const MDB_env *env); +static int mdb_env_write_meta(MDB_txn *txn); +#if defined(MDB_USE_POSIX_MUTEX) && !defined(MDB_ROBUST_SUPPORTED) /* Drop unused excl arg */ +# define mdb_env_close0(env, excl) mdb_env_close1(env) +#endif +static void mdb_env_close0(MDB_env *env, int excl); + +static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); +static int mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); +static void mdb_node_del(MDB_cursor *mc, int ksize); +static void mdb_node_shrink(MDB_page *mp, indx_t indx); +static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft); +static int mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data); +static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); +static size_t mdb_branch_size(MDB_env *env, MDB_val *key); + +static int mdb_rebalance(MDB_cursor *mc); +static int mdb_update_key(MDB_cursor *mc, MDB_val *key); + +static void mdb_cursor_pop(MDB_cursor *mc); +static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); + +static int mdb_cursor_del0(MDB_cursor *mc); +static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); +static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); +static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, + int *exactp); +static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); +static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); + +static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); +static void mdb_xcursor_init0(MDB_cursor *mc); +static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); +static void mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int force); + +static int mdb_drop0(MDB_cursor *mc, int subs); +static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); +static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); + +/** @cond */ +static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; +/** @endcond */ + +/** Compare two items pointing at size_t's of unknown alignment. */ +#ifdef MISALIGNED_OK +# define mdb_cmp_clong mdb_cmp_long +#else +# define mdb_cmp_clong mdb_cmp_cint +#endif + +#ifdef _WIN32 +static SECURITY_DESCRIPTOR mdb_null_sd; +static SECURITY_ATTRIBUTES mdb_all_sa; +static int mdb_sec_inited; + +struct MDB_name; +static int utf8_to_utf16(const char *src, struct MDB_name *dst, int xtra); +#endif + +/** Return the library version info. */ +char * ESECT +mdb_version(int *major, int *minor, int *patch) +{ + if (major) *major = MDB_VERSION_MAJOR; + if (minor) *minor = MDB_VERSION_MINOR; + if (patch) *patch = MDB_VERSION_PATCH; + return MDB_VERSION_STRING; +} + +/** Table of descriptions for LMDB @ref errors */ +static char *const mdb_errstr[] = { + "MDB_KEYEXIST: Key/data pair already exists", + "MDB_NOTFOUND: No matching key/data pair found", + "MDB_PAGE_NOTFOUND: Requested page not found", + "MDB_CORRUPTED: Located page was wrong type", + "MDB_PANIC: Update of meta page failed or environment had fatal error", + "MDB_VERSION_MISMATCH: Database environment version mismatch", + "MDB_INVALID: File is not an LMDB file", + "MDB_MAP_FULL: Environment mapsize limit reached", + "MDB_DBS_FULL: Environment maxdbs limit reached", + "MDB_READERS_FULL: Environment maxreaders limit reached", + "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", + "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", + "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", + "MDB_PAGE_FULL: Internal error - page has no more space", + "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", + "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", + "MDB_BAD_TXN: Transaction must abort, has a child, or is invalid", + "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", + "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", +}; + +char * +mdb_strerror(int err) +{ +#ifdef _WIN32 + /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. + * This works as long as no function between the call to mdb_strerror + * and the actual use of the message uses more than 4K of stack. + */ +#define MSGSIZE 1024 +#define PADSIZE 4096 + char buf[MSGSIZE+PADSIZE], *ptr = buf; +#endif + int i; + if (!err) + return ("Successful return: 0"); + + if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { + i = err - MDB_KEYEXIST; + return mdb_errstr[i]; + } + +#ifdef _WIN32 + /* These are the C-runtime error codes we use. The comment indicates + * their numeric value, and the Win32 error they would correspond to + * if the error actually came from a Win32 API. A major mess, we should + * have used LMDB-specific error codes for everything. + */ + switch(err) { + case ENOENT: /* 2, FILE_NOT_FOUND */ + case EIO: /* 5, ACCESS_DENIED */ + case ENOMEM: /* 12, INVALID_ACCESS */ + case EACCES: /* 13, INVALID_DATA */ + case EBUSY: /* 16, CURRENT_DIRECTORY */ + case EINVAL: /* 22, BAD_COMMAND */ + case ENOSPC: /* 28, OUT_OF_PAPER */ + return strerror(err); + default: + ; + } + buf[0] = 0; + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE); + return ptr; +#else + if (err < 0) + return "Invalid error code"; + return strerror(err); +#endif +} + +/** assert(3) variant in cursor context */ +#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) +/** assert(3) variant in transaction context */ +#define mdb_tassert(txn, expr) mdb_assert0((txn)->mt_env, expr, #expr) +/** assert(3) variant in environment context */ +#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) + +#ifndef NDEBUG +# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ + mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) + +static void ESECT +mdb_assert_fail(MDB_env *env, const char *expr_txt, + const char *func, const char *file, int line) +{ + char buf[400]; + sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", + file, line, expr_txt, func); + if (env->me_assert_func) + env->me_assert_func(env, buf); + fprintf(stderr, "%s\n", buf); + abort(); +} +#else +# define mdb_assert0(env, expr, expr_txt) ((void) 0) +#endif /* NDEBUG */ + +#if MDB_DEBUG +/** Return the page number of \b mp which may be sub-page, for debug output */ +static pgno_t +mdb_dbg_pgno(MDB_page *mp) +{ + pgno_t ret; + COPY_PGNO(ret, MP_PGNO(mp)); + return ret; +} + +/** Display a key in hexadecimal and return the address of the result. + * @param[in] key the key to display + * @param[in] buf the buffer to write into. Should always be #DKBUF. + * @return The key in hexadecimal form. + */ +char * +mdb_dkey(MDB_val *key, char *buf) +{ + char *ptr = buf; + unsigned char *c = key->mv_data; + unsigned int i; + + if (!key) + return ""; + + if (key->mv_size > DKBUF_MAXKEYSIZE) + return "MDB_MAXKEYSIZE"; + /* may want to make this a dynamic check: if the key is mostly + * printable characters, print it as-is instead of converting to hex. + */ +#if 1 + buf[0] = '\0'; + for (i=0; imv_size; i++) + ptr += sprintf(ptr, "%02x", *c++); +#else + sprintf(buf, "%.*s", key->mv_size, key->mv_data); +#endif + return buf; +} + +static const char * +mdb_leafnode_type(MDB_node *n) +{ + static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; + return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : + tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; +} + +/** Display all the keys in the page. */ +void +mdb_page_list(MDB_page *mp) +{ + pgno_t pgno = mdb_dbg_pgno(mp); + const char *type, *state = (MP_FLAGS(mp) & P_DIRTY) ? ", dirty" : ""; + MDB_node *node; + unsigned int i, nkeys, nsize, total = 0; + MDB_val key; + DKBUF; + + switch (MP_FLAGS(mp) & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { + case P_BRANCH: type = "Branch page"; break; + case P_LEAF: type = "Leaf page"; break; + case P_LEAF|P_SUBP: type = "Sub-page"; break; + case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; + case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; + case P_OVERFLOW: + fprintf(stderr, "Overflow page %"Z"u pages %u%s\n", + pgno, mp->mp_pages, state); + return; + case P_META: + fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n", + pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); + return; + default: + fprintf(stderr, "Bad page %"Z"u flags 0x%X\n", pgno, MP_FLAGS(mp)); + return; + } + + nkeys = NUMKEYS(mp); + fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state); + + for (i=0; imp_pad; + key.mv_data = LEAF2KEY(mp, i, nsize); + total += nsize; + fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); + continue; + } + node = NODEPTR(mp, i); + key.mv_size = node->mn_ksize; + key.mv_data = node->mn_data; + nsize = NODESIZE + key.mv_size; + if (IS_BRANCH(mp)) { + fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node), + DKEY(&key)); + total += nsize; + } else { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + nsize += sizeof(pgno_t); + else + nsize += NODEDSZ(node); + total += nsize; + nsize += sizeof(indx_t); + fprintf(stderr, "key %d: nsize %d, %s%s\n", + i, nsize, DKEY(&key), mdb_leafnode_type(node)); + } + total = EVEN(total); + } + fprintf(stderr, "Total: header %d + contents %d + unused %d\n", + IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + MP_LOWER(mp), total, SIZELEFT(mp)); +} + +void +mdb_cursor_chk(MDB_cursor *mc) +{ + unsigned int i; + MDB_node *node; + MDB_page *mp; + + if (!mc->mc_snum || !(mc->mc_flags & C_INITIALIZED)) return; + for (i=0; imc_top; i++) { + mp = mc->mc_pg[i]; + node = NODEPTR(mp, mc->mc_ki[i]); + if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) + printf("oops!\n"); + } + if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) + printf("ack!\n"); + if (XCURSOR_INITED(mc)) { + node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) && + mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) { + printf("blah!\n"); + } + } +} +#endif + +#if (MDB_DEBUG) > 2 +/** Count all the pages in each DB and in the freelist + * and make sure it matches the actual number of pages + * being used. + * All named DBs must be open for a correct count. + */ +static void mdb_audit(MDB_txn *txn) +{ + MDB_cursor mc; + MDB_val key, data; + MDB_ID freecount, count; + MDB_dbi i; + int rc; + + freecount = 0; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + mdb_tassert(txn, rc == MDB_NOTFOUND); + + count = 0; + for (i = 0; imt_numdbs; i++) { + MDB_xcursor mx; + if (!(txn->mt_dbflags[i] & DB_VALID)) + continue; + mdb_cursor_init(&mc, txn, i, &mx); + if (txn->mt_dbs[i].md_root == P_INVALID) + continue; + count += txn->mt_dbs[i].md_branch_pages + + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); + for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { + unsigned j; + MDB_page *mp; + mp = mc.mc_pg[mc.mc_top]; + for (j=0; jmn_flags & F_SUBDATA) { + MDB_db db; + memcpy(&db, NODEDATA(leaf), sizeof(db)); + count += db.md_branch_pages + db.md_leaf_pages + + db.md_overflow_pages; + } + } + } + mdb_tassert(txn, rc == MDB_NOTFOUND); + } + } + if (freecount + count + NUM_METAS != txn->mt_next_pgno) { + fprintf(stderr, "audit: %"Z"u freecount: %"Z"u count: %"Z"u total: %"Z"u next_pgno: %"Z"u\n", + txn->mt_txnid, freecount, count+NUM_METAS, + freecount+count+NUM_METAS, txn->mt_next_pgno); + } +} +#endif + +int +mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_cmp(a, b); +} + +int +mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t)) + dcmp = mdb_cmp_clong; +#endif + return dcmp(a, b); +} + +/** Allocate memory for a page. + * Re-use old malloc'd pages first for singletons, otherwise just malloc. + * Set #MDB_TXN_ERROR on failure. + */ +static MDB_page * +mdb_page_malloc(MDB_txn *txn, unsigned num) +{ + MDB_env *env = txn->mt_env; + MDB_page *ret = env->me_dpages; + size_t psize = env->me_psize, sz = psize, off; + /* For ! #MDB_NOMEMINIT, psize counts how much to init. + * For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. + */ + if (num == 1) { + if (ret) { + VGMEMP_ALLOC(env, ret, sz); + VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); + env->me_dpages = ret->mp_next; + return ret; + } + psize -= off = PAGEHDRSZ; + } else { + sz *= num; + off = sz - psize; + } + if ((ret = malloc(sz)) != NULL) { + VGMEMP_ALLOC(env, ret, sz); + if (!(env->me_flags & MDB_NOMEMINIT)) { + memset((char *)ret + off, 0, psize); + ret->mp_pad = 0; + } + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + return ret; +} +/** Free a single page. + * Saves single pages to a list, for future reuse. + * (This is not used for multi-page overflow pages.) + */ +static void +mdb_page_free(MDB_env *env, MDB_page *mp) +{ + mp->mp_next = env->me_dpages; + VGMEMP_FREE(env, mp); + env->me_dpages = mp; +} + +/** Free a dirty page */ +static void +mdb_dpage_free(MDB_env *env, MDB_page *dp) +{ + if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { + mdb_page_free(env, dp); + } else { + /* large pages just get freed directly */ + VGMEMP_FREE(env, dp); + free(dp); + } +} + +/** Return all dirty pages to dpage list */ +static void +mdb_dlist_free(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned i, n = dl[0].mid; + + for (i = 1; i <= n; i++) { + mdb_dpage_free(env, dl[i].mptr); + } + dl[0].mid = 0; +} + +/** Loosen or free a single page. + * Saves single pages to a list for future reuse + * in this same txn. It has been pulled from the freeDB + * and already resides on the dirty list, but has been + * deleted. Use these pages first before pulling again + * from the freeDB. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. + */ +static int +mdb_page_loose(MDB_cursor *mc, MDB_page *mp) +{ + int loose = 0; + pgno_t pgno = mp->mp_pgno; + MDB_txn *txn = mc->mc_txn; + + if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { + if (txn->mt_parent) { + MDB_ID2 *dl = txn->mt_u.dirty_list; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + /* ok, it's ours */ + loose = 1; + } + } + } else { + /* no parent txn, so it's just ours */ + loose = 1; + } + } + if (loose) { + DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), + mp->mp_pgno)); + NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; + txn->mt_loose_pgs = mp; + txn->mt_loose_count++; + mp->mp_flags |= P_LOOSE; + } else { + int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); + if (rc) + return rc; + } + + return MDB_SUCCESS; +} + +/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. + * @param[in] mc A cursor handle for the current operation. + * @param[in] pflags Flags of the pages to update: + * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it. + * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). + * @return 0 on success, non-zero on failure. + */ +static int +mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) +{ + enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP }; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m3, *m0 = mc; + MDB_xcursor *mx; + MDB_page *dp, *mp; + MDB_node *leaf; + unsigned i, j; + int rc = MDB_SUCCESS, level; + + /* Mark pages seen by cursors */ + if (mc->mc_flags & C_UNTRACK) + mc = NULL; /* will find mc in mt_cursors */ + for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) { + for (; mc; mc=mc->mc_next) { + if (!(mc->mc_flags & C_INITIALIZED)) + continue; + for (m3 = mc;; m3 = &mx->mx_cursor) { + mp = NULL; + for (j=0; jmc_snum; j++) { + mp = m3->mc_pg[j]; + if ((mp->mp_flags & Mask) == pflags) + mp->mp_flags ^= P_KEEP; + } + mx = m3->mc_xcursor; + /* Proceed to mx if it is at a sub-database */ + if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) + break; + if (! (mp && (mp->mp_flags & P_LEAF))) + break; + leaf = NODEPTR(mp, m3->mc_ki[j-1]); + if (!(leaf->mn_flags & F_SUBDATA)) + break; + } + } + if (i == 0) + break; + } + + if (all) { + /* Mark dirty root pages */ + for (i=0; imt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + pgno_t pgno = txn->mt_dbs[i].md_root; + if (pgno == P_INVALID) + continue; + if ((rc = mdb_page_get(m0, pgno, &dp, &level)) != MDB_SUCCESS) + break; + if ((dp->mp_flags & Mask) == pflags && level <= 1) + dp->mp_flags ^= P_KEEP; + } + } + } + + return rc; +} + +static int mdb_page_flush(MDB_txn *txn, int keep); + +/** Spill pages from the dirty list back to disk. + * This is intended to prevent running into #MDB_TXN_FULL situations, + * but note that they may still occur in a few cases: + * 1) our estimate of the txn size could be too small. Currently this + * seems unlikely, except with a large number of #MDB_MULTIPLE items. + * 2) child txns may run out of space if their parents dirtied a + * lot of pages and never spilled them. TODO: we probably should do + * a preemptive spill during #mdb_txn_begin() of a child txn, if + * the parent's dirty_room is below a given threshold. + * + * Otherwise, if not using nested txns, it is expected that apps will + * not run into #MDB_TXN_FULL any more. The pages are flushed to disk + * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. + * If the txn never references them again, they can be left alone. + * If the txn only reads them, they can be used without any fuss. + * If the txn writes them again, they can be dirtied immediately without + * going thru all of the work of #mdb_page_touch(). Such references are + * handled by #mdb_page_unspill(). + * + * Also note, we never spill DB root pages, nor pages of active cursors, + * because we'll need these back again soon anyway. And in nested txns, + * we can't spill a page in a child txn if it was already spilled in a + * parent txn. That would alter the parent txns' data even though + * the child hasn't committed yet, and we'd have no way to undo it if + * the child aborted. + * + * @param[in] m0 cursor A cursor handle identifying the transaction and + * database for which we are checking space. + * @param[in] key For a put operation, the key being stored. + * @param[in] data For a put operation, the data being stored. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) +{ + MDB_txn *txn = m0->mc_txn; + MDB_page *dp; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned int i, j, need; + int rc; + + if (m0->mc_flags & C_SUB) + return MDB_SUCCESS; + + /* Estimate how much space this op will take */ + i = m0->mc_db->md_depth; + /* Named DBs also dirty the main DB */ + if (m0->mc_dbi >= CORE_DBS) + i += txn->mt_dbs[MAIN_DBI].md_depth; + /* For puts, roughly factor in the key+data size */ + if (key) + i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; + i += i; /* double it for good measure */ + need = i; + + if (txn->mt_dirty_room > i) + return MDB_SUCCESS; + + if (!txn->mt_spill_pgs) { + txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); + if (!txn->mt_spill_pgs) + return ENOMEM; + } else { + /* purge deleted slots */ + MDB_IDL sl = txn->mt_spill_pgs; + unsigned int num = sl[0]; + j=0; + for (i=1; i<=num; i++) { + if (!(sl[i] & 1)) + sl[++j] = sl[i]; + } + sl[0] = j; + } + + /* Preserve pages which may soon be dirtied again */ + if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS) + goto done; + + /* Less aggressive spill - we originally spilled the entire dirty list, + * with a few exceptions for cursor pages and DB root pages. But this + * turns out to be a lot of wasted effort because in a large txn many + * of those pages will need to be used again. So now we spill only 1/8th + * of the dirty pages. Testing revealed this to be a good tradeoff, + * better than 1/2, 1/4, or 1/10. + */ + if (need < MDB_IDL_UM_MAX / 8) + need = MDB_IDL_UM_MAX / 8; + + /* Save the page IDs of all the pages we're flushing */ + /* flush from the tail forward, this saves a lot of shifting later on. */ + for (i=dl[0].mid; i && need; i--) { + MDB_ID pn = dl[i].mid << 1; + dp = dl[i].mptr; + if (dp->mp_flags & (P_LOOSE|P_KEEP)) + continue; + /* Can't spill twice, make sure it's not already in a parent's + * spill list. + */ + if (txn->mt_parent) { + MDB_txn *tx2; + for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { + if (tx2->mt_spill_pgs) { + j = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { + dp->mp_flags |= P_KEEP; + break; + } + } + } + if (tx2) + continue; + } + if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) + goto done; + need--; + } + mdb_midl_sort(txn->mt_spill_pgs); + + /* Flush the spilled part of dirty list */ + if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) + goto done; + + /* Reset any dirty pages we kept that page_flush didn't see */ + rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i); + +done: + txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; + return rc; +} + +/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ +static txnid_t +mdb_find_oldest(MDB_txn *txn) +{ + int i; + txnid_t mr, oldest = txn->mt_txnid - 1; + if (txn->mt_env->me_txns) { + MDB_reader *r = txn->mt_env->me_txns->mti_readers; + for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { + if (r[i].mr_pid) { + mr = r[i].mr_txnid; + if (oldest > mr) + oldest = mr; + } + } + } + return oldest; +} + +/** Add a page to the txn's dirty list */ +static void +mdb_page_dirty(MDB_txn *txn, MDB_page *mp) +{ + MDB_ID2 mid; + int rc, (*insert)(MDB_ID2L, MDB_ID2 *); + + if (txn->mt_flags & MDB_TXN_WRITEMAP) { + insert = mdb_mid2l_append; + } else { + insert = mdb_mid2l_insert; + } + mid.mid = mp->mp_pgno; + mid.mptr = mp; + rc = insert(txn->mt_u.dirty_list, &mid); + mdb_tassert(txn, rc == 0); + txn->mt_dirty_room--; +} + +/** Allocate page numbers and memory for writing. Maintain me_pglast, + * me_pghead and mt_next_pgno. Set #MDB_TXN_ERROR on failure. + * + * If there are free pages available from older transactions, they + * are re-used first. Otherwise allocate a new page at mt_next_pgno. + * Do not modify the freedB, just merge freeDB records into me_pghead[] + * and move me_pglast to say which records were consumed. Only this + * function can create me_pghead and move me_pglast/mt_next_pgno. + * @param[in] mc cursor A cursor handle identifying the transaction and + * database for which we are allocating. + * @param[in] num the number of pages to allocate. + * @param[out] mp Address of the allocated page(s). Requests for multiple pages + * will always be satisfied by a single contiguous chunk of memory. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) +{ +#ifdef MDB_PARANOID /* Seems like we can ignore this now */ + /* Get at most more freeDB records once me_pghead + * has enough pages. If not enough, use new pages from the map. + * If and mc is updating the freeDB, only get new + * records if me_pghead is empty. Then the freelist cannot play + * catch-up with itself by growing while trying to save it. + */ + enum { Paranoid = 1, Max_retries = 500 }; +#else + enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; +#endif + int rc, retry = num * 60; + MDB_txn *txn = mc->mc_txn; + MDB_env *env = txn->mt_env; + pgno_t pgno, *mop = env->me_pghead; + unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; + MDB_page *np; + txnid_t oldest = 0, last; + MDB_cursor_op op; + MDB_cursor m2; + int found_old = 0; + + /* If there are any loose pages, just use them */ + if (num == 1 && txn->mt_loose_pgs) { + np = txn->mt_loose_pgs; + txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); + txn->mt_loose_count--; + DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), + np->mp_pgno)); + *mp = np; + return MDB_SUCCESS; + } + + *mp = NULL; + + /* If our dirty list is already full, we can't do anything */ + if (txn->mt_dirty_room == 0) { + rc = MDB_TXN_FULL; + goto fail; + } + + for (op = MDB_FIRST;; op = MDB_NEXT) { + MDB_val key, data; + MDB_node *leaf; + pgno_t *idl; + + /* Seek a big enough contiguous page range. Prefer + * pages at the tail, just truncating the list. + */ + if (mop_len > n2) { + i = mop_len; + do { + pgno = mop[i]; + if (mop[i-n2] == pgno+n2) + goto search_done; + } while (--i > n2); + if (--retry < 0) + break; + } + + if (op == MDB_FIRST) { /* 1st iteration */ + /* Prepare to fetch more and coalesce */ + last = env->me_pglast; + oldest = env->me_pgoldest; + mdb_cursor_init(&m2, txn, FREE_DBI, NULL); + if (last) { + op = MDB_SET_RANGE; + key.mv_data = &last; /* will look up last+1 */ + key.mv_size = sizeof(last); + } + if (Paranoid && mc->mc_dbi == FREE_DBI) + retry = -1; + } + if (Paranoid && retry < 0 && mop_len) + break; + + last++; + /* Do not fetch more if the record will be too recent */ + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + rc = mdb_cursor_get(&m2, &key, NULL, op); + if (rc) { + if (rc == MDB_NOTFOUND) + break; + goto fail; + } + last = *(txnid_t*)key.mv_data; + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + np = m2.mc_pg[m2.mc_top]; + leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); + if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS) + goto fail; + + idl = (MDB_ID *) data.mv_data; + i = idl[0]; + if (!mop) { + if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { + rc = ENOMEM; + goto fail; + } + } else { + if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) + goto fail; + mop = env->me_pghead; + } + env->me_pglast = last; +#if (MDB_DEBUG) > 1 + DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", + last, txn->mt_dbs[FREE_DBI].md_root, i)); + for (j = i; j; j--) + DPRINTF(("IDL %"Z"u", idl[j])); +#endif + /* Merge in descending sorted order */ + mdb_midl_xmerge(mop, idl); + mop_len = mop[0]; + } + + /* Use new pages from the map when nothing suitable in the freeDB */ + i = 0; + pgno = txn->mt_next_pgno; + if (pgno + num >= env->me_maxpg) { + DPUTS("DB size maxed out"); + rc = MDB_MAP_FULL; + goto fail; + } + +search_done: + if (env->me_flags & MDB_WRITEMAP) { + np = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + if (!(np = mdb_page_malloc(txn, num))) { + rc = ENOMEM; + goto fail; + } + } + if (i) { + mop[0] = mop_len -= num; + /* Move any stragglers down */ + for (j = i-num; j < mop_len; ) + mop[++j] = mop[++i]; + } else { + txn->mt_next_pgno = pgno + num; + } + np->mp_pgno = pgno; + mdb_page_dirty(txn, np); + *mp = np; + + return MDB_SUCCESS; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Copy the used portions of a non-overflow page. + * @param[in] dst page to copy into + * @param[in] src page to copy from + * @param[in] psize size of a page + */ +static void +mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) +{ + enum { Align = sizeof(pgno_t) }; + indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; + + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. + */ + if ((unused &= -Align) && !IS_LEAF2(src)) { + upper = (upper + PAGEBASE) & -Align; + memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); + memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), + psize - upper); + } else { + memcpy(dst, src, psize - unused); + } +} + +/** Pull a page off the txn's spill list, if present. + * If a page being referenced was spilled to disk in this txn, bring + * it back and make it dirty/writable again. + * @param[in] txn the transaction handle. + * @param[in] mp the page being referenced. It must not be dirty. + * @param[out] ret the writable page, if any. ret is unchanged if + * mp wasn't spilled. + */ +static int +mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) +{ + MDB_env *env = txn->mt_env; + const MDB_txn *tx2; + unsigned x; + pgno_t pgno = mp->mp_pgno, pn = pgno << 1; + + for (tx2 = txn; tx2; tx2=tx2->mt_parent) { + if (!tx2->mt_spill_pgs) + continue; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + MDB_page *np; + int num; + if (txn->mt_dirty_room == 0) + return MDB_TXN_FULL; + if (IS_OVERFLOW(mp)) + num = mp->mp_pages; + else + num = 1; + if (env->me_flags & MDB_WRITEMAP) { + np = mp; + } else { + np = mdb_page_malloc(txn, num); + if (!np) + return ENOMEM; + if (num > 1) + memcpy(np, mp, num * env->me_psize); + else + mdb_page_copy(np, mp, env->me_psize); + } + if (tx2 == txn) { + /* If in current txn, this page is no longer spilled. + * If it happens to be the last page, truncate the spill list. + * Otherwise mark it as deleted by setting the LSB. + */ + if (x == txn->mt_spill_pgs[0]) + txn->mt_spill_pgs[0]--; + else + txn->mt_spill_pgs[x] |= 1; + } /* otherwise, if belonging to a parent txn, the + * page remains spilled until child commits + */ + + mdb_page_dirty(txn, np); + np->mp_flags |= P_DIRTY; + *ret = np; + break; + } + } + return MDB_SUCCESS; +} + +/** Touch a page: make it dirty and re-insert into tree with updated pgno. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc cursor pointing to the page to be touched + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_touch(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top], *np; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m2, *m3; + pgno_t pgno; + int rc; + + if (!F_ISSET(MP_FLAGS(mp), P_DIRTY)) { + if (txn->mt_flags & MDB_TXN_SPILLS) { + np = NULL; + rc = mdb_page_unspill(txn, mp, &np); + if (rc) + goto fail; + if (np) + goto done; + } + if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || + (rc = mdb_page_alloc(mc, 1, &np))) + goto fail; + pgno = np->mp_pgno; + DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc), + mp->mp_pgno, pgno)); + mdb_cassert(mc, mp->mp_pgno != pgno); + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* Update the parent page, if any, to point to the new page */ + if (mc->mc_top) { + MDB_page *parent = mc->mc_pg[mc->mc_top-1]; + MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); + SETPGNO(node, pgno); + } else { + mc->mc_db->md_root = pgno; + } + } else if (txn->mt_parent && !IS_SUBP(mp)) { + MDB_ID2 mid, *dl = txn->mt_u.dirty_list; + pgno = mp->mp_pgno; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + return 0; + } + } + mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); + /* No - copy it */ + np = mdb_page_malloc(txn, 1); + if (!np) + return ENOMEM; + mid.mid = pgno; + mid.mptr = np; + rc = mdb_mid2l_insert(dl, &mid); + mdb_cassert(mc, rc == 0); + } else { + return 0; + } + + mdb_page_copy(np, mp, txn->mt_env->me_psize); + np->mp_pgno = pgno; + np->mp_flags |= P_DIRTY; + +done: + /* Adjust cursors pointing to mp */ + mc->mc_pg[mc->mc_top] = np; + m2 = txn->mt_cursors[mc->mc_dbi]; + if (mc->mc_flags & C_SUB) { + for (; m2; m2=m2->mc_next) { + m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[mc->mc_top] == mp) + m3->mc_pg[mc->mc_top] = np; + } + } else { + for (; m2; m2=m2->mc_next) { + if (m2->mc_snum < mc->mc_snum) continue; + if (m2 == mc) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + m2->mc_pg[mc->mc_top] = np; + if (IS_LEAF(np)) + XCURSOR_REFRESH(m2, mc->mc_top, np); + } + } + } + return 0; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_env_sync(MDB_env *env, int force) +{ + int rc = 0; + if (env->me_flags & MDB_RDONLY) + return EACCES; + if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { + if (env->me_flags & MDB_WRITEMAP) { + int flags = ((env->me_flags & MDB_MAPASYNC) && !force) + ? MS_ASYNC : MS_SYNC; + if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) + rc = ErrCode(); +#ifdef _WIN32 + else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); +#endif + } else { +#ifdef BROKEN_FDATASYNC + if (env->me_flags & MDB_FSYNCONLY) { + if (fsync(env->me_fd)) + rc = ErrCode(); + } else +#endif + if (MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); + } + } + return rc; +} + +/** Back up parent txn's cursors, then grab the originals for tracking */ +static int +mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) +{ + MDB_cursor *mc, *bk; + MDB_xcursor *mx; + size_t size; + int i; + + for (i = src->mt_numdbs; --i >= 0; ) { + if ((mc = src->mt_cursors[i]) != NULL) { + size = sizeof(MDB_cursor); + if (mc->mc_xcursor) + size += sizeof(MDB_xcursor); + for (; mc; mc = bk->mc_next) { + bk = malloc(size); + if (!bk) + return ENOMEM; + *bk = *mc; + mc->mc_backup = bk; + mc->mc_db = &dst->mt_dbs[i]; + /* Kill pointers into src to reduce abuse: The + * user may not use mc until dst ends. But we need a valid + * txn pointer here for cursor fixups to keep working. + */ + mc->mc_txn = dst; + mc->mc_dbflag = &dst->mt_dbflags[i]; + if ((mx = mc->mc_xcursor) != NULL) { + *(MDB_xcursor *)(bk+1) = *mx; + mx->mx_cursor.mc_txn = dst; + } + mc->mc_next = dst->mt_cursors[i]; + dst->mt_cursors[i] = mc; + } + } + } + return MDB_SUCCESS; +} + +/** Close this write txn's cursors, give parent txn's cursors back to parent. + * @param[in] txn the transaction handle. + * @param[in] merge true to keep changes to parent cursors, false to revert. + * @return 0 on success, non-zero on failure. + */ +static void +mdb_cursors_close(MDB_txn *txn, unsigned merge) +{ + MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; + MDB_xcursor *mx; + int i; + + for (i = txn->mt_numdbs; --i >= 0; ) { + for (mc = cursors[i]; mc; mc = next) { + next = mc->mc_next; + if ((bk = mc->mc_backup) != NULL) { + if (merge) { + /* Commit changes to parent txn */ + mc->mc_next = bk->mc_next; + mc->mc_backup = bk->mc_backup; + mc->mc_txn = bk->mc_txn; + mc->mc_db = bk->mc_db; + mc->mc_dbflag = bk->mc_dbflag; + if ((mx = mc->mc_xcursor) != NULL) + mx->mx_cursor.mc_txn = bk->mc_txn; + } else { + /* Abort nested txn */ + *mc = *bk; + if ((mx = mc->mc_xcursor) != NULL) + *mx = *(MDB_xcursor *)(bk+1); + } + mc = bk; + } + /* Only malloced cursors are permanently tracked. */ + free(mc); + } + cursors[i] = NULL; + } +} + +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ +enum Pidlock_op { + Pidset, Pidcheck +}; +#else +enum Pidlock_op { + Pidset = F_SETLK, Pidcheck = F_GETLK +}; +#endif + +/** Set or check a pid lock. Set returns 0 on success. + * Check returns 0 if the process is certainly dead, nonzero if it may + * be alive (the lock exists or an error happened so we do not know). + * + * On Windows Pidset is a no-op, we merely check for the existence + * of the process with the given pid. On POSIX we use a single byte + * lock on the lockfile, set at an offset equal to the pid. + */ +static int +mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) +{ +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ + int ret = 0; + HANDLE h; + if (op == Pidcheck) { + h = OpenProcess(env->me_pidquery, FALSE, pid); + /* No documented "no such process" code, but other program use this: */ + if (!h) + return ErrCode() != ERROR_INVALID_PARAMETER; + /* A process exists until all handles to it close. Has it exited? */ + ret = WaitForSingleObject(h, 0) != 0; + CloseHandle(h); + } + return ret; +#else + for (;;) { + int rc; + struct flock lock_info; + memset(&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = pid; + lock_info.l_len = 1; + if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { + if (op == F_GETLK && lock_info.l_type != F_UNLCK) + rc = -1; + } else if ((rc = ErrCode()) == EINTR) { + continue; + } + return rc; + } +#endif +} + +/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). + * @param[in] txn the transaction handle to initialize + * @return 0 on success, non-zero on failure. + */ +static int +mdb_txn_renew0(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_txninfo *ti = env->me_txns; + MDB_meta *meta; + unsigned int i, nr, flags = txn->mt_flags; + uint16_t x; + int rc, new_notls = 0; + + if ((flags &= MDB_TXN_RDONLY) != 0) { + if (!ti) { + meta = mdb_env_pick_meta(env); + txn->mt_txnid = meta->mm_txnid; + txn->mt_u.reader = NULL; + } else { + MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : + pthread_getspecific(env->me_txkey); + if (r) { + if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) + return MDB_BAD_RSLOT; + } else { + MDB_PID_T pid = env->me_pid; + MDB_THR_T tid = pthread_self(); + mdb_mutexref_t rmutex = env->me_rmutex; + + if (!env->me_live_reader) { + rc = mdb_reader_pid(env, Pidset, pid); + if (rc) + return rc; + env->me_live_reader = 1; + } + + if (LOCK_MUTEX(rc, env, rmutex)) + return rc; + nr = ti->mti_numreaders; + for (i=0; imti_readers[i].mr_pid == 0) + break; + if (i == env->me_maxreaders) { + UNLOCK_MUTEX(rmutex); + return MDB_READERS_FULL; + } + r = &ti->mti_readers[i]; + /* Claim the reader slot, carefully since other code + * uses the reader table un-mutexed: First reset the + * slot, next publish it in mti_numreaders. After + * that, it is safe for mdb_env_close() to touch it. + * When it will be closed, we can finally claim it. + */ + r->mr_pid = 0; + r->mr_txnid = (txnid_t)-1; + r->mr_tid = tid; + if (i == nr) + ti->mti_numreaders = ++nr; + env->me_close_readers = nr; + r->mr_pid = pid; + UNLOCK_MUTEX(rmutex); + + new_notls = (env->me_flags & MDB_NOTLS); + if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { + r->mr_pid = 0; + return rc; + } + } + do /* LY: Retry on a race, ITS#7970. */ + r->mr_txnid = ti->mti_txnid; + while(r->mr_txnid != ti->mti_txnid); + txn->mt_txnid = r->mr_txnid; + txn->mt_u.reader = r; + meta = env->me_metas[txn->mt_txnid & 1]; + } + + } else { + /* Not yet touching txn == env->me_txn0, it may be active */ + if (ti) { + if (LOCK_MUTEX(rc, env, env->me_wmutex)) + return rc; + txn->mt_txnid = ti->mti_txnid; + meta = env->me_metas[txn->mt_txnid & 1]; + } else { + meta = mdb_env_pick_meta(env); + txn->mt_txnid = meta->mm_txnid; + } + txn->mt_txnid++; +#if MDB_DEBUG + if (txn->mt_txnid == mdb_debug_start) + mdb_debug = 1; +#endif + txn->mt_child = NULL; + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + txn->mt_dirty_room = MDB_IDL_UM_MAX; + txn->mt_u.dirty_list = env->me_dirty_list; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_free_pgs = env->me_free_pgs; + txn->mt_free_pgs[0] = 0; + txn->mt_spill_pgs = NULL; + env->me_txn = txn; + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); + } + + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db)); + + /* Moved to here to avoid a data race in read TXNs */ + txn->mt_next_pgno = meta->mm_last_pg+1; + + txn->mt_flags = flags; + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + for (i=CORE_DBS; imt_numdbs; i++) { + x = env->me_dbflags[i]; + txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_USRVALID|DB_STALE : 0; + } + txn->mt_dbflags[MAIN_DBI] = DB_VALID|DB_USRVALID; + txn->mt_dbflags[FREE_DBI] = DB_VALID; + + if (env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + rc = MDB_PANIC; + } else if (env->me_maxpg < txn->mt_next_pgno) { + rc = MDB_MAP_RESIZED; + } else { + return MDB_SUCCESS; + } + mdb_txn_end(txn, new_notls /*0 or MDB_END_SLOT*/ | MDB_END_FAIL_BEGIN); + return rc; +} + +int +mdb_txn_renew(MDB_txn *txn) +{ + int rc; + + if (!txn || !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY|MDB_TXN_FINISHED)) + return EINVAL; + + rc = mdb_txn_renew0(txn); + if (rc == MDB_SUCCESS) { + DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); + } + return rc; +} + +int +mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) +{ + MDB_txn *txn; + MDB_ntxn *ntxn; + int rc, size, tsize; + + flags &= MDB_TXN_BEGIN_FLAGS; + flags |= env->me_flags & MDB_WRITEMAP; + + if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */ + return EACCES; + + if (parent) { + /* Nested transactions: Max 1 child, write txns only, no writemap */ + flags |= parent->mt_flags; + if (flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_BLOCKED)) { + return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; + } + /* Child txns save MDB_pgstate and use own copy of cursors */ + size = env->me_maxdbs * (sizeof(MDB_db)+sizeof(MDB_cursor *)+1); + size += tsize = sizeof(MDB_ntxn); + } else if (flags & MDB_RDONLY) { + size = env->me_maxdbs * (sizeof(MDB_db)+1); + size += tsize = sizeof(MDB_txn); + } else { + /* Reuse preallocated write txn. However, do not touch it until + * mdb_txn_renew0() succeeds, since it currently may be active. + */ + txn = env->me_txn0; + goto renew; + } + if ((txn = calloc(1, size)) == NULL) { + DPRINTF(("calloc: %s", strerror(errno))); + return ENOMEM; + } + txn->mt_dbxs = env->me_dbxs; /* static */ + txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); + txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs; + txn->mt_flags = flags; + txn->mt_env = env; + + if (parent) { + unsigned int i; + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = parent->mt_dbiseqs; + txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); + if (!txn->mt_u.dirty_list || + !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) + { + free(txn->mt_u.dirty_list); + free(txn); + return ENOMEM; + } + txn->mt_txnid = parent->mt_txnid; + txn->mt_dirty_room = parent->mt_dirty_room; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_spill_pgs = NULL; + txn->mt_next_pgno = parent->mt_next_pgno; + parent->mt_flags |= MDB_TXN_HAS_CHILD; + parent->mt_child = txn; + txn->mt_parent = parent; + txn->mt_numdbs = parent->mt_numdbs; + memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + /* Copy parent's mt_dbflags, but clear DB_NEW */ + for (i=0; imt_numdbs; i++) + txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; + rc = 0; + ntxn = (MDB_ntxn *)txn; + ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ + if (env->me_pghead) { + size = MDB_IDL_SIZEOF(env->me_pghead); + env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); + if (env->me_pghead) + memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); + else + rc = ENOMEM; + } + if (!rc) + rc = mdb_cursor_shadow(parent, txn); + if (rc) + mdb_txn_end(txn, MDB_END_FAIL_BEGINCHILD); + } else { /* MDB_RDONLY */ + txn->mt_dbiseqs = env->me_dbiseqs; +renew: + rc = mdb_txn_renew0(txn); + } + if (rc) { + if (txn != env->me_txn0) + free(txn); + } else { + txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ + *ret = txn; + DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w', + (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); + } + + return rc; +} + +MDB_env * +mdb_txn_env(MDB_txn *txn) +{ + if(!txn) return NULL; + return txn->mt_env; +} + +size_t +mdb_txn_id(MDB_txn *txn) +{ + if(!txn) return 0; + return txn->mt_txnid; +} + +/** Export or close DBI handles opened in this txn. */ +static void +mdb_dbis_update(MDB_txn *txn, int keep) +{ + int i; + MDB_dbi n = txn->mt_numdbs; + MDB_env *env = txn->mt_env; + unsigned char *tdbflags = txn->mt_dbflags; + + for (i = n; --i >= CORE_DBS;) { + if (tdbflags[i] & DB_NEW) { + if (keep) { + env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; + } else { + char *ptr = env->me_dbxs[i].md_name.mv_data; + if (ptr) { + env->me_dbxs[i].md_name.mv_data = NULL; + env->me_dbxs[i].md_name.mv_size = 0; + env->me_dbflags[i] = 0; + env->me_dbiseqs[i]++; + free(ptr); + } + } + } + } + if (keep && env->me_numdbs < n) + env->me_numdbs = n; +} + +/** End a transaction, except successful commit of a nested transaction. + * May be called twice for readonly txns: First reset it, then abort. + * @param[in] txn the transaction handle to end + * @param[in] mode why and how to end the transaction + */ +static void +mdb_txn_end(MDB_txn *txn, unsigned mode) +{ + MDB_env *env = txn->mt_env; +#if MDB_DEBUG + static const char *const names[] = MDB_END_NAMES; +#endif + + /* Export or close DBI handles opened in this txn */ + mdb_dbis_update(txn, mode & MDB_END_UPDATE); + + DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + names[mode & MDB_END_OPMASK], + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + if (txn->mt_u.reader) { + txn->mt_u.reader->mr_txnid = (txnid_t)-1; + if (!(env->me_flags & MDB_NOTLS)) { + txn->mt_u.reader = NULL; /* txn does not own reader */ + } else if (mode & MDB_END_SLOT) { + txn->mt_u.reader->mr_pid = 0; + txn->mt_u.reader = NULL; + } /* else txn owns the slot until it does MDB_END_SLOT */ + } + txn->mt_numdbs = 0; /* prevent further DBI activity */ + txn->mt_flags |= MDB_TXN_FINISHED; + + } else if (!F_ISSET(txn->mt_flags, MDB_TXN_FINISHED)) { + pgno_t *pghead = env->me_pghead; + + if (!(mode & MDB_END_UPDATE)) /* !(already closed cursors) */ + mdb_cursors_close(txn, 0); + if (!(env->me_flags & MDB_WRITEMAP)) { + mdb_dlist_free(txn); + } + + txn->mt_numdbs = 0; + txn->mt_flags = MDB_TXN_FINISHED; + + if (!txn->mt_parent) { + mdb_midl_shrink(&txn->mt_free_pgs); + env->me_free_pgs = txn->mt_free_pgs; + /* me_pgstate: */ + env->me_pghead = NULL; + env->me_pglast = 0; + + env->me_txn = NULL; + mode = 0; /* txn == env->me_txn0, do not free() it */ + + /* The writer mutex was locked in mdb_txn_begin. */ + if (env->me_txns) + UNLOCK_MUTEX(env->me_wmutex); + } else { + txn->mt_parent->mt_child = NULL; + txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD; + env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; + mdb_midl_free(txn->mt_free_pgs); + free(txn->mt_u.dirty_list); + } + mdb_midl_free(txn->mt_spill_pgs); + + mdb_midl_free(pghead); + } + + if (mode & MDB_END_FREE) + free(txn); +} + +void +mdb_txn_reset(MDB_txn *txn) +{ + if (txn == NULL) + return; + + /* This call is only valid for read-only txns */ + if (!(txn->mt_flags & MDB_TXN_RDONLY)) + return; + + mdb_txn_end(txn, MDB_END_RESET); +} + +void +mdb_txn_abort(MDB_txn *txn) +{ + if (txn == NULL) + return; + + if (txn->mt_child) + mdb_txn_abort(txn->mt_child); + + mdb_txn_end(txn, MDB_END_ABORT|MDB_END_SLOT|MDB_END_FREE); +} + +/** Save the freelist as of this transaction to the freeDB. + * This changes the freelist. Keep trying until it stabilizes. + */ +static int +mdb_freelist_save(MDB_txn *txn) +{ + /* env->me_pghead[] can grow and shrink during this call. + * env->me_pglast and txn->mt_free_pgs[] can only grow. + * Page numbers cannot disappear from txn->mt_free_pgs[]. + */ + MDB_cursor mc; + MDB_env *env = txn->mt_env; + int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; + txnid_t pglast = 0, head_id = 0; + pgno_t freecnt = 0, *free_pgs, *mop; + ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; + + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + + if (env->me_pghead) { + /* Make sure first page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (!env->me_pghead && txn->mt_loose_pgs) { + /* Put loose page numbers in mt_free_pgs, since + * we may be unable to return them to me_pghead. + */ + MDB_page *mp = txn->mt_loose_pgs; + MDB_ID2 *dl = txn->mt_u.dirty_list; + unsigned x; + if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) + return rc; + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* must also remove from dirty list */ + if (txn->mt_flags & MDB_TXN_WRITEMAP) { + for (x=1; x<=dl[0].mid; x++) + if (dl[x].mid == mp->mp_pgno) + break; + mdb_tassert(txn, x <= dl[0].mid); + } else { + x = mdb_mid2l_search(dl, mp->mp_pgno); + mdb_tassert(txn, dl[x].mid == mp->mp_pgno); + mdb_dpage_free(env, mp); + } + dl[x].mptr = NULL; + } + { + /* squash freed slots out of the dirty list */ + unsigned y; + for (y=1; dl[y].mptr && y <= dl[0].mid; y++); + if (y <= dl[0].mid) { + for(x=y, y++;;) { + while (!dl[y].mptr && y <= dl[0].mid) y++; + if (y > dl[0].mid) break; + dl[x++] = dl[y++]; + } + dl[0].mid = x-1; + } else { + /* all slots freed */ + dl[0].mid = 0; + } + } + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + } + + /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ + clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) + ? SSIZE_MAX : maxfree_1pg; + + for (;;) { + /* Come back here after each Put() in case freelist changed */ + MDB_val key, data; + pgno_t *pgs; + ssize_t j; + + /* If using records from freeDB which we have not yet + * deleted, delete them and any we reserved for me_pghead. + */ + while (pglast < env->me_pglast) { + rc = mdb_cursor_first(&mc, &key, NULL); + if (rc) + return rc; + pglast = head_id = *(txnid_t *)key.mv_data; + total_room = head_room = 0; + mdb_tassert(txn, pglast <= env->me_pglast); + rc = mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } + + /* Save the IDL of pages freed by this txn, to a single record */ + if (freecnt < txn->mt_free_pgs[0]) { + if (!freecnt) { + /* Make sure last page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + free_pgs = txn->mt_free_pgs; + /* Write to last page of freeDB */ + key.mv_size = sizeof(txn->mt_txnid); + key.mv_data = &txn->mt_txnid; + do { + freecnt = free_pgs[0]; + data.mv_size = MDB_IDL_SIZEOF(free_pgs); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* Retry if mt_free_pgs[] grew during the Put() */ + free_pgs = txn->mt_free_pgs; + } while (freecnt < free_pgs[0]); + mdb_midl_sort(free_pgs); + memcpy(data.mv_data, free_pgs, data.mv_size); +#if (MDB_DEBUG) > 1 + { + unsigned int i = free_pgs[0]; + DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u", + txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); + for (; i; i--) + DPRINTF(("IDL %"Z"u", free_pgs[i])); + } +#endif + continue; + } + + mop = env->me_pghead; + mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; + + /* Reserve records for me_pghead[]. Split it if multi-page, + * to avoid searching freeDB for a page range. Use keys in + * range [1,me_pglast]: Smaller than txnid of oldest reader. + */ + if (total_room >= mop_len) { + if (total_room == mop_len || --more < 0) + break; + } else if (head_room >= maxfree_1pg && head_id > 1) { + /* Keep current record (overflow page), add a new one */ + head_id--; + head_room = 0; + } + /* (Re)write {key = head_id, IDL length = head_room} */ + total_room -= head_room; + head_room = mop_len - total_room; + if (head_room > maxfree_1pg && head_id > 1) { + /* Overflow multi-page for part of me_pghead */ + head_room /= head_id; /* amortize page sizes */ + head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); + } else if (head_room < 0) { + /* Rare case, not bothering to delete this record */ + head_room = 0; + } + key.mv_size = sizeof(head_id); + key.mv_data = &head_id; + data.mv_size = (head_room + 1) * sizeof(pgno_t); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* IDL is initially empty, zero out at least the length */ + pgs = (pgno_t *)data.mv_data; + j = head_room > clean_limit ? head_room : 0; + do { + pgs[j] = 0; + } while (--j >= 0); + total_room += head_room; + } + + /* Return loose page numbers to me_pghead, though usually none are + * left at this point. The pages themselves remain in dirty_list. + */ + if (txn->mt_loose_pgs) { + MDB_page *mp = txn->mt_loose_pgs; + unsigned count = txn->mt_loose_count; + MDB_IDL loose; + /* Room for loose pages + temp IDL with same */ + if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) + return rc; + mop = env->me_pghead; + loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) + loose[ ++count ] = mp->mp_pgno; + loose[0] = count; + mdb_midl_sort(loose); + mdb_midl_xmerge(mop, loose); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + mop_len = mop[0]; + } + + /* Fill in the reserved me_pghead records */ + rc = MDB_SUCCESS; + if (mop_len) { + MDB_val key, data; + + mop += mop_len; + rc = mdb_cursor_first(&mc, &key, &data); + for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { + txnid_t id = *(txnid_t *)key.mv_data; + ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; + MDB_ID save; + + mdb_tassert(txn, len >= 0 && id <= env->me_pglast); + key.mv_data = &id; + if (len > mop_len) { + len = mop_len; + data.mv_size = (len + 1) * sizeof(MDB_ID); + } + data.mv_data = mop -= len; + save = mop[0]; + mop[0] = len; + rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); + mop[0] = save; + if (rc || !(mop_len -= len)) + break; + } + } + return rc; +} + +/** Flush (some) dirty pages to the map, after clearing their dirty flag. + * @param[in] txn the transaction that's being committed + * @param[in] keep number of initial pages in dirty_list to keep dirty. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_flush(MDB_txn *txn, int keep) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned psize = env->me_psize, j; + int i, pagecount = dl[0].mid, rc; + size_t size = 0, pos = 0; + pgno_t pgno = 0; + MDB_page *dp = NULL; +#ifdef _WIN32 + OVERLAPPED ov; +#else + struct iovec iov[MDB_COMMIT_PAGES]; + ssize_t wpos = 0, wsize = 0, wres; + size_t next_pos = 1; /* impossible pos, so pos != next_pos */ + int n = 0; +#endif + + j = i = keep; + + if (env->me_flags & MDB_WRITEMAP) { + /* Clear dirty flags */ + while (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[++j] = dl[i]; + continue; + } + dp->mp_flags &= ~P_DIRTY; + } + goto done; + } + + /* Write the pages */ + for (;;) { + if (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[i].mid = 0; + continue; + } + pgno = dl[i].mid; + /* clear dirty flag */ + dp->mp_flags &= ~P_DIRTY; + pos = pgno * psize; + size = psize; + if (IS_OVERFLOW(dp)) size *= dp->mp_pages; + } +#ifdef _WIN32 + else break; + + /* Windows actually supports scatter/gather I/O, but only on + * unbuffered file handles. Since we're relying on the OS page + * cache for all our data, that's self-defeating. So we just + * write pages one at a time. We use the ov structure to set + * the write offset, to at least save the overhead of a Seek + * system call. + */ + DPRINTF(("committing page %"Z"u", pgno)); + memset(&ov, 0, sizeof(ov)); + ov.Offset = pos & 0xffffffff; + ov.OffsetHigh = pos >> 16 >> 16; + if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { + rc = ErrCode(); + DPRINTF(("WriteFile: %d", rc)); + return rc; + } +#else + /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ + if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { + if (n) { +retry_write: + /* Write previous page(s) */ +#ifdef MDB_USE_PWRITEV + wres = pwritev(env->me_fd, iov, n, wpos); +#else + if (n == 1) { + wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); + } else { +retry_seek: + if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { + rc = ErrCode(); + if (rc == EINTR) + goto retry_seek; + DPRINTF(("lseek: %s", strerror(rc))); + return rc; + } + wres = writev(env->me_fd, iov, n); + } +#endif + if (wres != wsize) { + if (wres < 0) { + rc = ErrCode(); + if (rc == EINTR) + goto retry_write; + DPRINTF(("Write error: %s", strerror(rc))); + } else { + rc = EIO; /* TODO: Use which error code? */ + DPUTS("short write, filesystem full?"); + } + return rc; + } + n = 0; + } + if (i > pagecount) + break; + wpos = pos; + wsize = 0; + } + DPRINTF(("committing page %"Z"u", pgno)); + next_pos = pos + size; + iov[n].iov_len = size; + iov[n].iov_base = (char *)dp; + wsize += size; + n++; +#endif /* _WIN32 */ + } + + /* MIPS has cache coherency issues, this is a no-op everywhere else + * Note: for any size >= on-chip cache size, entire on-chip cache is + * flushed. + */ + CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); + + for (i = keep; ++i <= pagecount; ) { + dp = dl[i].mptr; + /* This is a page we skipped above */ + if (!dl[i].mid) { + dl[++j] = dl[i]; + dl[j].mid = dp->mp_pgno; + continue; + } + mdb_dpage_free(env, dp); + } + +done: + i--; + txn->mt_dirty_room += i - j; + dl[0].mid = j; + return MDB_SUCCESS; +} + +int +mdb_txn_commit(MDB_txn *txn) +{ + int rc; + unsigned int i, end_mode; + MDB_env *env; + + if (txn == NULL) + return EINVAL; + + /* mdb_txn_end() mode for a commit which writes nothing */ + end_mode = MDB_END_EMPTY_COMMIT|MDB_END_UPDATE|MDB_END_SLOT|MDB_END_FREE; + + if (txn->mt_child) { + rc = mdb_txn_commit(txn->mt_child); + if (rc) + goto fail; + } + + env = txn->mt_env; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + goto done; + } + + if (txn->mt_flags & (MDB_TXN_FINISHED|MDB_TXN_ERROR)) { + DPUTS("txn has failed/finished, can't commit"); + if (txn->mt_parent) + txn->mt_parent->mt_flags |= MDB_TXN_ERROR; + rc = MDB_BAD_TXN; + goto fail; + } + + if (txn->mt_parent) { + MDB_txn *parent = txn->mt_parent; + MDB_page **lp; + MDB_ID2L dst, src; + MDB_IDL pspill; + unsigned x, y, len, ps_len; + + /* Append our free list to parent's */ + rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); + if (rc) + goto fail; + mdb_midl_free(txn->mt_free_pgs); + /* Failures after this must either undo the changes + * to the parent or set MDB_TXN_ERROR in the parent. + */ + + parent->mt_next_pgno = txn->mt_next_pgno; + parent->mt_flags = txn->mt_flags; + + /* Merge our cursors into parent's and close them */ + mdb_cursors_close(txn, 1); + + /* Update parent's DB table. */ + memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + parent->mt_numdbs = txn->mt_numdbs; + parent->mt_dbflags[FREE_DBI] = txn->mt_dbflags[FREE_DBI]; + parent->mt_dbflags[MAIN_DBI] = txn->mt_dbflags[MAIN_DBI]; + for (i=CORE_DBS; imt_numdbs; i++) { + /* preserve parent's DB_NEW status */ + x = parent->mt_dbflags[i] & DB_NEW; + parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; + } + + dst = parent->mt_u.dirty_list; + src = txn->mt_u.dirty_list; + /* Remove anything in our dirty list from parent's spill list */ + if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { + x = y = ps_len; + pspill[0] = (pgno_t)-1; + /* Mark our dirty pages as deleted in parent spill list */ + for (i=0, len=src[0].mid; ++i <= len; ) { + MDB_ID pn = src[i].mid << 1; + while (pn > pspill[x]) + x--; + if (pn == pspill[x]) { + pspill[x] = 1; + y = --x; + } + } + /* Squash deleted pagenums if we deleted any */ + for (x=y; ++x <= ps_len; ) + if (!(pspill[x] & 1)) + pspill[++y] = pspill[x]; + pspill[0] = y; + } + + /* Remove anything in our spill list from parent's dirty list */ + if (txn->mt_spill_pgs && txn->mt_spill_pgs[0]) { + for (i=1; i<=txn->mt_spill_pgs[0]; i++) { + MDB_ID pn = txn->mt_spill_pgs[i]; + if (pn & 1) + continue; /* deleted spillpg */ + pn >>= 1; + y = mdb_mid2l_search(dst, pn); + if (y <= dst[0].mid && dst[y].mid == pn) { + free(dst[y].mptr); + while (y < dst[0].mid) { + dst[y] = dst[y+1]; + y++; + } + dst[0].mid--; + } + } + } + + /* Find len = length of merging our dirty list with parent's */ + x = dst[0].mid; + dst[0].mid = 0; /* simplify loops */ + if (parent->mt_parent) { + len = x + src[0].mid; + y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; + for (i = x; y && i; y--) { + pgno_t yp = src[y].mid; + while (yp < dst[i].mid) + i--; + if (yp == dst[i].mid) { + i--; + len--; + } + } + } else { /* Simplify the above for single-ancestor case */ + len = MDB_IDL_UM_MAX - txn->mt_dirty_room; + } + /* Merge our dirty list with parent's */ + y = src[0].mid; + for (i = len; y; dst[i--] = src[y--]) { + pgno_t yp = src[y].mid; + while (yp < dst[x].mid) + dst[i--] = dst[x--]; + if (yp == dst[x].mid) + free(dst[x--].mptr); + } + mdb_tassert(txn, i == x); + dst[0].mid = len; + free(txn->mt_u.dirty_list); + parent->mt_dirty_room = txn->mt_dirty_room; + if (txn->mt_spill_pgs) { + if (parent->mt_spill_pgs) { + /* TODO: Prevent failure here, so parent does not fail */ + rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); + if (rc) + parent->mt_flags |= MDB_TXN_ERROR; + mdb_midl_free(txn->mt_spill_pgs); + mdb_midl_sort(parent->mt_spill_pgs); + } else { + parent->mt_spill_pgs = txn->mt_spill_pgs; + } + } + + /* Append our loose page list to parent's */ + for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(*lp)) + ; + *lp = txn->mt_loose_pgs; + parent->mt_loose_count += txn->mt_loose_count; + + parent->mt_child = NULL; + mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); + free(txn); + return rc; + } + + if (txn != env->me_txn) { + DPUTS("attempt to commit unknown transaction"); + rc = EINVAL; + goto fail; + } + + mdb_cursors_close(txn, 0); + + if (!txn->mt_u.dirty_list[0].mid && + !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) + goto done; + + DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); + + /* Update DB root pointers */ + if (txn->mt_numdbs > CORE_DBS) { + MDB_cursor mc; + MDB_dbi i; + MDB_val data; + data.mv_size = sizeof(MDB_db); + + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + for (i = CORE_DBS; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + if (TXN_DBI_CHANGED(txn, i)) { + rc = MDB_BAD_DBI; + goto fail; + } + data.mv_data = &txn->mt_dbs[i]; + rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, + F_SUBDATA); + if (rc) + goto fail; + } + } + } + + rc = mdb_freelist_save(txn); + if (rc) + goto fail; + + mdb_midl_free(env->me_pghead); + env->me_pghead = NULL; + mdb_midl_shrink(&txn->mt_free_pgs); + +#if (MDB_DEBUG) > 2 + mdb_audit(txn); +#endif + + if ((rc = mdb_page_flush(txn, 0)) || + (rc = mdb_env_sync(env, 0)) || + (rc = mdb_env_write_meta(txn))) + goto fail; + end_mode = MDB_END_COMMITTED|MDB_END_UPDATE; + +done: + mdb_txn_end(txn, end_mode); + return MDB_SUCCESS; + +fail: + mdb_txn_abort(txn); + return rc; +} + +/** Read the environment parameters of a DB environment before + * mapping it into memory. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_read_header(MDB_env *env, MDB_meta *meta) +{ + MDB_metabuf pbuf; + MDB_page *p; + MDB_meta *m; + int i, rc, off; + enum { Size = sizeof(pbuf) }; + + /* We don't know the page size yet, so use a minimum value. + * Read both meta pages so we can use the latest one. + */ + + for (i=off=0; imm_psize) { +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; + if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) + rc = 0; +#else + rc = pread(env->me_fd, &pbuf, Size, off); +#endif + if (rc != Size) { + if (rc == 0 && off == 0) + return ENOENT; + rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; + DPRINTF(("read: %s", mdb_strerror(rc))); + return rc; + } + + p = (MDB_page *)&pbuf; + + if (!F_ISSET(p->mp_flags, P_META)) { + DPRINTF(("page %"Z"u not a meta page", p->mp_pgno)); + return MDB_INVALID; + } + + m = METADATA(p); + if (m->mm_magic != MDB_MAGIC) { + DPUTS("meta has invalid magic"); + return MDB_INVALID; + } + + if (m->mm_version != MDB_DATA_VERSION) { + DPRINTF(("database is version %u, expected version %u", + m->mm_version, MDB_DATA_VERSION)); + return MDB_VERSION_MISMATCH; + } + + if (off == 0 || m->mm_txnid > meta->mm_txnid) + *meta = *m; + } + return 0; +} + +/** Fill in most of the zeroed #MDB_meta for an empty database environment */ +static void ESECT +mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) +{ + meta->mm_magic = MDB_MAGIC; + meta->mm_version = MDB_DATA_VERSION; + meta->mm_mapsize = env->me_mapsize; + meta->mm_psize = env->me_psize; + meta->mm_last_pg = NUM_METAS-1; + meta->mm_flags = env->me_flags & 0xffff; + meta->mm_flags |= MDB_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */ + meta->mm_dbs[FREE_DBI].md_root = P_INVALID; + meta->mm_dbs[MAIN_DBI].md_root = P_INVALID; +} + +/** Write the environment parameters of a freshly created DB environment. + * @param[in] env the environment handle + * @param[in] meta the #MDB_meta to write + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_init_meta(MDB_env *env, MDB_meta *meta) +{ + MDB_page *p, *q; + int rc; + unsigned int psize; +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + ov.Offset = pos; \ + rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) +#else + int len; +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + len = pwrite(fd, ptr, size, pos); \ + if (len == -1 && ErrCode() == EINTR) continue; \ + rc = (len >= 0); break; } while(1) +#endif + + DPUTS("writing new meta page"); + + psize = env->me_psize; + + p = calloc(NUM_METAS, psize); + if (!p) + return ENOMEM; + + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + + DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0); + if (!rc) + rc = ErrCode(); + else if ((unsigned) len == psize * NUM_METAS) + rc = MDB_SUCCESS; + else + rc = ENOSPC; + free(p); + return rc; +} + +/** Update the environment info to commit a transaction. + * @param[in] txn the transaction that's being committed + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_write_meta(MDB_txn *txn) +{ + MDB_env *env; + MDB_meta meta, metab, *mp; + unsigned flags; + size_t mapsize; + off_t off; + int rc, len, toggle; + char *ptr; + HANDLE mfd; +#ifdef _WIN32 + OVERLAPPED ov; +#else + int r2; +#endif + + toggle = txn->mt_txnid & 1; + DPRINTF(("writing meta page %d for root page %"Z"u", + toggle, txn->mt_dbs[MAIN_DBI].md_root)); + + env = txn->mt_env; + flags = env->me_flags; + mp = env->me_metas[toggle]; + mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; + /* Persist any increases of mapsize config */ + if (mapsize < env->me_mapsize) + mapsize = env->me_mapsize; + + if (flags & MDB_WRITEMAP) { + mp->mm_mapsize = mapsize; + mp->mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; + mp->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; + mp->mm_last_pg = txn->mt_next_pgno - 1; +#if (__GNUC__ * 100 + __GNUC_MINOR__ >= 404) && /* TODO: portability */ \ + !(defined(__i386__) || defined(__x86_64__)) + /* LY: issue a memory barrier, if not x86. ITS#7969 */ + __sync_synchronize(); +#endif + mp->mm_txnid = txn->mt_txnid; + if (!(flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { + unsigned meta_size = env->me_psize; + rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; + ptr = (char *)mp - PAGEHDRSZ; +#ifndef _WIN32 /* POSIX msync() requires ptr = start of OS page */ + r2 = (ptr - env->me_map) & (env->me_os_psize - 1); + ptr -= r2; + meta_size += r2; +#endif + if (MDB_MSYNC(ptr, meta_size, rc)) { + rc = ErrCode(); + goto fail; + } + } + goto done; + } + metab.mm_txnid = mp->mm_txnid; + metab.mm_last_pg = mp->mm_last_pg; + + meta.mm_mapsize = mapsize; + meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; + meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; + meta.mm_last_pg = txn->mt_next_pgno - 1; + meta.mm_txnid = txn->mt_txnid; + + off = offsetof(MDB_meta, mm_mapsize); + ptr = (char *)&meta + off; + len = sizeof(MDB_meta) - off; + off += (char *)mp - env->me_map; + + /* Write to the SYNC fd unless MDB_NOSYNC/MDB_NOMETASYNC. + * (me_mfd goes to the same file as me_fd, but writing to it + * also syncs to disk. Avoids a separate fdatasync() call.) + */ + mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd; +#ifdef _WIN32 + { + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) + rc = -1; + } +#else +retry_write: + rc = pwrite(mfd, ptr, len, off); +#endif + if (rc != len) { + rc = rc < 0 ? ErrCode() : EIO; +#ifndef _WIN32 + if (rc == EINTR) + goto retry_write; +#endif + DPUTS("write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Write some old data back, to prevent it from being used. + * Use the non-SYNC fd; we know it will fail anyway. + */ + meta.mm_last_pg = metab.mm_last_pg; + meta.mm_txnid = metab.mm_txnid; +#ifdef _WIN32 + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + WriteFile(env->me_fd, ptr, len, NULL, &ov); +#else + r2 = pwrite(env->me_fd, ptr, len, off); + (void)r2; /* Silence warnings. We don't care about pwrite's return value */ +#endif +fail: + env->me_flags |= MDB_FATAL_ERROR; + return rc; + } + /* MIPS has cache coherency issues, this is a no-op everywhere else */ + CACHEFLUSH(env->me_map + off, len, DCACHE); +done: + /* Memory ordering issues are irrelevant; since the entire writer + * is wrapped by wmutex, all of these changes will become visible + * after the wmutex is unlocked. Since the DB is multi-version, + * readers will get consistent data regardless of how fresh or + * how stale their view of these values is. + */ + if (env->me_txns) + env->me_txns->mti_txnid = txn->mt_txnid; + + return MDB_SUCCESS; +} + +/** Check both meta pages to see which one is newer. + * @param[in] env the environment handle + * @return newest #MDB_meta. + */ +static MDB_meta * +mdb_env_pick_meta(const MDB_env *env) +{ + MDB_meta *const *metas = env->me_metas; + return metas[ metas[0]->mm_txnid < metas[1]->mm_txnid ]; +} + +int ESECT +mdb_env_create(MDB_env **env) +{ + MDB_env *e; + + e = calloc(1, sizeof(MDB_env)); + if (!e) + return ENOMEM; + + e->me_maxreaders = DEFAULT_READERS; + e->me_maxdbs = e->me_numdbs = CORE_DBS; + e->me_fd = INVALID_HANDLE_VALUE; + e->me_lfd = INVALID_HANDLE_VALUE; + e->me_mfd = INVALID_HANDLE_VALUE; +#ifdef MDB_USE_POSIX_SEM + e->me_rmutex = SEM_FAILED; + e->me_wmutex = SEM_FAILED; +#endif + e->me_pid = getpid(); + GET_PAGESIZE(e->me_os_psize); + VGMEMP_CREATE(e,0,0); + *env = e; + return MDB_SUCCESS; +} + +static int ESECT +mdb_env_map(MDB_env *env, void *addr) +{ + MDB_page *p; + unsigned int flags = env->me_flags; +#ifdef _WIN32 + int rc; + HANDLE mh; + LONG sizelo, sizehi; + size_t msize; + + if (flags & MDB_RDONLY) { + /* Don't set explicit map size, use whatever exists */ + msize = 0; + sizelo = 0; + sizehi = 0; + } else { + msize = env->me_mapsize; + sizelo = msize & 0xffffffff; + sizehi = msize >> 16 >> 16; /* only needed on Win64 */ + + /* Windows won't create mappings for zero length files. + * and won't map more than the file size. + * Just set the maxsize right now. + */ + if (!(flags & MDB_WRITEMAP) && (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo + || !SetEndOfFile(env->me_fd) + || SetFilePointer(env->me_fd, 0, NULL, 0) != 0)) + return ErrCode(); + } + + mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? + PAGE_READWRITE : PAGE_READONLY, + sizehi, sizelo, NULL); + if (!mh) + return ErrCode(); + env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? + FILE_MAP_WRITE : FILE_MAP_READ, + 0, 0, msize, addr); + rc = env->me_map ? 0 : ErrCode(); + CloseHandle(mh); + if (rc) + return rc; +#else + int mmap_flags = MAP_SHARED; + int prot = PROT_READ; +#ifdef MAP_NOSYNC /* Used on FreeBSD */ + if (flags & MDB_NOSYNC) + mmap_flags |= MAP_NOSYNC; +#endif + if (flags & MDB_WRITEMAP) { + prot |= PROT_WRITE; + if (ftruncate(env->me_fd, env->me_mapsize) < 0) + return ErrCode(); + } + env->me_map = mmap(addr, env->me_mapsize, prot, mmap_flags, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } + + if (flags & MDB_NORDAHEAD) { + /* Turn off readahead. It's harmful when the DB is larger than RAM. */ +#ifdef MADV_RANDOM + madvise(env->me_map, env->me_mapsize, MADV_RANDOM); +#else +#ifdef POSIX_MADV_RANDOM + posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); +#endif /* POSIX_MADV_RANDOM */ +#endif /* MADV_RANDOM */ + } +#endif /* _WIN32 */ + + /* Can happen because the address argument to mmap() is just a + * hint. mmap() can pick another, e.g. if the range is in use. + * The MAP_FIXED flag would prevent that, but then mmap could + * instead unmap existing pages to make room for the new map. + */ + if (addr && env->me_map != addr) + return EBUSY; /* TODO: Make a new MDB_* error code? */ + + p = (MDB_page *)env->me_map; + env->me_metas[0] = METADATA(p); + env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_mapsize(MDB_env *env, size_t size) +{ + /* If env is already open, caller is responsible for making + * sure there are no active txns. + */ + if (env->me_map) { + int rc; + MDB_meta *meta; + void *old; + if (env->me_txn) + return EINVAL; + meta = mdb_env_pick_meta(env); + if (!size) + size = meta->mm_mapsize; + { + /* Silently round up to minimum if the size is too small */ + size_t minsize = (meta->mm_last_pg + 1) * env->me_psize; + if (size < minsize) + size = minsize; + } + munmap(env->me_map, env->me_mapsize); + env->me_mapsize = size; + old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; + rc = mdb_env_map(env, old); + if (rc) + return rc; + } + env->me_mapsize = size; + if (env->me_psize) + env->me_maxpg = env->me_mapsize / env->me_psize; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) +{ + if (env->me_map) + return EINVAL; + env->me_maxdbs = dbs + CORE_DBS; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) +{ + if (env->me_map || readers < 1) + return EINVAL; + env->me_maxreaders = readers; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) +{ + if (!env || !readers) + return EINVAL; + *readers = env->me_maxreaders; + return MDB_SUCCESS; +} + +static int ESECT +mdb_fsize(HANDLE fd, size_t *size) +{ +#ifdef _WIN32 + LARGE_INTEGER fsize; + + if (!GetFileSizeEx(fd, &fsize)) + return ErrCode(); + + *size = fsize.QuadPart; +#else + struct stat st; + + if (fstat(fd, &st)) + return ErrCode(); + + *size = st.st_size; +#endif + return MDB_SUCCESS; +} + + +#ifdef _WIN32 +typedef wchar_t mdb_nchar_t; +# define MDB_NAME(str) L##str +# define mdb_name_cpy wcscpy +#else +/** Character type for file names: char on Unix, wchar_t on Windows */ +typedef char mdb_nchar_t; +# define MDB_NAME(str) str /**< #mdb_nchar_t[] string literal */ +# define mdb_name_cpy strcpy /**< Copy name (#mdb_nchar_t string) */ +#endif + +/** Filename - string of #mdb_nchar_t[] */ +typedef struct MDB_name { + int mn_len; /**< Length */ + int mn_alloced; /**< True if #mn_val was malloced */ + mdb_nchar_t *mn_val; /**< Contents */ +} MDB_name; + +/** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */ +static const mdb_nchar_t *const mdb_suffixes[2][2] = { + { MDB_NAME("/data.mdb"), MDB_NAME("") }, + { MDB_NAME("/lock.mdb"), MDB_NAME("-lock") } +}; + +#define MDB_SUFFLEN 9 /**< Max string length in #mdb_suffixes[] */ + +/** Set up filename + scratch area for filename suffix, for opening files. + * It should be freed with #mdb_fname_destroy(). + * On Windows, paths are converted from char *UTF-8 to wchar_t *UTF-16. + * + * @param[in] path Pathname for #mdb_env_open(). + * @param[in] envflags Whether a subdir and/or lockfile will be used. + * @param[out] fname Resulting filename, with room for a suffix if necessary. + */ +static int ESECT +mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname) +{ + int no_suffix = F_ISSET(envflags, MDB_NOSUBDIR|MDB_NOLOCK); + fname->mn_alloced = 0; +#ifdef _WIN32 + return utf8_to_utf16(path, fname, no_suffix ? 0 : MDB_SUFFLEN); +#else + fname->mn_len = strlen(path); + if (no_suffix) + fname->mn_val = (char *) path; + else if ((fname->mn_val = malloc(fname->mn_len + MDB_SUFFLEN+1)) != NULL) { + fname->mn_alloced = 1; + strcpy(fname->mn_val, path); + } + else + return ENOMEM; + return MDB_SUCCESS; +#endif +} + +/** Destroy \b fname from #mdb_fname_init() */ +#define mdb_fname_destroy(fname) \ + do { if ((fname).mn_alloced) free((fname).mn_val); } while (0) + +#ifdef O_CLOEXEC /* POSIX.1-2008: Set FD_CLOEXEC atomically at open() */ +# define MDB_CLOEXEC O_CLOEXEC +#else +# define MDB_CLOEXEC 0 +#endif + +/** File type, access mode etc. for #mdb_fopen() */ +enum mdb_fopen_type { +#ifdef _WIN32 + MDB_O_RDONLY, MDB_O_RDWR, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS +#else + /* A comment in mdb_fopen() explains some O_* flag choices. */ + MDB_O_RDONLY= O_RDONLY, /**< for RDONLY me_fd */ + MDB_O_RDWR = O_RDWR |O_CREAT, /**< for me_fd */ + MDB_O_META = O_WRONLY|MDB_DSYNC |MDB_CLOEXEC, /**< for me_mfd */ + MDB_O_COPY = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */ + /** Bitmask for open() flags in enum #mdb_fopen_type. The other bits + * distinguish otherwise-equal MDB_O_* constants from each other. + */ + MDB_O_MASK = MDB_O_RDWR|MDB_CLOEXEC | MDB_O_RDONLY|MDB_O_META|MDB_O_COPY, + MDB_O_LOCKS = MDB_O_RDWR|MDB_CLOEXEC | ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */ +#endif +}; + +/** Open an LMDB file. + * @param[in] env The LMDB environment. + * @param[in,out] fname Path from from #mdb_fname_init(). A suffix is + * appended if necessary to create the filename, without changing mn_len. + * @param[in] which Determines file type, access mode, etc. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[out] res Resulting file handle. + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_fopen(const MDB_env *env, MDB_name *fname, + enum mdb_fopen_type which, mdb_mode_t mode, + HANDLE *res) +{ + int rc = MDB_SUCCESS; + HANDLE fd; +#ifdef _WIN32 + DWORD acc, share, disp, attrs; +#else + int flags; +#endif + + if (fname->mn_alloced) /* modifiable copy */ + mdb_name_cpy(fname->mn_val + fname->mn_len, + mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]); + + /* The directory must already exist. Usually the file need not. + * MDB_O_META requires the file because we already created it using + * MDB_O_RDWR. MDB_O_COPY must not overwrite an existing file. + * + * With MDB_O_COPY we do not want the OS to cache the writes, since + * the source data is already in the OS cache. + * + * The lockfile needs FD_CLOEXEC (close file descriptor on exec*()) + * to avoid the flock() issues noted under Caveats in lmdb.h. + * Also set it for other filehandles which the user cannot get at + * and close himself, which he may need after fork(). I.e. all but + * me_fd, which programs do use via mdb_env_get_fd(). + */ + +#ifdef _WIN32 + acc = GENERIC_READ|GENERIC_WRITE; + share = FILE_SHARE_READ|FILE_SHARE_WRITE; + disp = OPEN_ALWAYS; + attrs = FILE_ATTRIBUTE_NORMAL; + switch (which) { + case MDB_O_RDONLY: /* read-only datafile */ + acc = GENERIC_READ; + disp = OPEN_EXISTING; + break; + case MDB_O_META: /* for writing metapages */ + acc = GENERIC_WRITE; + disp = OPEN_EXISTING; + attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH; + break; + case MDB_O_COPY: /* mdb_env_copy() & co */ + acc = GENERIC_WRITE; + share = 0; + disp = CREATE_NEW; + attrs = FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH; + break; + default: break; /* silence gcc -Wswitch (not all enum values handled) */ + } + fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL); +#else + fd = open(fname->mn_val, which & MDB_O_MASK, mode); +#endif + + if (fd == INVALID_HANDLE_VALUE) + rc = ErrCode(); +#ifndef _WIN32 + else { + if (which != MDB_O_RDONLY && which != MDB_O_RDWR) { + /* Set CLOEXEC if we could not pass it to open() */ + if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1) + (void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC); + } + if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) { + /* This may require buffer alignment. There is no portable + * way to ask how much, so we require OS pagesize alignment. + */ +# ifdef F_NOCACHE /* __APPLE__ */ + (void) fcntl(fd, F_NOCACHE, 1); +# elif defined O_DIRECT + /* open(...O_DIRECT...) would break on filesystems without + * O_DIRECT support (ITS#7682). Try to set it here instead. + */ + if ((flags = fcntl(fd, F_GETFL)) != -1) + (void) fcntl(fd, F_SETFL, flags | O_DIRECT); +# endif + } + } +#endif /* !_WIN32 */ + + *res = fd; + return rc; +} + + +#ifdef BROKEN_FDATASYNC +#include +#include +#endif + +/** Further setup required for opening an LMDB environment + */ +static int ESECT +mdb_env_open2(MDB_env *env) +{ + unsigned int flags = env->me_flags; + int i, newenv = 0, rc; + MDB_meta meta; + +#ifdef _WIN32 + /* See if we should use QueryLimited */ + rc = GetVersion(); + if ((rc & 0xff) > 5) + env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; + else + env->me_pidquery = PROCESS_QUERY_INFORMATION; +#endif /* _WIN32 */ + +#ifdef BROKEN_FDATASYNC + /* ext3/ext4 fdatasync is broken on some older Linux kernels. + * https://lkml.org/lkml/2012/9/3/83 + * Kernels after 3.6-rc6 are known good. + * https://lkml.org/lkml/2012/9/10/556 + * See if the DB is on ext3/ext4, then check for new enough kernel + * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known + * to be patched. + */ + { + struct statfs st; + fstatfs(env->me_fd, &st); + while (st.f_type == 0xEF53) { + struct utsname uts; + int i; + uname(&uts); + if (uts.release[0] < '3') { + if (!strncmp(uts.release, "2.6.32.", 7)) { + i = atoi(uts.release+7); + if (i >= 60) + break; /* 2.6.32.60 and newer is OK */ + } else if (!strncmp(uts.release, "2.6.34.", 7)) { + i = atoi(uts.release+7); + if (i >= 15) + break; /* 2.6.34.15 and newer is OK */ + } + } else if (uts.release[0] == '3') { + i = atoi(uts.release+2); + if (i > 5) + break; /* 3.6 and newer is OK */ + if (i == 5) { + i = atoi(uts.release+4); + if (i >= 4) + break; /* 3.5.4 and newer is OK */ + } else if (i == 2) { + i = atoi(uts.release+4); + if (i >= 30) + break; /* 3.2.30 and newer is OK */ + } + } else { /* 4.x and newer is OK */ + break; + } + env->me_flags |= MDB_FSYNCONLY; + break; + } + } +#endif + + if ((i = mdb_env_read_header(env, &meta)) != 0) { + if (i != ENOENT) + return i; + DPUTS("new mdbenv"); + newenv = 1; + env->me_psize = env->me_os_psize; + if (env->me_psize > MAX_PAGESIZE) + env->me_psize = MAX_PAGESIZE; + memset(&meta, 0, sizeof(meta)); + mdb_env_init_meta0(env, &meta); + meta.mm_mapsize = DEFAULT_MAPSIZE; + } else { + env->me_psize = meta.mm_psize; + } + + /* Was a mapsize configured? */ + if (!env->me_mapsize) { + env->me_mapsize = meta.mm_mapsize; + } + { + /* Make sure mapsize >= committed data size. Even when using + * mm_mapsize, which could be broken in old files (ITS#7789). + */ + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; + } + meta.mm_mapsize = env->me_mapsize; + + if (newenv && !(flags & MDB_FIXEDMAP)) { + /* mdb_env_map() may grow the datafile. Write the metapages + * first, so the file will be valid if initialization fails. + * Except with FIXEDMAP, since we do not yet know mm_address. + * We could fill in mm_address later, but then a different + * program might end up doing that - one with a memory layout + * and map address which does not suit the main program. + */ + rc = mdb_env_init_meta(env, &meta); + if (rc) + return rc; + newenv = 0; + } + + rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); + if (rc) + return rc; + + if (newenv) { + if (flags & MDB_FIXEDMAP) + meta.mm_address = env->me_map; + i = mdb_env_init_meta(env, &meta); + if (i != MDB_SUCCESS) { + return i; + } + } + + env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; + env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2) + - sizeof(indx_t); +#if !(MDB_MAXKEYSIZE) + env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); +#endif + env->me_maxpg = env->me_mapsize / env->me_psize; + +#if MDB_DEBUG + { + MDB_meta *meta = mdb_env_pick_meta(env); + MDB_db *db = &meta->mm_dbs[MAIN_DBI]; + + DPRINTF(("opened database version %u, pagesize %u", + meta->mm_version, env->me_psize)); + DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); + DPRINTF(("depth: %u", db->md_depth)); + DPRINTF(("entries: %"Z"u", db->md_entries)); + DPRINTF(("branch pages: %"Z"u", db->md_branch_pages)); + DPRINTF(("leaf pages: %"Z"u", db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages)); + DPRINTF(("root: %"Z"u", db->md_root)); + } +#endif + + return MDB_SUCCESS; +} + + +/** Release a reader thread's slot in the reader lock table. + * This function is called automatically when a thread exits. + * @param[in] ptr This points to the slot in the reader lock table. + */ +static void +mdb_env_reader_dest(void *ptr) +{ + MDB_reader *reader = ptr; + +#ifndef _WIN32 + if (reader->mr_pid == getpid()) /* catch pthread_exit() in child process */ +#endif + /* We omit the mutex, so do this atomically (i.e. skip mr_txnid) */ + reader->mr_pid = 0; +} + +#ifdef _WIN32 +/** Junk for arranging thread-specific callbacks on Windows. This is + * necessarily platform and compiler-specific. Windows supports up + * to 1088 keys. Let's assume nobody opens more than 64 environments + * in a single process, for now. They can override this if needed. + */ +#ifndef MAX_TLS_KEYS +#define MAX_TLS_KEYS 64 +#endif +static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; +static int mdb_tls_nkeys; + +static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) +{ + int i; + switch(reason) { + case DLL_PROCESS_ATTACH: break; + case DLL_THREAD_ATTACH: break; + case DLL_THREAD_DETACH: + for (i=0; ime_txns->mti_txnid = meta->mm_txnid; + +#ifdef _WIN32 + { + OVERLAPPED ov; + /* First acquire a shared lock. The Unlock will + * then release the existing exclusive lock. + */ + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + rc = ErrCode(); + } else { + UnlockFile(env->me_lfd, 0, 0, 1, 0); + *excl = 0; + } + } +#else + { + struct flock lock_info; + /* The shared lock replaces the existing lock */ + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_RDLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + *excl = rc ? -1 : 0; /* error may mean we lost the lock */ + } +#endif + + return rc; +} + +/** Try to get exclusive lock, otherwise shared. + * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. + */ +static int ESECT +mdb_env_excl_lock(MDB_env *env, int *excl) +{ + int rc = 0; +#ifdef _WIN32 + if (LockFile(env->me_lfd, 0, 0, 1, 0)) { + *excl = 1; + } else { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + *excl = 0; + } else { + rc = ErrCode(); + } + } +#else + struct flock lock_info; + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (!rc) { + *excl = 1; + } else +# ifndef MDB_USE_POSIX_MUTEX + if (*excl < 0) /* always true when MDB_USE_POSIX_MUTEX */ +# endif + { + lock_info.l_type = F_RDLCK; + while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (rc == 0) + *excl = 0; + } +#endif + return rc; +} + +#ifdef MDB_USE_HASH +/* + * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code + * + * @(#) $Revision: 5.1 $ + * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + +typedef unsigned long long mdb_hash_t; +#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) + +/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer + * @param[in] val value to hash + * @param[in] hval initial value for hash + * @return 64 bit hash + * + * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the + * hval arg on the first call. + */ +static mdb_hash_t +mdb_hash_val(MDB_val *val, mdb_hash_t hval) +{ + unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ + unsigned char *end = s + val->mv_size; + /* + * FNV-1a hash each octet of the string + */ + while (s < end) { + /* xor the bottom with the current octet */ + hval ^= (mdb_hash_t)*s++; + + /* multiply by the 64 bit FNV magic prime mod 2^64 */ + hval += (hval << 1) + (hval << 4) + (hval << 5) + + (hval << 7) + (hval << 8) + (hval << 40); + } + /* return our new hash value */ + return hval; +} + +/** Hash the string and output the encoded hash. + * This uses modified RFC1924 Ascii85 encoding to accommodate systems with + * very short name limits. We don't care about the encoding being reversible, + * we just want to preserve as many bits of the input as possible in a + * small printable string. + * @param[in] str string to hash + * @param[out] encbuf an array of 11 chars to hold the hash + */ +static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; + +static void ESECT +mdb_pack85(unsigned long l, char *out) +{ + int i; + + for (i=0; i<5; i++) { + *out++ = mdb_a85[l % 85]; + l /= 85; + } +} + +static void ESECT +mdb_hash_enc(MDB_val *val, char *encbuf) +{ + mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); + + mdb_pack85(h, encbuf); + mdb_pack85(h>>32, encbuf+5); + encbuf[10] = '\0'; +} +#endif + +/** Open and/or initialize the lock region for the environment. + * @param[in] env The LMDB environment. + * @param[in] fname Filename + scratch area, from #mdb_fname_init(). + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl) +{ +#ifdef _WIN32 +# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT +#else +# define MDB_ERRCODE_ROFS EROFS +#endif + int rc; + off_t size, rsize; + + rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd); + if (rc) { + /* Omit lockfile if read-only env on read-only filesystem */ + if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { + return MDB_SUCCESS; + } + goto fail; + } + + if (!(env->me_flags & MDB_NOTLS)) { + rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); + if (rc) + goto fail; + env->me_flags |= MDB_ENV_TXKEY; +#ifdef _WIN32 + /* Windows TLS callbacks need help finding their TLS info. */ + if (mdb_tls_nkeys >= MAX_TLS_KEYS) { + rc = MDB_TLS_FULL; + goto fail; + } + mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; +#endif + } + + /* Try to get exclusive lock. If we succeed, then + * nobody is using the lock region and we should initialize it. + */ + if ((rc = mdb_env_excl_lock(env, excl))) goto fail; + +#ifdef _WIN32 + size = GetFileSize(env->me_lfd, NULL); +#else + size = lseek(env->me_lfd, 0, SEEK_END); + if (size == -1) goto fail_errno; +#endif + rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); + if (size < rsize && *excl > 0) { +#ifdef _WIN32 + if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize + || !SetEndOfFile(env->me_lfd)) + goto fail_errno; +#else + if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; +#endif + } else { + rsize = size; + size = rsize - sizeof(MDB_txninfo); + env->me_maxreaders = size/sizeof(MDB_reader) + 1; + } + { +#ifdef _WIN32 + HANDLE mh; + mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, + 0, 0, NULL); + if (!mh) goto fail_errno; + env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); + CloseHandle(mh); + if (!env->me_txns) goto fail_errno; +#else + void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, + env->me_lfd, 0); + if (m == MAP_FAILED) goto fail_errno; + env->me_txns = m; +#endif + } + if (*excl > 0) { +#ifdef _WIN32 + BY_HANDLE_FILE_INFORMATION stbuf; + struct { + DWORD volume; + DWORD nhigh; + DWORD nlow; + } idbuf; + MDB_val val; + char encbuf[11]; + + if (!mdb_sec_inited) { + InitializeSecurityDescriptor(&mdb_null_sd, + SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); + mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); + mdb_all_sa.bInheritHandle = FALSE; + mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; + mdb_sec_inited = 1; + } + if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.volume = stbuf.dwVolumeSerialNumber; + idbuf.nhigh = stbuf.nFileIndexHigh; + idbuf.nlow = stbuf.nFileIndexLow; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); + sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf); + env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + struct stat stbuf; + struct { + dev_t dev; + ino_t ino; + } idbuf; + MDB_val val; + char encbuf[11]; + +#if defined(__NetBSD__) +#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ +#endif + if (fstat(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.dev = stbuf.st_dev; + idbuf.ino = stbuf.st_ino; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); +#ifdef MDB_SHORT_SEMNAMES + encbuf[9] = '\0'; /* drop name from 15 chars to 14 chars */ +#endif + sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf); + /* Clean up after a previous run, if needed: Try to + * remove both semaphores before doing anything else. + */ + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + env->me_rmutex = sem_open(env->me_txns->mti_rmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#else /* MDB_USE_POSIX_MUTEX: */ + pthread_mutexattr_t mattr; + + /* Solaris needs this before initing a robust mutex. Otherwise + * it may skip the init and return EBUSY "seems someone already + * inited" or EINVAL "it was inited differently". + */ + memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex)); + memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex)); + + if ((rc = pthread_mutexattr_init(&mattr))) + goto fail; + + rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); +#ifdef MDB_ROBUST_SUPPORTED + if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); +#endif + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr); + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr); + pthread_mutexattr_destroy(&mattr); + if (rc) + goto fail; +#endif /* _WIN32 || MDB_USE_POSIX_SEM */ + + env->me_txns->mti_magic = MDB_MAGIC; + env->me_txns->mti_format = MDB_LOCK_FORMAT; + env->me_txns->mti_txnid = 0; + env->me_txns->mti_numreaders = 0; + + } else { + if (env->me_txns->mti_magic != MDB_MAGIC) { + DPUTS("lock region has invalid magic"); + rc = MDB_INVALID; + goto fail; + } + if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { + DPRINTF(("lock region has format+version 0x%x, expected 0x%x", + env->me_txns->mti_format, MDB_LOCK_FORMAT)); + rc = MDB_VERSION_MISMATCH; + goto fail; + } + rc = ErrCode(); + if (rc && rc != EACCES && rc != EAGAIN) { + goto fail; + } +#ifdef _WIN32 + env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#endif + } + return MDB_SUCCESS; + +fail_errno: + rc = ErrCode(); +fail: + return rc; +} + + /** Only a subset of the @ref mdb_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. + */ +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \ + MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) + +#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) +# error "Persistent DB flags & env flags overlap, but both go in mm_flags" +#endif + +int ESECT +mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) +{ + int rc, excl = -1; + MDB_name fname; + + if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) + return EINVAL; + + flags |= env->me_flags; + + rc = mdb_fname_init(path, flags, &fname); + if (rc) + return rc; + + if (flags & MDB_RDONLY) { + /* silently ignore WRITEMAP when we're only getting read access */ + flags &= ~MDB_WRITEMAP; + } else { + if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && + (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) + rc = ENOMEM; + } + env->me_flags = flags |= MDB_ENV_ACTIVE; + if (rc) + goto leave; + + env->me_path = strdup(path); + env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); + env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); + env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); + if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { + rc = ENOMEM; + goto leave; + } + env->me_dbxs[FREE_DBI].md_cmp = mdb_cmp_long; /* aligned MDB_INTEGERKEY */ + + /* For RDONLY, get lockfile after we know datafile exists */ + if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { + rc = mdb_env_setup_locks(env, &fname, mode, &excl); + if (rc) + goto leave; + } + + rc = mdb_fopen(env, &fname, + (flags & MDB_RDONLY) ? MDB_O_RDONLY : MDB_O_RDWR, + mode, &env->me_fd); + if (rc) + goto leave; + + if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { + rc = mdb_env_setup_locks(env, &fname, mode, &excl); + if (rc) + goto leave; + } + + if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { + if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) { + /* Synchronous fd for meta writes. Needed even with + * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. + */ + rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd); + if (rc) + goto leave; + } + DPRINTF(("opened dbenv %p", (void *) env)); + if (excl > 0) { + rc = mdb_env_share_locks(env, &excl); + if (rc) + goto leave; + } + if (!(flags & MDB_RDONLY)) { + MDB_txn *txn; + int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * + (sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1); + if ((env->me_pbuf = calloc(1, env->me_psize)) && + (txn = calloc(1, size))) + { + txn->mt_dbs = (MDB_db *)((char *)txn + tsize); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_env = env; + txn->mt_dbxs = env->me_dbxs; + txn->mt_flags = MDB_TXN_FINISHED; + env->me_txn0 = txn; + } else { + rc = ENOMEM; + } + } + } + +leave: + if (rc) { + mdb_env_close0(env, excl); + } + mdb_fname_destroy(fname); + return rc; +} + +/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ +static void ESECT +mdb_env_close0(MDB_env *env, int excl) +{ + int i; + + if (!(env->me_flags & MDB_ENV_ACTIVE)) + return; + + /* Doing this here since me_dbxs may not exist during mdb_env_close */ + if (env->me_dbxs) { + for (i = env->me_maxdbs; --i >= CORE_DBS; ) + free(env->me_dbxs[i].md_name.mv_data); + free(env->me_dbxs); + } + + free(env->me_pbuf); + free(env->me_dbiseqs); + free(env->me_dbflags); + free(env->me_path); + free(env->me_dirty_list); + free(env->me_txn0); + mdb_midl_free(env->me_free_pgs); + + if (env->me_flags & MDB_ENV_TXKEY) { + pthread_key_delete(env->me_txkey); +#ifdef _WIN32 + /* Delete our key from the global list */ + for (i=0; ime_txkey) { + mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; + mdb_tls_nkeys--; + break; + } +#endif + } + + if (env->me_map) { + munmap(env->me_map, env->me_mapsize); + } + if (env->me_mfd != INVALID_HANDLE_VALUE) + (void) close(env->me_mfd); + if (env->me_fd != INVALID_HANDLE_VALUE) + (void) close(env->me_fd); + if (env->me_txns) { + MDB_PID_T pid = getpid(); + /* Clearing readers is done in this function because + * me_txkey with its destructor must be disabled first. + * + * We skip the the reader mutex, so we touch only + * data owned by this process (me_close_readers and + * our readers), and clear each reader atomically. + */ + for (i = env->me_close_readers; --i >= 0; ) + if (env->me_txns->mti_readers[i].mr_pid == pid) + env->me_txns->mti_readers[i].mr_pid = 0; +#ifdef _WIN32 + if (env->me_rmutex) { + CloseHandle(env->me_rmutex); + if (env->me_wmutex) CloseHandle(env->me_wmutex); + } + /* Windows automatically destroys the mutexes when + * the last handle closes. + */ +#elif defined(MDB_USE_POSIX_SEM) + if (env->me_rmutex != SEM_FAILED) { + sem_close(env->me_rmutex); + if (env->me_wmutex != SEM_FAILED) + sem_close(env->me_wmutex); + /* If we have the filelock: If we are the + * only remaining user, clean up semaphores. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) { + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + } + } +#elif defined(MDB_ROBUST_SUPPORTED) + /* If we have the filelock: If we are the + * only remaining user, clean up robust + * mutexes. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) { + pthread_mutex_destroy(env->me_txns->mti_rmutex); + pthread_mutex_destroy(env->me_txns->mti_wmutex); + } +#endif + munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); + } + if (env->me_lfd != INVALID_HANDLE_VALUE) { +#ifdef _WIN32 + if (excl >= 0) { + /* Unlock the lockfile. Windows would have unlocked it + * after closing anyway, but not necessarily at once. + */ + UnlockFile(env->me_lfd, 0, 0, 1, 0); + } +#endif + (void) close(env->me_lfd); + } + + env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); +} + +void ESECT +mdb_env_close(MDB_env *env) +{ + MDB_page *dp; + + if (env == NULL) + return; + + VGMEMP_DESTROY(env); + while ((dp = env->me_dpages) != NULL) { + VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); + env->me_dpages = dp->mp_next; + free(dp); + } + + mdb_env_close0(env, 0); + free(env); +} + +/** Compare two items pointing at aligned size_t's */ +static int +mdb_cmp_long(const MDB_val *a, const MDB_val *b) +{ + return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : + *(size_t *)a->mv_data > *(size_t *)b->mv_data; +} + +/** Compare two items pointing at aligned unsigned int's. + * + * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, + * but #mdb_cmp_clong() is called instead if the data type is size_t. + */ +static int +mdb_cmp_int(const MDB_val *a, const MDB_val *b) +{ + return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : + *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; +} + +/** Compare two items pointing at unsigned ints of unknown alignment. + * Nodes and keys are guaranteed to be 2-byte aligned. + */ +static int +mdb_cmp_cint(const MDB_val *a, const MDB_val *b) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short *u, *c; + int x; + + u = (unsigned short *) ((char *) a->mv_data + a->mv_size); + c = (unsigned short *) ((char *) b->mv_data + a->mv_size); + do { + x = *--u - *--c; + } while(!x && u > (unsigned short *)a->mv_data); + return x; +#else + unsigned short *u, *c, *end; + int x; + + end = (unsigned short *) ((char *) a->mv_data + a->mv_size); + u = (unsigned short *)a->mv_data; + c = (unsigned short *)b->mv_data; + do { + x = *u++ - *c++; + } while(!x && u < end); + return x; +#endif +} + +/** Compare two items lexically */ +static int +mdb_cmp_memn(const MDB_val *a, const MDB_val *b) +{ + int diff; + ssize_t len_diff; + unsigned int len; + + len = a->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + len = b->mv_size; + len_diff = 1; + } + + diff = memcmp(a->mv_data, b->mv_data, len); + return diff ? diff : len_diff<0 ? -1 : len_diff; +} + +/** Compare two items in reverse byte order */ +static int +mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) +{ + const unsigned char *p1, *p2, *p1_lim; + ssize_t len_diff; + int diff; + + p1_lim = (const unsigned char *)a->mv_data; + p1 = (const unsigned char *)a->mv_data + a->mv_size; + p2 = (const unsigned char *)b->mv_data + b->mv_size; + + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + p1_lim += len_diff; + len_diff = 1; + } + + while (p1 > p1_lim) { + diff = *--p1 - *--p2; + if (diff) + return diff; + } + return len_diff<0 ? -1 : len_diff; +} + +/** Search for key within a page, using binary search. + * Returns the smallest entry larger or equal to the key. + * If exactp is non-null, stores whether the found entry was an exact match + * in *exactp (1 or 0). + * Updates the cursor index with the index of the found entry. + * If no entry larger or equal to the key is found, returns NULL. + */ +static MDB_node * +mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) +{ + unsigned int i = 0, nkeys; + int low, high; + int rc = 0; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NULL; + MDB_val nodekey; + MDB_cmp_func *cmp; + DKBUF; + + nkeys = NUMKEYS(mp); + + DPRINTF(("searching %u keys in %s %spage %"Z"u", + nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp))); + + low = IS_LEAF(mp) ? 0 : 1; + high = nkeys - 1; + cmp = mc->mc_dbx->md_cmp; + + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster mdb_cmp_int. + */ + if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { + if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) + cmp = mdb_cmp_long; + else + cmp = mdb_cmp_int; + } + + if (IS_LEAF2(mp)) { + nodekey.mv_size = mc->mc_db->md_pad; + node = NODEPTR(mp, 0); /* fake */ + while (low <= high) { + i = (low + high) >> 1; + nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); + rc = cmp(key, &nodekey); + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } else { + while (low <= high) { + i = (low + high) >> 1; + + node = NODEPTR(mp, i); + nodekey.mv_size = NODEKSZ(node); + nodekey.mv_data = NODEKEY(node); + + rc = cmp(key, &nodekey); +#if MDB_DEBUG + if (IS_LEAF(mp)) + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + else + DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i", + i, DKEY(&nodekey), NODEPGNO(node), rc)); +#endif + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } + + if (rc > 0) { /* Found entry is less than the key. */ + i++; /* Skip to get the smallest entry larger than key. */ + if (!IS_LEAF2(mp)) + node = NODEPTR(mp, i); + } + if (exactp) + *exactp = (rc == 0 && nkeys > 0); + /* store the key index */ + mc->mc_ki[mc->mc_top] = i; + if (i >= nkeys) + /* There is no entry larger or equal to the key. */ + return NULL; + + /* nodeptr is fake for LEAF2 */ + return node; +} + +#if 0 +static void +mdb_cursor_adjust(MDB_cursor *mc, func) +{ + MDB_cursor *m2; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { + func(mc, m2); + } + } +} +#endif + +/** Pop a page off the top of the cursor's stack. */ +static void +mdb_cursor_pop(MDB_cursor *mc) +{ + if (mc->mc_snum) { + DPRINTF(("popping page %"Z"u off db %d cursor %p", + mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); + + mc->mc_snum--; + if (mc->mc_snum) { + mc->mc_top--; + } else { + mc->mc_flags &= ~C_INITIALIZED; + } + } +} + +/** Push a page onto the top of the cursor's stack. + * Set #MDB_TXN_ERROR on failure. + */ +static int +mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) +{ + DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno, + DDBI(mc), (void *) mc)); + + if (mc->mc_snum >= CURSOR_STACK) { + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CURSOR_FULL; + } + + mc->mc_top = mc->mc_snum++; + mc->mc_pg[mc->mc_top] = mp; + mc->mc_ki[mc->mc_top] = 0; + + return MDB_SUCCESS; +} + +/** Find the address of the page corresponding to a given page number. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc the cursor accessing the page. + * @param[in] pgno the page number for the page to retrieve. + * @param[out] ret address of a pointer where the page's address will be stored. + * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) +{ + MDB_txn *txn = mc->mc_txn; + MDB_env *env = txn->mt_env; + MDB_page *p = NULL; + int level; + + if (! (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_WRITEMAP))) { + MDB_txn *tx2 = txn; + level = 1; + do { + MDB_ID2L dl = tx2->mt_u.dirty_list; + unsigned x; + /* Spilled pages were dirtied in this txn and flushed + * because the dirty list got full. Bring this page + * back in from the map (but don't unspill it here, + * leave that unless page_touch happens again). + */ + if (tx2->mt_spill_pgs) { + MDB_ID pn = pgno << 1; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + goto done; + } + } + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + p = dl[x].mptr; + goto done; + } + } + level++; + } while ((tx2 = tx2->mt_parent) != NULL); + } + + if (pgno < txn->mt_next_pgno) { + level = 0; + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + DPRINTF(("page %"Z"u not found", pgno)); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_NOTFOUND; + } + +done: + *ret = p; + if (lvl) + *lvl = level; + return MDB_SUCCESS; +} + +/** Finish #mdb_page_search() / #mdb_page_search_lowest(). + * The cursor is at the root page, set up the rest of it. + */ +static int +mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int rc; + DKBUF; + + while (IS_BRANCH(mp)) { + MDB_node *node; + indx_t i; + + DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); + /* Don't assert on branch pages in the FreeDB. We can get here + * while in the process of rebalancing a FreeDB branch page; we must + * let that proceed. ITS#8336 + */ + mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1); + DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0)))); + + if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { + i = 0; + if (flags & MDB_PS_LAST) { + i = NUMKEYS(mp) - 1; + /* if already init'd, see if we're already in right place */ + if (mc->mc_flags & C_INITIALIZED) { + if (mc->mc_ki[mc->mc_top] == i) { + mc->mc_top = mc->mc_snum++; + mp = mc->mc_pg[mc->mc_top]; + goto ready; + } + } + } + } else { + int exact; + node = mdb_node_search(mc, key, &exact); + if (node == NULL) + i = NUMKEYS(mp) - 1; + else { + i = mc->mc_ki[mc->mc_top]; + if (!exact) { + mdb_cassert(mc, i > 0); + i--; + } + } + DPRINTF(("following index %u for key [%s]", i, DKEY(key))); + } + + mdb_cassert(mc, i < NUMKEYS(mp)); + node = NODEPTR(mp, i); + + if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = i; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + +ready: + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc)) != 0) + return rc; + mp = mc->mc_pg[mc->mc_top]; + } + } + + if (!IS_LEAF(mp)) { + DPRINTF(("internal error, index points to a %02X page!?", + mp->mp_flags)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + + DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno, + key ? DKEY(key) : "null")); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + return MDB_SUCCESS; +} + +/** Search for the lowest key under the current branch page. + * This just bypasses a NUMKEYS check in the current page + * before calling mdb_page_search_root(), because the callers + * are all in situations where the current page is known to + * be underfilled. + */ +static int +mdb_page_search_lowest(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NODEPTR(mp, 0); + int rc; + + if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = 0; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); +} + +/** Search for the page a given key should be in. + * Push it and its parent pages on the cursor stack. + * @param[in,out] mc the cursor for this operation. + * @param[in] key the key to search for, or NULL for first/last page. + * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB + * are touched (updated with new page numbers). + * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. + * This is used by #mdb_cursor_first() and #mdb_cursor_last(). + * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) +{ + int rc; + pgno_t root; + + /* Make sure the txn is still viable, then find the root from + * the txn's db table and set it as the root of the cursor's stack. + */ + if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) { + DPUTS("transaction may not be used now"); + return MDB_BAD_TXN; + } else { + /* Make sure we're using an up-to-date root */ + if (*mc->mc_dbflag & DB_STALE) { + MDB_cursor mc2; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); + if (rc) + return rc; + { + MDB_val data; + int exact = 0; + uint16_t flags; + MDB_node *leaf = mdb_node_search(&mc2, + &mc->mc_dbx->md_name, &exact); + if (!exact) + return MDB_NOTFOUND; + if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) + return MDB_INCOMPATIBLE; /* not a named DB */ + rc = mdb_node_read(&mc2, leaf, &data); + if (rc) + return rc; + memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), + sizeof(uint16_t)); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. + */ + if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) + return MDB_INCOMPATIBLE; + memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); + } + *mc->mc_dbflag &= ~DB_STALE; + } + root = mc->mc_db->md_root; + + if (root == P_INVALID) { /* Tree is empty. */ + DPUTS("tree is empty"); + return MDB_NOTFOUND; + } + } + + mdb_cassert(mc, root > 1); + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) + if ((rc = mdb_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0) + return rc; + + mc->mc_snum = 1; + mc->mc_top = 0; + + DPRINTF(("db %d root page %"Z"u has flags 0x%X", + DDBI(mc), root, mc->mc_pg[0]->mp_flags)); + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc))) + return rc; + } + + if (flags & MDB_PS_ROOTONLY) + return MDB_SUCCESS; + + return mdb_page_search_root(mc, key, flags); +} + +static int +mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) +{ + MDB_txn *txn = mc->mc_txn; + pgno_t pg = mp->mp_pgno; + unsigned x = 0, ovpages = mp->mp_pages; + MDB_env *env = txn->mt_env; + MDB_IDL sl = txn->mt_spill_pgs; + MDB_ID pn = pg << 1; + int rc; + + DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages)); + /* If the page is dirty or on the spill list we just acquired it, + * so we should give it back to our current free list, if any. + * Otherwise put it onto the list of pages we freed in this txn. + * + * Won't create me_pghead: me_pglast must be inited along with it. + * Unsupported in nested txns: They would need to hide the page + * range in ancestor txns' dirty and spilled lists. + */ + if (env->me_pghead && + !txn->mt_parent && + ((mp->mp_flags & P_DIRTY) || + (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn))) + { + unsigned i, j; + pgno_t *mop; + MDB_ID2 *dl, ix, iy; + rc = mdb_midl_need(&env->me_pghead, ovpages); + if (rc) + return rc; + if (!(mp->mp_flags & P_DIRTY)) { + /* This page is no longer spilled */ + if (x == sl[0]) + sl[0]--; + else + sl[x] |= 1; + goto release; + } + /* Remove from dirty list */ + dl = txn->mt_u.dirty_list; + x = dl[0].mid--; + for (ix = dl[x]; ix.mptr != mp; ix = iy) { + if (x > 1) { + x--; + iy = dl[x]; + dl[x] = ix; + } else { + mdb_cassert(mc, x > 1); + j = ++(dl[0].mid); + dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + } + txn->mt_dirty_room++; + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dpage_free(env, mp); +release: + /* Insert in me_pghead */ + mop = env->me_pghead; + j = mop[0] + ovpages; + for (i = mop[0]; i && mop[i] < pg; i--) + mop[j--] = mop[i]; + while (j>i) + mop[j--] = pg++; + mop[0] += ovpages; + } else { + rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); + if (rc) + return rc; + } + mc->mc_db->md_overflow_pages -= ovpages; + return 0; +} + +/** Return the data associated with a given node. + * @param[in] mc The cursor for this operation. + * @param[in] leaf The node being read. + * @param[out] data Updated to point to the node's data. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) +{ + MDB_page *omp; /* overflow page */ + pgno_t pgno; + int rc; + + if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { + data->mv_size = NODEDSZ(leaf); + data->mv_data = NODEDATA(leaf); + return MDB_SUCCESS; + } + + /* Read overflow data. + */ + data->mv_size = NODEDSZ(leaf); + memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); + if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) { + DPRINTF(("read overflow page %"Z"u failed", pgno)); + return rc; + } + data->mv_data = METADATA(omp); + + return MDB_SUCCESS; +} + +int +mdb_get(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + MDB_cursor mc; + MDB_xcursor mx; + int exact = 0; + DKBUF; + + DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); + + if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); +} + +/** Find a sibling for a page. + * Replaces the page at the top of the cursor's stack with the + * specified sibling, if one exists. + * @param[in] mc The cursor for this operation. + * @param[in] move_right Non-zero if the right sibling is requested, + * otherwise the left sibling. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_cursor_sibling(MDB_cursor *mc, int move_right) +{ + int rc; + MDB_node *indx; + MDB_page *mp; + + if (mc->mc_snum < 2) { + return MDB_NOTFOUND; /* root has no siblings */ + } + + mdb_cursor_pop(mc); + DPRINTF(("parent page is page %"Z"u, index %u", + mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); + + if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) + : (mc->mc_ki[mc->mc_top] == 0)) { + DPRINTF(("no more keys left, moving to %s sibling", + move_right ? "right" : "left")); + if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { + /* undo cursor_pop before returning */ + mc->mc_top++; + mc->mc_snum++; + return rc; + } + } else { + if (move_right) + mc->mc_ki[mc->mc_top]++; + else + mc->mc_ki[mc->mc_top]--; + DPRINTF(("just moving to %s index key %u", + move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); + } + mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); + + indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((rc = mdb_page_get(mc, NODEPGNO(indx), &mp, NULL)) != 0) { + /* mc will be inconsistent if caller does mc_snum++ as above */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + return rc; + } + + mdb_cursor_push(mc, mp); + if (!move_right) + mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; + + return MDB_SUCCESS; +} + +/** Move the cursor to the next data item. */ +static int +mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if ((mc->mc_flags & C_DEL && op == MDB_NEXT_DUP)) + return MDB_NOTFOUND; + + if (!(mc->mc_flags & C_INITIALIZED)) + return mdb_cursor_first(mc, key, data); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_NEXT || op == MDB_NEXT_DUP) { + rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); + if (op != MDB_NEXT || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) + MDB_GET_KEY(leaf, key); + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_NEXT_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_next: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + if (mc->mc_flags & C_DEL) { + mc->mc_flags ^= C_DEL; + goto skip; + } + + if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { + DPUTS("=====> move to next sibling page"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]++; + +skip: + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the previous data item. */ +static int +mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + return rc; + mc->mc_ki[mc->mc_top]++; + } + + mp = mc->mc_pg[mc->mc_top]; + + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_PREV || op == MDB_PREV_DUP) { + rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); + if (op != MDB_PREV || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) { + MDB_GET_KEY(leaf, key); + mc->mc_flags &= ~C_EOF; + } + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_PREV_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + + mc->mc_flags &= ~(C_EOF|C_DEL); + + if (mc->mc_ki[mc->mc_top] == 0) { + DPUTS("=====> move to prev sibling page"); + if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; + DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]--; + + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (!IS_LEAF(mp)) + return MDB_CORRUPTED; + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Set the cursor on a specific data item. */ +static int +mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op, int *exactp) +{ + int rc; + MDB_page *mp; + MDB_node *leaf = NULL; + DKBUF; + + if (key->mv_size == 0) + return MDB_BAD_VALSIZE; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + /* See if we're already on the right page */ + if (mc->mc_flags & C_INITIALIZED) { + MDB_val nodekey; + + mp = mc->mc_pg[mc->mc_top]; + if (!NUMKEYS(mp)) { + mc->mc_ki[mc->mc_top] = 0; + return MDB_NOTFOUND; + } + if (MP_FLAGS(mp) & P_LEAF2) { + nodekey.mv_size = mc->mc_db->md_pad; + nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, 0); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. + */ + mc->mc_ki[mc->mc_top] = 0; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc > 0) { + unsigned int i; + unsigned int nkeys = NUMKEYS(mp); + if (nkeys > 1) { + if (MP_FLAGS(mp) & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + nkeys-1, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, nkeys-1); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* last node was the one we wanted */ + mc->mc_ki[mc->mc_top] = nkeys-1; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc < 0) { + if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + /* This is definitely the right page, skip search_page */ + if (MP_FLAGS(mp) & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + mc->mc_ki[mc->mc_top], nodekey.mv_size); + } else { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* current node was the one we wanted */ + if (exactp) + *exactp = 1; + goto set1; + } + } + rc = 0; + mc->mc_flags &= ~C_EOF; + goto set2; + } + } + /* If any parents have right-sibs, search. + * Otherwise, there's nothing further. + */ + for (i=0; imc_top; i++) + if (mc->mc_ki[i] < + NUMKEYS(mc->mc_pg[i])-1) + break; + if (i == mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = nkeys; + return MDB_NOTFOUND; + } + } + if (!mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = 0; + if (op == MDB_SET_RANGE && !exactp) { + rc = 0; + goto set1; + } else + return MDB_NOTFOUND; + } + } else { + mc->mc_pg[0] = 0; + } + + rc = mdb_page_search(mc, key, 0); + if (rc != MDB_SUCCESS) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + +set2: + leaf = mdb_node_search(mc, key, exactp); + if (exactp != NULL && !*exactp) { + /* MDB_SET specified and not an exact match. */ + return MDB_NOTFOUND; + } + + if (leaf == NULL) { + DPUTS("===> inexact leaf not found, goto sibling"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; /* no entries matched */ + } + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, 0); + } + +set1: + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + if (IS_LEAF2(mp)) { + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + } else { + int ex2, *ex2p; + if (op == MDB_GET_BOTH) { + ex2p = &ex2; + ex2 = 0; + } else { + ex2p = NULL; + } + rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); + if (rc != MDB_SUCCESS) + return rc; + } + } else if (data) { + if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { + MDB_val olddata; + MDB_cmp_func *dcmp; + if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS) + return rc; + dcmp = mc->mc_dbx->md_dcmp; +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) + dcmp = mdb_cmp_clong; +#endif + rc = dcmp(data, &olddata); + if (rc) { + if (op == MDB_GET_BOTH || rc > 0) + return MDB_NOTFOUND; + rc = 0; + } + *data = olddata; + + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + /* The key already matches in all other cases */ + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) + MDB_GET_KEY(leaf, key); + DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); + + return rc; +} + +/** Move the cursor to the first item in the database. */ +static int +mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + mc->mc_ki[mc->mc_top] = 0; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + if ( key ) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the last item in the database. */ +static int +mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_LAST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; + mc->mc_flags |= C_INITIALIZED|C_EOF; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + if (key) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +int +mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op) +{ + int rc; + int exact = 0; + int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); + + if (mc == NULL) + return EINVAL; + + if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + switch (op) { + case MDB_GET_CURRENT: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + } else { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int nkeys = NUMKEYS(mp); + if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { + mc->mc_ki[mc->mc_top] = nkeys; + rc = MDB_NOTFOUND; + break; + } + rc = MDB_SUCCESS; + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } else { + MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY(leaf, key); + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); + } else { + rc = mdb_node_read(mc, leaf, data); + } + } + } + } + break; + case MDB_GET_BOTH: + case MDB_GET_BOTH_RANGE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + /* FALLTHRU */ + case MDB_SET: + case MDB_SET_KEY: + case MDB_SET_RANGE: + if (key == NULL) { + rc = EINVAL; + } else { + rc = mdb_cursor_set(mc, key, data, op, + op == MDB_SET_RANGE ? NULL : &exact); + } + break; + case MDB_GET_MULTIPLE: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + rc = MDB_SUCCESS; + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || + (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) + break; + goto fetchm; + case MDB_NEXT_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + if (rc == MDB_SUCCESS) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + MDB_cursor *mx; +fetchm: + mx = &mc->mc_xcursor->mx_cursor; + data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * + mx->mc_db->md_pad; + data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); + mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_PREV_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_last(mc, key, data); + else + rc = MDB_SUCCESS; + if (rc == MDB_SUCCESS) { + MDB_cursor *mx = &mc->mc_xcursor->mx_cursor; + if (mx->mc_flags & C_INITIALIZED) { + rc = mdb_cursor_sibling(mx, 0); + if (rc == MDB_SUCCESS) + goto fetchm; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_NEXT: + case MDB_NEXT_DUP: + case MDB_NEXT_NODUP: + rc = mdb_cursor_next(mc, key, data, op); + break; + case MDB_PREV: + case MDB_PREV_DUP: + case MDB_PREV_NODUP: + rc = mdb_cursor_prev(mc, key, data, op); + break; + case MDB_FIRST: + rc = mdb_cursor_first(mc, key, data); + break; + case MDB_FIRST_DUP: + mfunc = mdb_cursor_first; + mmove: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) { + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + rc = MDB_NOTFOUND; + break; + } + mc->mc_flags &= ~C_EOF; + { + MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_GET_KEY(leaf, key); + rc = mdb_node_read(mc, leaf, data); + break; + } + } + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + case MDB_LAST: + rc = mdb_cursor_last(mc, key, data); + break; + case MDB_LAST_DUP: + mfunc = mdb_cursor_last; + goto mmove; + default: + DPRINTF(("unhandled/unimplemented cursor operation %u", op)); + rc = EINVAL; + break; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + return rc; +} + +/** Touch all the pages in the cursor stack. Set mc_top. + * Makes sure all the pages are writable, before attempting a write operation. + * @param[in] mc The cursor to operate on. + */ +static int +mdb_cursor_touch(MDB_cursor *mc) +{ + int rc = MDB_SUCCESS; + + if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & (DB_DIRTY|DB_DUPDATA))) { + /* Touch DB record of named DB */ + MDB_cursor mc2; + MDB_xcursor mcx; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); + if (rc) + return rc; + *mc->mc_dbflag |= DB_DIRTY; + } + mc->mc_top = 0; + if (mc->mc_snum) { + do { + rc = mdb_page_touch(mc); + } while (!rc && ++(mc->mc_top) < mc->mc_snum); + mc->mc_top = mc->mc_snum-1; + } + return rc; +} + +/** Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDB_NOSPILL 0x8000 + +int +mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, + unsigned int flags) +{ + MDB_env *env; + MDB_node *leaf = NULL; + MDB_page *fp, *mp, *sub_root = NULL; + uint16_t fp_flags; + MDB_val xdata, *rdata, dkey, olddata; + MDB_db dummy; + int do_sub = 0, insert_key, insert_data; + unsigned int mcount = 0, dcount = 0, nospill; + size_t nsize; + int rc, rc2; + unsigned int nflags; + DKBUF; + + if (mc == NULL || key == NULL) + return EINVAL; + + env = mc->mc_txn->mt_env; + + /* Check this first so counter will always be zero on any + * early failures. + */ + if (flags & MDB_MULTIPLE) { + dcount = data[1].mv_size; + data[1].mv_size = 0; + if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) + return MDB_INCOMPATIBLE; + } + + nospill = flags & MDB_NOSPILL; + flags &= ~MDB_NOSPILL; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (key->mv_size-1 >= ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; + +#if SIZE_MAX > MAXDATASIZE + if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) + return MDB_BAD_VALSIZE; +#else + if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; +#endif + + DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", + DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); + + dkey.mv_size = 0; + + if (flags & MDB_CURRENT) { + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + rc = MDB_SUCCESS; + } else if (mc->mc_db->md_root == P_INVALID) { + /* new database, cursor has nothing to point to */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + rc = MDB_NO_ROOT; + } else { + int exact = 0; + MDB_val d2; + if (flags & MDB_APPEND) { + MDB_val k2; + rc = mdb_cursor_last(mc, &k2, &d2); + if (rc == 0) { + rc = mc->mc_dbx->md_cmp(key, &k2); + if (rc > 0) { + rc = MDB_NOTFOUND; + mc->mc_ki[mc->mc_top]++; + } else { + /* new key is <= last key */ + rc = MDB_KEYEXIST; + } + } + } else { + rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); + } + if ((flags & MDB_NOOVERWRITE) && rc == 0) { + DPRINTF(("duplicate key [%s]", DKEY(key))); + *data = d2; + return MDB_KEYEXIST; + } + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + /* Cursor is positioned, check for room in the dirty list */ + if (!nospill) { + if (flags & MDB_MULTIPLE) { + rdata = &xdata; + xdata.mv_size = data->mv_size * dcount; + } else { + rdata = data; + } + if ((rc2 = mdb_page_spill(mc, key, rdata))) + return rc2; + } + + if (rc == MDB_NO_ROOT) { + MDB_page *np; + /* new database, write a root leaf page */ + DPUTS("allocating new root leaf page"); + if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { + return rc2; + } + mdb_cursor_push(mc, np); + mc->mc_db->md_root = np->mp_pgno; + mc->mc_db->md_depth++; + *mc->mc_dbflag |= DB_DIRTY; + if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) + == MDB_DUPFIXED) + MP_FLAGS(np) |= P_LEAF2; + mc->mc_flags |= C_INITIALIZED; + } else { + /* make sure all cursor pages are writable */ + rc2 = mdb_cursor_touch(mc); + if (rc2) + return rc2; + } + + insert_key = insert_data = rc; + if (insert_key) { + /* The key does not exist */ + DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + LEAFSIZE(key, data) > env->me_nodemax) + { + /* Too big for a node, insert in sub-DB. Set up an empty + * "old sub-page" for prep_subDB to expand to a full page. + */ + fp_flags = P_LEAF|P_DIRTY; + fp = env->me_pbuf; + fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ + MP_LOWER(fp) = MP_UPPER(fp) = (PAGEHDRSZ-PAGEBASE); + olddata.mv_size = PAGEHDRSZ; + goto prep_subDB; + } + } else { + /* there's only a key anyway, so this is a no-op */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + char *ptr; + unsigned int ksize = mc->mc_db->md_pad; + if (key->mv_size != ksize) + return MDB_BAD_VALSIZE; + ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + memcpy(ptr, key->mv_data, ksize); +fix_parent: + /* if overwriting slot 0 of leaf, need to + * update branch key if there is a parent page + */ + if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + unsigned short dtop = 1; + mc->mc_top--; + /* slot 0 is always an empty key, find real slot */ + while (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + mc->mc_top--; + dtop++; + } + if (mc->mc_ki[mc->mc_top]) + rc2 = mdb_update_key(mc, key); + else + rc2 = MDB_SUCCESS; + mc->mc_top += dtop; + if (rc2) + return rc2; + } + return MDB_SUCCESS; + } + +more: + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + olddata.mv_size = NODEDSZ(leaf); + olddata.mv_data = NODEDATA(leaf); + + /* DB has dups? */ + if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { + /* Prepare (sub-)page/sub-DB to accept the new item, + * if needed. fp: old sub-page or a header faking + * it. mp: new (sub-)page. offset: growth in page + * size. xdata: node data with new page or DB. + */ + unsigned i, offset = 0; + mp = fp = xdata.mv_data = env->me_pbuf; + mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + + /* Was a single item before, must convert now */ + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_cmp_func *dcmp; + /* Just overwrite the current item */ + if (flags == MDB_CURRENT) + goto current; + dcmp = mc->mc_dbx->md_dcmp; +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) + dcmp = mdb_cmp_clong; +#endif + /* does data match? */ + if (!dcmp(data, &olddata)) { + if (flags & (MDB_NODUPDATA|MDB_APPENDDUP)) + return MDB_KEYEXIST; + /* overwrite it */ + goto current; + } + + /* Back up original data item */ + dkey.mv_size = olddata.mv_size; + dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); + + /* Make sub-page header for the dup items, with dummy body */ + MP_FLAGS(fp) = P_LEAF|P_DIRTY|P_SUBP; + MP_LOWER(fp) = (PAGEHDRSZ-PAGEBASE); + xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + MP_FLAGS(fp) |= P_LEAF2; + fp->mp_pad = data->mv_size; + xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ + } else { + xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + + (dkey.mv_size & 1) + (data->mv_size & 1); + } + MP_UPPER(fp) = xdata.mv_size - PAGEBASE; + olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ + } else if (leaf->mn_flags & F_SUBDATA) { + /* Data is on sub-DB, just store it */ + flags |= F_DUPDATA|F_SUBDATA; + goto put_sub; + } else { + /* Data is on sub-page */ + fp = olddata.mv_data; + switch (flags) { + default: + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + offset = EVEN(NODESIZE + sizeof(indx_t) + + data->mv_size); + break; + } + offset = fp->mp_pad; + if (SIZELEFT(fp) < offset) { + offset *= 4; /* space for 4 more */ + break; + } + /* FALLTHRU */ /* Big enough MDB_DUPFIXED sub-page */ + case MDB_CURRENT: + MP_FLAGS(fp) |= P_DIRTY; + COPY_PGNO(MP_PGNO(fp), MP_PGNO(mp)); + mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; + flags |= F_DUPDATA; + goto put_sub; + } + xdata.mv_size = olddata.mv_size + offset; + } + + fp_flags = MP_FLAGS(fp); + if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { + /* Too big for a sub-page, convert to sub-DB */ + fp_flags &= ~P_SUBP; +prep_subDB: + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp_flags |= P_LEAF2; + dummy.md_pad = fp->mp_pad; + dummy.md_flags = MDB_DUPFIXED; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + dummy.md_flags |= MDB_INTEGERKEY; + } else { + dummy.md_pad = 0; + dummy.md_flags = 0; + } + dummy.md_depth = 1; + dummy.md_branch_pages = 0; + dummy.md_leaf_pages = 1; + dummy.md_overflow_pages = 0; + dummy.md_entries = NUMKEYS(fp); + xdata.mv_size = sizeof(MDB_db); + xdata.mv_data = &dummy; + if ((rc = mdb_page_alloc(mc, 1, &mp))) + return rc; + offset = env->me_psize - olddata.mv_size; + flags |= F_DUPDATA|F_SUBDATA; + dummy.md_root = mp->mp_pgno; + sub_root = mp; + } + if (mp != fp) { + MP_FLAGS(mp) = fp_flags | P_DIRTY; + MP_PAD(mp) = MP_PAD(fp); + MP_LOWER(mp) = MP_LOWER(fp); + MP_UPPER(mp) = MP_UPPER(fp) + offset; + if (fp_flags & P_LEAF2) { + memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); + } else { + memcpy((char *)mp + MP_UPPER(mp) + PAGEBASE, (char *)fp + MP_UPPER(fp) + PAGEBASE, + olddata.mv_size - MP_UPPER(fp) - PAGEBASE); + memcpy((char *)MP_PTRS(mp), (char *)MP_PTRS(fp), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0])); + for (i=0; imp_ptrs[i] += offset; + } + } + + rdata = &xdata; + flags |= F_DUPDATA; + do_sub = 1; + if (!insert_key) + mdb_node_del(mc, 0); + goto new_sub; + } +current: + /* LMDB passes F_SUBDATA in 'flags' to write a DB record */ + if ((leaf->mn_flags ^ flags) & F_SUBDATA) + return MDB_INCOMPATIBLE; + /* overflow page overwrites need special handling */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize); + + memcpy(&pg, olddata.mv_data, sizeof(pg)); + if ((rc2 = mdb_page_get(mc, pg, &omp, &level)) != 0) + return rc2; + ovpages = omp->mp_pages; + + /* Is the ov page large enough? */ + if (ovpages >= dpages) { + if (!(omp->mp_flags & P_DIRTY) && + (level || (env->me_flags & MDB_WRITEMAP))) + { + rc = mdb_page_unspill(mc->mc_txn, omp, &omp); + if (rc) + return rc; + level = 0; /* dirty in this txn or clean */ + } + /* Is it dirty? */ + if (omp->mp_flags & P_DIRTY) { + /* yes, overwrite it. Note in this case we don't + * bother to try shrinking the page if the new data + * is smaller than the overflow threshold. + */ + if (level > 1) { + /* It is writable only in a parent txn */ + size_t sz = (size_t) env->me_psize * ovpages, off; + MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); + MDB_ID2 id2; + if (!np) + return ENOMEM; + id2.mid = pg; + id2.mptr = np; + /* Note - this page is already counted in parent's dirty_room */ + rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); + mdb_cassert(mc, rc2 == 0); + /* Currently we make the page look as with put() in the + * parent txn, in case the user peeks at MDB_RESERVEd + * or unused parts. Some users treat ovpages specially. + */ + if (!(flags & MDB_RESERVE)) { + /* Skip the part where LMDB will put *data. + * Copy end of page, adjusting alignment so + * compiler may copy words instead of bytes. + */ + off = (PAGEHDRSZ + data->mv_size) & -(int)sizeof(size_t); + memcpy((size_t *)((char *)np + off), + (size_t *)((char *)omp + off), sz - off); + sz = PAGEHDRSZ; + } + memcpy(np, omp, sz); /* Copy beginning of page */ + omp = np; + } + SETDSZ(leaf, data->mv_size); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(omp); + else + memcpy(METADATA(omp), data->mv_data, data->mv_size); + return MDB_SUCCESS; + } + } + if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) + return rc2; + } else if (data->mv_size == olddata.mv_size) { + /* same size, just replace it. Note that we could + * also reuse this node if the new data is smaller, + * but instead we opt to shrink the node in that case. + */ + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = olddata.mv_data; + else if (!(mc->mc_flags & C_SUB)) + memcpy(olddata.mv_data, data->mv_data, data->mv_size); + else { + if (key->mv_size != NODEKSZ(leaf)) + goto new_ksize; + memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); + goto fix_parent; + } + return MDB_SUCCESS; + } +new_ksize: + mdb_node_del(mc, 0); + } + + rdata = data; + +new_sub: + nflags = flags & NODE_ADD_FLAGS; + nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); + if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { + if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) + nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ + if (!insert_key) + nflags |= MDB_SPLIT_REPLACE; + rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); + } else { + /* There is room already in this leaf page. */ + rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); + if (rc == 0) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp) continue; + if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) { + m3->mc_ki[i]++; + } + XCURSOR_REFRESH(m3, i, mp); + } + } + } + + if (rc == MDB_SUCCESS) { + /* Now store the actual data in the child DB. Note that we're + * storing the user data in the keys field, so there are strict + * size limits on dupdata. The actual data fields of the child + * DB are all zero size. + */ + if (do_sub) { + int xflags, new_dupdata; + size_t ecount; +put_sub: + xdata.mv_size = 0; + xdata.mv_data = ""; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((flags & (MDB_CURRENT|MDB_APPENDDUP)) == MDB_CURRENT) { + xflags = MDB_CURRENT|MDB_NOSPILL; + } else { + mdb_xcursor_init1(mc, leaf); + xflags = (flags & MDB_NODUPDATA) ? + MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; + } + if (sub_root) + mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; + new_dupdata = (int)dkey.mv_size; + /* converted, write the original data first */ + if (dkey.mv_size) { + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); + if (rc) + goto bad_sub; + /* we've done our job */ + dkey.mv_size = 0; + } + if (!(leaf->mn_flags & F_SUBDATA) || sub_root) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + MDB_xcursor *mx = mc->mc_xcursor; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[i] == mp) { + if (m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init2(m2, mx, new_dupdata); + } else if (!insert_key) { + XCURSOR_REFRESH(m2, i, mp); + } + } + } + } + ecount = mc->mc_xcursor->mx_db.md_entries; + if (flags & MDB_APPENDDUP) + xflags |= MDB_APPEND; + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); + if (flags & F_SUBDATA) { + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } + insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; + } + /* Increment count unless we just replaced an existing item. */ + if (insert_data) + mc->mc_db->md_entries++; + if (insert_key) { + /* Invalidate txn if we created an empty sub-DB */ + if (rc) + goto bad_sub; + /* If we succeeded and the key didn't exist before, + * make sure the cursor is marked valid. + */ + mc->mc_flags |= C_INITIALIZED; + } + if (flags & MDB_MULTIPLE) { + if (!rc) { + mcount++; + /* let caller know how many succeeded, if any */ + data[1].mv_size = mcount; + if (mcount < dcount) { + data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; + insert_key = insert_data = 0; + goto more; + } + } + } + return rc; +bad_sub: + if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ + rc = MDB_CORRUPTED; + } + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_cursor_del(MDB_cursor *mc, unsigned int flags) +{ + MDB_node *leaf; + MDB_page *mp; + int rc; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + + if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) + return rc; + + rc = mdb_cursor_touch(mc); + if (rc) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + if (!IS_LEAF(mp)) + return MDB_CORRUPTED; + if (IS_LEAF2(mp)) + goto del_key; + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (flags & MDB_NODUPDATA) { + /* mdb_cursor_del0() will subtract the final entry */ + mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + } else { + if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); + if (rc) + return rc; + /* If sub-DB still has entries, we're done */ + if (mc->mc_xcursor->mx_db.md_entries) { + if (leaf->mn_flags & F_SUBDATA) { + /* update subDB info */ + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } else { + MDB_cursor *m2; + /* shrink fake page */ + mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + /* fix other sub-DB cursors pointed at fake pages on this page */ + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + XCURSOR_REFRESH(m2, mc->mc_top, mp); + } + } + } + mc->mc_db->md_entries--; + return rc; + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + } + /* otherwise fall thru and delete the sub-DB */ + } + + if (leaf->mn_flags & F_SUBDATA) { + /* add all the child DB's pages to the free list */ + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto fail; + } + } + /* LMDB passes F_SUBDATA in 'flags' to delete a DB record */ + else if ((leaf->mn_flags ^ flags) & F_SUBDATA) { + rc = MDB_INCOMPATIBLE; + goto fail; + } + + /* add overflow pages to free list */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + + memcpy(&pg, NODEDATA(leaf), sizeof(pg)); + if ((rc = mdb_page_get(mc, pg, &omp, NULL)) || + (rc = mdb_ovpage_free(mc, omp))) + goto fail; + } + +del_key: + return mdb_cursor_del0(mc); + +fail: + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Allocate and initialize new pages for a database. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc a cursor on the database being added to. + * @param[in] flags flags defining what type of page is being allocated. + * @param[in] num the number of pages to allocate. This is usually 1, + * unless allocating overflow pages for a large record. + * @param[out] mp Address of a page, or NULL on failure. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) +{ + MDB_page *np; + int rc; + + if ((rc = mdb_page_alloc(mc, num, &np))) + return rc; + DPRINTF(("allocated new mpage %"Z"u, page size %u", + np->mp_pgno, mc->mc_txn->mt_env->me_psize)); + np->mp_flags = flags | P_DIRTY; + np->mp_lower = (PAGEHDRSZ-PAGEBASE); + np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE; + + if (IS_BRANCH(np)) + mc->mc_db->md_branch_pages++; + else if (IS_LEAF(np)) + mc->mc_db->md_leaf_pages++; + else if (IS_OVERFLOW(np)) { + mc->mc_db->md_overflow_pages += num; + np->mp_pages = num; + } + *mp = np; + + return 0; +} + +/** Calculate the size of a leaf node. + * The size depends on the environment's page size; if a data item + * is too large it will be put onto an overflow page and the node + * size will only include the key and not the data. Sizes are always + * rounded up to an even number of bytes, to guarantee 2-byte alignment + * of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @param[in] data The data for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) +{ + size_t sz; + + sz = LEAFSIZE(key, data); + if (sz > env->me_nodemax) { + /* put on overflow page */ + sz -= data->mv_size - sizeof(pgno_t); + } + + return EVEN(sz + sizeof(indx_t)); +} + +/** Calculate the size of a branch node. + * The size should depend on the environment's page size but since + * we currently don't support spilling large keys onto overflow + * pages, it's simply the size of the #MDB_node header plus the + * size of the key. Sizes are always rounded up to an even number + * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_branch_size(MDB_env *env, MDB_val *key) +{ + size_t sz; + + sz = INDXSIZE(key); + if (sz > env->me_nodemax) { + /* put on overflow page */ + /* not implemented */ + /* sz -= key->size - sizeof(pgno_t); */ + } + + return sz + sizeof(indx_t); +} + +/** Add a node to the page pointed to by the cursor. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc The cursor for this operation. + * @param[in] indx The index on the page where the new node should be added. + * @param[in] key The key for the new node. + * @param[in] data The data for the new node, if any. + * @param[in] pgno The page number, if adding a branch node. + * @param[in] flags Flags for the node. + * @return 0 on success, non-zero on failure. Possible errors are: + *
    + *
  • ENOMEM - failed to allocate overflow pages for the node. + *
  • MDB_PAGE_FULL - there is insufficient room in the page. This error + * should never happen since all callers already calculate the + * page's free space before calling this function. + *
+ */ +static int +mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) +{ + unsigned int i; + size_t node_size = NODESIZE; + ssize_t room; + indx_t ofs; + MDB_node *node; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_page *ofp = NULL; /* overflow page */ + void *ndata; + DKBUF; + + mdb_cassert(mc, MP_UPPER(mp) >= MP_LOWER(mp)); + + DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", + IS_LEAF(mp) ? "leaf" : "branch", + IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, + key ? key->mv_size : 0, key ? DKEY(key) : "null")); + + if (IS_LEAF2(mp)) { + /* Move higher keys up one slot. */ + int ksize = mc->mc_db->md_pad, dif; + char *ptr = LEAF2KEY(mp, indx, ksize); + dif = NUMKEYS(mp) - indx; + if (dif > 0) + memmove(ptr+ksize, ptr, dif*ksize); + /* insert new key */ + memcpy(ptr, key->mv_data, ksize); + + /* Just using these for counting */ + MP_LOWER(mp) += sizeof(indx_t); + MP_UPPER(mp) -= ksize - sizeof(indx_t); + return MDB_SUCCESS; + } + + room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); + if (key != NULL) + node_size += key->mv_size; + if (IS_LEAF(mp)) { + mdb_cassert(mc, key && data); + if (F_ISSET(flags, F_BIGDATA)) { + /* Data already on overflow page. */ + node_size += sizeof(pgno_t); + } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { + int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); + int rc; + /* Put data on overflow page. */ + DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", + data->mv_size, node_size+data->mv_size)); + node_size = EVEN(node_size + sizeof(pgno_t)); + if ((ssize_t)node_size > room) + goto full; + if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) + return rc; + DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno)); + flags |= F_BIGDATA; + goto update; + } else { + node_size += data->mv_size; + } + } + node_size = EVEN(node_size); + if ((ssize_t)node_size > room) + goto full; + +update: + /* Move higher pointers up one slot. */ + for (i = NUMKEYS(mp); i > indx; i--) + MP_PTRS(mp)[i] = MP_PTRS(mp)[i - 1]; + + /* Adjust free space offsets. */ + ofs = MP_UPPER(mp) - node_size; + mdb_cassert(mc, ofs >= MP_LOWER(mp) + sizeof(indx_t)); + MP_PTRS(mp)[indx] = ofs; + MP_UPPER(mp) = ofs; + MP_LOWER(mp) += sizeof(indx_t); + + /* Write the node data. */ + node = NODEPTR(mp, indx); + node->mn_ksize = (key == NULL) ? 0 : key->mv_size; + node->mn_flags = flags; + if (IS_LEAF(mp)) + SETDSZ(node,data->mv_size); + else + SETPGNO(node,pgno); + + if (key) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + if (IS_LEAF(mp)) { + ndata = NODEDATA(node); + if (ofp == NULL) { + if (F_ISSET(flags, F_BIGDATA)) + memcpy(ndata, data->mv_data, sizeof(pgno_t)); + else if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = ndata; + else + memcpy(ndata, data->mv_data, data->mv_size); + } else { + memcpy(ndata, &ofp->mp_pgno, sizeof(pgno_t)); + ndata = METADATA(ofp); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = ndata; + else + memcpy(ndata, data->mv_data, data->mv_size); + } + } + + return MDB_SUCCESS; + +full: + DPRINTF(("not enough room in page %"Z"u, got %u ptrs", + mdb_dbg_pgno(mp), NUMKEYS(mp))); + DPRINTF(("upper-lower = %u - %u = %"Z"d", MP_UPPER(mp),MP_LOWER(mp),room)); + DPRINTF(("node size = %"Z"u", node_size)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_FULL; +} + +/** Delete the specified node from a page. + * @param[in] mc Cursor pointing to the node to delete. + * @param[in] ksize The size of a node. Only used if the page is + * part of a #MDB_DUPFIXED database. + */ +static void +mdb_node_del(MDB_cursor *mc, int ksize) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + indx_t indx = mc->mc_ki[mc->mc_top]; + unsigned int sz; + indx_t i, j, numkeys, ptr; + MDB_node *node; + char *base; + + DPRINTF(("delete node %u on %s page %"Z"u", indx, + IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); + numkeys = NUMKEYS(mp); + mdb_cassert(mc, indx < numkeys); + + if (IS_LEAF2(mp)) { + int x = numkeys - 1 - indx; + base = LEAF2KEY(mp, indx, ksize); + if (x) + memmove(base, base + ksize, x * ksize); + MP_LOWER(mp) -= sizeof(indx_t); + MP_UPPER(mp) += ksize - sizeof(indx_t); + return; + } + + node = NODEPTR(mp, indx); + sz = NODESIZE + node->mn_ksize; + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + sz += sizeof(pgno_t); + else + sz += NODEDSZ(node); + } + sz = EVEN(sz); + + ptr = MP_PTRS(mp)[indx]; + for (i = j = 0; i < numkeys; i++) { + if (i != indx) { + MP_PTRS(mp)[j] = MP_PTRS(mp)[i]; + if (MP_PTRS(mp)[i] < ptr) + MP_PTRS(mp)[j] += sz; + j++; + } + } + + base = (char *)mp + MP_UPPER(mp) + PAGEBASE; + memmove(base + sz, base, ptr - MP_UPPER(mp)); + + MP_LOWER(mp) -= sizeof(indx_t); + MP_UPPER(mp) += sz; +} + +/** Compact the main page after deleting a node on a subpage. + * @param[in] mp The main page to operate on. + * @param[in] indx The index of the subpage on the main page. + */ +static void +mdb_node_shrink(MDB_page *mp, indx_t indx) +{ + MDB_node *node; + MDB_page *sp, *xp; + char *base; + indx_t delta, nsize, len, ptr; + int i; + + node = NODEPTR(mp, indx); + sp = (MDB_page *)NODEDATA(node); + delta = SIZELEFT(sp); + nsize = NODEDSZ(node) - delta; + + /* Prepare to shift upward, set len = length(subpage part to shift) */ + if (IS_LEAF2(sp)) { + len = nsize; + if (nsize & 1) + return; /* do not make the node uneven-sized */ + } else { + xp = (MDB_page *)((char *)sp + delta); /* destination subpage */ + for (i = NUMKEYS(sp); --i >= 0; ) + MP_PTRS(xp)[i] = MP_PTRS(sp)[i] - delta; + len = PAGEHDRSZ; + } + MP_UPPER(sp) = MP_LOWER(sp); + COPY_PGNO(MP_PGNO(sp), mp->mp_pgno); + SETDSZ(node, nsize); + + /* Shift upward */ + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + delta, base, (char *)sp + len - base); + + ptr = mp->mp_ptrs[indx]; + for (i = NUMKEYS(mp); --i >= 0; ) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] += delta; + } + mp->mp_upper += delta; +} + +/** Initial setup of a sorted-dups cursor. + * Sorted duplicates are implemented as a sub-database for the given key. + * The duplicate data items are actually keys of the sub-database. + * Operations on the duplicate data items are performed using a sub-cursor + * initialized when the sub-database is first accessed. This function does + * the preliminary setup of the sub-cursor, filling in the fields that + * depend only on the parent DB. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + */ +static void +mdb_xcursor_init0(MDB_cursor *mc) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + mx->mx_cursor.mc_xcursor = NULL; + mx->mx_cursor.mc_txn = mc->mc_txn; + mx->mx_cursor.mc_db = &mx->mx_db; + mx->mx_cursor.mc_dbx = &mx->mx_dbx; + mx->mx_cursor.mc_dbi = mc->mc_dbi; + mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + mx->mx_dbx.md_name.mv_size = 0; + mx->mx_dbx.md_name.mv_data = NULL; + mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; + mx->mx_dbx.md_dcmp = NULL; + mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; +} + +/** Final setup of a sorted-dups cursor. + * Sets up the fields that depend on the data from the main cursor. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + * @param[in] node The data containing the #MDB_db record for the + * sorted-dup database. + */ +static void +mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (node->mn_flags & F_SUBDATA) { + memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); + mx->mx_cursor.mc_pg[0] = 0; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + } else { + MDB_page *fp = NODEDATA(node); + mx->mx_db.md_pad = 0; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + mx->mx_db.md_entries = NUMKEYS(fp); + COPY_PGNO(mx->mx_db.md_root, MP_PGNO(fp)); + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; + mx->mx_cursor.mc_pg[0] = fp; + mx->mx_cursor.mc_ki[0] = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + mx->mx_db.md_flags = MDB_DUPFIXED; + mx->mx_db.md_pad = fp->mp_pad; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + mx->mx_db.md_flags |= MDB_INTEGERKEY; + } + } + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; +#if UINT_MAX < SIZE_MAX + if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) + mx->mx_dbx.md_cmp = mdb_cmp_clong; +#endif +} + + +/** Fixup a sorted-dups cursor due to underlying update. + * Sets up some fields that depend on the data from the main cursor. + * Almost the same as init1, but skips initialization steps if the + * xcursor had already been used. + * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up. + * @param[in] src_mx The xcursor of an up-to-date cursor. + * @param[in] new_dupdata True if converting from a non-#F_DUPDATA item. + */ +static void +mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (new_dupdata) { + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags |= C_INITIALIZED; + mx->mx_cursor.mc_ki[0] = 0; + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; +#if UINT_MAX < SIZE_MAX + mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; +#endif + } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { + return; + } + mx->mx_db = src_mx->mx_db; + mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); +} + +/** Initialize a cursor for a given transaction and database. */ +static void +mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) +{ + mc->mc_next = NULL; + mc->mc_backup = NULL; + mc->mc_dbi = dbi; + mc->mc_txn = txn; + mc->mc_db = &txn->mt_dbs[dbi]; + mc->mc_dbx = &txn->mt_dbxs[dbi]; + mc->mc_dbflag = &txn->mt_dbflags[dbi]; + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_pg[0] = 0; + mc->mc_ki[0] = 0; + mc->mc_flags = 0; + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { + mdb_tassert(txn, mx != NULL); + mc->mc_xcursor = mx; + mdb_xcursor_init0(mc); + } else { + mc->mc_xcursor = NULL; + } + if (*mc->mc_dbflag & DB_STALE) { + mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); + } +} + +int +mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) +{ + MDB_cursor *mc; + size_t size = sizeof(MDB_cursor); + + if (!ret || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + if (dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EINVAL; + + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) + size += sizeof(MDB_xcursor); + + if ((mc = malloc(size)) != NULL) { + mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; + } + } else { + return ENOMEM; + } + + *ret = mc; + + return MDB_SUCCESS; +} + +int +mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) +{ + if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi, DB_VALID)) + return EINVAL; + + if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); + return MDB_SUCCESS; +} + +/* Return the count of duplicate data items for the current key */ +int +mdb_cursor_count(MDB_cursor *mc, size_t *countp) +{ + MDB_node *leaf; + + if (mc == NULL || countp == NULL) + return EINVAL; + + if (mc->mc_xcursor == NULL) + return MDB_INCOMPATIBLE; + + if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (!mc->mc_snum) + return MDB_NOTFOUND; + + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + *countp = 1; + } else { + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + return EINVAL; + + *countp = mc->mc_xcursor->mx_db.md_entries; + } + return MDB_SUCCESS; +} + +void +mdb_cursor_close(MDB_cursor *mc) +{ + if (mc && !mc->mc_backup) { + /* remove from txn, if tracked */ + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + free(mc); + } +} + +MDB_txn * +mdb_cursor_txn(MDB_cursor *mc) +{ + if (!mc) return NULL; + return mc->mc_txn; +} + +MDB_dbi +mdb_cursor_dbi(MDB_cursor *mc) +{ + return mc->mc_dbi; +} + +/** Replace the key for a branch node with a new key. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc Cursor pointing to the node to operate on. + * @param[in] key The new key to use. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_update_key(MDB_cursor *mc, MDB_val *key) +{ + MDB_page *mp; + MDB_node *node; + char *base; + size_t len; + int delta, ksize, oksize; + indx_t ptr, i, numkeys, indx; + DKBUF; + + indx = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + node = NODEPTR(mp, indx); + ptr = mp->mp_ptrs[indx]; +#if MDB_DEBUG + { + MDB_val k2; + char kbuf2[DKBUF_MAXKEYSIZE*2+1]; + k2.mv_data = NODEKEY(node); + k2.mv_size = node->mn_ksize; + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u", + indx, ptr, + mdb_dkey(&k2, kbuf2), + DKEY(key), + mp->mp_pgno)); + } +#endif + + /* Sizes must be 2-byte aligned. */ + ksize = EVEN(key->mv_size); + oksize = EVEN(node->mn_ksize); + delta = ksize - oksize; + + /* Shift node contents if EVEN(key length) changed. */ + if (delta) { + if (delta > 0 && SIZELEFT(mp) < delta) { + pgno_t pgno; + /* not enough space left, do a delete and split */ + DPRINTF(("Not enough room, delta = %d, splitting...", delta)); + pgno = NODEPGNO(node); + mdb_node_del(mc, 0); + return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); + } + + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] -= delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + len = ptr - mp->mp_upper + NODESIZE; + memmove(base - delta, base, len); + mp->mp_upper -= delta; + + node = NODEPTR(mp, indx); + } + + /* But even if no shift was needed, update ksize */ + if (node->mn_ksize != key->mv_size) + node->mn_ksize = key->mv_size; + + if (key->mv_size) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + return MDB_SUCCESS; +} + +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); + +/** Perform \b act while tracking temporary cursor \b mn */ +#define WITH_CURSOR_TRACKING(mn, act) do { \ + MDB_cursor dummy, *tracked, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ + if ((mn).mc_flags & C_SUB) { \ + dummy.mc_flags = C_INITIALIZED; \ + dummy.mc_xcursor = (MDB_xcursor *)&(mn); \ + tracked = &dummy; \ + } else { \ + tracked = &(mn); \ + } \ + tracked->mc_next = *tp; \ + *tp = tracked; \ + { act; } \ + *tp = tracked->mc_next; \ +} while (0) + +/** Move a node from csrc to cdst. + */ +static int +mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) +{ + MDB_node *srcnode; + MDB_val key, data; + pgno_t srcpg; + MDB_cursor mn; + int rc; + unsigned short flags; + + DKBUF; + + /* Mark src and dst as dirty. */ + if ((rc = mdb_page_touch(csrc)) || + (rc = mdb_page_touch(cdst))) + return rc; + + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); + data.mv_size = 0; + data.mv_data = NULL; + srcpg = 0; + flags = 0; + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); + mdb_cassert(csrc, !((size_t)srcnode & 1)); + srcpg = NODEPGNO(srcnode); + flags = srcnode->mn_flags; + if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + unsigned int snum = csrc->mc_snum; + MDB_node *s2; + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(csrc); + if (rc) + return rc; + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + csrc->mc_snum = snum--; + csrc->mc_top = snum; + } else { + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + } + mn.mc_xcursor = NULL; + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { + unsigned int snum = cdst->mc_snum; + MDB_node *s2; + MDB_val bkey; + /* must find the lowest key below dst */ + mdb_cursor_copy(cdst, &mn); + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + bkey.mv_size = mn.mc_db->md_pad; + bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + bkey.mv_size = NODEKSZ(s2); + bkey.mv_data = NODEKEY(s2); + } + mn.mc_snum = snum--; + mn.mc_top = snum; + mn.mc_ki[snum] = 0; + rc = mdb_update_key(&mn, &bkey); + if (rc) + return rc; + } + + DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u", + IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", + csrc->mc_ki[csrc->mc_top], + DKEY(&key), + csrc->mc_pg[csrc->mc_top]->mp_pgno, + cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); + + /* Add the node to the destination page. + */ + rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); + if (rc != MDB_SUCCESS) + return rc; + + /* Delete the node from the source page. + */ + mdb_node_del(csrc, key.mv_size); + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + MDB_page *mpd, *mps; + + mps = csrc->mc_pg[csrc->mc_top]; + /* If we're adding on the left, bump others up */ + if (fromleft) { + mpd = cdst->mc_pg[csrc->mc_top]; + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) + continue; + if (m3 != cdst && + m3->mc_pg[csrc->mc_top] == mpd && + m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) { + m3->mc_ki[csrc->mc_top]++; + } + if (m3 !=csrc && + m3->mc_pg[csrc->mc_top] == mps && + m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + m3->mc_ki[csrc->mc_top-1]++; + } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); + } + } else + /* Adding on the right, bump others down */ + { + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) + continue; + if (m3->mc_pg[csrc->mc_top] == mps) { + if (!m3->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + m3->mc_ki[csrc->mc_top-1]--; + } else { + m3->mc_ki[csrc->mc_top]--; + } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); + } + } + } + } + + /* Update the parent separators. + */ + if (csrc->mc_ki[csrc->mc_top] == 0) { + if (csrc->mc_ki[csrc->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for source page %"Z"u to [%s]", + csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(csrc, &mn); + mn.mc_snum--; + mn.mc_top--; + /* We want mdb_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_update_key(&mn, &key)); + if (rc) + return rc; + } + if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + MDB_val nullkey; + indx_t ix = csrc->mc_ki[csrc->mc_top]; + nullkey.mv_size = 0; + csrc->mc_ki[csrc->mc_top] = 0; + rc = mdb_update_key(csrc, &nullkey); + csrc->mc_ki[csrc->mc_top] = ix; + mdb_cassert(csrc, rc == MDB_SUCCESS); + } + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + if (cdst->mc_ki[cdst->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for destination page %"Z"u to [%s]", + cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(cdst, &mn); + mn.mc_snum--; + mn.mc_top--; + /* We want mdb_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_update_key(&mn, &key)); + if (rc) + return rc; + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { + MDB_val nullkey; + indx_t ix = cdst->mc_ki[cdst->mc_top]; + nullkey.mv_size = 0; + cdst->mc_ki[cdst->mc_top] = 0; + rc = mdb_update_key(cdst, &nullkey); + cdst->mc_ki[cdst->mc_top] = ix; + mdb_cassert(cdst, rc == MDB_SUCCESS); + } + } + + return MDB_SUCCESS; +} + +/** Merge one page into another. + * The nodes from the page pointed to by \b csrc will + * be copied to the page pointed to by \b cdst and then + * the \b csrc page will be freed. + * @param[in] csrc Cursor pointing to the source page. + * @param[in] cdst Cursor pointing to the destination page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_page *psrc, *pdst; + MDB_node *srcnode; + MDB_val key, data; + unsigned nkeys; + int rc; + indx_t i, j; + + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + + DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); + + mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ + mdb_cassert(csrc, cdst->mc_snum > 1); + + /* Mark dst as dirty. */ + if ((rc = mdb_page_touch(cdst))) + return rc; + + /* get dst page again now that we've touched it. */ + pdst = cdst->mc_pg[cdst->mc_top]; + + /* Move all nodes from src to dst. + */ + j = nkeys = NUMKEYS(pdst); + if (IS_LEAF2(psrc)) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = METADATA(psrc); + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); + if (rc != MDB_SUCCESS) + return rc; + key.mv_data = (char *)key.mv_data + key.mv_size; + } + } else { + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + srcnode = NODEPTR(psrc, i); + if (i == 0 && IS_BRANCH(psrc)) { + MDB_cursor mn; + MDB_node *s2; + mdb_cursor_copy(csrc, &mn); + mn.mc_xcursor = NULL; + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + key.mv_size = mn.mc_db->md_pad; + key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + } else { + key.mv_size = srcnode->mn_ksize; + key.mv_data = NODEKEY(srcnode); + } + + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); + if (rc != MDB_SUCCESS) + return rc; + } + } + + DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", + pdst->mp_pgno, NUMKEYS(pdst), + (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); + + /* Unlink the src page from parent and add to free list. + */ + csrc->mc_top--; + mdb_node_del(csrc, 0); + if (csrc->mc_ki[csrc->mc_top] == 0) { + key.mv_size = 0; + rc = mdb_update_key(csrc, &key); + if (rc) { + csrc->mc_top++; + return rc; + } + } + csrc->mc_top++; + + psrc = csrc->mc_pg[csrc->mc_top]; + /* If not operating on FreeDB, allow this page to be reused + * in this txn. Otherwise just add to free list. + */ + rc = mdb_page_loose(csrc, psrc); + if (rc) + return rc; + if (IS_LEAF(psrc)) + csrc->mc_db->md_leaf_pages--; + else + csrc->mc_db->md_branch_pages--; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + unsigned int top = csrc->mc_top; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_snum < csrc->mc_snum) continue; + if (m3->mc_pg[top] == psrc) { + m3->mc_pg[top] = pdst; + m3->mc_ki[top] += nkeys; + m3->mc_ki[top-1] = cdst->mc_ki[top-1]; + } else if (m3->mc_pg[top-1] == csrc->mc_pg[top-1] && + m3->mc_ki[top-1] > csrc->mc_ki[top-1]) { + m3->mc_ki[top-1]--; + } + if (IS_LEAF(psrc)) + XCURSOR_REFRESH(m3, top, m3->mc_pg[top]); + } + } + { + unsigned int snum = cdst->mc_snum; + uint16_t depth = cdst->mc_db->md_depth; + mdb_cursor_pop(cdst); + rc = mdb_rebalance(cdst); + /* Did the tree height change? */ + if (depth != cdst->mc_db->md_depth) + snum += cdst->mc_db->md_depth - depth; + cdst->mc_snum = snum; + cdst->mc_top = snum-1; + } + return rc; +} + +/** Copy the contents of a cursor. + * @param[in] csrc The cursor to copy from. + * @param[out] cdst The cursor to copy to. + */ +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) +{ + unsigned int i; + + cdst->mc_txn = csrc->mc_txn; + cdst->mc_dbi = csrc->mc_dbi; + cdst->mc_db = csrc->mc_db; + cdst->mc_dbx = csrc->mc_dbx; + cdst->mc_snum = csrc->mc_snum; + cdst->mc_top = csrc->mc_top; + cdst->mc_flags = csrc->mc_flags; + + for (i=0; imc_snum; i++) { + cdst->mc_pg[i] = csrc->mc_pg[i]; + cdst->mc_ki[i] = csrc->mc_ki[i]; + } +} + +/** Rebalance the tree after a delete operation. + * @param[in] mc Cursor pointing to the page where rebalancing + * should begin. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_rebalance(MDB_cursor *mc) +{ + MDB_node *node; + int rc, fromleft; + unsigned int ptop, minkeys, thresh; + MDB_cursor mn; + indx_t oldki; + + if (IS_BRANCH(mc->mc_pg[mc->mc_top])) { + minkeys = 2; + thresh = 1; + } else { + minkeys = 1; + thresh = FILL_THRESHOLD; + } + DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", + IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); + + if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh && + NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { + DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); + return MDB_SUCCESS; + } + + if (mc->mc_snum < 2) { + MDB_page *mp = mc->mc_pg[0]; + if (IS_SUBP(mp)) { + DPUTS("Can't rebalance a subpage, ignoring"); + return MDB_SUCCESS; + } + if (NUMKEYS(mp) == 0) { + DPUTS("tree is completely empty"); + mc->mc_db->md_root = P_INVALID; + mc->mc_db->md_depth = 0; + mc->mc_db->md_leaf_pages = 0; + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + /* Adjust cursors pointing to mp */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (!(m3->mc_flags & C_INITIALIZED) || (m3->mc_snum < mc->mc_snum)) + continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum = 0; + m3->mc_top = 0; + m3->mc_flags &= ~C_INITIALIZED; + } + } + } + } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { + int i; + DPUTS("collapsing root page!"); + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); + rc = mdb_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL); + if (rc) + return rc; + mc->mc_db->md_depth--; + mc->mc_db->md_branch_pages--; + mc->mc_ki[0] = mc->mc_ki[1]; + for (i = 1; imc_db->md_depth; i++) { + mc->mc_pg[i] = mc->mc_pg[i+1]; + mc->mc_ki[i] = mc->mc_ki[i+1]; + } + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) continue; + if (!(m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_pg[0] == mp) { + for (i=0; imc_db->md_depth; i++) { + m3->mc_pg[i] = m3->mc_pg[i+1]; + m3->mc_ki[i] = m3->mc_ki[i+1]; + } + m3->mc_snum--; + m3->mc_top--; + } + } + } + } else + DPUTS("root page doesn't need rebalancing"); + return MDB_SUCCESS; + } + + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. + */ + ptop = mc->mc_top-1; + mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); + + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. + */ + + /* Find neighbors. + */ + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + + oldki = mc->mc_ki[mc->mc_top]; + if (mc->mc_ki[ptop] == 0) { + /* We're the leftmost leaf in our parent. + */ + DPUTS("reading right neighbor"); + mn.mc_ki[ptop]++; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = 0; + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + fromleft = 0; + } else { + /* There is at least one neighbor to the left. + */ + DPUTS("reading left neighbor"); + mn.mc_ki[ptop]--; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; + mc->mc_ki[mc->mc_top] = 0; + fromleft = 1; + } + + DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)", + mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); + + /* If the neighbor page is above threshold and has enough keys, + * move one key from it. Otherwise we should try to merge them. + * (A branch page must never have less than 2 keys.) + */ + if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { + rc = mdb_node_move(&mn, mc, fromleft); + if (fromleft) { + /* if we inserted on left, bump position up */ + oldki++; + } + } else { + if (!fromleft) { + rc = mdb_page_merge(&mn, mc); + } else { + oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); + mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; + /* We want mdb_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_page_merge(mc, &mn)); + mdb_cursor_copy(&mn, mc); + } + mc->mc_flags &= ~C_EOF; + } + mc->mc_ki[mc->mc_top] = oldki; + return rc; +} + +/** Complete a delete operation started by #mdb_cursor_del(). */ +static int +mdb_cursor_del0(MDB_cursor *mc) +{ + int rc; + MDB_page *mp; + indx_t ki; + unsigned int nkeys; + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + ki = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + mdb_node_del(mc, mc->mc_db->md_pad); + mc->mc_db->md_entries--; + { + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3 == mc || m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] == ki) { + m3->mc_flags |= C_DEL; + if (mc->mc_db->md_flags & MDB_DUPSORT) { + /* Sub-cursor referred into dataset which is gone */ + m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } + continue; + } else if (m3->mc_ki[mc->mc_top] > ki) { + m3->mc_ki[mc->mc_top]--; + } + XCURSOR_REFRESH(m3, mc->mc_top, mp); + } + } + } + rc = mdb_rebalance(mc); + if (rc) + goto fail; + + /* DB is totally empty now, just bail out. + * Other cursors adjustments were already done + * by mdb_rebalance and aren't needed here. + */ + if (!mc->mc_snum) { + mc->mc_flags |= C_EOF; + return rc; + } + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) { + /* if m3 points past last node in page, find next sibling */ + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(m3, 1); + if (rc == MDB_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + continue; + } + if (rc) + goto fail; + } + if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) { + MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]); + /* If this node has dupdata, it may need to be reinited + * because its data has moved. + * If the xcursor was not initd it must be reinited. + * Else if node points to a subDB, nothing is needed. + * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset. + */ + if (node->mn_flags & F_DUPDATA) { + if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + if (!(node->mn_flags & F_SUBDATA)) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } else { + mdb_xcursor_init1(m3, node); + rc = mdb_cursor_first(&m3->mc_xcursor->mx_cursor, NULL, NULL); + if (rc) + goto fail; + } + } + m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; + } + } + } + } + mc->mc_flags |= C_DEL; + +fail: + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_del(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { + /* must ignore any data */ + data = NULL; + } + + return mdb_del0(txn, dbi, key, data, 0); +} + +static int +mdb_del0(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + MDB_cursor_op op; + MDB_val rdata, *xdata; + int rc, exact = 0; + DKBUF; + + DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); + + mdb_cursor_init(&mc, txn, dbi, &mx); + + if (data) { + op = MDB_GET_BOTH; + rdata = *data; + xdata = &rdata; + } else { + op = MDB_SET; + xdata = NULL; + flags |= MDB_NODUPDATA; + } + rc = mdb_cursor_set(&mc, key, xdata, op, &exact); + if (rc == 0) { + /* let mdb_page_split know about this cursor if needed: + * delete will trigger a rebalance; if it needs to move + * a node from one page to another, it will have to + * update the parent's separator key(s). If the new sepkey + * is larger than the current one, the parent page may + * run out of space, triggering a split. We need this + * cursor to be consistent until the end of the rebalance. + */ + mc.mc_flags |= C_UNTRACK; + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = mdb_cursor_del(&mc, flags); + txn->mt_cursors[dbi] = mc.mc_next; + } + return rc; +} + +/** Split a page and insert a new node. + * Set #MDB_TXN_ERROR on failure. + * @param[in,out] mc Cursor pointing to the page and desired insertion index. + * The cursor will be updated to point to the actual page and index where + * the node got inserted after the split. + * @param[in] newkey The key for the newly inserted node. + * @param[in] newdata The data for the newly inserted node. + * @param[in] newpgno The page number, if the new node is a branch node. + * @param[in] nflags The #NODE_ADD_FLAGS for the new node. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, + unsigned int nflags) +{ + unsigned int flags; + int rc = MDB_SUCCESS, new_root = 0, did_split = 0; + indx_t newindx; + pgno_t pgno = 0; + int i, j, split_indx, nkeys, pmax; + MDB_env *env = mc->mc_txn->mt_env; + MDB_node *node; + MDB_val sepkey, rkey, xdata, *rdata = &xdata; + MDB_page *copy = NULL; + MDB_page *mp, *rp, *pp; + int ptop; + MDB_cursor mn; + DKBUF; + + mp = mc->mc_pg[mc->mc_top]; + newindx = mc->mc_ki[mc->mc_top]; + nkeys = NUMKEYS(mp); + + DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i", + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, + DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); + + /* Create a right sibling. */ + if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) + return rc; + rp->mp_pad = mp->mp_pad; + DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); + + /* Usually when splitting the root page, the cursor + * height is 1. But when called from mdb_update_key, + * the cursor height may be greater because it walks + * up the stack while finding the branch slot to update. + */ + if (mc->mc_top < 1) { + if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) + goto done; + /* shift current top to make room for new parent */ + for (i=mc->mc_snum; i>0; i--) { + mc->mc_pg[i] = mc->mc_pg[i-1]; + mc->mc_ki[i] = mc->mc_ki[i-1]; + } + mc->mc_pg[0] = pp; + mc->mc_ki[0] = 0; + mc->mc_db->md_root = pp->mp_pgno; + DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); + new_root = mc->mc_db->md_depth++; + + /* Add left (implicit) pointer. */ + if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { + /* undo the pre-push */ + mc->mc_pg[0] = mc->mc_pg[1]; + mc->mc_ki[0] = mc->mc_ki[1]; + mc->mc_db->md_root = mp->mp_pgno; + mc->mc_db->md_depth--; + goto done; + } + mc->mc_snum++; + mc->mc_top++; + ptop = 0; + } else { + ptop = mc->mc_top-1; + DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno)); + } + + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + mn.mc_pg[mn.mc_top] = rp; + mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; + + if (nflags & MDB_APPEND) { + mn.mc_ki[mn.mc_top] = 0; + sepkey = *newkey; + split_indx = newindx; + nkeys = 0; + } else { + + split_indx = (nkeys+1) / 2; + + if (IS_LEAF2(rp)) { + char *split, *ins; + int x; + unsigned int lsize, rsize, ksize; + /* Move half of the keys to the right sibling */ + x = mc->mc_ki[mc->mc_top] - split_indx; + ksize = mc->mc_db->md_pad; + split = LEAF2KEY(mp, split_indx, ksize); + rsize = (nkeys - split_indx) * ksize; + lsize = (nkeys - split_indx) * sizeof(indx_t); + mp->mp_lower -= lsize; + rp->mp_lower += lsize; + mp->mp_upper += rsize - lsize; + rp->mp_upper -= rsize - lsize; + sepkey.mv_size = ksize; + if (newindx == split_indx) { + sepkey.mv_data = newkey->mv_data; + } else { + sepkey.mv_data = split; + } + if (x<0) { + ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); + memcpy(rp->mp_ptrs, split, rsize); + sepkey.mv_data = rp->mp_ptrs; + memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); + memcpy(ins, newkey->mv_data, ksize); + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + } else { + if (x) + memcpy(rp->mp_ptrs, split, x * ksize); + ins = LEAF2KEY(rp, x, ksize); + memcpy(ins, newkey->mv_data, ksize); + memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); + rp->mp_lower += sizeof(indx_t); + rp->mp_upper -= ksize - sizeof(indx_t); + mc->mc_ki[mc->mc_top] = x; + } + } else { + int psize, nsize, k, keythresh; + + /* Maximum free space in an empty page */ + pmax = env->me_psize - PAGEHDRSZ; + /* Threshold number of keys considered "small" */ + keythresh = env->me_psize >> 7; + + if (IS_LEAF(mp)) + nsize = mdb_leaf_size(env, newkey, newdata); + else + nsize = mdb_branch_size(env, newkey); + nsize = EVEN(nsize); + + /* grab a page to hold a temporary copy */ + copy = mdb_page_malloc(mc->mc_txn, 1); + if (copy == NULL) { + rc = ENOMEM; + goto done; + } + copy->mp_pgno = mp->mp_pgno; + copy->mp_flags = mp->mp_flags; + copy->mp_lower = (PAGEHDRSZ-PAGEBASE); + copy->mp_upper = env->me_psize - PAGEBASE; + + /* prepare to insert */ + for (i=0, j=0; imp_ptrs[j++] = 0; + } + copy->mp_ptrs[j++] = mp->mp_ptrs[i]; + } + + /* When items are relatively large the split point needs + * to be checked, because being off-by-one will make the + * difference between success or failure in mdb_node_add. + * + * It's also relevant if a page happens to be laid out + * such that one half of its nodes are all "small" and + * the other half of its nodes are "large." If the new + * item is also "large" and falls on the half with + * "large" nodes, it also may not fit. + * + * As a final tweak, if the new item goes on the last + * spot on the page (and thus, onto the new page), bias + * the split so the new page is emptier than the old page. + * This yields better packing during sequential inserts. + */ + if (nkeys < keythresh || nsize > pmax/16 || newindx >= nkeys) { + /* Find split point */ + psize = 0; + if (newindx <= split_indx || newindx >= nkeys) { + i = 0; j = 1; + k = newindx >= nkeys ? nkeys : split_indx+1+IS_LEAF(mp); + } else { + i = nkeys; j = -1; + k = split_indx-1; + } + for (; i!=k; i+=j) { + if (i == newindx) { + psize += nsize; + node = NULL; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + psize += sizeof(pgno_t); + else + psize += NODEDSZ(node); + } + psize = EVEN(psize); + } + if (psize > pmax || i == k-j) { + split_indx = i + (j<0); + break; + } + } + } + if (split_indx == newindx) { + sepkey.mv_size = newkey->mv_size; + sepkey.mv_data = newkey->mv_data; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); + sepkey.mv_size = node->mn_ksize; + sepkey.mv_data = NODEKEY(node); + } + } + } + + DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); + + /* Copy separator key to the parent. + */ + if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { + int snum = mc->mc_snum; + mn.mc_snum--; + mn.mc_top--; + did_split = 1; + /* We want other splits to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0)); + if (rc) + goto done; + + /* root split? */ + if (mc->mc_snum > snum) { + ptop++; + } + /* Right page might now have changed parent. + * Check if left page also changed parent. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; imc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + if (mn.mc_ki[ptop]) { + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } else { + /* find right page's left sibling */ + mc->mc_ki[ptop] = mn.mc_ki[ptop]; + mdb_cursor_sibling(mc, 0); + } + } + } else { + mn.mc_top--; + rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); + mn.mc_top++; + } + if (rc != MDB_SUCCESS) { + goto done; + } + if (nflags & MDB_APPEND) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[mc->mc_top] = 0; + rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); + if (rc) + goto done; + for (i=0; imc_top; i++) + mc->mc_ki[i] = mn.mc_ki[i]; + } else if (!IS_LEAF2(mp)) { + /* Move nodes */ + mc->mc_pg[mc->mc_top] = rp; + i = split_indx; + j = 0; + do { + if (i == newindx) { + rkey.mv_data = newkey->mv_data; + rkey.mv_size = newkey->mv_size; + if (IS_LEAF(mp)) { + rdata = newdata; + } else + pgno = newpgno; + flags = nflags; + /* Update index for the new key. */ + mc->mc_ki[mc->mc_top] = j; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + rkey.mv_data = NODEKEY(node); + rkey.mv_size = node->mn_ksize; + if (IS_LEAF(mp)) { + xdata.mv_data = NODEDATA(node); + xdata.mv_size = NODEDSZ(node); + rdata = &xdata; + } else + pgno = NODEPGNO(node); + flags = node->mn_flags; + } + + if (!IS_LEAF(mp) && j == 0) { + /* First branch index doesn't need key data. */ + rkey.mv_size = 0; + } + + rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); + if (rc) + goto done; + if (i == nkeys) { + i = 0; + j = 0; + mc->mc_pg[mc->mc_top] = copy; + } else { + i++; + j++; + } + } while (i != split_indx); + + nkeys = NUMKEYS(copy); + for (i=0; imp_ptrs[i] = copy->mp_ptrs[i]; + mp->mp_lower = copy->mp_lower; + mp->mp_upper = copy->mp_upper; + memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), + env->me_psize - copy->mp_upper - PAGEBASE); + + /* reset back to original page */ + if (newindx < split_indx) { + mc->mc_pg[mc->mc_top] = mp; + } else { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<=ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + if (nflags & MDB_RESERVE) { + node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!(node->mn_flags & F_BIGDATA)) + newdata->mv_data = NODEDATA(node); + } + } else { + if (newindx >= split_indx) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<=ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + } + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + nkeys = NUMKEYS(mp); + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) + continue; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (new_root) { + int k; + /* sub cursors may be on different DB */ + if (m3->mc_pg[0] != mp) + continue; + /* root split */ + for (k=new_root; k>=0; k--) { + m3->mc_ki[k+1] = m3->mc_ki[k]; + m3->mc_pg[k+1] = m3->mc_pg[k]; + } + if (m3->mc_ki[0] >= nkeys) { + m3->mc_ki[0] = 1; + } else { + m3->mc_ki[0] = 0; + } + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum++; + m3->mc_top++; + } + if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) + m3->mc_ki[mc->mc_top]++; + if (m3->mc_ki[mc->mc_top] >= nkeys) { + m3->mc_pg[mc->mc_top] = rp; + m3->mc_ki[mc->mc_top] -= nkeys; + for (i=0; imc_top; i++) { + m3->mc_ki[i] = mn.mc_ki[i]; + m3->mc_pg[i] = mn.mc_pg[i]; + } + } + } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && + m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { + m3->mc_ki[ptop]++; + } + if (IS_LEAF(mp)) + XCURSOR_REFRESH(m3, mc->mc_top, m3->mc_pg[mc->mc_top]); + } + } + DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); + +done: + if (copy) /* tmp page */ + mdb_page_free(env, copy); + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_put(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned int flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + int rc; + + if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) + return EINVAL; + + if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + mdb_cursor_init(&mc, txn, dbi, &mx); + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = mdb_cursor_put(&mc, key, data, flags); + txn->mt_cursors[dbi] = mc.mc_next; + return rc; +} + +#ifndef MDB_WBUF +#define MDB_WBUF (1024*1024) +#endif +#define MDB_EOF 0x10 /**< #mdb_env_copyfd1() is done reading */ + + /** State needed for a double-buffering compacting copy. */ +typedef struct mdb_copy { + MDB_env *mc_env; + MDB_txn *mc_txn; + pthread_mutex_t mc_mutex; + pthread_cond_t mc_cond; /**< Condition variable for #mc_new */ + char *mc_wbuf[2]; + char *mc_over[2]; + int mc_wlen[2]; + int mc_olen[2]; + pgno_t mc_next_pgno; + HANDLE mc_fd; + int mc_toggle; /**< Buffer number in provider */ + int mc_new; /**< (0-2 buffers to write) | (#MDB_EOF at end) */ + /** Error code. Never cleared if set. Both threads can set nonzero + * to fail the copy. Not mutex-protected, LMDB expects atomic int. + */ + volatile int mc_error; +} mdb_copy; + + /** Dedicated writer thread for compacting copy. */ +static THREAD_RET ESECT CALL_CONV +mdb_env_copythr(void *arg) +{ + mdb_copy *my = arg; + char *ptr; + int toggle = 0, wsize, rc; +#ifdef _WIN32 + DWORD len; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + int len; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#ifdef SIGPIPE + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGPIPE); + if ((rc = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0) + my->mc_error = rc; +#endif +#endif + + pthread_mutex_lock(&my->mc_mutex); + for(;;) { + while (!my->mc_new) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + if (my->mc_new == 0 + MDB_EOF) /* 0 buffers, just EOF */ + break; + wsize = my->mc_wlen[toggle]; + ptr = my->mc_wbuf[toggle]; +again: + rc = MDB_SUCCESS; + while (wsize > 0 && !my->mc_error) { + DO_WRITE(rc, my->mc_fd, ptr, wsize, len); + if (!rc) { + rc = ErrCode(); +#if defined(SIGPIPE) && !defined(_WIN32) + if (rc == EPIPE) { + /* Collect the pending SIGPIPE, otherwise at least OS X + * gives it to the process on thread-exit (ITS#8504). + */ + int tmp; + sigwait(&set, &tmp); + } +#endif + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + if (rc) { + my->mc_error = rc; + } + /* If there's an overflow page tail, write it too */ + if (my->mc_olen[toggle]) { + wsize = my->mc_olen[toggle]; + ptr = my->mc_over[toggle]; + my->mc_olen[toggle] = 0; + goto again; + } + my->mc_wlen[toggle] = 0; + toggle ^= 1; + /* Return the empty buffer to provider */ + my->mc_new--; + pthread_cond_signal(&my->mc_cond); + } + pthread_mutex_unlock(&my->mc_mutex); + return (THREAD_RET)0; +#undef DO_WRITE +} + + /** Give buffer and/or #MDB_EOF to writer thread, await unused buffer. + * + * @param[in] my control structure. + * @param[in] adjust (1 to hand off 1 buffer) | (MDB_EOF when ending). + */ +static int ESECT +mdb_env_cthr_toggle(mdb_copy *my, int adjust) +{ + pthread_mutex_lock(&my->mc_mutex); + my->mc_new += adjust; + pthread_cond_signal(&my->mc_cond); + while (my->mc_new & 2) /* both buffers in use */ + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + pthread_mutex_unlock(&my->mc_mutex); + + my->mc_toggle ^= (adjust & 1); + /* Both threads reset mc_wlen, to be safe from threading errors */ + my->mc_wlen[my->mc_toggle] = 0; + return my->mc_error; +} + + /** Depth-first tree traversal for compacting copy. + * @param[in] my control structure. + * @param[in,out] pg database root. + * @param[in] flags includes #F_DUPDATA if it is a sorted-duplicate sub-DB. + */ +static int ESECT +mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) +{ + MDB_cursor mc = {0}; + MDB_node *ni; + MDB_page *mo, *mp, *leaf; + char *buf, *ptr; + int rc, toggle; + unsigned int i; + + /* Empty DB, nothing to do */ + if (*pg == P_INVALID) + return MDB_SUCCESS; + + mc.mc_snum = 1; + mc.mc_txn = my->mc_txn; + + rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL); + if (rc) + return rc; + rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); + if (rc) + return rc; + + /* Make cursor pages writable */ + buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); + if (buf == NULL) + return ENOMEM; + + for (i=0; imc_env->me_psize); + mc.mc_pg[i] = (MDB_page *)ptr; + ptr += my->mc_env->me_psize; + } + + /* This is writable space for a leaf page. Usually not needed. */ + leaf = (MDB_page *)ptr; + + toggle = my->mc_toggle; + while (mc.mc_snum > 0) { + unsigned n; + mp = mc.mc_pg[mc.mc_top]; + n = NUMKEYS(mp); + + if (IS_LEAF(mp)) { + if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { + for (i=0; imn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t)); + rc = mdb_page_get(&mc, pg, &omp, NULL); + if (rc) + goto done; + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + memcpy(mo, omp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno; + my->mc_next_pgno += omp->mp_pages; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (omp->mp_pages > 1) { + my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1); + my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + } else if (ni->mn_flags & F_SUBDATA) { + MDB_db db; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&db, NODEDATA(ni), sizeof(db)); + my->mc_toggle = toggle; + rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); + if (rc) + goto done; + toggle = my->mc_toggle; + memcpy(NODEDATA(ni), &db, sizeof(db)); + } + } + } + } else { + mc.mc_ki[mc.mc_top]++; + if (mc.mc_ki[mc.mc_top] < n) { + pgno_t pg; +again: + ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); + pg = NODEPGNO(ni); + rc = mdb_page_get(&mc, pg, &mp, NULL); + if (rc) + goto done; + mc.mc_top++; + mc.mc_snum++; + mc.mc_ki[mc.mc_top] = 0; + if (IS_BRANCH(mp)) { + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. + */ + mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); + goto again; + } else + mc.mc_pg[mc.mc_top] = mp; + continue; + } + } + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + mdb_page_copy(mo, mp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno++; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (mc.mc_top) { + /* Update parent if there is one */ + ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); + SETPGNO(ni, mo->mp_pgno); + mdb_cursor_pop(&mc); + } else { + /* Otherwise we're done */ + *pg = mo->mp_pgno; + break; + } + } +done: + free(buf); + return rc; +} + + /** Copy environment with compaction. */ +static int ESECT +mdb_env_copyfd1(MDB_env *env, HANDLE fd) +{ + MDB_meta *mm; + MDB_page *mp; + mdb_copy my = {0}; + MDB_txn *txn = NULL; + pthread_t thr; + pgno_t root, new_root; + int rc = MDB_SUCCESS; + +#ifdef _WIN32 + if (!(my.mc_mutex = CreateMutex(NULL, FALSE, NULL)) || + !(my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL))) { + rc = ErrCode(); + goto done; + } + my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); + if (my.mc_wbuf[0] == NULL) { + /* _aligned_malloc() sets errno, but we use Windows error codes */ + rc = ERROR_NOT_ENOUGH_MEMORY; + goto done; + } +#else + if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0) + return rc; + if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0) + goto done2; +#ifdef HAVE_MEMALIGN + my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); + if (my.mc_wbuf[0] == NULL) { + rc = errno; + goto done; + } +#else + { + void *p; + if ((rc = posix_memalign(&p, env->me_os_psize, MDB_WBUF*2)) != 0) + goto done; + my.mc_wbuf[0] = p; + } +#endif +#endif + memset(my.mc_wbuf[0], 0, MDB_WBUF*2); + my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; + my.mc_next_pgno = NUM_METAS; + my.mc_env = env; + my.mc_fd = fd; + rc = THREAD_CREATE(thr, mdb_env_copythr, &my); + if (rc) + goto done; + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + goto finish; + + mp = (MDB_page *)my.mc_wbuf[0]; + memset(mp, 0, NUM_METAS * env->me_psize); + mp->mp_pgno = 0; + mp->mp_flags = P_META; + mm = (MDB_meta *)METADATA(mp); + mdb_env_init_meta0(env, mm); + mm->mm_address = env->me_metas[0]->mm_address; + + mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); + mp->mp_pgno = 1; + mp->mp_flags = P_META; + *(MDB_meta *)METADATA(mp) = *mm; + mm = (MDB_meta *)METADATA(mp); + + /* Set metapage 1 with current main DB */ + root = new_root = txn->mt_dbs[MAIN_DBI].md_root; + if (root != P_INVALID) { + /* Count free pages + freeDB pages. Subtract from last_pg + * to find the new last_pg, which also becomes the new root. + */ + MDB_ID freecount = 0; + MDB_cursor mc; + MDB_val key, data; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + if (rc != MDB_NOTFOUND) + goto finish; + freecount += txn->mt_dbs[FREE_DBI].md_branch_pages + + txn->mt_dbs[FREE_DBI].md_leaf_pages + + txn->mt_dbs[FREE_DBI].md_overflow_pages; + + new_root = txn->mt_next_pgno - 1 - freecount; + mm->mm_last_pg = new_root; + mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; + mm->mm_dbs[MAIN_DBI].md_root = new_root; + } else { + /* When the DB is empty, handle it specially to + * fix any breakage like page leaks from ITS#8174. + */ + mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags; + } + if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) { + mm->mm_txnid = 1; /* use metapage 1 */ + } + + my.mc_wlen[0] = env->me_psize * NUM_METAS; + my.mc_txn = txn; + rc = mdb_env_cwalk(&my, &root, 0); + if (rc == MDB_SUCCESS && root != new_root) { + rc = MDB_INCOMPATIBLE; /* page leak or corrupt DB */ + } + +finish: + if (rc) + my.mc_error = rc; + mdb_env_cthr_toggle(&my, 1 | MDB_EOF); + rc = THREAD_FINISH(thr); + mdb_txn_abort(txn); + +done: +#ifdef _WIN32 + if (my.mc_wbuf[0]) _aligned_free(my.mc_wbuf[0]); + if (my.mc_cond) CloseHandle(my.mc_cond); + if (my.mc_mutex) CloseHandle(my.mc_mutex); +#else + free(my.mc_wbuf[0]); + pthread_cond_destroy(&my.mc_cond); +done2: + pthread_mutex_destroy(&my.mc_mutex); +#endif + return rc ? rc : my.mc_error; +} + + /** Copy environment as-is. */ +static int ESECT +mdb_env_copyfd0(MDB_env *env, HANDLE fd) +{ + MDB_txn *txn = NULL; + mdb_mutexref_t wmutex = NULL; + int rc; + size_t wsize, w3; + char *ptr; +#ifdef _WIN32 + DWORD len, w2; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + ssize_t len; + size_t w2; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. + */ + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + if (env->me_txns) { + /* We must start the actual read txn after blocking writers */ + mdb_txn_end(txn, MDB_END_RESET_TMP); + + /* Temporarily block writers until we snapshot the meta pages */ + wmutex = env->me_wmutex; + if (LOCK_MUTEX(rc, env, wmutex)) + goto leave; + + rc = mdb_txn_renew0(txn); + if (rc) { + UNLOCK_MUTEX(wmutex); + goto leave; + } + } + + wsize = env->me_psize * NUM_METAS; + ptr = env->me_map; + w2 = wsize; + while (w2 > 0) { + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + w2 -= len; + continue; + } else { + /* Non-blocking or async handles are not supported */ + rc = EIO; + break; + } + } + if (wmutex) + UNLOCK_MUTEX(wmutex); + + if (rc) + goto leave; + + w3 = txn->mt_next_pgno * env->me_psize; + { + size_t fsize = 0; + if ((rc = mdb_fsize(env->me_fd, &fsize))) + goto leave; + if (w3 > fsize) + w3 = fsize; + } + wsize = w3 - wsize; + while (wsize > 0) { + if (wsize > MAX_WRITE) + w2 = MAX_WRITE; + else + w2 = wsize; + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + +leave: + mdb_txn_abort(txn); + return rc; +} + +int ESECT +mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) +{ + if (flags & MDB_CP_COMPACT) + return mdb_env_copyfd1(env, fd); + else + return mdb_env_copyfd0(env, fd); +} + +int ESECT +mdb_env_copyfd(MDB_env *env, HANDLE fd) +{ + return mdb_env_copyfd2(env, fd, 0); +} + +int ESECT +mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) +{ + int rc; + MDB_name fname; + HANDLE newfd = INVALID_HANDLE_VALUE; + + rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname); + if (rc == MDB_SUCCESS) { + rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd); + mdb_fname_destroy(fname); + } + if (rc == MDB_SUCCESS) { + rc = mdb_env_copyfd2(env, newfd, flags); + if (close(newfd) < 0 && rc == MDB_SUCCESS) + rc = ErrCode(); + } + return rc; +} + +int ESECT +mdb_env_copy(MDB_env *env, const char *path) +{ + return mdb_env_copy2(env, path, 0); +} + +int ESECT +mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) +{ + if (flag & ~CHANGEABLE) + return EINVAL; + if (onoff) + env->me_flags |= flag; + else + env->me_flags &= ~flag; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_flags(MDB_env *env, unsigned int *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_flags & (CHANGEABLE|CHANGELESS); + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_userctx(MDB_env *env, void *ctx) +{ + if (!env) + return EINVAL; + env->me_userctx = ctx; + return MDB_SUCCESS; +} + +void * ESECT +mdb_env_get_userctx(MDB_env *env) +{ + return env ? env->me_userctx : NULL; +} + +int ESECT +mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) +{ + if (!env) + return EINVAL; +#ifndef NDEBUG + env->me_assert_func = func; +#endif + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_path(MDB_env *env, const char **arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_path; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_fd; + return MDB_SUCCESS; +} + +/** Common code for #mdb_stat() and #mdb_env_stat(). + * @param[in] env the environment to operate in. + * @param[in] db the #MDB_db record containing the stats to return. + * @param[out] arg the address of an #MDB_stat structure to receive the stats. + * @return 0, this function always succeeds. + */ +static int ESECT +mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) +{ + arg->ms_psize = env->me_psize; + arg->ms_depth = db->md_depth; + arg->ms_branch_pages = db->md_branch_pages; + arg->ms_leaf_pages = db->md_leaf_pages; + arg->ms_overflow_pages = db->md_overflow_pages; + arg->ms_entries = db->md_entries; + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_stat(MDB_env *env, MDB_stat *arg) +{ + MDB_meta *meta; + + if (env == NULL || arg == NULL) + return EINVAL; + + meta = mdb_env_pick_meta(env); + + return mdb_stat0(env, &meta->mm_dbs[MAIN_DBI], arg); +} + +int ESECT +mdb_env_info(MDB_env *env, MDB_envinfo *arg) +{ + MDB_meta *meta; + + if (env == NULL || arg == NULL) + return EINVAL; + + meta = mdb_env_pick_meta(env); + arg->me_mapaddr = meta->mm_address; + arg->me_last_pgno = meta->mm_last_pg; + arg->me_last_txnid = meta->mm_txnid; + + arg->me_mapsize = env->me_mapsize; + arg->me_maxreaders = env->me_maxreaders; + arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : 0; + return MDB_SUCCESS; +} + +/** Set the default comparison functions for a database. + * Called immediately after a database is opened to set the defaults. + * The user can then override them with #mdb_set_compare() or + * #mdb_set_dupsort(). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +static void +mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) +{ + uint16_t f = txn->mt_dbs[dbi].md_flags; + + txn->mt_dbxs[dbi].md_cmp = + (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : + (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; + + txn->mt_dbxs[dbi].md_dcmp = + !(f & MDB_DUPSORT) ? 0 : + ((f & MDB_INTEGERDUP) + ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) + : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); +} + +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) +{ + MDB_val key, data; + MDB_dbi i; + MDB_cursor mc; + MDB_db dummy; + int rc, dbflag, exact; + unsigned int unused = 0, seq; + char *namedup; + size_t len; + + if (flags & ~VALID_FLAGS) + return EINVAL; + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + /* main DB? */ + if (!name) { + *dbi = MAIN_DBI; + if (flags & PERSISTENT_FLAGS) { + uint16_t f2 = flags & PERSISTENT_FLAGS; + /* make sure flag changes get committed */ + if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { + txn->mt_dbs[MAIN_DBI].md_flags |= f2; + txn->mt_flags |= MDB_TXN_DIRTY; + } + } + mdb_default_cmp(txn, MAIN_DBI); + return MDB_SUCCESS; + } + + if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, MAIN_DBI); + } + + /* Is the DB already open? */ + len = strlen(name); + for (i=CORE_DBS; imt_numdbs; i++) { + if (!txn->mt_dbxs[i].md_name.mv_size) { + /* Remember this free slot */ + if (!unused) unused = i; + continue; + } + if (len == txn->mt_dbxs[i].md_name.mv_size && + !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { + *dbi = i; + return MDB_SUCCESS; + } + } + + /* If no free slot and max hit, fail */ + if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) + return MDB_DBS_FULL; + + /* Cannot mix named databases with some mainDB flags */ + if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) + return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; + + /* Find the DB info */ + dbflag = DB_NEW|DB_VALID|DB_USRVALID; + exact = 0; + key.mv_size = len; + key.mv_data = (void *)name; + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); + if (rc == MDB_SUCCESS) { + /* make sure this is actually a DB */ + MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) + return MDB_INCOMPATIBLE; + } else { + if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE)) + return rc; + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + } + + /* Done here so we cannot fail after creating a new DB */ + if ((namedup = strdup(name)) == NULL) + return ENOMEM; + + if (rc) { + /* MDB_NOTFOUND and MDB_CREATE: Create new DB */ + data.mv_size = sizeof(MDB_db); + data.mv_data = &dummy; + memset(&dummy, 0, sizeof(dummy)); + dummy.md_root = P_INVALID; + dummy.md_flags = flags & PERSISTENT_FLAGS; + WITH_CURSOR_TRACKING(mc, + rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA)); + dbflag |= DB_DIRTY; + } + + if (rc) { + free(namedup); + } else { + /* Got info, register DBI in this txn */ + unsigned int slot = unused ? unused : txn->mt_numdbs; + txn->mt_dbxs[slot].md_name.mv_data = namedup; + txn->mt_dbxs[slot].md_name.mv_size = len; + txn->mt_dbxs[slot].md_rel = NULL; + txn->mt_dbflags[slot] = dbflag; + /* txn-> and env-> are the same in read txns, use + * tmp variable to avoid undefined assignment + */ + seq = ++txn->mt_env->me_dbiseqs[slot]; + txn->mt_dbiseqs[slot] = seq; + + memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); + *dbi = slot; + mdb_default_cmp(txn, slot); + if (!unused) { + txn->mt_numdbs++; + } + } + + return rc; +} + +int ESECT +mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) +{ + if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + if (txn->mt_dbflags[dbi] & DB_STALE) { + MDB_cursor mc; + MDB_xcursor mx; + /* Stale, must read the DB's root. cursor_init does it for us. */ + mdb_cursor_init(&mc, txn, dbi, &mx); + } + return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); +} + +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) +{ + char *ptr; + if (dbi < CORE_DBS || dbi >= env->me_maxdbs) + return; + ptr = env->me_dbxs[dbi].md_name.mv_data; + /* If there was no name, this was already closed */ + if (ptr) { + env->me_dbxs[dbi].md_name.mv_data = NULL; + env->me_dbxs[dbi].md_name.mv_size = 0; + env->me_dbflags[dbi] = 0; + env->me_dbiseqs[dbi]++; + free(ptr); + } +} + +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) +{ + /* We could return the flags for the FREE_DBI too but what's the point? */ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; + return MDB_SUCCESS; +} + +/** Add all the DB's pages to the free list. + * @param[in] mc Cursor on the DB to free. + * @param[in] subs non-Zero to check for sub-DBs in this DB. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_drop0(MDB_cursor *mc, int subs) +{ + int rc; + + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc == MDB_SUCCESS) { + MDB_txn *txn = mc->mc_txn; + MDB_node *ni; + MDB_cursor mx; + unsigned int i; + + /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. + * This also avoids any P_LEAF2 pages, which have no nodes. + * Also if the DB doesn't have sub-DBs and has no overflow + * pages, omit scanning leaves. + */ + if ((mc->mc_flags & C_SUB) || + (!subs && !mc->mc_db->md_overflow_pages)) + mdb_cursor_pop(mc); + + mdb_cursor_copy(mc, &mx); + while (mc->mc_snum > 0) { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + unsigned n = NUMKEYS(mp); + if (IS_LEAF(mp)) { + for (i=0; imn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(mc, pg, &omp, NULL); + if (rc != 0) + goto done; + mdb_cassert(mc, IS_OVERFLOW(omp)); + rc = mdb_midl_append_range(&txn->mt_free_pgs, + pg, omp->mp_pages); + if (rc) + goto done; + mc->mc_db->md_overflow_pages -= omp->mp_pages; + if (!mc->mc_db->md_overflow_pages && !subs) + break; + } else if (subs && (ni->mn_flags & F_SUBDATA)) { + mdb_xcursor_init1(mc, ni); + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto done; + } + } + if (!subs && !mc->mc_db->md_overflow_pages) + goto pop; + } else { + if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) + goto done; + for (i=0; imt_free_pgs, pg); + } + } + if (!mc->mc_top) + break; + mc->mc_ki[mc->mc_top] = i; + rc = mdb_cursor_sibling(mc, 1); + if (rc) { + if (rc != MDB_NOTFOUND) + goto done; + /* no more siblings, go back to beginning + * of previous level. + */ +pop: + mdb_cursor_pop(mc); + mc->mc_ki[0] = 0; + for (i=1; imc_snum; i++) { + mc->mc_ki[i] = 0; + mc->mc_pg[i] = mx.mc_pg[i]; + } + } + } + /* free it */ + rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); +done: + if (rc) + txn->mt_flags |= MDB_TXN_ERROR; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + mc->mc_flags &= ~C_INITIALIZED; + return rc; +} + +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) +{ + MDB_cursor *mc, *m2; + int rc; + + if ((unsigned)del > 1 || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + + if (TXN_DBI_CHANGED(txn, dbi)) + return MDB_BAD_DBI; + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) + return rc; + + rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); + /* Invalidate the dropped DB's cursors */ + for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) + m2->mc_flags &= ~(C_INITIALIZED|C_EOF); + if (rc) + goto leave; + + /* Can't delete the main DB */ + if (del && dbi >= CORE_DBS) { + rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA); + if (!rc) { + txn->mt_dbflags[dbi] = DB_STALE; + mdb_dbi_close(txn->mt_env, dbi); + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + } else { + /* reset the DB record, mark it dirty */ + txn->mt_dbflags[dbi] |= DB_DIRTY; + txn->mt_dbs[dbi].md_depth = 0; + txn->mt_dbs[dbi].md_branch_pages = 0; + txn->mt_dbs[dbi].md_leaf_pages = 0; + txn->mt_dbs[dbi].md_overflow_pages = 0; + txn->mt_dbs[dbi].md_entries = 0; + txn->mt_dbs[dbi].md_root = P_INVALID; + + txn->mt_flags |= MDB_TXN_DIRTY; + } +leave: + mdb_cursor_close(mc); + return rc; +} + +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_cmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_dcmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_rel = rel; + return MDB_SUCCESS; +} + +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_relctx = ctx; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxkeysize(MDB_env *env) +{ + return ENV_MAXKEY(env); +} + +int ESECT +mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) +{ + unsigned int i, rdrs; + MDB_reader *mr; + char buf[64]; + int rc = 0, first = 1; + + if (!env || !func) + return -1; + if (!env->me_txns) { + return func("(no reader locks)\n", ctx); + } + rdrs = env->me_txns->mti_numreaders; + mr = env->me_txns->mti_readers; + for (i=0; i> 1; + cursor = base + pivot + 1; + val = pid - ids[cursor]; + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + /* found, so it's a duplicate */ + return -1; + } + } + + if( val > 0 ) { + ++cursor; + } + ids[0]++; + for (n = ids[0]; n > cursor; n--) + ids[n] = ids[n-1]; + ids[n] = pid; + return 0; +} + +int ESECT +mdb_reader_check(MDB_env *env, int *dead) +{ + if (!env) + return EINVAL; + if (dead) + *dead = 0; + return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; +} + +/** As #mdb_reader_check(). \b rlocked is set if caller locked #me_rmutex. */ +static int ESECT +mdb_reader_check0(MDB_env *env, int rlocked, int *dead) +{ + mdb_mutexref_t rmutex = rlocked ? NULL : env->me_rmutex; + unsigned int i, j, rdrs; + MDB_reader *mr; + MDB_PID_T *pids, pid; + int rc = MDB_SUCCESS, count = 0; + + rdrs = env->me_txns->mti_numreaders; + pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); + if (!pids) + return ENOMEM; + pids[0] = 0; + mr = env->me_txns->mti_readers; + for (i=0; ime_pid) { + if (mdb_pid_insert(pids, pid) == 0) { + if (!mdb_reader_pid(env, Pidcheck, pid)) { + /* Stale reader found */ + j = i; + if (rmutex) { + if ((rc = LOCK_MUTEX0(rmutex)) != 0) { + if ((rc = mdb_mutex_failed(env, rmutex, rc))) + break; + rdrs = 0; /* the above checked all readers */ + } else { + /* Recheck, a new process may have reused pid */ + if (mdb_reader_pid(env, Pidcheck, pid)) + j = rdrs; + } + } + for (; jme_rmutex); + if (!rlocked) { + /* Keep mti_txnid updated, otherwise next writer can + * overwrite data which latest meta page refers to. + */ + meta = mdb_env_pick_meta(env); + env->me_txns->mti_txnid = meta->mm_txnid; + /* env is hosed if the dead thread was ours */ + if (env->me_txn) { + env->me_flags |= MDB_FATAL_ERROR; + env->me_txn = NULL; + rc = MDB_PANIC; + } + } + DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'), + (rc ? "this process' env is hosed" : "recovering"))); + rc2 = mdb_reader_check0(env, rlocked, NULL); + if (rc2 == 0) + rc2 = mdb_mutex_consistent(mutex); + if (rc || (rc = rc2)) { + DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc))); + UNLOCK_MUTEX(mutex); + } + } else { +#ifdef _WIN32 + rc = ErrCode(); +#endif + DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc))); + } + + return rc; +} +#endif /* MDB_ROBUST_SUPPORTED */ + +#if defined(_WIN32) +/** Convert \b src to new wchar_t[] string with room for \b xtra extra chars */ +static int ESECT +utf8_to_utf16(const char *src, MDB_name *dst, int xtra) +{ + int rc, need = 0; + wchar_t *result = NULL; + for (;;) { /* malloc result, then fill it in */ + need = MultiByteToWideChar(CP_UTF8, 0, src, -1, result, need); + if (!need) { + rc = ErrCode(); + free(result); + return rc; + } + if (!result) { + result = malloc(sizeof(wchar_t) * (need + xtra)); + if (!result) + return ENOMEM; + continue; + } + dst->mn_alloced = 1; + dst->mn_len = need - 1; + dst->mn_val = result; + return MDB_SUCCESS; + } +} +#endif /* defined(_WIN32) */ +/** @} */ diff --git a/nostrdb/memchr.h b/nostrdb/memchr.h new file mode 100644 index 0000000000..9c7ae74318 --- /dev/null +++ b/nostrdb/memchr.h @@ -0,0 +1,72 @@ + + +#ifndef FAST_MEMCHR_H +#define FAST_MEMCHR_H + +#include + +#ifdef __ARM_NEON +#define vector_strchr neon_strchr +#else +#define vector_strchr native_memchr +#endif + +#ifdef __ARM_NEON +#include +static const char *neon_strchr(const char *str, char c, size_t length) { + const char* end = str + length; + + // Alignment handling + while (str < end && ((size_t)str & 0xF)) { + if (*str == c) + return str; + ++str; + } + + uint8x16_t searchChar = vdupq_n_u8(c); + + while (str + 16 <= end) { + uint8x16_t chunk = vld1q_u8((const uint8_t*)str); + uint8x16_t comparison = vceqq_u8(chunk, searchChar); + + // Check first 64 bits + uint64_t result0 = + vgetq_lane_u64(vreinterpretq_u64_u8(comparison), 0); + + if (result0) + return str + __builtin_ctzll(result0)/8; + + // Check second 64 bits + uint64_t result1 = vgetq_lane_u64(vreinterpretq_u64_u8(comparison), 1); + if (result1) + return str + 8 + __builtin_ctzll(result1)/8; + + str += 16; + } + + // Handle remaining unaligned characters + for (; str < end; ++str) { + if (*str == c) + return str; + } + + return NULL; +} +#endif + +static inline const char *native_memchr(const char *str, char c, size_t length) { + const void *result = memchr(str, c, length); + return (const char *) result; +} + +static inline const char *fast_strchr(const char *str, char c, size_t length) +{ + if (length >= 16) { + return vector_strchr(str, c, length); + } + + return native_memchr(str, c, length); +} + + +#endif // FAST_MEMCHR_H diff --git a/nostrdb/midl.c b/nostrdb/midl.c new file mode 100644 index 0000000000..b0ea5383b2 --- /dev/null +++ b/nostrdb/midl.c @@ -0,0 +1,359 @@ +/** @file midl.c + * @brief ldap bdb back-end ID List functions */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2021 The OpenLDAP Foundation. + * Portions Copyright 2001-2021 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include +#include +#include +#include +#include +#include "midl.h" + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup idls ID List Management + * @{ + */ +#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) + +unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +#if 0 /* superseded by append/sort */ +int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) +{ + unsigned x, i; + + x = mdb_midl_search( ids, id ); + assert( x > 0 ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0] && ids[x] == id ) { + /* duplicate */ + assert(0); + return -1; + } + + if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + /* no room */ + --ids[0]; + return -2; + + } else { + /* insert id */ + for (i=ids[0]; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = id; + } + + return 0; +} +#endif + +MDB_IDL mdb_midl_alloc(int num) +{ + MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); + if (ids) { + *ids++ = num; + *ids = 0; + } + return ids; +} + +void mdb_midl_free(MDB_IDL ids) +{ + if (ids) + free(ids-1); +} + +void mdb_midl_shrink( MDB_IDL *idp ) +{ + MDB_IDL ids = *idp; + if (*(--ids) > MDB_IDL_UM_MAX && + (ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID)))) + { + *ids++ = MDB_IDL_UM_MAX; + *idp = ids; + } +} + +static int mdb_midl_grow( MDB_IDL *idp, int num ) +{ + MDB_IDL idn = *idp-1; + /* grow it */ + idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); + if (!idn) + return ENOMEM; + *idn++ += num; + *idp = idn; + return 0; +} + +int mdb_midl_need( MDB_IDL *idp, unsigned num ) +{ + MDB_IDL ids = *idp; + num += ids[0]; + if (num > ids[-1]) { + num = (num + num/4 + (256 + 2)) & -256; + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) + return ENOMEM; + *ids++ = num - 2; + *idp = ids; + } + return 0; +} + +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] >= ids[-1]) { + if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0]++; + ids[ids[0]] = id; + return 0; +} + +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] + app[0] >= ids[-1]) { + if (mdb_midl_grow(idp, app[0])) + return ENOMEM; + ids = *idp; + } + memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); + ids[0] += app[0]; + return 0; +} + +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) +{ + MDB_ID *ids = *idp, len = ids[0]; + /* Too big? */ + if (len + n > ids[-1]) { + if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0] = len + n; + ids += len; + while (n) + ids[n--] = id++; + return 0; +} + +void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) +{ + MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; + idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ + old_id = idl[j]; + while (i) { + merge_id = merge[i--]; + for (; old_id < merge_id; old_id = idl[--j]) + idl[k--] = old_id; + idl[k--] = merge_id; + } + idl[0] = total; +} + +/* Quicksort + Insertion sort for small arrays */ + +#define SMALL 8 +#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } + +void +mdb_midl_sort( MDB_IDL ids ) +{ + /* Max possible depth of int-indexed tree * 2 items/level */ + int istack[sizeof(int)*CHAR_BIT * 2]; + int i,j,k,l,ir,jstack; + MDB_ID a, itmp; + + ir = (int)ids[0]; + l = 1; + jstack = 0; + for(;;) { + if (ir - l < SMALL) { /* Insertion sort */ + for (j=l+1;j<=ir;j++) { + a = ids[j]; + for (i=j-1;i>=1;i--) { + if (ids[i] >= a) break; + ids[i+1] = ids[i]; + } + ids[i+1] = a; + } + if (jstack == 0) break; + ir = istack[jstack--]; + l = istack[jstack--]; + } else { + k = (l + ir) >> 1; /* Choose median of left, center, right */ + MIDL_SWAP(ids[k], ids[l+1]); + if (ids[l] < ids[ir]) { + MIDL_SWAP(ids[l], ids[ir]); + } + if (ids[l+1] < ids[ir]) { + MIDL_SWAP(ids[l+1], ids[ir]); + } + if (ids[l] < ids[l+1]) { + MIDL_SWAP(ids[l], ids[l+1]); + } + i = l+1; + j = ir; + a = ids[l+1]; + for(;;) { + do i++; while(ids[i] > a); + do j--; while(ids[j] < a); + if (j < i) break; + MIDL_SWAP(ids[i],ids[j]); + } + ids[l+1] = ids[j]; + ids[j] = a; + jstack += 2; + if (ir-i+1 >= j-l) { + istack[jstack] = ir; + istack[jstack-1] = i; + ir = j-1; + } else { + istack[jstack] = j-1; + istack[jstack-1] = l; + l = i; + } + } + } +} + +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = (unsigned)ids[0].mid; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( id, ids[cursor].mid ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) +{ + unsigned x, i; + + x = mdb_mid2l_search( ids, id->mid ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0].mid && ids[x].mid == id->mid ) { + /* duplicate */ + return -1; + } + + if ( ids[0].mid >= MDB_IDL_UM_MAX ) { + /* too big */ + return -2; + + } else { + /* insert id */ + ids[0].mid++; + for (i=(unsigned)ids[0].mid; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = *id; + } + + return 0; +} + +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) +{ + /* Too big? */ + if (ids[0].mid >= MDB_IDL_UM_MAX) { + return -2; + } + ids[0].mid++; + ids[ids[0].mid] = *id; + return 0; +} + +/** @} */ +/** @} */ diff --git a/nostrdb/midl.h b/nostrdb/midl.h new file mode 100644 index 0000000000..dd6ae77c37 --- /dev/null +++ b/nostrdb/midl.h @@ -0,0 +1,186 @@ +/** @file midl.h + * @brief LMDB ID List header file. + * + * This file was originally part of back-bdb but has been + * modified for use in libmdb. Most of the macros defined + * in this file are unused, just left over from the original. + * + * This file is only used internally in libmdb and its definitions + * are not exposed publicly. + */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2021 The OpenLDAP Foundation. + * Portions Copyright 2001-2021 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _MDB_MIDL_H_ +#define _MDB_MIDL_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ + +/** @defgroup idls ID List Management + * @{ + */ + /** A generic unsigned ID number. These were entryIDs in back-bdb. + * Preferably it should have the same size as a pointer. + */ +typedef size_t MDB_ID; + + /** An IDL is an ID List, a sorted array of IDs. The first + * element of the array is a counter for how many actual + * IDs are in the list. In the original back-bdb code, IDLs are + * sorted in ascending order. For libmdb IDLs are sorted in + * descending order. + */ +typedef MDB_ID *MDB_IDL; + +/* IDL sizes - likely should be even bigger + * limiting factors: sizeof(ID), thread stack size + */ +#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ +#define MDB_IDL_DB_SIZE (1< #include +#include +#include "bindings/c/profile_json_parser.h" #include "secp256k1.h" #include "secp256k1_ecdh.h" #include "secp256k1_schnorrsig.h" +// the maximum number of things threads pop and push in bulk +static const int THREAD_QUEUE_BATCH = 4096; + +// the maximum size of inbox queues +static const int DEFAULT_QUEUE_SIZE = 1000000; + + +#define NDB_PARSED_ID (1 << 0) +#define NDB_PARSED_PUBKEY (1 << 1) +#define NDB_PARSED_SIG (1 << 2) +#define NDB_PARSED_CREATED_AT (1 << 3) +#define NDB_PARSED_KIND (1 << 4) +#define NDB_PARSED_CONTENT (1 << 5) +#define NDB_PARSED_TAGS (1 << 6) +#define NDB_PARSED_ALL (NDB_PARSED_ID|NDB_PARSED_PUBKEY|NDB_PARSED_SIG|NDB_PARSED_CREATED_AT|NDB_PARSED_KIND|NDB_PARSED_CONTENT|NDB_PARSED_TAGS) + + +// controls whether to continue or stop the json parser +enum ndb_idres { + NDB_IDRES_CONT, + NDB_IDRES_STOP, +}; + +// closure data for the id-detecting ingest controller +struct ndb_ingest_controller +{ + MDB_txn *read_txn; + struct ndb_lmdb *lmdb; +}; + +enum ndb_dbs { + NDB_DB_NOTE, + NDB_DB_META, + NDB_DB_PROFILE, + NDB_DB_NOTE_ID, + NDB_DB_PROFILE_PK, + NDB_DBS, +}; + struct ndb_json_parser { const char *json; int json_len; @@ -22,6 +68,895 @@ struct ndb_json_parser { int num_tokens; }; +// useful to pass to threads on its own +struct ndb_lmdb { + MDB_env *env; + MDB_dbi dbs[NDB_DBS]; +}; + +struct ndb_writer { + struct ndb_lmdb *lmdb; + + void *queue_buf; + int queue_buflen; + pthread_t thread_id; + + struct prot_queue inbox; +}; + +struct ndb_ingester { + struct threadpool tp; + struct ndb_writer *writer; +}; + + +struct ndb { + struct ndb_lmdb lmdb; + struct ndb_ingester ingester; + struct ndb_writer writer; + // lmdb environ handles, etc +}; + +// A clustered key with an id and a timestamp +struct ndb_tsid { + unsigned char id[32]; + uint64_t timestamp; +}; + +/** From LMDB: Compare two items lexically */ +static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) { + int diff; + ssize_t len_diff; + unsigned int len; + + len = a->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + len = b->mv_size; + len_diff = 1; + } + + diff = memcmp(a->mv_data, b->mv_data, len); + return diff ? diff : len_diff<0 ? -1 : len_diff; +} + +static int ndb_tsid_compare(const MDB_val *a, const MDB_val *b) +{ + struct ndb_tsid *tsa, *tsb; + MDB_val a2 = *a, b2 = *b; + a2.mv_size = sizeof(tsa->id); + b2.mv_size = sizeof(tsb->id); + + int cmp = mdb_cmp_memn(&a2, &b2); + if (cmp) return cmp; + + tsa = a->mv_data; + tsb = b->mv_data; + + if (tsa->timestamp < tsb->timestamp) + return -1; + else if (tsa->timestamp > tsb->timestamp) + return 1; + return 0; +} + +static inline void ndb_tsid_low(struct ndb_tsid *key, unsigned char *id) +{ + memcpy(key->id, id, 32); + key->timestamp = 0; +} + +static inline void ndb_tsid_init(struct ndb_tsid *key, unsigned char *id, + uint64_t timestamp) +{ + memcpy(key->id, id, 32); + key->timestamp = 0; +} + +// useful for range-searching for the latest key with a clustered created_at timen +static inline void ndb_tsid_high(struct ndb_tsid *key, const unsigned char *id) +{ + memcpy(key->id, id, 32); + key->timestamp = UINT64_MAX; +} + +enum ndb_ingester_msgtype { + NDB_INGEST_EVENT, // write json to the ingester queue for processing + NDB_INGEST_QUIT, // kill ingester thread immediately +}; + +enum ndb_writer_msgtype { + NDB_WRITER_QUIT, // kill thread immediately + NDB_WRITER_NOTE, // write a note to the db + NDB_WRITER_PROFILE, // write a profile to the db +}; + +struct ndb_ingester_event { + char *json; + int len; +}; + +struct ndb_writer_note { + struct ndb_note *note; + size_t note_len; +}; + +struct ndb_writer_profile { + struct ndb_writer_note note; + void *profile_flatbuf; + size_t profile_len; +}; + +struct ndb_ingester_msg { + enum ndb_ingester_msgtype type; + union { + struct ndb_ingester_event event; + }; +}; + +struct ndb_writer_msg { + enum ndb_writer_msgtype type; + union { + struct ndb_writer_note note; + struct ndb_writer_profile profile; + }; +}; + +int ndb_note_verify(void *ctx, unsigned char pubkey[32], unsigned char id[32], + unsigned char sig[64]) +{ + secp256k1_xonly_pubkey xonly_pubkey; + int ok; + + ok = secp256k1_xonly_pubkey_parse((secp256k1_context*)ctx, &xonly_pubkey, + pubkey) != 0; + if (!ok) return 0; + + ok = secp256k1_schnorrsig_verify((secp256k1_context*)ctx, sig, id, 32, + &xonly_pubkey) > 0; + if (!ok) return 0; + + return 1; +} + +static inline int ndb_writer_queue_msgs(struct ndb_writer *writer, + struct ndb_writer_msg *msgs, + int num_msgs) +{ + return prot_queue_push_all(&writer->inbox, msgs, num_msgs); +} + +static int ndb_writer_queue_note(struct ndb_writer *writer, + struct ndb_note *note, size_t note_len) +{ + struct ndb_writer_msg msg; + msg.type = NDB_WRITER_NOTE; + + msg.note.note = note; + msg.note.note_len = note_len; + + return prot_queue_push(&writer->inbox, &msg); +} + +// get some value based on a clustered id key +int ndb_get_tsid(MDB_txn *txn, struct ndb_lmdb *lmdb, enum ndb_dbs db, + const unsigned char *id, MDB_val *val) +{ + MDB_val k, v; + MDB_cursor *cur; + struct ndb_tsid tsid; + int success = 0; + + ndb_tsid_high(&tsid, id); + k.mv_data = &tsid; + k.mv_size = sizeof(tsid); + + mdb_cursor_open(txn, lmdb->dbs[db], &cur); + + // Position cursor at the next key greater than or equal to the specified key + if (mdb_cursor_get(cur, &k, &v, MDB_SET_RANGE)) { + // Failed :(. It could be the last element? + if (mdb_cursor_get(cur, &k, &v, MDB_LAST)) + goto cleanup; + } else { + // if set range worked and our key exists, it should be + // the one right before this one + if (mdb_cursor_get(cur, &k, &v, MDB_PREV)) + goto cleanup; + } + + if (memcmp(k.mv_data, id, 32) == 0) { + *val = v; + success = 1; + } + +cleanup: + mdb_cursor_close(cur); + return success; +} + +struct ndb_note *ndb_get_note_by_id(struct ndb *ndb, const unsigned char *id) +{ + MDB_val k, v; + MDB_txn *txn; + + if (mdb_txn_begin(ndb->lmdb.env, 0, 0, &txn)) { + ndb_debug("ndb_get_note_by_id: mdb_txn_begin failed\n"); + return NULL; + } + + if (!ndb_get_tsid(txn, &ndb->lmdb, NDB_DB_NOTE_ID, id, &k)) { + ndb_debug("ndb_get_note_by_id: ndb_get_tsid failed\n"); + return NULL; + } + + if (mdb_get(txn, ndb->lmdb.dbs[NDB_DB_NOTE], &k, &v)) { + ndb_debug("ndb_get_note_by_id: mdb_get note failed\n"); + return NULL; + } + + mdb_txn_abort(txn); + + return (struct ndb_note *)v.mv_data; +} + +static int ndb_has_note(MDB_txn *txn, struct ndb_lmdb *lmdb, const unsigned char *id) +{ + MDB_val val; + + if (!ndb_get_tsid(txn, lmdb, NDB_DB_NOTE_ID, id, &val)) + return 0; + + return 1; +} + +static enum ndb_idres ndb_ingester_json_controller(void *data, const char *hexid) +{ + unsigned char id[32]; + struct ndb_ingest_controller *c = data; + + hex_decode(hexid, 64, id, sizeof(id)); + + // let's see if we already have it + + if (!ndb_has_note(c->read_txn, c->lmdb, id)) + return NDB_IDRES_CONT; + + return NDB_IDRES_STOP; +} + + +static int ndb_process_profile_note(struct ndb_note *note, void **profile, + size_t *profile_len) +{ + int res; + + flatcc_builder_t builder; + flatcc_json_parser_t json_parser; + + flatcc_builder_init(&builder); + + //printf("parsing profile '%.*s'\n", note->content_length, ndb_note_content(note)); + res = profile_parse_json(&builder, &json_parser, + ndb_note_content(note), + note->content_length, + flatcc_json_parser_f_skip_unknown); + + if (res != 0) { + ndb_debug("profile_parse_json failed %d '%.*s'\n", res, + note->content_length, ndb_note_content(note)); + return 0; + } + + *profile = flatcc_builder_finalize_aligned_buffer(&builder, profile_len); + return 1; +} + + +static int ndb_ingester_process_event(secp256k1_context *ctx, + struct ndb_ingester *ingester, + struct ndb_ingester_event *ev, + struct ndb_writer_msg *out, + MDB_txn *read_txn + ) +{ + struct ndb_tce tce; + struct ndb_note *note; + struct ndb_ingest_controller controller; + struct ndb_id_cb cb; + void *buf, *flatbuf; + size_t bufsize, note_size, profile_len; + + // we will use this to check if we already have it in the DB during + // ID parsing + controller.read_txn = read_txn; + controller.lmdb = ingester->writer->lmdb; + cb.fn = ndb_ingester_json_controller; + cb.data = &controller; + + // since we're going to be passing this allocated note to a different + // thread, we can't use thread-local buffers. just allocate a block + bufsize = max(ev->len * 8.0, 4096); + buf = malloc(bufsize); + if (!buf) { + ndb_debug("couldn't malloc buf\n"); + return 0; + } + + note_size = + ndb_ws_event_from_json(ev->json, ev->len, &tce, buf, bufsize, &cb); + + if (note_size == -42) { + // we already have this! + //ndb_debug("already have id??\n"); + goto cleanup; + } else if (note_size == 0) { + ndb_debug("failed to parse '%.*s'\n", ev->len, ev->json); + goto cleanup; + } + + //ndb_debug("parsed evtype:%d '%.*s'\n", tce.evtype, ev->len, ev->json); + + switch (tce.evtype) { + case NDB_TCE_NOTICE: goto cleanup; + case NDB_TCE_EOSE: goto cleanup; + case NDB_TCE_OK: goto cleanup; + case NDB_TCE_EVENT: + note = tce.event.note; + if (note != buf) { + ndb_debug("note buffer not equal to malloc'd buffer\n"); + goto cleanup; + } + + // Verify! If it's an invalid note we don't need to + // bothter writing it to the database + if (!ndb_note_verify(ctx, note->pubkey, note->id, note->sig)) { + ndb_debug("signature verification failed\n"); + goto cleanup; + } + + // we didn't find anything. let's send it + // to the writer thread + note = realloc(note, note_size); + + if (note->kind == 0 && + ndb_process_profile_note(note, &flatbuf, &profile_len)) { + out->type = NDB_WRITER_PROFILE; + out->profile.note.note = note; + out->profile.note.note_len = note_size; + out->profile.profile_flatbuf = flatbuf; + out->profile.profile_len = profile_len; + } else { + out->type = NDB_WRITER_NOTE; + out->note.note = note; + out->note.note_len = note_size; + } + + // there's nothing left to do with the original json, so free it + free(ev->json); + return 1; + } + +cleanup: + free(ev->json); + free(buf); + + return 0; +} + +static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db) +{ + MDB_cursor *mc; + MDB_val key, val; + + if (mdb_cursor_open(txn, db, &mc)) + return 0; + + if (mdb_cursor_get(mc, &key, &val, MDB_LAST)) { + mdb_cursor_close(mc); + return 0; + } + + mdb_cursor_close(mc); + + assert(key.mv_size == 8); + return *((uint64_t*)key.mv_data); +} + +static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, + struct ndb_writer_profile *profile) +{ + uint64_t profile_key; + struct ndb_tsid tsid; + struct ndb_note *note; + int rc; + + MDB_val key, val; + MDB_dbi profile_db, pk_db; + + note = profile->note.note; + + // get dbs + profile_db = lmdb->dbs[NDB_DB_PROFILE]; + pk_db = lmdb->dbs[NDB_DB_PROFILE_PK]; + + // get new key + profile_key = ndb_get_last_key(txn, profile_db) + 1; + + // write profile to profile store + key.mv_data = &profile_key; + key.mv_size = sizeof(profile_key); + val.mv_data = profile->profile_flatbuf; + val.mv_size = profile->profile_len; + //ndb_debug("profile_len %ld\n", profile->profile_len); + + if ((rc = mdb_put(txn, profile_db, &key, &val, 0))) { + ndb_debug("write profile to db failed: %s\n", mdb_strerror(rc)); + return 0; + } + + // write profile_pk + created_at index + ndb_tsid_init(&tsid, note->pubkey, note->created_at); + + key.mv_data = &tsid; + key.mv_size = sizeof(tsid); + val.mv_data = &profile_key; + val.mv_size = sizeof(profile_key); + + if ((rc = mdb_put(txn, pk_db, &key, &val, 0))) { + ndb_debug("write profile_pk(%" PRIu64 ") to db failed: %s\n", + profile_key, mdb_strerror(rc)); + return 0; + } + + return 1; +} + +static uint64_t ndb_write_note(struct ndb_lmdb *lmdb, MDB_txn *txn, + struct ndb_writer_note *note) +{ + int rc; + uint64_t note_key; + struct ndb_tsid tsid; + MDB_dbi note_db, id_db; + MDB_val key, val; + + // get dbs + note_db = lmdb->dbs[NDB_DB_NOTE]; + id_db = lmdb->dbs[NDB_DB_NOTE_ID]; + + // get new key + note_key = ndb_get_last_key(txn, note_db) + 1; + + // write note to event store + key.mv_data = ¬e_key; + key.mv_size = sizeof(note_key); + val.mv_data = note->note; + val.mv_size = note->note_len; + + if ((rc = mdb_put(txn, note_db, &key, &val, 0))) { + ndb_debug("write note to db failed: %s\n", mdb_strerror(rc)); + return 0; + } + + // write id index key clustered with created_at + ndb_tsid_init(&tsid, note->note->id, note->note->created_at); + + key.mv_data = &tsid; + key.mv_size = sizeof(tsid); + val.mv_data = ¬e_key; + val.mv_size = sizeof(note_key); + + if ((rc = mdb_put(txn, id_db, &key, &val, 0))) { + ndb_debug("write note id index to db failed: %s\n", + mdb_strerror(rc)); + return 0; + } + + return note_key; +} + +static void *ndb_writer_thread(void *data) +{ + struct ndb_writer *writer = data; + struct ndb_writer_msg msgs[THREAD_QUEUE_BATCH], *msg; + int i, popped, done, any_note; + MDB_txn *txn; + + done = 0; + while (!done) { + txn = NULL; + popped = prot_queue_pop_all(&writer->inbox, msgs, THREAD_QUEUE_BATCH); + ndb_debug("writer popped %d items\n", popped); + + any_note = 0; + for (i = 0 ; i < popped; i++) { + msg = &msgs[i]; + switch (msg->type) { + case NDB_WRITER_NOTE: any_note = 1; break; + case NDB_WRITER_PROFILE: any_note = 1; break; + case NDB_WRITER_QUIT: break; + } + } + + if (any_note && mdb_txn_begin(writer->lmdb->env, NULL, 0, &txn)) + { + fprintf(stderr, "writer thread txn_begin failed"); + // should definitely not happen unless DB is full + // or something ? + assert(false); + } + + for (i = 0; i < popped; i++) { + msg = &msgs[i]; + + switch (msg->type) { + case NDB_WRITER_QUIT: + // quits are handled before this + done = 1; + continue; + case NDB_WRITER_PROFILE: + ndb_write_note(writer->lmdb, txn, &msg->note); + // TODO: save note_key with profile + ndb_write_profile(writer->lmdb, txn, &msg->profile); + break; + case NDB_WRITER_NOTE: + ndb_write_note(writer->lmdb, txn, &msg->note); + break; + } + } + + // commit writes + if (any_note && mdb_txn_commit(txn)) { + fprintf(stderr, "writer thread txn commit failed"); + assert(false); + } + + + // free notes + for (i = 0; i < popped; i++) { + msg = &msgs[i]; + if (msg->type == NDB_WRITER_NOTE) + free(msg->note.note); + else if (msg->type == NDB_WRITER_PROFILE) { + free(msg->profile.profile_flatbuf); + free(msg->profile.note.note); + } + } + } + + ndb_debug("quitting writer thread\n"); + return NULL; +} + +static void *ndb_ingester_thread(void *data) +{ + secp256k1_context *ctx; + struct thread *thread = data; + struct ndb_ingester *ingester = (struct ndb_ingester *)thread->ctx; + struct ndb_lmdb *lmdb = ingester->writer->lmdb; + struct ndb_ingester_msg msgs[THREAD_QUEUE_BATCH], *msg; + struct ndb_writer_msg outs[THREAD_QUEUE_BATCH], *out; + int i, to_write, popped, done, any_event; + MDB_txn *read_txn = NULL; + + ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + ndb_debug("started ingester thread\n"); + + done = 0; + while (!done) { + to_write = 0; + any_event = 0; + + popped = prot_queue_pop_all(&thread->inbox, msgs, THREAD_QUEUE_BATCH); + ndb_debug("ingester popped %d items\n", popped); + + for (i = 0; i < popped; i++) { + msg = &msgs[i]; + if (msg->type == NDB_INGEST_EVENT) { + any_event = 1; + break; + } + } + + if (any_event) + mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &read_txn); + + for (i = 0; i < popped; i++) { + msg = &msgs[i]; + switch (msg->type) { + case NDB_INGEST_QUIT: + done = 1; + break; + + case NDB_INGEST_EVENT: + out = &outs[to_write]; + if (ndb_ingester_process_event(ctx, ingester, + &msg->event, out, + read_txn)) { + to_write++; + } + } + } + + if (any_event) + mdb_txn_abort(read_txn); + + if (to_write > 0) { + //ndb_debug("pushing %d events to write queue\n", to_write); + if (!ndb_writer_queue_msgs(ingester->writer, outs, to_write)) { + ndb_debug("failed pushing %d events to write queue\n", to_write); + } + } + } + + ndb_debug("quitting ingester thread\n"); + secp256k1_context_destroy(ctx); + return NULL; +} + + +static int ndb_writer_init(struct ndb_writer *writer, struct ndb_lmdb *lmdb) +{ + writer->lmdb = lmdb; + writer->queue_buflen = sizeof(struct ndb_writer_msg) * DEFAULT_QUEUE_SIZE; + writer->queue_buf = malloc(writer->queue_buflen); + if (writer->queue_buf == NULL) { + fprintf(stderr, "ndb: failed to allocate space for writer queue"); + return 0; + } + + // init the writer queue. + prot_queue_init(&writer->inbox, writer->queue_buf, + writer->queue_buflen, sizeof(struct ndb_writer_msg)); + + // spin up the writer thread + if (pthread_create(&writer->thread_id, NULL, ndb_writer_thread, writer)) + { + fprintf(stderr, "ndb writer thread failed to create\n"); + return 0; + } + + return 1; +} + +// initialize the ingester queue and then spawn the thread +static int ndb_ingester_init(struct ndb_ingester *ingester, + struct ndb_writer *writer, int num_threads) +{ + int elem_size, num_elems; + static struct ndb_ingester_msg quit_msg = { .type = NDB_INGEST_QUIT }; + + // TODO: configurable queue sizes + elem_size = sizeof(struct ndb_ingester_msg); + num_elems = DEFAULT_QUEUE_SIZE; + + ingester->writer = writer; + + if (!threadpool_init(&ingester->tp, num_threads, elem_size, num_elems, + &quit_msg, ingester, ndb_ingester_thread)) + { + fprintf(stderr, "ndb ingester threadpool failed to init\n"); + return 0; + } + + return 1; +} + +static int ndb_writer_destroy(struct ndb_writer *writer) +{ + struct ndb_writer_msg msg; + + // kill thread + msg.type = NDB_WRITER_QUIT; + if (!prot_queue_push(&writer->inbox, &msg)) { + // queue is too full to push quit message. just kill it. + pthread_exit(&writer->thread_id); + } else { + pthread_join(writer->thread_id, NULL); + } + + // cleanup + prot_queue_destroy(&writer->inbox); + + free(writer->queue_buf); + + return 1; +} + +static int ndb_ingester_destroy(struct ndb_ingester *ingester) +{ + threadpool_destroy(&ingester->tp); + return 1; +} + +static int ndb_ingester_queue_event(struct ndb_ingester *ingester, + char *json, int len) +{ + struct ndb_ingester_msg msg; + msg.type = NDB_INGEST_EVENT; + + msg.event.json = json; + msg.event.len = len; + + return threadpool_dispatch(&ingester->tp, &msg); +} + +static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t mapsize) +{ + int rc; + MDB_txn *txn; + + if ((rc = mdb_env_create(&lmdb->env))) { + fprintf(stderr, "mdb_env_create failed, error %d\n", rc); + return 0; + } + + if ((rc = mdb_env_set_mapsize(lmdb->env, mapsize))) { + fprintf(stderr, "mdb_env_set_mapsize failed, error %d\n", rc); + return 0; + } + + if ((rc = mdb_env_set_maxdbs(lmdb->env, NDB_DBS))) { + fprintf(stderr, "mdb_env_set_mapsize failed, error %d\n", rc); + return 0; + } + + if ((rc = mdb_env_open(lmdb->env, filename, 0, 0664))) { + fprintf(stderr, "mdb_env_open failed, error %d\n", rc); + return 0; + } + + // Initialize DBs + if ((rc = mdb_txn_begin(lmdb->env, NULL, 0, &txn))) { + fprintf(stderr, "mdb_txn_begin failed, error %d\n", rc); + return 0; + } + + // note flatbuffer db + if ((rc = mdb_dbi_open(txn, "note", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NOTE]))) { + fprintf(stderr, "mdb_dbi_open event failed, error %d\n", rc); + return 0; + } + + // note metadata db + if ((rc = mdb_dbi_open(txn, "meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_META]))) { + fprintf(stderr, "mdb_dbi_open meta failed, error %d\n", rc); + return 0; + } + + // profile flatbuffer db + if ((rc = mdb_dbi_open(txn, "profile", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_PROFILE]))) { + fprintf(stderr, "mdb_dbi_open profile failed, error %d\n", rc); + return 0; + } + + // id+ts index flags + unsigned int tsid_flags = MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED; + + // index dbs + if ((rc = mdb_dbi_open(txn, "note_id", tsid_flags, &lmdb->dbs[NDB_DB_NOTE_ID]))) { + fprintf(stderr, "mdb_dbi_open id failed, error %d\n", rc); + return 0; + } + mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_ID], ndb_tsid_compare); + + if ((rc = mdb_dbi_open(txn, "profile_pk", tsid_flags, &lmdb->dbs[NDB_DB_PROFILE_PK]))) { + fprintf(stderr, "mdb_dbi_open id failed, error %d\n", rc); + return 0; + } + mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_PK], ndb_tsid_compare); + + + // Commit the transaction + if ((rc = mdb_txn_commit(txn))) { + fprintf(stderr, "mdb_txn_commit failed, error %d\n", rc); + return 0; + } + + return 1; +} + +int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingester_threads) +{ + struct ndb *ndb; + //MDB_dbi ind_id; // TODO: ind_pk, etc + + ndb = *pndb = calloc(1, sizeof(struct ndb)); + if (ndb == NULL) { + fprintf(stderr, "ndb_init: malloc failed\n"); + return 0; + } + + if (!ndb_init_lmdb(filename, &ndb->lmdb, mapsize)) + return 0; + + if (!ndb_writer_init(&ndb->writer, &ndb->lmdb)) { + fprintf(stderr, "ndb_writer_init failed"); + return 0; + } + + if (!ndb_ingester_init(&ndb->ingester, &ndb->writer, ingester_threads)) { + fprintf(stderr, "failed to initialize %d ingester thread(s)", + ingester_threads); + return 0; + } + + // Initialize LMDB environment and spin up threads + return 1; +} + +void ndb_destroy(struct ndb *ndb) +{ + if (ndb == NULL) + return; + + // ingester depends on writer and must be destroyed first + ndb_ingester_destroy(&ndb->ingester); + ndb_writer_destroy(&ndb->writer); + + mdb_env_close(ndb->lmdb.env); + + free(ndb); +} + +// Process a nostr event, ie: ["EVENT", "subid", {"content":"..."}...] +// +// This function returns as soon as possible, first copying the passed +// json and then queueing it up for processing. Worker threads then take +// the json and process it. +// +// Processing: +// +// 1. The event is parsed into ndb_notes and the signature is validated +// 2. A quick lookup is made on the database to see if we already have +// the note id, if we do we don't need to waste time on json parsing +// or note validation. +// 3. Once validation is done we pass it to the writer queue for writing +// to LMDB. +// +int ndb_process_event(struct ndb *ndb, const char *json, int json_len) +{ + // Since we need to return as soon as possible, and we're not + // making any assumptions about the lifetime of the string, we + // definitely need to copy the json here. In the future once we + // have our thread that manages a websocket connection, we can + // avoid the copy and just use the buffer we get from that + // thread. + char *json_copy = strdupn(json, json_len); + if (json_copy == NULL) + return 0; + + return ndb_ingester_queue_event(&ndb->ingester, json_copy, json_len); +} + +int ndb_process_events(struct ndb *ndb, const char *ldjson, size_t json_len) +{ + const char *start, *end, *very_end; + start = ldjson; + end = start + json_len; + very_end = ldjson + json_len; +#if DEBUG + int processed = 0; +#endif + + while ((end = fast_strchr(start, '\n', very_end - start))) { + //printf("processing '%.*s'\n", (int)(end-start), start); + if (!ndb_process_event(ndb, start, end - start)) { + ndb_debug("ndb_process_event failed\n"); + return 0; + } + start = end + 1; +#if DEBUG + processed++; +#endif + } + + ndb_debug("ndb_process_events: processed %d events\n", processed); + + return 1; +} + static inline int cursor_push_tag(struct cursor *cur, struct ndb_tag *tag) { return cursor_push_u16(cur, tag->count); @@ -97,15 +1032,34 @@ static inline int ndb_json_parser_init(struct ndb_json_parser *p, return 1; } -static inline int ndb_json_parser_parse(struct ndb_json_parser *p) +static inline int ndb_json_parser_parse(struct ndb_json_parser *p, + struct ndb_id_cb *cb) { + jsmntok_t *tok; int cap = ((unsigned char *)p->toks_end - (unsigned char*)p->toks)/sizeof(*p->toks); - p->num_tokens = - jsmn_parse(&p->json_parser, p->json, p->json_len, p->toks, cap); + int res = + jsmn_parse(&p->json_parser, p->json, p->json_len, p->toks, cap, cb != NULL); + + // got an ID! + if (res == -42) { + tok = &p->toks[p->json_parser.toknext-1]; + switch (cb->fn(cb->data, p->json + tok->start)) { + case NDB_IDRES_CONT: + res = jsmn_parse(&p->json_parser, p->json, p->json_len, + p->toks, cap, 0); + break; + case NDB_IDRES_STOP: + return -42; + } + } else if (res == 0) { + return 0; + } + + p->num_tokens = res; p->i = 0; - return p->num_tokens; + return 1; } static int cursor_push_unescaped_char(struct cursor *cur, char c1, char c2) @@ -117,7 +1071,6 @@ static int cursor_push_unescaped_char(struct cursor *cur, char c1, char c2) case 'b': return cursor_push_byte(cur, '\b'); case 'f': return cursor_push_byte(cur, '\f'); case '\\': return cursor_push_byte(cur, '\\'); - case '/': return cursor_push_byte(cur, '/'); case '"': return cursor_push_byte(cur, '"'); case 'u': // these aren't handled yet @@ -251,7 +1204,8 @@ static int ndb_event_commitment(struct ndb_note *ev, unsigned char *buf, int buf make_cursor(buf, buf + buflen, &cur); - snprintf(timebuf, sizeof(timebuf), "%d", ev->created_at); + // TODO: update in 2106 ... + snprintf(timebuf, sizeof(timebuf), "%d", (uint32_t)ev->created_at); snprintf(kindbuf, sizeof(kindbuf), "%d", ev->kind); ok = @@ -355,7 +1309,7 @@ int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note, unsigned char *end = builder->mem.end; unsigned char *start = (unsigned char*)(*note) + total_size; - ndb_builder_set_pubkey(builder, keypair->pubkey); + ndb_builder_set_pubkey(builder, keypair->pubkey); if (!ndb_calculate_id(builder->note, start, end - start)) return 0; @@ -654,7 +1608,8 @@ static int parse_unsigned_int(const char *start, int len, unsigned int *num) } int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, - unsigned char *buf, int bufsize) + unsigned char *buf, int bufsize, + struct ndb_id_cb *cb) { jsmntok_t *tok = NULL; int tok_len, res; @@ -664,11 +1619,22 @@ int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, tce->subid = ""; ndb_json_parser_init(&parser, json, len, buf, bufsize); - if ((res = ndb_json_parser_parse(&parser)) < 0) + if ((res = ndb_json_parser_parse(&parser, cb)) < 0) return res; - if (parser.num_tokens < 3 || parser.toks[0].type != JSMN_ARRAY) + if (parser.num_tokens < 3 || parser.toks[0].type != JSMN_ARRAY) { + /* + tok = &parser.toks[parser.json_parser.toknext-1]; + ndb_debug("failing at not enough takens (%d) or != JSMN_ARRAY @ '%.*s', '%.*s'\n", + parser.num_tokens, 10, json + parser.json_parser.pos, + toksize(tok), json + tok->start); + tok = &parser.toks[parser.json_parser.toknext-2]; + ndb_debug("failing at not enough takens (%d) or != JSMN_ARRAY @ '%.*s', '%.*s'\n", + parser.num_tokens, 10, json + parser.json_parser.pos, + toksize(tok), json + tok->start); + */ return 0; + } parser.i = 1; tok = &parser.toks[parser.i++]; @@ -738,7 +1704,9 @@ int ndb_parse_json_note(struct ndb_json_parser *parser, struct ndb_note **note) unsigned char hexbuf[64]; const char *json = parser->json; const char *start; - int i, tok_len; + int i, tok_len, parsed; + + parsed = 0; if (parser->toks[parser->i].type != JSMN_OBJECT) return 0; @@ -758,17 +1726,19 @@ int ndb_parse_json_note(struct ndb_json_parser *parser, struct ndb_note **note) // pubkey tok = &parser->toks[i+1]; hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf)); + parsed |= NDB_PARSED_PUBKEY; ndb_builder_set_pubkey(&parser->builder, hexbuf); } else if (tok_len == 2 && start[0] == 'i' && start[1] == 'd') { // id tok = &parser->toks[i+1]; hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf)); - // TODO: validate id + parsed |= NDB_PARSED_ID; ndb_builder_set_id(&parser->builder, hexbuf); } else if (tok_len == 3 && start[0] == 's' && start[1] == 'i' && start[2] == 'g') { // sig tok = &parser->toks[i+1]; hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf)); + parsed |= NDB_PARSED_SIG; ndb_builder_set_sig(&parser->builder, hexbuf); } else if (start[0] == 'k' && jsoneq(json, tok, tok_len, "kind")) { // kind @@ -779,6 +1749,7 @@ int ndb_parse_json_note(struct ndb_json_parser *parser, struct ndb_note **note) if (!parse_unsigned_int(start, toksize(tok), &parser->builder.note->kind)) return 0; + parsed |= NDB_PARSED_KIND; } else if (start[0] == 'c') { if (jsoneq(json, tok, tok_len, "created_at")) { // created_at @@ -786,9 +1757,12 @@ int ndb_parse_json_note(struct ndb_json_parser *parser, struct ndb_note **note) start = json + tok->start; if (tok->type != JSMN_PRIMITIVE || tok_len <= 0) return 0; - if (!parse_unsigned_int(start, toksize(tok), - &parser->builder.note->created_at)) + // TODO: update to int64 in 2106 ... xD + unsigned int bigi; + if (!parse_unsigned_int(start, toksize(tok), &bigi)) return 0; + parser->builder.note->created_at = bigi; + parsed |= NDB_PARSED_CREATED_AT; } else if (jsoneq(json, tok, tok_len, "content")) { // content tok = &parser->toks[i+1]; @@ -799,18 +1773,25 @@ int ndb_parse_json_note(struct ndb_json_parser *parser, struct ndb_note **note) json + tok->start, tok_len, &pstr, &written, pack_ids)) { + ndb_debug("ndb_builder_make_json_str failed\n"); return 0; } parser->builder.note->content_length = written; parser->builder.note->content = pstr; + parsed |= NDB_PARSED_CONTENT; } } else if (start[0] == 't' && jsoneq(json, tok, tok_len, "tags")) { tok = &parser->toks[i+1]; ndb_builder_process_json_tags(parser, tok); i += tok->size; + parsed |= NDB_PARSED_TAGS; } } + //ndb_debug("parsed %d = %d, &->%d", parsed, NDB_PARSED_ALL, parsed & NDB_PARSED_ALL); + if (parsed != NDB_PARSED_ALL) + return 0; + return ndb_builder_finalize(&parser->builder, note, NULL); } @@ -821,7 +1802,7 @@ int ndb_note_from_json(const char *json, int len, struct ndb_note **note, int res; ndb_json_parser_init(&parser, json, len, buf, bufsize); - if ((res = ndb_json_parser_parse(&parser)) < 0) + if ((res = ndb_json_parser_parse(&parser, NULL)) < 0) return res; if (parser.num_tokens < 1) @@ -850,7 +1831,7 @@ void ndb_builder_set_kind(struct ndb_builder *builder, uint32_t kind) builder->note->kind = kind; } -void ndb_builder_set_created_at(struct ndb_builder *builder, uint32_t created_at) +void ndb_builder_set_created_at(struct ndb_builder *builder, uint64_t created_at) { builder->note->created_at = created_at; } diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index 0239623ec2..51d27eef81 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -7,7 +7,20 @@ #define NDB_PACKED_STR 0x1 #define NDB_PACKED_ID 0x2 +//#define DEBUG 1 + +#ifdef DEBUG +#define ndb_debug(...) printf(__VA_ARGS__) +#else +#define ndb_debug(...) (void)0 +#endif + struct ndb_json_parser; +struct ndb; + +struct ndb_t { + struct ndb *ndb; +}; // To-client event types enum tce_type { @@ -17,6 +30,15 @@ enum tce_type { NDB_TCE_EOSE = 0x4, }; +// function pointer for controlling what to do after we parse an id +typedef enum ndb_idres (*ndb_id_fn)(void *, const char *); + +// id callback + closure data +struct ndb_id_cb { + ndb_id_fn fn; + void *data; +}; + struct ndb_str { unsigned char flag; union { @@ -80,6 +102,7 @@ struct ndb_tag { }; struct ndb_tags { + uint16_t padding; uint16_t count; struct ndb_tag tag[0]; }; @@ -92,12 +115,12 @@ struct ndb_note { unsigned char pubkey[32]; unsigned char sig[64]; - uint32_t created_at; + uint64_t created_at; uint32_t kind; uint32_t content_length; union ndb_packed_str content; uint32_t strings; - + uint32_t reserved[4]; // expansion slots // nothing can come after tags since it contains variadic data struct ndb_tags tags; }; @@ -126,16 +149,24 @@ int ndb_calculate_id(struct ndb_note *note, unsigned char *buf, int buflen); int ndb_sign_id(struct ndb_keypair *keypair, unsigned char id[32], unsigned char sig[64]); int ndb_create_keypair(struct ndb_keypair *key); int ndb_decode_key(const char *secstr, struct ndb_keypair *keypair); +int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[32], unsigned char signature[64]); -// BUILDER +// NDB +int ndb_init(struct ndb **ndb, const char *dbdir, size_t mapsize, int ingester_threads); +int ndb_process_event(struct ndb *, const char *json, int len); +int ndb_process_events(struct ndb *, const char *ldjson, size_t len); +int ndb_get_profile(struct ndb *, unsigned char pubkey[32], void **out); +struct ndb_note *ndb_get_note_by_id(struct ndb *, const unsigned char *id); +void ndb_destroy(struct ndb *); +// BUILDER int ndb_parse_json_note(struct ndb_json_parser *, struct ndb_note **); -int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, unsigned char *buf, int bufsize); +int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, unsigned char *buf, int bufsize, struct ndb_id_cb *); int ndb_note_from_json(const char *json, int len, struct ndb_note **, unsigned char *buf, int buflen); int ndb_builder_init(struct ndb_builder *builder, unsigned char *buf, int bufsize); int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note, struct ndb_keypair *privkey); int ndb_builder_set_content(struct ndb_builder *builder, const char *content, int len); -void ndb_builder_set_created_at(struct ndb_builder *builder, uint32_t created_at); +void ndb_builder_set_created_at(struct ndb_builder *builder, uint64_t created_at); void ndb_builder_set_sig(struct ndb_builder *builder, unsigned char *sig); void ndb_builder_set_pubkey(struct ndb_builder *builder, unsigned char *pubkey); void ndb_builder_set_id(struct ndb_builder *builder, unsigned char *id); diff --git a/nostrdb/protected_queue.h b/nostrdb/protected_queue.h new file mode 100644 index 0000000000..c2212b6996 --- /dev/null +++ b/nostrdb/protected_queue.h @@ -0,0 +1,235 @@ +/* + * This header file provides a thread-safe queue implementation for generic + * data elements. It uses POSIX threads (pthreads) to ensure thread safety. + * The queue allows for pushing and popping elements, with the ability to + * block or non-block on pop operations. Users are responsible for providing + * memory for the queue buffer and ensuring its correct lifespan. + * + * Author: William Casarin + * Inspired-by: https://github.com/hoytech/hoytech-cpp/blob/master/hoytech/protected_queue.h + */ + +#ifndef PROT_QUEUE_H +#define PROT_QUEUE_H + +#include +#include +#include +#include +#include "cursor.h" +#include "util.h" + +#define BUFFER_SIZE 100 + +/* + * The prot_queue structure represents a thread-safe queue that can hold + * generic data elements. + */ +struct prot_queue { + unsigned char *buf; + size_t buflen; + + int head; + int tail; + int count; + int elem_size; + + pthread_mutex_t mutex; + pthread_cond_t cond; +}; + + +/* + * Initialize the queue. + * Params: + * q - Pointer to the queue. + * buf - Buffer for holding data elements. + * buflen - Length of the buffer. + * elem_size - Size of each data element. + * Returns 1 if successful, 0 otherwise. + */ +static inline int prot_queue_init(struct prot_queue* q, void* buf, + size_t buflen, int elem_size) +{ + // buffer elements must fit nicely in the buffer + if (buflen == 0 || buflen % elem_size != 0) + return 0; + + q->head = 0; + q->tail = 0; + q->count = 0; + q->buf = buf; + q->buflen = buflen; + q->elem_size = elem_size; + + pthread_mutex_init(&q->mutex, NULL); + pthread_cond_init(&q->cond, NULL); + + return 1; +} + +/* + * Return the capacity of the queue. + * q - Pointer to the queue. + */ +static inline size_t prot_queue_capacity(struct prot_queue *q) { + return q->buflen / q->elem_size; +} + +/* + * Push an element onto the queue. + * Params: + * q - Pointer to the queue. + * data - Pointer to the data element to be pushed. + * + * Returns 1 if successful, 0 if the queue is full. + */ +static int prot_queue_push(struct prot_queue* q, void *data) +{ + int cap; + + pthread_mutex_lock(&q->mutex); + + cap = prot_queue_capacity(q); + if (q->count == cap) { + // only signal if the push was sucessful + pthread_mutex_unlock(&q->mutex); + return 0; + } + + memcpy(&q->buf[q->tail * q->elem_size], data, q->elem_size); + q->tail = (q->tail + 1) % cap; + q->count++; + + pthread_cond_signal(&q->cond); + pthread_mutex_unlock(&q->mutex); + + return 1; +} + +/* + * Push multiple elements onto the queue. + * Params: + * q - Pointer to the queue. + * data - Pointer to the data elements to be pushed. + * count - Number of elements to push. + * + * Returns the number of elements successfully pushed, 0 if the queue is full or if there is not enough contiguous space. + */ +static int prot_queue_push_all(struct prot_queue* q, void *data, int count) +{ + int cap; + int first_copy_count, second_copy_count; + + pthread_mutex_lock(&q->mutex); + + cap = prot_queue_capacity(q); + if (q->count + count > cap) { + pthread_mutex_unlock(&q->mutex); + return 0; // Return failure if the queue is full + } + + first_copy_count = min(count, cap - q->tail); // Elements until the end of the buffer + second_copy_count = count - first_copy_count; // Remaining elements if wrap around + + memcpy(&q->buf[q->tail * q->elem_size], data, first_copy_count * q->elem_size); + q->tail = (q->tail + first_copy_count) % cap; + + if (second_copy_count > 0) { + // If there is a wrap around, copy the remaining elements + memcpy(&q->buf[q->tail * q->elem_size], (char *)data + first_copy_count * q->elem_size, second_copy_count * q->elem_size); + q->tail = (q->tail + second_copy_count) % cap; + } + + q->count += count; + + pthread_cond_signal(&q->cond); // Signal a waiting thread + pthread_mutex_unlock(&q->mutex); + + return count; +} + +/* + * Try to pop an element from the queue without blocking. + * Params: + * q - Pointer to the queue. + * data - Pointer to where the popped data will be stored. + * Returns 1 if successful, 0 if the queue is empty. + */ +static inline int prot_queue_try_pop(struct prot_queue *q, void *data) { + pthread_mutex_lock(&q->mutex); + + if (q->count == 0) { + pthread_mutex_unlock(&q->mutex); + return 0; + } + + memcpy(data, &q->buf[q->head * q->elem_size], q->elem_size); + q->head = (q->head + 1) % prot_queue_capacity(q); + q->count--; + + pthread_mutex_unlock(&q->mutex); + return 1; +} + +/* + * Wait until we have elements, and then pop multiple elements from the queue + * up to the specified maximum. + * + * Params: + * q - Pointer to the queue. + * buffer - Pointer to the buffer where popped data will be stored. + * max_items - Maximum number of items to pop from the queue. + * Returns the actual number of items popped. + */ +static int prot_queue_pop_all(struct prot_queue *q, void *dest, int max_items) { + pthread_mutex_lock(&q->mutex); + + // Wait until there's at least one item to pop + while (q->count == 0) { + pthread_cond_wait(&q->cond, &q->mutex); + } + + int items_until_end = (q->buflen - q->head * q->elem_size) / q->elem_size; + int items_to_pop = min(q->count, max_items); + items_to_pop = min(items_to_pop, items_until_end); + + memcpy(dest, &q->buf[q->head * q->elem_size], items_to_pop * q->elem_size); + q->head = (q->head + items_to_pop) % prot_queue_capacity(q); + q->count -= items_to_pop; + + pthread_mutex_unlock(&q->mutex); + + return items_to_pop; +} + +/* + * Pop an element from the queue. Blocks if the queue is empty. + * Params: + * q - Pointer to the queue. + * data - Pointer to where the popped data will be stored. + */ +static inline void prot_queue_pop(struct prot_queue *q, void *data) { + pthread_mutex_lock(&q->mutex); + + while (q->count == 0) + pthread_cond_wait(&q->cond, &q->mutex); + + memcpy(data, &q->buf[q->head * q->elem_size], q->elem_size); + q->head = (q->head + 1) % prot_queue_capacity(q); + q->count--; + + pthread_mutex_unlock(&q->mutex); +} + +/* + * Destroy the queue. Releases resources associated with the queue. + * Params: + * q - Pointer to the queue. + */ +static inline void prot_queue_destroy(struct prot_queue* q) { + pthread_mutex_destroy(&q->mutex); + pthread_cond_destroy(&q->cond); +} + +#endif // PROT_QUEUE_H diff --git a/nostrdb/threadpool.h b/nostrdb/threadpool.h new file mode 100644 index 0000000000..dca952e279 --- /dev/null +++ b/nostrdb/threadpool.h @@ -0,0 +1,103 @@ + +#ifndef THREADPOOL_H +#define THREADPOOL_H + +#include "protected_queue.h" + +struct thread +{ + pthread_t thread_id; + struct prot_queue inbox; + void *qmem; + void *ctx; +}; + +struct threadpool +{ + int num_threads; + struct thread *pool; + int next_thread; + void *quit_msg; +}; + +static int threadpool_init(struct threadpool *tp, int num_threads, + int q_elem_size, int q_num_elems, + void *quit_msg, void *ctx, void* (*thread_fn)(void*)) +{ + int i; + struct thread *t; + + if (num_threads <= 0) + return 0; + + tp->num_threads = num_threads; + tp->pool = malloc(sizeof(*tp->pool) * num_threads); + tp->quit_msg = quit_msg; + tp->next_thread = -1; + + if (tp->pool == NULL) { + fprintf(stderr, "threadpool_init: couldn't allocate memory for pool"); + return 0; + } + + for (i = 0; i < num_threads; i++) { + t = &tp->pool[i]; + t->qmem = malloc(q_elem_size * q_num_elems); + t->ctx = ctx; + + if (t->qmem == NULL) { + fprintf(stderr, "threadpool_init: couldn't allocate memory for queue"); + return 0; + } + + if (!prot_queue_init(&t->inbox, t->qmem, q_elem_size * q_num_elems, q_elem_size)) { + fprintf(stderr, "threadpool_init: couldn't init queue. buffer alignment is wrong."); + return 0; + } + + if (pthread_create(&t->thread_id, NULL, thread_fn, t) != 0) { + fprintf(stderr, "threadpool_init: failed to create thread\n"); + return 0; + } + } + + return 1; +} + +static inline struct thread *threadpool_next_thread(struct threadpool *tp) +{ + tp->next_thread = (tp->next_thread + 1) % tp->num_threads; + return &tp->pool[tp->next_thread]; +} + +static inline int threadpool_dispatch(struct threadpool *tp, void *msg) +{ + struct thread *t = threadpool_next_thread(tp); + return prot_queue_push(&t->inbox, msg); +} + +static inline int threadpool_dispatch_all(struct threadpool *tp, void *msgs, + int num_msgs) +{ + struct thread *t = threadpool_next_thread(tp); + return prot_queue_push_all(&t->inbox, msgs, num_msgs); +} + +static inline void threadpool_destroy(struct threadpool *tp) +{ + struct thread *t; + + for (uint64_t i = 0; i < tp->num_threads; i++) { + t = &tp->pool[i]; + if (!prot_queue_push(&t->inbox, tp->quit_msg)) { + pthread_exit(&t->thread_id); + } else { + pthread_join(t->thread_id, NULL); + } + prot_queue_destroy(&t->inbox); + free(t->qmem); + } + free(tp->pool); +} + +#endif // THREADPOOL_H diff --git a/nostrdb/util.h b/nostrdb/util.h new file mode 100644 index 0000000000..e60e388002 --- /dev/null +++ b/nostrdb/util.h @@ -0,0 +1,33 @@ + +#ifndef NDB_UTIL_H +#define NDB_UTIL_H + +static inline int min(int a, int b) { + return a < b ? a : b; +} + +static inline int max(int a, int b) { + return a > b ? a : b; +} + +static inline void* memdup(const void* src, size_t size) { + void* dest = malloc(size); + if (dest == NULL) { + return NULL; // Memory allocation failed + } + memcpy(dest, src, size); + return dest; +} + +static inline char *strdupn(const char *src, size_t size) { + char* dest = malloc(size+1); + if (dest == NULL) { + return NULL; // Memory allocation failed + } + memcpy(dest, src, size); + dest[size] = '\0'; + return dest; +} + +#endif // NDB_UTIL_H + From 35b67dc08d4b61bf98922cc94cffcb3037c77710 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Fri, 25 Aug 2023 18:13:42 -0700 Subject: [PATCH 036/111] nostrdb: initial Ndb class --- damus.xcodeproj/project.pbxproj | 4 ++- damus/TestData.swift | 5 ++++ nostrdb/Ndb.swift | 47 +++++++++++++++++++++++++++++++++ nostrdb/Test/NdbTests.swift | 23 ++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 nostrdb/Ndb.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 4c1bb4d328..332ef3120d 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -103,6 +103,7 @@ 4C30AC7629A5770900E2BD5A /* NotificationItemView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C30AC7529A5770900E2BD5A /* NotificationItemView.swift */; }; 4C30AC7829A577AB00E2BD5A /* EventCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C30AC7729A577AB00E2BD5A /* EventCache.swift */; }; 4C30AC8029A6A53F00E2BD5A /* ProfilePicturesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C30AC7F29A6A53F00E2BD5A /* ProfilePicturesView.swift */; }; + 4C32B9332A99845B00DC3548 /* Ndb.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C478E242A9932C100489948 /* Ndb.swift */; }; 4C363A8428233689006E126D /* Parser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C363A8328233689006E126D /* Parser.swift */; }; 4C363A8828236948006E126D /* BlocksView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C363A8728236948006E126D /* BlocksView.swift */; }; 4C363A8C28236B92006E126D /* PubkeyView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C363A8B28236B92006E126D /* PubkeyView.swift */; }; @@ -1732,6 +1733,7 @@ 4C90548A2A6AEDEE00811EEC /* NdbNote.swift */, 4C5D5C9C2A6B2CB40024563C /* AsciiCharacter.swift */, 4CDD1ADF2A6B305F001CD4DF /* NdbTagElem.swift */, + 4C478E242A9932C100489948 /* Ndb.swift */, 4CDD1AE12A6B3074001CD4DF /* NdbTagsIterator.swift */, 4CE9FBB82A6B3B26007E485C /* nostrdb.c */, 4C4793032A993DB900489948 /* midl.c */, @@ -1749,7 +1751,6 @@ 4C78EFD72A707C4D007E8197 /* secp256k1_schnorrsig.h */, 4C78EFDA2A707C67007E8197 /* secp256k1_extrakeys.h */, 4C78EFD92A707C4D007E8197 /* secp256k1.h */, - 4C478E242A9932C100489948 /* Ndb.swift */, ); path = nostrdb; sourceTree = ""; @@ -2403,6 +2404,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 4C32B9332A99845B00DC3548 /* Ndb.swift in Sources */, 4C4793082A993E8900489948 /* refmap.c in Sources */, 4C4793072A993E6200489948 /* emitter.c in Sources */, 4C4793062A993E5300489948 /* json_parser.c in Sources */, diff --git a/damus/TestData.swift b/damus/TestData.swift index 5e8bfec9c7..2f225ba201 100644 --- a/damus/TestData.swift +++ b/damus/TestData.swift @@ -58,6 +58,11 @@ func test_damus_state() -> DamusState { return damus } +let test_wire_events = """ +["EVENT","s",{"id":"d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349","pubkey":"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245","created_at":1650049978,"kind":0,"tags":[],"content":"{\\"name\\\":\\"jb55\\",\\"picture\\":\\\"http://cdn.jb55.com/img/red-me.jpg\\",\\"about\\":\\"bitcoin, lightning and nostr dev\\",\\"nip05\\":\\"jb55.com\\"}","sig":"1315045da793c4825de1517149172bf35a6da39d91b7787afb3263721e07bc816cb898996ed8d69af05d6efcd1c926a089bd66cad870bcc361405c11ba302c51"}] +["EVENT","s",{"id":"b2e03951843b191b5d9d1969f48db0156b83cc7dbd841f543f109362e24c4a9c","pubkey":"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245","created_at":1650050002,"kind":1,"tags":[],"content":"hello, this is my new key","sig":"4342eff1d78a82b42522cd26ec66a5293eca997f81d4b80efd02230d3d27317fb63d42656e8f32383562f075a2b6d999b60dcf70e2df18cf5e8b3801faeb0bd6"}] +""" + let test_failing_nostr_report = """ { "id": "99198ecb6a34372b7e39b88280bab3394654a00f5f8504466fac2d6acb569663", diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift new file mode 100644 index 0000000000..b12e57ca89 --- /dev/null +++ b/nostrdb/Ndb.swift @@ -0,0 +1,47 @@ +// +// Ndb.swift +// damus +// +// Created by William Casarin on 2023-08-25. +// + +import Foundation + +class Ndb { + let ndb: ndb_t + + init?() { + var ndb_p: OpaquePointer? = nil + + let dir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: "") + + let ok = dir!.withCString { testdir in + return ndb_init(&ndb_p, testdir, 1024 * 1024 * 700, 4) != 0 + } + + if !ok { + return nil + } + + self.ndb = ndb_t(ndb: ndb_p) + } + + func lookup_note(_ id: NoteId) -> NdbNote? { + id.id.withUnsafeBytes { bs in + guard let note_p = ndb_get_note_by_id(ndb.ndb, bs) else { + return nil + } + return NdbNote(note: note_p, owned_size: nil) + } + } + + func process_events(_ str: String) -> Bool { + return str.withCString { cstr in + return ndb_process_events(ndb.ndb, cstr, str.utf8.count) != 0 + } + } + + deinit { + ndb_destroy(ndb.ndb) + } +} diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift index 2b1e9fb177..54c1b84633 100644 --- a/nostrdb/Test/NdbTests.swift +++ b/nostrdb/Test/NdbTests.swift @@ -31,6 +31,29 @@ final class NdbTests: XCTestCase { } + func test_ndb_init() { + + do { + let ndb = Ndb()! + let ok = ndb.process_events(test_wire_events) + XCTAssertTrue(ok) + } + + do { + let ndb = Ndb()! + let id1 = NoteId(hex: "d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349")! + let note1 = ndb.lookup_note(id1) + XCTAssertNotNil(note1) + let id = NoteId(hex: "b2e03951843b191b5d9d1969f48db0156b83cc7dbd841f543f109362e24c4a9c")! + let note = ndb.lookup_note(id) + XCTAssertNotNil(note) + guard let note else { return } + XCTAssertEqual(note.pubkey, Pubkey(hex: "32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245")!) + } + + + } + func test_ndb_note() throws { let note = NdbNote.owned_from_json(json: test_contact_list_json) XCTAssertNotNil(note) From 4c0166bd31842cfeb0e79cc041cf01e5e0ed9b03 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sat, 26 Aug 2023 20:43:54 -0700 Subject: [PATCH 037/111] add swift flatbuffers --- damus.xcodeproj/project.pbxproj | 93 ++ flatbuffers/ByteBuffer.swift | 462 +++++++++ .../Documentation.docc/Documentation.md | 22 + .../Resources/code/fbs/monster_step_1.fbs | 1 + .../Resources/code/fbs/monster_step_2.fbs | 1 + .../Resources/code/fbs/monster_step_3.fbs | 6 + .../Resources/code/fbs/monster_step_4.fbs | 12 + .../Resources/code/fbs/monster_step_5.fbs | 18 + .../Resources/code/fbs/monster_step_6.fbs | 25 + .../Resources/code/fbs/monster_step_7.fbs | 27 + .../Resources/code/swift/swift_code_1.swift | 1 + .../Resources/code/swift/swift_code_10.swift | 71 ++ .../Resources/code/swift/swift_code_11.swift | 11 + .../Resources/code/swift/swift_code_12.swift | 19 + .../Resources/code/swift/swift_code_13.swift | 26 + .../Resources/code/swift/swift_code_2.swift | 2 + .../Resources/code/swift/swift_code_3.swift | 7 + .../Resources/code/swift/swift_code_4.swift | 10 + .../Resources/code/swift/swift_code_5.swift | 22 + .../Resources/code/swift/swift_code_6.swift | 26 + .../Resources/code/swift/swift_code_7.swift | 29 + .../Resources/code/swift/swift_code_8.swift | 40 + .../Resources/code/swift/swift_code_9.swift | 62 ++ .../images/tutorial_cover_image_1.png | Bin 0 -> 20176 bytes .../Tutorial_Table_of_Contents.tutorial | 14 + .../create_your_first_buffer.tutorial | 72 ++ .../creating_flatbuffer_schema.tutorial | 47 + .../Tutorials/reading_bytebuffer.tutorial | 27 + flatbuffers/Enum.swift | 59 ++ flatbuffers/FbConstants.swift | 122 +++ flatbuffers/FlatBufferBuilder.swift | 920 ++++++++++++++++++ flatbuffers/FlatBufferObject.swift | 68 ++ flatbuffers/FlatBuffersUtils.swift | 41 + flatbuffers/FlatbuffersErrors.swift | 77 ++ flatbuffers/Int+extension.swift | 51 + flatbuffers/Message.swift | 69 ++ flatbuffers/Mutable.swift | 88 ++ flatbuffers/NativeObject.swift | 57 ++ flatbuffers/Offset.swift | 32 + flatbuffers/Root.swift | 119 +++ flatbuffers/String+extension.swift | 113 +++ flatbuffers/Struct.swift | 51 + flatbuffers/Table.swift | 240 +++++ flatbuffers/TableVerifier.swift | 207 ++++ flatbuffers/VeriferOptions.swift | 56 ++ flatbuffers/Verifiable.swift | 215 ++++ flatbuffers/Verifier.swift | 217 +++++ 47 files changed, 3955 insertions(+) create mode 100644 flatbuffers/ByteBuffer.swift create mode 100644 flatbuffers/Documentation.docc/Documentation.md create mode 100644 flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_1.fbs create mode 100644 flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_2.fbs create mode 100644 flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_3.fbs create mode 100644 flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_4.fbs create mode 100644 flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_5.fbs create mode 100644 flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_6.fbs create mode 100644 flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_7.fbs create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_1.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_10.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_11.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_12.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_13.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_2.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_3.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_4.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_5.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_6.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_7.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_8.swift create mode 100644 flatbuffers/Documentation.docc/Resources/code/swift/swift_code_9.swift create mode 100644 flatbuffers/Documentation.docc/Resources/images/tutorial_cover_image_1.png create mode 100644 flatbuffers/Documentation.docc/Tutorials/Tutorial_Table_of_Contents.tutorial create mode 100644 flatbuffers/Documentation.docc/Tutorials/create_your_first_buffer.tutorial create mode 100644 flatbuffers/Documentation.docc/Tutorials/creating_flatbuffer_schema.tutorial create mode 100644 flatbuffers/Documentation.docc/Tutorials/reading_bytebuffer.tutorial create mode 100644 flatbuffers/Enum.swift create mode 100644 flatbuffers/FbConstants.swift create mode 100644 flatbuffers/FlatBufferBuilder.swift create mode 100644 flatbuffers/FlatBufferObject.swift create mode 100644 flatbuffers/FlatBuffersUtils.swift create mode 100644 flatbuffers/FlatbuffersErrors.swift create mode 100644 flatbuffers/Int+extension.swift create mode 100644 flatbuffers/Message.swift create mode 100644 flatbuffers/Mutable.swift create mode 100644 flatbuffers/NativeObject.swift create mode 100644 flatbuffers/Offset.swift create mode 100644 flatbuffers/Root.swift create mode 100644 flatbuffers/String+extension.swift create mode 100644 flatbuffers/Struct.swift create mode 100644 flatbuffers/Table.swift create mode 100644 flatbuffers/TableVerifier.swift create mode 100644 flatbuffers/VeriferOptions.swift create mode 100644 flatbuffers/Verifiable.swift create mode 100644 flatbuffers/Verifier.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 332ef3120d..171c505551 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -104,6 +104,28 @@ 4C30AC7829A577AB00E2BD5A /* EventCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C30AC7729A577AB00E2BD5A /* EventCache.swift */; }; 4C30AC8029A6A53F00E2BD5A /* ProfilePicturesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C30AC7F29A6A53F00E2BD5A /* ProfilePicturesView.swift */; }; 4C32B9332A99845B00DC3548 /* Ndb.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C478E242A9932C100489948 /* Ndb.swift */; }; + 4C32B9342A9AD01A00DC3548 /* NdbProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C478E2C2A9935D300489948 /* NdbProfile.swift */; }; + 4C32B94C2A9AD44700DC3548 /* FbConstants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9372A9AD44700DC3548 /* FbConstants.swift */; }; + 4C32B94D2A9AD44700DC3548 /* Offset.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9382A9AD44700DC3548 /* Offset.swift */; }; + 4C32B94E2A9AD44700DC3548 /* Mutable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9392A9AD44700DC3548 /* Mutable.swift */; }; + 4C32B94F2A9AD44700DC3548 /* Int+extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B93A2A9AD44700DC3548 /* Int+extension.swift */; }; + 4C32B9502A9AD44700DC3548 /* FlatBufferBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B93B2A9AD44700DC3548 /* FlatBufferBuilder.swift */; }; + 4C32B9512A9AD44700DC3548 /* FlatbuffersErrors.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B93C2A9AD44700DC3548 /* FlatbuffersErrors.swift */; }; + 4C32B9522A9AD44700DC3548 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B93D2A9AD44700DC3548 /* Message.swift */; }; + 4C32B9532A9AD44700DC3548 /* Verifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B93E2A9AD44700DC3548 /* Verifier.swift */; }; + 4C32B9542A9AD44700DC3548 /* FlatBuffersUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B93F2A9AD44700DC3548 /* FlatBuffersUtils.swift */; }; + 4C32B9552A9AD44700DC3548 /* ByteBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9402A9AD44700DC3548 /* ByteBuffer.swift */; }; + 4C32B9562A9AD44700DC3548 /* TableVerifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9412A9AD44700DC3548 /* TableVerifier.swift */; }; + 4C32B9572A9AD44700DC3548 /* Root.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9422A9AD44700DC3548 /* Root.swift */; }; + 4C32B9582A9AD44700DC3548 /* VeriferOptions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9432A9AD44700DC3548 /* VeriferOptions.swift */; }; + 4C32B9592A9AD44700DC3548 /* Table.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9442A9AD44700DC3548 /* Table.swift */; }; + 4C32B95A2A9AD44700DC3548 /* Verifiable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9452A9AD44700DC3548 /* Verifiable.swift */; }; + 4C32B95B2A9AD44700DC3548 /* NativeObject.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9462A9AD44700DC3548 /* NativeObject.swift */; }; + 4C32B95C2A9AD44700DC3548 /* String+extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9472A9AD44700DC3548 /* String+extension.swift */; }; + 4C32B95D2A9AD44700DC3548 /* Documentation.docc in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9482A9AD44700DC3548 /* Documentation.docc */; }; + 4C32B95E2A9AD44700DC3548 /* FlatBufferObject.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B9492A9AD44700DC3548 /* FlatBufferObject.swift */; }; + 4C32B95F2A9AD44700DC3548 /* Enum.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B94A2A9AD44700DC3548 /* Enum.swift */; }; + 4C32B9602A9AD44700DC3548 /* Struct.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C32B94B2A9AD44700DC3548 /* Struct.swift */; }; 4C363A8428233689006E126D /* Parser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C363A8328233689006E126D /* Parser.swift */; }; 4C363A8828236948006E126D /* BlocksView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C363A8728236948006E126D /* BlocksView.swift */; }; 4C363A8C28236B92006E126D /* PubkeyView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C363A8B28236B92006E126D /* PubkeyView.swift */; }; @@ -617,6 +639,27 @@ 4C30AC7529A5770900E2BD5A /* NotificationItemView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NotificationItemView.swift; sourceTree = ""; }; 4C30AC7729A577AB00E2BD5A /* EventCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventCache.swift; sourceTree = ""; }; 4C30AC7F29A6A53F00E2BD5A /* ProfilePicturesView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilePicturesView.swift; sourceTree = ""; }; + 4C32B9372A9AD44700DC3548 /* FbConstants.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FbConstants.swift; sourceTree = ""; }; + 4C32B9382A9AD44700DC3548 /* Offset.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Offset.swift; sourceTree = ""; }; + 4C32B9392A9AD44700DC3548 /* Mutable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Mutable.swift; sourceTree = ""; }; + 4C32B93A2A9AD44700DC3548 /* Int+extension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Int+extension.swift"; sourceTree = ""; }; + 4C32B93B2A9AD44700DC3548 /* FlatBufferBuilder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlatBufferBuilder.swift; sourceTree = ""; }; + 4C32B93C2A9AD44700DC3548 /* FlatbuffersErrors.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlatbuffersErrors.swift; sourceTree = ""; }; + 4C32B93D2A9AD44700DC3548 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; + 4C32B93E2A9AD44700DC3548 /* Verifier.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Verifier.swift; sourceTree = ""; }; + 4C32B93F2A9AD44700DC3548 /* FlatBuffersUtils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlatBuffersUtils.swift; sourceTree = ""; }; + 4C32B9402A9AD44700DC3548 /* ByteBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ByteBuffer.swift; sourceTree = ""; }; + 4C32B9412A9AD44700DC3548 /* TableVerifier.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TableVerifier.swift; sourceTree = ""; }; + 4C32B9422A9AD44700DC3548 /* Root.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Root.swift; sourceTree = ""; }; + 4C32B9432A9AD44700DC3548 /* VeriferOptions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VeriferOptions.swift; sourceTree = ""; }; + 4C32B9442A9AD44700DC3548 /* Table.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Table.swift; sourceTree = ""; }; + 4C32B9452A9AD44700DC3548 /* Verifiable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Verifiable.swift; sourceTree = ""; }; + 4C32B9462A9AD44700DC3548 /* NativeObject.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NativeObject.swift; sourceTree = ""; }; + 4C32B9472A9AD44700DC3548 /* String+extension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "String+extension.swift"; sourceTree = ""; }; + 4C32B9482A9AD44700DC3548 /* Documentation.docc */ = {isa = PBXFileReference; lastKnownFileType = folder.documentationcatalog; path = Documentation.docc; sourceTree = ""; }; + 4C32B9492A9AD44700DC3548 /* FlatBufferObject.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlatBufferObject.swift; sourceTree = ""; }; + 4C32B94A2A9AD44700DC3548 /* Enum.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Enum.swift; sourceTree = ""; }; + 4C32B94B2A9AD44700DC3548 /* Struct.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Struct.swift; sourceTree = ""; }; 4C363A8328233689006E126D /* Parser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Parser.swift; sourceTree = ""; }; 4C363A8728236948006E126D /* BlocksView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BlocksView.swift; sourceTree = ""; }; 4C363A8B28236B92006E126D /* PubkeyView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PubkeyView.swift; sourceTree = ""; }; @@ -1329,6 +1372,34 @@ path = Notifications; sourceTree = ""; }; + 4C32B9362A9AD44700DC3548 /* flatbuffers */ = { + isa = PBXGroup; + children = ( + 4C32B9372A9AD44700DC3548 /* FbConstants.swift */, + 4C32B9382A9AD44700DC3548 /* Offset.swift */, + 4C32B9392A9AD44700DC3548 /* Mutable.swift */, + 4C32B93A2A9AD44700DC3548 /* Int+extension.swift */, + 4C32B93B2A9AD44700DC3548 /* FlatBufferBuilder.swift */, + 4C32B93C2A9AD44700DC3548 /* FlatbuffersErrors.swift */, + 4C32B93D2A9AD44700DC3548 /* Message.swift */, + 4C32B93E2A9AD44700DC3548 /* Verifier.swift */, + 4C32B93F2A9AD44700DC3548 /* FlatBuffersUtils.swift */, + 4C32B9402A9AD44700DC3548 /* ByteBuffer.swift */, + 4C32B9412A9AD44700DC3548 /* TableVerifier.swift */, + 4C32B9422A9AD44700DC3548 /* Root.swift */, + 4C32B9432A9AD44700DC3548 /* VeriferOptions.swift */, + 4C32B9442A9AD44700DC3548 /* Table.swift */, + 4C32B9452A9AD44700DC3548 /* Verifiable.swift */, + 4C32B9462A9AD44700DC3548 /* NativeObject.swift */, + 4C32B9472A9AD44700DC3548 /* String+extension.swift */, + 4C32B9482A9AD44700DC3548 /* Documentation.docc */, + 4C32B9492A9AD44700DC3548 /* FlatBufferObject.swift */, + 4C32B94A2A9AD44700DC3548 /* Enum.swift */, + 4C32B94B2A9AD44700DC3548 /* Struct.swift */, + ); + path = flatbuffers; + sourceTree = ""; + }; 4C478E2A2A9935D300489948 /* bindings */ = { isa = PBXGroup; children = ( @@ -1967,6 +2038,7 @@ 4CE6DEDA27F7A08100C66700 = { isa = PBXGroup; children = ( + 4C32B9362A9AD44700DC3548 /* flatbuffers */, 4C9054862A6AEB4500811EEC /* nostrdb */, 4C19AE4A2A5CEF7C00C90DB7 /* nostrscript */, 4C06670728FDE62900038D2A /* damus-c */, @@ -2414,16 +2486,19 @@ 4CE9FBBA2A6B3C63007E485C /* nostrdb.c in Sources */, 4C3AC79D2833036D00E1F516 /* FollowingView.swift in Sources */, 5CF72FC229B9142F00124A13 /* ShareAction.swift in Sources */, + 4C32B9522A9AD44700DC3548 /* Message.swift in Sources */, 4C8D1A6C29F1DFC200ACDF75 /* FriendIcon.swift in Sources */, 4C30AC7829A577AB00E2BD5A /* EventCache.swift in Sources */, 4C285C8428385690008A31F1 /* CreateAccountView.swift in Sources */, 4CDD1AE22A6B3074001CD4DF /* NdbTagsIterator.swift in Sources */, 4C216F34286F5ACD00040376 /* DMView.swift in Sources */, + 4C32B9572A9AD44700DC3548 /* Root.swift in Sources */, 4C3EA64428FF558100C48A62 /* sha256.c in Sources */, 4CCF9AAF2A1FDBDB00E03CFB /* VideoPlayer.swift in Sources */, 504323A72A34915F006AE6DC /* RelayModel.swift in Sources */, 4CA9276A2A290FC00098A105 /* ContextButton.swift in Sources */, 4CF0ABF62985CD5500D66079 /* UserSearch.swift in Sources */, + 4C32B9542A9AD44700DC3548 /* FlatBuffersUtils.swift in Sources */, 4C363AA828297703006E126D /* InsertSort.swift in Sources */, 4C285C86283892E7008A31F1 /* CreateAccountModel.swift in Sources */, 4C64987C286D03E000EAE2B3 /* DirectMessagesView.swift in Sources */, @@ -2461,6 +2536,7 @@ 4CA5588329F33F5B00DC6A45 /* StringCodable.swift in Sources */, 4C75EFB92804A2740006080F /* EventView.swift in Sources */, 4C15C7152A55DE7A00D0A0DB /* ReactionsSettingsView.swift in Sources */, + 4C32B94C2A9AD44700DC3548 /* FbConstants.swift in Sources */, 4C8D00C829DF791C0036AF10 /* CompatibleAttribute.swift in Sources */, 4C7D09742A0AEF9000943473 /* AlbyGradient.swift in Sources */, 4C687C272A6039500092C550 /* TestData.swift in Sources */, @@ -2470,8 +2546,11 @@ 4CB883B6297730E400DC99E7 /* LNUrls.swift in Sources */, 4C7FF7D52823313F009601DB /* Mentions.swift in Sources */, BA4AB0AE2A63B9270070A32A /* AddEmojiView.swift in Sources */, + 4C32B94D2A9AD44700DC3548 /* Offset.swift in Sources */, 4C633350283D40E500B1C9C3 /* HomeModel.swift in Sources */, 4C987B57283FD07F0042CE38 /* FollowersModel.swift in Sources */, + 4C32B9552A9AD44700DC3548 /* ByteBuffer.swift in Sources */, + 4C32B95B2A9AD44700DC3548 /* NativeObject.swift in Sources */, 3AB72AB9298ECF30004BB58C /* Translator.swift in Sources */, 4C363A9028247A1D006E126D /* NostrLink.swift in Sources */, 4C3D52B6298DB4E6001C5831 /* ZapEvent.swift in Sources */, @@ -2492,6 +2571,7 @@ 4C7D096D2A0AEA0400943473 /* CodeScanner.swift in Sources */, 4CE4F9DE2852768D00C00DD9 /* ConfigView.swift in Sources */, 3A48E7B029DFBE9D006E787E /* MutedThreadsManager.swift in Sources */, + 4C32B94E2A9AD44700DC3548 /* Mutable.swift in Sources */, 4C285C8E28399BFE008A31F1 /* SaveKeysView.swift in Sources */, F7F0BA25297892BD009531F3 /* SwipeToDismiss.swift in Sources */, 4C8D00CA29DF80350036AF10 /* TruncatedText.swift in Sources */, @@ -2502,6 +2582,7 @@ 4CB8838F296F781C00DC99E7 /* ReactionsView.swift in Sources */, BA4AB0B02A63B94D0070A32A /* EmojiListItemView.swift in Sources */, 4C75EFB328049D640006080F /* NostrEvent.swift in Sources */, + 4C32B9582A9AD44700DC3548 /* VeriferOptions.swift in Sources */, 4CA2EFA0280E37AC0044ACD8 /* TimelineView.swift in Sources */, 4C30AC7629A5770900E2BD5A /* NotificationItemView.swift in Sources */, 4C86F7C42A76C44C00EC0817 /* ZappingNotify.swift in Sources */, @@ -2515,6 +2596,7 @@ 4C90BD18283A9EE5008EE7EF /* LoginView.swift in Sources */, 4CB8838B296F6E1E00DC99E7 /* NIP05Badge.swift in Sources */, 4CA3FA1029F593D000FDB3C3 /* ZapTypePicker.swift in Sources */, + 4C32B95D2A9AD44700DC3548 /* Documentation.docc in Sources */, 4C3EA66828FF5F9900C48A62 /* hex.c in Sources */, E9E4ED0B295867B900DD7078 /* ThreadView.swift in Sources */, 4CD348EF29C3659D00497EB2 /* ImageUploadModel.swift in Sources */, @@ -2549,6 +2631,7 @@ 4C4E137B2A76D5FB00BDD832 /* MuteThreadNotify.swift in Sources */, 4CC7AAE7297EFA7B00430951 /* Zap.swift in Sources */, 4C12535A2A76C9960004F4B8 /* UnfollowNotify.swift in Sources */, + 4C32B95C2A9AD44700DC3548 /* String+extension.swift in Sources */, 4C3BEFD22819DB9B00B3DE84 /* ProfileModel.swift in Sources */, 4CA352AA2A76BF3A003BB08B /* LocalNotificationNotify.swift in Sources */, 4C7D09682A0AE9B200943473 /* NWCScannerView.swift in Sources */, @@ -2560,6 +2643,7 @@ 4CCEB7B029B5415A0078AA28 /* SearchingProfileView.swift in Sources */, BAB68BED29543FA3007BA466 /* SelectWalletView.swift in Sources */, 3169CAE6294E69C000EE4006 /* EmptyTimelineView.swift in Sources */, + 4C32B9602A9AD44700DC3548 /* Struct.swift in Sources */, 4CC7AAF0297F11C700430951 /* SelectedEventView.swift in Sources */, 4CC7AAF8297F1CEE00430951 /* EventProfile.swift in Sources */, 4C687C242A5FA86D0092C550 /* SearchHeaderView.swift in Sources */, @@ -2633,6 +2717,7 @@ 5C513FCC2984ACA60072348F /* QRCodeView.swift in Sources */, 4CC14FF52A740BB7007AEB17 /* NoteId.swift in Sources */, 4C19AE512A5CEF7C00C90DB7 /* NostrScript.swift in Sources */, + 4C32B95E2A9AD44700DC3548 /* FlatBufferObject.swift in Sources */, 4C3EA64F28FF59F200C48A62 /* tal.c in Sources */, 4CB88393296F798300DC99E7 /* ReactionsModel.swift in Sources */, 5C42E78C29DB76D90086AAC1 /* EmptyUserSearchView.swift in Sources */, @@ -2676,6 +2761,7 @@ 4C8D00CF29E38B950036AF10 /* nostr_bech32.c in Sources */, 4C1253562A76C8C60004F4B8 /* BroadcastNotify.swift in Sources */, 4C3BEFD42819DE8F00B3DE84 /* NostrKind.swift in Sources */, + 4C32B9532A9AD44700DC3548 /* Verifier.swift in Sources */, 4C3EA66028FF5E7700C48A62 /* node_id.c in Sources */, 4C687C212A5F7ED00092C550 /* DamusBackground.swift in Sources */, 4CA352A02A76AE80003BB08B /* Notify.swift in Sources */, @@ -2685,9 +2771,11 @@ 4C363A962827096D006E126D /* PostBlock.swift in Sources */, 4CA9275F2A2902B20098A105 /* LongformPreview.swift in Sources */, 4C5F9116283D855D0052CD1C /* EventsModel.swift in Sources */, + 4C32B94F2A9AD44700DC3548 /* Int+extension.swift in Sources */, 4CEE2AED2805B22500AB5EEF /* NostrRequest.swift in Sources */, 4C06670E28FDEAA000038D2A /* utf8.c in Sources */, 4C3EA66D28FF782800C48A62 /* amount.c in Sources */, + 4C32B9562A9AD44700DC3548 /* TableVerifier.swift in Sources */, 4C5E54062A9671F800FF6E60 /* UserStatusSheet.swift in Sources */, F71694F42A6732B7001F4053 /* GradientFollowButton.swift in Sources */, 4C3AC7A728369BA200E1F516 /* SearchHomeView.swift in Sources */, @@ -2695,8 +2783,10 @@ 4C363A922825FCF2006E126D /* ProfileUpdate.swift in Sources */, 4CB9D4A92992D2F400A9A7E4 /* FollowsYou.swift in Sources */, 4C3BEFDA281DCA1400B3DE84 /* LikeCounter.swift in Sources */, + 4C32B9502A9AD44700DC3548 /* FlatBufferBuilder.swift in Sources */, 50A60D142A28BEEE00186190 /* RelayLog.swift in Sources */, 4CB88389296AF99A00DC99E7 /* EventDetailBar.swift in Sources */, + 4C32B9512A9AD44700DC3548 /* FlatbuffersErrors.swift in Sources */, 4CE8794E2996B16A00F758CC /* RelayToggle.swift in Sources */, 4C3AC79B28306D7B00E1F516 /* Contacts.swift in Sources */, 4C3EA63D28FF52D600C48A62 /* bolt11.c in Sources */, @@ -2708,6 +2798,7 @@ 4C12536C2A76D4B00004F4B8 /* RepostedNotify.swift in Sources */, 4CB55EF3295E5D59007FD187 /* RecommendedRelayView.swift in Sources */, 4CE4F0F229D4FCFA005914DB /* DebouncedOnChange.swift in Sources */, + 4C32B9592A9AD44700DC3548 /* Table.swift in Sources */, 4C5D5C9D2A6B2CB40024563C /* AsciiCharacter.swift in Sources */, 4CF0ABEC29844B4700D66079 /* AnyDecodable.swift in Sources */, 4C9146FE2A2A87C200DDEA40 /* nostrscript.c in Sources */, @@ -2746,6 +2837,7 @@ 4C75EFA427FA577B0006080F /* PostView.swift in Sources */, 4C30AC7229A5677A00E2BD5A /* NotificationsView.swift in Sources */, 4C1A9A2129DDD3E100516EAC /* KeySettingsView.swift in Sources */, + 4C32B95A2A9AD44700DC3548 /* Verifiable.swift in Sources */, 4C73C5142A4437C10062CAC0 /* ZapUserView.swift in Sources */, 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */, 4C1A9A1D29DDCF9B00516EAC /* NotificationSettingsView.swift in Sources */, @@ -2762,6 +2854,7 @@ 4CB9D4A72992D02B00A9A7E4 /* ProfileNameView.swift in Sources */, 4CE4F0F429D779B5005914DB /* PostBox.swift in Sources */, 4C9B0DF32A65C46800CBDA21 /* ProfileEditButton.swift in Sources */, + 4C32B95F2A9AD44700DC3548 /* Enum.swift in Sources */, 4C2859622A12A7F0004746F7 /* GoldSupportGradient.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/flatbuffers/ByteBuffer.swift b/flatbuffers/ByteBuffer.swift new file mode 100644 index 0000000000..f5c681ac65 --- /dev/null +++ b/flatbuffers/ByteBuffer.swift @@ -0,0 +1,462 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// `ByteBuffer` is the interface that stores the data for a `Flatbuffers` object +/// it allows users to write and read data directly from memory thus the use of its +/// functions should be used +@frozen +public struct ByteBuffer { + + /// Storage is a container that would hold the memory pointer to solve the issue of + /// deallocating the memory that was held by (memory: UnsafeMutableRawPointer) + @usableFromInline + final class Storage { + // This storage doesn't own the memory, therefore, we won't deallocate on deinit. + private let unowned: Bool + /// pointer to the start of the buffer object in memory + var memory: UnsafeMutableRawPointer + /// Capacity of UInt8 the buffer can hold + var capacity: Int + + @usableFromInline + init(count: Int, alignment: Int) { + memory = UnsafeMutableRawPointer.allocate( + byteCount: count, + alignment: alignment) + capacity = count + unowned = false + } + + @usableFromInline + init(memory: UnsafeMutableRawPointer, capacity: Int, unowned: Bool) { + self.memory = memory + self.capacity = capacity + self.unowned = unowned + } + + deinit { + if !unowned { + memory.deallocate() + } + } + + @usableFromInline + func copy(from ptr: UnsafeRawPointer, count: Int) { + assert( + !unowned, + "copy should NOT be called on a buffer that is built by assumingMemoryBound") + memory.copyMemory(from: ptr, byteCount: count) + } + + @usableFromInline + func initialize(for size: Int) { + assert( + !unowned, + "initalize should NOT be called on a buffer that is built by assumingMemoryBound") + memset(memory, 0, size) + } + + /// Reallocates the buffer incase the object to be written doesnt fit in the current buffer + /// - Parameter size: Size of the current object + @usableFromInline + func reallocate(_ size: Int, writerSize: Int, alignment: Int) { + let currentWritingIndex = capacity &- writerSize + while capacity <= writerSize &+ size { + capacity = capacity << 1 + } + + /// solution take from Apple-NIO + capacity = capacity.convertToPowerofTwo + + let newData = UnsafeMutableRawPointer.allocate( + byteCount: capacity, + alignment: alignment) + memset(newData, 0, capacity &- writerSize) + memcpy( + newData.advanced(by: capacity &- writerSize), + memory.advanced(by: currentWritingIndex), + writerSize) + memory.deallocate() + memory = newData + } + } + + @usableFromInline var _storage: Storage + + /// The size of the elements written to the buffer + their paddings + private var _writerSize: Int = 0 + /// Aliginment of the current memory being written to the buffer + var alignment = 1 + /// Current Index which is being used to write to the buffer, it is written from the end to the start of the buffer + var writerIndex: Int { _storage.capacity &- _writerSize } + + /// Reader is the position of the current Writer Index (capacity - size) + public var reader: Int { writerIndex } + /// Current size of the buffer + public var size: UOffset { UOffset(_writerSize) } + /// Public Pointer to the buffer object in memory. This should NOT be modified for any reason + public var memory: UnsafeMutableRawPointer { _storage.memory } + /// Current capacity for the buffer + public var capacity: Int { _storage.capacity } + + /// Constructor that creates a Flatbuffer object from a UInt8 + /// - Parameter bytes: Array of UInt8 + public init(bytes: [UInt8]) { + var b = bytes + _storage = Storage(count: bytes.count, alignment: alignment) + _writerSize = _storage.capacity + b.withUnsafeMutableBytes { bufferPointer in + self._storage.copy(from: bufferPointer.baseAddress!, count: bytes.count) + } + } + + #if !os(WASI) + /// Constructor that creates a Flatbuffer from the Swift Data type object + /// - Parameter data: Swift data Object + public init(data: Data) { + var b = data + _storage = Storage(count: data.count, alignment: alignment) + _writerSize = _storage.capacity + b.withUnsafeMutableBytes { bufferPointer in + self._storage.copy(from: bufferPointer.baseAddress!, count: data.count) + } + } + #endif + + /// Constructor that creates a Flatbuffer instance with a size + /// - Parameter size: Length of the buffer + init(initialSize size: Int) { + let size = size.convertToPowerofTwo + _storage = Storage(count: size, alignment: alignment) + _storage.initialize(for: size) + } + + #if swift(>=5.0) && !os(WASI) + /// Constructor that creates a Flatbuffer object from a ContiguousBytes + /// - Parameters: + /// - contiguousBytes: Binary stripe to use as the buffer + /// - count: amount of readable bytes + public init( + contiguousBytes: Bytes, + count: Int) + { + _storage = Storage(count: count, alignment: alignment) + _writerSize = _storage.capacity + contiguousBytes.withUnsafeBytes { buf in + _storage.copy(from: buf.baseAddress!, count: buf.count) + } + } + #endif + + /// Constructor that creates a Flatbuffer from unsafe memory region without copying + /// - Parameter assumingMemoryBound: The unsafe memory region + /// - Parameter capacity: The size of the given memory region + public init( + assumingMemoryBound memory: UnsafeMutableRawPointer, + capacity: Int) + { + _storage = Storage(memory: memory, capacity: capacity, unowned: true) + _writerSize = capacity + } + + /// Creates a copy of the buffer that's being built by calling sizedBuffer + /// - Parameters: + /// - memory: Current memory of the buffer + /// - count: count of bytes + init(memory: UnsafeMutableRawPointer, count: Int) { + _storage = Storage(count: count, alignment: alignment) + _storage.copy(from: memory, count: count) + _writerSize = _storage.capacity + } + + /// Creates a copy of the existing flatbuffer, by copying it to a different memory. + /// - Parameters: + /// - memory: Current memory of the buffer + /// - count: count of bytes + /// - removeBytes: Removes a number of bytes from the current size + init( + memory: UnsafeMutableRawPointer, + count: Int, + removing removeBytes: Int) + { + _storage = Storage(count: count, alignment: alignment) + _storage.copy(from: memory, count: count) + _writerSize = removeBytes + } + + /// Fills the buffer with padding by adding to the writersize + /// - Parameter padding: Amount of padding between two to be serialized objects + @inline(__always) + @usableFromInline + mutating func fill(padding: Int) { + assert(padding >= 0, "Fill should be larger than or equal to zero") + ensureSpace(size: padding) + _writerSize = _writerSize &+ (MemoryLayout.size &* padding) + } + + /// Adds an array of type Scalar to the buffer memory + /// - Parameter elements: An array of Scalars + @inline(__always) + @usableFromInline + mutating func push(elements: [T]) { + let size = elements.count &* MemoryLayout.size + ensureSpace(size: size) + elements.reversed().forEach { s in + push(value: s, len: MemoryLayout.size(ofValue: s)) + } + } + + /// Adds an object of type NativeStruct into the buffer + /// - Parameters: + /// - value: Object that will be written to the buffer + /// - size: size to subtract from the WriterIndex + @usableFromInline + @inline(__always) + mutating func push(struct value: T, size: Int) { + ensureSpace(size: size) + var v = value + memcpy(_storage.memory.advanced(by: writerIndex &- size), &v, size) + _writerSize = _writerSize &+ size + } + + /// Adds an object of type Scalar into the buffer + /// - Parameters: + /// - value: Object that will be written to the buffer + /// - len: Offset to subtract from the WriterIndex + @inline(__always) + @usableFromInline + mutating func push(value: T, len: Int) { + ensureSpace(size: len) + var v = value + memcpy(_storage.memory.advanced(by: writerIndex &- len), &v, len) + _writerSize = _writerSize &+ len + } + + /// Adds a string to the buffer using swift.utf8 object + /// - Parameter str: String that will be added to the buffer + /// - Parameter len: length of the string + @inline(__always) + @usableFromInline + mutating func push(string str: String, len: Int) { + ensureSpace(size: len) + if str.utf8 + .withContiguousStorageIfAvailable({ self.push(bytes: $0, len: len) }) != + nil + { + } else { + let utf8View = str.utf8 + for c in utf8View.reversed() { + push(value: c, len: 1) + } + } + } + + /// Writes a string to Bytebuffer using UTF8View + /// - Parameters: + /// - bytes: Pointer to the view + /// - len: Size of string + @usableFromInline + @inline(__always) + mutating func push( + bytes: UnsafeBufferPointer, + len: Int) -> Bool + { + memcpy( + _storage.memory.advanced(by: writerIndex &- len), + UnsafeRawPointer(bytes.baseAddress!), + len) + _writerSize = _writerSize &+ len + return true + } + + /// Write stores an object into the buffer directly or indirectly. + /// + /// Direct: ignores the capacity of buffer which would mean we are referring to the direct point in memory + /// indirect: takes into respect the current capacity of the buffer (capacity - index), writing to the buffer from the end + /// - Parameters: + /// - value: Value that needs to be written to the buffer + /// - index: index to write to + /// - direct: Should take into consideration the capacity of the buffer + @inline(__always) + func write(value: T, index: Int, direct: Bool = false) { + var index = index + if !direct { + index = _storage.capacity &- index + } + assert(index < _storage.capacity, "Write index is out of writing bound") + assert(index >= 0, "Writer index should be above zero") + _storage.memory.storeBytes(of: value, toByteOffset: index, as: T.self) + } + + /// Makes sure that buffer has enouch space for each of the objects that will be written into it + /// - Parameter size: size of object + @discardableResult + @usableFromInline + @inline(__always) + mutating func ensureSpace(size: Int) -> Int { + if size &+ _writerSize > _storage.capacity { + _storage.reallocate(size, writerSize: _writerSize, alignment: alignment) + } + assert(size < FlatBufferMaxSize, "Buffer can't grow beyond 2 Gigabytes") + return size + } + + /// pops the written VTable if it's already written into the buffer + /// - Parameter size: size of the `VTable` + @usableFromInline + @inline(__always) + mutating func pop(_ size: Int) { + assert( + (_writerSize &- size) > 0, + "New size should NOT be a negative number") + memset(_storage.memory.advanced(by: writerIndex), 0, _writerSize &- size) + _writerSize = size + } + + /// Clears the current size of the buffer + @inline(__always) + mutating public func clearSize() { + _writerSize = 0 + } + + /// Clears the current instance of the buffer, replacing it with new memory + @inline(__always) + mutating public func clear() { + _writerSize = 0 + alignment = 1 + _storage.initialize(for: _storage.capacity) + } + + /// Reads an object from the buffer + /// - Parameters: + /// - def: Type of the object + /// - position: the index of the object in the buffer + @inline(__always) + public func read(def: T.Type, position: Int) -> T { + _storage.memory.advanced(by: position).load(as: T.self) + } + + /// Reads a slice from the memory assuming a type of T + /// - Parameters: + /// - index: index of the object to be read from the buffer + /// - count: count of bytes in memory + @inline(__always) + public func readSlice( + index: Int, + count: Int) -> [T] + { + assert( + index + count <= _storage.capacity, + "Reading out of bounds is illegal") + let start = _storage.memory.advanced(by: index) + .assumingMemoryBound(to: T.self) + let array = UnsafeBufferPointer(start: start, count: count) + return Array(array) + } + + #if !os(WASI) + /// Reads a string from the buffer and encodes it to a swift string + /// - Parameters: + /// - index: index of the string in the buffer + /// - count: length of the string + /// - type: Encoding of the string + @inline(__always) + public func readString( + at index: Int, + count: Int, + type: String.Encoding = .utf8) -> String? + { + assert( + index + count <= _storage.capacity, + "Reading out of bounds is illegal") + let start = _storage.memory.advanced(by: index) + .assumingMemoryBound(to: UInt8.self) + let bufprt = UnsafeBufferPointer(start: start, count: count) + return String(bytes: Array(bufprt), encoding: type) + } + #else + /// Reads a string from the buffer and encodes it to a swift string + /// - Parameters: + /// - index: index of the string in the buffer + /// - count: length of the string + /// - type: Encoding of the string + @inline(__always) + public func readString( + at index: Int, + count: Int) -> String? + { + assert( + index + count <= _storage.capacity, + "Reading out of bounds is illegal") + let start = _storage.memory.advanced(by: index) + .assumingMemoryBound(to: UInt8.self) + let bufprt = UnsafeBufferPointer(start: start, count: count) + return String(cString: bufprt.baseAddress!) + } + #endif + + /// Creates a new Flatbuffer object that's duplicated from the current one + /// - Parameter removeBytes: the amount of bytes to remove from the current Size + @inline(__always) + public func duplicate(removing removeBytes: Int = 0) -> ByteBuffer { + assert(removeBytes > 0, "Can NOT remove negative bytes") + assert( + removeBytes < _storage.capacity, + "Can NOT remove more bytes than the ones allocated") + return ByteBuffer( + memory: _storage.memory, + count: _storage.capacity, + removing: _writerSize &- removeBytes) + } + + /// Returns the written bytes into the ``ByteBuffer`` + public var underlyingBytes: [UInt8] { + let cp = capacity &- writerIndex + let start = memory.advanced(by: writerIndex) + .bindMemory(to: UInt8.self, capacity: cp) + + let ptr = UnsafeBufferPointer(start: start, count: cp) + return Array(ptr) + } + + /// SkipPrefix Skips the first 4 bytes in case one of the following + /// functions are called `getPrefixedSizeCheckedRoot` & `getPrefixedSizeRoot` + /// which allows us to skip the first 4 bytes instead of recreating the buffer + @discardableResult + @usableFromInline + @inline(__always) + mutating func skipPrefix() -> Int32 { + _writerSize = _writerSize &- MemoryLayout.size + return read(def: Int32.self, position: 0) + } + +} + +extension ByteBuffer: CustomDebugStringConvertible { + + public var debugDescription: String { + """ + buffer located at: \(_storage.memory), with capacity of \(_storage.capacity) + { writerSize: \(_writerSize), readerSize: \(reader), writerIndex: \(writerIndex) } + """ + } +} diff --git a/flatbuffers/Documentation.docc/Documentation.md b/flatbuffers/Documentation.docc/Documentation.md new file mode 100644 index 0000000000..a1510808eb --- /dev/null +++ b/flatbuffers/Documentation.docc/Documentation.md @@ -0,0 +1,22 @@ +# ``FlatBuffers`` + +FlatBuffers: Memory Efficient Serialization Library + +## Overview + +- Access to serialized data without parsing/unpacking - What sets FlatBuffers apart is that it represents hierarchical data in a flat binary buffer in such a way that it can still be accessed directly without parsing/unpacking, while also still supporting data structure evolution (forwards/backwards compatibility). +- Memory efficiency and speed - The only memory needed to access your data is that of the buffer. It requires 0 additional allocations (in C++, other languages may vary). FlatBuffers is also very suitable for use with mmap (or streaming), requiring only part of the buffer to be in memory. Access is close to the speed of raw struct access with only one extra indirection (a kind of vtable) to allow for format evolution and optional fields. It is aimed at projects where spending time and space (many memory allocations) to be able to access or construct serialized data is undesirable, such as in games or any other performance sensitive applications. See the benchmarks for details. +- Flexible - Optional fields means not only do you get great forwards and backwards compatibility (increasingly important for long-lived games: don't have to update all data with each new version!). It also means you have a lot of choice in what data you write and what data you don't, and how you design data structures. +- Tiny code footprint - Small amounts of generated code, and just a single small header as the minimum dependency, which is very easy to integrate. Again, see the benchmark section for details. +- Strongly typed - Errors happen at compile time rather than manually having to write repetitive and error prone run-time checks. Useful code can be generated for you. + +## Topics + +### Read this first + +- + +### Where to start + +- ``FlatBufferBuilder`` +- ``ByteBuffer`` diff --git a/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_1.fbs b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_1.fbs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_1.fbs @@ -0,0 +1 @@ + diff --git a/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_2.fbs b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_2.fbs new file mode 100644 index 0000000000..a43897845e --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_2.fbs @@ -0,0 +1 @@ +enum Color:byte { red, green, blue } diff --git a/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_3.fbs b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_3.fbs new file mode 100644 index 0000000000..d31a29cd02 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_3.fbs @@ -0,0 +1,6 @@ +enum Color:byte { red, green, blue } + +struct Vec3 { + x:float; + y:float; +} diff --git a/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_4.fbs b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_4.fbs new file mode 100644 index 0000000000..51f7bb1aa6 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_4.fbs @@ -0,0 +1,12 @@ +enum Color:byte { red, green, blue } + +struct Vec3 { + x:float; + y:float; +} + +table Monster { + pos:Vec3; + color:Color = Blue; +} + diff --git a/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_5.fbs b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_5.fbs new file mode 100644 index 0000000000..8d0b72956b --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_5.fbs @@ -0,0 +1,18 @@ +enum Color:byte { red, green, blue } + +struct Vec3 { + x:float; + y:float; +} + +table Monster { + pos:Vec3; + color:Color = Blue; + + mana:short = 150; + hp:short = 100; + name:string; + equipped:Equipment; + weapons:[Weapon]; + path:[Vec3]; +} diff --git a/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_6.fbs b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_6.fbs new file mode 100644 index 0000000000..10c3eaf67d --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_6.fbs @@ -0,0 +1,25 @@ +enum Color:byte { red, green, blue } + +union Equipment { Weapon } // Optionally add more tables. + +struct Vec3 { + x:float; + y:float; +} + +table Monster { + pos:Vec3; + color:Color = Blue; + + mana:short = 150; + hp:short = 100; + name:string; + equipped:Equipment; + weapons:[Weapon]; + path:[Vec3]; +} + +table Weapon { + name:string; + damage:short; +} diff --git a/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_7.fbs b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_7.fbs new file mode 100644 index 0000000000..b4dde6ced6 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/fbs/monster_step_7.fbs @@ -0,0 +1,27 @@ +enum Color:byte { red, green, blue } + +union Equipment { Weapon } // Optionally add more tables. + +struct Vec3 { + x:float; + y:float; +} + +table Monster { + pos:Vec3; + color:Color = Blue; + + mana:short = 150; + hp:short = 100; + name:string; + equipped:Equipment; + weapons:[Weapon]; + path:[Vec3]; +} + +table Weapon { + name:string; + damage:short; +} + +root_type Monster; // flatc --swift monster.fbs diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_1.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_1.swift new file mode 100644 index 0000000000..fecc4ab449 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_1.swift @@ -0,0 +1 @@ +import Foundation diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_10.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_10.swift new file mode 100644 index 0000000000..51d4fbfcdb --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_10.swift @@ -0,0 +1,71 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) + + let weapon1Name = builder.create(string: "Sword") + let weapon2Name = builder.create(string: "Axe") + + // start creating the weapon by calling startWeapon + let weapon1Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon1Name, &builder) + Weapon.add(damage: 3, &builder) + // end the object by passing the start point for the weapon 1 + let sword = Weapon.endWeapon(&builder, start: weapon1Start) + + let weapon2Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon2Name, &builder) + Weapon.add(damage: 5, &builder) + let axe = Weapon.endWeapon(&builder, start: weapon2Start) + + // Create a FlatBuffer `vector` that contains offsets to the sword and axe + // we created above. + let weaponsOffset = builder.createVector(ofOffsets: [sword, axe]) + + // Name of the Monster. + let name = builder.create(string: "Orc") + + let pathOffset = fbb.createVector(ofStructs: [ + Vec3(x: 0, y: 0), + Vec3(x: 5, y: 5), + ]) + + // startVector(len, elementSize: MemoryLayout.size) + // for o in offsets.reversed() { + // push(element: o) + // } + // endVector(len: len) + + let orc = Monster.createMonster( + &builder, + pos: Vec3(x: 1, y: 2), + hp: 300, + nameOffset: name, + color: .red, + weaponsVectorOffset: weaponsOffset, + equippedType: .weapon, + equippedOffset: axe, + pathOffset: pathOffset) + + // let start = Monster.startMonster(&builder) + // Monster.add(pos: Vec3(x: 1, y: 2), &builder) + // Monster.add(hp: 300, &builder) + // Monster.add(name: name, &builder) + // Monster.add(color: .red, &builder) + // Monster.addVectorOf(weapons: weaponsOffset, &builder) + // Monster.add(equippedType: .weapon, &builder) + // Monster.addVectorOf(paths: weaponsOffset, &builder) + // Monster.add(equipped: axe, &builder) + // var orc = Monster.endMonster(&builder, start: start) + + // Call `finish(offset:)` to instruct the builder that this monster is complete. + builder.finish(offset: orc) + // This must be called after `finish()`. + // `sizedByteArray` returns the finished buf of type [UInt8]. + let buf = builder.sizedByteArray + + // or you can use to get an object of type Data + let bufData = ByteBuffer(data: builder.sizedBuffer) +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_11.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_11.swift new file mode 100644 index 0000000000..07d2d8d2b0 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_11.swift @@ -0,0 +1,11 @@ +import FlatBuffers +import Foundation + +func run() { + // create a ByteBuffer(:) from an [UInt8] or Data() + let buf = [] // Get your data + var byteBuffer = ByteBuffer(bytes: buf) + // Get an accessor to the root object inside the buffer. + let monster: Monster = try! getCheckedRoot(byteBuffer: &byteBuffer) + // let monster: Monster = getRoot(byteBuffer: &byteBuffer) +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_12.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_12.swift new file mode 100644 index 0000000000..0d9ff69432 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_12.swift @@ -0,0 +1,19 @@ +import FlatBuffers +import Foundation + +func run() { + // create a ByteBuffer(:) from an [UInt8] or Data() + let buf = [] // Get your data + var byteBuffer = ByteBuffer(bytes: buf) + // Get an accessor to the root object inside the buffer. + let monster: Monster = try! getCheckedRoot(byteBuffer: &byteBuffer) + // let monster: Monster = getRoot(byteBuffer: &byteBuffer) + + let hp = monster.hp + let mana = monster.mana + let name = monster.name // returns an optional string + + let pos = monster.pos + let x = pos.x + let y = pos.y +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_13.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_13.swift new file mode 100644 index 0000000000..1372d6fc08 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_13.swift @@ -0,0 +1,26 @@ +import FlatBuffers +import Foundation + +func run() { + // create a ByteBuffer(:) from an [UInt8] or Data() + let buf = [] // Get your data + var byteBuffer = ByteBuffer(bytes: buf) + // Get an accessor to the root object inside the buffer. + let monster: Monster = try! getCheckedRoot(byteBuffer: &byteBuffer) + // let monster: Monster = getRoot(byteBuffer: &byteBuffer) + + let hp = monster.hp + let mana = monster.mana + let name = monster.name // returns an optional string + + let pos = monster.pos + let x = pos.x + let y = pos.y + + // Get and check if the monster has an equipped item + if monster.equippedType == .weapon { + let _weapon = monster.equipped(type: Weapon.self) + let name = _weapon.name // should return "Axe" + let dmg = _weapon.damage // should return 5 + } +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_2.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_2.swift new file mode 100644 index 0000000000..ddd066e7a8 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_2.swift @@ -0,0 +1,2 @@ +import FlatBuffers +import Foundation diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_3.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_3.swift new file mode 100644 index 0000000000..bacdaa5513 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_3.swift @@ -0,0 +1,7 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_4.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_4.swift new file mode 100644 index 0000000000..87546993bf --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_4.swift @@ -0,0 +1,10 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) + + let weapon1Name = builder.create(string: "Sword") + let weapon2Name = builder.create(string: "Axe") +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_5.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_5.swift new file mode 100644 index 0000000000..12e0d4ca66 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_5.swift @@ -0,0 +1,22 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) + + let weapon1Name = builder.create(string: "Sword") + let weapon2Name = builder.create(string: "Axe") + + // start creating the weapon by calling startWeapon + let weapon1Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon1Name, &builder) + Weapon.add(damage: 3, &builder) + // end the object by passing the start point for the weapon 1 + let sword = Weapon.endWeapon(&builder, start: weapon1Start) + + let weapon2Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon2Name, &builder) + Weapon.add(damage: 5, &builder) + let axe = Weapon.endWeapon(&builder, start: weapon2Start) +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_6.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_6.swift new file mode 100644 index 0000000000..bfb4f7e515 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_6.swift @@ -0,0 +1,26 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) + + let weapon1Name = builder.create(string: "Sword") + let weapon2Name = builder.create(string: "Axe") + + // start creating the weapon by calling startWeapon + let weapon1Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon1Name, &builder) + Weapon.add(damage: 3, &builder) + // end the object by passing the start point for the weapon 1 + let sword = Weapon.endWeapon(&builder, start: weapon1Start) + + let weapon2Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon2Name, &builder) + Weapon.add(damage: 5, &builder) + let axe = Weapon.endWeapon(&builder, start: weapon2Start) + + // Create a FlatBuffer `vector` that contains offsets to the sword and axe + // we created above. + let weaponsOffset = builder.createVector(ofOffsets: [sword, axe]) +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_7.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_7.swift new file mode 100644 index 0000000000..97264b018d --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_7.swift @@ -0,0 +1,29 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) + + let weapon1Name = builder.create(string: "Sword") + let weapon2Name = builder.create(string: "Axe") + + // start creating the weapon by calling startWeapon + let weapon1Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon1Name, &builder) + Weapon.add(damage: 3, &builder) + // end the object by passing the start point for the weapon 1 + let sword = Weapon.endWeapon(&builder, start: weapon1Start) + + let weapon2Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon2Name, &builder) + Weapon.add(damage: 5, &builder) + let axe = Weapon.endWeapon(&builder, start: weapon2Start) + + // Create a FlatBuffer `vector` that contains offsets to the sword and axe + // we created above. + let weaponsOffset = builder.createVector(ofOffsets: [sword, axe]) + + // Name of the Monster. + let name = builder.create(string: "Orc") +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_8.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_8.swift new file mode 100644 index 0000000000..a0c2819809 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_8.swift @@ -0,0 +1,40 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) + + let weapon1Name = builder.create(string: "Sword") + let weapon2Name = builder.create(string: "Axe") + + // start creating the weapon by calling startWeapon + let weapon1Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon1Name, &builder) + Weapon.add(damage: 3, &builder) + // end the object by passing the start point for the weapon 1 + let sword = Weapon.endWeapon(&builder, start: weapon1Start) + + let weapon2Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon2Name, &builder) + Weapon.add(damage: 5, &builder) + let axe = Weapon.endWeapon(&builder, start: weapon2Start) + + // Create a FlatBuffer `vector` that contains offsets to the sword and axe + // we created above. + let weaponsOffset = builder.createVector(ofOffsets: [sword, axe]) + + // Name of the Monster. + let name = builder.create(string: "Orc") + + let pathOffset = fbb.createVector(ofStructs: [ + Vec3(x: 0, y: 0), + Vec3(x: 5, y: 5), + ]) + + // startVector(len, elementSize: MemoryLayout.size) + // for o in offsets.reversed() { + // push(element: o) + // } + // endVector(len: len) +} diff --git a/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_9.swift b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_9.swift new file mode 100644 index 0000000000..51ce8fd2c2 --- /dev/null +++ b/flatbuffers/Documentation.docc/Resources/code/swift/swift_code_9.swift @@ -0,0 +1,62 @@ +import FlatBuffers +import Foundation + +func run() { + // create a `FlatBufferBuilder`, which will be used to serialize objects + let builder = FlatBufferBuilder(initialSize: 1024) + + let weapon1Name = builder.create(string: "Sword") + let weapon2Name = builder.create(string: "Axe") + + // start creating the weapon by calling startWeapon + let weapon1Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon1Name, &builder) + Weapon.add(damage: 3, &builder) + // end the object by passing the start point for the weapon 1 + let sword = Weapon.endWeapon(&builder, start: weapon1Start) + + let weapon2Start = Weapon.startWeapon(&builder) + Weapon.add(name: weapon2Name, &builder) + Weapon.add(damage: 5, &builder) + let axe = Weapon.endWeapon(&builder, start: weapon2Start) + + // Create a FlatBuffer `vector` that contains offsets to the sword and axe + // we created above. + let weaponsOffset = builder.createVector(ofOffsets: [sword, axe]) + + // Name of the Monster. + let name = builder.create(string: "Orc") + + let pathOffset = fbb.createVector(ofStructs: [ + Vec3(x: 0, y: 0), + Vec3(x: 5, y: 5), + ]) + + // startVector(len, elementSize: MemoryLayout.size) + // for o in offsets.reversed() { + // push(element: o) + // } + // endVector(len: len) + + let orc = Monster.createMonster( + &builder, + pos: Vec3(x: 1, y: 2), + hp: 300, + nameOffset: name, + color: .red, + weaponsVectorOffset: weaponsOffset, + equippedType: .weapon, + equippedOffset: axe, + pathOffset: pathOffset) + + // let start = Monster.startMonster(&builder) + // Monster.add(pos: Vec3(x: 1, y: 2), &builder) + // Monster.add(hp: 300, &builder) + // Monster.add(name: name, &builder) + // Monster.add(color: .red, &builder) + // Monster.addVectorOf(weapons: weaponsOffset, &builder) + // Monster.add(equippedType: .weapon, &builder) + // Monster.addVectorOf(paths: weaponsOffset, &builder) + // Monster.add(equipped: axe, &builder) + // var orc = Monster.endMonster(&builder, start: start) +} diff --git a/flatbuffers/Documentation.docc/Resources/images/tutorial_cover_image_1.png b/flatbuffers/Documentation.docc/Resources/images/tutorial_cover_image_1.png new file mode 100644 index 0000000000000000000000000000000000000000..0e64fe601486ac93ce649ff9d5fc8b8ad2300944 GIT binary patch literal 20176 zcmeFXV|3*~*Dn}$?2g&7ZL4EDCmq|iI(Da%bnFw`wmY_Mn;p){bKm>kJ99tG*ZDAK zty+cuM(I@5{_P#1q#%g|j|UG128JXpC9VPn1`YrvYS=HJ7Th+Tb}%qRTT3x9C227+ z5+x^lGfNv&Ffgfz#AFz)1Zj+)k95*fQlfC;!ZJtlzx4`&r$}L?Fl~a#e=8Zu!P2_j zDfQGvQES|(iBHdBpyC@bg&923hj|reWQ2K9M~-ySE{|ESZ5qlA|`~e+DSzbrc5F~N^JllC9G#myz^YU&ak$X zU}3Ga!k(}lq;S;8?ywVB&SsdG2AUAebFP2z86{dc1=vyaPl7S*YpCCeEwR+pDZo9M zzsOU1@VEz%ueF^wskdAo+G6jX?HS)Rs$?ku+eR(y*<8CH^y42%fj$EUhQwj^op?;_s&bA!!5U#*cQB<$DYQ&rqon*lC**Qs15r1Ot6pQ0bkSdb z_J#(m?zES8q!>+_b*7GYMD18j$o@u!+=P?IjB70a5tpa07t|Rvw7$dkU{(KNS{@eJ z{&u6)c0Y_-!#8%Bl}Ane;@zW-S?Dc!An;jAXQk)iIIyd9^3rl1XM!D zEPRN?G#w?y;CL{FzUjF)()n&=ml2AB6&as5f8R54iXosUYzlj*4e@zMd0p=9z!Lwy`u6 zt>sm0=T7G070E>Dgy@GrhJ~Te`N@DF2i_vQwz58jb{ zHE-!LBjck4#C7B*vP~3QG^; z0~>LVdA`06E%6E?8I#p_K|qE?bult9*gC@29fJo$UhZ4%I**$0qwtYHf!XTi>=f0= zU)UZ&02ici*~&CS91GxZ_<4E_CP;!B90<#d*B@l^8=10)Hfl>SIl$5YD+NVHghvZ} zArQ#`VF}qS$ajmx0&J+8#sT@q7uzk$W*B!xDi3h4Zk?%E5?Hw0Z;1UkHNrM%=KYY@ zVu5k6EKULGgtA5MPOF`%9LO;2Gm1%~-R0BAFq*qTQ5TYkx4Nyu2%{fq-|fD8 z|AhO>`wn~ABZOfAr5ecg?PoVdG}#_%A({*vW^iV3UvS-(-U)d-1$u-WP+%H+BkH*S z$Bx{N$`0j@pAlP4=2P&JENNV{^h7?13V(zoEqP-6Lc&A5LVRT$ue2{<1%L%00>lAu z0q7Oz74Q`T0IK>*OM%%Fs?&JeVembV+MoPE?HTPsYvP?U7vztschnDmNMu+kl3BA= zq)YNL3f79+rxz9)XC4-b$`}^1igZhSg!5b6*%PLO9BWx{8Q@oKuZ zx(>IxwG!xNT@b9(u9LMkxq9E@cwKm1dqI22-aj1a&UsfV&NR)&PGKK0&23it13us& zjkVag5e1FGjbV1_^Jafq7}$4n{NxRUE<5zmdKS5&121bli-RG#HouWi@uH4#N)ydV(yX8 zkQ+>sV_UXyTFCcq?z!sTe%&_hLqYx+_{!nRD$ky2*}8rgX<3!pqWvdz&ZcA8vd+%A z(6iXH@*mzU#(^#tVw3?w%>Z6h0#;WYqPC5OU|GCt=)AF|6|QYb^_tnT)v~>*eWH=N zQN!j*0LQ?}B=4$MjBAu@Y)r^FPP#T{183J#yzQIK_r_D3$<@xfx+d8=9g{_a$F+^W zS=RbC`quuN0SBl%%=5V`+o5FB(~~TdjtR!;7$6>#@`oW7vA zkiV!B$l!k>QYYf#x6F#mau&cB(D&r=hU?(;UcWZ;h`rX^TiI=`5`Yj&n?;u-2}5A zVFharZ47<_aRYA+%?P&_p#<5Jz>1I-b{+RQ`uy*N>7{8GqBBB52qCif5Zu<-0ehJ7 zw(_>=b_Loq8VwpgDUk%=`+@j^#Dln}CQa!SV0W2 z80pAm)LZTa8V#&f_Ai8NUSlULkGnRz(!Bw6@fyze4`zP)q7dBzdO7=s!yzG+8U z+;rDI=wy~oGg7!RsCJ1nGYXzZ4fz$I^YM6Mm;AwNTxXF=c6v zpN{XC*^ZrTJ3pO1F+t@9as)CO{&q;b(somNC_Ovm5#cFwZWaFqgja4#A=@DFOT^D? zqj=QS^Vz6u4|uJAeS-!-r$S4TKgaVAr_vm#7AjgM^UBH=5UN5JR%Wh{Ma7-}A-jpV z85D(lQuhD8o%5t{T}<6vTv}`oIH?z#y)Tq4)KA@JHy(~QABYf5J&EW7~ z$Xh;JUOgaju05z9YshMIusduo_G~*;JvbcyXz8yoQ6JVvn89LaVAtWc-%hv$c9t;| zB#+40Gr|9;H)ZM6YHxOpIE=rS6-&aMJcyTvl+ZNONLSL-#&p*!{hn^Y ze~o0tp^moP>O*2SMwYX4m3+y?*7@k5mBCDBq{(NAcEihDXRh(ts_i85y#M_BYW9WH zy5l=c7qOLF+8O=kKg(nj&7q6_b#p5pYpaKp3vIoq1zM}N2Zg28MkXJ9<4oE0#<#Jz zeVA_Wfq+f_vDfQ`o|GVluvg^yHM60-O!=r?3@fO52G!U>){PEOXtUu3{^z{Wz!m;z{On*oAG3QRt%5#_K&@m z*~8^SMLpP7hclz@0kY&*Mvfm3EkFDRUPiXFJNyOLPG$lA^N&`>>Ha#ZFU1cSi=vg8 ztwerrYe*L{qg=aojhiwpu7?l7tAy@Q?pd4TT`woeK>g>-@ixU~{RiPkkpZ#UuzAo< zZ|&+U+I`BqlGG%k66kyk{@5F^zu6RYWAu^n23(KrXpcL`9uBpVwypRPe|R4v97nxh z%n04RFFnm{?<@L;zIzs!|4=-nTP_P&l-pQVMx!Vl8l(!UYpUTIHT3A3JEKOo>Ig7|jU(ExZ z^bQHJodxpe98@>ea<}kDVL*T)pFyoVxJ6c}U=VEx>?ER2U!sfqeZS`xzH+u=7htzZ zK*?+iy7T-4+L$2LFxyGO17v?hnrca#$;*S$g37R9P~doA(4Z1HC<%h&|94peoEi-B zKkX1;U}2VEQ2#}v0Lnjqzd`Af=YQmo@u6Tapc*v6HE(owJ3#%LUS#9H;@#K}y>h3=E6nGl5I1kY9o1FIcK+xoFAD@fzFPG8&rL z8<{eC*gAa50ps`J1r=>gT?|P)Y;EkEc|8P3|3kqGDt|UJk&^s}#Kl^GR7+loM9kjF zl!TL!nUR@P5T1mDgx|@;j8{cm;=kBIH33o!7Z(R!CMI`xcSd(MMtdi7CKetZ9wug1 zCRSDk5Cwy?r=5$T2ZNn6+5ZsoKjnyfF6`?)lwk*Jlk6GMC+3%gV}T)hhun#72ftIN%aL(O?k+ zT;%El3`a59Afmy@3`n}gN0}hVx?!0iz{$GB@tM1S2>hoeSO1F4{7Hh!3?Ww!4mmde z9|jQlCD{K@{(sDj@GI$W7Gnuol|6J?Cu!x8ECD{#iHwjLb1B8sh4MlHZ?~6D7D~XT zlA{D@6{NmG$s{TO51N|tWubajh97Koj$@>mRx@kmje!jzAsD#e5+urK)atYmtuCh* zUN3U>l~-5NAkJ_~2<=*v-1dJrs#*M=N4+5k_4b>3yskGU?CIf_y2mpGc^kek%D*yr zbXV6~_fgVb2g@ow=MvWP)Tg%`L`js2l?<)s3k3=klQ-HoiIW&|O*ffXcKsl5Sxs`n zh<#^NCVPTjjyv3oo@j{NJ)I6G#4u?!fKK~#DkU=oaE_=6j32MJD4Bx3k$$W7ab|wk zJefSVMOhT{gHhN_BI~U}b}gN^hm+HR|D2B(@2!uI;>cXyUp+@Xh2okiS4!tgRnK`f z(;p))UfWyh#<^jPqJ;Y-IoUGbw&CH0BTQAtm`A=0>+wWg?eH30MGXi<)%y9)as_DWV z1Ij8plk7SwLJHLR75NwUq<&!7mMxqY%gmn~ycuI%C1O*hIU@IC1qi;usQx0r4*P-7 zm9DtycdKs0)N5`H`6ZSF(YqiAo6|)}|M|44%Ow8rdZJB7d3?00dhg|K7sYabX_Udz z@>@KEj@t>BgGGN+t@9yr+T92OT5_zq$XhCqi_ga4!-Q*6JlzJi1tLP2AFbj&Zcw0| z&9oNh!tQs|uIJgjIm!M==4!nC({FH>i#2Uo03f^>jp6Un-SO<7Z@ynfvdCZx6#bPc z@-|-F5f1PKbT}9qLik*~l!_&2Q<7a@pIGp?%z|-a*+G9`8mAwOPyrsKE3!;wq7|!d z)oXoBqyE5;7ne|qaT+78KY|`xU;;vp<*ISb3d!P8Ssxo6-WK;GnDu7kGQ4hg7Jq#c zX>@+s**jumQY$gYMz-1C0Ni_S>y!3Z;-t|J^#ytHM$7gZ*p(II? z6^@0W9TXPRD0e`>=k{k%h)t!7Vb!C8?nEhy@3JN%DvT(7Kd>(xtO_2J4qw2p>m*S! z`$!}N0lTKY)?kAmjqea&7GM>5BpwwDA&5MTNx8B|`gdO*YA3?NaT+CJdEdw=Njniu z3}$M%ZbR&Y^PE>@UzSNVYUUPWaHb~+hTbxQU^tPE>;8mq3Zf@T$gikr!)H7`DUH-o zvL44u4NU_a8Uc$n6xH)~pA3;tfj~%M==C{RR_pS$ud-abF73EX<6Tr1n~8nbYpdAaU@Oienk%rTX{l_a7PsrG}J=s!O&z zy50HHphKpgZ3&&j=1R^0Nu^y}npUGc_&WXdc5-%5a=ZK8w|;x^K6q0B6{yH7n#||b zzeSQ3DU<0aR!@SCN@8YwZgb|s4)!`J^d?-VYgWYPa$>Y}acoK!A;jnQl~BN|UvoN_ zEQ=)q3tBMay;3A6k*Pw|9hUG_$rE+i z@pLpwVaS}Vh84z7g&Nn@hn+sPje~NI0e%_J+Y zK=a2X3;|dsRqnv7i=6ikf++C>3w<9y{`1YI$r^_HV1@c@|Az?2=LfzEwK|^A^Q7Yf zsA-Kdy&%BGeB>4h8Nt8nu*-p=o!Q@{)1-iM@<3^I8JTRqUUGSD=wOt+;_aXxOvlgo zCo|*n_6>WU{wrS%+1e-K3niIpJ&)q3=miV~Vu^-^EKYRnr+yKx-l_g_baAk=TZyi0 z>#Q^;D#Dnq#!Yn1TNE^T!jTQ5L-#h-)~El4;^rMJlSFIFx!*))g5>}!E-#r*qe?l= zheI7YuuPEVr0@$HsN1r0h}tH7*hb0%h3LwUW)19k*;r8P7~Ho@KK8gMB2J$Ecz=3}?^J*^REs|ZlFfvt(*Zy)@m5gL3SR;2|L**U%R+A9Z{(d6c z`4=lQgDUGn{>W95^is}qR%;qlTDt?0m%O2>y*UT=P_qgkLhSq0nZ+^R?|pY+j|(oG zh=KI7S&U{2x*l*T32BX<@R$JH*&46yYaWi*1p}II@xbpTJ|Qw+EVl=)NC8Js=b=LP z`0S14;(;>Vzi{TP1K+3ZxXTaX3_B$0X7|y=&UD+{W-1B!5PbpPSo#2RM?(9t%mopR zHZ3=EHO`uzEeB|Bv71b~qI~^f2I)_ue|Rtw(egXZ5nSK5#j_w`*CiALKA&{01pg{?ZD)qVFKZM90u!^UM)=T5HT4mS>(A4+a zh?k6Y@qJ>U{pOP2YB2Y+Ad|1r9z)>t$Dp&t)X@Pd@Xi`-x_;T#<=mC)XnYEd1-sVg zU+mzJe=Ff4w;R3S?Iynvr_HeAs%Fvkf@YCjLC&H)9SYkevv1x+3{Gue- zC%7c)le+sd^qbZ&(DW^a5jFfK@)O4xiv1?H!VPB!E4yK}P)ax*$+&*Rze3ly7Vb8@ zb7`@lslT#;M+tV>ekjQ6xf87Q_})mNSngHtVR@EeeD+nSw#2@JaUpy4^zBN{&ekt% z8pDwkFp5MI=TXPoT>wdiT9k1ozLF%w8I^i14I2t8n2+;b#_GKwH=V1ML#f@%9A&kc zTStLJeB4CUr7#t95@_9$x4-Vv@}yh5a}VAPas&vxXosFqe*|4^GpqXW3%j^MoPxC{ z3NNjWXh|`yi}qJyNnULD-dk*$rA$^avV>o27@Oi;bB4 z#X|5WC4w`c$mnu!8I^ya~=1NUM-g7|Z|g#}J+BwwGcnMOn<0eGm4H`);q) z<6h2G&*yB5P%iPXA7yjmD_xU=>QY^kD<^RtR*Fi%QV^<^p2u2?^$Ng(J-1#2)?*P$(D|4bb9;N2F*X8$(m>AQ0eNcz`d(dn zla%DKXz9-tkp_|PU$i=XZl+e|g){S%!_$Auv4fSFGRqbHV5}MV&DWJ;FiKWV zIQN~-YE^+4jze+EkP8?xVsX6lz24+nv^VgVh0I(yR^+uL@#VIMEs1V<`AXwGO79i^ zJ7EG}egwxfrKT;IgONyminyicVn>qsb*)2mZk0`)Nw_F^R)j`&#ZViMi2?B_6gojc zcdG>ahUM?nS#pe<;{(xJn|w(sgfNYYReW9zZKg`3#ccFI!>7!hw?24r-F87auXl?f z7Q@GUW`Ac4enmN`$>$SECSDZAAyVY5NW%Mwf9 zd1<&do<_Kyd&hh~9bR7V-Ub*wDKp?NX%b~Jy>cY2TB)3-o~yi6!*s?}Eba7;3|(YY z=VNChR)@-8LTWgYALUH?bzTMWCrmijzz`z>_?tZ``f#i#%?gdab~&Hib&ZWw+yvZk zbKpP)3LbAft7)qRadd?(vkr1Vf`c4$DMY&MozI%MK5tBWb=5Dm9E|tWutm1xd2GwWU*|NwmPrqWs%!sx!bHbrfcpz9C*o0MLyJJUWv5U=k_Q|W zc^!K9M|~)9X2VwtMQ7{|oAL-OS%+lr*Xld#x!oEP03Sxa1Es;8^#gvaJJ6P{v7m&! zu_zAs(0Nu4vn4(_Or*U44(Ty(0_!$5tz}7mnk=bB)h#0unamM9hJ9BfF%ED^uBOqX zz$3(J=(GqqU7!11ttF1R^w#^%)q)N?eS#f=jU=%;U%E&>_N$aBYwXk<2O zQnoJLCGL?b)~#;GEB-%%I)$n4NTx5Hktk_XOjNk;@ukGFqr5}W7ANFb(#?; zJ;^p`T0h@YuldZ3TQ@<=8a_sioC1kyDyH+KX-TJb7Ak;eTq_aU@iwykeN{8@O#Jx^ z&xx!$^?sU-9~-#IL=(HW)*og(hb>jWazQjO3p(t{I+_+ z;5zy9pKu*V+U{kmwVz~P&vNBV8z8tc;zPX%2qB;3b7q=)WLY;g5C2u^Q!deSC@*a~-@3i^+ zXXEKHh5LTeiUO**wfP-#81X1U`*)sYDsQxN%!Vp?``cVNNa^G(@X%!vY&-wF7<|MCP@{LJqqfKZl>o;4=XS1bUcCIx%Q8DCGU#)Q3#_ zr&NvXUuf0K@C#-2?5Nbw&dLe5q@XBOuEiT$gUIiR(6X+3ybV*yRw^WV10j}vS3Q}w z=ygya@3fcri3J^X+=VQW!g)Elvce=9<|4KKyS+=*>;Dw*H9|q zj_Eak4|504|F9dq8Mtceecz{gk?s_Q+)xUr2u4hWi*Pk9zz6`p|T)-=_@KRzWP<) zi{CM6Uni;%jz>bEblk55`rR;#8rH*Y$*kYs$H=bY1eM%j*BL0OYqHg@cPIv!tKuzX z@hX4K+o2=J?1t@?*bd9Q6cfa8G4AGg54~<=` z-KR~tOvRoCGQw_r`+b0REyuzI(G=V4R^HybgOYjn!SMaA@Ca%fPo?ZgwlI(55`%-V zLZ*H@5Vnpih4?$ELLWv^YeqZk@;*wWMJ9pMsxIVA4u%@iaoj6EB)j^6j zhd>AAdkEg;8nSB*Ng3RG;08W{aS8s%rEVV;qo0Q#v`6;}I59bujfpPKgb~MBYlF3k%93O| z$`UCwIZ9^cx)N&xI)k0YCW|-(v*DV>A{e7yMm8tbz`B>ZTG zfL_PmUpG#NP_LY8E8wZ!jjk6>b;0n)$u=ua!FYM4Q>(@Zv_cb|$~c)&XNA$R7+J3T zQu9{x1I6W;`zkeWuSLZEN5u(^CT`)NH~0MWb}%W7~KrQrV<|E%5`rFay zJgZziEgpdxs5Mv;tdt|gvqheob(-?kx)%(D1PimHcKaD!RUxrS7fTs@p*h53HVl^s zyum+Oypaz4`w#9OjtWk>0voVv1VYriVV}PLj96e&$%y@rCRj8BU<@iZ;~)rxkN&9- zmGb`;HWDKsgHm!WZ-MwJpAfp6 zCP4yvKAjN!gKM8IiT`&s!!2(FJWfgnW7_;>+0s=;d^$&wY$R45zvqog5`AuPi_^g5-LNDv_rhF(wL7gk@9=dye~sBaZQ6AYJMKOi z*>8u>POXtqwdP*P&s8g9$Ycw4oUT^^{vr7C8@C!0b&0q6t>dM=Z}9!Pzk-t>srVe; zLUy&Azv)O)o&t)?tjc+wT>V3{I2k5!D+p6aRe!6tp?PcdtH3=JUJ?Bn!D%a?IN5f* zY!0?RkvSB934-!np02)upnRyew@0Ca7Uzkzzvm{oMT{*_h@1ftg#QJ_d&^&QVb z^}O3kD=O^eUc{=_>`;u+rdY^RkAY!Szvn$gOAFsauT&?y!jaVEg~ zL28D5(EQ6P&hF}6Sa@>jD!<6EE#`{B9JiQfv+Pd8Gf#Vw4U3<~C<8`%#9uaN+xOyu zSk6Ut=9B2*f2+PSssyu|{k1TYw7>ov*JrlI@Ehbn8A+u}woQ6_c~~xNqcEzaGoOgU zU|_X}WSP~iQ8C3W_=G=K?K{l|@4F)~sE+z0cD-uN`K5pZq;N|YJ5}NlDuWH5*mR@) zro?KKUGCe<MwtjnS}Po(_@|XC*-jwvWrxW^xJoVnMhrgMe?B2 z!A{JdiQI!QWM6WjT-)@I!FG+ywJ2L9%r9ag1*!oPQ|`di$&CG-{Cs-vi!~lzuf6c- z8;fXs7dy6yZ0I!(P{XW0#P$+cFD&Y#({rwwbZ?zB7*l0i# zVZB(+ilyVh@WwzY!TO0`sLc4Ik_BRC<11oY7^*6awT;tyyRPp)YHT5_PYb#?t1$jD@_x`>Wgtq`0U1DB}b7c0h&)3 z2jmzP55+nSc9NjY-;_U*`o`hgWROLH6sMEEDWa+CoJrvea`VLGmQhTTq1ysO@{(ty zrN32hTCS|I-U+(iko(@$)G-B#g&|LGGvCeC_E?l6pUu%SOZMG9+0IIc*#jDXK$lX4 zejNQf2`uQU-gQOKw8*_2kVG%d?(lZQ=hfeW!{1A5sLWY$t*njqQo4L?*_;8n$86KP zfxpI5#qRSTHx?&=Dq3B33=AKZskVet;<0X{#9e$@i;51h8j|FuyK?n-?^Db7NAp4^ zPABupS$XSi?x`)KiB-acyaE=3Xx+bUcAi)|sboyHEifv|HHH2Q(jAghIM;w7nZ$D6 zFxF#&C{oDGPu5Sd;FDfFI?y?i*d7wWL_K^mmnfA^swQ9AXE5PEy=1wNW=mWOJ?-%(^TcYALg5%7Lbv404s6(2Xcju^=8??p5|b5DnZyg zb=9c4roQaw3oO3eKb(Qemh`!UWqRf3pV0Ue3qqnoU3+*lflGx{IbC8uxfHC#`ag)8 zOl4SRK=tV-E=1?Hixu+u|8ld2nQ5qO6&IX}#nRH4X#MIABnQM($ZFs*do&mDHYWt@ zNO$#$pJJEP$Hhi0?DMRoO7~Q%bFtjqte(sSJDzO3tgP^w2jpQteG52NRS{I)ZN9b@m zWL&qYGCyCLub1MkF^nJt{q-2N#84LN9>D^UiV}ojxDD5is_$zR?x9nO6vfM!u)i>h zeR;TUP!&seBP_ltP?w`-@#ORg7s6Nb5m(028?LqhKkz>3h0rcJ@kFq+!!QEvCd4$0 z`9|)H8_-GQ#UB(1jetv3rS<~Q?w=YJq`RwitR11u3fJILhFl1Oxl5XABRVe^X+Mig z82M9B2=aq@5IWn6^F?RE=r-`jh+&4CmPU$I0= zR@UifkLW9pklb>{C&O(REGBeC@t0sNScp>;8mfN4M@{cZU+j`E4pql9>G@iXK)wgY zbX3R1GrrhN+|R^-BE@Vh4P>h0wPm&>{(^)NfCeP%G+=8xJQQ@deF-ND>G=nVB3Mq9 ze!ZyYm*sZ8VmYYl{OPbvf%K-?UMgLv(9Fi;w4;r}8>-0*3tQUgO;y|k7i0K`y4-9i zFrLYl{y^s8p(C!v!B#WsBc>-Yooe$)6ahOoLkYNZ`bk+PeOD?$=D$9OGXuLU1N zF|D%o*Hr04kM+c>{rneS$wZ>2&dt)X#V6*2$KJXeZ{O3-6KK*^lYfjR7vFm#j7bT= zL_$6VEzwChwWBW$jY3b7>ZZ|ZtZ$KuZ6vg6qy?Al*M}ihsxa40u1GygvI9-s(pPf@dOY zqUhZc)c19^Q4DiqgrGQLRdFzclD~F51!>;?^5$KKg(YUy!tTp3*Ba}rzuiq~cZ450eG=UV&4i|A0vREfHe)-&` zJM7!QgBpaQhD5_>C{=QugC$$?-i<_w1ZS?cLSV8IUQCMtcZmkk*Vo>1gwIUS|Ryf7RT1G*j4H3#lDdh$7 zIQ{#!9nueQyBo_F_WhYvOp62THQms*4F!xFV=GvTTXUUWE=IIQqGXAI;wGt5x}lb(B^|Xz07b-g2C6 zV#(ha1Cc!CXp*|c%S5V9&mHhWkKAOja>arfl@ke#WD^(ly`|-E5Q{i~SEr{E@ktP*PRFL|C9bP448zPVUh;FsO#>)yPXh{FrQ9V*zl(+$n&4+ETdmxl z?RtKsv3Vy^$V;Sgw#8I-TztW8Am}@vUh%|x5jk67zKxm$N;JyVCrSPRqEP~%-_z4M z^FT;iQ9kU@X72+!EjX3u-8Ip-PLu<<*_?m3ovyU#@mUMuSX@i3wlE@pu#XsQ)?v%ADRwFpo;3y#n*h{;sQGVo38z8 zRhW_YKh{v>KD@fNTHC{tLGOEkfQApZ)h$oLD;|xqh@(<1l$$G+_bye+jkmtU!a_bG zd|B?mxfk7HhHZU#M#9taD*4Fjcp^^7WxfnLfutKz1IqvMB!2kb$H_w>o5V@4J(pE$ zBk%**u==5}kVvpOT#GCi8V0iHJUkpPzDj`QYZW^f)>z~tCeo?{?wsoIWleJnd&Etg zG>&=*Nz0`pG3i8jLafKmjPfDtI{S<+>*vcfnlH|N_f`k9U*?k?5;}V+35pEHhN09}$b1L-=(P!N(}7;zoXp{;+SLrSTy(xYa-q*(0U^<0~> zK`6lfpy7`qa4hlgq0}0Eg)jM`L;XV!$&GIQA)WBKw)5uN%^@j4m`LOL1#K0lF z`TzwWetg)gZfb!7Qcpjl56?5ZM*fL;l}(PX9DyXeMp!$!olRnTb$s_BSAL9{mFVVtw!WyYL&UoMtz)MDx0d}4hGZ>0jLXE zr%ph=OFM-MKvTi%zgxO&zuifp>7wkh2kpViDZBOBR5O3}b#uPIVBk#{Kc`wd!sI74 zTV&Kvu3d-|=6T_A8xD4Y6F>6k^mBpJbI-2Uma7YDWHI4wpkb`fXZvLr>h}uM0@x*J z-H&G|ctPMsro51tX5|n4xDKCXuKJfI{eoH}po))a>vkyj^j<`he>G%C2qx>9M;?a~ zPFyhf@K;=1(|pZJEeTnyaju#0<|we>1>Lt2pEPs|nHdD8`-G8dSU-O6Ldt4Qa&Hav z$p_!Kkt8~McJN$~05sT&08&M9l6p^I`p3ubrZqqUufTi$b-!knUV))l{L#(;Lg;t| zZ1956ci0Ml2|HTwTvt{5XV0Vq27qdmhHbP;`|e73Zcuc+DoB=pJSyNLdQA;N^t5o} z?v@@!;FWs5Y;%g*^=wJmYqw!qVx{C$mh}IWHSbj?8~~wKs%BZD@9A^|n&S9zeV|E+ zidwa>NZwK$Sr`o&S)bHP(Z*Thm2*Rz(6_T*VSO0ThibJllP-)<%$ z>+t1DKOwrF2ZROT@;V>XFQ0J{%H{5VM?Y&D{TNBEI%-K)EmD~IjD)+q*gU%07E2|v zpGamRRxbgJ?;2Gox@S_#l%xANsa4$wmY2b~Q$el14eJ9sSm4fY04C-fB%{ zRB_q9xYr?o5pSM`?G0SGNCijbyXh{vH3jVxvOG7qv1C!dQ=&TZRKK z{>Baw=OI9|0b|WjC@rEKky#6lxcU(7C-k)9@hn_MFRQcNi^guC6luCKoIrKdz(PiU zYk-Ih4tvf~>8t+)lgOw$m@}ro^k+|eRaSCU=!u!bW^So?GOlTZK+>H{$K@99}?y?wfFsdsZ`Qw5MMRCVub2Jx}uc^c!`FB|O( zT@;)I?fW-J(^q$Jln6~Z7(F4>V|f`Yx1k?hjV$;9Zv)V9I-}3$mSo_6y-amC8@nmX zsSkIH9qFIe!MU)Z6ErJ2I@vRsF#j84!@_z)Bc0dnB)&v)J2xG!Dpq}AJaWpZkG|1Z z66yB1#qhmNX(O5xL$vH?0zxh5(BX~nXQf0UV(j1`eBLFI&tTrL=eogPR9*L$-uL+X zGYt6*swgCh%d6xo@ull=lFLT9r6S47#G5jJot`(cDZY)M|gdGwy08bPK+PROqN2hg~k>-jAVSt!(?@Q3%-OFQl=P*q^;F|kp zx~KqXH9_R%>_pCP ze*B9x_y))uhhoaO^cJO~rysker0?9%KVMQ{E0m90QKlfNUvL>S>MxLpl~SJQboVVS z+SBfF^-xKm;;Ml7h^NtF9y?U@FXN4|vlo37ni3iDa^Jys+3LQGmrrCC!VlK?PWN~y zVj=JQS9CP648~vKO+&xczvB`4eELVs*%#OfE}E>AJvbSbv-P_4jqKM(i<`(>35JgB zw{(+_`()j_?LWiu40I7!SH0}ap2c%e_Ef{GGAl@^fVTWKWn@w7lc8rM`yv=(FY#Ih z&kx<~^s@FR6@OYiV6gK0T8lHgQ_2~wnaOiF|H41G(Q)mi_T=ndP+*-_!>^H6=G)~<# zJ5B2O(qB?>xMEbyaMpxnORvhu6F;qmv@_AJe}d&^Tj4?arEipi(7`s_@A@Q9d1EUcrLNIv@- zd5&*|i)cbY4D~w=s3-!UJN7&A75+g5hczw_f6IVigW2Jo_oe@DJT6PCKfAD9<2O3C z3WzPz7UPUoXNwv9N<>1B15fV{D9U+*?o!L|uW`F{nz?R|jmd6=33yitTU`AE;5N>% zx%ZHBrpu^+DC7n_;*6Jh?EZAND8ufY+?mx0L0jA-?*z)o?@ns`TyPct?!G)|5@yAl zL>*?jbL4T_UM7tE!e`4+$mFM4z6<^*q_Ix)_wYo0)N(q|v$0Ir!Zy$8y9jJWtli!> za3Z{^pk>UFll(w!wj!&xJ^Sghz2?a^^7;emVYU^(2TR@_9JT&W&s~BIVM)YBG{_DmSoijrD%bVJ|Z)Cf`!$oN?FJ9etl1p!8n$Ae}cBtG2aoAlB&?*r7!@utD zGUeh7#u7?{B6}ZG#_Ysj3*@AUNU$ayuR*F5g>Ew%kASs3%&7YFZRrg;8n!FAt~^Hj zDC6n>+Ibzq)qY`s;s6T@Ht7q4;N2_s{>07fV8K>kOW;6(6o7}~G~C3lSHJT;KMQ4h zo28Fqm~Gl?N$UIlOI;HG6?I6N%fUpn?RX;s`MhfM{PTcgM);NpEodIcw(g*!*!0x9 z->Cj|)t)_0mwb5)E{oZ;3X(`(k?x9)rS+^1jiS`bvfIp>edk8#I6mAmVOBL>U)YqF>o>&zyu$hjDwJ~31#Zu7 z<2|3}XjXprzNyvj-Jd^upr~yO(9{J``N<^H7;CRr>cpFxCa|;xFVMTdGzdU$c%Hh> zt`{e?kCWLyQhyPA&fsdj#vV)fb8zk2;r9_tKIvEIl{U2KnX#_FwY#!vG3q;jT0E8J z>l<_(75m~>i$G;;5 z_iwc|P4NsI-aO%@GthvMURmB7vI@%9<(suw3A^;x9i*>suNggyE%XG1o$}jszqCgd zy*l~jh`y|C>H%3zNATodS|#&@DM`&Hztq4Hg>G--1fP8asq9-GanvhEKk95S(iOPi z%xy@!KcuK`|H}FM>V^)InvznZ^yc7+#RY_^4^vWl2|Wh7-H1X@i>nHG#Cpu)I3ML^ zI(2W5d2OwaNyI319XHsxqHtJG%TyWpe4(vB7++rYjhVD}R(U0IPu@OSpp7>!TZ@JsOf@a;0q(#=~9PRf3{rfQuY z=cr@~n%BOvgyOfP?rgbUc>zT=o_^0)ENcS1V_x4X3e8mN^3|WCPb%$?sF#>ktYiro z7mNUU7gy<%ce(WaAURVbFQR9ZwZBT{ZdPp4JG$y~({n;yma>)c(biJ?Q@)JprR{YN&QU-^}iB`xoqMf6eFnobTs(KHvBAePu^kYU_(WF3^=T*cVjm z5*I(_X&uPcTF`#R#9MeUXGO={+m6|KDAn1hvJ|HO2uzY8K35EO@F~R`M{iem60$|6 z+&8jlANDT-vzNYtD*0gdQ__TQF6^c^9>#6$3Tv`Oh9{qX+O4u*&J|f8{Lwb9Vp+7} zeS4wWyRjo=UFPiWf3|pf=jY&?n7fLbWn7H{oF5E4y=9d{X0Us*1`E`=bCN3Y>q{#Z zEn%&9@amPqYK2hx^8{qJ&Y8>6&KAQT)@ktavP|f1fg9WuPQQ^aLC(IT-Qw{)i^-lOg- zcJ381-txWUWD>po!>0B2DxdMvk#)pka$j3j{Wezd^7~BFe=4DICe{?Q>1$3g zNSi=)cXt!t?Q!MK-Jmf=X&#%*4Y|Q{ww@OsAkzS)kn*};gSn`c*B`3XPz1{g%*fYo zEPq6B>&-0SHI3`uy4w){`K`h0XK7e`FzJ0yXs`1JcYegh%pu>cOnXPx2HCQf!{MKq zxl(|)4&Eq!RJ{&z(JH&^dkE`j zU@&=9F)HC&VE;ztBBNZdP3=P_@!b6QfIR|1qbK5Zw>y_R?gq!n?Mm4coqsd8hnLVg z|8|~O<*~y5zG{84hTBryxmQAZ)?UK1vul%L_K=~6r<{sx$Ae47f2N$SVG&toRXI85 zan(z)J{&84a)PxsWDps3Gg9#M;}2C5%DKwm^~-XH*fdwx&PsDRNrfeqIG0w;%k*PX z12-94?qf|+c?_D3CYZs<7kQKo%L&i>xdDGKuVNn{vy5uS$=?H#8>tWD?qYfB^U&UK zl``-HTnVUc)*yww@gA;aGYjJY;ozB_ZR5T59qh+LocRTqNR5Y3KTu%^Doz%G<9R2h zRRc*}00fMJ{!KU>4NEdcCkjL0I@MxktPO4%>zl4p2q4n{XfhkgtdxN`b{rs{tFAP4 zh*uAQQGZr-ifi;=jF=L6-pvMLWz<``Fs{IDA}0+Ah&d9-?5mt92D&b0!jefYu?KWi zfnO@XFK%gWI50nJD1jcc8JS?70B*Sgw^FZ?YXQT*UIC1Ma=$Qh+75=G*}-UEsSFNz`_LZ_;*#$S&mU`VKH7HG8tB9+A<&mkXeoxG!UZ#Z zDf!I`8dHZ4G0HR7enq__=DomTF& zdU$WR@R=ZA9tEX5+AOJ5hIs9WI>k|ga-y9yj^S$mWRv~4yIVdN?IVfQg&;@<5ah*}>tH}|F%Send4y{K z0woB7ArX@fAP_1I#Bu$=S4F|ja+c86q*e}6_>zuWq=owzvkUp05IQGckeBf!30ko% zN+bLjx8uLQbH5j1HG^`|I_NmlpJO_d4y|tzU!lP1zv-#lwB;Wh6z|G6%5SL%sEC>i zsKOM!^GMjUj?k!#$(L&}TLJ&jBVSs)9_%Cm)#CVJVDyRb1j9R8aQfS?9Wk>5r^&3f z^f=RQeV2{m5rUxix)fB+zDIEMpJBi2y^$*P2~1J~>A+$JRJlYS^He|)X$8}SZ3iTGJxdx1-1t#F zDMD$i=#)Cu#H3zXgK{XM_v2k?*|OC2ZhL8{(ZpxR%_v(O(rloF9d;~r>>O!z_abi3 zbosy}XH$w}bCK4t7cK@y4*;_|)Mz~sb6!4Uf~PoH0bW4#G?gJp!43*YUuiF)_C}_g zZF0!5`F%M$k$04!mIW�C@3%ug6|-S7ZMsAFtiKqbNK07YST6hsu{-y%6pt`qFaB zuT!!|Tv&6620zBRn>kOumAz*I?fw97|HA>GtYGj^FmyO`8KdF{vj7TlfOEK)QsFa? z?|Ts|9AxL!&)AC(#hr1;DZ4-|p)JOQc(T>LVb7m|7@*y`M6X(#^`-5yo~Y5^t9WNu zRR4qQ&s1SD$=EoXwf>f%9n_6;$~(+mTQiFc$qbc*9LPuvqP<#s8lr;HJaYW_@t(HW zf;v}GIH4yyJL_h?4Yd{iXLqcECL8Oyqmr4P?)q_S5PN$_ak9><&<@N2P(L|TK5BK8 z$Ud{-G(6F#I4B8aI00p8z}vy(?=vEgows`(P68{Thw{^}ZQGX8}zQOSN{dG=a*&x literal 0 HcmV?d00001 diff --git a/flatbuffers/Documentation.docc/Tutorials/Tutorial_Table_of_Contents.tutorial b/flatbuffers/Documentation.docc/Tutorials/Tutorial_Table_of_Contents.tutorial new file mode 100644 index 0000000000..009116fc4c --- /dev/null +++ b/flatbuffers/Documentation.docc/Tutorials/Tutorial_Table_of_Contents.tutorial @@ -0,0 +1,14 @@ +@Tutorials(name: "Starting with FlatBuffers") { + @Intro(title: "Starting with FlatBuffers") { + FlatBuffers is an efficient cross platform serialization library for C++, + C#, C, Go, Java, Kotlin, JavaScript, Lobster, Lua, TypeScript, PHP, Python, Rust and Swift. + It was originally created at Google for game development and other performance-critical applications. + } + @Chapter(name: "Generating your code") { + Start by generating your first FlatBuffers objects. + @Image(source: tutorial_cover_image_1.png, alt: "A code structure for a base struct in flatbuffers") + @TutorialReference(tutorial: "doc:creating_flatbuffer_schema") + @TutorialReference(tutorial: "doc:create_your_first_buffer") + @TutorialReference(tutorial: "doc:reading_bytebuffer") + } +} diff --git a/flatbuffers/Documentation.docc/Tutorials/create_your_first_buffer.tutorial b/flatbuffers/Documentation.docc/Tutorials/create_your_first_buffer.tutorial new file mode 100644 index 0000000000..2f8089f7d3 --- /dev/null +++ b/flatbuffers/Documentation.docc/Tutorials/create_your_first_buffer.tutorial @@ -0,0 +1,72 @@ +@Tutorial(time: 5) { + @Intro(title: "After having our code generated") { + After generating the code from the previous section, we will know start creating our monster object. + We will create a monster object called orc. + } + + @Section(title: "Building your first buffer") { + @ContentAndMedia {} + @Steps { + @Step { + Starting with a new file, we will create our very first Flatbuffer. + @Code(name: "ViewController.swift", file: "swift_code_1.swift") + } + @Step { + First, we need to import ``FlatBuffers`` + @Code(name: "ViewController.swift", file: "swift_code_2.swift") + } + @Step { + We need to create an instance of the `FlatBufferBuilder`, which will contain the buffer as it grows. + You can pass an initial size of the buffer (here 1024 bytes), which will grow automatically if needed. + @Code(name: "ViewController.swift", file: "swift_code_3.swift") + } + @Step { + After creating the builder, we can start serializing our data. Before we make our orc Monster, + let's create some Weapons: a Sword and an Axe. However we will start by naming our weapons as `Sword` and `Axe` + @Code(name: "ViewController.swift", file: "swift_code_4.swift") + } + @Step { + After naming the weapons, we will create two weapon objects with the damage that the weapon is going to deal. + That's done by calling the `start` Method on each table you will be creating, in this case its called `startWeapon` + and finished by calling `end`. + @Code(name: "ViewController.swift", file: "swift_code_5.swift") + } + @Step { + We will take our (Sword and Axe) serialized data and serialize their offsets as a vector of tables into our `ByteBuffer`. + So we can reference them later on from our Monster Object + @Code(name: "ViewController.swift", file: "swift_code_6.swift") + } + @Step { + We will add our Monster name as a string value just like we did with the weapons. + @Code(name: "ViewController.swift", file: "swift_code_7.swift") + } + + @Step { + We will create a path that our monster should be using while roaming in its den. To create a vector of paths we would us + `createVector(ofStructs: [])` which will take a Native `Swift` struct that has been padded to fit the `FlatBuffers` standards. + + There are usually two ways of creating vectors in `FlatBuffers` which you can see in commented out code. + And thus there are multiple convenience methods that will cover all the bases + when trying to create a vector so that you dont have to create it with `start` and `end` + @Code(name: "ViewController.swift", file: "swift_code_8.swift") + } + + @Step { + Now to serialize our data into our `Monster` object. Which again there are two ways of doing, by calling the `create` method or + by serializing the objects yourself. What we added to our Monster were the `Equipped Type` and the `Equipped` union itself, which + allows the Monster to have the `Axe` as his equipped weapon. + + Important: Unlike structs, you should not nest tables or other objects, + which is why we created all the `strings/vectors/tables` that this monster refers to before start. + If you try to create any of them between start and end, you will get an `assert`. + @Code(name: "ViewController.swift", file: "swift_code_9.swift") + } + + @Step { + Finally you can just finalize the buffer by calling `builder.finish` and get the Byte array from the buffer. + @Code(name: "ViewController.swift", file: "swift_code_10.swift") + } + + } + } + } diff --git a/flatbuffers/Documentation.docc/Tutorials/creating_flatbuffer_schema.tutorial b/flatbuffers/Documentation.docc/Tutorials/creating_flatbuffer_schema.tutorial new file mode 100644 index 0000000000..0fcd362ef9 --- /dev/null +++ b/flatbuffers/Documentation.docc/Tutorials/creating_flatbuffer_schema.tutorial @@ -0,0 +1,47 @@ +@Tutorial(time: 2) { + @Intro(title: "Creating a schema") { + You will need to have the FlatBuffer compiler to be installed on your device + } + + @Section(title: "Creating a schema") { + @ContentAndMedia {} + @Steps { + @Step { + Start by creating a new empty folder called `monster.fbs`. We want to create a Monster table, that contains + position, color, and basic information about the monster. + @Code(name: "monster.fbs", file: "monster_step_1.fbs") + } + @Step { + We will start by adding our Color object. We will be using an enumerate, to represent this object + @Code(name: "monster.fbs", file: "monster_step_2.fbs") + } + @Step { + We will add a position object and will use a struct to represent that type of data. Where we will need the monsters + x and y positions. + @Code(name: "monster.fbs", file: "monster_step_3.fbs") + } + @Step { + Then we will be creating our Monster object of type table. This will contain the current position of our + monster and its color + @Code(name: "monster.fbs", file: "monster_step_4.fbs") + } + @Step { + Our Monster is missing a name, mana, hp, name, equipped Weapon, weapons, and path. We will be adding these + fields to our table with a proper data type for each. Example; weapons, and path would be a vector of data. + @Code(name: "monster.fbs", file: "monster_step_5.fbs") + } + @Step { + Now we are missing two data types here, `Weapon` and `Equipment`. And since Equipment can be a weapon, we will be using + a `Union` enumerate that can contain all the equipment that you would want your monster to have. And the weapon can simply + have a name and amount of damage + @Code(name: "monster.fbs", file: "monster_step_6.fbs") + } + @Step { + And to finalize our monster table, we can add a root type of type Monster. + Then run the command `flatc --swift monster.fbs` + Note: Make sure to import the file to your xcode project. + @Code(name: "monster.fbs", file: "monster_step_7.fbs") + } + } + } + } diff --git a/flatbuffers/Documentation.docc/Tutorials/reading_bytebuffer.tutorial b/flatbuffers/Documentation.docc/Tutorials/reading_bytebuffer.tutorial new file mode 100644 index 0000000000..2c4609f7e3 --- /dev/null +++ b/flatbuffers/Documentation.docc/Tutorials/reading_bytebuffer.tutorial @@ -0,0 +1,27 @@ +@Tutorial(time: 2) { + @Intro(title: "Reading ByteBuffers") { + After getting our ByteBuffer created, we can now read it. + } + + @Section(title: "Reading your first buffer") { + @ContentAndMedia {} + @Steps { + @Step { + After fetching the data from disk or network you need to access that data, and that can be done. + By simply calling `getCheckedRoot`, which checks if the data is valid before enabling you to read from a corrupt buffer. + however, if you are sure that the data is 100% correct you can simply call `getRoot` + @Code(name: "ViewController.swift", file: "swift_code_11.swift") + } + @Step { + Now since we have a Monster object, all the fields can be accessed by simply fetching the data. Note, Deprecated fields will not + show up + @Code(name: "ViewController.swift", file: "swift_code_12.swift") + } + @Step { + And you can access union types as easy as this + @Code(name: "ViewController.swift", file: "swift_code_13.swift") + } + } + } + } + diff --git a/flatbuffers/Enum.swift b/flatbuffers/Enum.swift new file mode 100644 index 0000000000..ab5db06c9b --- /dev/null +++ b/flatbuffers/Enum.swift @@ -0,0 +1,59 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Enum is a protocol that all flatbuffers enums should conform to +/// Since it allows us to get the actual `ByteSize` and `Value` from +/// a swift enum. +public protocol Enum { + /// associatedtype that the type of the enum should conform to + associatedtype T: Scalar & Verifiable + /// Size of the current associatedtype in the enum + static var byteSize: Int { get } + /// The current value the enum hosts + var value: T { get } +} + +extension Enum where Self: Verifiable { + + /// Verifies that the current value is which the bounds of the buffer, and if + /// the current `Value` is aligned properly + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - position: Current position within the buffer + /// - type: The type of the object to be verified + /// - Throws: Errors coming from `inBuffer` function + public static func verify( + _ verifier: inout Verifier, + at position: Int, + of type: T.Type) throws where T: Verifiable + { + try verifier.inBuffer(position: position, of: type.self) + } + +} + +/// UnionEnum is a Protocol that allows us to create Union type of enums +/// and their value initializers. Since an `init` was required by +/// the verifier +public protocol UnionEnum: Enum { + init?(value: T) throws +} diff --git a/flatbuffers/FbConstants.swift b/flatbuffers/FbConstants.swift new file mode 100644 index 0000000000..272c572e4a --- /dev/null +++ b/flatbuffers/FbConstants.swift @@ -0,0 +1,122 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +#if os(Linux) +import CoreFoundation +#else +import Foundation +#endif +#else +import SwiftOverlayShims +#endif + +/// A boolean to see if the system is littleEndian +let isLitteEndian: Bool = { + let number: UInt32 = 0x12345678 + return number == number.littleEndian +}() +/// Constant for the file id length +let FileIdLength = 4 +/// Type aliases +public typealias Byte = UInt8 +public typealias UOffset = UInt32 +public typealias SOffset = Int32 +public typealias VOffset = UInt16 +/// Maximum size for a buffer +public let FlatBufferMaxSize = UInt32 + .max << ((MemoryLayout.size * 8 - 1) - 1) + +/// Protocol that All Scalars should conform to +/// +/// Scalar is used to conform all the numbers that can be represented in a FlatBuffer. It's used to write/read from the buffer. +public protocol Scalar: Equatable { + associatedtype NumericValue + var convertedEndian: NumericValue { get } +} + +extension Scalar where Self: Verifiable {} + +extension Scalar where Self: FixedWidthInteger { + /// Converts the value from BigEndian to LittleEndian + /// + /// Converts values to little endian on machines that work with BigEndian, however this is NOT TESTED yet. + public var convertedEndian: NumericValue { + self as! Self.NumericValue + } +} + +extension Double: Scalar, Verifiable { + public typealias NumericValue = UInt64 + + public var convertedEndian: UInt64 { + bitPattern.littleEndian + } +} + +extension Float32: Scalar, Verifiable { + public typealias NumericValue = UInt32 + + public var convertedEndian: UInt32 { + bitPattern.littleEndian + } +} + +extension Bool: Scalar, Verifiable { + public var convertedEndian: UInt8 { + self == true ? 1 : 0 + } + + public typealias NumericValue = UInt8 +} + +extension Int: Scalar, Verifiable { + public typealias NumericValue = Int +} + +extension Int8: Scalar, Verifiable { + public typealias NumericValue = Int8 +} + +extension Int16: Scalar, Verifiable { + public typealias NumericValue = Int16 +} + +extension Int32: Scalar, Verifiable { + public typealias NumericValue = Int32 +} + +extension Int64: Scalar, Verifiable { + public typealias NumericValue = Int64 +} + +extension UInt8: Scalar, Verifiable { + public typealias NumericValue = UInt8 +} + +extension UInt16: Scalar, Verifiable { + public typealias NumericValue = UInt16 +} + +extension UInt32: Scalar, Verifiable { + public typealias NumericValue = UInt32 +} + +extension UInt64: Scalar, Verifiable { + public typealias NumericValue = UInt64 +} + +public func FlatBuffersVersion_23_5_26() {} diff --git a/flatbuffers/FlatBufferBuilder.swift b/flatbuffers/FlatBufferBuilder.swift new file mode 100644 index 0000000000..f96ad61141 --- /dev/null +++ b/flatbuffers/FlatBufferBuilder.swift @@ -0,0 +1,920 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// ``FlatBufferBuilder`` builds a `FlatBuffer` through manipulating its internal state. +/// +/// This is done by creating a ``ByteBuffer`` that hosts the incoming data and +/// has a hardcoded growth limit of `2GiB` which is set by the Flatbuffers standards. +/// +/// ```swift +/// var builder = FlatBufferBuilder() +/// ``` +/// The builder should be always created as a variable, since it would be passed into the writers +/// +@frozen +public struct FlatBufferBuilder { + + /// Storage for the Vtables used in the buffer are stored in here, so they would be written later in EndTable + @usableFromInline internal var _vtableStorage = VTableStorage() + /// Flatbuffer data will be written into + @usableFromInline internal var _bb: ByteBuffer + + /// Reference Vtables that were already written to the buffer + private var _vtables: [UOffset] = [] + /// A check if the buffer is being written into by a different table + private var isNested = false + /// Dictonary that stores a map of all the strings that were written to the buffer + private var stringOffsetMap: [String: Offset] = [:] + /// A check to see if finish(::) was ever called to retreive data object + private var finished = false + /// A check to see if the buffer should serialize Default values + private var serializeDefaults: Bool + + /// Current alignment for the buffer + var _minAlignment: Int = 0 { + didSet { + _bb.alignment = _minAlignment + } + } + + /// Gives a read access to the buffer's size + public var size: UOffset { _bb.size } + + #if !os(WASI) + /// Data representation of the buffer + /// + /// Should only be used after ``finish(offset:addPrefix:)`` is called + public var data: Data { + assert(finished, "Data shouldn't be called before finish()") + return Data( + bytes: _bb.memory.advanced(by: _bb.writerIndex), + count: _bb.capacity &- _bb.writerIndex) + } + #endif + + /// Returns the underlying bytes in the ``ByteBuffer`` + /// + /// Note: This should be used with caution. + public var fullSizedByteArray: [UInt8] { + let ptr = UnsafeBufferPointer( + start: _bb.memory.assumingMemoryBound(to: UInt8.self), + count: _bb.capacity) + return Array(ptr) + } + + /// Returns the written bytes into the ``ByteBuffer`` + /// + /// Should only be used after ``finish(offset:addPrefix:)`` is called + public var sizedByteArray: [UInt8] { + assert(finished, "Data shouldn't be called before finish()") + return _bb.underlyingBytes + } + + /// Returns the original ``ByteBuffer`` + /// + /// Returns the current buffer that was just created + /// with the offsets, and data written to it. + public var buffer: ByteBuffer { _bb } + + /// Returns a newly created sized ``ByteBuffer`` + /// + /// returns a new buffer that is sized to the data written + /// to the main buffer + public var sizedBuffer: ByteBuffer { + assert(finished, "Data shouldn't be called before finish()") + return ByteBuffer( + memory: _bb.memory.advanced(by: _bb.reader), + count: Int(_bb.size)) + } + + // MARK: - Init + + /// Initialize the buffer with a size + /// - Parameters: + /// - initialSize: Initial size for the buffer + /// - force: Allows default to be serialized into the buffer + /// + /// This initializes a new builder with an initialSize that would initialize + /// a new ``ByteBuffer``. ``FlatBufferBuilder`` by default doesnt serialize defaults + /// however the builder can be force by passing true for `serializeDefaults` + public init( + initialSize: Int32 = 1024, + serializeDefaults force: Bool = false) + { + assert(initialSize > 0, "Size should be greater than zero!") + guard isLitteEndian else { + fatalError( + "Reading/Writing a buffer in big endian machine is not supported on swift") + } + serializeDefaults = force + _bb = ByteBuffer(initialSize: Int(initialSize)) + } + + /// Clears the builder and the buffer from the written data. + mutating public func clear() { + _minAlignment = 0 + isNested = false + stringOffsetMap.removeAll(keepingCapacity: true) + _vtables.removeAll(keepingCapacity: true) + _vtableStorage.clear() + _bb.clear() + } + + // MARK: - Create Tables + + /// Checks if the required fields were serialized into the buffer + /// - Parameters: + /// - table: offset for the table + /// - fields: Array of all the important fields to be serialized + /// + /// *NOTE: Never call this function, this is only supposed to be called + /// by the generated code* + @inline(__always) + mutating public func require(table: Offset, fields: [Int32]) { + for field in fields { + let start = _bb.capacity &- Int(table.o) + let startTable = start &- Int(_bb.read(def: Int32.self, position: start)) + let isOkay = _bb.read( + def: VOffset.self, + position: startTable &+ Int(field)) != 0 + assert(isOkay, "Flatbuffers requires the following field") + } + } + + /// Finished the buffer by adding the file id and then calling finish + /// - Parameters: + /// - offset: Offset of the table + /// - fileId: Takes the fileId + /// - prefix: if false it wont add the size of the buffer + /// + /// ``finish(offset:fileId:addPrefix:)`` should be called at the end of creating + /// a table + /// ```swift + /// var root = SomeObject + /// .createObject(&builder, + /// name: nameOffset) + /// builder.finish( + /// offset: root, + /// fileId: "ax1a", + /// addPrefix: true) + /// ``` + /// File id would append a file id name at the end of the written bytes before, + /// finishing the buffer. + /// + /// Whereas, if `addPrefix` is true, the written bytes would + /// include the size of the current buffer. + mutating public func finish( + offset: Offset, + fileId: String, + addPrefix prefix: Bool = false) + { + let size = MemoryLayout.size + preAlign( + len: size &+ (prefix ? size : 0) &+ FileIdLength, + alignment: _minAlignment) + assert(fileId.count == FileIdLength, "Flatbuffers requires file id to be 4") + _bb.push(string: fileId, len: 4) + finish(offset: offset, addPrefix: prefix) + } + + /// Finished the buffer by adding the file id, offset, and prefix to it. + /// - Parameters: + /// - offset: Offset of the table + /// - prefix: if false it wont add the size of the buffer + /// + /// ``finish(offset:addPrefix:)`` should be called at the end of creating + /// a table + /// ```swift + /// var root = SomeObject + /// .createObject(&builder, + /// name: nameOffset) + /// builder.finish( + /// offset: root, + /// addPrefix: true) + /// ``` + /// If `addPrefix` is true, the written bytes would + /// include the size of the current buffer. + mutating public func finish( + offset: Offset, + addPrefix prefix: Bool = false) + { + notNested() + let size = MemoryLayout.size + preAlign(len: size &+ (prefix ? size : 0), alignment: _minAlignment) + push(element: refer(to: offset.o)) + if prefix { push(element: _bb.size) } + _vtableStorage.clear() + finished = true + } + + /// ``startTable(with:)`` will let the builder know, that a new object is being serialized. + /// + /// The function will fatalerror if called while there is another object being serialized. + /// ```swift + /// let start = Monster + /// .startMonster(&fbb) + /// ``` + /// - Parameter numOfFields: Number of elements to be written to the buffer + /// - Returns: Offset of the newly started table + @inline(__always) + mutating public func startTable(with numOfFields: Int) -> UOffset { + notNested() + isNested = true + _vtableStorage.start(count: numOfFields) + return _bb.size + } + + /// ``endTable(at:)`` will let the ``FlatBufferBuilder`` know that the + /// object that's written to it is completed + /// + /// This would be called after all the elements are serialized, + /// it will add the current vtable into the ``ByteBuffer``. + /// The functions will `fatalError` in case the object is called + /// without ``startTable(with:)``, or the object has exceeded the limit of 2GB. + /// + /// - Parameter startOffset:Start point of the object written + /// - returns: The root of the table + mutating public func endTable(at startOffset: UOffset) -> UOffset { + assert(isNested, "Calling endtable without calling starttable") + let sizeofVoffset = MemoryLayout.size + let vTableOffset = push(element: SOffset(0)) + + let tableObjectSize = vTableOffset &- startOffset + assert(tableObjectSize < 0x10000, "Buffer can't grow beyond 2 Gigabytes") + let _max = Int(_vtableStorage.maxOffset) &+ sizeofVoffset + + _bb.fill(padding: _max) + _bb.write( + value: VOffset(tableObjectSize), + index: _bb.writerIndex &+ sizeofVoffset, + direct: true) + _bb.write(value: VOffset(_max), index: _bb.writerIndex, direct: true) + + var itr = 0 + while itr < _vtableStorage.writtenIndex { + let loaded = _vtableStorage.load(at: itr) + itr = itr &+ _vtableStorage.size + guard loaded.offset != 0 else { continue } + let _index = (_bb.writerIndex &+ Int(loaded.position)) + _bb.write( + value: VOffset(vTableOffset &- loaded.offset), + index: _index, + direct: true) + } + + _vtableStorage.clear() + let vt_use = _bb.size + + var isAlreadyAdded: Int? + + let vt2 = _bb.memory.advanced(by: _bb.writerIndex) + let len2 = vt2.load(fromByteOffset: 0, as: Int16.self) + + for table in _vtables { + let position = _bb.capacity &- Int(table) + let vt1 = _bb.memory.advanced(by: position) + let len1 = _bb.read(def: Int16.self, position: position) + if len2 != len1 || 0 != memcmp(vt1, vt2, Int(len2)) { continue } + + isAlreadyAdded = Int(table) + break + } + + if let offset = isAlreadyAdded { + let vTableOff = Int(vTableOffset) + let space = _bb.capacity &- vTableOff + _bb.write(value: Int32(offset &- vTableOff), index: space, direct: true) + _bb.pop(_bb.capacity &- space) + } else { + _bb.write(value: Int32(vt_use &- vTableOffset), index: Int(vTableOffset)) + _vtables.append(_bb.size) + } + isNested = false + return vTableOffset + } + + // MARK: - Builds Buffer + + /// Asserts to see if the object is not nested + @inline(__always) + @usableFromInline + mutating internal func notNested() { + assert(!isNested, "Object serialization must not be nested") + } + + /// Changes the minimuim alignment of the buffer + /// - Parameter size: size of the current alignment + @inline(__always) + @usableFromInline + mutating internal func minAlignment(size: Int) { + if size > _minAlignment { + _minAlignment = size + } + } + + /// Gets the padding for the current element + /// - Parameters: + /// - bufSize: Current size of the buffer + the offset of the object to be written + /// - elementSize: Element size + @inline(__always) + @usableFromInline + mutating internal func padding( + bufSize: UInt32, + elementSize: UInt32) -> UInt32 + { + ((~bufSize) &+ 1) & (elementSize - 1) + } + + /// Prealigns the buffer before writting a new object into the buffer + /// - Parameters: + /// - len:Length of the object + /// - alignment: Alignment type + @inline(__always) + @usableFromInline + mutating internal func preAlign(len: Int, alignment: Int) { + minAlignment(size: alignment) + _bb.fill(padding: Int(padding( + bufSize: _bb.size &+ UOffset(len), + elementSize: UOffset(alignment)))) + } + + /// Prealigns the buffer before writting a new object into the buffer + /// - Parameters: + /// - len: Length of the object + /// - type: Type of the object to be written + @inline(__always) + @usableFromInline + mutating internal func preAlign(len: Int, type: T.Type) { + preAlign(len: len, alignment: MemoryLayout.size) + } + + /// Refers to an object that's written in the buffer + /// - Parameter off: the objects index value + @inline(__always) + @usableFromInline + mutating internal func refer(to off: UOffset) -> UOffset { + let size = MemoryLayout.size + preAlign(len: size, alignment: size) + return _bb.size &- off &+ UInt32(size) + } + + /// Tracks the elements written into the buffer + /// - Parameters: + /// - offset: The offset of the element witten + /// - position: The position of the element + @inline(__always) + @usableFromInline + mutating internal func track(offset: UOffset, at position: VOffset) { + _vtableStorage.add(loc: FieldLoc(offset: offset, position: position)) + } + + // MARK: - Inserting Vectors + + /// ``startVector(_:elementSize:)`` creates a new vector within buffer + /// + /// The function checks if there is a current object being written, if + /// the check passes it creates a buffer alignment of `length * elementSize` + /// ```swift + /// builder.startVector( + /// int32Values.count, elementSize: 4) + /// ``` + /// + /// - Parameters: + /// - len: Length of vector to be created + /// - elementSize: Size of object type to be written + @inline(__always) + mutating public func startVector(_ len: Int, elementSize: Int) { + notNested() + isNested = true + preAlign(len: len &* elementSize, type: UOffset.self) + preAlign(len: len &* elementSize, alignment: elementSize) + } + + /// ``endVector(len:)`` ends the currently created vector + /// + /// Calling ``endVector(len:)`` requires the length, of the current + /// vector. The length would be pushed to indicate the count of numbers + /// within the vector. If ``endVector(len:)`` is called without + /// ``startVector(_:elementSize:)`` it asserts. + /// + /// ```swift + /// let vectorOffset = builder. + /// endVector(len: int32Values.count) + /// ``` + /// + /// - Parameter len: Length of the buffer + /// - Returns: Returns the current ``Offset`` in the ``ByteBuffer`` + @inline(__always) + mutating public func endVector(len: Int) -> Offset { + assert(isNested, "Calling endVector without calling startVector") + isNested = false + return Offset(offset: push(element: Int32(len))) + } + + /// Creates a vector of type ``Scalar`` into the ``ByteBuffer`` + /// + /// ``createVector(_:)-4swl0`` writes a vector of type Scalars into + /// ``ByteBuffer``. This is a convenient method instead of calling, + /// ``startVector(_:elementSize:)`` and then ``endVector(len:)`` + /// ```swift + /// let vectorOffset = builder. + /// createVector([1, 2, 3, 4]) + /// ``` + /// + /// The underlying implementation simply calls ``createVector(_:size:)-4lhrv`` + /// + /// - Parameter elements: elements to be written into the buffer + /// - returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector(_ elements: [T]) -> Offset { + createVector(elements, size: elements.count) + } + + /// Creates a vector of type Scalar in the buffer + /// + /// ``createVector(_:)-4swl0`` writes a vector of type Scalars into + /// ``ByteBuffer``. This is a convenient method instead of calling, + /// ``startVector(_:elementSize:)`` and then ``endVector(len:)`` + /// ```swift + /// let vectorOffset = builder. + /// createVector([1, 2, 3, 4], size: 4) + /// ``` + /// + /// - Parameter elements: Elements to be written into the buffer + /// - Parameter size: Count of elements + /// - returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector( + _ elements: [T], + size: Int) -> Offset + { + let size = size + startVector(size, elementSize: MemoryLayout.size) + _bb.push(elements: elements) + return endVector(len: size) + } + + /// Creates a vector of type ``Enum`` into the ``ByteBuffer`` + /// + /// ``createVector(_:)-9h189`` writes a vector of type ``Enum`` into + /// ``ByteBuffer``. This is a convenient method instead of calling, + /// ``startVector(_:elementSize:)`` and then ``endVector(len:)`` + /// ```swift + /// let vectorOffset = builder. + /// createVector([.swift, .cpp]) + /// ``` + /// + /// The underlying implementation simply calls ``createVector(_:size:)-7cx6z`` + /// + /// - Parameter elements: elements to be written into the buffer + /// - returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector(_ elements: [T]) -> Offset { + createVector(elements, size: elements.count) + } + + /// Creates a vector of type ``Enum`` into the ``ByteBuffer`` + /// + /// ``createVector(_:)-9h189`` writes a vector of type ``Enum`` into + /// ``ByteBuffer``. This is a convenient method instead of calling, + /// ``startVector(_:elementSize:)`` and then ``endVector(len:)`` + /// ```swift + /// let vectorOffset = builder. + /// createVector([.swift, .cpp]) + /// ``` + /// + /// - Parameter elements: Elements to be written into the buffer + /// - Parameter size: Count of elements + /// - returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector( + _ elements: [T], + size: Int) -> Offset + { + let size = size + startVector(size, elementSize: T.byteSize) + for e in elements.reversed() { + _bb.push(value: e.value, len: T.byteSize) + } + return endVector(len: size) + } + + /// Creates a vector of already written offsets + /// + /// ``createVector(ofOffsets:)`` creates a vector of ``Offset`` into + /// ``ByteBuffer``. This is a convenient method instead of calling, + /// ``startVector(_:elementSize:)`` and then ``endVector(len:)``. + /// + /// The underlying implementation simply calls ``createVector(ofOffsets:len:)`` + /// + /// ```swift + /// let namesOffsets = builder. + /// createVector(ofOffsets: [name1, name2]) + /// ``` + /// - Parameter offsets: Array of offsets of type ``Offset`` + /// - returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector(ofOffsets offsets: [Offset]) -> Offset { + createVector(ofOffsets: offsets, len: offsets.count) + } + + /// Creates a vector of already written offsets + /// + /// ``createVector(ofOffsets:)`` creates a vector of ``Offset`` into + /// ``ByteBuffer``. This is a convenient method instead of calling, + /// ``startVector(_:elementSize:)`` and then ``endVector(len:)`` + /// + /// ```swift + /// let namesOffsets = builder. + /// createVector(ofOffsets: [name1, name2]) + /// ``` + /// + /// - Parameter offsets: Array of offsets of type ``Offset`` + /// - Parameter size: Count of elements + /// - returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector( + ofOffsets offsets: [Offset], + len: Int) -> Offset + { + startVector(len, elementSize: MemoryLayout.size) + for o in offsets.reversed() { + push(element: o) + } + return endVector(len: len) + } + + /// Creates a vector of strings + /// + /// ``createVector(ofStrings:)`` creates a vector of `String` into + /// ``ByteBuffer``. This is a convenient method instead of manually + /// creating the string offsets, you simply pass it to this function + /// and it would write the strings into the ``ByteBuffer``. + /// After that it calls ``createVector(ofOffsets:)`` + /// + /// ```swift + /// let namesOffsets = builder. + /// createVector(ofStrings: ["Name", "surname"]) + /// ``` + /// + /// - Parameter str: Array of string + /// - returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector(ofStrings str: [String]) -> Offset { + var offsets: [Offset] = [] + for s in str { + offsets.append(create(string: s)) + } + return createVector(ofOffsets: offsets) + } + + /// Creates a vector of type ``NativeStruct``. + /// + /// Any swift struct in the generated code, should confirm to + /// ``NativeStruct``. Since the generated swift structs are padded + /// to the `FlatBuffers` standards. + /// + /// ```swift + /// let offsets = builder. + /// createVector(ofStructs: [NativeStr(num: 1), NativeStr(num: 2)]) + /// ``` + /// + /// - Parameter structs: A vector of ``NativeStruct`` + /// - Returns: ``Offset`` of the vector + @inline(__always) + mutating public func createVector(ofStructs structs: [T]) + -> Offset + { + startVector( + structs.count * MemoryLayout.size, + elementSize: MemoryLayout.alignment) + for i in structs.reversed() { + _ = create(struct: i) + } + return endVector(len: structs.count) + } + + // MARK: - Inserting Structs + + /// Writes a ``NativeStruct`` into the ``ByteBuffer`` + /// + /// Adds a native struct that's build and padded according + /// to `FlatBuffers` standards. with a predefined position. + /// + /// ```swift + /// let offset = builder.create( + /// struct: NativeStr(num: 1), + /// position: 10) + /// ``` + /// + /// - Parameters: + /// - s: ``NativeStruct`` to be inserted into the ``ByteBuffer`` + /// - position: The predefined position of the object + /// - Returns: ``Offset`` of written struct + @inline(__always) + @discardableResult + mutating public func create( + struct s: T, position: VOffset) -> Offset + { + let offset = create(struct: s) + _vtableStorage.add(loc: FieldLoc( + offset: _bb.size, + position: VOffset(position))) + return offset + } + + /// Writes a ``NativeStruct`` into the ``ByteBuffer`` + /// + /// Adds a native struct that's build and padded according + /// to `FlatBuffers` standards, directly into the buffer without + /// a predefined position. + /// + /// ```swift + /// let offset = builder.create( + /// struct: NativeStr(num: 1)) + /// ``` + /// + /// - Parameters: + /// - s: ``NativeStruct`` to be inserted into the ``ByteBuffer`` + /// - Returns: ``Offset`` of written struct + @inline(__always) + @discardableResult + mutating public func create( + struct s: T) -> Offset + { + let size = MemoryLayout.size + preAlign(len: size, alignment: MemoryLayout.alignment) + _bb.push(struct: s, size: size) + return Offset(offset: _bb.size) + } + + // MARK: - Inserting Strings + + /// Insets a string into the buffer of type `UTF8` + /// + /// Adds a swift string into ``ByteBuffer`` by encoding it + /// using `UTF8` + /// + /// ```swift + /// let nameOffset = builder + /// .create(string: "welcome") + /// ``` + /// + /// - Parameter str: String to be serialized + /// - returns: ``Offset`` of inserted string + @inline(__always) + mutating public func create(string str: String?) -> Offset { + guard let str = str else { return Offset() } + let len = str.utf8.count + notNested() + preAlign(len: len &+ 1, type: UOffset.self) + _bb.fill(padding: 1) + _bb.push(string: str, len: len) + push(element: UOffset(len)) + return Offset(offset: _bb.size) + } + + /// Insets a shared string into the buffer of type `UTF8` + /// + /// Adds a swift string into ``ByteBuffer`` by encoding it + /// using `UTF8`. The function will check if the string, + /// is already written to the ``ByteBuffer`` + /// + /// ```swift + /// let nameOffset = builder + /// .createShared(string: "welcome") + /// + /// + /// let secondOffset = builder + /// .createShared(string: "welcome") + /// + /// assert(nameOffset.o == secondOffset.o) + /// ``` + /// + /// - Parameter str: String to be serialized + /// - returns: ``Offset`` of inserted string + @inline(__always) + mutating public func createShared(string str: String?) -> Offset { + guard let str = str else { return Offset() } + if let offset = stringOffsetMap[str] { + return offset + } + let offset = create(string: str) + stringOffsetMap[str] = offset + return offset + } + + // MARK: - Inseting offsets + + /// Writes the ``Offset`` of an already written table + /// + /// Writes the ``Offset`` of a table if not empty into the + /// ``ByteBuffer`` + /// + /// - Parameters: + /// - offset: ``Offset`` of another object to be written + /// - position: The predefined position of the object + @inline(__always) + mutating public func add(offset: Offset, at position: VOffset) { + if offset.isEmpty { return } + add(element: refer(to: offset.o), def: 0, at: position) + } + + /// Pushes a value of type ``Offset`` into the ``ByteBuffer`` + /// - Parameter o: ``Offset`` + /// - returns: Current position of the ``Offset`` + @inline(__always) + @discardableResult + mutating public func push(element o: Offset) -> UOffset { + push(element: refer(to: o.o)) + } + + // MARK: - Inserting Scalars to Buffer + + /// Writes a ``Scalar`` value into ``ByteBuffer`` + /// + /// ``add(element:def:at:)`` takes in a default value, and current value + /// and the position within the `VTable`. The default value would not + /// be serialized if the value is the same as the current value or + /// `serializeDefaults` is equal to false. + /// + /// If serializing defaults is important ``init(initialSize:serializeDefaults:)``, + /// passing true for `serializeDefaults` would do the job. + /// + /// ```swift + /// // Adds 10 to the buffer + /// builder.add(element: Int(10), def: 1, position 12) + /// ``` + /// + /// *NOTE: Never call this manually* + /// + /// - Parameters: + /// - element: Element to insert + /// - def: Default value for that element + /// - position: The predefined position of the element + @inline(__always) + mutating public func add( + element: T, + def: T, + at position: VOffset) + { + if element == def && !serializeDefaults { return } + track(offset: push(element: element), at: position) + } + + /// Writes a optional ``Scalar`` value into ``ByteBuffer`` + /// + /// Takes an optional value to be written into the ``ByteBuffer`` + /// + /// *NOTE: Never call this manually* + /// + /// - Parameters: + /// - element: Optional element of type scalar + /// - position: The predefined position of the element + @inline(__always) + mutating public func add(element: T?, at position: VOffset) { + guard let element = element else { return } + track(offset: push(element: element), at: position) + } + + /// Pushes a values of type ``Scalar`` into the ``ByteBuffer`` + /// + /// *NOTE: Never call this manually* + /// + /// - Parameter element: Element to insert + /// - returns: Postion of the Element + @inline(__always) + @discardableResult + mutating public func push(element: T) -> UOffset { + let size = MemoryLayout.size + preAlign( + len: size, + alignment: size) + _bb.push(value: element, len: size) + return _bb.size + } + +} + +extension FlatBufferBuilder: CustomDebugStringConvertible { + + public var debugDescription: String { + """ + buffer debug: + \(_bb) + builder debug: + { finished: \(finished), serializeDefaults: \(serializeDefaults), isNested: \(isNested) } + """ + } + + /// VTableStorage is a class to contain the VTable buffer that would be serialized into buffer + @usableFromInline + internal class VTableStorage { + /// Memory check since deallocating each time we want to clear would be expensive + /// and memory leaks would happen if we dont deallocate the first allocated memory. + /// memory is promised to be available before adding `FieldLoc` + private var memoryInUse = false + /// Size of FieldLoc in memory + let size = MemoryLayout.stride + /// Memeory buffer + var memory: UnsafeMutableRawBufferPointer! + /// Capacity of the current buffer + var capacity: Int = 0 + /// Maximuim offset written to the class + var maxOffset: VOffset = 0 + /// number of fields written into the buffer + var numOfFields: Int = 0 + /// Last written Index + var writtenIndex: Int = 0 + + /// Creates the memory to store the buffer in + @usableFromInline + @inline(__always) + init() { + memory = UnsafeMutableRawBufferPointer.allocate( + byteCount: 0, + alignment: 0) + } + + @inline(__always) + deinit { + memory.deallocate() + } + + /// Builds a buffer with byte count of fieldloc.size * count of field numbers + /// - Parameter count: number of fields to be written + @inline(__always) + func start(count: Int) { + assert(count >= 0, "number of fields should NOT be negative") + let capacity = count &* size + ensure(space: capacity) + } + + /// Adds a FieldLoc into the buffer, which would track how many have been written, + /// and max offset + /// - Parameter loc: Location of encoded element + @inline(__always) + func add(loc: FieldLoc) { + memory.baseAddress?.advanced(by: writtenIndex).storeBytes( + of: loc, + as: FieldLoc.self) + writtenIndex = writtenIndex &+ size + numOfFields = numOfFields &+ 1 + maxOffset = max(loc.position, maxOffset) + } + + /// Clears the data stored related to the encoded buffer + @inline(__always) + func clear() { + maxOffset = 0 + numOfFields = 0 + writtenIndex = 0 + } + + /// Ensure that the buffer has enough space instead of recreating the buffer each time. + /// - Parameter space: space required for the new vtable + @inline(__always) + func ensure(space: Int) { + guard space &+ writtenIndex > capacity else { return } + memory.deallocate() + memory = UnsafeMutableRawBufferPointer.allocate( + byteCount: space, + alignment: size) + capacity = space + } + + /// Loads an object of type `FieldLoc` from buffer memory + /// - Parameter index: index of element + /// - Returns: a FieldLoc at index + @inline(__always) + func load(at index: Int) -> FieldLoc { + memory.load(fromByteOffset: index, as: FieldLoc.self) + } + + } + + internal struct FieldLoc { + var offset: UOffset + var position: VOffset + } + +} diff --git a/flatbuffers/FlatBufferObject.swift b/flatbuffers/FlatBufferObject.swift new file mode 100644 index 0000000000..705c934638 --- /dev/null +++ b/flatbuffers/FlatBufferObject.swift @@ -0,0 +1,68 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// NativeStruct is a protocol that indicates if the struct is a native `swift` struct +/// since now we will be serializing native structs into the buffer. +public protocol NativeStruct {} + +/// FlatbuffersInitializable is a protocol that allows any object to be +/// Initialized from a ByteBuffer +public protocol FlatbuffersInitializable { + /// Any flatbuffers object that confirms to this protocol is going to be + /// initializable through this initializer + init(_ bb: ByteBuffer, o: Int32) +} + +/// FlatbufferObject structures all the Flatbuffers objects +public protocol FlatBufferObject: FlatbuffersInitializable { + var __buffer: ByteBuffer! { get } +} + +/// ``ObjectAPIPacker`` is a protocol that allows object to pack and unpack from a +/// ``NativeObject`` to a flatbuffers Object and vice versa. +public protocol ObjectAPIPacker { + /// associatedtype to the object that should be unpacked. + associatedtype T + + /// ``pack(_:obj:)-3ptws`` tries to pacs the variables of a native Object into the `ByteBuffer` by using + /// a FlatBufferBuilder + /// - Parameters: + /// - builder: FlatBufferBuilder that will host incoming data + /// - obj: Object of associatedtype to the current implementer + /// + /// ``pack(_:obj:)-3ptws`` can be called by passing through an already initialized ``FlatBufferBuilder`` + /// or it can be called by using the public API that will create a new ``FlatBufferBuilder`` + static func pack(_ builder: inout FlatBufferBuilder, obj: inout T?) -> Offset + + /// ``pack(_:obj:)-20ipk`` packs the variables of a native Object into the `ByteBuffer` by using + /// the FlatBufferBuilder + /// - Parameters: + /// - builder: FlatBufferBuilder that will host incoming data + /// - obj: Object of associatedtype to the current implementer + /// + /// ``pack(_:obj:)-20ipk`` can be called by passing through an already initialized ``FlatBufferBuilder`` + /// or it can be called by using the public API that will create a new ``FlatBufferBuilder`` + static func pack(_ builder: inout FlatBufferBuilder, obj: inout T) -> Offset + + /// ``unpack()`` unpacks a ``FlatBuffers`` object into a Native swift object. + mutating func unpack() -> T +} diff --git a/flatbuffers/FlatBuffersUtils.swift b/flatbuffers/FlatBuffersUtils.swift new file mode 100644 index 0000000000..338988df75 --- /dev/null +++ b/flatbuffers/FlatBuffersUtils.swift @@ -0,0 +1,41 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// FlatBuffersUtils hosts some utility functions that might be useful +public enum FlatBuffersUtils { + + /// Gets the size of the prefix + /// - Parameter bb: Flatbuffer object + public static func getSizePrefix(bb: ByteBuffer) -> Int32 { + bb.read(def: Int32.self, position: bb.reader) + } + + /// Removes the prefix by duplicating the Flatbuffer this call is expensive since its + /// creates a new buffer use `readPrefixedSizeCheckedRoot` instead + /// unless a completely new buffer is required + /// - Parameter bb: Flatbuffer object + /// + /// + public static func removeSizePrefix(bb: ByteBuffer) -> ByteBuffer { + bb.duplicate(removing: MemoryLayout.size) + } +} diff --git a/flatbuffers/FlatbuffersErrors.swift b/flatbuffers/FlatbuffersErrors.swift new file mode 100644 index 0000000000..1a9284ebac --- /dev/null +++ b/flatbuffers/FlatbuffersErrors.swift @@ -0,0 +1,77 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Collection of thrown from the Flatbuffer verifier +public enum FlatbuffersErrors: Error, Equatable { + + /// Thrown when verifying a file id that doesnt match buffer id + case bufferIdDidntMatchPassedId + /// Prefixed size doesnt match the current (readable) buffer size + case prefixedSizeNotEqualToBufferSize + /// Thrown when buffer is bigger than the allowed 2GiB + case exceedsMaxSizeAllowed + /// Thrown when there is an missaligned pointer at position + /// of type + case missAlignedPointer(position: Int, type: String) + /// Thrown when trying to read a value that goes out of the + /// current buffer bounds + case outOfBounds(position: UInt, end: Int) + /// Thrown when the signed offset is out of the bounds of the + /// current buffer + case signedOffsetOutOfBounds(offset: Int, position: Int) + /// Thrown when a required field doesnt exist within the buffer + case requiredFieldDoesntExist(position: VOffset, name: String) + /// Thrown when a string is missing its NULL Terminator `\0`, + /// this can be disabled in the `VerifierOptions` + case missingNullTerminator(position: Int, str: String?) + /// Thrown when the verifier has reached the maximum tables allowed, + /// this can be disabled in the `VerifierOptions` + case maximumTables + /// Thrown when the verifier has reached the maximum depth allowed, + /// this can be disabled in the `VerifierOptions` + case maximumDepth + /// Thrown when the verifier is presented with an unknown union case + case unknownUnionCase + /// thrown when a value for a union is not found within the buffer + case valueNotFound(key: Int?, keyName: String, field: Int?, fieldName: String) + /// thrown when the size of the keys vector doesnt match fields vector + case unionVectorSize( + keyVectorSize: Int, + fieldVectorSize: Int, + unionKeyName: String, + fieldName: String) + case apparentSizeTooLarge + +} + +#if !os(WASI) + +extension FlatbuffersErrors { + public static func == ( + lhs: FlatbuffersErrors, + rhs: FlatbuffersErrors) -> Bool + { + lhs.localizedDescription == rhs.localizedDescription + } +} + +#endif diff --git a/flatbuffers/Int+extension.swift b/flatbuffers/Int+extension.swift new file mode 100644 index 0000000000..c8cd0e3641 --- /dev/null +++ b/flatbuffers/Int+extension.swift @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +extension Int { + + /// Moves the current int into the nearest power of two + /// + /// This is used since the UnsafeMutableRawPointer will face issues when writing/reading + /// if the buffer alignment exceeds that actual size of the buffer + var convertToPowerofTwo: Int { + guard self > 0 else { return 1 } + var n = UOffset(self) + + #if arch(arm) || arch(i386) + let max = UInt32(Int.max) + #else + let max = UInt32.max + #endif + + n -= 1 + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + if n != max { + n += 1 + } + + return Int(n) + } +} diff --git a/flatbuffers/Message.swift b/flatbuffers/Message.swift new file mode 100644 index 0000000000..172a339db6 --- /dev/null +++ b/flatbuffers/Message.swift @@ -0,0 +1,69 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// FlatBufferGRPCMessage protocol that should allow us to invoke +/// initializers directly from the GRPC generated code +public protocol FlatBufferGRPCMessage { + + /// Raw pointer which would be pointing to the beginning of the readable bytes + var rawPointer: UnsafeMutableRawPointer { get } + + /// Size of readable bytes in the buffer + var size: Int { get } + + init(byteBuffer: ByteBuffer) +} + +/// Message is a wrapper around Buffers to to able to send Flatbuffers `Buffers` through the +/// GRPC library +public struct Message: FlatBufferGRPCMessage { + internal var buffer: ByteBuffer + + /// Returns the an object of type T that would be read from the buffer + public var object: T { + T.init( + buffer, + o: Int32(buffer.read(def: UOffset.self, position: buffer.reader)) + + Int32(buffer.reader)) + } + + public var rawPointer: UnsafeMutableRawPointer { + buffer.memory.advanced(by: buffer.reader) } + + public var size: Int { Int(buffer.size) } + + /// Initializes the message with the type Flatbuffer.Bytebuffer that is transmitted over + /// GRPC + /// - Parameter byteBuffer: Flatbuffer ByteBuffer object + public init(byteBuffer: ByteBuffer) { + buffer = byteBuffer + } + + /// Initializes the message by copying the buffer to the message to be sent. + /// from the builder + /// - Parameter builder: FlatbufferBuilder that has the bytes created in + /// - Note: Use `builder.finish(offset)` before passing the builder without prefixing anything to it + public init(builder: inout FlatBufferBuilder) { + buffer = builder.sizedBuffer + builder.clear() + } +} diff --git a/flatbuffers/Mutable.swift b/flatbuffers/Mutable.swift new file mode 100644 index 0000000000..7a1a3d5bca --- /dev/null +++ b/flatbuffers/Mutable.swift @@ -0,0 +1,88 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Mutable is a protocol that allows us to mutate Scalar values within a ``ByteBuffer`` +public protocol Mutable { + /// makes Flatbuffer accessed within the Protocol + var bb: ByteBuffer { get } + /// makes position of the ``Table``/``struct`` accessed within the Protocol + var postion: Int32 { get } +} + +extension Mutable { + + /// Mutates the memory in the buffer, this is only called from the access function of ``Table`` and ``struct`` + /// - Parameters: + /// - value: New value to be inserted to the buffer + /// - index: index of the Element + func mutate(value: T, o: Int32) -> Bool { + guard o != 0 else { return false } + bb.write(value: value, index: Int(o), direct: true) + return true + } +} + +extension Mutable where Self == Table { + + /// Mutates a value by calling mutate with respect to the position in a ``Table`` + /// - Parameters: + /// - value: New value to be inserted to the buffer + /// - index: index of the Element + public func mutate(_ value: T, index: Int32) -> Bool { + guard index != 0 else { return false } + return mutate(value: value, o: index + postion) + } + + /// Directly mutates the element by calling mutate + /// + /// Mutates the Element at index ignoring the current position by calling mutate + /// - Parameters: + /// - value: New value to be inserted to the buffer + /// - index: index of the Element + public func directMutate(_ value: T, index: Int32) -> Bool { + mutate(value: value, o: index) + } +} + +extension Mutable where Self == Struct { + + /// Mutates a value by calling mutate with respect to the position in the struct + /// - Parameters: + /// - value: New value to be inserted to the buffer + /// - index: index of the Element + public func mutate(_ value: T, index: Int32) -> Bool { + mutate(value: value, o: index + postion) + } + + /// Directly mutates the element by calling mutate + /// + /// Mutates the Element at index ignoring the current position by calling mutate + /// - Parameters: + /// - value: New value to be inserted to the buffer + /// - index: index of the Element + public func directMutate(_ value: T, index: Int32) -> Bool { + mutate(value: value, o: index) + } +} + +extension Struct: Mutable {} +extension Table: Mutable {} diff --git a/flatbuffers/NativeObject.swift b/flatbuffers/NativeObject.swift new file mode 100644 index 0000000000..9c72b50b7a --- /dev/null +++ b/flatbuffers/NativeObject.swift @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// NativeObject is a protocol that all of the `Object-API` generated code should be +/// conforming to since it allows developers the ease of use to pack and unpack their +/// Flatbuffers objects +public protocol NativeObject {} + +extension NativeObject { + + /// Serialize is a helper function that serailizes the data from the Object API to a bytebuffer directly th + /// - Parameter type: Type of the Flatbuffer object + /// - Returns: returns the encoded sized ByteBuffer + public func serialize(type: T.Type) -> ByteBuffer + where T.T == Self + { + var builder = FlatBufferBuilder(initialSize: 1024) + return serialize(builder: &builder, type: type.self) + } + + /// Serialize is a helper function that serailizes the data from the Object API to a bytebuffer directly. + /// + /// - Parameters: + /// - builder: A FlatBufferBuilder + /// - type: Type of the Flatbuffer object + /// - Returns: returns the encoded sized ByteBuffer + /// - Note: The `serialize(builder:type)` can be considered as a function that allows you to create smaller builder instead of the default `1024`. + /// It can be considered less expensive in terms of memory allocation + public func serialize( + builder: inout FlatBufferBuilder, + type: T.Type) -> ByteBuffer where T.T == Self + { + var s = self + let root = type.pack(&builder, obj: &s) + builder.finish(offset: root) + return builder.sizedBuffer + } +} diff --git a/flatbuffers/Offset.swift b/flatbuffers/Offset.swift new file mode 100644 index 0000000000..e433f35a26 --- /dev/null +++ b/flatbuffers/Offset.swift @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Offset object for all the Objects that are written into the buffer +public struct Offset { + /// Offset of the object in the buffer + public var o: UOffset + /// Returns false if the offset is equal to zero + public var isEmpty: Bool { o == 0 } + + public init(offset: UOffset) { o = offset } + public init() { o = 0 } +} diff --git a/flatbuffers/Root.swift b/flatbuffers/Root.swift new file mode 100644 index 0000000000..6269148bb6 --- /dev/null +++ b/flatbuffers/Root.swift @@ -0,0 +1,119 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Takes in a prefixed sized buffer, where the prefixed size would be skipped. +/// And would verify that the buffer passed is a valid `Flatbuffers` Object. +/// - Parameters: +/// - byteBuffer: Buffer that needs to be checked and read +/// - options: Verifier options +/// - Throws: FlatbuffersErrors +/// - Returns: Returns a valid, checked Flatbuffers object +/// +/// ``getPrefixedSizeCheckedRoot(byteBuffer:options:)`` would skip the first Bytes in +/// the ``ByteBuffer`` and verifies the buffer by calling ``getCheckedRoot(byteBuffer:options:)`` +public func getPrefixedSizeCheckedRoot( + byteBuffer: inout ByteBuffer, + fileId: String? = nil, + options: VerifierOptions = .init()) throws -> T +{ + byteBuffer.skipPrefix() + return try getCheckedRoot( + byteBuffer: &byteBuffer, + fileId: fileId, + options: options) +} + +/// Takes in a prefixed sized buffer, where we check if the sized buffer is equal to prefix size. +/// And would verify that the buffer passed is a valid `Flatbuffers` Object. +/// - Parameters: +/// - byteBuffer: Buffer that needs to be checked and read +/// - options: Verifier options +/// - Throws: FlatbuffersErrors +/// - Returns: Returns a valid, checked Flatbuffers object +/// +/// ``getPrefixedSizeCheckedRoot(byteBuffer:options:)`` would skip the first Bytes in +/// the ``ByteBuffer`` and verifies the buffer by calling ``getCheckedRoot(byteBuffer:options:)`` +public func getCheckedPrefixedSizeRoot( + byteBuffer: inout ByteBuffer, + fileId: String? = nil, + options: VerifierOptions = .init()) throws -> T +{ + let prefix = byteBuffer.skipPrefix() + if prefix != byteBuffer.size { + throw FlatbuffersErrors.prefixedSizeNotEqualToBufferSize + } + return try getCheckedRoot( + byteBuffer: &byteBuffer, + fileId: fileId, + options: options) +} + +/// Takes in a prefixed sized buffer, where the prefixed size would be skipped. +/// Returns a `NON-Checked` flatbuffers object +/// - Parameter byteBuffer: Buffer that contains data +/// - Returns: Returns a Flatbuffers object +/// +/// ``getPrefixedSizeCheckedRoot(byteBuffer:options:)`` would skip the first Bytes in +/// the ``ByteBuffer`` and then calls ``getRoot(byteBuffer:)`` +public func getPrefixedSizeRoot(byteBuffer: inout ByteBuffer) + -> T +{ + byteBuffer.skipPrefix() + return getRoot(byteBuffer: &byteBuffer) + +} + +/// Verifies that the buffer passed is a valid `Flatbuffers` Object. +/// - Parameters: +/// - byteBuffer: Buffer that needs to be checked and read +/// - options: Verifier options +/// - Throws: FlatbuffersErrors +/// - Returns: Returns a valid, checked Flatbuffers object +/// +/// ``getCheckedRoot(byteBuffer:options:)`` Takes in a ``ByteBuffer`` and verifies +/// that by creating a ``Verifier`` and checkes if all the `Bytes` and correctly aligned +/// and within the ``ByteBuffer`` range. +public func getCheckedRoot( + byteBuffer: inout ByteBuffer, + fileId: String? = nil, + options: VerifierOptions = .init()) throws -> T +{ + var verifier = try Verifier(buffer: &byteBuffer, options: options) + if let fileId = fileId { + try verifier.verify(id: fileId) + } + try ForwardOffset.verify(&verifier, at: 0, of: T.self) + return T.init( + byteBuffer, + o: Int32(byteBuffer.read(def: UOffset.self, position: byteBuffer.reader)) + + Int32(byteBuffer.reader)) +} + +/// Returns a `NON-Checked` flatbuffers object +/// - Parameter byteBuffer: Buffer that contains data +/// - Returns: Returns a Flatbuffers object +public func getRoot(byteBuffer: inout ByteBuffer) -> T { + T.init( + byteBuffer, + o: Int32(byteBuffer.read(def: UOffset.self, position: byteBuffer.reader)) + + Int32(byteBuffer.reader)) +} diff --git a/flatbuffers/String+extension.swift b/flatbuffers/String+extension.swift new file mode 100644 index 0000000000..35c83cbdbe --- /dev/null +++ b/flatbuffers/String+extension.swift @@ -0,0 +1,113 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +extension String: Verifiable { + + /// Verifies that the current value is which the bounds of the buffer, and if + /// the current `Value` is aligned properly + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - position: Current position within the buffer + /// - type: The type of the object to be verified + /// - Throws: Errors coming from `inBuffer`, `missingNullTerminator` and `outOfBounds` + public static func verify( + _ verifier: inout Verifier, + at position: Int, + of type: T.Type) throws where T: Verifiable + { + + let range = try String.verifyRange(&verifier, at: position, of: UInt8.self) + /// Safe &+ since we already check for overflow in verify range + let stringLen = range.start &+ range.count + + if stringLen >= verifier.capacity { + throw FlatbuffersErrors.outOfBounds( + position: UInt(clamping: stringLen.magnitude), + end: verifier.capacity) + } + + let isNullTerminated = verifier._buffer.read( + def: UInt8.self, + position: stringLen) == 0 + + if !verifier._options._ignoreMissingNullTerminators && !isNullTerminated { + let str = verifier._buffer.readString(at: range.start, count: range.count) + throw FlatbuffersErrors.missingNullTerminator( + position: position, + str: str) + } + } +} + +extension String: FlatbuffersInitializable { + + /// Initailizes a string from a Flatbuffers ByteBuffer + /// - Parameters: + /// - bb: ByteBuffer containing the readable string + /// - o: Current position + public init(_ bb: ByteBuffer, o: Int32) { + let v = Int(o) + let count = bb.read(def: Int32.self, position: v) + self = bb.readString( + at: MemoryLayout.size + v, + count: Int(count)) ?? "" + } +} + +extension String: ObjectAPIPacker { + + public static func pack( + _ builder: inout FlatBufferBuilder, + obj: inout String?) -> Offset + { + guard var obj = obj else { return Offset() } + return pack(&builder, obj: &obj) + } + + public static func pack( + _ builder: inout FlatBufferBuilder, + obj: inout String) -> Offset + { + builder.create(string: obj) + } + + public mutating func unpack() -> String { + self + } + +} + +extension String: NativeObject { + + public func serialize(type: T.Type) -> ByteBuffer + where T.T == Self + { + fatalError("serialize should never be called from string directly") + } + + public func serialize( + builder: inout FlatBufferBuilder, + type: T.Type) -> ByteBuffer where T.T == Self + { + fatalError("serialize should never be called from string directly") + } +} diff --git a/flatbuffers/Struct.swift b/flatbuffers/Struct.swift new file mode 100644 index 0000000000..04cfba0928 --- /dev/null +++ b/flatbuffers/Struct.swift @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Struct is a representation of a mutable `Flatbuffers` struct +/// since native structs are value types and cant be mutated +@frozen +public struct Struct { + + /// Hosting Bytebuffer + public private(set) var bb: ByteBuffer + /// Current position of the struct + public private(set) var postion: Int32 + + /// Initializer for a mutable flatbuffers struct + /// - Parameters: + /// - bb: Current hosting Bytebuffer + /// - position: Current position for the struct in the ByteBuffer + public init(bb: ByteBuffer, position: Int32 = 0) { + self.bb = bb + postion = position + } + + /// Reads data from the buffer directly at offset O + /// - Parameters: + /// - type: Type of data to be read + /// - o: Current offset of the data + /// - Returns: Data of Type T that conforms to type Scalar + public func readBuffer(of type: T.Type, at o: Int32) -> T { + let r = bb.read(def: T.self, position: Int(o + postion)) + return r + } +} diff --git a/flatbuffers/Table.swift b/flatbuffers/Table.swift new file mode 100644 index 0000000000..0da5919a84 --- /dev/null +++ b/flatbuffers/Table.swift @@ -0,0 +1,240 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// `Table` is a Flatbuffers object that can read, +/// mutate scalar fields within a valid flatbuffers buffer +@frozen +public struct Table { + + /// Hosting Bytebuffer + public private(set) var bb: ByteBuffer + /// Current position of the table within the buffer + public private(set) var postion: Int32 + + /// Initializer for the table interface to allow generated code to read + /// data from memory + /// - Parameters: + /// - bb: ByteBuffer that stores data + /// - position: Current table position + /// - Note: This will `CRASH` if read on a big endian machine + public init(bb: ByteBuffer, position: Int32 = 0) { + guard isLitteEndian else { + fatalError( + "Reading/Writing a buffer in big endian machine is not supported on swift") + } + self.bb = bb + postion = position + } + + /// Gets the offset of the current field within the buffer by reading + /// the vtable + /// - Parameter o: current offset + /// - Returns: offset of field within buffer + public func offset(_ o: Int32) -> Int32 { + let vtable = postion - bb.read(def: Int32.self, position: Int(postion)) + return o < bb + .read(def: VOffset.self, position: Int(vtable)) ? Int32(bb.read( + def: Int16.self, + position: Int(vtable + o))) : 0 + } + + /// Gets the indirect offset of the current stored object + /// (applicable only for object arrays) + /// - Parameter o: current offset + /// - Returns: offset of field within buffer + public func indirect(_ o: Int32) -> Int32 { + o + bb.read(def: Int32.self, position: Int(o)) + } + + /// String reads from the buffer with respect to position of the current table. + /// - Parameter offset: Offset of the string + public func string(at offset: Int32) -> String? { + directString(at: offset + postion) + } + + /// Direct string reads from the buffer disregarding the position of the table. + /// It would be preferable to use string unless the current position of the table + /// is not needed + /// - Parameter offset: Offset of the string + public func directString(at offset: Int32) -> String? { + var offset = offset + offset += bb.read(def: Int32.self, position: Int(offset)) + let count = bb.read(def: Int32.self, position: Int(offset)) + let position = Int(offset) + MemoryLayout.size + return bb.readString(at: position, count: Int(count)) + } + + /// Reads from the buffer with respect to the position in the table. + /// - Parameters: + /// - type: Type of Element that needs to be read from the buffer + /// - o: Offset of the Element + public func readBuffer(of type: T.Type, at o: Int32) -> T { + directRead(of: T.self, offset: o + postion) + } + + /// Reads from the buffer disregarding the position of the table. + /// It would be used when reading from an + /// ``` + /// let offset = __t.offset(10) + /// //Only used when the we already know what is the + /// // position in the table since __t.vector(at:) + /// // returns the index with respect to the position + /// __t.directRead(of: Byte.self, + /// offset: __t.vector(at: offset) + index * 1) + /// ``` + /// - Parameters: + /// - type: Type of Element that needs to be read from the buffer + /// - o: Offset of the Element + public func directRead(of type: T.Type, offset o: Int32) -> T { + let r = bb.read(def: T.self, position: Int(o)) + return r + } + + /// Returns that current `Union` object at a specific offset + /// by adding offset to the current position of table + /// - Parameter o: offset + /// - Returns: A flatbuffers object + public func union(_ o: Int32) -> T { + let o = o + postion + return directUnion(o) + } + + /// Returns a direct `Union` object at a specific offset + /// - Parameter o: offset + /// - Returns: A flatbuffers object + public func directUnion(_ o: Int32) -> T { + T.init(bb, o: o + bb.read(def: Int32.self, position: Int(o))) + } + + /// Returns a vector of type T at a specific offset + /// This should only be used by `Scalars` + /// - Parameter off: Readable offset + /// - Returns: Returns a vector of type [T] + public func getVector(at off: Int32) -> [T]? { + let o = offset(off) + guard o != 0 else { return nil } + return bb.readSlice(index: Int(vector(at: o)), count: Int(vector(count: o))) + } + + /// Vector count gets the count of Elements within the array + /// - Parameter o: start offset of the vector + /// - returns: Count of elements + public func vector(count o: Int32) -> Int32 { + var o = o + o += postion + o += bb.read(def: Int32.self, position: Int(o)) + return bb.read(def: Int32.self, position: Int(o)) + } + + /// Vector start index in the buffer + /// - Parameter o:start offset of the vector + /// - returns: the start index of the vector + public func vector(at o: Int32) -> Int32 { + var o = o + o += postion + return o + bb.read(def: Int32.self, position: Int(o)) + 4 + } + + /// Reading an indirect offset of a table. + /// - Parameters: + /// - o: position within the buffer + /// - fbb: ByteBuffer + /// - Returns: table offset + static public func indirect(_ o: Int32, _ fbb: ByteBuffer) -> Int32 { + o + fbb.read(def: Int32.self, position: Int(o)) + } + + /// Gets a vtable value according to an table Offset and a field offset + /// - Parameters: + /// - o: offset relative to entire buffer + /// - vOffset: Field offset within a vtable + /// - fbb: ByteBuffer + /// - Returns: an position of a field + static public func offset( + _ o: Int32, + vOffset: Int32, + fbb: ByteBuffer) -> Int32 + { + let vTable = Int32(fbb.capacity) - o + return vTable + Int32(fbb.read( + def: Int16.self, + position: Int(vTable + vOffset - fbb.read( + def: Int32.self, + position: Int(vTable))))) + } + + /// Compares two objects at offset A and offset B within a ByteBuffer + /// - Parameters: + /// - off1: first offset to compare + /// - off2: second offset to compare + /// - fbb: Bytebuffer + /// - Returns: returns the difference between + static public func compare( + _ off1: Int32, + _ off2: Int32, + fbb: ByteBuffer) -> Int32 + { + let memorySize = Int32(MemoryLayout.size) + let _off1 = off1 + fbb.read(def: Int32.self, position: Int(off1)) + let _off2 = off2 + fbb.read(def: Int32.self, position: Int(off2)) + let len1 = fbb.read(def: Int32.self, position: Int(_off1)) + let len2 = fbb.read(def: Int32.self, position: Int(_off2)) + let startPos1 = _off1 + memorySize + let startPos2 = _off2 + memorySize + let minValue = min(len1, len2) + for i in 0...minValue { + let b1 = fbb.read(def: Int8.self, position: Int(i + startPos1)) + let b2 = fbb.read(def: Int8.self, position: Int(i + startPos2)) + if b1 != b2 { + return Int32(b2 - b1) + } + } + return len1 - len2 + } + + /// Compares two objects at offset A and array of `Bytes` within a ByteBuffer + /// - Parameters: + /// - off1: Offset to compare to + /// - key: bytes array to compare to + /// - fbb: Bytebuffer + /// - Returns: returns the difference between + static public func compare( + _ off1: Int32, + _ key: [Byte], + fbb: ByteBuffer) -> Int32 + { + let memorySize = Int32(MemoryLayout.size) + let _off1 = off1 + fbb.read(def: Int32.self, position: Int(off1)) + let len1 = fbb.read(def: Int32.self, position: Int(_off1)) + let len2 = Int32(key.count) + let startPos1 = _off1 + memorySize + let minValue = min(len1, len2) + for i in 0.. Int? { + if field >= _vtableLength { + return nil + } + + /// Reading the offset for the field needs to be read. + let offset: VOffset = try _verifier.getValue( + at: Int(clamping: _vtable &+ Int(field))) + + if offset > 0 { + return Int(clamping: _position &+ Int(offset)) + } + return nil + } + + /// Visits all the fields within the table to validate the integrity + /// of the data + /// - Parameters: + /// - field: voffset of the current field to be read + /// - fieldName: fieldname to report data Errors. + /// - required: If the field has to be available in the buffer + /// - type: Type of field to be read + /// - Throws: A `FlatbuffersErrors` where the field is corrupt + public mutating func visit( + field: VOffset, + fieldName: String, + required: Bool, + type: T.Type) throws where T: Verifiable + { + let derefValue = try dereference(field) + + if let value = derefValue { + try T.verify(&_verifier, at: value, of: T.self) + return + } + if required { + throw FlatbuffersErrors.requiredFieldDoesntExist( + position: field, + name: fieldName) + } + } + + /// Visits all the fields for a union object within the table to + /// validate the integrity of the data + /// - Parameters: + /// - key: Current Key Voffset + /// - field: Current field Voffset + /// - unionKeyName: Union key name + /// - fieldName: Field key name + /// - required: indicates if an object is required to be present + /// - completion: Completion is a handler that WILL be called in the generated + /// - Throws: A `FlatbuffersErrors` where the field is corrupt + public mutating func visit( + unionKey key: VOffset, + unionField field: VOffset, + unionKeyName: String, + fieldName: String, + required: Bool, + completion: @escaping (inout Verifier, T, Int) throws -> Void) throws + where T: UnionEnum + { + let keyPos = try dereference(key) + let valPos = try dereference(field) + + if keyPos == nil && valPos == nil { + if required { + throw FlatbuffersErrors.requiredFieldDoesntExist( + position: key, + name: unionKeyName) + } + return + } + + if let _key = keyPos, + let _val = valPos + { + /// verifiying that the key is within the buffer + try T.T.verify(&_verifier, at: _key, of: T.T.self) + guard let _enum = try T.init(value: _verifier._buffer.read( + def: T.T.self, + position: _key)) else + { + throw FlatbuffersErrors.unknownUnionCase + } + /// we are assuming that Unions will always be of type Uint8 + try completion( + &_verifier, + _enum, + _val) + return + } + throw FlatbuffersErrors.valueNotFound( + key: keyPos, + keyName: unionKeyName, + field: valPos, + fieldName: fieldName) + } + + /// Visits and validates all the objects within a union vector + /// - Parameters: + /// - key: Current Key Voffset + /// - field: Current field Voffset + /// - unionKeyName: Union key name + /// - fieldName: Field key name + /// - required: indicates if an object is required to be present + /// - completion: Completion is a handler that WILL be called in the generated + /// - Throws: A `FlatbuffersErrors` where the field is corrupt + public mutating func visitUnionVector( + unionKey key: VOffset, + unionField field: VOffset, + unionKeyName: String, + fieldName: String, + required: Bool, + completion: @escaping (inout Verifier, T, Int) throws -> Void) throws + where T: UnionEnum + { + let keyVectorPosition = try dereference(key) + let offsetVectorPosition = try dereference(field) + + if let keyPos = keyVectorPosition, + let valPos = offsetVectorPosition + { + try UnionVector.verify( + &_verifier, + keyPosition: keyPos, + fieldPosition: valPos, + unionKeyName: unionKeyName, + fieldName: fieldName, + completion: completion) + return + } + if required { + throw FlatbuffersErrors.requiredFieldDoesntExist( + position: field, + name: fieldName) + } + } + + /// Finishs the current Table verifier, and subtracts the current + /// table from the incremented depth. + public mutating func finish() { + _verifier.finish() + } +} diff --git a/flatbuffers/VeriferOptions.swift b/flatbuffers/VeriferOptions.swift new file mode 100644 index 0000000000..a760ffbab0 --- /dev/null +++ b/flatbuffers/VeriferOptions.swift @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// `VerifierOptions` is a set of options to verify a flatbuffer +public struct VerifierOptions { + + /// Maximum `Apparent` size if the buffer can be expanded into a DAG tree + internal var _maxApparentSize: UOffset + + /// Maximum table count allowed in a buffer + internal var _maxTableCount: UOffset + + /// Maximum depth allowed in a buffer + internal var _maxDepth: UOffset + + /// Ignoring missing null terminals in strings + internal var _ignoreMissingNullTerminators: Bool + + /// initializes the set of options for the verifier + /// - Parameters: + /// - maxDepth: Maximum depth allowed in a buffer + /// - maxTableCount: Maximum table count allowed in a buffer + /// - maxApparentSize: Maximum `Apparent` size if the buffer can be expanded into a DAG tree + /// - ignoreMissingNullTerminators: Ignoring missing null terminals in strings *Currently not supported in swift* + public init( + maxDepth: UOffset = 64, + maxTableCount: UOffset = 1000000, + maxApparentSize: UOffset = 1 << 31, + ignoreMissingNullTerminators: Bool = false) + { + _maxDepth = maxDepth + _maxTableCount = maxTableCount + _maxApparentSize = maxApparentSize + _ignoreMissingNullTerminators = ignoreMissingNullTerminators + } + +} diff --git a/flatbuffers/Verifiable.swift b/flatbuffers/Verifiable.swift new file mode 100644 index 0000000000..b445c4ce13 --- /dev/null +++ b/flatbuffers/Verifiable.swift @@ -0,0 +1,215 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Verifiable is a protocol all swift flatbuffers object should conform to, +/// since swift is similar to `cpp` and `rust` where the data is read directly +/// from `unsafeMemory` thus the need to verify if the buffer received is a valid one +public protocol Verifiable { + + /// Verifies that the current value is which the bounds of the buffer, and if + /// the current `Value` is aligned properly + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - position: Current position within the buffer + /// - type: The type of the object to be verified + /// - Throws: Errors coming from `inBuffer` function + static func verify( + _ verifier: inout Verifier, + at position: Int, + of type: T.Type) throws where T: Verifiable +} + +extension Verifiable { + + /// Verifies if the current range to be read is within the bounds of the buffer, + /// and if the range is properly aligned + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - position: Current position within the buffer + /// - type: The type of the object to be verified + /// - Throws: Erros thrown from `isAligned` & `rangeInBuffer` + /// - Returns: a tuple of the start position and the count of objects within the range + @discardableResult + public static func verifyRange( + _ verifier: inout Verifier, + at position: Int, of type: T.Type) throws -> (start: Int, count: Int) + { + let len: UOffset = try verifier.getValue(at: position) + let intLen = Int(len) + let start = Int(clamping: (position &+ MemoryLayout.size).magnitude) + try verifier.isAligned(position: start, type: type.self) + try verifier.rangeInBuffer(position: start, size: intLen) + return (start, intLen) + } +} + +extension Verifiable where Self: Scalar { + + /// Verifies that the current value is which the bounds of the buffer, and if + /// the current `Value` is aligned properly + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - position: Current position within the buffer + /// - type: The type of the object to be verified + /// - Throws: Errors coming from `inBuffer` function + public static func verify( + _ verifier: inout Verifier, + at position: Int, + of type: T.Type) throws where T: Verifiable + { + try verifier.inBuffer(position: position, of: type.self) + } +} + +// MARK: - ForwardOffset + +/// ForwardOffset is a container to wrap around the Generic type to be verified +/// from the flatbuffers object. +public enum ForwardOffset: Verifiable where U: Verifiable { + + /// Verifies that the current value is which the bounds of the buffer, and if + /// the current `Value` is aligned properly + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - position: Current position within the buffer + /// - type: The type of the object to be verified + /// - Throws: Errors coming from `inBuffer` function + public static func verify( + _ verifier: inout Verifier, + at position: Int, + of type: T.Type) throws where T: Verifiable + { + let offset: UOffset = try verifier.getValue(at: position) + let nextOffset = Int(clamping: (Int(offset) &+ position).magnitude) + try U.verify(&verifier, at: nextOffset, of: U.self) + } +} + +// MARK: - Vector + +/// Vector is a container to wrap around the Generic type to be verified +/// from the flatbuffers object. +public enum Vector: Verifiable where U: Verifiable, S: Verifiable { + + /// Verifies that the current value is which the bounds of the buffer, and if + /// the current `Value` is aligned properly + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - position: Current position within the buffer + /// - type: The type of the object to be verified + /// - Throws: Errors coming from `inBuffer` function + public static func verify( + _ verifier: inout Verifier, + at position: Int, + of type: T.Type) throws where T: Verifiable + { + /// checks if the next verification type S is equal to U of type forwardOffset + /// This had to be done since I couldnt find a solution for duplicate call functions + /// A fix will be appreciated + if U.self is ForwardOffset.Type { + let range = try verifyRange(&verifier, at: position, of: UOffset.self) + for index in stride( + from: range.start, + to: Int(clamping: range.start &+ range.count), + by: MemoryLayout.size) + { + try U.verify(&verifier, at: index, of: U.self) + } + } else { + try S.verifyRange(&verifier, at: position, of: S.self) + } + } +} + +// MARK: - UnionVector + +/// UnionVector is a container to wrap around the Generic type to be verified +/// from the flatbuffers object. +public enum UnionVector where S: UnionEnum { + + /// Completion handler for the function Verify, that passes the verifier + /// enum type and position of union field + public typealias Completion = (inout Verifier, S, Int) throws -> Void + + /// Verifies if the current range to be read is within the bounds of the buffer, + /// and if the range is properly aligned. It also verifies if the union type is a + /// *valid/supported* union type. + /// - Parameters: + /// - verifier: Verifier that hosts the buffer + /// - keyPosition: Current union key position within the buffer + /// - fieldPosition: Current union field position within the buffer + /// - unionKeyName: Name of key to written if error is presented + /// - fieldName: Name of field to written if error is presented + /// - completion: Completion is a handler that WILL be called in the generated + /// code to verify the actual objects + /// - Throws: FlatbuffersErrors + public static func verify( + _ verifier: inout Verifier, + keyPosition: Int, + fieldPosition: Int, + unionKeyName: String, + fieldName: String, + completion: @escaping Completion) throws + { + /// Get offset for union key vectors and offset vectors + let keyOffset: UOffset = try verifier.getValue(at: keyPosition) + let fieldOffset: UOffset = try verifier.getValue(at: fieldPosition) + + /// Check if values are within the buffer, returns the start position of vectors, and vector counts + /// Using &+ is safe since we already verified that the value is within the buffer, where the max is + /// going to be 2Gib and swift supports Int64 by default + let keysRange = try S.T.verifyRange( + &verifier, + at: Int(keyOffset) &+ keyPosition, + of: S.T.self) + let offsetsRange = try UOffset.verifyRange( + &verifier, + at: Int(fieldOffset) &+ fieldPosition, + of: UOffset.self) + + guard keysRange.count == offsetsRange.count else { + throw FlatbuffersErrors.unionVectorSize( + keyVectorSize: keysRange.count, + fieldVectorSize: offsetsRange.count, + unionKeyName: unionKeyName, + fieldName: fieldName) + } + + var count = 0 + /// Iterate over the vector of keys and offsets. + while count < keysRange.count { + + /// index of readable enum value in array + let keysIndex = MemoryLayout.size * count + guard let _enum = try S.init(value: verifier._buffer.read( + def: S.T.self, + position: keysRange.start + keysIndex)) else + { + throw FlatbuffersErrors.unknownUnionCase + } + /// index of readable offset value in array + let fieldIndex = MemoryLayout.size * count + try completion(&verifier, _enum, offsetsRange.start + fieldIndex) + count += 1 + } + } +} diff --git a/flatbuffers/Verifier.swift b/flatbuffers/Verifier.swift new file mode 100644 index 0000000000..6daf6f50ce --- /dev/null +++ b/flatbuffers/Verifier.swift @@ -0,0 +1,217 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !os(WASI) +import Foundation +#else +import SwiftOverlayShims +#endif + +/// Verifier that check if the buffer passed into it is a valid, +/// safe, aligned Flatbuffers object since swift read from `unsafeMemory` +public struct Verifier { + + /// Flag to check for alignment if true + fileprivate let _checkAlignment: Bool + /// Capacity of the current buffer + fileprivate var _capacity: Int + /// Current ApparentSize + fileprivate var _apparentSize: UOffset = 0 + /// Amount of tables present within a buffer + fileprivate var _tableCount = 0 + + /// Capacity of the buffer + internal var capacity: Int { _capacity } + /// Current reached depth within the buffer + internal var _depth = 0 + /// Current verifiable ByteBuffer + internal var _buffer: ByteBuffer + /// Options for verification + internal let _options: VerifierOptions + + /// Initializer for the verifier + /// - Parameters: + /// - buffer: Bytebuffer that is required to be verified + /// - options: `VerifierOptions` that set the rule for some of the verification done + /// - checkAlignment: If alignment check is required to be preformed + /// - Throws: `exceedsMaxSizeAllowed` if capacity of the buffer is more than 2GiB + public init( + buffer: inout ByteBuffer, + options: VerifierOptions = .init(), + checkAlignment: Bool = true) throws + { + guard buffer.capacity < FlatBufferMaxSize else { + throw FlatbuffersErrors.exceedsMaxSizeAllowed + } + + _buffer = buffer + _capacity = buffer.capacity + _checkAlignment = checkAlignment + _options = options + } + + /// Resets the verifier to initial state + public mutating func reset() { + _depth = 0 + _tableCount = 0 + } + + /// Checks if the value of type `T` is aligned properly in the buffer + /// - Parameters: + /// - position: Current position + /// - type: Type of value to check + /// - Throws: `missAlignedPointer` if the pointer is not aligned properly + public mutating func isAligned(position: Int, type: T.Type) throws { + + /// If check alignment is false this mutating function doesnt continue + if !_checkAlignment { return } + + /// advance pointer to position X + let ptr = _buffer._storage.memory.advanced(by: position) + /// Check if the pointer is aligned + if Int(bitPattern: ptr) & (MemoryLayout.alignment &- 1) == 0 { + return + } + + throw FlatbuffersErrors.missAlignedPointer( + position: position, + type: String(describing: T.self)) + } + + /// Checks if the value of Size "X" is within the range of the buffer + /// - Parameters: + /// - position: Current postion to be read + /// - size: `Byte` Size of readable object within the buffer + /// - Throws: `outOfBounds` if the value is out of the bounds of the buffer + /// and `apparentSizeTooLarge` if the apparent size is bigger than the one specified + /// in `VerifierOptions` + public mutating func rangeInBuffer(position: Int, size: Int) throws { + let end = UInt(clamping: (position &+ size).magnitude) + if end > _buffer.capacity { + throw FlatbuffersErrors.outOfBounds(position: end, end: capacity) + } + _apparentSize = _apparentSize &+ UInt32(size) + if _apparentSize > _options._maxApparentSize { + throw FlatbuffersErrors.apparentSizeTooLarge + } + } + + /// Validates if a value of type `T` is aligned and within the bounds of + /// the buffer + /// - Parameters: + /// - position: Current readable position + /// - type: Type of value to check + /// - Throws: FlatbuffersErrors + public mutating func inBuffer(position: Int, of type: T.Type) throws { + try isAligned(position: position, type: type) + try rangeInBuffer(position: position, size: MemoryLayout.size) + } + + /// Visits a table at the current position and validates if the table meets + /// the rules specified in the `VerifierOptions` + /// - Parameter position: Current position to be read + /// - Throws: FlatbuffersErrors + /// - Returns: A `TableVerifier` at the current readable table + public mutating func visitTable(at position: Int) throws -> TableVerifier { + let vtablePosition = try derefOffset(position: position) + let vtableLength: VOffset = try getValue(at: vtablePosition) + + let length = Int(vtableLength) + try isAligned( + position: Int(clamping: (vtablePosition + length).magnitude), + type: VOffset.self) + try rangeInBuffer(position: vtablePosition, size: length) + + _tableCount += 1 + + if _tableCount > _options._maxTableCount { + throw FlatbuffersErrors.maximumTables + } + + _depth += 1 + + if _depth > _options._maxDepth { + throw FlatbuffersErrors.maximumDepth + } + + return TableVerifier( + position: position, + vtable: vtablePosition, + vtableLength: length, + verifier: &self) + } + + /// Validates if a value of type `T` is within the buffer and returns it + /// - Parameter position: Current position to be read + /// - Throws: `inBuffer` errors + /// - Returns: a value of type `T` usually a `VTable` or a table offset + internal mutating func getValue(at position: Int) throws -> T { + try inBuffer(position: position, of: T.self) + return _buffer.read(def: T.self, position: position) + } + + /// derefrences an offset within a vtable to get the position of the field + /// in the bytebuffer + /// - Parameter position: Current readable position + /// - Throws: `inBuffer` errors & `signedOffsetOutOfBounds` + /// - Returns: Current readable position for a field + @inline(__always) + internal mutating func derefOffset(position: Int) throws -> Int { + try inBuffer(position: position, of: Int32.self) + + let offset = _buffer.read(def: Int32.self, position: position) + // switching to int32 since swift's default Int is int64 + // this should be safe since we already checked if its within + // the buffer + let _int32Position = UInt32(position) + + let reportedOverflow: (partialValue: UInt32, overflow: Bool) + if offset > 0 { + reportedOverflow = _int32Position + .subtractingReportingOverflow(offset.magnitude) + } else { + reportedOverflow = _int32Position + .addingReportingOverflow(offset.magnitude) + } + + /// since `subtractingReportingOverflow` & `addingReportingOverflow` returns true, + /// if there is overflow we return failure + if reportedOverflow.overflow || reportedOverflow.partialValue > _buffer + .capacity + { + throw FlatbuffersErrors.signedOffsetOutOfBounds( + offset: Int(offset), + position: position) + } + + return Int(reportedOverflow.partialValue) + } + + /// finishes the current iteration of verification on an object + internal mutating func finish() { + _depth -= 1 + } + + mutating func verify(id: String) throws { + let size = MemoryLayout.size + let str = _buffer.readString(at: size, count: size) + if id == str { + return + } + throw FlatbuffersErrors.bufferIdDidntMatchPassedId + } + +} From 699f77d9e13d5b5c9e20320d1d95105d861f363f Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sat, 26 Aug 2023 20:44:08 -0700 Subject: [PATCH 038/111] add extended virtual memory entitlement This will allow larger nostrdb databases --- damus/damus.entitlements | 2 ++ 1 file changed, 2 insertions(+) diff --git a/damus/damus.entitlements b/damus/damus.entitlements index 29f0e7ef65..4bb26140f8 100644 --- a/damus/damus.entitlements +++ b/damus/damus.entitlements @@ -9,6 +9,8 @@ applinks:damus.io webcredentials:damus.io + com.apple.developer.kernel.extended-virtual-addressing + com.apple.security.app-sandbox com.apple.security.device.audio-input From 92bbc9766d821c410a344e412ebad5e677428cc3 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sat, 26 Aug 2023 20:45:42 -0700 Subject: [PATCH 039/111] project: disable compile warnings for lmdb and nostrdb --- damus.xcodeproj/project.pbxproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 171c505551..80329548a2 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -173,8 +173,8 @@ 4C3EA67D28FFBBA300C48A62 /* InvoicesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3EA67C28FFBBA200C48A62 /* InvoicesView.swift */; }; 4C3EA67F28FFC01D00C48A62 /* InvoiceView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3EA67E28FFC01D00C48A62 /* InvoiceView.swift */; }; 4C42812C298C848200DBF26F /* TranslateView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C42812B298C848200DBF26F /* TranslateView.swift */; }; - 4C4793012A993CDA00489948 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4793002A993B9A00489948 /* mdb.c */; }; - 4C4793042A993DC000489948 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4793032A993DB900489948 /* midl.c */; }; + 4C4793012A993CDA00489948 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4793002A993B9A00489948 /* mdb.c */; settings = {COMPILER_FLAGS = "-w"; }; }; + 4C4793042A993DC000489948 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4793032A993DB900489948 /* midl.c */; settings = {COMPILER_FLAGS = "-w"; }; }; 4C4793052A993E3200489948 /* builder.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792942A9939BD00489948 /* builder.c */; }; 4C4793062A993E5300489948 /* json_parser.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792C82A9939BD00489948 /* json_parser.c */; }; 4C4793072A993E6200489948 /* emitter.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792CF2A9939BD00489948 /* emitter.c */; }; @@ -351,7 +351,7 @@ 4CE879552996BAB900F758CC /* RelayPaidDetail.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CE879542996BAB900F758CC /* RelayPaidDetail.swift */; }; 4CE879582996C45300F758CC /* ZapsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CE879572996C45300F758CC /* ZapsView.swift */; }; 4CE8795B2996C47A00F758CC /* ZapsModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CE8795A2996C47A00F758CC /* ZapsModel.swift */; }; - 4CE9FBBA2A6B3C63007E485C /* nostrdb.c in Sources */ = {isa = PBXBuildFile; fileRef = 4CE9FBB82A6B3B26007E485C /* nostrdb.c */; }; + 4CE9FBBA2A6B3C63007E485C /* nostrdb.c in Sources */ = {isa = PBXBuildFile; fileRef = 4CE9FBB82A6B3B26007E485C /* nostrdb.c */; settings = {COMPILER_FLAGS = "-w"; }; }; 4CEE2AED2805B22500AB5EEF /* NostrRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2AEC2805B22500AB5EEF /* NostrRequest.swift */; }; 4CEE2AF1280B216B00AB5EEF /* EventDetailView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2AF0280B216B00AB5EEF /* EventDetailView.swift */; }; 4CEE2AF3280B25C500AB5EEF /* ProfilePicView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2AF2280B25C500AB5EEF /* ProfilePicView.swift */; }; From caffa0398b469d189d37f87f15d80a296c45a508 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sat, 26 Aug 2023 20:45:59 -0700 Subject: [PATCH 040/111] nostrdb: profile flatbuffers in nostrdb working! --- damus.xcodeproj/project.pbxproj | 1 + nostrdb/Ndb.swift | 23 +++++++++++---- nostrdb/Test/NdbTests.swift | 16 +++++++---- nostrdb/bindings/swift/NdbProfile.swift | 2 -- nostrdb/nostrdb.c | 38 ++++++++++++++++++------- nostrdb/nostrdb.h | 4 +-- 6 files changed, 60 insertions(+), 24 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 80329548a2..069c0a26f5 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -2476,6 +2476,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 4C32B9342A9AD01A00DC3548 /* NdbProfile.swift in Sources */, 4C32B9332A99845B00DC3548 /* Ndb.swift in Sources */, 4C4793082A993E8900489948 /* refmap.c in Sources */, 4C4793072A993E6200489948 /* emitter.c in Sources */, diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index b12e57ca89..32035b7e63 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -10,13 +10,15 @@ import Foundation class Ndb { let ndb: ndb_t + static var db_path: String { + (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: ""))! + } + init?() { var ndb_p: OpaquePointer? = nil - let dir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: "") - - let ok = dir!.withCString { testdir in - return ndb_init(&ndb_p, testdir, 1024 * 1024 * 700, 4) != 0 + let ok = Ndb.db_path.withCString { testdir in + return ndb_init(&ndb_p, testdir, 1024 * 1024 * 1024 * 32, 4) != 0 } if !ok { @@ -28,13 +30,24 @@ class Ndb { func lookup_note(_ id: NoteId) -> NdbNote? { id.id.withUnsafeBytes { bs in - guard let note_p = ndb_get_note_by_id(ndb.ndb, bs) else { + guard let note_p = ndb_get_note_by_id(ndb.ndb, bs, nil) else { return nil } return NdbNote(note: note_p, owned_size: nil) } } + func lookup_profile(_ pubkey: Pubkey) -> NdbProfile? { + return pubkey.id.withUnsafeBytes { pk_bytes in + var size: Int = 0 + guard let profile_p = ndb_get_profile_by_pubkey(ndb.ndb, pk_bytes, &size) else { + return nil + } + + return NdbProfile(.init(memory: profile_p, count: size), o: 0) + } + } + func process_events(_ str: String) -> Bool { return str.withCString { cstr in return ndb_process_events(ndb.ndb, cstr, str.utf8.count) != 0 diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift index 54c1b84633..846fff0c5d 100644 --- a/nostrdb/Test/NdbTests.swift +++ b/nostrdb/Test/NdbTests.swift @@ -12,6 +12,8 @@ final class NdbTests: XCTestCase { override func setUpWithError() throws { // Put setup code here. This method is called before the invocation of each test method in the class. + try FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb") + try FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb") } override func tearDownWithError() throws { @@ -41,14 +43,18 @@ final class NdbTests: XCTestCase { do { let ndb = Ndb()! - let id1 = NoteId(hex: "d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349")! - let note1 = ndb.lookup_note(id1) - XCTAssertNotNil(note1) - let id = NoteId(hex: "b2e03951843b191b5d9d1969f48db0156b83cc7dbd841f543f109362e24c4a9c")! + let id = NoteId(hex: "d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349")! let note = ndb.lookup_note(id) XCTAssertNotNil(note) guard let note else { return } - XCTAssertEqual(note.pubkey, Pubkey(hex: "32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245")!) + let pk = Pubkey(hex: "32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245")! + XCTAssertEqual(note.pubkey, pk) + + let profile = ndb.lookup_profile(pk) + XCTAssertNotNil(profile) + guard let profile else { return } + + XCTAssertEqual(profile.name, "jb55") } diff --git a/nostrdb/bindings/swift/NdbProfile.swift b/nostrdb/bindings/swift/NdbProfile.swift index 847cafdcb0..dab53bcdfb 100644 --- a/nostrdb/bindings/swift/NdbProfile.swift +++ b/nostrdb/bindings/swift/NdbProfile.swift @@ -2,8 +2,6 @@ // swiftlint:disable all // swiftformat:disable all -import FlatBuffers - public struct NdbProfile: FlatBufferObject, Verifiable { static func validateVersion() { FlatBuffersVersion_23_5_26() } diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index ea876d4e66..5689e6c257 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -275,29 +275,47 @@ int ndb_get_tsid(MDB_txn *txn, struct ndb_lmdb *lmdb, enum ndb_dbs db, return success; } -struct ndb_note *ndb_get_note_by_id(struct ndb *ndb, const unsigned char *id) +static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind, + enum ndb_dbs store, const unsigned char *pk, + size_t *len) { MDB_val k, v; MDB_txn *txn; + void *res = NULL; + if (len) + *len = 0; if (mdb_txn_begin(ndb->lmdb.env, 0, 0, &txn)) { ndb_debug("ndb_get_note_by_id: mdb_txn_begin failed\n"); return NULL; } - if (!ndb_get_tsid(txn, &ndb->lmdb, NDB_DB_NOTE_ID, id, &k)) { - ndb_debug("ndb_get_note_by_id: ndb_get_tsid failed\n"); - return NULL; + if (!ndb_get_tsid(txn, &ndb->lmdb, ind, pk, &k)) { + ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n"); + goto cleanup; } - if (mdb_get(txn, ndb->lmdb.dbs[NDB_DB_NOTE], &k, &v)) { - ndb_debug("ndb_get_note_by_id: mdb_get note failed\n"); - return NULL; + if (mdb_get(txn, ndb->lmdb.dbs[store], &k, &v)) { + ndb_debug("ndb_get_profile_by_pubkey: mdb_get note failed\n"); + goto cleanup; } + res = v.mv_data; + if (len) + *len = v.mv_size; +cleanup: mdb_txn_abort(txn); + return res; +} - return (struct ndb_note *)v.mv_data; +void *ndb_get_profile_by_pubkey(struct ndb *ndb, const unsigned char *pk, size_t *len) +{ + return ndb_lookup_tsid(ndb, NDB_DB_PROFILE_PK, NDB_DB_PROFILE, pk, len); +} + +struct ndb_note *ndb_get_note_by_id(struct ndb *ndb, const unsigned char *id, size_t *len) +{ + return ndb_lookup_tsid(ndb, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len); } static int ndb_has_note(MDB_txn *txn, struct ndb_lmdb *lmdb, const unsigned char *id) @@ -486,8 +504,8 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, // write profile to profile store key.mv_data = &profile_key; key.mv_size = sizeof(profile_key); - val.mv_data = profile->profile_flatbuf; - val.mv_size = profile->profile_len; + val.mv_data = profile->profile_flatbuf + 4; + val.mv_size = profile->profile_len - 4; //ndb_debug("profile_len %ld\n", profile->profile_len); if ((rc = mdb_put(txn, profile_db, &key, &val, 0))) { diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index 51d27eef81..d5062ffe44 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -155,8 +155,8 @@ int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[3 int ndb_init(struct ndb **ndb, const char *dbdir, size_t mapsize, int ingester_threads); int ndb_process_event(struct ndb *, const char *json, int len); int ndb_process_events(struct ndb *, const char *ldjson, size_t len); -int ndb_get_profile(struct ndb *, unsigned char pubkey[32], void **out); -struct ndb_note *ndb_get_note_by_id(struct ndb *, const unsigned char *id); +void *ndb_get_profile_by_pubkey(struct ndb *, const unsigned char *pubkey, size_t *len); +struct ndb_note *ndb_get_note_by_id(struct ndb *, const unsigned char *id, size_t *len); void ndb_destroy(struct ndb *); // BUILDER From 50d55572be1d295a79be3cf0e6e87448ce004cdf Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 08:00:03 -0700 Subject: [PATCH 041/111] Fix crash when long pressing custom reactions Changelog-Fixed: Fix crash when long pressing custom reactions --- damus/Views/ActionBar/EventActionBar.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift index bc96c2df39..f314fd62b4 100644 --- a/damus/Views/ActionBar/EventActionBar.swift +++ b/damus/Views/ActionBar/EventActionBar.swift @@ -329,7 +329,9 @@ struct LikeButton: View { for (index, _) in emojis.enumerated() { DispatchQueue.main.asyncAfter(deadline: .now() + 0.1 * Double(index)) { withAnimation(.interpolatingSpring(stiffness: 170, damping: 8)) { - showEmojis[index] = 1 + if index < showEmojis.count { + showEmojis[index] = 1 + } } } } From c44c0d08631566e7e9ba955a48ba4fd8db3e8b6c Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sat, 26 Aug 2023 21:38:21 -0700 Subject: [PATCH 042/111] profile: remove deleted flag it's not used anymore --- damus/Nostr/Nostr.swift | 5 ----- damus/Util/AccountDeletion.swift | 1 - 2 files changed, 6 deletions(-) diff --git a/damus/Nostr/Nostr.swift b/damus/Nostr/Nostr.swift index 1e439f2547..ea927b7a79 100644 --- a/damus/Nostr/Nostr.swift +++ b/damus/Nostr/Nostr.swift @@ -79,11 +79,6 @@ class Profile: Codable { set(s) { set_val("reactions", s) } } - var deleted: Bool? { - get { return get_val("deleted"); } - set(s) { set_val("deleted", s) } - } - var display_name: String? { get { return str("display_name"); } set(s) { set_str("display_name", s) } diff --git a/damus/Util/AccountDeletion.swift b/damus/Util/AccountDeletion.swift index 70654e69fe..faf9c06295 100644 --- a/damus/Util/AccountDeletion.swift +++ b/damus/Util/AccountDeletion.swift @@ -10,7 +10,6 @@ import Foundation func created_deleted_account_profile(keypair: FullKeypair) -> NostrEvent? { let profile = Profile() - profile.deleted = true profile.about = "account deleted" profile.name = "nobody" From 0bbc2c6348428fa43c830dac2eb22b328c9d8830 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sat, 26 Aug 2023 21:58:37 -0700 Subject: [PATCH 043/111] ndb: save in documents instead of cache dir This is more long term storage --- nostrdb/Ndb.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index 32035b7e63..df97e1b854 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -11,7 +11,7 @@ class Ndb { let ndb: ndb_t static var db_path: String { - (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: ""))! + (FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: ""))! } init?() { From 984c7b6932b89c3fe05b312e04775d145ae3fc56 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sat, 26 Aug 2023 22:11:12 -0700 Subject: [PATCH 044/111] ndb: ensure profile flatbuffers are not copied These are pointers into LMDB's virtual memory map of the database. No copy required. --- nostrdb/Ndb.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index df97e1b854..ccefba04a7 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -44,7 +44,8 @@ class Ndb { return nil } - return NdbProfile(.init(memory: profile_p, count: size), o: 0) + let buf = ByteBuffer(assumingMemoryBound: profile_p, capacity: size) + return NdbProfile(buf, o: 0) } } From ba6792640d108b71b1e86bc0a33a53c7f17b8852 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 08:17:25 -0700 Subject: [PATCH 045/111] flatbuffers: update bindings, add verifier --- .../bindings/c/flatbuffers_common_builder.h | 6 +- nostrdb/bindings/c/meta_builder.h | 4 +- nostrdb/bindings/c/meta_json_parser.h | 6 +- nostrdb/bindings/c/meta_reader.h | 6 +- nostrdb/bindings/c/meta_verifier.h | 6 +- nostrdb/bindings/c/profile_builder.h | 53 +++++- nostrdb/bindings/c/profile_json_parser.h | 168 +++++++++++++++--- nostrdb/bindings/c/profile_reader.h | 36 +++- nostrdb/bindings/c/profile_verifier.h | 38 +++- nostrdb/bindings/swift/NdbProfile.swift | 127 ++++++++++++- nostrdb/flatcc/flatcc_alloc.h | 2 +- nostrdb/flatcc/flatcc_epilogue.h | 2 +- nostrdb/flatcc/flatcc_flatbuffers.h | 12 +- nostrdb/flatcc/flatcc_json_parser.h | 4 +- nostrdb/flatcc/flatcc_json_printer.h | 4 +- nostrdb/flatcc/flatcc_portable.h | 2 +- nostrdb/flatcc/flatcc_prologue.h | 2 +- nostrdb/flatcc/flatcc_rtconfig.h | 2 +- nostrdb/flatcc/flatcc_unaligned.h | 2 +- nostrdb/flatcc/flatcc_verifier.h | 2 +- nostrdb/flatcc/json_printer.c | 16 +- .../reflection/flatbuffers_common_reader.h | 1 + nostrdb/flatcc/refmap.c | 2 +- nostrdb/flatcc/verifier.c | 8 +- 24 files changed, 434 insertions(+), 77 deletions(-) diff --git a/nostrdb/bindings/c/flatbuffers_common_builder.h b/nostrdb/bindings/c/flatbuffers_common_builder.h index a4be1ce6e8..6266b98946 100644 --- a/nostrdb/bindings/c/flatbuffers_common_builder.h +++ b/nostrdb/bindings/c/flatbuffers_common_builder.h @@ -5,9 +5,9 @@ /* Common FlatBuffers build functionality for C. */ -#include "flatcc/flatcc_prologue.h" +#include "flatcc_prologue.h" #ifndef FLATBUILDER_H -#include "flatcc/flatcc_builder.h" +#include "flatcc_builder.h" #endif typedef flatcc_builder_t flatbuffers_builder_t; typedef flatcc_builder_ref_t flatbuffers_ref_t; @@ -681,5 +681,5 @@ __flatbuffers_build_scalar(flatbuffers_, flatbuffers_double, double) __flatbuffers_build_string(flatbuffers_) __flatbuffers_build_buffer(flatbuffers_) -#include "flatcc/flatcc_epilogue.h" +#include "flatcc_epilogue.h" #endif /* FLATBUFFERS_COMMON_BUILDER_H */ diff --git a/nostrdb/bindings/c/meta_builder.h b/nostrdb/bindings/c/meta_builder.h index 19521e17eb..1ae48f3413 100644 --- a/nostrdb/bindings/c/meta_builder.h +++ b/nostrdb/bindings/c/meta_builder.h @@ -9,7 +9,7 @@ #ifndef FLATBUFFERS_COMMON_BUILDER_H #include "flatbuffers_common_builder.h" #endif -#include "flatcc/flatcc_prologue.h" +#include "flatcc_prologue.h" #ifndef flatbuffers_identifier #define flatbuffers_identifier 0 #endif @@ -48,5 +48,5 @@ static NdbEventMeta_ref_t NdbEventMeta_clone(flatbuffers_builder_t *B, NdbEventM __flatbuffers_memoize_end(B, t, NdbEventMeta_end(B)); } -#include "flatcc/flatcc_epilogue.h" +#include "flatcc_epilogue.h" #endif /* META_BUILDER_H */ diff --git a/nostrdb/bindings/c/meta_json_parser.h b/nostrdb/bindings/c/meta_json_parser.h index 1c18728fa1..b008106284 100644 --- a/nostrdb/bindings/c/meta_json_parser.h +++ b/nostrdb/bindings/c/meta_json_parser.h @@ -3,8 +3,8 @@ /* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ -#include "flatcc/flatcc_json_parser.h" -#include "flatcc/flatcc_prologue.h" +#include "flatcc_json_parser.h" +#include "flatcc_prologue.h" /* * Parses the default root table or struct of the schema and constructs a FlatBuffer. @@ -112,5 +112,5 @@ static int meta_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, return 0; } -#include "flatcc/flatcc_epilogue.h" +#include "flatcc_epilogue.h" #endif /* META_JSON_PARSER_H */ diff --git a/nostrdb/bindings/c/meta_reader.h b/nostrdb/bindings/c/meta_reader.h index 1e72408e62..e72746d0d2 100644 --- a/nostrdb/bindings/c/meta_reader.h +++ b/nostrdb/bindings/c/meta_reader.h @@ -6,11 +6,11 @@ #ifndef FLATBUFFERS_COMMON_READER_H #include "flatbuffers_common_reader.h" #endif -#include "flatcc/flatcc_flatbuffers.h" +#include "flatcc_flatbuffers.h" #ifndef __alignas_is_defined #include #endif -#include "flatcc/flatcc_prologue.h" +#include "flatcc_prologue.h" #ifndef flatbuffers_identifier #define flatbuffers_identifier 0 #endif @@ -49,5 +49,5 @@ __flatbuffers_table_as_root(NdbEventMeta) __flatbuffers_define_scalar_field(0, NdbEventMeta, received_at, flatbuffers_int32, int32_t, INT32_C(0)) -#include "flatcc/flatcc_epilogue.h" +#include "flatcc_epilogue.h" #endif /* META_READER_H */ diff --git a/nostrdb/bindings/c/meta_verifier.h b/nostrdb/bindings/c/meta_verifier.h index 713ba38b37..d2b2df7224 100644 --- a/nostrdb/bindings/c/meta_verifier.h +++ b/nostrdb/bindings/c/meta_verifier.h @@ -6,8 +6,8 @@ #ifndef META_READER_H #include "meta_reader.h" #endif -#include "flatcc/flatcc_verifier.h" -#include "flatcc/flatcc_prologue.h" +#include "flatcc_verifier.h" +#include "flatcc_prologue.h" static int NdbEventMeta_verify_table(flatcc_table_verifier_descriptor_t *td); @@ -38,5 +38,5 @@ static inline int NdbEventMeta_verify_as_root_with_type_hash(const void *buf, si return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &NdbEventMeta_verify_table); } -#include "flatcc/flatcc_epilogue.h" +#include "flatcc_epilogue.h" #endif /* META_VERIFIER_H */ diff --git a/nostrdb/bindings/c/profile_builder.h b/nostrdb/bindings/c/profile_builder.h index 185fba73e3..97341d552e 100644 --- a/nostrdb/bindings/c/profile_builder.h +++ b/nostrdb/bindings/c/profile_builder.h @@ -9,7 +9,7 @@ #ifndef FLATBUFFERS_COMMON_BUILDER_H #include "flatbuffers_common_builder.h" #endif -#include "flatcc/flatcc_prologue.h" +#include "flatcc_prologue.h" #ifndef flatbuffers_identifier #define flatbuffers_identifier 0 #endif @@ -20,19 +20,29 @@ static const flatbuffers_voffset_t __NdbProfile_required[] = { 0 }; typedef flatbuffers_ref_t NdbProfile_ref_t; static NdbProfile_ref_t NdbProfile_clone(flatbuffers_builder_t *B, NdbProfile_table_t t); -__flatbuffers_build_table(flatbuffers_, NdbProfile, 11) +__flatbuffers_build_table(flatbuffers_, NdbProfile, 12) + +static const flatbuffers_voffset_t __NdbProfileRecord_required[] = { 0 }; +typedef flatbuffers_ref_t NdbProfileRecord_ref_t; +static NdbProfileRecord_ref_t NdbProfileRecord_clone(flatbuffers_builder_t *B, NdbProfileRecord_table_t t); +__flatbuffers_build_table(flatbuffers_, NdbProfileRecord, 4) #define __NdbProfile_formal_args ,\ flatbuffers_string_ref_t v0, flatbuffers_string_ref_t v1, flatbuffers_string_ref_t v2, flatbuffers_string_ref_t v3,\ flatbuffers_string_ref_t v4, flatbuffers_string_ref_t v5, flatbuffers_bool_t v6, flatbuffers_string_ref_t v7,\ - flatbuffers_string_ref_t v8, int32_t v9, int32_t v10 + flatbuffers_string_ref_t v8, int32_t v9, int32_t v10, flatbuffers_string_ref_t v11 #define __NdbProfile_call_args ,\ v0, v1, v2, v3,\ v4, v5, v6, v7,\ - v8, v9, v10 + v8, v9, v10, v11 static inline NdbProfile_ref_t NdbProfile_create(flatbuffers_builder_t *B __NdbProfile_formal_args); __flatbuffers_build_table_prolog(flatbuffers_, NdbProfile, NdbProfile_file_identifier, NdbProfile_type_identifier) +#define __NdbProfileRecord_formal_args , NdbProfile_ref_t v0, uint64_t v1, uint64_t v2, flatbuffers_string_ref_t v3 +#define __NdbProfileRecord_call_args , v0, v1, v2, v3 +static inline NdbProfileRecord_ref_t NdbProfileRecord_create(flatbuffers_builder_t *B __NdbProfileRecord_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, NdbProfileRecord, NdbProfileRecord_file_identifier, NdbProfileRecord_type_identifier) + __flatbuffers_build_string_field(0, flatbuffers_, NdbProfile_name, NdbProfile) __flatbuffers_build_string_field(1, flatbuffers_, NdbProfile_website, NdbProfile) __flatbuffers_build_string_field(2, flatbuffers_, NdbProfile_about, NdbProfile) @@ -44,6 +54,7 @@ __flatbuffers_build_string_field(7, flatbuffers_, NdbProfile_picture, NdbProfile __flatbuffers_build_string_field(8, flatbuffers_, NdbProfile_nip05, NdbProfile) __flatbuffers_build_scalar_field(9, flatbuffers_, NdbProfile_damus_donation, flatbuffers_int32, int32_t, 4, 4, INT32_C(0), NdbProfile) __flatbuffers_build_scalar_field(10, flatbuffers_, NdbProfile_damus_donation_v2, flatbuffers_int32, int32_t, 4, 4, INT32_C(0), NdbProfile) +__flatbuffers_build_string_field(11, flatbuffers_, NdbProfile_lud06, NdbProfile) static inline NdbProfile_ref_t NdbProfile_create(flatbuffers_builder_t *B __NdbProfile_formal_args) { @@ -58,6 +69,7 @@ static inline NdbProfile_ref_t NdbProfile_create(flatbuffers_builder_t *B __NdbP || NdbProfile_nip05_add(B, v8) || NdbProfile_damus_donation_add(B, v9) || NdbProfile_damus_donation_v2_add(B, v10) + || NdbProfile_lud06_add(B, v11) || NdbProfile_reactions_add(B, v6)) { return 0; } @@ -78,11 +90,42 @@ static NdbProfile_ref_t NdbProfile_clone(flatbuffers_builder_t *B, NdbProfile_ta || NdbProfile_nip05_pick(B, t) || NdbProfile_damus_donation_pick(B, t) || NdbProfile_damus_donation_v2_pick(B, t) + || NdbProfile_lud06_pick(B, t) || NdbProfile_reactions_pick(B, t)) { return 0; } __flatbuffers_memoize_end(B, t, NdbProfile_end(B)); } -#include "flatcc/flatcc_epilogue.h" +__flatbuffers_build_table_field(0, flatbuffers_, NdbProfileRecord_profile, NdbProfile, NdbProfileRecord) +__flatbuffers_build_scalar_field(1, flatbuffers_, NdbProfileRecord_received_at, flatbuffers_uint64, uint64_t, 8, 8, UINT64_C(0), NdbProfileRecord) +__flatbuffers_build_scalar_field(2, flatbuffers_, NdbProfileRecord_note_key, flatbuffers_uint64, uint64_t, 8, 8, UINT64_C(0), NdbProfileRecord) +__flatbuffers_build_string_field(3, flatbuffers_, NdbProfileRecord_lnurl, NdbProfileRecord) + +static inline NdbProfileRecord_ref_t NdbProfileRecord_create(flatbuffers_builder_t *B __NdbProfileRecord_formal_args) +{ + if (NdbProfileRecord_start(B) + || NdbProfileRecord_received_at_add(B, v1) + || NdbProfileRecord_note_key_add(B, v2) + || NdbProfileRecord_profile_add(B, v0) + || NdbProfileRecord_lnurl_add(B, v3)) { + return 0; + } + return NdbProfileRecord_end(B); +} + +static NdbProfileRecord_ref_t NdbProfileRecord_clone(flatbuffers_builder_t *B, NdbProfileRecord_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (NdbProfileRecord_start(B) + || NdbProfileRecord_received_at_pick(B, t) + || NdbProfileRecord_note_key_pick(B, t) + || NdbProfileRecord_profile_pick(B, t) + || NdbProfileRecord_lnurl_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, NdbProfileRecord_end(B)); +} + +#include "flatcc_epilogue.h" #endif /* PROFILE_BUILDER_H */ diff --git a/nostrdb/bindings/c/profile_json_parser.h b/nostrdb/bindings/c/profile_json_parser.h index f404b6679f..5c33a7442b 100644 --- a/nostrdb/bindings/c/profile_json_parser.h +++ b/nostrdb/bindings/c/profile_json_parser.h @@ -19,6 +19,7 @@ static int profile_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, const char *buf, size_t bufsiz, int flags); static const char *NdbProfile_parse_json_table(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_builder_ref_t *result); +static const char *NdbProfileRecord_parse_json_table(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_builder_ref_t *result); static const char *profile_local_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, int *value_type, uint64_t *value, int *aggregate); static const char *profile_global_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, @@ -33,12 +34,12 @@ static const char *NdbProfile_parse_json_table(flatcc_json_parser_t *ctx, const uint64_t w; *result = 0; - if (flatcc_builder_start_table(ctx->ctx, 11)) goto failed; + if (flatcc_builder_start_table(ctx->ctx, 12)) goto failed; buf = flatcc_json_parser_object_start(ctx, buf, end, &more); while (more) { buf = flatcc_json_parser_symbol_start(ctx, buf, end); w = flatcc_json_parser_symbol_part(buf, end); - if (w < 0x6c75643136000000) { /* branch "lud16" */ + if (w < 0x6c75643036000000) { /* branch "lud06" */ if (w < 0x64616d75735f646f) { /* branch "damus_do" */ if ((w & 0xffffffffffff0000) == 0x62616e6e65720000) { /* "banner" */ buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 6); @@ -137,31 +138,46 @@ static const char *NdbProfile_parse_json_table(flatcc_json_parser_t *ctx, const } /* descend "display_" */ } /* descend "damus_do" */ } /* branch "damus_do" */ - } else { /* branch "lud16" */ + } else { /* branch "lud06" */ if (w < 0x6e69703035000000) { /* branch "nip05" */ - if ((w & 0xffffffff00000000) == 0x6e616d6500000000) { /* "name" */ - buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 4); - if (mark != buf) { - buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); - if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 0))) goto failed; - *pref = ref; - } else { - buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); - } - } else { /* "name" */ - if ((w & 0xffffffffff000000) == 0x6c75643136000000) { /* "lud16" */ + if (w < 0x6c75643136000000) { /* branch "lud16" */ + if ((w & 0xffffffffff000000) == 0x6c75643036000000) { /* "lud06" */ buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 5); if (mark != buf) { buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); - if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 3))) goto failed; + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 11))) goto failed; *pref = ref; } else { buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); } - } else { /* "lud16" */ + } else { /* "lud06" */ buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); - } /* "lud16" */ - } /* "name" */ + } /* "lud06" */ + } else { /* branch "lud16" */ + if ((w & 0xffffffff00000000) == 0x6e616d6500000000) { /* "name" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 4); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 0))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "name" */ + if ((w & 0xffffffffff000000) == 0x6c75643136000000) { /* "lud16" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 5); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 3))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "lud16" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "lud16" */ + } /* "name" */ + } /* branch "lud16" */ } else { /* branch "nip05" */ if (w < 0x7069637475726500) { /* branch "picture" */ if ((w & 0xffffffffff000000) == 0x6e69703035000000) { /* "nip05" */ @@ -233,7 +249,7 @@ static const char *NdbProfile_parse_json_table(flatcc_json_parser_t *ctx, const } /* branch "reaction" */ } /* branch "picture" */ } /* branch "nip05" */ - } /* branch "lud16" */ + } /* branch "lud06" */ buf = flatcc_json_parser_object_end(ctx, buf, end, &more); } if (ctx->error) goto failed; @@ -248,6 +264,118 @@ static inline int NdbProfile_parse_json_as_root(flatcc_builder_t *B, flatcc_json return flatcc_json_parser_table_as_root(B, ctx, buf, bufsiz, flags, fid, NdbProfile_parse_json_table); } +static const char *NdbProfileRecord_parse_json_table(flatcc_json_parser_t *ctx, const char *buf, const char *end, flatcc_builder_ref_t *result) +{ + int more; + void *pval; + flatcc_builder_ref_t ref, *pref; + const char *mark; + uint64_t w; + + *result = 0; + if (flatcc_builder_start_table(ctx->ctx, 4)) goto failed; + buf = flatcc_json_parser_object_start(ctx, buf, end, &more); + while (more) { + buf = flatcc_json_parser_symbol_start(ctx, buf, end); + w = flatcc_json_parser_symbol_part(buf, end); + if (w < 0x6e6f74655f6b6579) { /* branch "note_key" */ + if ((w & 0xffffffffff000000) == 0x6c6e75726c000000) { /* "lnurl" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 5); + if (mark != buf) { + buf = flatcc_json_parser_build_string(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 3))) goto failed; + *pref = ref; + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "lnurl" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "lnurl" */ + } else { /* branch "note_key" */ + if (w < 0x70726f66696c6500) { /* branch "profile" */ + if (w == 0x6e6f74655f6b6579) { /* "note_key" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 8); + if (mark != buf) { + uint64_t val = 0; + static flatcc_json_parser_integral_symbol_f *symbolic_parsers[] = { + profile_local_json_parser_enum, + profile_global_json_parser_enum, 0 }; + buf = flatcc_json_parser_uint64(ctx, (mark = buf), end, &val); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_uint64(ctx, (mark = buf), end, symbolic_parsers, &val); + if (buf == mark || buf == end) goto failed; + } + if (val != UINT64_C(0) || (ctx->flags & flatcc_json_parser_f_force_add)) { + if (!(pval = flatcc_builder_table_add(ctx->ctx, 2, 8, 8))) goto failed; + flatbuffers_uint64_write_to_pe(pval, val); + } + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "note_key" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "note_key" */ + } else { /* branch "profile" */ + if ((w & 0xffffffffffffff00) == 0x70726f66696c6500) { /* "profile" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 7); + if (mark != buf) { + buf = NdbProfile_parse_json_table(ctx, buf, end, &ref); + if (!ref || !(pref = flatcc_builder_table_add_offset(ctx->ctx, 0))) goto failed; + *pref = ref; + } else { + goto pfguard1; + } + } else { /* "profile" */ + goto pfguard1; + } /* "profile" */ + goto endpfguard1; +pfguard1: + if (w == 0x7265636569766564) { /* descend "received" */ + buf += 8; + w = flatcc_json_parser_symbol_part(buf, end); + if ((w & 0xffffff0000000000) == 0x5f61740000000000) { /* "_at" */ + buf = flatcc_json_parser_match_symbol(ctx, (mark = buf), end, 3); + if (mark != buf) { + uint64_t val = 0; + static flatcc_json_parser_integral_symbol_f *symbolic_parsers[] = { + profile_local_json_parser_enum, + profile_global_json_parser_enum, 0 }; + buf = flatcc_json_parser_uint64(ctx, (mark = buf), end, &val); + if (mark == buf) { + buf = flatcc_json_parser_symbolic_uint64(ctx, (mark = buf), end, symbolic_parsers, &val); + if (buf == mark || buf == end) goto failed; + } + if (val != UINT64_C(0) || (ctx->flags & flatcc_json_parser_f_force_add)) { + if (!(pval = flatcc_builder_table_add(ctx->ctx, 1, 8, 8))) goto failed; + flatbuffers_uint64_write_to_pe(pval, val); + } + } else { + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } + } else { /* "_at" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* "_at" */ + } else { /* descend "received" */ + buf = flatcc_json_parser_unmatched_symbol(ctx, buf, end); + } /* descend "received" */ +endpfguard1: + (void)0; + } /* branch "profile" */ + } /* branch "note_key" */ + buf = flatcc_json_parser_object_end(ctx, buf, end, &more); + } + if (ctx->error) goto failed; + if (!(*result = flatcc_builder_end_table(ctx->ctx))) goto failed; + return buf; +failed: + return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_runtime); +} + +static inline int NdbProfileRecord_parse_json_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx, const char *buf, size_t bufsiz, int flags, const char *fid) +{ + return flatcc_json_parser_table_as_root(B, ctx, buf, bufsiz, flags, fid, NdbProfileRecord_parse_json_table); +} + static const char *profile_local_json_parser_enum(flatcc_json_parser_t *ctx, const char *buf, const char *end, int *value_type, uint64_t *value, int *aggregate) { @@ -271,7 +399,7 @@ static int profile_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, ctx = ctx ? ctx : &parser; flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags); if (flatcc_builder_start_buffer(B, 0, 0, 0)) return -1; - NdbProfile_parse_json_table(ctx, buf, buf + bufsiz, &root); + NdbProfileRecord_parse_json_table(ctx, buf, buf + bufsiz, &root); if (ctx->error) { return ctx->error; } diff --git a/nostrdb/bindings/c/profile_reader.h b/nostrdb/bindings/c/profile_reader.h index ddc8fad446..069ec2cbc5 100644 --- a/nostrdb/bindings/c/profile_reader.h +++ b/nostrdb/bindings/c/profile_reader.h @@ -6,11 +6,11 @@ #ifndef FLATBUFFERS_COMMON_READER_H #include "flatbuffers_common_reader.h" #endif -#include "flatcc/flatcc_flatbuffers.h" +#include "flatcc_flatbuffers.h" #ifndef __alignas_is_defined #include #endif -#include "flatcc/flatcc_prologue.h" +#include "flatcc_prologue.h" #ifndef flatbuffers_identifier #define flatbuffers_identifier 0 #endif @@ -23,6 +23,10 @@ typedef const struct NdbProfile_table *NdbProfile_table_t; typedef struct NdbProfile_table *NdbProfile_mutable_table_t; typedef const flatbuffers_uoffset_t *NdbProfile_vec_t; typedef flatbuffers_uoffset_t *NdbProfile_mutable_vec_t; +typedef const struct NdbProfileRecord_table *NdbProfileRecord_table_t; +typedef struct NdbProfileRecord_table *NdbProfileRecord_mutable_table_t; +typedef const flatbuffers_uoffset_t *NdbProfileRecord_vec_t; +typedef flatbuffers_uoffset_t *NdbProfileRecord_mutable_vec_t; #ifndef NdbProfile_file_identifier #define NdbProfile_file_identifier 0 #endif @@ -35,6 +39,18 @@ typedef flatbuffers_uoffset_t *NdbProfile_mutable_vec_t; #ifndef NdbProfile_file_extension #define NdbProfile_file_extension "bin" #endif +#ifndef NdbProfileRecord_file_identifier +#define NdbProfileRecord_file_identifier 0 +#endif +/* deprecated, use NdbProfileRecord_file_identifier */ +#ifndef NdbProfileRecord_identifier +#define NdbProfileRecord_identifier 0 +#endif +#define NdbProfileRecord_type_hash ((flatbuffers_thash_t)0xa1a8569d) +#define NdbProfileRecord_type_identifier "\x9d\x56\xa8\xa1" +#ifndef NdbProfileRecord_file_extension +#define NdbProfileRecord_file_extension "bin" +#endif @@ -57,7 +73,21 @@ __flatbuffers_define_string_field(7, NdbProfile, picture, 0) __flatbuffers_define_string_field(8, NdbProfile, nip05, 0) __flatbuffers_define_scalar_field(9, NdbProfile, damus_donation, flatbuffers_int32, int32_t, INT32_C(0)) __flatbuffers_define_scalar_field(10, NdbProfile, damus_donation_v2, flatbuffers_int32, int32_t, INT32_C(0)) +__flatbuffers_define_string_field(11, NdbProfile, lud06, 0) + +struct NdbProfileRecord_table { uint8_t unused__; }; + +static inline size_t NdbProfileRecord_vec_len(NdbProfileRecord_vec_t vec) +__flatbuffers_vec_len(vec) +static inline NdbProfileRecord_table_t NdbProfileRecord_vec_at(NdbProfileRecord_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(NdbProfileRecord_table_t, vec, i, 0) +__flatbuffers_table_as_root(NdbProfileRecord) + +__flatbuffers_define_table_field(0, NdbProfileRecord, profile, NdbProfile_table_t, 0) +__flatbuffers_define_scalar_field(1, NdbProfileRecord, received_at, flatbuffers_uint64, uint64_t, UINT64_C(0)) +__flatbuffers_define_scalar_field(2, NdbProfileRecord, note_key, flatbuffers_uint64, uint64_t, UINT64_C(0)) +__flatbuffers_define_string_field(3, NdbProfileRecord, lnurl, 0) -#include "flatcc/flatcc_epilogue.h" +#include "flatcc_epilogue.h" #endif /* PROFILE_READER_H */ diff --git a/nostrdb/bindings/c/profile_verifier.h b/nostrdb/bindings/c/profile_verifier.h index 3fc2ae960f..9595ab7aa2 100644 --- a/nostrdb/bindings/c/profile_verifier.h +++ b/nostrdb/bindings/c/profile_verifier.h @@ -6,10 +6,11 @@ #ifndef PROFILE_READER_H #include "profile_reader.h" #endif -#include "flatcc/flatcc_verifier.h" -#include "flatcc/flatcc_prologue.h" +#include "flatcc_verifier.h" +#include "flatcc_prologue.h" static int NdbProfile_verify_table(flatcc_table_verifier_descriptor_t *td); +static int NdbProfileRecord_verify_table(flatcc_table_verifier_descriptor_t *td); static int NdbProfile_verify_table(flatcc_table_verifier_descriptor_t *td) { @@ -25,6 +26,7 @@ static int NdbProfile_verify_table(flatcc_table_verifier_descriptor_t *td) if ((ret = flatcc_verify_string_field(td, 8, 0) /* nip05 */)) return ret; if ((ret = flatcc_verify_field(td, 9, 4, 4) /* damus_donation */)) return ret; if ((ret = flatcc_verify_field(td, 10, 4, 4) /* damus_donation_v2 */)) return ret; + if ((ret = flatcc_verify_string_field(td, 11, 0) /* lud06 */)) return ret; return flatcc_verify_ok; } @@ -48,5 +50,35 @@ static inline int NdbProfile_verify_as_root_with_type_hash(const void *buf, size return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &NdbProfile_verify_table); } -#include "flatcc/flatcc_epilogue.h" +static int NdbProfileRecord_verify_table(flatcc_table_verifier_descriptor_t *td) +{ + int ret; + if ((ret = flatcc_verify_table_field(td, 0, 0, &NdbProfile_verify_table) /* profile */)) return ret; + if ((ret = flatcc_verify_field(td, 1, 8, 8) /* received_at */)) return ret; + if ((ret = flatcc_verify_field(td, 2, 8, 8) /* note_key */)) return ret; + if ((ret = flatcc_verify_string_field(td, 3, 0) /* lnurl */)) return ret; + return flatcc_verify_ok; +} + +static inline int NdbProfileRecord_verify_as_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, NdbProfileRecord_identifier, &NdbProfileRecord_verify_table); +} + +static inline int NdbProfileRecord_verify_as_typed_root(const void *buf, size_t bufsiz) +{ + return flatcc_verify_table_as_root(buf, bufsiz, NdbProfileRecord_type_identifier, &NdbProfileRecord_verify_table); +} + +static inline int NdbProfileRecord_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid) +{ + return flatcc_verify_table_as_root(buf, bufsiz, fid, &NdbProfileRecord_verify_table); +} + +static inline int NdbProfileRecord_verify_as_root_with_type_hash(const void *buf, size_t bufsiz, flatbuffers_thash_t thash) +{ + return flatcc_verify_table_as_typed_root(buf, bufsiz, thash, &NdbProfileRecord_verify_table); +} + +#include "flatcc_epilogue.h" #endif /* PROFILE_VERIFIER_H */ diff --git a/nostrdb/bindings/swift/NdbProfile.swift b/nostrdb/bindings/swift/NdbProfile.swift index dab53bcdfb..513dfa428d 100644 --- a/nostrdb/bindings/swift/NdbProfile.swift +++ b/nostrdb/bindings/swift/NdbProfile.swift @@ -2,6 +2,8 @@ // swiftlint:disable all // swiftformat:disable all + + public struct NdbProfile: FlatBufferObject, Verifiable { static func validateVersion() { FlatBuffersVersion_23_5_26() } @@ -23,6 +25,7 @@ public struct NdbProfile: FlatBufferObject, Verifiable { case nip05 = 20 case damusDonation = 22 case damusDonationV2 = 24 + case lud06 = 26 var v: Int32 { Int32(self.rawValue) } var p: VOffset { self.rawValue } } @@ -46,7 +49,9 @@ public struct NdbProfile: FlatBufferObject, Verifiable { public var nip05SegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.nip05.v) } public var damusDonation: Int32 { let o = _accessor.offset(VTOFFSET.damusDonation.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } public var damusDonationV2: Int32 { let o = _accessor.offset(VTOFFSET.damusDonationV2.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } - public static func startNdbProfile(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 11) } + public var lud06: String? { let o = _accessor.offset(VTOFFSET.lud06.v); return o == 0 ? nil : _accessor.string(at: o) } + public var lud06SegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.lud06.v) } + public static func startNdbProfile(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 12) } public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: name, at: VTOFFSET.name.p) } public static func add(website: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: website, at: VTOFFSET.website.p) } public static func add(about: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: about, at: VTOFFSET.about.p) } @@ -59,6 +64,7 @@ public struct NdbProfile: FlatBufferObject, Verifiable { public static func add(nip05: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: nip05, at: VTOFFSET.nip05.p) } public static func add(damusDonation: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: damusDonation, def: 0, at: VTOFFSET.damusDonation.p) } public static func add(damusDonationV2: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: damusDonationV2, def: 0, at: VTOFFSET.damusDonationV2.p) } + public static func add(lud06: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: lud06, at: VTOFFSET.lud06.p) } public static func endNdbProfile(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } public static func createNdbProfile( _ fbb: inout FlatBufferBuilder, @@ -72,7 +78,8 @@ public struct NdbProfile: FlatBufferObject, Verifiable { pictureOffset picture: Offset = Offset(), nip05Offset nip05: Offset = Offset(), damusDonation: Int32 = 0, - damusDonationV2: Int32 = 0 + damusDonationV2: Int32 = 0, + lud06Offset lud06: Offset = Offset() ) -> Offset { let __start = NdbProfile.startNdbProfile(&fbb) NdbProfile.add(name: name, &fbb) @@ -86,6 +93,7 @@ public struct NdbProfile: FlatBufferObject, Verifiable { NdbProfile.add(nip05: nip05, &fbb) NdbProfile.add(damusDonation: damusDonation, &fbb) NdbProfile.add(damusDonationV2: damusDonationV2, &fbb) + NdbProfile.add(lud06: lud06, &fbb) return NdbProfile.endNdbProfile(&fbb, start: __start) } @@ -102,7 +110,122 @@ public struct NdbProfile: FlatBufferObject, Verifiable { try _v.visit(field: VTOFFSET.nip05.p, fieldName: "nip05", required: false, type: ForwardOffset.self) try _v.visit(field: VTOFFSET.damusDonation.p, fieldName: "damusDonation", required: false, type: Int32.self) try _v.visit(field: VTOFFSET.damusDonationV2.p, fieldName: "damusDonationV2", required: false, type: Int32.self) + try _v.visit(field: VTOFFSET.lud06.p, fieldName: "lud06", required: false, type: ForwardOffset.self) _v.finish() } } +extension NdbProfile: Encodable { + + enum CodingKeys: String, CodingKey { + case name = "name" + case website = "website" + case about = "about" + case lud16 = "lud16" + case banner = "banner" + case displayName = "display_name" + case reactions = "reactions" + case picture = "picture" + case nip05 = "nip05" + case damusDonation = "damus_donation" + case damusDonationV2 = "damus_donation_v2" + case lud06 = "lud06" + } + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encodeIfPresent(name, forKey: .name) + try container.encodeIfPresent(website, forKey: .website) + try container.encodeIfPresent(about, forKey: .about) + try container.encodeIfPresent(lud16, forKey: .lud16) + try container.encodeIfPresent(banner, forKey: .banner) + try container.encodeIfPresent(displayName, forKey: .displayName) + if reactions != true { + try container.encodeIfPresent(reactions, forKey: .reactions) + } + try container.encodeIfPresent(picture, forKey: .picture) + try container.encodeIfPresent(nip05, forKey: .nip05) + if damusDonation != 0 { + try container.encodeIfPresent(damusDonation, forKey: .damusDonation) + } + if damusDonationV2 != 0 { + try container.encodeIfPresent(damusDonationV2, forKey: .damusDonationV2) + } + try container.encodeIfPresent(lud06, forKey: .lud06) + } +} + +public struct NdbProfileRecord: FlatBufferObject, Verifiable { + + static func validateVersion() { FlatBuffersVersion_23_5_26() } + public var __buffer: ByteBuffer! { return _accessor.bb } + private var _accessor: Table + + private init(_ t: Table) { _accessor = t } + public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + + private enum VTOFFSET: VOffset { + case profile = 4 + case receivedAt = 6 + case noteKey = 8 + case lnurl = 10 + var v: Int32 { Int32(self.rawValue) } + var p: VOffset { self.rawValue } + } + + public var profile: NdbProfile? { let o = _accessor.offset(VTOFFSET.profile.v); return o == 0 ? nil : NdbProfile(_accessor.bb, o: _accessor.indirect(o + _accessor.postion)) } + public var receivedAt: UInt64 { let o = _accessor.offset(VTOFFSET.receivedAt.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) } + public var noteKey: UInt64 { let o = _accessor.offset(VTOFFSET.noteKey.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) } + public var lnurl: String? { let o = _accessor.offset(VTOFFSET.lnurl.v); return o == 0 ? nil : _accessor.string(at: o) } + public var lnurlSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.lnurl.v) } + public static func startNdbProfileRecord(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 4) } + public static func add(profile: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: profile, at: VTOFFSET.profile.p) } + public static func add(receivedAt: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: receivedAt, def: 0, at: VTOFFSET.receivedAt.p) } + public static func add(noteKey: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: noteKey, def: 0, at: VTOFFSET.noteKey.p) } + public static func add(lnurl: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: lnurl, at: VTOFFSET.lnurl.p) } + public static func endNdbProfileRecord(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } + public static func createNdbProfileRecord( + _ fbb: inout FlatBufferBuilder, + profileOffset profile: Offset = Offset(), + receivedAt: UInt64 = 0, + noteKey: UInt64 = 0, + lnurlOffset lnurl: Offset = Offset() + ) -> Offset { + let __start = NdbProfileRecord.startNdbProfileRecord(&fbb) + NdbProfileRecord.add(profile: profile, &fbb) + NdbProfileRecord.add(receivedAt: receivedAt, &fbb) + NdbProfileRecord.add(noteKey: noteKey, &fbb) + NdbProfileRecord.add(lnurl: lnurl, &fbb) + return NdbProfileRecord.endNdbProfileRecord(&fbb, start: __start) + } + + public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + var _v = try verifier.visitTable(at: position) + try _v.visit(field: VTOFFSET.profile.p, fieldName: "profile", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.receivedAt.p, fieldName: "receivedAt", required: false, type: UInt64.self) + try _v.visit(field: VTOFFSET.noteKey.p, fieldName: "noteKey", required: false, type: UInt64.self) + try _v.visit(field: VTOFFSET.lnurl.p, fieldName: "lnurl", required: false, type: ForwardOffset.self) + _v.finish() + } +} + +extension NdbProfileRecord: Encodable { + + enum CodingKeys: String, CodingKey { + case profile = "profile" + case receivedAt = "received_at" + case noteKey = "note_key" + case lnurl = "lnurl" + } + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encodeIfPresent(profile, forKey: .profile) + if receivedAt != 0 { + try container.encodeIfPresent(receivedAt, forKey: .receivedAt) + } + if noteKey != 0 { + try container.encodeIfPresent(noteKey, forKey: .noteKey) + } + try container.encodeIfPresent(lnurl, forKey: .lnurl) + } +} + diff --git a/nostrdb/flatcc/flatcc_alloc.h b/nostrdb/flatcc/flatcc_alloc.h index c07462d577..3cd47db3e5 100644 --- a/nostrdb/flatcc/flatcc_alloc.h +++ b/nostrdb/flatcc/flatcc_alloc.h @@ -69,7 +69,7 @@ extern "C" { #ifndef FLATCC_USE_GENERIC_ALIGNED_ALLOC #ifndef FLATCC_NO_PALIGNED_ALLOC -#include "paligned_alloc.h" +#include "portable/paligned_alloc.h" #else #if !defined(__aligned_free_is_defined) || !__aligned_free_is_defined #define aligned_free free diff --git a/nostrdb/flatcc/flatcc_epilogue.h b/nostrdb/flatcc/flatcc_epilogue.h index dc724f6c98..915f0050f3 100644 --- a/nostrdb/flatcc/flatcc_epilogue.h +++ b/nostrdb/flatcc/flatcc_epilogue.h @@ -4,5 +4,5 @@ } #endif -#include "pdiagnostic_pop.h" +#include "portable/pdiagnostic_pop.h" diff --git a/nostrdb/flatcc/flatcc_flatbuffers.h b/nostrdb/flatcc/flatcc_flatbuffers.h index 210c9f2a42..7e2ca96ba7 100644 --- a/nostrdb/flatcc/flatcc_flatbuffers.h +++ b/nostrdb/flatcc/flatcc_flatbuffers.h @@ -4,7 +4,7 @@ * * Outside include guard to handle scope counter. */ -#include "pstatic_assert.h" +#include "portable/pstatic_assert.h" #ifndef FLATCC_FLATBUFFERS_H #define FLATCC_FLATBUFFERS_H @@ -17,14 +17,14 @@ extern "C" { #define flatcc_flatbuffers_defined #ifdef FLATCC_PORTABLE -#include "flatcc/flatcc_portable.h" +#include "flatcc_portable.h" #endif -#include "pwarnings.h" +#include "portable/pwarnings.h" /* Needed by C99 compilers without FLATCC_PORTABLE. */ -#include "pstdalign.h" +#include "portable/pstdalign.h" /* Handle fallthrough attribute in switch statements. */ -#include "pattributes.h" +#include "portable/pattributes.h" #include "flatcc_alloc.h" #include "flatcc_assert.h" @@ -37,7 +37,7 @@ extern "C" { * "flatcc_endian.h" requires the preceeding include files, * or compatible definitions. */ -#include "pendian.h" +#include "portable/pendian.h" #include "flatcc_types.h" #include "flatcc_endian.h" #include "flatcc_identifier.h" diff --git a/nostrdb/flatcc/flatcc_json_parser.h b/nostrdb/flatcc/flatcc_json_parser.h index ed7151c2fd..1d4a1f8abb 100644 --- a/nostrdb/flatcc/flatcc_json_parser.h +++ b/nostrdb/flatcc/flatcc_json_parser.h @@ -20,7 +20,7 @@ extern "C" { #include "flatcc_unaligned.h" #define PDIAGNOSTIC_IGNORE_UNUSED -#include "pdiagnostic_push.h" +#include "portable/pdiagnostic_push.h" enum flatcc_json_parser_flags { flatcc_json_parser_f_skip_unknown = 1, @@ -886,7 +886,7 @@ int flatcc_json_parser_struct_as_root(flatcc_builder_t *B, flatcc_json_parser_t const char *buf, size_t bufsiz, int flags, const char *fid, flatcc_json_parser_struct_f *parser); -#include "pdiagnostic_pop.h" +#include "portable/pdiagnostic_pop.h" #ifdef __cplusplus } diff --git a/nostrdb/flatcc/flatcc_json_printer.h b/nostrdb/flatcc/flatcc_json_printer.h index 0ce49c1461..612a7ada58 100644 --- a/nostrdb/flatcc/flatcc_json_printer.h +++ b/nostrdb/flatcc/flatcc_json_printer.h @@ -32,8 +32,8 @@ extern "C" { #include #include -#include "flatcc/flatcc_rtconfig.h" -#include "flatcc/flatcc_flatbuffers.h" +#include "flatcc_rtconfig.h" +#include "flatcc_flatbuffers.h" /* -DFLATCC_PORTABLE may help if inttypes.h is missing. */ #ifndef PRId64 diff --git a/nostrdb/flatcc/flatcc_portable.h b/nostrdb/flatcc/flatcc_portable.h index 9b0eb0c3f7..dbf583b799 100644 --- a/nostrdb/flatcc/flatcc_portable.h +++ b/nostrdb/flatcc/flatcc_portable.h @@ -5,7 +5,7 @@ extern "C" { #endif -#include "flatcc/portable/portable_basic.h" +#include "portable/portable_basic.h" #ifdef __cplusplus } diff --git a/nostrdb/flatcc/flatcc_prologue.h b/nostrdb/flatcc/flatcc_prologue.h index 36344c4c07..d7f8c337b6 100644 --- a/nostrdb/flatcc/flatcc_prologue.h +++ b/nostrdb/flatcc/flatcc_prologue.h @@ -1,7 +1,7 @@ /* Include guard intentionally left out. */ #define PDIAGNOSTIC_IGNORE_UNUSED -#include "pdiagnostic_push.h" +#include "portable/pdiagnostic_push.h" #ifdef __cplusplus extern "C" { diff --git a/nostrdb/flatcc/flatcc_rtconfig.h b/nostrdb/flatcc/flatcc_rtconfig.h index 59727b6516..6e6582cb56 100644 --- a/nostrdb/flatcc/flatcc_rtconfig.h +++ b/nostrdb/flatcc/flatcc_rtconfig.h @@ -8,7 +8,7 @@ extern "C" { /* Include portability layer here since all other files depend on it. */ #ifdef FLATCC_PORTABLE -#include "flatcc/portable/portable.h" +#include "portable/portable.h" #endif /* diff --git a/nostrdb/flatcc/flatcc_unaligned.h b/nostrdb/flatcc/flatcc_unaligned.h index 5ea26cede6..5f31d405d5 100644 --- a/nostrdb/flatcc/flatcc_unaligned.h +++ b/nostrdb/flatcc/flatcc_unaligned.h @@ -5,7 +5,7 @@ extern "C" { #endif -#include "punaligned.h" +#include "portable/punaligned.h" #define FLATCC_ALLOW_UNALIGNED_ACCESS PORTABLE_UNALIGNED_ACCESS diff --git a/nostrdb/flatcc/flatcc_verifier.h b/nostrdb/flatcc/flatcc_verifier.h index 7e0d296651..7181eeb498 100644 --- a/nostrdb/flatcc/flatcc_verifier.h +++ b/nostrdb/flatcc/flatcc_verifier.h @@ -51,7 +51,7 @@ extern "C" { * */ -#include "flatcc/flatcc_types.h" +#include "flatcc_types.h" #define FLATCC_VERIFY_ERROR_MAP(XX)\ XX(ok, "ok")\ diff --git a/nostrdb/flatcc/json_printer.c b/nostrdb/flatcc/json_printer.c index 4ebe1c1db1..2b8e78d51d 100644 --- a/nostrdb/flatcc/json_printer.c +++ b/nostrdb/flatcc/json_printer.c @@ -6,8 +6,8 @@ #include #include -#include "flatcc/flatcc_rtconfig.h" -#include "flatcc/flatcc_assert.h" +#include "flatcc_rtconfig.h" +#include "flatcc_assert.h" /* * Grisu significantly improves printing speed of floating point values @@ -18,13 +18,13 @@ #define PORTABLE_USE_GRISU3 1 #endif -#include "flatcc/flatcc_flatbuffers.h" -#include "flatcc/flatcc_json_printer.h" -#include "flatcc/flatcc_identifier.h" +#include "flatcc_flatbuffers.h" +#include "flatcc_json_printer.h" +#include "flatcc_identifier.h" -#include "flatcc/portable/pprintint.h" -#include "flatcc/portable/pprintfp.h" -#include "flatcc/portable/pbase64.h" +#include "portable/pprintint.h" +#include "portable/pprintfp.h" +#include "portable/pbase64.h" #define RAISE_ERROR(err) flatcc_json_printer_set_error(ctx, flatcc_json_printer_error_##err) diff --git a/nostrdb/flatcc/reflection/flatbuffers_common_reader.h b/nostrdb/flatcc/reflection/flatbuffers_common_reader.h index c575308689..6379c58644 100644 --- a/nostrdb/flatcc/reflection/flatbuffers_common_reader.h +++ b/nostrdb/flatcc/reflection/flatbuffers_common_reader.h @@ -64,6 +64,7 @@ static inline TK ## _option_t N ## _ ## NK ## _option(N ## _table_t t__tmp)\ #define __flatbuffers_offset_field(T, ID, t, r, adjust)\ {\ flatbuffers_uoffset_t *elem__tmp;\ + printf("got here\n"); __flatbuffers_read_vt(ID, offset__tmp, t)\ if (offset__tmp) {\ elem__tmp = (flatbuffers_uoffset_t *)((uint8_t *)(t) + offset__tmp);\ diff --git a/nostrdb/flatcc/refmap.c b/nostrdb/flatcc/refmap.c index d8c6034fbb..e1fa5931e0 100644 --- a/nostrdb/flatcc/refmap.c +++ b/nostrdb/flatcc/refmap.c @@ -179,7 +179,7 @@ flatcc_refmap_ref_t flatcc_refmap_find(flatcc_refmap_t *refmap, const void *src) #include #ifndef FLATCC_REFMAP_H -#include "flatcc/flatcc_refmap.h" +#include "flatcc_refmap.h" #endif #define test(x) do { if (!(x)) { fprintf(stderr, "%02d: refmap test failed\n", __LINE__); exit(-1); } } while (0) diff --git a/nostrdb/flatcc/verifier.c b/nostrdb/flatcc/verifier.c index 9c43bf613d..1cc68d79dc 100644 --- a/nostrdb/flatcc/verifier.c +++ b/nostrdb/flatcc/verifier.c @@ -6,10 +6,10 @@ */ #include -#include "flatcc/flatcc_rtconfig.h" -#include "flatcc/flatcc_flatbuffers.h" -#include "flatcc/flatcc_verifier.h" -#include "flatcc/flatcc_identifier.h" +#include "flatcc_rtconfig.h" +#include "flatcc_flatbuffers.h" +#include "flatcc_verifier.h" +#include "flatcc_identifier.h" /* Customization for testing. */ #if FLATCC_DEBUG_VERIFY From 2f60888fb1853c480593462e4ff6e7a079ce6d94 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 08:18:25 -0700 Subject: [PATCH 046/111] ndb: remove patch from copy script, just use sed --- nostrdb/copy-ndb | 4 +- nostrdb/flatcc.patch | 289 ------------------------------------------- 2 files changed, 3 insertions(+), 290 deletions(-) delete mode 100644 nostrdb/flatcc.patch diff --git a/nostrdb/copy-ndb b/nostrdb/copy-ndb index 2e2f1eb60e..a9784cf5c5 100755 --- a/nostrdb/copy-ndb +++ b/nostrdb/copy-ndb @@ -10,4 +10,6 @@ cp ~/src/c/nostrdb/deps/lmdb/midl.c . cp -r ~/src/c/nostrdb/deps/flatcc/include/flatcc/* flatcc cp ~/src/c/nostrdb/deps/flatcc/src/runtime/* flatcc cp -r ~/src/c/nostrdb/bindings . -patch -p2 < flatcc.patch +#patch -p2 < flatcc.patch +sed -i"" 's,import FlatBuffers,,' bindings/swift/NdbProfile.swift +sed -i"" 's,^#include "flatcc/,#include ",g' bindings/c/*.h flatcc/*.{c,h} diff --git a/nostrdb/flatcc.patch b/nostrdb/flatcc.patch deleted file mode 100644 index fda1c3c4c3..0000000000 --- a/nostrdb/flatcc.patch +++ /dev/null @@ -1,289 +0,0 @@ -diff --git b/nostrdb/bindings/c/flatbuffers_common_reader.h a/nostrdb/bindings/c/flatbuffers_common_reader.h -index c575308689b9..49e479e29980 100644 ---- b/nostrdb/bindings/c/flatbuffers_common_reader.h -+++ a/nostrdb/bindings/c/flatbuffers_common_reader.h -@@ -5,8 +5,8 @@ - - /* Common FlatBuffers read functionality for C. */ - --#include "flatcc/flatcc_prologue.h" --#include "flatcc/flatcc_flatbuffers.h" -+#include "flatcc_prologue.h" -+#include "flatcc_flatbuffers.h" - - - #define __flatbuffers_read_scalar_at_byteoffset(N, p, o) N ## _read_from_pe((uint8_t *)(p) + (o)) -@@ -574,5 +574,5 @@ static inline N ## _ ## K ## t N ## _as_typed_root(const void *buffer__tmp)\ - #define __flatbuffers_struct_as_root(N) __flatbuffers_buffer_as_root(N, struct_) - #define __flatbuffers_table_as_root(N) __flatbuffers_buffer_as_root(N, table_) - --#include "flatcc/flatcc_epilogue.h" -+#include "flatcc_epilogue.h" - #endif /* FLATBUFFERS_COMMON_H */ -diff --git b/nostrdb/bindings/c/profile_json_parser.h a/nostrdb/bindings/c/profile_json_parser.h -index a7caaaec6d37..f404b6679fe0 100644 ---- b/nostrdb/bindings/c/profile_json_parser.h -+++ a/nostrdb/bindings/c/profile_json_parser.h -@@ -3,8 +3,8 @@ - - /* Generated by flatcc 0.6.1 FlatBuffers schema compiler for C by dvide.com */ - --#include "flatcc/flatcc_json_parser.h" --#include "flatcc/flatcc_prologue.h" -+#include "flatcc_json_parser.h" -+#include "flatcc_prologue.h" - - /* - * Parses the default root table or struct of the schema and constructs a FlatBuffer. -@@ -280,5 +280,5 @@ static int profile_parse_json(flatcc_builder_t *B, flatcc_json_parser_t *ctx, - return 0; - } - --#include "flatcc/flatcc_epilogue.h" -+#include "flatcc_epilogue.h" - #endif /* PROFILE_JSON_PARSER_H */ -diff --git b/nostrdb/flatcc/builder.c a/nostrdb/flatcc/builder.c -index 9f54d884ff53..c5155a85e407 100644 ---- b/nostrdb/flatcc/builder.c -+++ a/nostrdb/flatcc/builder.c -@@ -16,8 +16,8 @@ - #include - #include - --#include "flatcc/flatcc_builder.h" --#include "flatcc/flatcc_emitter.h" -+#include "flatcc_builder.h" -+#include "flatcc_emitter.h" - - /* - * `check` is designed to handle incorrect use errors that can be -diff --git b/nostrdb/flatcc/emitter.c a/nostrdb/flatcc/emitter.c -index 089ea00b2060..dbeffacd97ed 100644 ---- b/nostrdb/flatcc/emitter.c -+++ a/nostrdb/flatcc/emitter.c -@@ -1,7 +1,7 @@ - #include - --#include "flatcc/flatcc_rtconfig.h" --#include "flatcc/flatcc_emitter.h" -+#include "flatcc_rtconfig.h" -+#include "flatcc_emitter.h" - - static int advance_front(flatcc_emitter_t *E) - { -diff --git b/nostrdb/flatcc/flatcc_alloc.h a/nostrdb/flatcc/flatcc_alloc.h -index 155364c1e2ba..c07462d57754 100644 ---- b/nostrdb/flatcc/flatcc_alloc.h -+++ a/nostrdb/flatcc/flatcc_alloc.h -@@ -69,7 +69,7 @@ extern "C" { - #ifndef FLATCC_USE_GENERIC_ALIGNED_ALLOC - - #ifndef FLATCC_NO_PALIGNED_ALLOC --#include "flatcc/portable/paligned_alloc.h" -+#include "paligned_alloc.h" - #else - #if !defined(__aligned_free_is_defined) || !__aligned_free_is_defined - #define aligned_free free -diff --git b/nostrdb/flatcc/flatcc_emitter.h a/nostrdb/flatcc/flatcc_emitter.h -index b8c83b94d3de..11756f23f4d0 100644 ---- b/nostrdb/flatcc/flatcc_emitter.h -+++ a/nostrdb/flatcc/flatcc_emitter.h -@@ -16,9 +16,9 @@ extern "C" { - #include - #include - --#include "flatcc/flatcc_types.h" --#include "flatcc/flatcc_iov.h" --#include "flatcc/flatcc_alloc.h" -+#include "flatcc_types.h" -+#include "flatcc_iov.h" -+#include "flatcc_alloc.h" - - /* - * The buffer steadily grows during emission but the design allows for -diff --git b/nostrdb/flatcc/flatcc_endian.h a/nostrdb/flatcc/flatcc_endian.h -index 0592f3132380..d16f72c89a11 100644 ---- b/nostrdb/flatcc/flatcc_endian.h -+++ a/nostrdb/flatcc/flatcc_endian.h -@@ -66,7 +66,7 @@ extern "C" { - #define htobe8(n) (n) - #endif - --#include "flatcc/flatcc_accessors.h" -+#include "flatcc_accessors.h" - - /* This is the binary encoding endianness, usually LE for flatbuffers. */ - #if FLATBUFFERS_PROTOCOL_IS_LE -diff --git b/nostrdb/flatcc/flatcc_epilogue.h a/nostrdb/flatcc/flatcc_epilogue.h -index 496857ba1251..dc724f6c98ed 100644 ---- b/nostrdb/flatcc/flatcc_epilogue.h -+++ a/nostrdb/flatcc/flatcc_epilogue.h -@@ -4,5 +4,5 @@ - } - #endif - --#include "flatcc/portable/pdiagnostic_pop.h" -+#include "pdiagnostic_pop.h" - -diff --git b/nostrdb/flatcc/flatcc_flatbuffers.h a/nostrdb/flatcc/flatcc_flatbuffers.h -index 4bfc7435251a..210c9f2a420d 100644 ---- b/nostrdb/flatcc/flatcc_flatbuffers.h -+++ a/nostrdb/flatcc/flatcc_flatbuffers.h -@@ -4,7 +4,7 @@ - * - * Outside include guard to handle scope counter. - */ --#include "flatcc/portable/pstatic_assert.h" -+#include "pstatic_assert.h" - - #ifndef FLATCC_FLATBUFFERS_H - #define FLATCC_FLATBUFFERS_H -@@ -19,15 +19,15 @@ extern "C" { - #ifdef FLATCC_PORTABLE - #include "flatcc/flatcc_portable.h" - #endif --#include "flatcc/portable/pwarnings.h" -+#include "pwarnings.h" - /* Needed by C99 compilers without FLATCC_PORTABLE. */ --#include "flatcc/portable/pstdalign.h" -+#include "pstdalign.h" - - /* Handle fallthrough attribute in switch statements. */ --#include "flatcc/portable/pattributes.h" -+#include "pattributes.h" - --#include "flatcc/flatcc_alloc.h" --#include "flatcc/flatcc_assert.h" -+#include "flatcc_alloc.h" -+#include "flatcc_assert.h" - - #define __FLATBUFFERS_PASTE2(a, b) a ## b - #define __FLATBUFFERS_PASTE3(a, b, c) a ## b ## c -@@ -37,10 +37,10 @@ extern "C" { - * "flatcc_endian.h" requires the preceeding include files, - * or compatible definitions. - */ --#include "flatcc/portable/pendian.h" --#include "flatcc/flatcc_types.h" --#include "flatcc/flatcc_endian.h" --#include "flatcc/flatcc_identifier.h" -+#include "pendian.h" -+#include "flatcc_types.h" -+#include "flatcc_endian.h" -+#include "flatcc_identifier.h" - - #ifndef FLATBUFFERS_WRAP_NAMESPACE - #define FLATBUFFERS_WRAP_NAMESPACE(ns, x) ns ## _ ## x -diff --git b/nostrdb/flatcc/flatcc_json_parser.h a/nostrdb/flatcc/flatcc_json_parser.h -index 1907fc7fc635..ed7151c2fd6b 100644 ---- b/nostrdb/flatcc/flatcc_json_parser.h -+++ a/nostrdb/flatcc/flatcc_json_parser.h -@@ -15,12 +15,12 @@ extern "C" { - #include - #include - --#include "flatcc/flatcc_rtconfig.h" --#include "flatcc/flatcc_builder.h" --#include "flatcc/flatcc_unaligned.h" -+#include "flatcc_rtconfig.h" -+#include "flatcc_builder.h" -+#include "flatcc_unaligned.h" - - #define PDIAGNOSTIC_IGNORE_UNUSED --#include "flatcc/portable/pdiagnostic_push.h" -+#include "pdiagnostic_push.h" - - enum flatcc_json_parser_flags { - flatcc_json_parser_f_skip_unknown = 1, -@@ -886,7 +886,7 @@ int flatcc_json_parser_struct_as_root(flatcc_builder_t *B, flatcc_json_parser_t - const char *buf, size_t bufsiz, int flags, const char *fid, - flatcc_json_parser_struct_f *parser); - --#include "flatcc/portable/pdiagnostic_pop.h" -+#include "pdiagnostic_pop.h" - - #ifdef __cplusplus - } -diff --git b/nostrdb/flatcc/flatcc_prologue.h a/nostrdb/flatcc/flatcc_prologue.h -index 3a74ed6040db..36344c4c071f 100644 ---- b/nostrdb/flatcc/flatcc_prologue.h -+++ a/nostrdb/flatcc/flatcc_prologue.h -@@ -1,7 +1,7 @@ - /* Include guard intentionally left out. */ - - #define PDIAGNOSTIC_IGNORE_UNUSED --#include "flatcc/portable/pdiagnostic_push.h" -+#include "pdiagnostic_push.h" - - #ifdef __cplusplus - extern "C" { -diff --git b/nostrdb/flatcc/flatcc_refmap.h a/nostrdb/flatcc/flatcc_refmap.h -index 062d94f5d35d..beafa301d042 100644 ---- b/nostrdb/flatcc/flatcc_refmap.h -+++ a/nostrdb/flatcc/flatcc_refmap.h -@@ -50,7 +50,7 @@ - extern "C" { - #endif - --#include "flatcc/flatcc_types.h" -+#include "flatcc_types.h" - - #ifndef FLATCC_REFMAP_MIN_BUCKETS - /* 8 buckets gives us 5 useful initial entries with a load factor of 0.7 */ -diff --git b/nostrdb/flatcc/flatcc_unaligned.h a/nostrdb/flatcc/flatcc_unaligned.h -index a7dc546111cd..5ea26cede6ee 100644 ---- b/nostrdb/flatcc/flatcc_unaligned.h -+++ a/nostrdb/flatcc/flatcc_unaligned.h -@@ -5,7 +5,7 @@ - extern "C" { - #endif - --#include "flatcc/portable/punaligned.h" -+#include "punaligned.h" - - #define FLATCC_ALLOW_UNALIGNED_ACCESS PORTABLE_UNALIGNED_ACCESS - -diff --git b/nostrdb/flatcc/json_parser.c a/nostrdb/flatcc/json_parser.c -index 0e3aeea9834c..06f778da33f3 100644 ---- b/nostrdb/flatcc/json_parser.c -+++ a/nostrdb/flatcc/json_parser.c -@@ -1,6 +1,6 @@ --#include "flatcc/flatcc_rtconfig.h" --#include "flatcc/flatcc_json_parser.h" --#include "flatcc/flatcc_assert.h" -+#include "flatcc_rtconfig.h" -+#include "flatcc_json_parser.h" -+#include "flatcc_assert.h" - - #define uoffset_t flatbuffers_uoffset_t - #define soffset_t flatbuffers_soffset_t -@@ -16,8 +16,8 @@ - #if FLATCC_USE_GRISU3 && !defined(PORTABLE_USE_GRISU3) - #define PORTABLE_USE_GRISU3 1 - #endif --#include "flatcc/portable/pparsefp.h" --#include "flatcc/portable/pbase64.h" -+#include "portable/pparsefp.h" -+#include "portable/pbase64.h" - - #if FLATCC_USE_SSE4_2 - #ifdef __SSE4_2__ -diff --git b/nostrdb/flatcc/refmap.c a/nostrdb/flatcc/refmap.c -index a2497f02247b..d8c6034fbb12 100644 ---- b/nostrdb/flatcc/refmap.c -+++ a/nostrdb/flatcc/refmap.c -@@ -13,10 +13,10 @@ - #include - #include - --#include "flatcc/flatcc_rtconfig.h" --#include "flatcc/flatcc_refmap.h" --#include "flatcc/flatcc_alloc.h" --#include "flatcc/flatcc_assert.h" -+#include "flatcc_rtconfig.h" -+#include "flatcc_refmap.h" -+#include "flatcc_alloc.h" -+#include "flatcc_assert.h" - - #define _flatcc_refmap_calloc FLATCC_CALLOC - #define _flatcc_refmap_free FLATCC_FREE From 882f6e2534ae90160b9a513961f171a803109566 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 08:19:03 -0700 Subject: [PATCH 047/111] ndb: update nostrdb, fix alignment issues --- nostrdb/nostrdb.c | 181 ++++++++++++++++++++++++++++++++++++++-------- nostrdb/nostrdb.h | 2 +- 2 files changed, 151 insertions(+), 32 deletions(-) diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index 5689e6c257..b84f6a62bf 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -15,6 +15,8 @@ #include #include "bindings/c/profile_json_parser.h" +#include "bindings/c/profile_builder.h" +#include "bindings/c/profile_verifier.h" #include "secp256k1.h" #include "secp256k1_ecdh.h" #include "secp256k1_schnorrsig.h" @@ -35,6 +37,10 @@ static const int DEFAULT_QUEUE_SIZE = 1000000; #define NDB_PARSED_TAGS (1 << 6) #define NDB_PARSED_ALL (NDB_PARSED_ID|NDB_PARSED_PUBKEY|NDB_PARSED_SIG|NDB_PARSED_CREATED_AT|NDB_PARSED_KIND|NDB_PARSED_CONTENT|NDB_PARSED_TAGS) +struct ndb_profile_record_builder { + flatcc_builder_t *builder; + void *flatbuf; +}; // controls whether to continue or stop the json parser enum ndb_idres { @@ -183,8 +189,7 @@ struct ndb_writer_note { struct ndb_writer_profile { struct ndb_writer_note note; - void *profile_flatbuf; - size_t profile_len; + struct ndb_profile_record_builder record; }; struct ndb_ingester_msg { @@ -275,6 +280,32 @@ int ndb_get_tsid(MDB_txn *txn, struct ndb_lmdb *lmdb, enum ndb_dbs db, return success; } +static void *ndb_lookup_by_key(struct ndb *ndb, uint64_t key, + enum ndb_dbs store, size_t *len) +{ + MDB_val k, v; + MDB_txn *txn; + + k.mv_data = &key; + k.mv_size = sizeof(key); + + if (mdb_txn_begin(ndb->lmdb.env, 0, 0, &txn)) { + ndb_debug("ndb_get_note_by_id: mdb_txn_begin failed\n"); + return NULL; + } + + if (mdb_get(txn, ndb->lmdb.dbs[store], &k, &v)) { + ndb_debug("ndb_get_profile_by_pubkey: mdb_get note failed\n"); + mdb_txn_abort(txn); + return NULL; + } + + if (len) + *len = v.mv_size; + + return v.mv_data; +} + static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind, enum ndb_dbs store, const unsigned char *pk, size_t *len) @@ -291,7 +322,7 @@ static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind, } if (!ndb_get_tsid(txn, &ndb->lmdb, ind, pk, &k)) { - ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n"); + //ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n"); goto cleanup; } @@ -301,6 +332,7 @@ static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind, } res = v.mv_data; + assert(((uint64_t)res % 4) == 0); if (len) *len = v.mv_size; cleanup: @@ -318,6 +350,11 @@ struct ndb_note *ndb_get_note_by_id(struct ndb *ndb, const unsigned char *id, si return ndb_lookup_tsid(ndb, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len); } +struct ndb_note *ndb_get_note_by_key(struct ndb *ndb, uint64_t key, size_t *len) +{ + return ndb_lookup_by_key(ndb, key, NDB_DB_NOTE, len); +} + static int ndb_has_note(MDB_txn *txn, struct ndb_lmdb *lmdb, const unsigned char *id) { MDB_val val; @@ -343,30 +380,83 @@ static enum ndb_idres ndb_ingester_json_controller(void *data, const char *hexid return NDB_IDRES_STOP; } +static int ndbprofile_parse_json(flatcc_builder_t *B, + const char *buf, size_t bufsiz, int flags, NdbProfile_ref_t *profile) +{ + flatcc_json_parser_t parser, *ctx = &parser; + flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags); + + if (flatcc_builder_start_buffer(B, 0, 0, 0)) + return 0; + + NdbProfile_parse_json_table(ctx, buf, buf + bufsiz, profile); + if (ctx->error) + return 0; + + if (!flatcc_builder_end_buffer(B, *profile)) + return 0; + + ctx->end_loc = buf; + + + return 1; +} + +void ndb_profile_record_builder_init(struct ndb_profile_record_builder *b) +{ + b->builder = malloc(sizeof(*b->builder)); + b->flatbuf = NULL; +} -static int ndb_process_profile_note(struct ndb_note *note, void **profile, - size_t *profile_len) +void ndb_profile_record_builder_free(struct ndb_profile_record_builder *b) +{ + if (b->builder) + free(b->builder); + if (b->flatbuf) + free(b->flatbuf); + + b->builder = NULL; + b->flatbuf = NULL; +} + +static int ndb_process_profile_note(struct ndb_note *note, + struct ndb_profile_record_builder *profile) { int res; - flatcc_builder_t builder; - flatcc_json_parser_t json_parser; + NdbProfile_ref_t profile_table; + flatcc_builder_t *builder; - flatcc_builder_init(&builder); + ndb_profile_record_builder_init(profile); + builder = profile->builder; + flatcc_builder_init(builder); - //printf("parsing profile '%.*s'\n", note->content_length, ndb_note_content(note)); - res = profile_parse_json(&builder, &json_parser, - ndb_note_content(note), - note->content_length, - flatcc_json_parser_f_skip_unknown); + NdbProfileRecord_start_as_root(builder); - if (res != 0) { + //printf("parsing profile '%.*s'\n", note->content_length, ndb_note_content(note)); + if (!(res = ndbprofile_parse_json(builder, ndb_note_content(note), + note->content_length, + flatcc_json_parser_f_skip_unknown, + &profile_table))) + { ndb_debug("profile_parse_json failed %d '%.*s'\n", res, - note->content_length, ndb_note_content(note)); + note->content_length, ndb_note_content(note)); + ndb_profile_record_builder_free(profile); return 0; } - *profile = flatcc_builder_finalize_aligned_buffer(&builder, profile_len); + uint64_t received_at = time(NULL); + const char *lnurl = "fixme"; + + NdbProfileRecord_profile_add(builder, profile_table); + NdbProfileRecord_received_at_add(builder, received_at); + + flatcc_builder_ref_t lnurl_off; + lnurl_off = flatcc_builder_create_string_str(builder, lnurl); + + NdbProfileRecord_lnurl_add(builder, lnurl_off); + + //*profile = flatcc_builder_finalize_aligned_buffer(builder, profile_len); return 1; } @@ -382,8 +472,8 @@ static int ndb_ingester_process_event(secp256k1_context *ctx, struct ndb_note *note; struct ndb_ingest_controller controller; struct ndb_id_cb cb; - void *buf, *flatbuf; - size_t bufsize, note_size, profile_len; + void *buf; + size_t bufsize, note_size; // we will use this to check if we already have it in the DB during // ID parsing @@ -436,14 +526,17 @@ static int ndb_ingester_process_event(secp256k1_context *ctx, // we didn't find anything. let's send it // to the writer thread note = realloc(note, note_size); + assert(((uint64_t)note % 4) == 0); + + if (note->kind == 0) { + struct ndb_profile_record_builder *b = + &out->profile.record; + + ndb_process_profile_note(note, b); - if (note->kind == 0 && - ndb_process_profile_note(note, &flatbuf, &profile_len)) { out->type = NDB_WRITER_PROFILE; out->profile.note.note = note; out->profile.note.note_len = note_size; - out->profile.profile_flatbuf = flatbuf; - out->profile.profile_len = profile_len; } else { out->type = NDB_WRITER_NOTE; out->note.note = note; @@ -482,11 +575,14 @@ static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db) } static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, - struct ndb_writer_profile *profile) + struct ndb_writer_profile *profile, + uint64_t note_key) { uint64_t profile_key; struct ndb_tsid tsid; struct ndb_note *note; + void *flatbuf; + size_t flatbuf_len; int rc; MDB_val key, val; @@ -494,6 +590,20 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, note = profile->note.note; + // add note_key to profile record + NdbProfileRecord_note_key_add(profile->record.builder, note_key); + NdbProfileRecord_end_as_root(profile->record.builder); + + flatbuf = profile->record.flatbuf = + flatcc_builder_finalize_aligned_buffer(profile->record.builder, &flatbuf_len); + + assert(((uint64_t)flatbuf % 8) == 0); + + // TODO: this may not be safe!? + flatbuf_len = (flatbuf_len + 7) & ~7; + + //assert(NdbProfileRecord_verify_as_root(flatbuf, flatbuf_len) == 0); + // get dbs profile_db = lmdb->dbs[NDB_DB_PROFILE]; pk_db = lmdb->dbs[NDB_DB_PROFILE_PK]; @@ -504,8 +614,8 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, // write profile to profile store key.mv_data = &profile_key; key.mv_size = sizeof(profile_key); - val.mv_data = profile->profile_flatbuf + 4; - val.mv_size = profile->profile_len - 4; + val.mv_data = flatbuf; + val.mv_size = flatbuf_len; //ndb_debug("profile_len %ld\n", profile->profile_len); if ((rc = mdb_put(txn, profile_db, &key, &val, 0))) { @@ -579,13 +689,14 @@ static void *ndb_writer_thread(void *data) struct ndb_writer *writer = data; struct ndb_writer_msg msgs[THREAD_QUEUE_BATCH], *msg; int i, popped, done, any_note; + uint64_t note_nkey; MDB_txn *txn; done = 0; while (!done) { txn = NULL; popped = prot_queue_pop_all(&writer->inbox, msgs, THREAD_QUEUE_BATCH); - ndb_debug("writer popped %d items\n", popped); + //ndb_debug("writer popped %d items\n", popped); any_note = 0; for (i = 0 ; i < popped; i++) { @@ -614,9 +725,14 @@ static void *ndb_writer_thread(void *data) done = 1; continue; case NDB_WRITER_PROFILE: - ndb_write_note(writer->lmdb, txn, &msg->note); - // TODO: save note_key with profile - ndb_write_profile(writer->lmdb, txn, &msg->profile); + note_nkey = + ndb_write_note(writer->lmdb, txn, &msg->note); + if (msg->profile.record.builder) { + // only write if parsing didn't fail + ndb_write_profile(writer->lmdb, txn, + &msg->profile, + note_nkey); + } break; case NDB_WRITER_NOTE: ndb_write_note(writer->lmdb, txn, &msg->note); @@ -637,8 +753,8 @@ static void *ndb_writer_thread(void *data) if (msg->type == NDB_WRITER_NOTE) free(msg->note.note); else if (msg->type == NDB_WRITER_PROFILE) { - free(msg->profile.profile_flatbuf); free(msg->profile.note.note); + ndb_profile_record_builder_free(&msg->profile.record); } } } @@ -667,7 +783,7 @@ static void *ndb_ingester_thread(void *data) any_event = 0; popped = prot_queue_pop_all(&thread->inbox, msgs, THREAD_QUEUE_BATCH); - ndb_debug("ingester popped %d items\n", popped); + //ndb_debug("ingester popped %d items\n", popped); for (i = 0; i < popped; i++) { msg = &msgs[i]; @@ -1336,6 +1452,9 @@ int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note, return 0; } + // make sure we're aligned as a whole + total_size = (total_size + 7) & ~7; + assert((total_size % 8) == 0); return total_size; } diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index d5062ffe44..d78e34a974 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -120,7 +120,6 @@ struct ndb_note { uint32_t content_length; union ndb_packed_str content; uint32_t strings; - uint32_t reserved[4]; // expansion slots // nothing can come after tags since it contains variadic data struct ndb_tags tags; }; @@ -157,6 +156,7 @@ int ndb_process_event(struct ndb *, const char *json, int len); int ndb_process_events(struct ndb *, const char *ldjson, size_t len); void *ndb_get_profile_by_pubkey(struct ndb *, const unsigned char *pubkey, size_t *len); struct ndb_note *ndb_get_note_by_id(struct ndb *, const unsigned char *id, size_t *len); +struct ndb_note *ndb_get_note_by_key(struct ndb *, uint64_t key, size_t *len); void ndb_destroy(struct ndb *); // BUILDER From 5657512370a76b757c26719c9f5981c72bf0d6bc Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 10:09:25 -0700 Subject: [PATCH 048/111] ndb: restore escaped slash fix --- nostrdb/nostrdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index b84f6a62bf..db67aa971e 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -1205,6 +1205,7 @@ static int cursor_push_unescaped_char(struct cursor *cur, char c1, char c2) case 'b': return cursor_push_byte(cur, '\b'); case 'f': return cursor_push_byte(cur, '\f'); case '\\': return cursor_push_byte(cur, '\\'); + case '/': return cursor_push_byte(cur, '/'); case '"': return cursor_push_byte(cur, '"'); case 'u': // these aren't handled yet From 8e92e28fafb06cfff32988d4ba4ce1f601e920b4 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 10:34:37 -0700 Subject: [PATCH 049/111] test: optionally remove lmdb db otherwise tests fail on CI --- nostrdb/Test/NdbTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift index 846fff0c5d..6466d7a07f 100644 --- a/nostrdb/Test/NdbTests.swift +++ b/nostrdb/Test/NdbTests.swift @@ -12,8 +12,8 @@ final class NdbTests: XCTestCase { override func setUpWithError() throws { // Put setup code here. This method is called before the invocation of each test method in the class. - try FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb") - try FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb") + try? FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb") + try? FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb") } override func tearDownWithError() throws { From c71b0ee91688f4a975000e33b5dc37a0f0e74c6d Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 11:46:03 -0700 Subject: [PATCH 050/111] blocks: pass keypair instead of privkey to avoid pubkey gen Generating a pubkey is quite slow, so pass a keypair instead of privkey --- damus/Components/TranslateView.swift | 8 ++-- damus/ContentParsing.swift | 4 +- damus/ContentView.swift | 4 +- damus/Models/DamusState.swift | 4 +- damus/Models/HomeModel.swift | 36 ++++++++-------- damus/Models/MutedThreadsManager.swift | 8 ++-- damus/Models/SearchHomeModel.swift | 4 +- damus/Models/SearchModel.swift | 4 +- damus/Models/ThreadModel.swift | 18 ++++---- damus/Nostr/NostrEvent.swift | 4 +- damus/Util/EventCache.swift | 18 ++++---- damus/Util/Keys.swift | 4 ++ damus/Util/ReplyCounter.swift | 4 +- damus/Views/DMView.swift | 2 +- damus/Views/Events/Components/ReplyPart.swift | 8 ++-- damus/Views/Events/EventMenu.swift | 6 +-- damus/Views/Events/EventShell.swift | 6 +-- .../Views/Events/Longform/LongformView.swift | 4 +- damus/Views/Events/MutedEventView.swift | 4 +- damus/Views/Events/SelectedEventView.swift | 6 +-- damus/Views/NoteContentView.swift | 10 ++--- damus/Views/SearchHomeView.swift | 2 +- damus/Views/ThreadView.swift | 6 +-- nostrdb/NdbNote.swift | 41 +++++++++---------- 24 files changed, 108 insertions(+), 107 deletions(-) diff --git a/damus/Components/TranslateView.swift b/damus/Components/TranslateView.swift index e78a68d088..bbe7fa5f6a 100644 --- a/damus/Components/TranslateView.swift +++ b/damus/Components/TranslateView.swift @@ -64,7 +64,7 @@ struct TranslateView: View { guard let note_language = translations_model.note_language else { return } - let res = await translate_note(profiles: damus_state.profiles, privkey: damus_state.keypair.privkey, event: event, settings: damus_state.settings, note_lang: note_language) + let res = await translate_note(profiles: damus_state.profiles, keypair: damus_state.keypair, event: event, settings: damus_state.settings, note_lang: note_language) DispatchQueue.main.async { self.translations_model.state = res } @@ -125,11 +125,11 @@ struct TranslateView_Previews: PreviewProvider { } } -func translate_note(profiles: Profiles, privkey: Privkey?, event: NostrEvent, settings: UserSettingsStore, note_lang: String) async -> TranslateStatus { - +func translate_note(profiles: Profiles, keypair: Keypair, event: NostrEvent, settings: UserSettingsStore, note_lang: String) async -> TranslateStatus { + // If the note language is different from our preferred languages, send a translation request. let translator = Translator(settings) - let originalContent = event.get_content(privkey) + let originalContent = event.get_content(keypair) let translated_note = try? await translator.translate(originalContent, from: note_lang, to: current_language()) guard let translated_note else { diff --git a/damus/ContentParsing.swift b/damus/ContentParsing.swift index 6f2f389ddf..d7bf0186a1 100644 --- a/damus/ContentParsing.swift +++ b/damus/ContentParsing.swift @@ -11,9 +11,9 @@ enum NoteContent { case note(NostrEvent) case content(String, TagsSequence?) - init(note: NostrEvent, privkey: Privkey?) { + init(note: NostrEvent, keypair: Keypair) { if note.known_kind == .dm { - self = .content(note.get_content(privkey), note.tags) + self = .content(note.get_content(keypair), note.tags) } else { self = .note(note) } diff --git a/damus/ContentView.swift b/damus/ContentView.swift index a3075dfd7b..bc9628a505 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -59,11 +59,11 @@ enum Sheets: Identifiable { enum FilterState : Int { case posts_and_replies = 1 case posts = 0 - + func filter(ev: NostrEvent) -> Bool { switch self { case .posts: - return ev.known_kind == .boost || !ev.is_reply(nil) + return ev.known_kind == .boost || !ev.is_reply(.empty) case .posts_and_replies: return true } diff --git a/damus/Models/DamusState.swift b/damus/Models/DamusState.swift index c26d43b82d..891db8ea8c 100644 --- a/damus/Models/DamusState.swift +++ b/damus/Models/DamusState.swift @@ -43,8 +43,8 @@ struct DamusState { // thread zaps if let ev = zap.event, !settings.nozaps, zap.is_in_thread { // [nozaps]: thread zaps are only available outside of the app store - replies.count_replies(ev, privkey: self.keypair.privkey) - events.add_replies(ev: ev, privkey: self.keypair.privkey) + replies.count_replies(ev, keypair: self.keypair) + events.add_replies(ev: ev, keypair: self.keypair) } // associate with events as well diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index 7ced11899d..039ceb6605 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -253,7 +253,7 @@ class HomeModel { process_zap_event(damus_state: damus_state, ev: ev) { zapres in guard case .done(let zap) = zapres, zap.target.pubkey == self.damus_state.keypair.pubkey, - should_show_event(privkey: self.damus_state.keypair.privkey, hellthreads: self.damus_state.muted_threads, contacts: self.damus_state.contacts, ev: zap.request.ev) else { + should_show_event(keypair: self.damus_state.keypair, hellthreads: self.damus_state.muted_threads, contacts: self.damus_state.contacts, ev: zap.request.ev) else { return } @@ -299,7 +299,7 @@ class HomeModel { return false } - return !damus_state.contacts.is_muted(ev.pubkey) && !damus_state.muted_threads.isMutedThread(ev, privkey: damus_state.keypair.privkey) + return !damus_state.contacts.is_muted(ev.pubkey) && !damus_state.muted_threads.isMutedThread(ev, keypair: damus_state.keypair) } } @@ -599,7 +599,7 @@ class HomeModel { // don't show notifications from ourselves guard ev.pubkey != damus_state.pubkey, event_has_our_pubkey(ev, our_pubkey: self.damus_state.pubkey), - should_show_event(privkey: self.damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { + should_show_event(keypair: self.damus_state.keypair, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { return } @@ -637,13 +637,13 @@ class HomeModel { func handle_text_event(sub_id: String, _ ev: NostrEvent) { - guard should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { + guard should_show_event(keypair: damus_state.keypair, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { return } // TODO: will we need to process this in other places like zap request contents, etc? process_image_metadatas(cache: damus_state.events, ev: ev) - damus_state.replies.count_replies(ev, privkey: self.damus_state.keypair.privkey) + damus_state.replies.count_replies(ev, keypair: self.damus_state.keypair) damus_state.events.insert(ev) if sub_id == home_subid { @@ -657,14 +657,14 @@ class HomeModel { notification_status.new_events = notifs if damus_state.settings.dm_notification && ev.age < HomeModel.event_max_age_for_notification { - let convo = ev.decrypted(privkey: self.damus_state.keypair.privkey) ?? NSLocalizedString("New encrypted direct message", comment: "Notification that the user has received a new direct message") + let convo = ev.decrypted(keypair: self.damus_state.keypair) ?? NSLocalizedString("New encrypted direct message", comment: "Notification that the user has received a new direct message") let notify = LocalNotification(type: .dm, event: ev, target: ev, content: convo) create_local_notification(profiles: damus_state.profiles, notify: notify) } } func handle_dm(_ ev: NostrEvent) { - guard should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { + guard should_show_event(keypair: damus_state.keypair, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) else { return } @@ -1129,12 +1129,12 @@ func event_has_our_pubkey(_ ev: NostrEvent, our_pubkey: Pubkey) -> Bool { } -func should_show_event(privkey: Privkey?, hellthreads: MutedThreadsManager, contacts: Contacts, ev: NostrEvent) -> Bool { +func should_show_event(keypair: Keypair, hellthreads: MutedThreadsManager, contacts: Contacts, ev: NostrEvent) -> Bool { if contacts.is_muted(ev.pubkey) { return false } - if hellthreads.isMutedThread(ev, privkey: privkey) { + if hellthreads.isMutedThread(ev, keypair: keypair) { return false } @@ -1221,11 +1221,11 @@ func create_in_app_event_zap_notification(profiles: Profiles, zap: Zap, locale: } } -func render_notification_content_preview(cache: EventCache, ev: NostrEvent, profiles: Profiles, privkey: Privkey?) -> String { - +func render_notification_content_preview(cache: EventCache, ev: NostrEvent, profiles: Profiles, keypair: Keypair) -> String { + let prefix_len = 300 - let artifacts = cache.get_cache_data(ev.id).artifacts.artifacts ?? render_note_content(ev: ev, profiles: profiles, privkey: privkey) - + let artifacts = cache.get_cache_data(ev.id).artifacts.artifacts ?? render_note_content(ev: ev, profiles: profiles, keypair: keypair) + // special case for longform events if ev.known_kind == .longform { let longform = LongformEvent(event: ev) @@ -1255,7 +1255,7 @@ func process_local_notification(damus_state: DamusState, event ev: NostrEvent) { } // Don't show notifications from muted threads. - if damus_state.muted_threads.isMutedThread(ev, privkey: damus_state.keypair.privkey) { + if damus_state.muted_threads.isMutedThread(ev, keypair: damus_state.keypair) { return } @@ -1265,12 +1265,12 @@ func process_local_notification(damus_state: DamusState, event ev: NostrEvent) { } if type == .text, damus_state.settings.mention_notification { - let blocks = ev.blocks(damus_state.keypair.privkey).blocks + let blocks = ev.blocks(damus_state.keypair).blocks for case .mention(let mention) in blocks { guard case .pubkey(let pk) = mention.ref, pk == damus_state.keypair.pubkey else { continue } - let content_preview = render_notification_content_preview(cache: damus_state.events, ev: ev, profiles: damus_state.profiles, privkey: damus_state.keypair.privkey) + let content_preview = render_notification_content_preview(cache: damus_state.events, ev: ev, profiles: damus_state.profiles, keypair: damus_state.keypair) let notify = LocalNotification(type: .mention, event: ev, target: ev, content: content_preview) create_local_notification(profiles: damus_state.profiles, notify: notify ) } @@ -1278,7 +1278,7 @@ func process_local_notification(damus_state: DamusState, event ev: NostrEvent) { damus_state.settings.repost_notification, let inner_ev = ev.get_inner_event(cache: damus_state.events) { - let content_preview = render_notification_content_preview(cache: damus_state.events, ev: inner_ev, profiles: damus_state.profiles, privkey: damus_state.keypair.privkey) + let content_preview = render_notification_content_preview(cache: damus_state.events, ev: inner_ev, profiles: damus_state.profiles, keypair: damus_state.keypair) let notify = LocalNotification(type: .repost, event: ev, target: inner_ev, content: content_preview) create_local_notification(profiles: damus_state.profiles, notify: notify) } else if type == .like, @@ -1286,7 +1286,7 @@ func process_local_notification(damus_state: DamusState, event ev: NostrEvent) { let evid = ev.referenced_ids.last, let liked_event = damus_state.events.lookup(evid) { - let content_preview = render_notification_content_preview(cache: damus_state.events, ev: liked_event, profiles: damus_state.profiles, privkey: damus_state.keypair.privkey) + let content_preview = render_notification_content_preview(cache: damus_state.events, ev: liked_event, profiles: damus_state.profiles, keypair: damus_state.keypair) let notify = LocalNotification(type: .like, event: ev, target: liked_event, content: content_preview) create_local_notification(profiles: damus_state.profiles, notify: notify) } diff --git a/damus/Models/MutedThreadsManager.swift b/damus/Models/MutedThreadsManager.swift index 5845ccc212..80903e4a1b 100644 --- a/damus/Models/MutedThreadsManager.swift +++ b/damus/Models/MutedThreadsManager.swift @@ -56,13 +56,13 @@ class MutedThreadsManager: ObservableObject { self.keypair = keypair } - func isMutedThread(_ ev: NostrEvent, privkey: Privkey?) -> Bool { - return _mutedThreadsSet.contains(ev.thread_id(privkey: privkey)) + func isMutedThread(_ ev: NostrEvent, keypair: Keypair) -> Bool { + return _mutedThreadsSet.contains(ev.thread_id(keypair: keypair)) } func updateMutedThread(_ ev: NostrEvent) { - let threadId = ev.thread_id(privkey: nil) - if isMutedThread(ev, privkey: keypair.privkey) { + let threadId = ev.thread_id(keypair: keypair) + if isMutedThread(ev, keypair: keypair) { mutedThreads = mutedThreads.filter { $0 != threadId } _mutedThreadsSet.remove(threadId) notify(.unmute_thread(ev)) diff --git a/damus/Models/SearchHomeModel.swift b/damus/Models/SearchHomeModel.swift index 6493bc71b2..fcc76e1080 100644 --- a/damus/Models/SearchHomeModel.swift +++ b/damus/Models/SearchHomeModel.swift @@ -35,7 +35,7 @@ class SearchHomeModel: ObservableObject { } func filter_muted() { - events.filter { should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: $0) } + events.filter { should_show_event(keypair: damus_state.keypair, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: $0) } self.objectWillChange.send() } @@ -60,7 +60,7 @@ class SearchHomeModel: ObservableObject { guard sub_id == self.base_subid || sub_id == self.profiles_subid else { return } - if ev.is_textlike && should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) && !ev.is_reply(nil) + if ev.is_textlike && should_show_event(keypair: damus_state.keypair, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: ev) && !ev.is_reply(damus_state.keypair) { if !damus_state.settings.multiple_events_per_pubkey && seen_pubkey.contains(ev.pubkey) { return diff --git a/damus/Models/SearchModel.swift b/damus/Models/SearchModel.swift index a782b26103..a80eb5558a 100644 --- a/damus/Models/SearchModel.swift +++ b/damus/Models/SearchModel.swift @@ -28,7 +28,7 @@ class SearchModel: ObservableObject { func filter_muted() { self.events.filter { - should_show_event(privkey: state.keypair.privkey, hellthreads: state.muted_threads, contacts: state.contacts, ev: $0) + should_show_event(keypair: state.keypair, hellthreads: state.muted_threads, contacts: state.contacts, ev: $0) } self.objectWillChange.send() } @@ -57,7 +57,7 @@ class SearchModel: ObservableObject { return } - guard should_show_event(privkey: state.keypair.privkey, hellthreads: state.muted_threads, contacts: state.contacts, ev: ev) else { + guard should_show_event(keypair: state.keypair, hellthreads: state.muted_threads, contacts: state.contacts, ev: ev) else { return } diff --git a/damus/Models/ThreadModel.swift b/damus/Models/ThreadModel.swift index 1a40b654b3..22b48acf39 100644 --- a/damus/Models/ThreadModel.swift +++ b/damus/Models/ThreadModel.swift @@ -18,7 +18,7 @@ class ThreadModel: ObservableObject { self.event_map = Set() self.event = event self.original_event = event - add_event(event, privkey: damus_state.keypair.privkey) + add_event(event, keypair: damus_state.keypair) } var is_original: Bool { @@ -46,9 +46,9 @@ class ThreadModel: ObservableObject { } @discardableResult - func set_active_event(_ ev: NostrEvent, privkey: Privkey?) -> Bool { + func set_active_event(_ ev: NostrEvent, keypair: Keypair) -> Bool { self.event = ev - add_event(ev, privkey: privkey) + add_event(ev, keypair: keypair) //self.objectWillChange.send() return false @@ -59,8 +59,8 @@ class ThreadModel: ObservableObject { var event_filter = NostrFilter() var ref_events = NostrFilter() - let thread_id = event.thread_id(privkey: nil) - + let thread_id = event.thread_id(keypair: .empty) + ref_events.referenced_ids = [thread_id, event.id] ref_events.kinds = [.text] ref_events.limit = 1000 @@ -85,14 +85,14 @@ class ThreadModel: ObservableObject { damus_state.pool.subscribe(sub_id: meta_subid, filters: meta_filters, handler: handle_event) } - func add_event(_ ev: NostrEvent, privkey: Privkey?) { + func add_event(_ ev: NostrEvent, keypair: Keypair) { if event_map.contains(ev) { return } let the_ev = damus_state.events.upsert(ev) - damus_state.replies.count_replies(the_ev, privkey: privkey) - damus_state.events.add_replies(ev: the_ev, privkey: privkey) + damus_state.replies.count_replies(ev, keypair: keypair) + damus_state.events.add_replies(ev: ev, keypair: keypair) event_map.insert(ev) objectWillChange.send() @@ -112,7 +112,7 @@ class ThreadModel: ObservableObject { } } else if ev.is_textlike { - self.add_event(ev, privkey: damus_state.keypair.privkey) + self.add_event(ev, keypair: damus_state.keypair) } } diff --git a/damus/Nostr/NostrEvent.swift b/damus/Nostr/NostrEvent.swift index 173734528e..9768991eda 100644 --- a/damus/Nostr/NostrEvent.swift +++ b/damus/Nostr/NostrEvent.swift @@ -928,8 +928,8 @@ func validate_event(ev: NostrEvent) -> ValidationResult { return ok ? .ok : .bad_sig } -func first_eref_mention(ev: NostrEvent, privkey: Privkey?) -> Mention? { - let blocks = ev.blocks(privkey).blocks.filter { block in +func first_eref_mention(ev: NostrEvent, keypair: Keypair) -> Mention? { + let blocks = ev.blocks(keypair).blocks.filter { block in guard case .mention(let mention) = block, case .note = mention.ref else { return false diff --git a/damus/Util/EventCache.swift b/damus/Util/EventCache.swift index 57235f8c87..d48fea6c73 100644 --- a/damus/Util/EventCache.swift +++ b/damus/Util/EventCache.swift @@ -228,13 +228,13 @@ class EventCache { return model } - func parent_events(event: NostrEvent, privkey: Privkey?) -> [NostrEvent] { + func parent_events(event: NostrEvent, keypair: Keypair) -> [NostrEvent] { var parents: [NostrEvent] = [] var ev = event while true { - guard let direct_reply = ev.direct_replies(privkey).last, + guard let direct_reply = ev.direct_replies(keypair).last, let next_ev = lookup(direct_reply), next_ev != ev else { break @@ -247,8 +247,8 @@ class EventCache { return parents.reversed() } - func add_replies(ev: NostrEvent, privkey: Privkey?) { - for reply in ev.direct_replies(privkey) { + func add_replies(ev: NostrEvent, keypair: Keypair) { + for reply in ev.direct_replies(keypair) { replies.add(id: reply, reply_id: ev.id) } } @@ -420,7 +420,7 @@ func preload_event(plan: PreloadPlan, state: DamusState) async { print("Preloading event \(plan.event.content)") if artifacts == nil && plan.load_artifacts { - let arts = render_note_content(ev: plan.event, profiles: profiles, privkey: our_keypair.privkey) + let arts = render_note_content(ev: plan.event, profiles: profiles, keypair: our_keypair) artifacts = arts // we need these asap @@ -441,8 +441,8 @@ func preload_event(plan: PreloadPlan, state: DamusState) async { } if plan.load_preview, note_artifact_is_separated(kind: plan.event.known_kind) { - let arts = artifacts ?? render_note_content(ev: plan.event, profiles: profiles, privkey: our_keypair.privkey) - + let arts = artifacts ?? render_note_content(ev: plan.event, profiles: profiles, keypair: our_keypair) + // only separated artifacts have previews if case .separated(let sep) = arts { let preview = await load_preview(artifacts: sep) @@ -456,13 +456,13 @@ func preload_event(plan: PreloadPlan, state: DamusState) async { } } - let note_language = plan.data.translations_model.note_language ?? plan.event.note_language(our_keypair.privkey) ?? current_language() + let note_language = plan.data.translations_model.note_language ?? plan.event.note_language(our_keypair) ?? current_language() var translations: TranslateStatus? = nil // We have to recheck should_translate here now that we have note_language if plan.load_translations && should_translate(event: plan.event, our_keypair: our_keypair, settings: settings, note_lang: note_language) && settings.auto_translate { - translations = await translate_note(profiles: profiles, privkey: our_keypair.privkey, event: plan.event, settings: settings, note_lang: note_language) + translations = await translate_note(profiles: profiles, keypair: our_keypair, event: plan.event, settings: settings, note_lang: note_language) } let ts = translations diff --git a/damus/Util/Keys.swift b/damus/Util/Keys.swift index 8d23b35695..1fc33ba060 100644 --- a/damus/Util/Keys.swift +++ b/damus/Util/Keys.swift @@ -33,6 +33,10 @@ struct Keypair { let pubkey_bech32: String let privkey_bech32: String? + static var empty: Keypair { + Keypair(pubkey: .empty, privkey: nil) + } + func to_full() -> FullKeypair? { guard let privkey = self.privkey else { return nil diff --git a/damus/Util/ReplyCounter.swift b/damus/Util/ReplyCounter.swift index 7a3ca3638c..2f22f1d4c0 100644 --- a/damus/Util/ReplyCounter.swift +++ b/damus/Util/ReplyCounter.swift @@ -28,7 +28,7 @@ class ReplyCounter { return replies[evid] ?? 0 } - func count_replies(_ event: NostrEvent, privkey: Privkey?) { + func count_replies(_ event: NostrEvent, keypair: Keypair) { guard event.is_textlike else { return } @@ -39,7 +39,7 @@ class ReplyCounter { counted.insert(event.id) - for reply in event.direct_replies(privkey) { + for reply in event.direct_replies(keypair) { if event.pubkey == our_pubkey { self.our_replies[reply] = event } diff --git a/damus/Views/DMView.swift b/damus/Views/DMView.swift index 1290cd8b5b..567b922f5d 100644 --- a/damus/Views/DMView.swift +++ b/damus/Views/DMView.swift @@ -17,7 +17,7 @@ struct DMView: View { var Mention: some View { Group { - if let mention = first_eref_mention(ev: event, privkey: damus_state.keypair.privkey) { + if let mention = first_eref_mention(ev: event, keypair: damus_state.keypair) { BuilderEventView(damus: damus_state, event_id: mention.ref) } else { EmptyView() diff --git a/damus/Views/Events/Components/ReplyPart.swift b/damus/Views/Events/Components/ReplyPart.swift index 3b6dc7b646..5fc6668062 100644 --- a/damus/Views/Events/Components/ReplyPart.swift +++ b/damus/Views/Events/Components/ReplyPart.swift @@ -10,11 +10,11 @@ import SwiftUI struct ReplyPart: View { let events: EventCache let event: NostrEvent - let privkey: Privkey? + let keypair: Keypair let profiles: Profiles var replying_to: NostrEvent? { - guard let note_ref = event.event_refs(privkey).first(where: { evref in evref.is_direct_reply != nil })?.is_direct_reply else { + guard let note_ref = event.event_refs(keypair).first(where: { evref in evref.is_direct_reply != nil })?.is_direct_reply else { return nil } @@ -23,7 +23,7 @@ struct ReplyPart: View { var body: some View { Group { - if event_is_reply(event.event_refs(privkey)) { + if event_is_reply(event.event_refs(keypair)) { ReplyDescription(event: event, replying_to: replying_to, profiles: profiles) } else { EmptyView() @@ -34,6 +34,6 @@ struct ReplyPart: View { struct ReplyPart_Previews: PreviewProvider { static var previews: some View { - ReplyPart(events: test_damus_state().events, event: test_note, privkey: nil, profiles: test_damus_state().profiles) + ReplyPart(events: test_damus_state().events, event: test_note, keypair: Keypair(pubkey: .empty, privkey: nil), profiles: test_damus_state().profiles) } } diff --git a/damus/Views/Events/EventMenu.swift b/damus/Views/Events/EventMenu.swift index 6b6c8b39ac..cd7c430f36 100644 --- a/damus/Views/Events/EventMenu.swift +++ b/damus/Views/Events/EventMenu.swift @@ -54,7 +54,7 @@ struct MenuItems: View { let bookmarked = bookmarks.isBookmarked(event) self._isBookmarked = State(initialValue: bookmarked) - let muted_thread = muted_threads.isMutedThread(event, privkey: keypair.privkey) + let muted_thread = muted_threads.isMutedThread(event, keypair: keypair) self._isMutedThread = State(initialValue: muted_thread) self.bookmarks = bookmarks @@ -68,7 +68,7 @@ struct MenuItems: View { var body: some View { Group { Button { - UIPasteboard.general.string = event.get_content(keypair.privkey) + UIPasteboard.general.string = event.get_content(keypair) } label: { Label(NSLocalizedString("Copy text", comment: "Context menu option for copying the text from an note."), image: "copy2") } @@ -106,7 +106,7 @@ struct MenuItems: View { if event.known_kind != .dm { Button { self.muted_threads.updateMutedThread(event) - let muted = self.muted_threads.isMutedThread(event, privkey: self.keypair.privkey) + let muted = self.muted_threads.isMutedThread(event, keypair: self.keypair) isMutedThread = muted } label: { let imageName = isMutedThread ? "mute" : "mute" diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift index ac8acb0908..7f0bd2d6e1 100644 --- a/damus/Views/Events/EventShell.swift +++ b/damus/Views/Events/EventShell.swift @@ -39,7 +39,7 @@ struct EventShell: View { return nil } - return first_eref_mention(ev: event, privkey: state.keypair.privkey) + return first_eref_mention(ev: event, keypair: state.keypair) } func Mention(_ mention: Mention) -> some View { @@ -71,7 +71,7 @@ struct EventShell: View { UserStatusView(status: state.profiles.profile_data(pubkey).status, show_general: state.settings.show_general_statuses, show_music: state.settings.show_music_statuses) if !options.contains(.no_replying_to) { - ReplyPart(events: state.events, event: event, privkey: state.keypair.privkey, profiles: state.profiles) + ReplyPart(events: state.events, event: event, keypair: state.keypair, profiles: state.profiles) } content @@ -98,7 +98,7 @@ struct EventShell: View { VStack(alignment: .leading, spacing: 2) { EventTop(state: state, event: event, pubkey: pubkey, is_anon: is_anon) UserStatusView(status: state.profiles.profile_data(pubkey).status, show_general: state.settings.show_general_statuses, show_music: state.settings.show_music_statuses) - ReplyPart(events: state.events, event: event, privkey: state.keypair.privkey, profiles: state.profiles) + ReplyPart(events: state.events, event: event, keypair: state.keypair, profiles: state.profiles) } } .padding(.horizontal) diff --git a/damus/Views/Events/Longform/LongformView.swift b/damus/Views/Events/Longform/LongformView.swift index 270081032f..135953e12b 100644 --- a/damus/Views/Events/Longform/LongformView.swift +++ b/damus/Views/Events/Longform/LongformView.swift @@ -76,8 +76,8 @@ let test_longform_event = LongformEvent.parse(from: NostrEvent( struct LongformView_Previews: PreviewProvider { static var previews: some View { let st = test_damus_state() - let artifacts = render_note_content(ev: test_longform_event.event, profiles: st.profiles, privkey: nil) - + let artifacts = render_note_content(ev: test_longform_event.event, profiles: st.profiles, keypair: Keypair(pubkey: .empty, privkey: nil)) + let model = NoteArtifactsModel(state: .loaded(artifacts)) ScrollView { LongformView(state: st, event: test_longform_event, artifacts: model) diff --git a/damus/Views/Events/MutedEventView.swift b/damus/Views/Events/MutedEventView.swift index 10140389a5..720a0ce832 100644 --- a/damus/Views/Events/MutedEventView.swift +++ b/damus/Views/Events/MutedEventView.swift @@ -18,11 +18,11 @@ struct MutedEventView: View { self.damus_state = damus_state self.event = event self.selected = selected - self._shown = State(initialValue: should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: event)) + self._shown = State(initialValue: should_show_event(keypair: damus_state.keypair, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: event)) } var should_mute: Bool { - return !should_show_event(privkey: damus_state.keypair.privkey, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: event) + return !should_show_event(keypair: damus_state.keypair, hellthreads: damus_state.muted_threads, contacts: damus_state.contacts, ev: event) } var MutedBox: some View { diff --git a/damus/Views/Events/SelectedEventView.swift b/damus/Views/Events/SelectedEventView.swift index bfa46aad06..31f73428d4 100644 --- a/damus/Views/Events/SelectedEventView.swift +++ b/damus/Views/Events/SelectedEventView.swift @@ -19,7 +19,7 @@ struct SelectedEventView: View { @StateObject var bar: ActionBarModel var replying_to: NostrEvent? { - guard let note_ref = event.event_refs(damus.keypair.privkey).first(where: { evref in evref.is_direct_reply != nil })?.is_direct_reply else { + guard let note_ref = event.event_refs(damus.keypair).first(where: { evref in evref.is_direct_reply != nil })?.is_direct_reply else { return nil } @@ -50,14 +50,14 @@ struct SelectedEventView: View { .minimumScaleFactor(0.75) .lineLimit(1) - if event_is_reply(event.event_refs(damus.keypair.privkey)) { + if event_is_reply(event.event_refs(damus.keypair)) { ReplyDescription(event: event, replying_to: replying_to, profiles: damus.profiles) .padding(.horizontal) } EventBody(damus_state: damus, event: event, size: size, options: [.wide]) - if let mention = first_eref_mention(ev: event, privkey: damus.keypair.privkey) { + if let mention = first_eref_mention(ev: event, keypair: damus.keypair) { BuilderEventView(damus: damus, event_id: mention.ref) .padding(.horizontal) } diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index 82069515ae..49df0eed16 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -36,7 +36,7 @@ struct NoteContentView: View { @ObservedObject var settings: UserSettingsStore var note_artifacts: NoteArtifacts { - return self.artifacts_model.state.artifacts ?? .separated(.just_content(event.get_content(damus_state.keypair.privkey))) + return self.artifacts_model.state.artifacts ?? .separated(.just_content(event.get_content(damus_state.keypair))) } init(damus_state: DamusState, event: NostrEvent, show_images: Bool, size: EventViewKind, options: EventViewOptions) { @@ -180,7 +180,7 @@ struct NoteContentView: View { } await preload_event(plan: plan, state: damus_state) } else if force_artifacts { - let arts = render_note_content(ev: event, profiles: damus_state.profiles, privkey: damus_state.keypair.privkey) + let arts = render_note_content(ev: event, profiles: damus_state.profiles, keypair: damus_state.keypair) self.artifacts_model.state = .loaded(arts) } } @@ -228,7 +228,7 @@ struct NoteContentView: View { var body: some View { ArtifactContent .onReceive(handle_notify(.profile_updated)) { profile in - let blocks = event.blocks(damus_state.keypair.privkey) + let blocks = event.blocks(damus_state.keypair) for block in blocks.blocks { switch block { case .mention(let m): @@ -394,8 +394,8 @@ func note_artifact_is_separated(kind: NostrKind?) -> Bool { return kind != .longform } -func render_note_content(ev: NostrEvent, profiles: Profiles, privkey: Privkey?) -> NoteArtifacts { - let blocks = ev.blocks(privkey) +func render_note_content(ev: NostrEvent, profiles: Profiles, keypair: Keypair) -> NoteArtifacts { + let blocks = ev.blocks(keypair) if ev.known_kind == .longform { return .longform(LongformContent(ev.content)) diff --git a/damus/Views/SearchHomeView.swift b/damus/Views/SearchHomeView.swift index 0c49b9c146..9b5a3e347f 100644 --- a/damus/Views/SearchHomeView.swift +++ b/damus/Views/SearchHomeView.swift @@ -50,7 +50,7 @@ struct SearchHomeView: View { damus: damus_state, show_friend_icon: true, filter: { ev in - if damus_state.muted_threads.isMutedThread(ev, privkey: self.damus_state.keypair.privkey) { + if damus_state.muted_threads.isMutedThread(ev, keypair: self.damus_state.keypair) { return false } diff --git a/damus/Views/ThreadView.swift b/damus/Views/ThreadView.swift index 86e4662ec1..1db5e31dbe 100644 --- a/damus/Views/ThreadView.swift +++ b/damus/Views/ThreadView.swift @@ -14,7 +14,7 @@ struct ThreadView: View { @Environment(\.dismiss) var dismiss var parent_events: [NostrEvent] { - state.events.parent_events(event: thread.event, privkey: state.keypair.privkey) + state.events.parent_events(event: thread.event, keypair: state.keypair) } var child_events: [NostrEvent] { @@ -34,7 +34,7 @@ struct ThreadView: View { selected: false) .padding(.horizontal) .onTapGesture { - thread.set_active_event(parent_event, privkey: self.state.keypair.privkey) + thread.set_active_event(parent_event, keypair: self.state.keypair) scroll_to_event(scroller: reader, id: parent_event.id, delay: 0.1, animate: false) } @@ -77,7 +77,7 @@ struct ThreadView: View { ) .padding(.horizontal) .onTapGesture { - thread.set_active_event(child_event, privkey: state.keypair.privkey) + thread.set_active_event(child_event, keypair: state.keypair) scroll_to_event(scroller: reader, id: child_event.id, delay: 0.1, animate: false) } diff --git a/nostrdb/NdbNote.swift b/nostrdb/NdbNote.swift index d8b9fa59a4..4c0b2d5ec2 100644 --- a/nostrdb/NdbNote.swift +++ b/nostrdb/NdbNote.swift @@ -268,8 +268,8 @@ extension NdbNote { return !too_big } - func get_blocks(privkey: Privkey?) -> Blocks { - return parse_note_content(content: .init(note: self, privkey: privkey)) + func get_blocks(keypair: Keypair) -> Blocks { + return parse_note_content(content: .init(note: self, keypair: keypair)) } func get_inner_event(cache: EventCache) -> NostrEvent? { @@ -314,41 +314,38 @@ extension NdbNote { References(tags: self.tags) } - func event_refs(_ privkey: Privkey?) -> [EventRef] { + func event_refs(_ keypair: Keypair) -> [EventRef] { if let rs = _event_refs { return rs } - let refs = interpret_event_refs_ndb(blocks: self.blocks(privkey).blocks, tags: self.tags) + let refs = interpret_event_refs_ndb(blocks: self.blocks(keypair).blocks, tags: self.tags) self._event_refs = refs return refs } - func get_content(_ privkey: Privkey?) -> String { + func get_content(_ keypair: Keypair) -> String { if known_kind == .dm { - return decrypted(privkey: privkey) ?? "*failed to decrypt content*" + return decrypted(keypair: keypair) ?? "*failed to decrypt content*" } return content } - func blocks(_ privkey: Privkey?) -> Blocks { + func blocks(_ keypair: Keypair) -> Blocks { if let bs = _blocks { return bs } - let blocks = get_blocks(privkey: privkey) + let blocks = get_blocks(keypair: keypair) self._blocks = blocks return blocks } // NDBTODO: switch this to operating on bytes not strings - func decrypted(privkey: Privkey?) -> String? { + func decrypted(keypair: Keypair) -> String? { if let decrypted_content { return decrypted_content } - guard let privkey, - let our_pubkey = privkey_to_pubkey(privkey: privkey) else { - return nil - } + let our_pubkey = keypair.pubkey // NDBTODO: don't hex encode var pubkey = self.pubkey @@ -359,14 +356,14 @@ extension NdbNote { } // NDBTODO: pass data to pubkey - let dec = decrypt_dm(privkey, pubkey: pubkey, content: self.content, encoding: .base64) + let dec = decrypt_dm(keypair.privkey, pubkey: pubkey, content: self.content, encoding: .base64) self.decrypted_content = dec return dec } - public func direct_replies(_ privkey: Privkey?) -> [NoteId] { - return event_refs(privkey).reduce(into: []) { acc, evref in + public func direct_replies(_ keypair: Keypair) -> [NoteId] { + return event_refs(keypair).reduce(into: []) { acc, evref in if let direct_reply = evref.is_direct_reply { acc.append(direct_reply.note_id) } @@ -374,8 +371,8 @@ extension NdbNote { } // NDBTODO: just use Id - public func thread_id(privkey: Privkey?) -> NoteId { - for ref in event_refs(privkey) { + public func thread_id(keypair: Keypair) -> NoteId { + for ref in event_refs(keypair) { if let thread_id = ref.is_thread_id { return thread_id.note_id } @@ -405,16 +402,16 @@ extension NdbNote { } */ - func is_reply(_ privkey: Privkey?) -> Bool { - return event_is_reply(self.event_refs(privkey)) + func is_reply(_ keypair: Keypair) -> Bool { + return event_is_reply(self.event_refs(keypair)) } - func note_language(_ privkey: Privkey?) -> String? { + func note_language(_ keypair: Keypair) -> String? { assert(!Thread.isMainThread, "This function must not be run on the main thread.") // Rely on Apple's NLLanguageRecognizer to tell us which language it thinks the note is in // and filter on only the text portions of the content as URLs and hashtags confuse the language recognizer. - let originalBlocks = self.blocks(privkey).blocks + let originalBlocks = self.blocks(keypair).blocks let originalOnlyText = originalBlocks.compactMap { $0.is_text }.joined(separator: " ") // Only accept language recognition hypothesis if there's at least a 50% probability that it's accurate. From dd29e871465332828ddeed59b151177166f7601e Mon Sep 17 00:00:00 2001 From: Jon Marrs Date: Wed, 30 Aug 2023 16:22:04 -0700 Subject: [PATCH 051/111] test: pass keypair instead of privkey for test cases Tests were not building due to recent changes in the Damus source code that replaced privkey with keypair. This patch extends those changes to the test cases, allowing the tests to build and pass. Signed-off-by: Jon Marrs Signed-off-by: William Casarin --- damusTests/LongPostTests.swift | 2 +- damusTests/NoteContentViewTests.swift | 4 ++-- damusTests/ReplyTests.swift | 14 +++++++------- damusTests/damusTests.swift | 2 +- nostrdb/Test/NdbTests.swift | 8 ++++---- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/damusTests/LongPostTests.swift b/damusTests/LongPostTests.swift index 6cc34a1f8a..cad2a83cbb 100644 --- a/damusTests/LongPostTests.swift +++ b/damusTests/LongPostTests.swift @@ -34,7 +34,7 @@ final class LongPostTests: XCTestCase { XCTAssertEqual(subid, "subid") XCTAssertTrue(ev.should_show_event) XCTAssertTrue(!ev.too_big) - XCTAssertTrue(should_show_event(privkey: test_keypair.privkey, hellthreads: test_damus_state().muted_threads, contacts: contacts, ev: ev)) + XCTAssertTrue(should_show_event(keypair: test_keypair, hellthreads: test_damus_state().muted_threads, contacts: contacts, ev: ev)) XCTAssertTrue(validate_event(ev: ev) == .ok ) } diff --git a/damusTests/NoteContentViewTests.swift b/damusTests/NoteContentViewTests.swift index 9cd740c90e..93bb8a589a 100644 --- a/damusTests/NoteContentViewTests.swift +++ b/damusTests/NoteContentViewTests.swift @@ -12,7 +12,7 @@ class NoteContentViewTests: XCTestCase { func testRenderBlocksWithNonLatinHashtags() { let content = "Damusはかっこいいです #cool #かっこいい" let note = NostrEvent(content: content, keypair: test_keypair, tags: [["t", "かっこいい"]])! - let parsed: Blocks = parse_note_content(content: .init(note: note, privkey: test_keypair.privkey)) + let parsed: Blocks = parse_note_content(content: .init(note: note, keypair: test_keypair)) let testState = test_damus_state() @@ -31,7 +31,7 @@ class NoteContentViewTests: XCTestCase { func testParseImageBlockInContentWithEscapedSlashes() { let testJSONWithEscapedSlashes = "{\"tags\":[],\"pubkey\":\"f8e6c64342f1e052480630e27e1016dce35fc3a614e60434fef4aa2503328ca9\",\"content\":\"https:\\/\\/cdn.nostr.build\\/i\\/5c1d3296f66c2630131bf123106486aeaf051ed8466031c0e0532d70b33cddb2.jpg\",\"created_at\":1691864981,\"kind\":1,\"sig\":\"fc0033aa3d4df50b692a5b346fa816fdded698de2045e36e0642a021391468c44ca69c2471adc7e92088131872d4aaa1e90ea6e1ad97f3cc748f4aed96dfae18\",\"id\":\"e8f6eca3b161abba034dac9a02bb6930ecde9fd2fb5d6c5f22a05526e11382cb\"}" let testNote = NostrEvent.owned_from_json(json: testJSONWithEscapedSlashes)! - let parsed = parse_note_content(content: .init(note: testNote, privkey: test_keypair.privkey)) + let parsed = parse_note_content(content: .init(note: testNote, keypair: test_keypair)) XCTAssertTrue((parsed.blocks[0].is_url != nil), "NoteContentView does not correctly parse an image block when url in JSON content contains optional escaped slashes.") } diff --git a/damusTests/ReplyTests.swift b/damusTests/ReplyTests.swift index 78d37c6ab1..7f2b900805 100644 --- a/damusTests/ReplyTests.swift +++ b/damusTests/ReplyTests.swift @@ -23,7 +23,7 @@ class ReplyTests: XCTestCase { let content = "this is #[0] a mention" let tags = [evid.tag] let ev = NostrEvent(content: content, keypair: test_keypair, tags: tags)! - let blocks = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let blocks = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks let event_refs = interpret_event_refs(blocks: blocks, tags: ev.tags) XCTAssertEqual(event_refs.count, 1) @@ -76,7 +76,7 @@ class ReplyTests: XCTestCase { let mentioned_id = NoteId(hex: "5a534797e8cd3b9f4c1cf63e20e48bd0e8bd7f8c4d6353fbd576df000f6f54d3")! let tags = [thread_id.tag, mentioned_id.tag] let ev = NostrEvent(content: content, keypair: test_keypair, tags: tags)! - let blocks = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let blocks = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks let event_refs = interpret_event_refs(blocks: blocks, tags: ev.tags) XCTAssertEqual(event_refs.count, 2) @@ -93,7 +93,7 @@ class ReplyTests: XCTestCase { func testEmptyMention() throws { let content = "this is some & content" let ev = NostrEvent(content: content, keypair: test_keypair, tags: [])! - let blocks = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let blocks = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks let post_blocks = parse_post_blocks(content: content) let post_tags = make_post_tags(post_blocks: post_blocks, tags: []) let event_refs = interpret_event_refs(blocks: blocks, tags: ev.tags) @@ -108,7 +108,7 @@ class ReplyTests: XCTestCase { let content = "#[10]" let tags: [[String]] = [[],[],[],[],[],[],[],[],[],[],["p", "3e999f94e2cb34ef44a64b351141ac4e51b5121b2d31aed4a6c84602a1144692"]] let ev = NostrEvent(content: content, keypair: test_keypair, tags: tags)! - let blocks = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let blocks = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks let mentions = blocks.filter { $0.is_mention != nil } XCTAssertEqual(mentions.count, 1) } @@ -145,7 +145,7 @@ class ReplyTests: XCTestCase { let reply_id = NoteId(hex: "80093e9bdb495728f54cda2bad4aed096877189552b3d41264e73b9a9595be22")! let tags = [thread_id.tag, reply_id.tag] let ev = NostrEvent(content: content, keypair: test_keypair, tags: tags)! - let blocks = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let blocks = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks let event_refs = interpret_event_refs(blocks: blocks, tags: ev.tags) XCTAssertEqual(event_refs.count, 2) @@ -265,7 +265,7 @@ class ReplyTests: XCTestCase { func testNoReply() throws { let content = "this is a #[0] reply" let ev = NostrEvent(content: content, keypair: test_keypair, tags: [])! - let blocks = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let blocks = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks let event_refs = interpret_event_refs(blocks: blocks, tags:ev.tags) XCTAssertEqual(event_refs.count, 0) @@ -275,7 +275,7 @@ class ReplyTests: XCTestCase { let note_id = NoteId(hex: "53f60f5114c06f069ffe9da2bc033e533d09cae44d37a8462154a663771a4ce6")! let tags = [note_id.tag] let ev = NostrEvent(content: "this is #[0] a mention", keypair: test_keypair, tags: tags)! - let parsed = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let parsed = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) diff --git a/damusTests/damusTests.swift b/damusTests/damusTests.swift index dd66ec3844..dbdf3dabac 100644 --- a/damusTests/damusTests.swift +++ b/damusTests/damusTests.swift @@ -190,7 +190,7 @@ class damusTests: XCTestCase { func testParseMentionOnlyText() { let tags = [["e", "event_id"]] let ev = NostrEvent(content: "there is no mention here", keypair: test_keypair, tags: tags)! - let parsed = parse_note_content(content: .init(note: ev, privkey: test_keypair.privkey)).blocks + let parsed = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift index 6466d7a07f..d54d98eee1 100644 --- a/nostrdb/Test/NdbTests.swift +++ b/nostrdb/Test/NdbTests.swift @@ -150,7 +150,7 @@ final class NdbTests: XCTestCase { return } self.measure(options: longer_iter()) { - let blocks = event.blocks(nil).blocks + let blocks = event.blocks(test_keypair).blocks let xs = interpret_event_refs(blocks: blocks, tags: event.tags) XCTAssertEqual(xs.count, 1) } @@ -161,7 +161,7 @@ final class NdbTests: XCTestCase { return } self.measure(options: longer_iter()) { - let blocks = note.blocks(nil).blocks + let blocks = note.blocks(test_keypair).blocks let xs = interpret_event_refs_ndb(blocks: blocks, tags: note.tags) XCTAssertEqual(xs.count, 1) } @@ -180,8 +180,8 @@ final class NdbTests: XCTestCase { XCTAssertEqual(note.pubkey, event.pubkey) XCTAssertEqual(note.id, event.id) - let ev_blocks = event.blocks(nil) - let note_blocks = note.blocks(nil) + let ev_blocks = event.blocks(test_keypair) + let note_blocks = note.blocks(test_keypair) XCTAssertEqual(ev_blocks, note_blocks) From a64f898df7fbdc4901f634d183c31585b7dd0e7f Mon Sep 17 00:00:00 2001 From: Grimless Date: Mon, 21 Aug 2023 17:17:21 -0400 Subject: [PATCH 052/111] Move the Block helper type to its own file, collapse the various standalone functions for parsing block data, and refactor consumers to initialize a Block with given data and access its members as needed. Closes: https://github.com/damus-io/damus/pull/1528 Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 + damus/ContentParsing.swift | 2 +- damus/Models/Mentions.swift | 227 +----------------------------- damus/Types/Block.swift | 214 ++++++++++++++++++++++++++++ damus/Util/Zap.swift | 2 +- damus/Views/DMChatView.swift | 4 +- damus/Views/NoteContentView.swift | 10 +- nostrdb/NdbNote.swift | 10 +- 8 files changed, 244 insertions(+), 229 deletions(-) create mode 100644 damus/Types/Block.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 069c0a26f5..664d7cfab9 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -404,6 +404,7 @@ 643EA5C8296B764E005081BB /* RelayFilterView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 643EA5C7296B764E005081BB /* RelayFilterView.swift */; }; 647D9A8D2968520300A295DE /* SideMenuView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 647D9A8C2968520300A295DE /* SideMenuView.swift */; }; 64FBD06F296255C400D9D3B2 /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64FBD06E296255C400D9D3B2 /* Theme.swift */; }; + 7527271E2A93FF0100214108 /* Block.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7527271D2A93FF0100214108 /* Block.swift */; }; 7C60CAEF298471A1009C80D6 /* CoreSVG.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */; }; 7C902AE32981D55B002AB16E /* ZoomableScrollView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */; }; 7C95CAEE299DCEF1009DCB67 /* KFOptionSetter+.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */; }; @@ -1078,6 +1079,7 @@ 643EA5C7296B764E005081BB /* RelayFilterView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterView.swift; sourceTree = ""; }; 647D9A8C2968520300A295DE /* SideMenuView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SideMenuView.swift; sourceTree = ""; }; 64FBD06E296255C400D9D3B2 /* Theme.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Theme.swift; sourceTree = ""; }; + 7527271D2A93FF0100214108 /* Block.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Block.swift; sourceTree = ""; }; 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreSVG.swift; sourceTree = ""; }; 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ZoomableScrollView.swift; sourceTree = ""; }; 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KFOptionSetter+.swift"; sourceTree = ""; }; @@ -1953,6 +1955,7 @@ isa = PBXGroup; children = ( 4CC14FED2A73FCBB007AEB17 /* Ids */, + 7527271D2A93FF0100214108 /* Block.swift */, ); path = Types; sourceTree = ""; @@ -2795,6 +2798,7 @@ 7CFF6317299FEFE5005D382A /* SelectableText.swift in Sources */, 4CA352A82A76B37E003BB08B /* NewMutesNotify.swift in Sources */, 4CFF8F6929CC9ED1008DB934 /* ImageContainerView.swift in Sources */, + 7527271E2A93FF0100214108 /* Block.swift in Sources */, 4C54AA0729A540BA003E4487 /* NotificationsModel.swift in Sources */, 4C12536C2A76D4B00004F4B8 /* RepostedNotify.swift in Sources */, 4CB55EF3295E5D59007FD187 /* RecommendedRelayView.swift in Sources */, diff --git a/damus/ContentParsing.swift b/damus/ContentParsing.swift index d7bf0186a1..96de616230 100644 --- a/damus/ContentParsing.swift +++ b/damus/ContentParsing.swift @@ -27,7 +27,7 @@ func parsed_blocks_finish(bs: inout note_blocks, tags: TagsSequence?) -> Blocks while (i < bs.num_blocks) { let block = bs.blocks[i] - if let converted = convert_block(block, tags: tags) { + if let converted = Block(block, tags: tags) { out.append(converted) } diff --git a/damus/Models/Mentions.swift b/damus/Models/Mentions.swift index d6cc781add..2132ca8ed4 100644 --- a/damus/Models/Mentions.swift +++ b/damus/Models/Mentions.swift @@ -123,147 +123,11 @@ struct LightningInvoice { } } -enum Block: Equatable { - static func == (lhs: Block, rhs: Block) -> Bool { - switch (lhs, rhs) { - case (.text(let a), .text(let b)): - return a == b - case (.mention(let a), .mention(let b)): - return a == b - case (.hashtag(let a), .hashtag(let b)): - return a == b - case (.url(let a), .url(let b)): - return a == b - case (.invoice(let a), .invoice(let b)): - return a.string == b.string - case (_, _): - return false - } - } - - case text(String) - case mention(Mention) - case hashtag(String) - case url(URL) - case invoice(Invoice) - case relay(String) - - var is_invoice: Invoice? { - if case .invoice(let invoice) = self { - return invoice - } - return nil - } - - var is_hashtag: String? { - if case .hashtag(let htag) = self { - return htag - } - return nil - } - - var is_url: URL? { - if case .url(let url) = self { - return url - } - - return nil - } - - var is_text: String? { - if case .text(let txt) = self { - return txt - } - return nil - } - - var is_note_mention: Bool { - if case .mention(let mention) = self, - case .note = mention.ref { - return true - } - return false - } - - var is_mention: Mention? { - if case .mention(let m) = self { - return m - } - return nil - } -} - -func render_blocks(blocks: [Block]) -> String { - return blocks.reduce("") { str, block in - switch block { - case .mention(let m): - if let idx = m.index { - return str + "#[\(idx)]" - } - - switch m.ref { - case .pubkey(let pk): return str + "nostr:\(pk.npub)" - case .note(let note_id): return str + "nostr:\(note_id.bech32)" - } - case .relay(let relay): - return str + relay - case .text(let txt): - return str + txt - case .hashtag(let htag): - return str + "#" + htag - case .url(let url): - return str + url.absoluteString - case .invoice(let inv): - return str + inv.string - } - } -} - struct Blocks: Equatable { let words: Int let blocks: [Block] } -func strblock_to_string(_ s: str_block_t) -> String? { - let len = s.end - s.start - let bytes = Data(bytes: s.start, count: len) - return String(bytes: bytes, encoding: .utf8) -} - -func convert_block(_ b: block_t, tags: TagsSequence?) -> Block? { - if b.type == BLOCK_HASHTAG { - guard let str = strblock_to_string(b.block.str) else { - return nil - } - return .hashtag(str) - } else if b.type == BLOCK_TEXT { - guard let str = strblock_to_string(b.block.str) else { - return nil - } - return .text(str) - } else if b.type == BLOCK_MENTION_INDEX { - return convert_mention_index_block(ind: Int(b.block.mention_index), tags: tags) - } else if b.type == BLOCK_URL { - return convert_url_block(b.block.str) - } else if b.type == BLOCK_INVOICE { - return convert_invoice_block(b.block.invoice) - } else if b.type == BLOCK_MENTION_BECH32 { - return convert_mention_bech32_block(b.block.mention_bech32) - } - - return nil -} - -func convert_url_block(_ b: str_block) -> Block? { - guard let str = strblock_to_string(b) else { - return nil - } - guard let url = URL(string: str) else { - return .text(str) - } - return .url(url) -} - func maybe_pointee(_ p: UnsafeMutablePointer!) -> T? { guard p != nil else { return nil @@ -326,75 +190,6 @@ func format_msats(_ msat: Int64, locale: Locale = Locale.current) -> String { return String(format: format, locale: locale, sats.decimalValue as NSDecimalNumber, formattedSats) } -func convert_invoice_block(_ b: invoice_block) -> Block? { - guard let invstr = strblock_to_string(b.invstr) else { - return nil - } - - guard var b11 = maybe_pointee(b.bolt11) else { - return nil - } - - guard let description = convert_invoice_description(b11: b11) else { - return nil - } - - let amount: Amount = maybe_pointee(b11.msat).map { .specific(Int64($0.millisatoshis)) } ?? .any - let payment_hash = Data(bytes: &b11.payment_hash, count: 32) - let created_at = b11.timestamp - - tal_free(b.bolt11) - return .invoice(Invoice(description: description, amount: amount, string: invstr, expiry: b11.expiry, payment_hash: payment_hash, created_at: created_at)) -} - -func convert_mention_bech32_block(_ b: mention_bech32_block) -> Block? -{ - switch b.bech32.type { - case NOSTR_BECH32_NOTE: - let note = b.bech32.data.note; - let note_id = NoteId(Data(bytes: note.event_id, count: 32)) - return .mention(.any(.note(note_id))) - - case NOSTR_BECH32_NEVENT: - let nevent = b.bech32.data.nevent; - let note_id = NoteId(Data(bytes: nevent.event_id, count: 32)) - return .mention(.any(.note(note_id))) - - case NOSTR_BECH32_NPUB: - let npub = b.bech32.data.npub - let pubkey = Pubkey(Data(bytes: npub.pubkey, count: 32)) - return .mention(.any(.pubkey(pubkey))) - - case NOSTR_BECH32_NSEC: - let nsec = b.bech32.data.nsec - let privkey = Privkey(Data(bytes: nsec.nsec, count: 32)) - guard let pubkey = privkey_to_pubkey(privkey: privkey) else { return nil } - return .mention(.any(.pubkey(pubkey))) - - case NOSTR_BECH32_NPROFILE: - let nprofile = b.bech32.data.nprofile - let pubkey = Pubkey(Data(bytes: nprofile.pubkey, count: 32)) - return .mention(.any(.pubkey(pubkey))) - - case NOSTR_BECH32_NRELAY: - let nrelay = b.bech32.data.nrelay - guard let relay_str = strblock_to_string(nrelay.relay) else { - return nil - } - return .relay(relay_str) - - case NOSTR_BECH32_NADDR: - // TODO: wtf do I do with this - guard let naddr = strblock_to_string(b.str) else { - return nil - } - return .text("nostr:" + naddr) - - default: - return nil - } -} - func convert_invoice_description(b11: bolt11) -> InvoiceDescription? { if let desc = b11.description { return .description(String(cString: desc)) @@ -407,24 +202,6 @@ func convert_invoice_description(b11: bolt11) -> InvoiceDescription? { return nil } -func convert_mention_index_block(ind: Int, tags: TagsSequence?) -> Block? -{ - guard let tags, - ind >= 0, - ind + 1 <= tags.count - else { - return .text("#[\(ind)]") - } - - let tag = tags[ind] - - guard let mention = MentionRef.from_tag(tag: tag) else { - return .text("#[\(ind)]") - } - - return .mention(.any(mention, index: ind)) -} - func find_tag_ref(type: String, id: String, tags: [[String]]) -> Int? { var i: Int = 0 for tag in tags { @@ -474,7 +251,9 @@ func post_to_event(post: NostrPost, keypair: FullKeypair) -> NostrEvent? { let tags = post.references.map({ r in r.tag }) + post.tags let post_blocks = parse_post_blocks(content: post.content) let post_tags = make_post_tags(post_blocks: post_blocks, tags: tags) - let content = render_blocks(blocks: post_tags.blocks) + let content = post_tags.blocks + .map(\.asString) + .joined(separator: "") return NostrEvent(content: content, keypair: keypair.to_keypair(), kind: post.kind.rawValue, tags: post_tags.tags) } diff --git a/damus/Types/Block.swift b/damus/Types/Block.swift new file mode 100644 index 0000000000..e478f6007b --- /dev/null +++ b/damus/Types/Block.swift @@ -0,0 +1,214 @@ +// +// Block.swift +// damus +// +// Created by Kyle Roucis on 2023-08-21. +// + +import Foundation + + +fileprivate extension String { + /// Failable initializer to build a Swift.String from a C-backed `str_block_t`. + init?(_ s: str_block_t) { + let len = s.end - s.start + let bytes = Data(bytes: s.start, count: len) + self.init(bytes: bytes, encoding: .utf8) + } +} + +/// Represents a block of data stored by the NOSTR protocol. This can be +/// simple text, a hashtag, a url, a relay reference, a mention ref and +/// potentially more in the future. +enum Block: Equatable { + static func == (lhs: Block, rhs: Block) -> Bool { + switch (lhs, rhs) { + case (.text(let a), .text(let b)): + return a == b + case (.mention(let a), .mention(let b)): + return a == b + case (.hashtag(let a), .hashtag(let b)): + return a == b + case (.url(let a), .url(let b)): + return a == b + case (.invoice(let a), .invoice(let b)): + return a.string == b.string + case (_, _): + return false + } + } + + case text(String) + case mention(Mention) + case hashtag(String) + case url(URL) + case invoice(Invoice) + case relay(String) +} +extension Block { + /// Failable initializer for the C-backed type `block_t`. This initializer will inspect + /// the underlying block type and build the appropriate enum value as needed. + init?(_ block: block_t, tags: TagsSequence? = nil) { + switch block.type { + case BLOCK_HASHTAG: + guard let str = String(block.block.str) else { + return nil + } + self = .hashtag(str) + case BLOCK_TEXT: + guard let str = String(block.block.str) else { + return nil + } + self = .text(str) + case BLOCK_MENTION_INDEX: + guard let b = Block(index: Int(block.block.mention_index), tags: tags) else { + return nil + } + self = b + case BLOCK_URL: + guard let b = Block(block.block.str) else { + return nil + } + self = b + case BLOCK_INVOICE: + guard let b = Block(invoice: block.block.invoice) else { + return nil + } + self = b + case BLOCK_MENTION_BECH32: + guard let b = Block(bech32: block.block.mention_bech32) else { + return nil + } + self = b + default: + return nil + } + } +} +fileprivate extension Block { + /// Failable initializer for the C-backed type `str_block_t`. + init?(_ b: str_block_t) { + guard let str = String(b) else { + return nil + } + + if let url = URL(string: str) { + self = .url(url) + } + else { + self = .text(str) + } + } +} +fileprivate extension Block { + /// Failable initializer for a block index and a tag sequence. + init?(index: Int, tags: TagsSequence? = nil) { + guard let tags, + index >= 0, + index + 1 <= tags.count + else { + self = .text("#[\(index)]") + return + } + + let tag = tags[index] + + if let mention = MentionRef.from_tag(tag: tag) { + self = .mention(.any(mention, index: index)) + } + else { + self = .text("#[\(index)]") + } + } +} +fileprivate extension Block { + /// Failable initializer for the C-backed type `invoice_block_t`. + init?(invoice: invoice_block_t) { + guard let invstr = String(invoice.invstr) else { + return nil + } + + guard var b11 = maybe_pointee(invoice.bolt11) else { + return nil + } + + guard let description = convert_invoice_description(b11: b11) else { + return nil + } + + let amount: Amount = maybe_pointee(b11.msat).map { .specific(Int64($0.millisatoshis)) } ?? .any + let payment_hash = Data(bytes: &b11.payment_hash, count: 32) + let created_at = b11.timestamp + + tal_free(invoice.bolt11) + self = .invoice(Invoice(description: description, amount: amount, string: invstr, expiry: b11.expiry, payment_hash: payment_hash, created_at: created_at)) + } +} +fileprivate extension Block { + /// Failable initializer for the C-backed type `mention_bech32_block_t`. This initializer will inspect the + /// bech32 type code and build the appropriate enum type. + init?(bech32 b: mention_bech32_block_t) { + switch b.bech32.type { + case NOSTR_BECH32_NOTE: + let note = b.bech32.data.note; + let note_id = NoteId(Data(bytes: note.event_id, count: 32)) + self = .mention(.any(.note(note_id))) + case NOSTR_BECH32_NEVENT: + let nevent = b.bech32.data.nevent; + let note_id = NoteId(Data(bytes: nevent.event_id, count: 32)) + self = .mention(.any(.note(note_id))) + case NOSTR_BECH32_NPUB: + let npub = b.bech32.data.npub + let pubkey = Pubkey(Data(bytes: npub.pubkey, count: 32)) + self = .mention(.any(.pubkey(pubkey))) + case NOSTR_BECH32_NSEC: + let nsec = b.bech32.data.nsec + let privkey = Privkey(Data(bytes: nsec.nsec, count: 32)) + guard let pubkey = privkey_to_pubkey(privkey: privkey) else { return nil } + self = .mention(.any(.pubkey(pubkey))) + case NOSTR_BECH32_NPROFILE: + let nprofile = b.bech32.data.nprofile + let pubkey = Pubkey(Data(bytes: nprofile.pubkey, count: 32)) + self = .mention(.any(.pubkey(pubkey))) + case NOSTR_BECH32_NRELAY: + let nrelay = b.bech32.data.nrelay + guard let relay_str = String(nrelay.relay) else { + return nil + } + self = .relay(relay_str) + case NOSTR_BECH32_NADDR: + // TODO: wtf do I do with this + guard let naddr = String(b.str) else { + return nil + } + self = .text("nostr:" + naddr) + default: + return nil + } + } +} +extension Block { + var asString: String { + switch self { + case .mention(let m): + if let idx = m.index { + return "#[\(idx)]" + } + + switch m.ref { + case .pubkey(let pk): return "nostr:\(pk.npub)" + case .note(let note_id): return "nostr:\(note_id.bech32)" + } + case .relay(let relay): + return relay + case .text(let txt): + return txt + case .hashtag(let htag): + return "#" + htag + case .url(let url): + return url.absoluteString + case .invoice(let inv): + return inv.string + } + } +} diff --git a/damus/Util/Zap.swift b/damus/Util/Zap.swift index 7749de723d..931810a99e 100644 --- a/damus/Util/Zap.swift +++ b/damus/Util/Zap.swift @@ -393,7 +393,7 @@ func decode_bolt11(_ s: String) -> Invoice? { let block = bs.blocks[0] - guard let converted = convert_block(block, tags: nil) else { + guard let converted = Block(block) else { blocks_free(&bs) return nil } diff --git a/damus/Views/DMChatView.swift b/damus/Views/DMChatView.swift index a905c68a32..18bf288fc1 100644 --- a/damus/Views/DMChatView.swift +++ b/damus/Views/DMChatView.swift @@ -130,7 +130,9 @@ struct DMChatView: View, KeyboardReadable { func send_message() { let tags = [["p", pubkey.hex()]] let post_blocks = parse_post_blocks(content: dms.draft) - let content = render_blocks(blocks: post_blocks) + let content = post_blocks + .map(\.asString) + .joined(separator: "") guard let dm = create_dm(content, to_pk: pubkey, tags: tags, keypair: damus_state.keypair) else { print("error creating dm") diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index 49df0eed16..633a4986fc 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -445,7 +445,15 @@ func render_blocks(blocks bs: Blocks, profiles: Profiles) -> NoteArtifactsSepara let blocks = bs.blocks let one_note_ref = blocks - .filter({ $0.is_note_mention }) + .filter({ + if case .mention(let mention) = $0, + case .note = mention.ref { + return true + } + else { + return false + } + }) .count == 1 var ind: Int = -1 diff --git a/nostrdb/NdbNote.swift b/nostrdb/NdbNote.swift index 4c0b2d5ec2..5f479a23e4 100644 --- a/nostrdb/NdbNote.swift +++ b/nostrdb/NdbNote.swift @@ -412,7 +412,15 @@ extension NdbNote { // Rely on Apple's NLLanguageRecognizer to tell us which language it thinks the note is in // and filter on only the text portions of the content as URLs and hashtags confuse the language recognizer. let originalBlocks = self.blocks(keypair).blocks - let originalOnlyText = originalBlocks.compactMap { $0.is_text }.joined(separator: " ") + let originalOnlyText = originalBlocks.compactMap { + if case .text(let txt) = $0 { + return txt + } + else { + return nil + } + } + .joined(separator: " ") // Only accept language recognition hypothesis if there's at least a 50% probability that it's accurate. let languageRecognizer = NLLanguageRecognizer() From 6ee0be40e9d49318cb8c23b27bb12b20a0d23dcc Mon Sep 17 00:00:00 2001 From: Grimless Date: Fri, 1 Sep 2023 11:32:01 -0400 Subject: [PATCH 053/111] Create helper extensions for Block and update tests for the Block helper model Closes: https://github.com/damus-io/damus/pull/1528 Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 ++ damusTests/HashtagTests.swift | 48 +++++++++++---------- damusTests/InvoiceTests.swift | 39 ++++++++++++----- damusTests/Models/Block+Tests.swift | 56 ++++++++++++++++++++++++ damusTests/NoteContentViewTests.swift | 2 +- damusTests/ReplyTests.swift | 62 +++++++++++++-------------- damusTests/UrlTests.swift | 45 +++++++++++++------ damusTests/damusTests.swift | 38 +++++++++++----- 8 files changed, 205 insertions(+), 89 deletions(-) create mode 100644 damusTests/Models/Block+Tests.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 664d7cfab9..cc79932be5 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -405,6 +405,7 @@ 647D9A8D2968520300A295DE /* SideMenuView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 647D9A8C2968520300A295DE /* SideMenuView.swift */; }; 64FBD06F296255C400D9D3B2 /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64FBD06E296255C400D9D3B2 /* Theme.swift */; }; 7527271E2A93FF0100214108 /* Block.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7527271D2A93FF0100214108 /* Block.swift */; }; + 75AD872B2AA23A460085EF2C /* Block+Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 75AD872A2AA23A460085EF2C /* Block+Tests.swift */; }; 7C60CAEF298471A1009C80D6 /* CoreSVG.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */; }; 7C902AE32981D55B002AB16E /* ZoomableScrollView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */; }; 7C95CAEE299DCEF1009DCB67 /* KFOptionSetter+.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */; }; @@ -1080,6 +1081,7 @@ 647D9A8C2968520300A295DE /* SideMenuView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SideMenuView.swift; sourceTree = ""; }; 64FBD06E296255C400D9D3B2 /* Theme.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Theme.swift; sourceTree = ""; }; 7527271D2A93FF0100214108 /* Block.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Block.swift; sourceTree = ""; }; + 75AD872A2AA23A460085EF2C /* Block+Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Block+Tests.swift"; sourceTree = ""; }; 7C60CAEE298471A1009C80D6 /* CoreSVG.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreSVG.swift; sourceTree = ""; }; 7C902AE22981D55B002AB16E /* ZoomableScrollView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ZoomableScrollView.swift; sourceTree = ""; }; 7C95CAED299DCEF1009DCB67 /* KFOptionSetter+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KFOptionSetter+.swift"; sourceTree = ""; }; @@ -2272,6 +2274,7 @@ isa = PBXGroup; children = ( F944F56D29EA9CCC0067B3BF /* DamusParseContentTests.swift */, + 75AD872A2AA23A460085EF2C /* Block+Tests.swift */, ); path = Models; sourceTree = ""; @@ -2889,6 +2892,7 @@ 4CB883AA297612FF00DC99E7 /* ZapTests.swift in Sources */, 4CB8839A297322D200DC99E7 /* DMTests.swift in Sources */, 4C9054852A6AEAA000811EEC /* NdbTests.swift in Sources */, + 75AD872B2AA23A460085EF2C /* Block+Tests.swift in Sources */, F944F56E29EA9CCC0067B3BF /* DamusParseContentTests.swift in Sources */, 3A5E47C72A4A76C800C0D090 /* TrieTests.swift in Sources */, 4CB883AE2976FA9300DC99E7 /* FormatTests.swift in Sources */, diff --git a/damusTests/HashtagTests.swift b/damusTests/HashtagTests.swift index 822eb11dc9..ffdaac67ab 100644 --- a/damusTests/HashtagTests.swift +++ b/damusTests/HashtagTests.swift @@ -8,15 +8,17 @@ import XCTest @testable import damus + final class HashtagTests: XCTestCase { func testParseHashtag() { let parsed = parse_note_content(content: .content("some hashtag #bitcoin derp",nil)).blocks XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "some hashtag ") - XCTAssertEqual(parsed[1].is_hashtag, "bitcoin") - XCTAssertEqual(parsed[2].is_text, " derp") + + XCTAssertEqual(parsed[0].asText, "some hashtag ") + XCTAssertEqual(parsed[1].asHashtag, "bitcoin") + XCTAssertEqual(parsed[2].asText, " derp") } func testHashtagWithComma() { @@ -24,9 +26,9 @@ final class HashtagTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "some hashtag ") - XCTAssertEqual(parsed[1].is_hashtag, "bitcoin") - XCTAssertEqual(parsed[2].is_text, ", cool") + XCTAssertEqual(parsed[0].asText, "some hashtag ") + XCTAssertEqual(parsed[1].asHashtag, "bitcoin") + XCTAssertEqual(parsed[2].asText, ", cool") } func testHashtagWithEmoji() { @@ -36,14 +38,14 @@ final class HashtagTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "some hashtag ") - XCTAssertEqual(parsed[1].is_hashtag, "bitcoin☕️") - XCTAssertEqual(parsed[2].is_text, " cool") + XCTAssertEqual(parsed[0].asText, "some hashtag ") + XCTAssertEqual(parsed[1].asHashtag, "bitcoin☕️") + XCTAssertEqual(parsed[2].asText, " cool") XCTAssertEqual(post_blocks.count, 3) - XCTAssertEqual(post_blocks[0].is_text, "some hashtag ") - XCTAssertEqual(post_blocks[1].is_hashtag, "bitcoin☕️") - XCTAssertEqual(post_blocks[2].is_text, " cool") + XCTAssertEqual(post_blocks[0].asText, "some hashtag ") + XCTAssertEqual(post_blocks[1].asHashtag, "bitcoin☕️") + XCTAssertEqual(post_blocks[2].asText, " cool") } func testPowHashtag() { @@ -53,12 +55,12 @@ final class HashtagTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 2) - XCTAssertEqual(parsed[0].is_text, "pow! ") - XCTAssertEqual(parsed[1].is_hashtag, "ぽわ〜") + XCTAssertEqual(parsed[0].asText, "pow! ") + XCTAssertEqual(parsed[1].asHashtag, "ぽわ〜") XCTAssertEqual(post_blocks.count, 2) - XCTAssertEqual(post_blocks[0].is_text, "pow! ") - XCTAssertEqual(post_blocks[1].is_hashtag, "ぽわ〜") + XCTAssertEqual(post_blocks[0].asText, "pow! ") + XCTAssertEqual(post_blocks[1].asHashtag, "ぽわ〜") } func testHashtagWithAccents() { @@ -66,8 +68,8 @@ final class HashtagTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 2) - XCTAssertEqual(parsed[0].is_text, "hello from ") - XCTAssertEqual(parsed[1].is_hashtag, "türkiye") + XCTAssertEqual(parsed[0].asText, "hello from ") + XCTAssertEqual(parsed[1].asHashtag, "türkiye") } func testHashtagWithNonLatinCharacters() { @@ -75,9 +77,9 @@ final class HashtagTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "this is a ") - XCTAssertEqual(parsed[1].is_hashtag, "시험") - XCTAssertEqual(parsed[2].is_text, " hope it works") + XCTAssertEqual(parsed[0].asText, "this is a ") + XCTAssertEqual(parsed[1].asHashtag, "시험") + XCTAssertEqual(parsed[2].asText, " hope it works") } func testParseHashtagEnd() { @@ -85,8 +87,8 @@ final class HashtagTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 2) - XCTAssertEqual(parsed[0].is_text, "some hashtag ") - XCTAssertEqual(parsed[1].is_hashtag, "bitcoin") + XCTAssertEqual(parsed[0].asText, "some hashtag ") + XCTAssertEqual(parsed[1].asHashtag, "bitcoin") } } diff --git a/damusTests/InvoiceTests.swift b/damusTests/InvoiceTests.swift index 1db6d2e737..a65a47ad8c 100644 --- a/damusTests/InvoiceTests.swift +++ b/damusTests/InvoiceTests.swift @@ -8,6 +8,19 @@ import XCTest @testable import damus + +extension Block { + var asInvoice: Invoice? { + switch self { + case .invoice(let invoice): + return invoice + default: + return nil + } + } +} + + final class InvoiceTests: XCTestCase { override func setUpWithError() throws { @@ -24,8 +37,9 @@ final class InvoiceTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) - XCTAssertNotNil(parsed[0].is_invoice) - guard let invoice = parsed[0].is_invoice else { + let invoiceOrNil = parsed[0].asInvoice + XCTAssertNotNil(invoiceOrNil) + guard let invoice = invoiceOrNil else { return } XCTAssertEqual(invoice.amount, .any) @@ -42,9 +56,10 @@ LNBC1P3MR5UJSP5G7SA48YD4JWTTPCHWMY4QYN4UWZQCJQ8NMWKD6QE3HCRVYTDLH9SPP57YM9TSA9NN XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 2) - XCTAssertNotNil(parsed[0].is_invoice) - XCTAssertEqual(parsed[1].is_text, " hi there") - guard let invoice = parsed[0].is_invoice else { + let invoiceOrNil = parsed[0].asInvoice + XCTAssertNotNil(invoiceOrNil) + XCTAssertEqual(parsed[1].asText, " hi there") + guard let invoice = invoiceOrNil else { return } XCTAssertEqual(invoice.amount, .any) @@ -58,8 +73,9 @@ LNBC1P3MR5UJSP5G7SA48YD4JWTTPCHWMY4QYN4UWZQCJQ8NMWKD6QE3HCRVYTDLH9SPP57YM9TSA9NN XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) - XCTAssertNotNil(parsed[0].is_invoice) - guard let invoice = parsed[0].is_invoice else { + let invoiceOrNil = parsed[0].asInvoice + XCTAssertNotNil(invoiceOrNil) + guard let invoice = invoiceOrNil else { return } XCTAssertEqual(invoice.amount, .specific(10000)) @@ -74,7 +90,7 @@ LNBC1P3MR5UJSP5G7SA48YD4JWTTPCHWMY4QYN4UWZQCJQ8NMWKD6QE3HCRVYTDLH9SPP57YM9TSA9NN XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) - XCTAssertNotNil(parsed[0].is_invoice) + XCTAssertNotNil(parsed[0].asInvoice) } func testParseInvoiceWithPrefixCapitalized() throws { @@ -83,7 +99,7 @@ LNBC1P3MR5UJSP5G7SA48YD4JWTTPCHWMY4QYN4UWZQCJQ8NMWKD6QE3HCRVYTDLH9SPP57YM9TSA9NN XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) - XCTAssertNotNil(parsed[0].is_invoice) + XCTAssertNotNil(parsed[0].asInvoice) } func testParseInvoice() throws { @@ -92,8 +108,9 @@ LNBC1P3MR5UJSP5G7SA48YD4JWTTPCHWMY4QYN4UWZQCJQ8NMWKD6QE3HCRVYTDLH9SPP57YM9TSA9NN XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) - XCTAssertNotNil(parsed[0].is_invoice) - guard let invoice = parsed[0].is_invoice else { + let invoiceOrNil = parsed[0].asInvoice + XCTAssertNotNil(invoiceOrNil) + guard let invoice = invoiceOrNil else { return } XCTAssertEqual(invoice.amount, .specific(10000)) diff --git a/damusTests/Models/Block+Tests.swift b/damusTests/Models/Block+Tests.swift new file mode 100644 index 0000000000..e7972c0df2 --- /dev/null +++ b/damusTests/Models/Block+Tests.swift @@ -0,0 +1,56 @@ +// +// Block+Tests.swift +// damusTests +// +// Created by Kyle Roucis on 9/1/23. +// + +import Foundation +@testable import damus + + +extension Block { + var asText: String? { + switch self { + case .text(let text): + return text + default: + return nil + } + } + + var isText: Bool { + return self.asText != nil + } + + var asURL: URL? { + switch self { + case .url(let url): + return url + default: + return nil + } + } + + var isURL: Bool { + return self.asURL != nil + } + + var asMention: Mention? { + switch self { + case .mention(let mention): + return mention + default: + return nil + } + } + + var asHashtag: String? { + switch self { + case .hashtag(let hashtag): + return hashtag + default: + return nil + } + } +} diff --git a/damusTests/NoteContentViewTests.swift b/damusTests/NoteContentViewTests.swift index 93bb8a589a..a376b7c75c 100644 --- a/damusTests/NoteContentViewTests.swift +++ b/damusTests/NoteContentViewTests.swift @@ -33,7 +33,7 @@ class NoteContentViewTests: XCTestCase { let testNote = NostrEvent.owned_from_json(json: testJSONWithEscapedSlashes)! let parsed = parse_note_content(content: .init(note: testNote, keypair: test_keypair)) - XCTAssertTrue((parsed.blocks[0].is_url != nil), "NoteContentView does not correctly parse an image block when url in JSON content contains optional escaped slashes.") + XCTAssertTrue((parsed.blocks[0].asURL != nil), "NoteContentView does not correctly parse an image block when url in JSON content contains optional escaped slashes.") } } diff --git a/damusTests/ReplyTests.swift b/damusTests/ReplyTests.swift index 7f2b900805..109ac625d8 100644 --- a/damusTests/ReplyTests.swift +++ b/damusTests/ReplyTests.swift @@ -41,7 +41,7 @@ class ReplyTests: XCTestCase { let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 1) - XCTAssertEqual(blocks[0].is_text, "what @") + XCTAssertEqual(blocks[0].asString, "what @") } func testHashtagsInQuote() { @@ -49,25 +49,25 @@ class ReplyTests: XCTestCase { let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 3) - XCTAssertEqual(blocks[0].is_text, "This is my \"") - XCTAssertEqual(blocks[1].is_hashtag, "awesome") - XCTAssertEqual(blocks[2].is_text, " post\"") + XCTAssertEqual(blocks[0].asText, "This is my \"") + XCTAssertEqual(blocks[1].asHashtag, "awesome") + XCTAssertEqual(blocks[2].asText, " post\"") } func testHashtagAtStartWorks() { let content = "#hashtag" let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 1) - XCTAssertEqual(blocks[0].is_hashtag, "hashtag") + XCTAssertEqual(blocks[0].asHashtag, "hashtag") } func testGroupOfHashtags() { let content = "#hashtag#what#nope" let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 3) - XCTAssertEqual(blocks[0].is_hashtag, "hashtag") - XCTAssertEqual(blocks[1].is_hashtag, "what") - XCTAssertEqual(blocks[2].is_hashtag, "nope") + XCTAssertEqual(blocks[0].asHashtag, "hashtag") + XCTAssertEqual(blocks[1].asHashtag, "what") + XCTAssertEqual(blocks[2].asHashtag, "nope") } func testRootReplyWithMention() throws { @@ -109,7 +109,7 @@ class ReplyTests: XCTestCase { let tags: [[String]] = [[],[],[],[],[],[],[],[],[],[],["p", "3e999f94e2cb34ef44a64b351141ac4e51b5121b2d31aed4a6c84602a1144692"]] let ev = NostrEvent(content: content, keypair: test_keypair, tags: tags)! let blocks = parse_note_content(content: .init(note: ev, keypair: test_keypair)).blocks - let mentions = blocks.filter { $0.is_mention != nil } + let mentions = blocks.filter { $0.asMention != nil } XCTAssertEqual(mentions.count, 1) } @@ -129,14 +129,14 @@ class ReplyTests: XCTestCase { XCTAssertEqual(post_note.content, expected_render) let blocks = parse_note_content(content: .content(post_note.content,nil)).blocks - let rendered = render_blocks(blocks: blocks) + let rendered = blocks.map { $0.asString }.joined(separator: "") XCTAssertEqual(rendered, expected_render) XCTAssertEqual(blocks.count, 3) - XCTAssertEqual(blocks[0].is_mention, .any(.pubkey(pk))) - XCTAssertEqual(blocks[1].is_text, "\n") - XCTAssertEqual(blocks[2].is_mention, .any(.pubkey(pk))) + XCTAssertEqual(blocks[0].asMention, .any(.pubkey(pk))) + XCTAssertEqual(blocks[1].asText, "\n") + XCTAssertEqual(blocks[2].asMention, .any(.pubkey(pk))) } func testThreadedReply() throws { @@ -279,9 +279,9 @@ class ReplyTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "this is ") - XCTAssertNotNil(parsed[1].is_mention) - XCTAssertEqual(parsed[2].is_text, " a mention") + XCTAssertEqual(parsed[0].asText, "this is ") + XCTAssertNotNil(parsed[1].asMention) + XCTAssertEqual(parsed[2].asText, " a mention") } func testEmptyPostReference() throws { @@ -295,8 +295,8 @@ class ReplyTests: XCTestCase { let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 2) - XCTAssertEqual(blocks[0].is_mention, .any(.pubkey(pk))) - XCTAssertEqual(blocks[1].is_text, " hello there") + XCTAssertEqual(blocks[0].asMention, .any(.pubkey(pk))) + XCTAssertEqual(blocks[1].asText, " hello there") } @@ -306,8 +306,8 @@ class ReplyTests: XCTestCase { let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 2) - XCTAssertEqual(blocks[1].is_mention, .any(.pubkey(pk))) - XCTAssertEqual(blocks[0].is_text, "this is a ") + XCTAssertEqual(blocks[1].asMention, .any(.pubkey(pk))) + XCTAssertEqual(blocks[0].asText, "this is a ") } func testNpubMention() throws { @@ -320,7 +320,7 @@ class ReplyTests: XCTestCase { XCTAssertEqual(ev.tags.count, 2) XCTAssertEqual(blocks.count, 3) - XCTAssertEqual(blocks[1].is_mention, .any(.pubkey(pk))) + XCTAssertEqual(blocks[1].asMention, .any(.pubkey(pk))) XCTAssertEqual(ev.content, "this is a nostr:npub1xtscya34g58tk0z605fvr788k263gsu6cy9x0mhnm87echrgufzsevkk5s mention") } @@ -335,7 +335,7 @@ class ReplyTests: XCTestCase { XCTAssertEqual(ev.tags.count, 2) XCTAssertEqual(blocks.count, 3) - XCTAssertEqual(blocks[1].is_mention, .any(.pubkey(pk))) + XCTAssertEqual(blocks[1].asMention, .any(.pubkey(pk))) XCTAssertEqual(ev.content, "this is a nostr:npub1enu46e5x2qtcmm72ttzsx6fmve5wkauftassz78l3mvluh8efqhqejf3v4 mention") } @@ -402,9 +402,9 @@ class ReplyTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "this is a ") - XCTAssertEqual(parsed[1].is_mention, .any(.pubkey(id))) - XCTAssertEqual(parsed[2].is_text, " event mention") + XCTAssertEqual(parsed[0].asText, "this is a ") + XCTAssertEqual(parsed[1].asMention, .any(.pubkey(id))) + XCTAssertEqual(parsed[2].asText, " event mention") guard case .text(let t1) = parsed[0] else { XCTAssertTrue(false) @@ -425,9 +425,9 @@ class ReplyTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "this is a ") - XCTAssertEqual(parsed[1].is_mention, .any(.note(id))) - XCTAssertEqual(parsed[2].is_text, " event mention") + XCTAssertEqual(parsed[0].asText, "this is a ") + XCTAssertEqual(parsed[1].asMention, .any(.note(id))) + XCTAssertEqual(parsed[2].asText, " event mention") guard case .text(let t1) = parsed[0] else { XCTAssertTrue(false) @@ -447,9 +447,9 @@ class ReplyTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].is_text, "this is ") - XCTAssertEqual(parsed[1].is_text, "#[0]") - XCTAssertEqual(parsed[2].is_text, " a mention") + XCTAssertEqual(parsed[0].asText, "this is ") + XCTAssertEqual(parsed[1].asText, "#[0]") + XCTAssertEqual(parsed[2].asText, " a mention") } } diff --git a/damusTests/UrlTests.swift b/damusTests/UrlTests.swift index d125706605..48886fdd59 100644 --- a/damusTests/UrlTests.swift +++ b/damusTests/UrlTests.swift @@ -26,65 +26,86 @@ final class UrlTests: XCTestCase { } func testParseUrlTrailingParenthesis() { + let testURL = URL(string: "https://en.m.wikipedia.org/wiki/Delicious_(website)") + XCTAssertNotNil(testURL) + let testString = "https://en.m.wikipedia.org/wiki/Delicious_(website)" + let parsed = parse_note_content(content: .content(testString, nil)).blocks XCTAssertNotNil(parsed) - XCTAssertEqual(parsed[0].is_url?.absoluteString, testString) + XCTAssertEqual(parsed[0].asURL, testURL) } func testParseUrlTrailingParenthesisAndInitialParenthesis() { + let testURL = URL(string: "https://en.m.wikipedia.org/wiki/Delicious_(website)") + XCTAssertNotNil(testURL) + let testString = "( https://en.m.wikipedia.org/wiki/Delicious_(website)" let parsed = parse_note_content(content: .content(testString, nil)).blocks - + XCTAssertNotNil(parsed) - XCTAssertEqual(parsed[1].is_url?.absoluteString, "https://en.m.wikipedia.org/wiki/Delicious_(website)") + XCTAssertEqual(parsed[1].asURL, testURL) } func testParseUrlTrailingParenthesisShouldntParse() { + let testURL = URL(string: "https://jb55.com") + XCTAssertNotNil(testURL) + let testString = "(https://jb55.com)" let parsed = parse_note_content(content: .content(testString, nil)).blocks XCTAssertNotNil(parsed) - XCTAssertEqual(parsed[1].is_url?.absoluteString, "https://jb55.com") + XCTAssertEqual(parsed[1].asURL, testURL) } func testParseSmartParens() { + let testURL = URL(string: "https://nostr-con.com/simplex") + XCTAssertNotNil(testURL) + let testString = "(https://nostr-con.com/simplex)" let parsed = parse_note_content(content: .content(testString, nil)).blocks XCTAssertNotNil(parsed) - XCTAssertEqual(parsed[1].is_url?.absoluteString, "https://nostr-con.com/simplex") + XCTAssertEqual(parsed[1].asURL, testURL) } func testLinkIsNotAHashtag() { let link = "https://github.com/damus-io/damus/blob/b7513f28fa1d31c2747865067256ad1d7cf43aac/damus/Nostr/NostrEvent.swift#L560" + let testURL = URL(string: link) + XCTAssertNotNil(testURL) let content = "my \(link) link" let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 3) - XCTAssertEqual(blocks[0].is_text, "my ") - XCTAssertEqual(blocks[1].is_url, URL(string: link)!) - XCTAssertEqual(blocks[2].is_text, " link") + XCTAssertEqual(blocks[0].asText, "my ") + XCTAssertEqual(blocks[1].asURL, testURL) + XCTAssertEqual(blocks[2].asText, " link") } func testParseUrlUpper() { + let testURL = URL(string: "HTTPS://jb55.COM") + XCTAssertNotNil(testURL) + let parsed = parse_note_content(content: .content("a HTTPS://jb55.COM b", nil)).blocks XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[1].is_url?.absoluteString, "HTTPS://jb55.COM") + XCTAssertEqual(parsed[1].asURL, testURL) } func testUrlAnchorsAreNotHashtags() { + let testURL = URL(string: "https://jb55.com/index.html#buybitcoin") + XCTAssertNotNil(testURL) + let content = "this is my link: https://jb55.com/index.html#buybitcoin this is not a hashtag!" let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 3) - XCTAssertEqual(blocks[0].is_text, "this is my link: ") - XCTAssertEqual(blocks[1].is_url, URL(string: "https://jb55.com/index.html#buybitcoin")!) - XCTAssertEqual(blocks[2].is_text, " this is not a hashtag!") + XCTAssertEqual(blocks[0].asText, "this is my link: ") + XCTAssertEqual(blocks[1].asURL, testURL) + XCTAssertEqual(blocks[2].asText, " this is not a hashtag!") } } diff --git a/damusTests/damusTests.swift b/damusTests/damusTests.swift index dbdf3dabac..1228aab202 100644 --- a/damusTests/damusTests.swift +++ b/damusTests/damusTests.swift @@ -8,6 +8,7 @@ import XCTest @testable import damus + class damusTests: XCTestCase { override func setUpWithError() throws { @@ -77,9 +78,15 @@ class damusTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertNotNil(parsed[0].is_text) - XCTAssertNotNil(parsed[1].is_url) - XCTAssertNotNil(parsed[2].is_text) + + XCTAssertTrue(parsed[0].isText) + XCTAssertFalse(parsed[0].isURL) + + XCTAssertTrue(parsed[1].isURL) + XCTAssertFalse(parsed[1].isText) + + XCTAssertTrue(parsed[2].isText) + XCTAssertFalse(parsed[2].isURL) } func testStringArrayStorage() { @@ -126,7 +133,11 @@ class damusTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[1].is_url?.absoluteString, "https://jb55.com") + + let url = URL(string: "https://jb55.com") + XCTAssertNotNil(url) + + XCTAssertEqual(parsed[1].asURL, url) } func testParseUrlEnd() { @@ -134,8 +145,13 @@ class damusTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 2) - XCTAssertEqual(parsed[0].is_text, "a ") - XCTAssertEqual(parsed[1].is_url?.absoluteString, "https://jb55.com") + + XCTAssertEqual(parsed[0].asString, "a ") + + let url = URL(string: "https://jb55.com") + XCTAssertNotNil(url) + + XCTAssertEqual(parsed[1].asURL, url) } func testParseUrlStart() { @@ -143,8 +159,8 @@ class damusTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 2) - XCTAssertEqual(parsed[0].is_url?.absoluteString, "https://jb55.com") - XCTAssertEqual(parsed[1].is_text, " br") +// XCTAssertEqual(parsed[0].is_url?.absoluteString, "https://jb55.com") +// XCTAssertEqual(parsed[1].is_text, " br") } func testNoParseUrlWithOnlyWhitespace() { @@ -152,7 +168,7 @@ class damusTests: XCTestCase { let parsed = parse_note_content(content: .content(testString,nil)).blocks XCTAssertNotNil(parsed) - XCTAssertEqual(parsed[0].is_text, testString) +// XCTAssertEqual(parsed[0].is_text, testString) } func testNoParseUrlTrailingCharacters() { @@ -160,7 +176,7 @@ class damusTests: XCTestCase { let parsed = parse_note_content(content: .content(testString,nil)).blocks XCTAssertNotNil(parsed) - XCTAssertEqual(parsed[0].is_url?.absoluteString, "https://foo.bar") +// XCTAssertEqual(parsed[0].is_url?.absoluteString, "https://foo.bar") } @@ -194,7 +210,7 @@ class damusTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) - XCTAssertEqual(parsed[0].is_text, "there is no mention here") +// XCTAssertEqual(parsed[0].is_text, "there is no mention here") guard case .text(let txt) = parsed[0] else { XCTAssertTrue(false) From f6f7d13f1296743a4c9c95280092ce5e60d82e51 Mon Sep 17 00:00:00 2001 From: Grimless Date: Fri, 1 Sep 2023 11:40:27 -0400 Subject: [PATCH 054/111] Properly implement top-level tests and fix one test using the wrong Block conversion property Closes: https://github.com/damus-io/damus/pull/1528 Reviewed-by: William Casarin Signed-off-by: William Casarin --- damusTests/ReplyTests.swift | 2 +- damusTests/damusTests.swift | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/damusTests/ReplyTests.swift b/damusTests/ReplyTests.swift index 109ac625d8..ac96d1b9e2 100644 --- a/damusTests/ReplyTests.swift +++ b/damusTests/ReplyTests.swift @@ -41,7 +41,7 @@ class ReplyTests: XCTestCase { let blocks = parse_post_blocks(content: content) XCTAssertEqual(blocks.count, 1) - XCTAssertEqual(blocks[0].asString, "what @") + XCTAssertEqual(blocks[0].asText, "what @") } func testHashtagsInQuote() { diff --git a/damusTests/damusTests.swift b/damusTests/damusTests.swift index 1228aab202..1756ed17b6 100644 --- a/damusTests/damusTests.swift +++ b/damusTests/damusTests.swift @@ -159,8 +159,13 @@ class damusTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 2) -// XCTAssertEqual(parsed[0].is_url?.absoluteString, "https://jb55.com") -// XCTAssertEqual(parsed[1].is_text, " br") + + let testURL = URL(string: "https://jb55.com") + XCTAssertNotNil(testURL) + + XCTAssertEqual(parsed[0].asURL, testURL) + + XCTAssertEqual(parsed[1].asText, " br") } func testNoParseUrlWithOnlyWhitespace() { @@ -168,15 +173,19 @@ class damusTests: XCTestCase { let parsed = parse_note_content(content: .content(testString,nil)).blocks XCTAssertNotNil(parsed) -// XCTAssertEqual(parsed[0].is_text, testString) + XCTAssertFalse(parsed[0].isURL) + XCTAssertEqual(parsed[0].asText, testString) } func testNoParseUrlTrailingCharacters() { let testString = "https://foo.bar, " let parsed = parse_note_content(content: .content(testString,nil)).blocks + let testURL = URL(string: "https://foo.bar") + XCTAssertNotNil(testURL) + XCTAssertNotNil(parsed) -// XCTAssertEqual(parsed[0].is_url?.absoluteString, "https://foo.bar") + XCTAssertEqual(parsed[0].asURL, testURL) } @@ -210,7 +219,7 @@ class damusTests: XCTestCase { XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 1) -// XCTAssertEqual(parsed[0].is_text, "there is no mention here") + XCTAssertEqual(parsed[0].asText, "there is no mention here") guard case .text(let txt) = parsed[0] else { XCTAssertTrue(false) From b18a0c573e9dc42bb424157812209e153927758b Mon Sep 17 00:00:00 2001 From: Grimless Date: Sat, 2 Sep 2023 12:34:39 -0400 Subject: [PATCH 055/111] profile: move the "Follow you" badge into the profile header Move the "Follow you" badge into the profile header he profile header out-of-line with the often long and already space-constrained username/display name text Changelog-Changed: Move the "Follow you" badge into the profile header Closes: https://github.com/damus-io/damus/pull/1529 Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 - damus/Models/HomeModel.swift | 2 +- damus/Util/DisplayName.swift | 20 ++-- damus/Views/EventView.swift | 12 --- .../Notifications/NotificationsView.swift | 2 +- damus/Views/Profile/EventProfileName.swift | 6 +- damus/Views/Profile/FollowsYou.swift | 29 ------ damus/Views/Profile/ProfileName.swift | 2 +- damus/Views/Profile/ProfileNameView.swift | 94 ++++++++++++++++--- damus/Views/Profile/ProfileView.swift | 88 ++++------------- 10 files changed, 114 insertions(+), 145 deletions(-) delete mode 100644 damus/Views/Profile/FollowsYou.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index cc79932be5..82e92fc81e 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -301,7 +301,6 @@ 4CB883B6297730E400DC99E7 /* LNUrls.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CB883B5297730E400DC99E7 /* LNUrls.swift */; }; 4CB8FC232A41ABA800763C51 /* AboutView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CB8FC222A41ABA500763C51 /* AboutView.swift */; }; 4CB9D4A72992D02B00A9A7E4 /* ProfileNameView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CB9D4A62992D02B00A9A7E4 /* ProfileNameView.swift */; }; - 4CB9D4A92992D2F400A9A7E4 /* FollowsYou.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CB9D4A82992D2F400A9A7E4 /* FollowsYou.swift */; }; 4CBCA930297DB57F00EC6B2F /* WebsiteLink.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CBCA92F297DB57F00EC6B2F /* WebsiteLink.swift */; }; 4CC14FEF2A73FCCB007AEB17 /* IdType.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CC14FEE2A73FCCB007AEB17 /* IdType.swift */; }; 4CC14FF12A73FCDB007AEB17 /* Pubkey.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CC14FF02A73FCDB007AEB17 /* Pubkey.swift */; }; @@ -972,7 +971,6 @@ 4CB883B5297730E400DC99E7 /* LNUrls.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LNUrls.swift; sourceTree = ""; }; 4CB8FC222A41ABA500763C51 /* AboutView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AboutView.swift; sourceTree = ""; }; 4CB9D4A62992D02B00A9A7E4 /* ProfileNameView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileNameView.swift; sourceTree = ""; }; - 4CB9D4A82992D2F400A9A7E4 /* FollowsYou.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FollowsYou.swift; sourceTree = ""; }; 4CBCA92F297DB57F00EC6B2F /* WebsiteLink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebsiteLink.swift; sourceTree = ""; }; 4CC14FEE2A73FCCB007AEB17 /* IdType.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IdType.swift; sourceTree = ""; }; 4CC14FF02A73FCDB007AEB17 /* Pubkey.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Pubkey.swift; sourceTree = ""; }; @@ -1943,7 +1941,6 @@ 4CEE2AF2280B25C500AB5EEF /* ProfilePicView.swift */, 4C8682862814DE470026224F /* ProfileView.swift */, 4CB9D4A62992D02B00A9A7E4 /* ProfileNameView.swift */, - 4CB9D4A82992D2F400A9A7E4 /* FollowsYou.swift */, 4C9F18E329ABDE6D008C55EC /* MaybeAnonPfpView.swift */, 4C9BB83329C12D9900FC4E37 /* EventProfileName.swift */, 4C8D1A6B29F1DFC200ACDF75 /* FriendIcon.swift */, @@ -2788,7 +2785,6 @@ 4C3AC7A728369BA200E1F516 /* SearchHomeView.swift in Sources */, 4CB883B0297705DD00DC99E7 /* ZapButton.swift in Sources */, 4C363A922825FCF2006E126D /* ProfileUpdate.swift in Sources */, - 4CB9D4A92992D2F400A9A7E4 /* FollowsYou.swift in Sources */, 4C3BEFDA281DCA1400B3DE84 /* LikeCounter.swift in Sources */, 4C32B9502A9AD44700DC3548 /* FlatBufferBuilder.swift in Sources */, 50A60D142A28BEEE00186190 /* RelayLog.swift in Sources */, diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index 039ceb6605..af7abc3e83 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -1168,7 +1168,7 @@ func zap_notification_body(profiles: Profiles, zap: Zap, locale: Locale = Locale let profile = profiles.lookup(id: pk) let sats = NSNumber(value: (Double(zap.invoice.amount) / 1000.0)) let formattedSats = format_msats_abbrev(zap.invoice.amount) - let name = Profile.displayName(profile: profile, pubkey: pk).display_name.truncate(maxLength: 50) + let name = Profile.displayName(profile: profile, pubkey: pk).displayName.truncate(maxLength: 50) if src.content.isEmpty { let format = localizedStringFormat(key: "zap_notification_no_message", locale: locale) diff --git a/damus/Util/DisplayName.swift b/damus/Util/DisplayName.swift index 567d2e18b6..4c4ec693a4 100644 --- a/damus/Util/DisplayName.swift +++ b/damus/Util/DisplayName.swift @@ -7,22 +7,16 @@ import Foundation - -struct BothNames { - let username: String - let display_name: String -} - enum DisplayName { - case both(BothNames) + case both(username: String, displayName: String) case one(String) - var display_name: String { + var displayName: String { switch self { case .one(let one): return one - case .both(let b): - return b.display_name + case .both(username: _, displayName: let displayName): + return displayName } } @@ -30,8 +24,8 @@ enum DisplayName { switch self { case .one(let one): return one - case .both(let b): - return b.username + case .both(username: let username, displayName: _): + return username } } } @@ -50,7 +44,7 @@ func parse_display_name(profile: Profile?, pubkey: Pubkey) -> DisplayName { let disp_name = profile.display_name?.isEmpty == false ? profile.display_name : nil if let name, let disp_name, name != disp_name { - return .both(BothNames(username: name, display_name: disp_name)) + return .both(username: name, displayName: disp_name) } if let one = name ?? disp_name { diff --git a/damus/Views/EventView.swift b/damus/Views/EventView.swift index a8bd587b6e..a3d49a7e7a 100644 --- a/damus/Views/EventView.swift +++ b/damus/Views/EventView.swift @@ -71,18 +71,6 @@ func should_show_images(settings: UserSettingsStore, contacts: Contacts, ev: Nos return false } -extension View { - func pubkey_context_menu(pubkey: Pubkey) -> some View { - return self.contextMenu { - Button { - UIPasteboard.general.string = pubkey.npub - } label: { - Label(NSLocalizedString("Copy Account ID", comment: "Context menu option for copying the ID of the account that created the note."), image: "copy2") - } - } - } -} - func format_relative_time(_ created_at: UInt32) -> String { return time_ago_since(Date(timeIntervalSince1970: Double(created_at))) diff --git a/damus/Views/Notifications/NotificationsView.swift b/damus/Views/Notifications/NotificationsView.swift index 32618b6107..76b5500e93 100644 --- a/damus/Views/Notifications/NotificationsView.swift +++ b/damus/Views/Notifications/NotificationsView.swift @@ -87,7 +87,7 @@ struct NotificationsView: View { var mystery: some View { VStack(spacing: 20) { - Text("Wake up, \(Profile.displayName(profile: state.profiles.lookup(id: state.pubkey), pubkey: state.pubkey).display_name.truncate(maxLength: 50))", comment: "Text telling the user to wake up, where the argument is their display name.") + Text("Wake up, \(Profile.displayName(profile: state.profiles.lookup(id: state.pubkey), pubkey: state.pubkey).displayName.truncate(maxLength: 50))", comment: "Text telling the user to wake up, where the argument is their display name.") Text("You are dreaming...", comment: "Text telling the user that they are dreaming.") } .id("what") diff --git a/damus/Views/Profile/EventProfileName.swift b/damus/Views/Profile/EventProfileName.swift index 9e604b4e5d..a63909fb10 100644 --- a/damus/Views/Profile/EventProfileName.swift +++ b/damus/Views/Profile/EventProfileName.swift @@ -63,11 +63,11 @@ struct EventProfileName: View { Text(one) .font(.body.weight(.bold)) - case .both(let both): - Text(both.display_name) + case .both(username: let username, displayName: let displayName): + Text(verbatim: displayName) .font(.body.weight(.bold)) - Text(verbatim: "@\(both.username)") + Text(verbatim: username) .foregroundColor(.gray) .font(eventviewsize_to_font(size, font_size: damus_state.settings.font_size)) } diff --git a/damus/Views/Profile/FollowsYou.swift b/damus/Views/Profile/FollowsYou.swift deleted file mode 100644 index 071dcffae1..0000000000 --- a/damus/Views/Profile/FollowsYou.swift +++ /dev/null @@ -1,29 +0,0 @@ -// -// FollowsYou.swift -// damus -// -// Created by William Casarin on 2023-02-07. -// - -import SwiftUI - -struct FollowsYou: View { - - var body: some View { - Text("Follows you", comment: "Text to indicate that a user is following your profile.") - .padding([.leading, .trailing], 6.0) - .padding([.top, .bottom], 2.0) - .foregroundColor(.gray) - .background { - RoundedRectangle(cornerRadius: 5.0) - .foregroundColor(DamusColors.adaptableGrey) - } - .font(.footnote) - } -} - -struct FollowsYou_Previews: PreviewProvider { - static var previews: some View { - FollowsYou() - } -} diff --git a/damus/Views/Profile/ProfileName.swift b/damus/Views/Profile/ProfileName.swift index 5ac2ccbe43..2a6acfd4b3 100644 --- a/damus/Views/Profile/ProfileName.swift +++ b/damus/Views/Profile/ProfileName.swift @@ -57,7 +57,7 @@ struct ProfileName: View { } var name_choice: String { - return prefix == "@" ? current_display_name.username.truncate(maxLength: 50) : current_display_name.display_name.truncate(maxLength: 50) + return prefix == "@" ? current_display_name.username.truncate(maxLength: 50) : current_display_name.displayName.truncate(maxLength: 50) } var onlyzapper: Bool { diff --git a/damus/Views/Profile/ProfileNameView.swift b/damus/Views/Profile/ProfileNameView.swift index 0042ba2a7b..0d1c523c60 100644 --- a/damus/Views/Profile/ProfileNameView.swift +++ b/damus/Views/Profile/ProfileNameView.swift @@ -7,10 +7,87 @@ import SwiftUI +fileprivate struct KeyView: View { + let pubkey: Pubkey + + @Environment(\.colorScheme) var colorScheme + + @State private var isCopied = false + + func keyColor() -> Color { + colorScheme == .light ? DamusColors.black : DamusColors.white + } + + private func copyPubkey(_ pubkey: String) { + UIPasteboard.general.string = pubkey + UIImpactFeedbackGenerator(style: .medium).impactOccurred() + withAnimation { + isCopied = true + DispatchQueue.main.asyncAfter(deadline: .now() + 3) { + withAnimation { + isCopied = false + } + } + } + } + + func pubkey_context_menu(pubkey: Pubkey) -> some View { + return self.contextMenu { + Button { + UIPasteboard.general.string = pubkey.npub + } label: { + Label(NSLocalizedString("Copy Account ID", comment: "Context menu option for copying the ID of the account that created the note."), image: "copy2") + } + } + } + + var body: some View { + let bech32 = pubkey.npub + + HStack { + Text(verbatim: "\(abbrev_pubkey(bech32, amount: 16))") + .font(.footnote) + .foregroundColor(keyColor()) + .padding(5) + .padding([.leading, .trailing], 5) + .background(RoundedRectangle(cornerRadius: 11).foregroundColor(DamusColors.adaptableGrey)) + + if isCopied { + HStack { + Image("check-circle") + .resizable() + .frame(width: 20, height: 20) + Text(NSLocalizedString("Copied", comment: "Label indicating that a user's key was copied.")) + .font(.footnote) + .layoutPriority(1) + } + .foregroundColor(DamusColors.green) + } else { + HStack { + Button { + copyPubkey(bech32) + } label: { + Label { + Text("Public key", comment: "Label indicating that the text is a user's public account key.") + } icon: { + Image("copy2") + .resizable() + .contentShape(Rectangle()) + .foregroundColor(.accentColor) + .frame(width: 20, height: 20) + } + .labelStyle(IconOnlyLabelStyle()) + .symbolRenderingMode(.hierarchical) + } + } + } + } + } +} + struct ProfileNameView: View { let pubkey: Pubkey let profile: Profile? - let follows_you: Bool let damus: DamusState var spacing: CGFloat { 10.0 } @@ -23,22 +100,15 @@ struct ProfileNameView: View { HStack(alignment: .center, spacing: spacing) { ProfileName(pubkey: pubkey, profile: profile, damus: damus) .font(.title3.weight(.bold)) - if follows_you { - FollowsYou() - } } - case .both(let both): - Text(both.display_name) + case .both(username: _, displayName: let displayName): + Text(displayName) .font(.title3.weight(.bold)) HStack(alignment: .center, spacing: spacing) { ProfileName(pubkey: pubkey, profile: profile, prefix: "@", damus: damus) .font(.callout) .foregroundColor(.gray) - - if follows_you { - FollowsYou() - } } } @@ -54,9 +124,9 @@ struct ProfileNameView: View { struct ProfileNameView_Previews: PreviewProvider { static var previews: some View { VStack { - ProfileNameView(pubkey: test_note.pubkey, profile: nil, follows_you: true, damus: test_damus_state()) + ProfileNameView(pubkey: test_note.pubkey, profile: nil, damus: test_damus_state()) - ProfileNameView(pubkey: test_note.pubkey, profile: nil, follows_you: false, damus: test_damus_state()) + ProfileNameView(pubkey: test_note.pubkey, profile: nil, damus: test_damus_state()) } } } diff --git a/damus/Views/Profile/ProfileView.swift b/damus/Views/Profile/ProfileView.swift index 9860e6635f..6cab5c5e16 100644 --- a/damus/Views/Profile/ProfileView.swift +++ b/damus/Views/Profile/ProfileView.swift @@ -255,6 +255,18 @@ struct ProfileView: View { .profile_button_style(scheme: colorScheme) } } + + private var followsYouBadge: some View { + Text("Follows you", comment: "Text to indicate that a user is following your profile.") + .padding([.leading, .trailing], 6.0) + .padding([.top, .bottom], 2.0) + .foregroundColor(.gray) + .background { + RoundedRectangle(cornerRadius: 5.0) + .foregroundColor(DamusColors.adaptableGrey) + } + .font(.footnote) + } func actionSection(profile_data: Profile?) -> some View { return Group { @@ -310,12 +322,16 @@ struct ProfileView: View { } Spacer() + + let follows_you = profile.pubkey != damus_state.pubkey && profile.follows(pubkey: damus_state.pubkey) + if follows_you { + followsYouBadge + } actionSection(profile_data: profile_data) } - - let follows_you = profile.pubkey != damus_state.pubkey && profile.follows(pubkey: damus_state.pubkey) - ProfileNameView(pubkey: profile.pubkey, profile: profile_data, follows_you: follows_you, damus: damus_state) + + ProfileNameView(pubkey: profile.pubkey, profile: profile_data, damus: damus_state) } } @@ -489,72 +505,6 @@ struct ProfileView_Previews: PreviewProvider { } } -struct KeyView: View { - let pubkey: Pubkey - - @Environment(\.colorScheme) var colorScheme - - @State private var isCopied = false - - func keyColor() -> Color { - colorScheme == .light ? DamusColors.black : DamusColors.white - } - - private func copyPubkey(_ pubkey: String) { - UIPasteboard.general.string = pubkey - UIImpactFeedbackGenerator(style: .medium).impactOccurred() - withAnimation { - isCopied = true - DispatchQueue.main.asyncAfter(deadline: .now() + 3) { - withAnimation { - isCopied = false - } - } - } - } - - var body: some View { - let bech32 = pubkey.npub - - HStack { - Text(verbatim: "\(abbrev_pubkey(bech32, amount: 16))") - .font(.footnote) - .foregroundColor(keyColor()) - .padding(5) - .padding([.leading, .trailing], 5) - .background(RoundedRectangle(cornerRadius: 11).foregroundColor(DamusColors.adaptableGrey)) - - if isCopied != true { - Button { - copyPubkey(bech32) - } label: { - Label { - Text("Public key", comment: "Label indicating that the text is a user's public account key.") - } icon: { - Image("copy2") - .resizable() - .contentShape(Rectangle()) - .foregroundColor(.accentColor) - .frame(width: 20, height: 20) - } - .labelStyle(IconOnlyLabelStyle()) - .symbolRenderingMode(.hierarchical) - } - } else { - HStack { - Image("check-circle") - .resizable() - .frame(width: 20, height: 20) - Text(NSLocalizedString("Copied", comment: "Label indicating that a user's key was copied.")) - .font(.footnote) - .layoutPriority(1) - } - .foregroundColor(DamusColors.green) - } - } - } -} - extension View { func profile_button_style(scheme: ColorScheme) -> some View { self.symbolRenderingMode(.palette) From 53734ea4836f4fec6515c738a8db58febfdb963b Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Wed, 6 Sep 2023 11:16:14 -0500 Subject: [PATCH 056/111] video: add AVPlayerView, a simple wrapper for AVPlayerViewController Closes: https://github.com/damus-io/damus/pull/1539 Reviewed-by: William Casarin Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 ++++ damus/Views/Video/AVPlayerView.swift | 31 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 damus/Views/Video/AVPlayerView.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 82e92fc81e..25bec02cf8 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -388,6 +388,7 @@ 504323A72A34915F006AE6DC /* RelayModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 504323A62A34915F006AE6DC /* RelayModel.swift */; }; 504323A92A3495B6006AE6DC /* RelayModelCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 504323A82A3495B6006AE6DC /* RelayModelCache.swift */; }; 5053ACA72A56DF3B00851AE3 /* DeveloperSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5053ACA62A56DF3B00851AE3 /* DeveloperSettingsView.swift */; }; + 50A16FFB2AA6C06600DFEC1F /* AVPlayerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */; }; 50A50A8D29A09E1C00C01BE7 /* RequestTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */; }; 50A60D142A28BEEE00186190 /* RelayLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A60D132A28BEEE00186190 /* RelayLog.swift */; }; 50B5685329F97CB400A23243 /* CredentialHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50B5685229F97CB400A23243 /* CredentialHandler.swift */; }; @@ -1063,6 +1064,7 @@ 504323A62A34915F006AE6DC /* RelayModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayModel.swift; sourceTree = ""; }; 504323A82A3495B6006AE6DC /* RelayModelCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayModelCache.swift; sourceTree = ""; }; 5053ACA62A56DF3B00851AE3 /* DeveloperSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeveloperSettingsView.swift; sourceTree = ""; }; + 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVPlayerView.swift; sourceTree = ""; }; 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RequestTests.swift; sourceTree = ""; }; 50A60D132A28BEEE00186190 /* RelayLog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayLog.swift; sourceTree = ""; }; 50B5685229F97CB400A23243 /* CredentialHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialHandler.swift; sourceTree = ""; }; @@ -1359,6 +1361,7 @@ children = ( 4C1A9A2929DDF54400516EAC /* DamusVideoPlayer.swift */, 4CCF9AAE2A1FDBDB00E03CFB /* VideoPlayer.swift */, + 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */, ); path = Video; sourceTree = ""; @@ -2795,6 +2798,7 @@ 4C3EA63D28FF52D600C48A62 /* bolt11.c in Sources */, 4C9BB83129C0ED4F00FC4E37 /* DisplayName.swift in Sources */, 7CFF6317299FEFE5005D382A /* SelectableText.swift in Sources */, + 50A16FFB2AA6C06600DFEC1F /* AVPlayerView.swift in Sources */, 4CA352A82A76B37E003BB08B /* NewMutesNotify.swift in Sources */, 4CFF8F6929CC9ED1008DB934 /* ImageContainerView.swift in Sources */, 7527271E2A93FF0100214108 /* Block.swift in Sources */, diff --git a/damus/Views/Video/AVPlayerView.swift b/damus/Views/Video/AVPlayerView.swift new file mode 100644 index 0000000000..7a69aef782 --- /dev/null +++ b/damus/Views/Video/AVPlayerView.swift @@ -0,0 +1,31 @@ +// +// AVPlayerView.swift +// damus +// +// Created by Bryan Montz on 9/4/23. +// + +import Foundation +import AVKit +import SwiftUI + +struct AVPlayerView: UIViewControllerRepresentable { + + let player: AVPlayer + + func makeUIViewController(context: Context) -> AVPlayerViewController { + AVPlayerViewController() + } + + func updateUIViewController(_ uiViewController: AVPlayerViewController, context: Context) { + if uiViewController.player == nil { + uiViewController.player = player + player.play() + } + } + + static func dismantleUIViewController(_ uiViewController: AVPlayerViewController, coordinator: ()) { + uiViewController.player?.pause() + uiViewController.player = nil + } +} From dec07df2c147735de2c8d1702d34b0203f263000 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Wed, 6 Sep 2023 11:23:33 -0500 Subject: [PATCH 057/111] video: add VideoController, which hold cached metadata and mute states Closes: https://github.com/damus-io/damus/pull/1539 Reviewed-by: William Casarin Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 +++ damus/ContentView.swift | 3 +- damus/Models/DamusState.swift | 4 ++- damus/Views/Video/VideoController.swift | 40 +++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 damus/Views/Video/VideoController.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 25bec02cf8..d78d57f641 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -389,6 +389,7 @@ 504323A92A3495B6006AE6DC /* RelayModelCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 504323A82A3495B6006AE6DC /* RelayModelCache.swift */; }; 5053ACA72A56DF3B00851AE3 /* DeveloperSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5053ACA62A56DF3B00851AE3 /* DeveloperSettingsView.swift */; }; 50A16FFB2AA6C06600DFEC1F /* AVPlayerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */; }; + 50A16FFF2AA76A0900DFEC1F /* VideoController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A16FFE2AA76A0900DFEC1F /* VideoController.swift */; }; 50A50A8D29A09E1C00C01BE7 /* RequestTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */; }; 50A60D142A28BEEE00186190 /* RelayLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A60D132A28BEEE00186190 /* RelayLog.swift */; }; 50B5685329F97CB400A23243 /* CredentialHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50B5685229F97CB400A23243 /* CredentialHandler.swift */; }; @@ -1065,6 +1066,7 @@ 504323A82A3495B6006AE6DC /* RelayModelCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayModelCache.swift; sourceTree = ""; }; 5053ACA62A56DF3B00851AE3 /* DeveloperSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeveloperSettingsView.swift; sourceTree = ""; }; 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVPlayerView.swift; sourceTree = ""; }; + 50A16FFE2AA76A0900DFEC1F /* VideoController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoController.swift; sourceTree = ""; }; 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RequestTests.swift; sourceTree = ""; }; 50A60D132A28BEEE00186190 /* RelayLog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayLog.swift; sourceTree = ""; }; 50B5685229F97CB400A23243 /* CredentialHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialHandler.swift; sourceTree = ""; }; @@ -1360,6 +1362,7 @@ isa = PBXGroup; children = ( 4C1A9A2929DDF54400516EAC /* DamusVideoPlayer.swift */, + 50A16FFE2AA76A0900DFEC1F /* VideoController.swift */, 4CCF9AAE2A1FDBDB00E03CFB /* VideoPlayer.swift */, 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */, ); @@ -2621,6 +2624,7 @@ 4CEE2AF7280B2DEA00AB5EEF /* ProfileName.swift in Sources */, 4CC7AAEB297F0AEC00430951 /* BuilderEventView.swift in Sources */, 31D2E847295218AF006D67F8 /* Shimmer.swift in Sources */, + 50A16FFF2AA76A0900DFEC1F /* VideoController.swift in Sources */, F7908E97298B1FDF00AB113A /* NIPURLBuilder.swift in Sources */, 4C285C8228385570008A31F1 /* CarouselView.swift in Sources */, 3A3040F129A8FF97008A0F29 /* LocalizationUtil.swift in Sources */, diff --git a/damus/ContentView.swift b/damus/ContentView.swift index bc9628a505..8782d5835a 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -654,7 +654,8 @@ struct ContentView: View { wallet: WalletModel(settings: settings), nav: self.navigationCoordinator, user_search_cache: user_search_cache, - music: MusicController(onChange: music_changed) + music: MusicController(onChange: music_changed), + video: VideoController() ) home.damus_state = self.damus_state! diff --git a/damus/Models/DamusState.swift b/damus/Models/DamusState.swift index 891db8ea8c..5811235ea6 100644 --- a/damus/Models/DamusState.swift +++ b/damus/Models/DamusState.swift @@ -33,6 +33,7 @@ struct DamusState { let nav: NavigationCoordinator let user_search_cache: UserSearchCache let music: MusicController? + let video: VideoController @discardableResult func add_zap(zap: Zapping) -> Bool { @@ -89,7 +90,8 @@ struct DamusState { wallet: WalletModel(settings: UserSettingsStore()), nav: NavigationCoordinator(), user_search_cache: user_search_cache, - music: nil + music: nil, + video: VideoController() ) } } diff --git a/damus/Views/Video/VideoController.swift b/damus/Views/Video/VideoController.swift new file mode 100644 index 0000000000..8f524976c6 --- /dev/null +++ b/damus/Views/Video/VideoController.swift @@ -0,0 +1,40 @@ +// +// VideoController.swift +// damus +// +// Created by Bryan Montz on 9/3/23. +// + +import Combine +import Foundation + +struct VideoMetadata { + let has_audio: Bool + let size: CGSize +} + +final class VideoController: ObservableObject { + private var mute_states: [URL: Bool] = [:] + private var metadatas: [URL: VideoMetadata] = [:] + + @Published var focused_model_id: UUID? + + func toggle_should_mute_video(url: URL) { + let state = mute_states[url] ?? true + mute_states[url] = !state + + objectWillChange.send() + } + + func should_mute_video(url: URL) -> Bool { + mute_states[url] ?? true + } + + func set_metadata(_ metadata: VideoMetadata, url: URL) { + metadatas[url] = metadata + } + + func metadata(for url: URL) -> VideoMetadata? { + metadatas[url] + } +} From f1f3abfb98da7bd1f88defab3dc49199ab68e763 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Wed, 6 Sep 2023 11:25:07 -0500 Subject: [PATCH 058/111] video: add DamusVideoPlayerViewModel Closes: https://github.com/damus-io/damus/pull/1539 Reviewed-by: William Casarin Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 + .../Video/DamusVideoPlayerViewModel.swift | 105 ++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 damus/Views/Video/DamusVideoPlayerViewModel.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index d78d57f641..a4e198b08f 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -389,6 +389,7 @@ 504323A92A3495B6006AE6DC /* RelayModelCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 504323A82A3495B6006AE6DC /* RelayModelCache.swift */; }; 5053ACA72A56DF3B00851AE3 /* DeveloperSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5053ACA62A56DF3B00851AE3 /* DeveloperSettingsView.swift */; }; 50A16FFB2AA6C06600DFEC1F /* AVPlayerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */; }; + 50A16FFD2AA7525700DFEC1F /* DamusVideoPlayerViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A16FFC2AA7525700DFEC1F /* DamusVideoPlayerViewModel.swift */; }; 50A16FFF2AA76A0900DFEC1F /* VideoController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A16FFE2AA76A0900DFEC1F /* VideoController.swift */; }; 50A50A8D29A09E1C00C01BE7 /* RequestTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */; }; 50A60D142A28BEEE00186190 /* RelayLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A60D132A28BEEE00186190 /* RelayLog.swift */; }; @@ -1066,6 +1067,7 @@ 504323A82A3495B6006AE6DC /* RelayModelCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayModelCache.swift; sourceTree = ""; }; 5053ACA62A56DF3B00851AE3 /* DeveloperSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeveloperSettingsView.swift; sourceTree = ""; }; 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVPlayerView.swift; sourceTree = ""; }; + 50A16FFC2AA7525700DFEC1F /* DamusVideoPlayerViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusVideoPlayerViewModel.swift; sourceTree = ""; }; 50A16FFE2AA76A0900DFEC1F /* VideoController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoController.swift; sourceTree = ""; }; 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RequestTests.swift; sourceTree = ""; }; 50A60D132A28BEEE00186190 /* RelayLog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayLog.swift; sourceTree = ""; }; @@ -1362,6 +1364,7 @@ isa = PBXGroup; children = ( 4C1A9A2929DDF54400516EAC /* DamusVideoPlayer.swift */, + 50A16FFC2AA7525700DFEC1F /* DamusVideoPlayerViewModel.swift */, 50A16FFE2AA76A0900DFEC1F /* VideoController.swift */, 4CCF9AAE2A1FDBDB00E03CFB /* VideoPlayer.swift */, 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */, @@ -2733,6 +2736,7 @@ 4CB88393296F798300DC99E7 /* ReactionsModel.swift in Sources */, 5C42E78C29DB76D90086AAC1 /* EmptyUserSearchView.swift in Sources */, 4CB88396296F7F8B00DC99E7 /* ReactionView.swift in Sources */, + 50A16FFD2AA7525700DFEC1F /* DamusVideoPlayerViewModel.swift in Sources */, 4CFF8F6B29CD0079008DB934 /* RepostedEvent.swift in Sources */, 4C8682872814DE470026224F /* ProfileView.swift in Sources */, 5C0707D12A1ECB38004E7B51 /* DamusLogoGradient.swift in Sources */, diff --git a/damus/Views/Video/DamusVideoPlayerViewModel.swift b/damus/Views/Video/DamusVideoPlayerViewModel.swift new file mode 100644 index 0000000000..09551a2721 --- /dev/null +++ b/damus/Views/Video/DamusVideoPlayerViewModel.swift @@ -0,0 +1,105 @@ +// +// DamusVideoPlayerViewModel.swift +// damus +// +// Created by Bryan Montz on 9/5/23. +// + +import AVFoundation +import Combine +import Foundation +import SwiftUI + +@MainActor +final class DamusVideoPlayerViewModel: ObservableObject { + + private let url: URL + private let player_item: AVPlayerItem + let player: AVPlayer + private let controller: VideoController + let id = UUID() + + @Published var has_audio = false + @Binding var video_size: CGSize? + @Published var is_muted = true + @Published var is_loading = true + + private var cancellables = Set() + + private var is_scrolled_into_view = false { + didSet { + if is_scrolled_into_view && !oldValue { + // we have just scrolled from out of view into view + controller.focused_model_id = id + } else if !is_scrolled_into_view && oldValue { + // we have just scrolled from in view to out of view + if controller.focused_model_id == id { + controller.focused_model_id = nil + } + } + } + } + + init(url: URL, video_size: Binding, controller: VideoController) { + self.url = url + player_item = AVPlayerItem(url: url) + player = AVPlayer(playerItem: player_item) + self.controller = controller + _video_size = video_size + + Task { + await load() + } + + is_muted = controller.should_mute_video(url: url) + player.isMuted = is_muted + + NotificationCenter.default.addObserver( + self, + selector: #selector(did_play_to_end), + name: Notification.Name.AVPlayerItemDidPlayToEndTime, + object: player_item + ) + + controller.$focused_model_id + .sink { [weak self] model_id in + model_id == self?.id ? self?.player.play() : self?.player.pause() + } + .store(in: &cancellables) + } + + private func load() async { + if let meta = controller.metadata(for: url) { + has_audio = meta.has_audio + video_size = meta.size + } else { + has_audio = await video_has_audio(player: player) + if let video_size = await get_video_size(player: player) { + self.video_size = video_size + let meta = VideoMetadata(has_audio: has_audio, size: video_size) + controller.set_metadata(meta, url: url) + } + } + + is_loading = false + } + + func did_tap_mute_button() { + is_muted.toggle() + player.isMuted = is_muted + controller.toggle_should_mute_video(url: url) + } + + func set_view_is_visible(_ is_visible: Bool) { + is_scrolled_into_view = is_visible + } + + func view_did_disappear() { + set_view_is_visible(false) + } + + @objc private func did_play_to_end() { + player.seek(to: CMTime.zero) + player.play() + } +} From 3569da568776dd2e015ca4130c23813c8889d71d Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Wed, 6 Sep 2023 11:29:45 -0500 Subject: [PATCH 059/111] video: switch player to use new view model pass VideoController through containing views Closes: https://github.com/damus-io/damus/pull/1539 Reviewed-by: William Casarin Signed-off-by: William Casarin --- damus/Components/ImageCarousel.swift | 8 +- damus/Views/Images/ImageContainerView.swift | 6 +- damus/Views/Images/ImageView.swift | 6 +- damus/Views/Video/DamusVideoPlayer.swift | 116 ++++++++++---------- 4 files changed, 69 insertions(+), 67 deletions(-) diff --git a/damus/Components/ImageCarousel.swift b/damus/Components/ImageCarousel.swift index f99b672959..c7553518bd 100644 --- a/damus/Components/ImageCarousel.swift +++ b/damus/Components/ImageCarousel.swift @@ -112,10 +112,6 @@ struct ImageCarousel: View { } } - func video_model(_ url: URL) -> VideoPlayerModel { - return state.events.get_video_player_model(url: url) - } - func Media(geo: GeometryProxy, url: MediaUrl, index: Int) -> some View { Group { switch url { @@ -125,7 +121,7 @@ struct ImageCarousel: View { open_sheet = true } case .video(let url): - DamusVideoPlayer(url: url, model: video_model(url), video_size: $video_size) + DamusVideoPlayer(url: url, video_size: $video_size, controller: state.video) .onChange(of: video_size) { size in guard let size else { return } @@ -194,7 +190,7 @@ struct ImageCarousel: View { } .tabViewStyle(PageTabViewStyle(indexDisplayMode: .never)) .fullScreenCover(isPresented: $open_sheet) { - ImageView(cache: state.events, urls: urls, disable_animation: state.settings.disable_animation) + ImageView(video_controller: state.video, urls: urls, disable_animation: state.settings.disable_animation) } .frame(height: height) .onChange(of: selectedIndex) { value in diff --git a/damus/Views/Images/ImageContainerView.swift b/damus/Views/Images/ImageContainerView.swift index cd4499a793..64396ad30e 100644 --- a/damus/Views/Images/ImageContainerView.swift +++ b/damus/Views/Images/ImageContainerView.swift @@ -10,7 +10,7 @@ import Kingfisher struct ImageContainerView: View { - let cache: EventCache + let video_controller: VideoController let url: MediaUrl @State private var image: UIImage? @@ -47,7 +47,7 @@ struct ImageContainerView: View { case .image(let url): Img(url: url) case .video(let url): - DamusVideoPlayer(url: url, model: cache.get_video_player_model(url: url), video_size: .constant(nil)) + DamusVideoPlayer(url: url, video_size: .constant(nil), controller: video_controller) } } } @@ -57,6 +57,6 @@ let test_image_url = URL(string: "https://jb55.com/red-me.jpg")! struct ImageContainerView_Previews: PreviewProvider { static var previews: some View { - ImageContainerView(cache: test_damus_state().events, url: .image(test_image_url), disable_animation: false) + ImageContainerView(video_controller: test_damus_state().video, url: .image(test_image_url), disable_animation: false) } } diff --git a/damus/Views/Images/ImageView.swift b/damus/Views/Images/ImageView.swift index a51133c794..850dc583dd 100644 --- a/damus/Views/Images/ImageView.swift +++ b/damus/Views/Images/ImageView.swift @@ -8,7 +8,7 @@ import SwiftUI struct ImageView: View { - let cache: EventCache + let video_controller: VideoController let urls: [MediaUrl] @Environment(\.presentationMode) var presentationMode @@ -39,7 +39,7 @@ struct ImageView: View { TabView(selection: $selectedIndex) { ForEach(urls.indices, id: \.self) { index in ZoomableScrollView { - ImageContainerView(cache: cache, url: urls[index], disable_animation: disable_animation) + ImageContainerView(video_controller: video_controller, url: urls[index], disable_animation: disable_animation) .aspectRatio(contentMode: .fit) .padding(.top, Theme.safeAreaInsets?.top) .padding(.bottom, Theme.safeAreaInsets?.bottom) @@ -80,6 +80,6 @@ struct ImageView: View { struct ImageView_Previews: PreviewProvider { static var previews: some View { let url: MediaUrl = .image(URL(string: "https://jb55.com/red-me.jpg")!) - ImageView(cache: test_damus_state().events, urls: [url], disable_animation: false) + ImageView(video_controller: test_damus_state().video, urls: [url], disable_animation: false) } } diff --git a/damus/Views/Video/DamusVideoPlayer.swift b/damus/Views/Video/DamusVideoPlayer.swift index ca64e42edd..bbca2a3758 100644 --- a/damus/Views/Video/DamusVideoPlayer.swift +++ b/damus/Views/Video/DamusVideoPlayer.swift @@ -17,79 +17,85 @@ func globalCoordinate(localX x: CGFloat, localY y: CGFloat, } struct DamusVideoPlayer: View { - var url: URL - @ObservedObject var model: VideoPlayerModel - @Binding var video_size: CGSize? + let url: URL + @StateObject var model: DamusVideoPlayerViewModel @EnvironmentObject private var orientationTracker: OrientationTracker - var mute_icon: String { - if model.has_audio == false || model.muted { - return "speaker.slash" - } else { - return "speaker" - } - } - - var mute_icon_color: Color { - switch self.model.has_audio { - case .none: - return .white - case .some(let has_audio): - return has_audio ? .white : .red - } - } - - var MuteIcon: some View { - ZStack { - Circle() - .opacity(0.2) - .frame(width: 32, height: 32) - .foregroundColor(.black) - - Image(systemName: mute_icon) - .padding() - .foregroundColor(mute_icon_color) - } + init(url: URL, video_size: Binding, controller: VideoController) { + self.url = url + _model = StateObject(wrappedValue: DamusVideoPlayerViewModel(url: url, video_size: video_size, controller: controller)) } var body: some View { GeometryReader { geo in let localFrame = geo.frame(in: .local) let centerY = globalCoordinate(localX: 0, localY: localFrame.midY, localGeometry: geo).y - let delta = localFrame.height / 2 - ZStack(alignment: .bottomTrailing) { - VideoPlayer(url: url, model: model) - if model.has_audio == true { - MuteIcon - .zIndex(11.0) - .onTapGesture { - self.model.muted = !self.model.muted - } + ZStack { + AVPlayerView(player: model.player) + + if model.is_loading { + ProgressView() + .progressViewStyle(.circular) + .tint(.white) + .scaleEffect(CGSize(width: 1.5, height: 1.5)) } - } - .onChange(of: model.size) { size in - guard let size else { - return + + if model.has_audio { + mute_button } - video_size = size } .onChange(of: centerY) { _ in - /// pause video when it is scrolled beyond visible range - let isBelowTop = centerY + delta > 100, /// 100 =~ approx. bottom (y) of ContentView's TabView - isAboveBottom = centerY - delta < orientationTracker.deviceMajorAxis - if isBelowTop && isAboveBottom { - model.start() - } else { - model.stop() + update_is_visible(centerY: centerY) + } + .onAppear { + update_is_visible(centerY: centerY) + } + } + .onDisappear { + model.view_did_disappear() + } + } + + private func update_is_visible(centerY: CGFloat) { + let isBelowTop = centerY > 100, /// 100 =~ approx. bottom (y) of ContentView's TabView + isAboveBottom = centerY < orientationTracker.deviceMajorAxis + model.set_view_is_visible(isBelowTop && isAboveBottom) + } + + private var mute_icon: String { + !model.has_audio || model.is_muted ? "speaker.slash" : "speaker" + } + + private var mute_icon_color: Color { + model.has_audio ? .white : .red + } + + private var mute_button: some View { + HStack { + Spacer() + VStack { + Spacer() + + Button { + model.did_tap_mute_button() + } label: { + ZStack { + Circle() + .opacity(0.2) + .frame(width: 32, height: 32) + .foregroundColor(.black) + + Image(systemName: mute_icon) + .padding() + .foregroundColor(mute_icon_color) + } } } } } } struct DamusVideoPlayer_Previews: PreviewProvider { - @StateObject static var model: VideoPlayerModel = VideoPlayerModel() - static var previews: some View { - DamusVideoPlayer(url: URL(string: "http://cdn.jb55.com/s/zaps-build.mp4")!, model: model, video_size: .constant(nil)) + DamusVideoPlayer(url: URL(string: "http://cdn.jb55.com/s/zaps-build.mp4")!, video_size: .constant(nil), controller: VideoController()) } } From 9cf53a9e93286ef881e57e5c29f2364f91e2ea7e Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Wed, 6 Sep 2023 11:49:06 -0500 Subject: [PATCH 060/111] video: remove VideoPlayer and switch to VideoController for cache Closes: https://github.com/damus-io/damus/pull/1539 Reviewed-by: William Casarin Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 8 +- damus/Components/ImageCarousel.swift | 2 +- damus/Util/AVPlayer+Additions.swift | 35 ++ damus/Util/EventCache.swift | 26 -- .../Video/DamusVideoPlayerViewModel.swift | 12 + damus/Views/Video/VideoController.swift | 4 + damus/Views/Video/VideoPlayer.swift | 350 ------------------ 7 files changed, 56 insertions(+), 381 deletions(-) create mode 100644 damus/Util/AVPlayer+Additions.swift delete mode 100644 damus/Views/Video/VideoPlayer.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index a4e198b08f..acf8a6806b 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -317,7 +317,6 @@ 4CC7AAFA297F64AC00430951 /* EventMenu.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CC7AAF9297F64AC00430951 /* EventMenu.swift */; }; 4CCEB7AE29B53D260078AA28 /* SearchingEventView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CCEB7AD29B53D260078AA28 /* SearchingEventView.swift */; }; 4CCEB7B029B5415A0078AA28 /* SearchingProfileView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CCEB7AF29B5415A0078AA28 /* SearchingProfileView.swift */; }; - 4CCF9AAF2A1FDBDB00E03CFB /* VideoPlayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CCF9AAE2A1FDBDB00E03CFB /* VideoPlayer.swift */; }; 4CCF9AB22A1FE80C00E03CFB /* GSPlayer in Frameworks */ = {isa = PBXBuildFile; productRef = 4CCF9AB12A1FE80C00E03CFB /* GSPlayer */; }; 4CD348EF29C3659D00497EB2 /* ImageUploadModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CD348EE29C3659D00497EB2 /* ImageUploadModel.swift */; }; 4CD7641B28A1641400B6928F /* EndBlock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CD7641A28A1641400B6928F /* EndBlock.swift */; }; @@ -394,6 +393,7 @@ 50A50A8D29A09E1C00C01BE7 /* RequestTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */; }; 50A60D142A28BEEE00186190 /* RelayLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A60D132A28BEEE00186190 /* RelayLog.swift */; }; 50B5685329F97CB400A23243 /* CredentialHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50B5685229F97CB400A23243 /* CredentialHandler.swift */; }; + 50C3E08A2AA8E3F7006A4BC0 /* AVPlayer+Additions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50C3E0892AA8E3F7006A4BC0 /* AVPlayer+Additions.swift */; }; 50DA11262A16A23F00236234 /* Launch.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 50DA11252A16A23F00236234 /* Launch.storyboard */; }; 5C0707D12A1ECB38004E7B51 /* DamusLogoGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C0707D02A1ECB38004E7B51 /* DamusLogoGradient.swift */; }; 5C42E78C29DB76D90086AAC1 /* EmptyUserSearchView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C42E78B29DB76D90086AAC1 /* EmptyUserSearchView.swift */; }; @@ -990,7 +990,6 @@ 4CC7AAF9297F64AC00430951 /* EventMenu.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventMenu.swift; sourceTree = ""; }; 4CCEB7AD29B53D260078AA28 /* SearchingEventView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchingEventView.swift; sourceTree = ""; }; 4CCEB7AF29B5415A0078AA28 /* SearchingProfileView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchingProfileView.swift; sourceTree = ""; }; - 4CCF9AAE2A1FDBDB00E03CFB /* VideoPlayer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoPlayer.swift; sourceTree = ""; }; 4CD348EE29C3659D00497EB2 /* ImageUploadModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageUploadModel.swift; sourceTree = ""; }; 4CD7641A28A1641400B6928F /* EndBlock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EndBlock.swift; sourceTree = ""; }; 4CDA128929E9D10C0006FA5A /* SignalView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SignalView.swift; sourceTree = ""; }; @@ -1072,6 +1071,7 @@ 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RequestTests.swift; sourceTree = ""; }; 50A60D132A28BEEE00186190 /* RelayLog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayLog.swift; sourceTree = ""; }; 50B5685229F97CB400A23243 /* CredentialHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialHandler.swift; sourceTree = ""; }; + 50C3E0892AA8E3F7006A4BC0 /* AVPlayer+Additions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVPlayer+Additions.swift"; sourceTree = ""; }; 50DA11252A16A23F00236234 /* Launch.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = Launch.storyboard; sourceTree = ""; }; 5C0707D02A1ECB38004E7B51 /* DamusLogoGradient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusLogoGradient.swift; sourceTree = ""; }; 5C42E78B29DB76D90086AAC1 /* EmptyUserSearchView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmptyUserSearchView.swift; sourceTree = ""; }; @@ -1366,7 +1366,6 @@ 4C1A9A2929DDF54400516EAC /* DamusVideoPlayer.swift */, 50A16FFC2AA7525700DFEC1F /* DamusVideoPlayerViewModel.swift */, 50A16FFE2AA76A0900DFEC1F /* VideoController.swift */, - 4CCF9AAE2A1FDBDB00E03CFB /* VideoPlayer.swift */, 50A16FFA2AA6C06600DFEC1F /* AVPlayerView.swift */, ); path = Video; @@ -1791,6 +1790,7 @@ 3A8CC6CB2A2CFEF900940F5F /* StringUtil.swift */, D2277EE92A089BD5006C3807 /* Router.swift */, 4C2B10272A7B0F5C008AA43E /* Log.swift */, + 50C3E0892AA8E3F7006A4BC0 /* AVPlayer+Additions.swift */, ); path = Util; sourceTree = ""; @@ -2507,7 +2507,6 @@ 4C216F34286F5ACD00040376 /* DMView.swift in Sources */, 4C32B9572A9AD44700DC3548 /* Root.swift in Sources */, 4C3EA64428FF558100C48A62 /* sha256.c in Sources */, - 4CCF9AAF2A1FDBDB00E03CFB /* VideoPlayer.swift in Sources */, 504323A72A34915F006AE6DC /* RelayModel.swift in Sources */, 4CA9276A2A290FC00098A105 /* ContextButton.swift in Sources */, 4CF0ABF62985CD5500D66079 /* UserSearch.swift in Sources */, @@ -2554,6 +2553,7 @@ 4C7D09742A0AEF9000943473 /* AlbyGradient.swift in Sources */, 4C687C272A6039500092C550 /* TestData.swift in Sources */, 3AA247FD297E3CFF0090C62D /* RepostsModel.swift in Sources */, + 50C3E08A2AA8E3F7006A4BC0 /* AVPlayer+Additions.swift in Sources */, 4C198DF229F88C6B004C165C /* BlurHashDecode.swift in Sources */, F75BA12F29A18EF500E10810 /* BookmarksView.swift in Sources */, 4CB883B6297730E400DC99E7 /* LNUrls.swift in Sources */, diff --git a/damus/Components/ImageCarousel.swift b/damus/Components/ImageCarousel.swift index c7553518bd..811b9ff2b8 100644 --- a/damus/Components/ImageCarousel.swift +++ b/damus/Components/ImageCarousel.swift @@ -105,7 +105,7 @@ struct ImageCarousel: View { } } .onAppear { - if self.image_fill == nil, let size = state.events.lookup_media_size(url: url) { + if self.image_fill == nil, let size = state.video.size_for_url(url) { let fill = ImageFill.calculate_image_fill(geo_size: geo_size, img_size: size, maxHeight: maxHeight, fillHeight: fillHeight) self.image_fill = fill } diff --git a/damus/Util/AVPlayer+Additions.swift b/damus/Util/AVPlayer+Additions.swift new file mode 100644 index 0000000000..f870d92140 --- /dev/null +++ b/damus/Util/AVPlayer+Additions.swift @@ -0,0 +1,35 @@ +// +// AVPlayer+Additions.swift +// damus +// +// Created by Bryan Montz on 9/6/23. +// + +import AVFoundation +import Foundation +import UIKit + +extension AVPlayer { +#if !os(macOS) + var currentImage: UIImage? { + guard + let playerItem = currentItem, + let cgImage = try? AVAssetImageGenerator(asset: playerItem.asset).copyCGImage(at: currentTime(), actualTime: nil) + else { return nil } + + return UIImage(cgImage: cgImage) + } +#else + var currentImage: NSImage? { + guard + let playerItem = currentItem, + let cgImage = try? AVAssetImageGenerator(asset: playerItem.asset).copyCGImage(at: currentTime(), actualTime: nil) + else { + return nil + } + let width: CGFloat = CGFloat(cgImage.width) + let height: CGFloat = CGFloat(cgImage.height) + return NSImage(cgImage: cgImage, size: NSMakeSize(width, height)) + } +#endif +} diff --git a/damus/Util/EventCache.swift b/damus/Util/EventCache.swift index d48fea6c73..29d3229c46 100644 --- a/damus/Util/EventCache.swift +++ b/damus/Util/EventCache.swift @@ -141,7 +141,6 @@ class EventCache { private var replies = ReplyMap() private var cancellable: AnyCancellable? private var image_metadata: [String: ImageMetadataState] = [:] // lowercased URL key - private var video_meta: [URL: VideoPlayerModel] = [:] private var event_data: [NoteId: EventData] = [:] //private var thread_latest: [String: Int64] @@ -204,30 +203,6 @@ class EventCache { return image_metadata[url.absoluteString.lowercased()] } - @MainActor - func lookup_media_size(url: URL) -> CGSize? { - if let img_meta = lookup_img_metadata(url: url) { - return img_meta.meta.dim?.size - } - - return get_video_player_model(url: url).size - } - - func store_video_player_model(url: URL, meta: VideoPlayerModel) { - video_meta[url] = meta - } - - @MainActor - func get_video_player_model(url: URL) -> VideoPlayerModel { - if let model = video_meta[url] { - return model - } - - let model = VideoPlayerModel() - video_meta[url] = model - return model - } - func parent_events(event: NostrEvent, keypair: Keypair) -> [NostrEvent] { var parents: [NostrEvent] = [] @@ -289,7 +264,6 @@ class EventCache { private func prune() { events = [:] - video_meta = [:] event_data = [:] replies.replies = [:] } diff --git a/damus/Views/Video/DamusVideoPlayerViewModel.swift b/damus/Views/Video/DamusVideoPlayerViewModel.swift index 09551a2721..761b8a4d5a 100644 --- a/damus/Views/Video/DamusVideoPlayerViewModel.swift +++ b/damus/Views/Video/DamusVideoPlayerViewModel.swift @@ -10,6 +10,18 @@ import Combine import Foundation import SwiftUI +func get_video_size(player: AVPlayer) async -> CGSize? { + let res = Task.detached(priority: .background) { + return player.currentImage?.size + } + return await res.value +} + +func video_has_audio(player: AVPlayer) async -> Bool { + let tracks = try? await player.currentItem?.asset.load(.tracks) + return tracks?.filter({ t in t.mediaType == .audio }).first != nil +} + @MainActor final class DamusVideoPlayerViewModel: ObservableObject { diff --git a/damus/Views/Video/VideoController.swift b/damus/Views/Video/VideoController.swift index 8f524976c6..9377d5cb4d 100644 --- a/damus/Views/Video/VideoController.swift +++ b/damus/Views/Video/VideoController.swift @@ -37,4 +37,8 @@ final class VideoController: ObservableObject { func metadata(for url: URL) -> VideoMetadata? { metadatas[url] } + + func size_for_url(_ url: URL) -> CGSize? { + metadatas[url]?.size + } } diff --git a/damus/Views/Video/VideoPlayer.swift b/damus/Views/Video/VideoPlayer.swift deleted file mode 100644 index 2a0a04d492..0000000000 --- a/damus/Views/Video/VideoPlayer.swift +++ /dev/null @@ -1,350 +0,0 @@ -// -// VideoPlayer.swift -// damus -// -// Created by William Casarin on 2023-05-25. -// - -import Foundation -// -// VideoPlayer.swift -// VideoPlayer -// -// Created by Gesen on 2019/7/7. -// Copyright © 2019 Gesen. All rights reserved. -// - -import AVFoundation -import GSPlayer -import SwiftUI - -public enum VideoState { - /// From the first load to get the first frame of the video - case loading - - /// Playing now - case playing(totalDuration: Double) - - /// Pause, will be called repeatedly when the buffer progress changes - case paused(playProgress: Double, bufferProgress: Double) - - /// An error occurred and cannot continue playing - case error(NSError) -} - -enum VideoHandler { - case onBufferChanged((Double) -> Void) - case onPlayToEndTime(() -> Void) - case onReplay(() -> Void) - case onStateChanged((VideoState) -> Void) -} - -@MainActor -public class VideoPlayerModel: ObservableObject { - @Published var autoReplay: Bool = true - @Published var muted: Bool = true - @Published var play: Bool = true - @Published var size: CGSize? = nil - @Published var has_audio: Bool? = nil - @Published var contentMode: UIView.ContentMode = .scaleAspectFill - - fileprivate var time: CMTime? - - var handlers: [VideoHandler] = [] - - init() { - } - - func stop() { - self.play = false - } - - func start() { - self.play = true - } - - func mute() { - self.muted = true - } - - func unmute() { - self.muted = false - } - - /// Whether the video will be automatically replayed until the end of the video playback. - func autoReplay(_ value: Bool) -> Self { - autoReplay = value - return self - } - - /// Whether the video is muted, only for this instance. - func mute(_ value: Bool) -> Self { - muted = value - return self - } - - /// A string defining how the video is displayed within an AVPlayerLayer bounds rect. - /// scaleAspectFill -> resizeAspectFill, scaleAspectFit -> resizeAspect, other -> resize - func contentMode(_ value: UIView.ContentMode) -> Self { - contentMode = value - return self - } - - /// Trigger a callback when the buffer progress changes, - /// the value is between 0 and 1. - func onBufferChanged(_ handler: @escaping (Double) -> Void) -> Self { - self.handlers.append(.onBufferChanged(handler)) - return self - } - - /// Playing to the end. - func onPlayToEndTime(_ handler: @escaping () -> Void) -> Self { - self.handlers.append(.onPlayToEndTime(handler)) - return self - } - - /// Replay after playing to the end. - func onReplay(_ handler: @escaping () -> Void) -> Self { - self.handlers.append(.onReplay(handler)) - return self - } - - /// Playback status changes, such as from play to pause. - func onStateChanged(_ handler: @escaping (VideoState) -> Void) -> Self { - self.handlers.append(.onStateChanged(handler)) - return self - } -} - -@available(iOS 13, *) -public struct VideoPlayer { - private(set) var url: URL - - @ObservedObject var model: VideoPlayerModel - - /// Init video player instance. - /// - Parameters: - /// - url: http/https URL - /// - play: play/pause - /// - time: current time - public init(url: URL, model: VideoPlayerModel) { - self.url = url - self._model = ObservedObject(wrappedValue: model) - } -} - -@available(iOS 13, *) -public extension VideoPlayer { - - /// Set the preload size, the default value is 1024 * 1024, unit is byte. - static var preloadByteCount: Int { - get { VideoPreloadManager.shared.preloadByteCount } - set { VideoPreloadManager.shared.preloadByteCount = newValue } - } - - /// Set the video urls to be preload queue. - /// Preloading will automatically cache a short segment of the beginning of the video - /// and decide whether to start or pause the preload based on the buffering of the currently playing video. - /// - Parameter urls: URL array - static func preload(urls: [URL]) { - VideoPreloadManager.shared.set(waiting: urls) - } - - /// Set custom http header, such as token. - static func customHTTPHeaderFields(transform: @escaping (URL) -> [String: String]?) { - VideoLoadManager.shared.customHTTPHeaderFields = transform - } - - /// Get the total size of the video cache. - static func calculateCachedSize() -> UInt { - return VideoCacheManager.calculateCachedSize() - } - - /// Clean up all caches. - static func cleanAllCache() { - try? VideoCacheManager.cleanAllCache() - } -} - -func get_video_size(player: AVPlayer) async -> CGSize? { - let res = Task.detached(priority: .background) { - return player.currentImage?.size - } - return await res.value -} - -func video_has_audio(player: AVPlayer) async -> Bool { - let tracks = try? await player.currentItem?.asset.load(.tracks) - return tracks?.filter({ t in t.mediaType == .audio }).first != nil -} - -@available(iOS 13, *) -extension VideoPlayer: UIViewRepresentable { - - public func makeUIView(context: Context) -> VideoPlayerView { - let uiView = VideoPlayerView() - - uiView.playToEndTime = { - if self.model.autoReplay == false { - self.model.play = false - } - DispatchQueue.main.async { - for handler in model.handlers { - if case .onPlayToEndTime(let cb) = handler { - cb() - } - } - } - } - - uiView.contentMode = self.model.contentMode - - uiView.replay = { - DispatchQueue.main.async { - for handler in model.handlers { - if case .onReplay(let cb) = handler { - cb() - } - } - } - } - - uiView.stateDidChanged = { [unowned uiView] _ in - let state: VideoState = uiView.convertState() - - if case .playing = state { - context.coordinator.startObserver(uiView: uiView) - - if let player = uiView.player { - Task { - let has_audio = await video_has_audio(player: player) - let size = await get_video_size(player: player) - Task { @MainActor in - if let size { - self.model.size = size - } - self.model.has_audio = has_audio - } - } - } - - } else { - context.coordinator.stopObserver(uiView: uiView) - } - - DispatchQueue.main.async { - for handler in model.handlers { - if case .onStateChanged(let cb) = handler { - cb(state) - } - } - } - } - - return uiView - } - - public func makeCoordinator() -> Coordinator { - Coordinator(self) - } - - public func updateUIView(_ uiView: VideoPlayerView, context: Context) { - if context.coordinator.observingURL != url { - context.coordinator.clean() - context.coordinator.observingURL = url - } - - if model.play { - uiView.play(for: url) - } else { - uiView.pause(reason: .userInteraction) - } - - uiView.isMuted = model.muted - uiView.isAutoReplay = model.autoReplay - - if let observerTime = context.coordinator.observerTime, let modelTime = model.time, - modelTime != observerTime && modelTime.isValid && modelTime.isNumeric { - uiView.seek(to: modelTime, completion: { _ in }) - } - } - - public static func dismantleUIView(_ uiView: VideoPlayerView, coordinator: VideoPlayer.Coordinator) { - uiView.pause(reason: .hidden) - } - - public class Coordinator: NSObject { - var videoPlayer: VideoPlayer - var observingURL: URL? - var observer: Any? - var observerTime: CMTime? - var observerBuffer: Double? - - init(_ videoPlayer: VideoPlayer) { - self.videoPlayer = videoPlayer - } - - @MainActor - func startObserver(uiView: VideoPlayerView) { - guard observer == nil else { return } - - observer = uiView.addPeriodicTimeObserver(forInterval: .init(seconds: 0.25, preferredTimescale: 60)) { [weak self, unowned uiView] time in - guard let `self` = self else { return } - - Task { @MainActor in - self.videoPlayer.model.time = time - } - self.observerTime = time - - self.updateBuffer(uiView: uiView) - } - } - - func stopObserver(uiView: VideoPlayerView) { - guard let observer = observer else { return } - - uiView.removeTimeObserver(observer) - - self.observer = nil - } - - func clean() { - self.observingURL = nil - self.observer = nil - self.observerTime = nil - self.observerBuffer = nil - } - - @MainActor - func updateBuffer(uiView: VideoPlayerView) { - let bufferProgress = uiView.bufferProgress - guard bufferProgress != observerBuffer else { return } - - for handler in videoPlayer.model.handlers { - if case .onBufferChanged(let cb) = handler { - DispatchQueue.main.async { - cb(bufferProgress) - } - } - } - - observerBuffer = bufferProgress - } - } -} - -private extension VideoPlayerView { - - func convertState() -> VideoState { - switch state { - case .none, .loading: - return .loading - case .playing: - return .playing(totalDuration: totalDuration) - case .paused(let p, let b): - return .paused(playProgress: p, bufferProgress: b) - case .error(let error): - return .error(error) - } - } -} From 54ba64535d717d5f9c48f2147586bdf3b509c626 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Wed, 6 Sep 2023 11:51:44 -0500 Subject: [PATCH 061/111] video: remove GSPlayer dependency Changelog-Fixed: Fixed audio in video playing twice Closes: https://github.com/damus-io/damus/pull/1539 Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index acf8a6806b..24b28c8dda 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -317,7 +317,6 @@ 4CC7AAFA297F64AC00430951 /* EventMenu.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CC7AAF9297F64AC00430951 /* EventMenu.swift */; }; 4CCEB7AE29B53D260078AA28 /* SearchingEventView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CCEB7AD29B53D260078AA28 /* SearchingEventView.swift */; }; 4CCEB7B029B5415A0078AA28 /* SearchingProfileView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CCEB7AF29B5415A0078AA28 /* SearchingProfileView.swift */; }; - 4CCF9AB22A1FE80C00E03CFB /* GSPlayer in Frameworks */ = {isa = PBXBuildFile; productRef = 4CCF9AB12A1FE80C00E03CFB /* GSPlayer */; }; 4CD348EF29C3659D00497EB2 /* ImageUploadModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CD348EE29C3659D00497EB2 /* ImageUploadModel.swift */; }; 4CD7641B28A1641400B6928F /* EndBlock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CD7641A28A1641400B6928F /* EndBlock.swift */; }; 4CDA128A29E9D10C0006FA5A /* SignalView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CDA128929E9D10C0006FA5A /* SignalView.swift */; }; @@ -1127,7 +1126,6 @@ buildActionMask = 2147483647; files = ( 4C06670428FC7EC500038D2A /* Kingfisher in Frameworks */, - 4CCF9AB22A1FE80C00E03CFB /* GSPlayer in Frameworks */, 4C649881286E0EE300EAE2B3 /* secp256k1 in Frameworks */, 4C27C9322A64766F007DBC75 /* MarkdownUI in Frameworks */, ); @@ -2305,7 +2303,6 @@ packageProductDependencies = ( 4C649880286E0EE300EAE2B3 /* secp256k1 */, 4C06670328FC7EC500038D2A /* Kingfisher */, - 4CCF9AB12A1FE80C00E03CFB /* GSPlayer */, 4C27C9312A64766F007DBC75 /* MarkdownUI */, ); productName = damus; @@ -3437,11 +3434,6 @@ package = 4C64987F286E0EE300EAE2B3 /* XCRemoteSwiftPackageReference "secp256k1" */; productName = secp256k1; }; - 4CCF9AB12A1FE80C00E03CFB /* GSPlayer */ = { - isa = XCSwiftPackageProductDependency; - package = 4CCF9AB02A1FE80B00E03CFB /* XCRemoteSwiftPackageReference "GSPlayer" */; - productName = GSPlayer; - }; /* End XCSwiftPackageProductDependency section */ /* Begin XCVersionGroup section */ From 42234b1cf31a53c041b74352e7bf65368246a70c Mon Sep 17 00:00:00 2001 From: William Casarin Date: Tue, 5 Sep 2023 17:05:20 -0700 Subject: [PATCH 062/111] remove timeline render logs --- damus/Views/Timeline/InnerTimelineView.swift | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/damus/Views/Timeline/InnerTimelineView.swift b/damus/Views/Timeline/InnerTimelineView.swift index 1a229262cb..98a986a874 100644 --- a/damus/Views/Timeline/InnerTimelineView.swift +++ b/damus/Views/Timeline/InnerTimelineView.swift @@ -12,15 +12,11 @@ struct InnerTimelineView: View { @ObservedObject var events: EventHolder let state: DamusState let filter: (NostrEvent) -> Bool - - static var count: Int = 0 - + init(events: EventHolder, damus: DamusState, filter: @escaping (NostrEvent) -> Bool) { self.events = events self.state = damus self.filter = filter - print("rendering InnerTimelineView \(InnerTimelineView.count)") - InnerTimelineView.count += 1 } var event_options: EventViewOptions { From 5c87b8e610a87156b96f244d27373446729ea2dc Mon Sep 17 00:00:00 2001 From: petrikaj <52623440+petrikaj@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:35:08 +0300 Subject: [PATCH 063/111] transations: add finnish translation Changelog-Added: Finnish translations Closes: https://github.com/damus-io/damus/pull/1535 Reviewed-by: William Casarin Signed-off-by: William Casarin --- damus/fi.lproj/InfoPlist.strings | Bin 0 -> 1442 bytes damus/fi.lproj/Localizable.strings | 1165 ++++++++++++++++++++++++ damus/fi.lproj/Localizable.stringsdict | 359 ++++++++ 3 files changed, 1524 insertions(+) create mode 100644 damus/fi.lproj/InfoPlist.strings create mode 100644 damus/fi.lproj/Localizable.strings create mode 100644 damus/fi.lproj/Localizable.stringsdict diff --git a/damus/fi.lproj/InfoPlist.strings b/damus/fi.lproj/InfoPlist.strings new file mode 100644 index 0000000000000000000000000000000000000000..2c73ba4ce4d7d82fa396414dc30d233b1e335a71 GIT binary patch literal 1442 zcmb`H-A=+l5QXR3r`T{wR35;1fe;cCB_zfNs1&vQ6haFL@5?iAseWe)Db*@m7{YFM zcjnB@*)#k3(XyrmyR((07TeTDtYS+nvjwZjHmp0}^ZOJ+3w7J3-NUDm-qQR z4T!30Rn?TQ)GHQeQfA6dk^RbdW=9qqyfW4~yva&T1#oo#iR<<|uRAPGvG9nmG%-gL z@2&`SMp;jwy5Dtk;Gzzt#HdF&Upr!9;yJ#0hGOl16tqQ!;ykBDX>o(@wqQ%U(xQ|p zt({_V;N0cd2YbNhg>2OWdR5vW>7_YzoK#(Ma~{7XeigmIF40ag-KJXAGvhD)sdEl* umee)!T`54Sno>{EV-a&gJ!5vI&*J6ytA~Y9uvfeXiYCP>dbM}2o!u{eE(8Gp literal 0 HcmV?d00001 diff --git a/damus/fi.lproj/Localizable.strings b/damus/fi.lproj/Localizable.strings new file mode 100644 index 0000000000..4cd1373602 --- /dev/null +++ b/damus/fi.lproj/Localizable.strings @@ -0,0 +1,1165 @@ +/* Notification that a user reacted to a note that the current user was tagged in */ +"reacted_tagged_in_1" = "%@ reagoi viestiin, johon sinut oli merkitty"; + +/* Notification that 2 users reacted to a note that the current user was tagged in */ +"reacted_tagged_in_2" = "%@ ja %@ reagoivat viestiin, johon sinut oli merkitty"; + +/* Notification that a user reposted a note that the current user was tagged in */ +"reposted_tagged_in_1" = "%@ jakoi uudelleen viestin, johon sinut oli merkitty"; + +/* Notification that 2 users reposted a note that the current user was tagged in */ +"reposted_tagged_in_2" = "%@ ja %@ jakoivat uudelleen viestin, johon sinut oli merkitty"; + +/* Notification that a user zapped a note that the current user was tagged in */ +"zapped_tagged_in_1" = "%@ zappasi viestiä, johon sinut oli merkitty"; + +/* Notification that 2 users zapped a note that the current user was tagged in */ +"zapped_tagged_in_2" = "%@ ja %@ zappasivat viestin, johon sinut oli merkitty"; + +/* Notification that a user reacted to the current user's note */ +"reacted_your_note_1" = "%@ reagoi viestiisi"; + +/* Notification that 2 users reacted to the current user's profile */ +"reacted_your_note_2" = "%@ ja %@ reagoivat viestiisi"; + +/* Notification that a user reposted the current user's note */ +"reposted_your_note_1" = "%@ jakoi viestisi uudelleen"; + +/* Notification that 2 users reposted the current user's note */ +"reposted_your_note_2" = "%@ ja %@ jakoivat viestisi uudelleen"; + +/* Notification that a user zapped the current user's note */ +"zapped_your_note_1" = "%@ zappasi viestisi"; + +/* Notification that 2 users zapped the current user's note */ +"zapped_your_note_2" = "%@ ja %@ zappasivat viestisi"; + +/* Notification that a user reacted to the current user's profile */ +"reacted_your_profile_1" = "%@ reagoi profiiliisi"; + +/* Notification that 2 users reacted to the current user's profile */ +"reacted_your_profile_2" = "%@ ja %@ reagoivat profiiliisi"; + +/* Notification that a user reposted the current user's profile */ +"reposted_your_profile_1" = "%@ jakoi profiilisi uudelleen"; + +/* Notification that 2 users reposted the current user's profile */ +"reposted_your_profile_2" = "%@ ja %@ jakoivat profiilisi uudelleen"; + +/* Notification that a user zapped the current user's profile */ +"zapped_your_profile_1" = "%@ zappasi profiiliasi"; + +/* Notification that 2 users zapped the current user's profile */ +"zapped_your_profile_2" = "%@ ja %@ zappasivat profiiliasi"; +/* Sentence composed of 2 variables to describe how many imports were performed from loading a NostrScript. In source English, the first variable is the number of imports, and the second variable is 'Import' or 'Imports'. + Sentence composed of 2 variables to describe how many people are following a user. In source English, the first variable is the number of followers, and the second variable is 'Follower' or 'Followers'. + Sentence composed of 2 variables to describe how many profiles a user is following. In source English, the first variable is the number of profiles being followed, and the second variable is 'Following'. + Sentence composed of 2 variables to describe how many reactions there are on a post. In source English, the first variable is the number of reactions, and the second variable is 'Reaction' or 'Reactions'. + Sentence composed of 2 variables to describe how many relay servers a user is connected. In source English, the first variable is the number of relay servers, and the second variable is 'Relay' or 'Relays'. + Sentence composed of 2 variables to describe how many reposts. In source English, the first variable is the number of reposts, and the second variable is 'Repost' or 'Reposts'. + Sentence composed of 2 variables to describe how many zap payments there are on a post. In source English, the first variable is the number of zap payments, and the second variable is 'Zap' or 'Zaps'. */ +"%@ %@" = "%1$@%2$@"; + +/* Alert message that informs a user was muted. */ +"%@ has been muted" = "%@ on mykistetty"; + +/* When a note or profile is not found when searching for it via its note id */ +"%@ not found" = "%@ ei löytynyt"; + +/* Reacted by heading in local notification */ +"%@ reacted with %@" = "%1$@ reagoi merkinnällä %2$@"; + +/* Percentage of additional zap that should be sent to support Damus development. */ +"%@%" = "%1$@%2$"; + +/* Explanation of what is done to keep personally identifiable information private. There is a heading that precedes this explanation which is a variable to this string. */ +"%@. Creating an account doesn't require a phone number, email or name. Get started right away with zero friction." = "%@. Tilin luominen ei vaadi puhelinnumeroa, sähköpostia tai nimeä. Aloita heti kitkattomasti."; + +/* Explanation of what is done to keep private data encrypted. There is a heading that precedes this explanation which is a variable to this string. */ +"%@. End-to-End encrypted private messaging. Keep Big Tech out of your DMs" = "%@. Päästä päähän salatut yksityiset viestit. Pidä suuryritykset poissa viesteistäsi."; + +/* Explanation of what can be done by users to earn money. There is a heading that precedes this explanation which is a variable to this string. */ +"%@. Tip your friends and stack sats with Bitcoin⚡️, the native currency of the internet." = "%@. Tippaa ystäviäsi ja kasaa satseja Bitcoineilla, Internetin omalla valuutalla."; + +/* Fraction of how many of the user's relay servers that are operational. */ +"%@/%@" = "%1$@/%2$@"; + +/* Description of why the Nostr address is invalid. */ +"'%@' is an invalid Nostr address. It should look like an email address." = "'%@' on virheellinen Nostr-osoite. Sen pitäisi näyttää sähköpostiosoitteelta."; + +/* No comment provided by engineer. */ +"@" = "@"; + +/* Label to prompt for about text entry for user to describe about themself. */ +"About" = "Tietoja"; + +/* Label for About Me section of user profile form. */ +"About Me" = "Tietoja minusta"; + +/* Placeholder text for About Me description. */ +"Absolute Boss" = "Täyspomo"; + +/* Button to accept the end user license agreement before being allowed into the app. */ +"Accept" = "Hyväksy"; + +/* Section header for accessibility settings */ +"Accessibility" = "Saavutettavuus"; + +/* Title for confirmation dialog to either share, report, or mute a profile. */ +"Actions" = "Toiminnot"; + +/* Button to confirm adding user inputted relay. */ +"Add" = "Lisää"; + +/* Button label to re-add all original participants as profiles to reply to in a note */ +"Add all" = "Lisää kaikki"; + +/* Context menu option for adding a note bookmark. */ +"Add bookmark" = "Lisää kirjanmerkki"; + +/* Button text to add bookmark to a note. */ +"Add Bookmark" = "Lisää kirjanmerkki"; + +/* Header text to prompt user to optionally provide additional information when reporting a user or note. */ +"Additional information" = "Lisätietoja"; + +/* Label to display relay contact user. */ +"Admin" = "Ylläpitäjä"; + +/* Label for filter for all notifications. */ +"All" = "Kaikki"; + +/* Ask the user if they already have an account on Nostr */ +"Already on Nostr?" = "Onko sinulla jo tili Nostrissa?"; + +/* Setting to always show and never blur images */ +"Always show images" = "Näytä kuvat aina"; + +/* Text indicating that they can contribute zaps to support Damus development. */ +"An additional percentage of each zap will be sent to support Damus development" = "Lisäprosentti jokaisesta zapista lähetetään Damuksen kehittämisen tukemiseksi"; + +/* Toggle to enable or disable image animation */ +"Animations" = "Animaatiot"; + +/* Button text to indicate that the zap type is a anonymous zap. + Picker option to indicate that a zap should be sent anonymously and not identify the user as who sent it. + Placeholder display name of anonymous user. */ +"Anonymous" = "Anonyymi"; + +/* Any amount of sats */ +"Any" = "Mikä tahansa"; + +/* Prompt for optional entry of API Key to use translation server. */ +"API Key (optional)" = "API-avain (valinnainen)"; + +/* Prompt for required entry of API Key to use translation server. */ +"API Key (required)" = "API-avain (pakollinen)"; + +/* Navigation title for text and appearance settings. + Section header for text and appearance settings */ +"Appearance" = "Ulkoasu"; + +/* Text asking the user if they are lost in the app. */ +"Are you lost?" = "Oletko eksyksissä?"; + +/* Prompt to ask user if they want to attach their Nostr Wallet Connect lightning wallet. */ +"Are you sure you want to attach this wallet?" = "Haluatko varmasti liittää tämän lompakon?"; + +/* Alert for deleting all of the bookmarks. */ +"Are you sure you want to delete all of your bookmarks?" = "Haluatko varmasti poistaa kaikki kirjanmerkkisi?"; + +/* Alert message asking if the user wants to upload an image. */ +"Are you sure you want to upload this image?" = "Haluatko varmasti ladata tämän kuvan?"; + +/* Alert message asking if the user wants to upload media. */ +"Are you sure you want to upload this media?" = "Haluatko varmasti ladata tämän median?"; + +/* Text for button to attach Nostr Wallet Connect lightning wallet. */ +"Attach" = "Liitä"; + +/* Navigation title for attaching Nostr Wallet Connect lightning wallet. */ +"Attach a Wallet" = "Liitä lompakko"; + +/* Button to attach an Alby Wallet, a service that provides a Lightning wallet for zapping sats. Alby is the name of the service and should not be translated. */ +"Attach Alby Wallet" = "Liitä Alby-lompakko"; + +/* Text for button to attach Nostr Wallet Connect lightning wallet. */ +"Attach Wallet" = "Liitä lompakko"; + +/* Toggle to automatically translate notes. */ +"Automatically translate notes" = "Käännä viestit automaattisesti"; + +/* Label for Banner Image section of user profile form. */ +"Banner Image" = "Bannerikuva"; + +/* Reminder to user that they should save their account information. */ +"Before we get started, you'll need to save your account info, otherwise you won't be able to login in the future if you ever uninstall Damus." = "Ennen aloittamista sinun on tallennettava tilisi tiedot, muuten et pysty kirjautumaan sisään tulevaisuudessa, jos poistat Damus-sovelluksen."; + +/* Label for Bitcoin Lightning Tips section of user profile form. */ +"Bitcoin Lightning Tips" = "Bitcoin Lightning tipit"; + +/* Sidebar menu label for Bookmarks view. + Title of bookmarks view */ +"Bookmarks" = "Kirjanmerkit"; + +/* Button to broadcast note to all your relays + Context menu option for broadcasting the user's note to all of the user's connected relay servers. */ +"Broadcast" = "Lähetä"; + +/* Alert button to cancel out of alert for muting a user. + Button to cancel a repost. + Button to cancel out of alert that creates a new mutelist. + Button to cancel out of posting a note. + Button to cancel out of view adding user inputted relay. + Button to cancel the upload. + Cancel deleting bookmarks. + Cancel deleting the user. + Cancel out of logging out the user. + Cancel out of search view. + Text for button to cancel out of connecting Nostr Wallet Connect lightning ewallet. */ +"Cancel" = "Peruuta"; + +/* Option to select photo from library */ +"Choose from Library" = "Valitse kirjastosta"; + +/* Button for clearing bookmarks data. */ +"Clear All" = "Tyhjennä kaikki"; + +/* Button to clear image cache. */ +"Clear Cache" = "Tyhjennä välimuisti"; + +/* Button to connect to recommended relay server. */ +"Connect" = "Yhdistä"; + +/* Button to connect to the relay. + Label for section for adding a relay server. */ +"Connect To Relay" = "Yhdistä välittäjään"; + +/* Section title for relay servers that are connected. */ +"Connected Relays" = "Yhdistetyt välittäjät"; + +/* Label to display relay contact information. */ +"Contact" = "Yhteystiedot"; + +/* Continue with bookmarks. + Continue with deleting the user. */ +"Continue" = "Jatka"; + +/* Label indicating that a user's key was copied. */ +"Copied" = "Kopioitu"; + +/* Button to copy a relay server address. */ +"Copy" = "Kopioi"; + +/* Context menu option for copying the ID of the account that created the note. */ +"Copy Account ID" = "Kopioi tilin tunniste"; + +/* Context menu option to copy an image into clipboard. */ +"Copy Image" = "Kopioi kuva"; + +/* Context menu option to copy the URL of an image into clipboard. */ +"Copy Image URL" = "Kopioi kuvan URL"; + +/* Title of section for copying a Lightning invoice identifier. */ +"Copy invoice" = "Kopioi lasku"; + +/* Button to copy link to note */ +"Copy Link" = "Kopioi linkki"; + +/* Context menu option for copying a user's Lightning URL. */ +"Copy LNURL" = "Kopioi LNURL"; + +/* Context menu option for copying the ID of the note. */ +"Copy note ID" = "Kopioi viestin tunniste"; + +/* Context menu option for copying the JSON text from the note. */ +"Copy note JSON" = "Kopioi viestin JSON"; + +/* Button to copy report ID. */ +"Copy Report ID" = "Kopioi ilmoituksen tunniste"; + +/* Context menu option for copying the text from an note. */ +"Copy text" = "Kopioi teksti"; + +/* Label for button in context menu to copy URL of the selected uploaded media asset. */ +"Copy URL" = "Kopioi URL-osoite"; + +/* Context menu option for copying the ID of the user who created the note. */ +"Copy user public key" = "Kopioi käyttäjän julkinen avain"; + +/* Indicates that there are no users found. */ +"Could not find the user you're looking for" = "Etsimääsi käyttäjää ei löytynyt"; + +/* Alert message to indicate that the muted user could not be found. */ +"Could not find user to mute..." = "Mykistettävää käyttäjää ei löytynyt..."; + +/* Button to navigate to create account view. */ +"Create account" = "Luo tili"; + +/* Button to create account. */ +"Create account now" = "Luo tili nyt"; + +/* Title of alert prompting the user to create a new mutelist. */ +"Create new mutelist" = "Luo uusi mykistyslista"; + +/* Example description about Bitcoin creator(s), Satoshi Nakamoto. */ +"Creator(s) of Bitcoin. Absolute legend." = "Bitcoinin luoja(t). Absoluuttinen legenda."; + +/* Dropdown option for selecting a custom translation server. */ +"Custom" = "Mukautettu"; + +/* Dropdown option for selecting DeepL as the translation service. */ +"DeepL (Proprietary, Higher Accuracy)" = "DeepL (kaupallinen, tarkempi)"; + +/* Button to pay a Lightning invoice with the user's default Lightning wallet. */ +"Default Wallet" = "Oletuslompakko"; + +/* Title for section in zap settings that controls the default zap amount in sats. */ +"Default Zap Amount in sats" = "Oletus zap-määrä satseissa"; + +/* Button for deleting the users account. + Button to remove a user from their mutelist. */ +"Delete" = "Poista"; + +/* Button to delete the user's account. */ +"Delete Account" = "Poista tili"; + +/* Label to display relay description. */ +"Description" = "Kuvaus"; + +/* Navigation title for developer settings + Section header for developer settings */ +"Developer" = "Kehittäjä"; + +/* Setting to enable developer mode */ +"Developer Mode" = "Kehittäjätila"; + +/* Section header for Developer Settings view */ +"Developer Mode enables features and options that may help developers diagnose issues and improve this app. Most users will not need Developer Mode." = "Kehittäjätila ottaa käyttöön ominaisuuksia ja vaihtoehtoja, jotka voivat auttaa kehittäjiä diagnosoimaan ongelmia ja parantamaan tätä sovellusta. Useimmat käyttäjät eivät tarvitse kehittäjätilaa."; + +/* Button to disconnect from a relay server. */ +"Disconnect" = "Katkaise yhteys"; + +/* Button to disconnect from the relay. */ +"Disconnect From Relay" = "Katkaise yhteys välittäjään"; + +/* Text for button to disconnect from Nostr Wallet Connect lightning wallet. */ +"Disconnect Wallet" = "Katkaise lompakon yhteys"; + +/* Label to prompt display name entry. */ +"Display name" = "Näyttönimi"; + +/* Navigation title for DMs view, where DM is the English abbreviation for Direct Message. + Navigation title for view of DMs, where DM is an English abbreviation for Direct Message. + Picker option for DM selector for seeing only DMs that have been responded to. DM is the English abbreviation for Direct Message. + Setting to enable DM Local Notification + Toolbar label for DMs view, where DM is the English abbreviation for Direct Message. */ +"DMs" = "Yksityisviestit"; + +/* Button that, when tapped, will finish adding a different user's relays to your relay by hiding the + buttons next to the relays. + Button to dismiss wallet selection view for paying Lightning invoice. */ +"Done" = "Valmis"; + +/* Heading indicating that this application allows users to earn money. */ +"Earn Money" = "Ansaitse rahaa"; + +/* Button to edit user's profile. */ +"Edit" = "Muokkaa"; + +/* Heading indicating that this application keeps private messaging end-to-end encrypted. */ +"Encrypted" = "Salattu"; + +/* Prompt for user to enter an account key to login. */ +"Enter your account key" = "Syötä tilisi avain"; + +/* Message to display when there was an error fetching a lightning invoice while attempting to zap. */ +"Error fetching lightning invoice" = "Virhe noutaessa lightning-laskua"; + +/* Error message indicating why saving keys failed. */ +"Error: %@" = "Virhe: %@"; + +/* Navigation title of view that shows the EULA, an acronym for End User License Agreement. */ +"EULA" = "Käyttöehdot"; + +/* Description of public zap type where the zap is sent publicly and identifies the user who sent it. */ +"Everyone will see that you zapped" = "Kaikki näkevät, että zappasit"; + +/* Indication that the execution of running a NostrScript finished successfully, while providing a numeric return code. */ +"Executed successfully, returned with code %@" = "Suoritettu onnistuneesti, palautti koodin %@"; + +/* NostrScript error message when it fails to initialize a module. */ +"Failed to initialize" = "Alustus epäonnistui"; + +/* NostrScript error message when it fails to parse a script. */ +"Failed to parse" = "Jäsentäminen epäonnistui"; + +/* Button label text for filtering relay servers. */ +"Filter" = "Suodatin"; + +/* Button to follow a user. */ +"Follow" = "Seuraa"; + +/* Button to follow a user back. */ +"Follow Back" = "Seuraa takaisin"; + +/* Button to follow a given hashtag. */ +"Follow hashtag" = "Seuraa aihetunnistetta"; + +/* Text on QR code view to prompt viewer looking at screen to follow the user. */ +"Follow me on Nostr" = "Seuraa minua Nostrissa"; + +/* Text to indicate that the user is followed by one of our follows. */ +"Followed by %@" = "Seuraaja: %@"; + +/* Text to indicate that the user is followed by two of our follows. */ +"Followed by %@ & %@" = "Seuraajat: %1$@ & %2$@"; + +/* Text to indicate that the user is followed by three of our follows. */ +"Followed by %@, %@ & %@" = "Seuraajat: %1$@, %2$@ & %3$@"; + +/* Label describing followers of a user. + Navigation bar title for view that shows who is following a user. */ +"Followers" = "Seuraajat"; + +/* Navigation bar title for view that shows who is following a user. */ +"Followers You Know" = "Tuntemasi seuraajat"; + +/* Navigation bar title for view that shows who a user is following. */ +"Following" = "Seurattavat"; + +/* Label to indicate that the user is in the process of following another user. */ +"Following..." = "Seurataan..."; + +/* Text to indicate that a user is following your profile. */ +"Follows you" = "Seuraa sinua"; + +/* Dropdown option for selecting Free plan for DeepL translation service. */ +"Free" = "Ilmainen"; + +/* Button to navigate to DeepL website to get a translation API key. */ +"Get API Key" = "Hanki API-avain"; + +/* Button to navigate to nokyctranslate website to get a translation API key. */ +"Get API Key with BTC/Lightning" = "Hanki API-avain BTC:llä/Lightningilla"; + +/* Text indicating the goal of developing Damus which the user can help with. */ +"Help build the future of decentralized communication on the web." = "Auta rakentamaan hajautetun viestinnän tulevaisuutta verkossa."; + +/* Button to hide a note from a user who has been muted. */ +"Hide" = "Piilota"; + +/* Section footer describing OnlyZaps mode */ +"Hide all 🤙's" = "Piilota kaikki 🤙:t"; + +/* Navigation bar title for Home view where notes and replies appear from those who the user is following. */ +"Home" = "Etusivu"; + +/* Placeholder example text for profile picture URL. */ +"https://example.com/pic.jpg" = "https://esimerkki.com/kuva.jpg"; + +/* Placeholder example text for website URL for user profile. */ +"https://jb55.com" = "https://jb55.com"; + +/* Description of report type for illegal content. */ +"Illegal Content" = "Laiton sisältö"; + +/* Prompt selection of user's image uploader */ +"Image uploader" = "Kuvan lataaja"; + +/* Section title for images configuration. */ +"Images" = "Kuvat"; + +/* Description of report type for impersonation. */ +"Impersonation" = "Henkilönä esiintyminen"; + +/* Error message indicating that an invalid account key was entered for login. */ +"Invalid key" = "Virheellinen avain"; + +/* Message to display when there was an error attempting to zap due to an invalid lightning address. */ +"Invalid lightning address" = "Virheellinen lightning-osoite"; + +/* Error message when an invalid Nostr wallet connection string is provided. */ +"Invalid Nostr wallet connection string" = "Virheellinen Nostr-lompakon yhteysmerkkijono"; + +/* Title of alerting as invalid tip address. */ +"Invalid Tip Address" = "Virheellinen tippiosoite"; + +/* Placeholder example text for identifier used for Nostr addresses. */ +"jb55@jb55.com" = "jb55@jb55.com"; + +/* Navigation title for managing keys. + Settings section for managing keys */ +"Keys" = "Avaimet"; + +/* Button that opens up a webpage where the user can learn more about Nostr. */ +"Learn more about Nostr" = "Lue lisää Nostrista"; + +/* Moves the post button to the left side of the screen */ +"Left Handed" = "Vasenkätinen"; + +/* Button to continue to login page. */ +"Let's get started!" = "Aloitetaan!"; + +/* Button to complete account creation and start using the app. */ +"Let's go!" = "Mennään!"; + +/* Dropdown option for selecting LibreTranslate as the translation service. */ +"LibreTranslate (Open Source)" = "LibreTranslate (Avoin lähdekoodi)"; + +/* Placeholder text for entry of Lightning Address or LNURL. */ +"Lightning Address or LNURL" = "Lightning-osoite tai LNURL"; + +/* Indicates that the view is for paying a Lightning invoice. */ +"Lightning Invoice" = "Lightning-lasku"; + +/* Accessibility Label for Like button */ +"Like" = "Tykkää"; + +/* Setting to enable Like Local Notification */ +"Likes" = "Tykkäykset"; + +/* Face ID usage description shown when trying to access private key */ +"Local authentication to access private key" = "Paikallinen tunnistautuminen yksityisen avaimen käyttöön"; + +/* Dropdown option label for system default for Lightning wallet. */ +"Local default" = "Paikallinen oletus"; + +/* Section header for damus local notifications user configuration */ +"Local Notifications" = "Paikalliset ilmoitukset"; + +/* Label to display developer mode logs. */ +"Log" = "Loki"; + +/* Button to log into account. + Button to navigate to login view. */ +"Login" = "Kirjaudu sisään"; + +/* Alert for logging out the user. + Button for logging out the user. */ +"Logout" = "Kirjaudu ulos"; + +/* Label that appears when searching for note or profile */ +"Looking for %@..." = "Etsitään %@..."; + +/* Button label to indicate that tapping it will make the selected zap type be the default for future zaps. */ +"Make Default" = "Aseta oletukseksi"; + +/* Reminder message in alert to get customer to verify that their private security account key is saved saved before logging out. */ +"Make sure your nsec account key is saved before you logout or you will lose access to this account" = "Varmista, että nsec-tilin avain on tallennettu ennen uloskirjautumista, muuten menetät pääsyn tähän tiliin"; + +/* Mentioned by heading in local notification */ +"Mentioned by %@" = "Maininta käyttäjältä %@"; + +/* Label for filter for seeing mention notifications (replies, etc). + Setting to enable Mention Local Notification */ +"Mentions" = "Maininnat"; + +/* Alert button to mute a user. + Button to mute a profile. */ +"Mute" = "Mykistä"; + +/* Alert message prompt to ask if a user should be muted. */ +"Mute %@?" = "Mykistä %@?"; + +/* Context menu option for muting a conversation. */ +"Mute conversation" = "Mykistä keskustelu"; + +/* Context menu option for muting users. */ +"Mute user" = "Mykistä käyttäjä"; + +/* Title of alert for muting a user. */ +"Mute User" = "Mykistä käyttäjä"; + +/* Sidebar menu label for muted users view. */ +"Muted" = "Mykistetyt"; + +/* Navigation title of view to see list of muted users. */ +"Muted Users" = "Mykistetyt käyttäjät"; + +/* Notification that the user has received a new direct message */ +"New encrypted direct message" = "Uusi salattu yksityisviesti"; + +/* Ask the user if they are new to Nostr */ +"New to Nostr?" = "Uusi Nostrissa?"; + +/* Button to cancel out of posting a note after being alerted that it looks like they might be posting a private key. */ +"No" = "Ei"; + +/* Text indicating that there is no data available to show for specific metadata about a relay server. */ +"No data available" = "Ei saatavilla olevia tietoja"; + +/* Alert message prompt that asks if the user wants to create a new mute list, overwriting previous mute lists. */ +"No mute list found, create a new one? This will overwrite any previous mute lists." = "Mykistyslistaa ei löytynyt, luodaanko uusi? Tämä korvaa aiemmat mykistyslistat."; + +/* Description of anonymous zap type where the zap is sent anonymously and does not identify the user who sent it. */ +"No one will see that you zapped" = "Kukaan ei näe, että zappasit"; + +/* Description of non-zap type where sats are sent to the user's wallet as a regular Lightning payment, not as a zap. */ +"No zaps will be sent, only a lightning payment." = "Zappeja ei lähetetä, vain lightning-maksu."; + +/* Dropdown option for selecting NoKYCTranslate.com as the translation service. */ +"NoKYCTranslate.com (Prepay with BTC)" = "NoKYCTranslate.com (Ennakkomaksu BTC:llä)"; + +/* Button text to indicate that the zap type is a private zap. + Picker option to indicate that sats should be sent to the user's wallet as a regular Lightning payment, not as a zap. */ +"None" = "Ei mitään"; + +/* No search results. */ +"none" = "ei mitään"; + +/* Dropdown option for selecting no translation service. */ +"none_translation_service" = "Ei mitään"; + +/* Label for the Nostr Address section of user profile form. */ +"Nostr Address" = "Nostr-osoite"; + +/* Description about what is Nostr. */ +"Nostr is a protocol, designed for simplicity, that aims to create a censorship-resistant global social network" = "Nostr on yksinkertaiseksi suunniteltu protokolla, jonka tavoitteena on luoda sensuurinkestävä maailmanlaajuinen sosiaalinen verkosto"; + +/* Navigation title for the view showing NostrScript. */ +"NostrScript" = "NostrScript"; + +/* Text indicating that there was an error with loading NostrScript. There is a more descriptive error message shown separately underneath. */ +"NostrScript Error" = "NostrScript-virhe"; + +/* Alert user that they might be attempting to paste a private key and ask them to confirm. */ +"Note contains \"nsec1\" private key. Are you sure?" = "Viesti sisältää \"nsec1\" yksityisavaimen. Oletko varma?"; + +/* Text to indicate that what is being shown is a note from a user who has been muted. */ +"Note from a user you've muted" = "Viesti käyttäjältä, jonka olet mykistänyt"; + +/* Label for filter for seeing only notes (instead of notes and replies). + Label for filter for seeing only your notes (instead of notes and replies). */ +"Notes" = "Viestit"; + +/* Label for filter for seeing notes and replies (instead of only notes). + Label for filter for seeing your notes and replies (instead of only your notes). */ +"Notes & Replies" = "Viestit & Vastaukset"; + +/* Indicates that there are no notes in the timeline to view. */ +"Nothing to see here. Check back later!" = "Täällä ei ole mitään nähtävää. Tarkista myöhemmin uudelleen!"; + +/* Section header for notification indicator dot settings */ +"Notification Dots" = "Ilmoituspisteet"; + +/* Section header for Notification Preferences */ +"Notification Preference" = "Ilmoitusasetukset"; + +/* Section header for Damus notifications + Toolbar label for Notifications view. */ +"Notifications" = "Ilmoitukset"; + +/* String indicating that a given timestamp just occurred */ +"now" = "nyt"; + +/* Prompt for user to enter in an account key to login. This text shows the characters the key could start with if it was a private key. */ +"nsec1..." = "nsec1..."; + +/* Description of report type for nudity. */ +"Nudity" = "Alastomuus"; + +/* Button to dismiss the alert. */ +"Ok" = "OK"; + +/* Help text on green lock icon that explains that only the current user can see the message of a zap event and who sent the zap. */ +"Only you can see this message and who sent it." = "Vain sinä voit nähdä tämän viestin ja kuka sen lähetti."; + +/* Section header for enabling OnlyZaps mode (hide reactions) */ +"OnlyZaps" = "VainZapit"; + +/* Non-tappable text in context menu that shows up when the zap button on profile is long pressed to indicate that the user has enabled OnlyZaps, meaning that they would like to be only zapped and not accept reactions to their notes. */ +"OnlyZaps Enabled" = "VainZapit käytössä"; + +/* Setting toggle to hide reactions. */ +"OnlyZaps mode" = "VainZapit -tila"; + +/* Prompt for user to enter optional additional information when reporting an account or content. */ +"Optional" = "Valinnainen"; + +/* Label indicating that a form input is optional. */ +"optional" = "valinnainen"; + +/* Section header that indicates the relay server requires payment. */ +"Paid Relay" = "Maksullinen välittäjä"; + +/* Button to paste a Nostr Wallet Connect string to connect the wallet for use in Damus for zaps. */ +"Paste" = "Liitä"; + +/* Button to pay a Lightning invoice. */ +"Pay" = "Maksa"; + +/* Navigation bar title for view to pay Lightning invoice. */ +"Pay the Lightning invoice" = "Maksa lightning-lasku"; + +/* Alert for deleting the users account. + Section title for deleting the user */ +"Permanently Delete Account" = "Poista tili pysyvästi"; + +/* Prompt selection of DeepL subscription plan to perform machine translations on notes */ +"Plan" = "Vaihtoehto"; + +/* Instructions on how to filter a specific timeline feed by choosing relay servers to filter on. */ +"Please choose relays from the list below to filter the current feed:" = "Valitse alla olevasta listasta välittäjät, joilla haluat suodattaa nykyistä syötettä:"; + +/* Button to post a note. */ +"Post" = "Lähetä"; + +/* Button text to indicate that the zap type is a private zap. + Heading indicating that this application keeps personally identifiable information private. A sentence describing what is done to keep data private comes after this heading. + Picker option to indicate that a zap should be sent privately and not identify the user to the public. */ +"Private" = "Yksityinen"; + +/* Label to indicate that the text below is the user's private key used by only the user themself as a secret to login to access their account. + Title of the secure field that holds the user's private key. */ +"Private Key" = "Yksityisavain"; + +/* Title of notification when a private zap is received. */ +"Private Zap" = "Yksityinen zap"; + +/* Description of private zap type where the zap is sent privately and does not identify the user to the public. */ +"private_zap_description" = "Vain '%@' näkee, että olet zappannut häntä"; + +/* Dropdown option for selecting Pro plan for DeepL translation service. */ +"Pro" = "Pro"; + +/* Description of report type for profanity. */ +"Profanity" = "Hävyttömyydet"; + +/* Sidebar menu label for Profile view. */ +"Profile" = "Profiili"; + +/* Label for Profile Picture section of user profile form. */ +"Profile Picture" = "Profiilikuva"; + +/* Button text to indicate that the zap type is a public zap. + Picker option to indicate that a zap should be sent publicly and identify the user as who sent it. */ +"Public" = "Julkinen"; + +/* Section title for the user's public account ID. */ +"Public Account ID" = "Julkinen tilin tunniste"; + +/* Label to indicate that text below is the user's public key used by others to uniquely refer to the user. + Label to indicate the public key of the account. */ +"Public Key" = "Julkinen avain"; + +/* Label indicating that the text is a user's public account key. */ +"Public key" = "Julkinen avain"; + +/* Button to view profile's qr code. */ +"QR Code" = "QR-koodi"; + +/* Button to compose a quoted note */ +"Quote" = "Lainaa"; + +/* Indication that a NostrScript was run until it reached a suspended state. */ +"Ran to suspension." = "Ajettiin keskeytykseen asti."; + +/* Navigation bar title for Reactions view. */ +"Reactions" = "Reaktiot"; + +/* Section title for recommend relay servers that could be added as part of configuration */ +"Recommended Relays" = "Suositellut välittäjät"; + +/* Button to reject the end user license agreement, which disallows the user from being let into the app. */ +"Reject" = "Hylkää"; + +/* Label to display relay address. */ +"Relay" = "Välitin"; + +/* Navigation bar title that shows the list of relays for a user. + Sidebar menu label for Relays view. + Title of relays view */ +"Relays" = "Välittäjät"; + +/* Description of what was done as a result of sending a report to relay servers. */ +"Relays have been notified and clients will be able to use this information to filter content. Thank you!" = "Välittäjät on ilmoitettu ja asiakkaat voivat käyttää tätä tietoa sisällön suodattamiseen. Kiitos!"; + +/* Button label to remove all participants from a note reply. */ +"Remove all" = "Poista kaikki"; + +/* Button text to remove bookmark from a note. */ +"Remove Bookmark" = "Poista kirjanmerkki"; + +/* Context menu option for removing a note bookmark. */ +"Remove bookmark" = "Poista kirjanmerkki"; + +/* Accessibility label for reply button */ +"Reply" = "Vastaa"; + +/* Text indicating that the view is used for editing which participants are replied to in a note. */ +"Replying to" = "Vastaanottajat"; + +/* Indicating that the user is replying to the following listed people. + Indicating that the user is replying to the themself and no one else, where the parameter is 'self' in US English. + Label to indicate that the user is replying to 1 user. */ +"Replying to %@" = "Vastaa käyttäjälle %@"; + +/* Label to indicate that the user is replying to 2 users. */ +"Replying to %@ & %@" = "Vastaa käyttäjille %1$@ & %2$@"; + +/* Label to indicate that the user is replying to themself. */ +"Replying to self" = "Vastaan itselleni"; + +/* Button to report a profile. + Context menu option for reporting content. + Label indicating that the current view is for the user to report content. */ +"Report" = "Ilmoita"; + +/* Label indicating that the text underneath is the identifier of the report that was sent to relay servers. */ +"Report ID:" = "Ilmoituksen tunniste:"; + +/* Button to report a note. */ +"Report Note" = "Ilmoita vististä"; + +/* Message indicating that a report was successfully sent to relay servers. */ +"Report sent!" = "Ilmoitus lähetetty!"; + +/* Button to report a user. */ +"Report User" = "Ilmoita käyttäjästä"; + +/* Button to repost a note */ +"Repost" = "Jaa uudelleen"; + +/* Text indicating that the note was reposted (i.e. re-shared). */ +"Reposted" = "Jaettu uudelleen"; + +/* Reposted by heading in local notification */ +"Reposted by %@" = "Jaettu uudelleen käyttäjän %@ toimesta"; + +/* Accessibility label for boosts button + Navigation bar title for Reposts view. + Setting to enable Repost Local Notification */ +"Reposts" = "Uudelleenjakamiset"; + +/* Picker option for DM selector for seeing only message requests (DMs that someone else sent the user which has not been responded to yet). DM is the English abbreviation for Direct Message. */ +"Requests" = "Pyynnöt"; + +/* Button to retry completing account creation after an error occurred. */ +"Retry" = "Yritä uudelleen"; + +/* Button that runs a NostrScript. */ +"Run" = "Suorita"; + +/* Indication that the execution of a NostrScript is running. */ +"Running..." = "Suoritetaan..."; + +/* Indication that a runtime error occurred when running a NostrScript. */ +"Runtime error" = "Suoritusaikainen virhe"; + +/* Name of Bitcoin creator(s). */ +"Satoshi Nakamoto" = "Satoshi Nakamoto"; + +/* Button for saving profile. */ +"Save" = "Tallenna"; + +/* Context menu option to save an image. */ +"Save Image" = "Tallenna kuva"; + +/* Text to prompt scanning a QR code of a user's pubkey to open their profile. */ +"Scan a user's pubkey" = "Skannaa käyttäjän julkinen avain"; + +/* Button to switch to scan QR Code page. */ +"Scan Code" = "Skannaa koodi"; + +/* Text on QR code view to prompt viewer looking at screen to scan the QR code on screen with their device camera. */ +"Scan the code" = "Skannaa koodi"; + +/* Navigation link to search hashtag. */ +"Search hashtag: #%@" = "Etsi aihetunniste: #%@"; + +/* Placeholder text to prompt entry of search query. */ +"Search..." = "Haku..."; + +/* Navigation title for universe/search settings. + Section header for search/universe settings */ +"Search/Universe" = "Haku/Kaikki"; + +/* Section title for user's secret account login key. */ +"Secret Account Login Key" = "Salainen kirjautumisavain"; + +/* Title of section for selecting a Lightning wallet to pay a Lightning invoice. */ +"Select a Lightning wallet" = "Valitse salamalompakko"; + +/* Prompt selection of user's default wallet */ +"Select default wallet" = "Valitse oletuslompakko"; + +/* Part of a larger sentence 'Replying to self' in US English. 'self' indicates that the user is replying to themself and no one else. */ +"self" = "itselleni"; + +/* Text prompt for user to send a message to the other user. */ +"Send a message to start the conversation..." = "Lähetä viesti aloittaaksesi keskustelun..."; + +/* Placeholder text for a comment to send as part of a zap to the user. */ +"Send a message with your zap..." = "Lähetä viesti zapin mukana..."; + +/* Prompt selection of LibreTranslate server to perform machine translations on notes */ +"Server" = "Palvelin"; + +/* Prompt selection of translation service provider. */ +"Service" = "Palvelu"; + +/* Navigation title for Settings view. + Sidebar menu label for accessing the app settings */ +"Settings" = "Asetukset"; + +/* Button to share a note + Button to share an image. + Button to share the link to a profile. */ +"Share" = "Jaa"; + +/* Title text to indicate that the buttons below are meant to be used to share a note with others. */ +"Share Note" = "Jaa viesti"; + +/* Button to present iOS share sheet */ +"Share Via..." = "Jaa kautta..."; + +/* Button to show a note from a user who has been muted. + Toggle to show or hide user's secret account login key. */ +"Show" = "Näytä"; + +/* Button that, when tapped, will show + buttons next to a user's relays. */ +"Show +" = "Näytä +"; + +/* Button to show less of a long profile description. */ +"Show less" = "Näytä vähemmän"; + +/* Button to show entire note. + Button to show more of a long profile description. */ +"Show more" = "Näytä lisää"; + +/* Setting to Show notifications only associated to users your follow */ +"Show only from users you follow" = "Näytä vain käyttäjiltä, joita seuraat"; + +/* Toggle to show notes that are only in the device's preferred languages on the Universe feed and hide notes that are in other languages. */ +"Show only preferred languages on Universe feed" = "Näytä Universum-syötteessä vain suosikkikieliä"; + +/* Toggle to show or hide selection of wallet. */ +"Show wallet selector" = "Näytä lompakonvalitsin"; + +/* Title of view to log into an account. */ +"Sign in" = "Kirjaudu sisään"; + +/* Section title for signing out */ +"Sign Out" = "Kirjaudu ulos"; + +/* Sidebar menu label to sign out of the account. */ +"Sign out" = "Kirjaudu ulos"; + +/* Description about why Nostr is needed. */ +"Social media has developed into a key way information flows around the world. Unfortunately, our current social media systems are broken" = "Sosiaalinen media on kehittynyt tärkeäksi tavaksi, jolla tieto liikkuu maailmassa. Valitettavasti nykyiset sosiaalisen median järjestelmät ovat rikki"; + +/* Label to display relay software. */ +"Software" = "Ohjelmisto"; + +/* Description of report type for spam. + Section header for Universe/Search spam */ +"Spam" = "Roskaposti"; + +/* Text calling for the user to support Damus through zaps */ +"Support Damus" = "Tue Damusta"; + +/* Label to display relay's supported NIPs. */ +"Supported NIPs" = "Tuetut NIP:t"; + +/* Option to take a photo with the camera */ +"Take Photo" = "Ota valokuva"; + +/* Section header for damus text truncation user configuration */ +"Text Truncation" = "Tekstin typistys"; + +/* Button to close out of alert that informs that the action to muted a user was successful. */ +"Thanks!" = "Kiitos!"; + +/* Giving the description of the alert message. */ +"The address should either begin with LNURL or should look like an email address." = "Osoitteen tulisi alkaa LNURL:llä tai näyttää sähköpostiosoitteelta."; + +/* Quick description of what Damus is */ +"The go-to iOS Nostr client" = "iOS:n Nostr-asiakasohjelma"; + +/* Footer description that explains that the relay server requires payment to post. */ +"This is a paid relay, you must pay for notes to be accepted." = "Tämä on maksullinen välittäjä, merkintöjä varten pitää maksaa."; + +/* Warning that the inputted account key is a public key and the result of what happens because of it. */ +"This is a public key, you will not be able to make notes or interact in any way. This is used for viewing accounts from their perspective." = "Tämä on julkinen avain, et voi tehdä merkintöjä tai vuorovaikuttaa millään tavalla. Tätä käytetään tilien tarkastelemiseen niiden näkökulmasta."; + +/* Label to describe that a public key is the user's account ID and what they can do with it. */ +"This is your account ID, you can give this to your friends so that they can follow you. Tap to copy." = "Tämä on tilisi tunniste. Voit antaa sen ystävillesi, jotta he voivat seurata sinua. Napauta kopioidaksesi."; + +/* Label to describe that a private key is the user's secret account key and what they should do with it. */ +"This is your secret account key. You need this to access your account. Don't share this with anyone! Save it in a password manager and keep it safe!" = "Tämä on salainen tiliavaimasi. Tarvitset tätä päästäksesi tilillesi. Älä jaa tätä kenenkään kanssa! Tallenna se salasananhallintaohjelmaan ja pidä turvassa!"; + +/* An error message that appears when the user attempts to add a relay that has already been added. */ +"This relay is already in your list" = "Tämä välittäjä on jo listallasi"; + +/* Navigation bar title for note thread. */ +"Thread" = "Ketju"; + +/* Text indicating that this zap is the one with the highest amount of sats. */ +"Top Zap" = "Paras zap"; + +/* Toggle to translate direct messages. */ +"Translate DMs" = "Käännä ykstyisviestit"; + +/* Button to translate note from different language. */ +"Translate Note" = "Käännä viesti"; + +/* Button to indicate that the note has been translated from a different language. */ +"Translated from %@" = "Käännetty kielestä %@"; + +/* Navigation title for translation settings. + Section header for text and appearance settings */ +"Translation" = "Käännös"; + +/* Section title for selecting the translation service. */ +"Translations" = "Käännökset"; + +/* Setting to truncate text in mention notifications */ +"Truncate notification mention text" = "Typistä maininta-ilmoitusten teksti"; + +/* Setting to truncate text in timeline */ +"Truncate timeline text" = "Typistä aikajanan teksti"; + +/* Text field prompt asking user to type DELETE in all caps to confirm that they want to proceed with deleting their account. */ +"Type %@ to delete" = "Kirjoita %@ poistaaksesi"; + +/* Text box prompt to ask user to type their note. */ +"Type your note here..." = "Kirjoita viestisi tähän..."; + +/* Button to unfollow a user. */ +"Unfollow" = "Lopeta seuraaminen"; + +/* Button to unfollow a given hashtag. */ +"Unfollow hashtag" = "Lopeta aihetunnisteen seuraaminen"; + +/* Label to indicate that the user is in the process of unfollowing another user. */ +"Unfollowing..." = "Lopetetaan seuraamista..."; + +/* Toolbar label for the universal view where notes from all connected relay servers appear. */ +"Universe 🛸" = "Kaikki 🛸"; + +/* Button to unmute a profile. */ +"Unmute" = "Poista mykistys"; + +/* Context menu option for unmuting a conversation. */ +"Unmute conversation" = "Poista keskustelun mykistys"; + +/* Text indicating that the long-form note title is untitled. */ +"Untitled" = "Nimeämätön"; + +/* Button to proceed with uploading. */ +"Upload" = "Lataa"; + +/* Example URL to LibreTranslate server */ +"URL" = "URL-osoite"; + +/* Alert message that informs a user was d. */ +"User has been muted" = "Käyttäjä on mykistetty"; + +/* Alert message to indicate the user has been muted */ +"User muted" = "Käyttäjä mykistetty"; + +/* Label for Username section of user profile form. */ +"Username" = "Käyttäjätunnus"; + +/* Label to display relay software version. + Section title for displaying the version number of the Damus app. */ +"Version" = "Versio"; + +/* Setting to only see 1 event per user (npub) in the search/universe */ +"View multiple events per user" = "Näytä useita tapahtumia käyttäjää kohden"; + +/* Button to switch to view users QR Code */ +"View QR Code" = "Näytä QR-koodi"; + +/* Text telling the user to wake up, where the argument is their display name. */ +"Wake up, %@" = "Herää, %@"; + +/* Sidebar menu label for Wallet view. */ +"wallet" = "lompakko"; + +/* Navigation title for Wallet view + Sidebar menu label for Wallet view. + Title for section in zap settings that controls the Lightning wallet selection. */ +"Wallet" = "Lompakko"; + +/* Alert for deleting the users account. */ +"WARNING:\n\nTHIS WILL SIGN AN EVENT THAT DELETES THIS ACCOUNT.\n\nYOU WILL NO LONGER BE ABLE TO LOG INTO DAMUS USING THIS ACCOUNT KEY.\n\n ARE YOU SURE YOU WANT TO CONTINUE?" = "VAROITUS:\n\nTÄMÄ ALLEKIRJOITTAA TAPAHTUMAN, JOKA POISTAA TÄMÄN TILIN.\n\nET VOIT ENÄÄ KIRJAUTUA DAMUKSEEN TÄLLÄ TILIAVAIMELLA.\n\nHALUATKO VARMASTI JATKAA?"; + +/* Label for Website section of user profile form. */ +"Website" = "Verkkosivusto"; + +/* Welcome text shown on the first screen when user is not logged in. */ +"Welcome to Damus" = "Tervetuloa Damukseen"; + +/* Welcoming message to the reader. The variable is 'you', the reader. */ +"Welcome to the social network %@ control." = "Tervetuloa sosiaaliseen verkostoon, jota %@ hallitset."; + +/* Welcome text */ +"Welcome to the social network you control" = "Tervetuloa sosiaaliseen verkostoon, jota hallitset"; + +/* Text to welcome user. */ +"Welcome, %@!" = "Tervetuloa, %@!"; + +/* Header text to prompt user what issue they want to report. */ +"What do you want to report?" = "Mitä haluat ilmoittaa?"; + +/* Heading text for section describing what is Nostr. */ +"What is Nostr?" = "Mikä on Nostr?"; + +/* Heading text for section describing why Nostr is needed. */ +"Why we need Nostr?" = "Miksi Nostria tarvitaan?"; + +/* Placeholder example for relay server address. */ +"wss://some.relay.com" = "wss://joku.valittaja.com"; + +/* Text of button that confirms to overwrite the existing mutelist. */ +"Yes, Overwrite" = "Kyllä, korvaa"; + +/* Button to proceed with posting a note even though it looks like they might be posting a private key. */ +"Yes, Post with Private Key" = "Kyllä, jaa yksityisavaimella"; + +/* You, in this context, is the person who controls their own social network. You is used in the context of a larger sentence that welcomes the reader to the social network that they control themself. */ +"you" = "sinä"; + +/* Text telling the user that they are dreaming. */ +"You are dreaming..." = "Sinä uneksit..."; + +/* Text indicating that there are no bookmarks to be viewed */ +"You have no bookmarks yet, add them in the context menu" = "Sinulla ei ole vielä kirjanmerkkejä, lisää niitä pikavalikosta"; + +/* Label for Your Name section of user profile form. */ +"Your Name" = "Nimesi"; + +/* Footer text to inform user what will happen when the report is submitted. */ +"Your report will be sent to the relays you are connected to" = "Ilmoituksesi lähetetään välittäjille, joihin olet yhteydessä"; + +/* Accessibility label for zap button + Text underneath the number of sats indicating that it's the amount used for zaps. + Title of notification when a non-private zap is received. */ +"Zap" = "Zap"; + +/* Message to display when sending a zap from the user's connected wallet failed. */ +"Zap attempt from connected wallet failed." = "Yhdistetystä lompakosta lähetetty zap-yritys epäonnistui."; + +/* Message to display when a zap from the user's connected wallet was canceled. */ +"Zap attempt from connected wallet was canceled." = "Yhdistetystä lompakosta lähetetty zap-yritys peruutettiin."; + +/* Text to indicate that the buttons below it is for choosing the type of zap to send. */ +"Zap type" = "Zap-tyyppi"; + +/* Button to send a zap. */ +"Zap User" = "Zap käyttäjälle"; + +/* Setting to enable vibration on zap */ +"Zap Vibration" = "Zap-värinä"; + +/* Text to indicate that the app is in the process of sending a zap. */ +"Zapping..." = "Zappaa..."; + +/* Label for filter for zap notifications. + Navigation bar title for the Zaps view. + Navigation title for zap settings. + Section header for zap settings + Setting to enable Zap Local Notification + Title for section in zap settings that controls general zap preferences. */ +"Zaps" = "Zapit"; \ No newline at end of file diff --git a/damus/fi.lproj/Localizable.stringsdict b/damus/fi.lproj/Localizable.stringsdict new file mode 100644 index 0000000000..56af52214e --- /dev/null +++ b/damus/fi.lproj/Localizable.stringsdict @@ -0,0 +1,359 @@ + + + + + followed_by_three_and_others + + NSStringLocalizedFormatKey + %#@OTHERS@ + OTHERS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Seuraajina %2$@, %3$@, %4$@ & %1$d muuta + other + Seuraajina %2$@, %3$@, %4$@ & %1$d muuta + + + followers_count + + NSStringLocalizedFormatKey + %#@FOLLOWERS@ + FOLLOWERS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Seuraaja + other + Seuraajat + + + following_count + + NSStringLocalizedFormatKey + %#@FOLLOWING@ + FOLLOWING + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Seuraa + other + Seuraa + + + imports_count + + NSStringLocalizedFormatKey + %#@IMPORTS@ + IMPORTS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Tuonti + other + Tuontia + + + reacted_tagged_in_3 + + NSStringLocalizedFormatKey + %#@REACTED@ + REACTED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muu reagoi viestiin, johon sinut on merkitty + other + %2$@ ja %1$d muuta reagoi viestiin, johon sinut on merkitty + + + reacted_your_note_3 + + NSStringLocalizedFormatKey + %#@REACTED@ + REACTED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muu reagoi viestiisi + other + %2$@ ja %1$d muuta reagoi viestiisi + + + reacted_your_profile_3 + + NSStringLocalizedFormatKey + %#@REACTED@ + REACTED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muu reagoi profiiliisi + other + %2$@ ja %1$d muuta reagoi profiiliisi + + + reactions_count + + NSStringLocalizedFormatKey + %#@REACTIONS@ + REACTIONS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Reaktio + other + Reaktiot + + + relays_count + + NSStringLocalizedFormatKey + %#@RELAYS@ + RELAYS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Välittäjä + other + Välittäjät + + + replying_to_two_and_others + + NSStringLocalizedFormatKey + %#@OTHERS@ + OTHERS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Vastaa käyttäjille %2$@, %3$@ & %1$d muille + other + Vastaa käyttäjille %2$@, %3$@ & %1$d muille + + + reposted_tagged_in_3 + + NSStringLocalizedFormatKey + %#@REPOSTED@ + REPOSTED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muuta jakoivat uudelleen viestin, johon sinut on merkitty + other + %2$@ ja %1$d muuta jakoivat uudelleen viestin, johon sinut on merkitty + + + reposted_your_note_3 + + NSStringLocalizedFormatKey + %#@REPOSTED@ + REPOSTED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muuta jakoivat uudelleen viestisi + other + %2$@ ja %1$d muuta jakoivat uudelleen viestisi + + + reposted_your_profile_3 + + NSStringLocalizedFormatKey + %#@REPOSTED@ + REPOSTED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muuta jakoivat profiilisi + other + %2$@ ja %1$d muuta jakoivat profiilisi + + + reposts_count + + NSStringLocalizedFormatKey + %#@REPOSTS@ + REPOSTS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Jaettu uudelleen + other + Jaettu uudelleen + + + sats + + NSStringLocalizedFormatKey + %#@SATS@ + SATS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + sat + other + satsia + + + sats_count + + NSStringLocalizedFormatKey + %1$#@SATS@ + SATS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + @ + one + %2$@ sat + other + %2$@ satsia + + + zap_notification_no_message + + NSStringLocalizedFormatKey + %1$#@NOTIFICATION@ + NOTIFICATION + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + @ + one + Sait %2$@ satin käyttäjältä %3$@ + other + Sait %2$@ satsia käyttäjältä %3$@ + + + zap_notification_with_message + + NSStringLocalizedFormatKey + %1$#@NOTIFICATION@ + NOTIFICATION + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + @ + one + Sait %2$@ satin käyttäjältä %3$@: "%4$@" + other + Sait %2$@ satsia käyttäjältä %3$@: "%4$@" + + + zapped_tagged_in_3 + + NSStringLocalizedFormatKey + %#@ZAPPED@ + ZAPPED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muuta zappasivat viestin, johon sinut on merkitty + other + %2$@ ja %1$d muuta zappasivat viestin, johon sinut on merkitty + + + zapped_your_note_3 + + NSStringLocalizedFormatKey + %#@ZAPPED@ + ZAPPED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muuta zappasivat viestisi + other + %2$@ ja %1$d muuta zappasivat viestisi + + zapped_your_profile_3 + + NSStringLocalizedFormatKey + %#@ZAPPED@ + ZAPPED + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + %2$@ ja %1$d muuta zappasivat profiiliasi + other + %2$@ ja %1$d muuta zappasivat profiiliasi + + + zaps_count + + NSStringLocalizedFormatKey + %#@ZAPS@ + ZAPS + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + d + one + Zap + other + Zapit + + + + \ No newline at end of file From 3e15f15a57e78809c2e479fa4d840df4326ecb89 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:03:44 -0600 Subject: [PATCH 064/111] colors: add color sets from figma --- .../DamusDangerBorder.colorset/Contents.json | 38 +++++++++++++++++++ .../DamusDangerPrimary.colorset/Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../DamusNeutral1.colorset/Contents.json | 38 +++++++++++++++++++ .../DamusNeutral3.colorset/Contents.json | 38 +++++++++++++++++++ .../DamusNeutral6.colorset/Contents.json | 38 +++++++++++++++++++ .../DamusSuccessBorder.colorset/Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../DamusWarningBorder.colorset/Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ .../Contents.json | 38 +++++++++++++++++++ 18 files changed, 684 insertions(+) create mode 100644 damus/Assets.xcassets/Colors/DamusDangerBorder.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusDangerPrimary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusDangerQuaternary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusDangerSecondary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusDangerTertiary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusNeutral1.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusNeutral3.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusNeutral6.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusSuccessBorder.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusSuccessPrimary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusSuccessQuaternary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusSuccessSecondary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusSuccessTertiary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusWarningBorder.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusWarningPrimary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusWarningQuaternary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusWarningSecondary.colorset/Contents.json create mode 100644 damus/Assets.xcassets/Colors/DamusWarningTertiary.colorset/Contents.json diff --git a/damus/Assets.xcassets/Colors/DamusDangerBorder.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusDangerBorder.colorset/Contents.json new file mode 100644 index 0000000000..7fadd5c1b2 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusDangerBorder.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xE3", + "green" : "0xD7", + "red" : "0xF7" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x20", + "green" : "0x13", + "red" : "0x61" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusDangerPrimary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusDangerPrimary.colorset/Contents.json new file mode 100644 index 0000000000..4e59e9409f --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusDangerPrimary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x63", + "green" : "0x11", + "red" : "0xF5" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x6E", + "green" : "0x20", + "red" : "0xF8" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusDangerQuaternary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusDangerQuaternary.colorset/Contents.json new file mode 100644 index 0000000000..6062bb59db --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusDangerQuaternary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xEE", + "green" : "0xE8", + "red" : "0xF7" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x35", + "green" : "0x04", + "red" : "0x8B" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusDangerSecondary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusDangerSecondary.colorset/Contents.json new file mode 100644 index 0000000000..75ea47ba1f --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusDangerSecondary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x3D", + "green" : "0x07", + "red" : "0x9C" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x44", + "green" : "0x06", + "red" : "0xB2" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusDangerTertiary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusDangerTertiary.colorset/Contents.json new file mode 100644 index 0000000000..a721e9dfca --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusDangerTertiary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x2D", + "green" : "0x05", + "red" : "0x75" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xD8", + "green" : "0xC2", + "red" : "0xFF" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusNeutral1.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusNeutral1.colorset/Contents.json new file mode 100644 index 0000000000..1d836e6ef5 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusNeutral1.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xFA", + "green" : "0xFA", + "red" : "0xF9" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x24", + "green" : "0x22", + "red" : "0x20" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusNeutral3.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusNeutral3.colorset/Contents.json new file mode 100644 index 0000000000..e97f26c653 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusNeutral3.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xE3", + "green" : "0xE1", + "red" : "0xDD" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x2A", + "green" : "0x26", + "red" : "0x23" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusNeutral6.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusNeutral6.colorset/Contents.json new file mode 100644 index 0000000000..2e99fc87f0 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusNeutral6.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x59", + "green" : "0x53", + "red" : "0x4A" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x85", + "green" : "0x7A", + "red" : "0x6A" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusSuccessBorder.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusSuccessBorder.colorset/Contents.json new file mode 100644 index 0000000000..6f7f2c7552 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusSuccessBorder.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xE4", + "green" : "0xF1", + "red" : "0xD6" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x38", + "green" : "0x5C", + "red" : "0x12" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusSuccessPrimary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusSuccessPrimary.colorset/Contents.json new file mode 100644 index 0000000000..2795f54ff5 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusSuccessPrimary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x5A", + "green" : "0xAB", + "red" : "0x04" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x64", + "green" : "0xBF", + "red" : "0x03" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusSuccessQuaternary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusSuccessQuaternary.colorset/Contents.json new file mode 100644 index 0000000000..d117edea5c --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusSuccessQuaternary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xF0", + "green" : "0xF7", + "red" : "0xE8" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x1F", + "green" : "0x33", + "red" : "0x0A" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusSuccessSecondary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusSuccessSecondary.colorset/Contents.json new file mode 100644 index 0000000000..52897867c4 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusSuccessSecondary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x34", + "green" : "0x64", + "red" : "0x02" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x3F", + "green" : "0x79", + "red" : "0x02" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusSuccessTertiary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusSuccessTertiary.colorset/Contents.json new file mode 100644 index 0000000000..9dfe396e0c --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusSuccessTertiary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x1F", + "green" : "0x3C", + "red" : "0x01" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xE4", + "green" : "0xFF", + "red" : "0xAD" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusWarningBorder.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusWarningBorder.colorset/Contents.json new file mode 100644 index 0000000000..aa3b08a4ed --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusWarningBorder.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xD1", + "green" : "0xEE", + "red" : "0xFE" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x12", + "green" : "0x43", + "red" : "0x5C" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusWarningPrimary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusWarningPrimary.colorset/Contents.json new file mode 100644 index 0000000000..6dcd2a89ab --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusWarningPrimary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x1C", + "green" : "0xAD", + "red" : "0xF9" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x2C", + "green" : "0xB5", + "red" : "0xFC" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusWarningQuaternary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusWarningQuaternary.colorset/Contents.json new file mode 100644 index 0000000000..a0ffe76e74 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusWarningQuaternary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xE1", + "green" : "0xF4", + "red" : "0xFE" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x0A", + "green" : "0x25", + "red" : "0x33" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusWarningSecondary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusWarningSecondary.colorset/Contents.json new file mode 100644 index 0000000000..3728491d14 --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusWarningSecondary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x06", + "green" : "0x85", + "red" : "0xC6" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x03", + "green" : "0x93", + "red" : "0xDD" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/damus/Assets.xcassets/Colors/DamusWarningTertiary.colorset/Contents.json b/damus/Assets.xcassets/Colors/DamusWarningTertiary.colorset/Contents.json new file mode 100644 index 0000000000..e6e5fe862e --- /dev/null +++ b/damus/Assets.xcassets/Colors/DamusWarningTertiary.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors" : [ + { + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0x04", + "green" : "0x6A", + "red" : "0x9F" + } + }, + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "color" : { + "color-space" : "srgb", + "components" : { + "alpha" : "1.000", + "blue" : "0xCC", + "green" : "0xF5", + "red" : "0xFF" + } + }, + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} From 20b6627799b79b7a445d9c779d61fa5ad2ebadb6 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:12:06 -0600 Subject: [PATCH 065/111] colors: add variables for the new color assets --- damus/Components/DamusColors.swift | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/damus/Components/DamusColors.swift b/damus/Components/DamusColors.swift index 6875affe7f..4eb7f43d49 100644 --- a/damus/Components/DamusColors.swift +++ b/damus/Components/DamusColors.swift @@ -22,5 +22,23 @@ class DamusColors { static let purple = Color("DamusPurple") static let deepPurple = Color("DamusDeepPurple") static let blue = Color("DamusBlue") + static let success = Color("DamusSuccessPrimary") + static let successSecondary = Color("DamusSuccessSecondary") + static let successTertiary = Color("DamusSuccessTertiary") + static let successQuaternary = Color("DamusSuccessQuaternary") + static let successBorder = Color("DamusSuccessBorder") + static let warning = Color("DamusWarningPrimary") + static let warningSecondary = Color("DamusWarningSecondary") + static let warningTertiary = Color("DamusWarningTertiary") + static let warningQuaternary = Color("DamusWarningQuaternary") + static let warningBorder = Color("DamusWarningBorder") + static let danger = Color("DamusDangerPrimary") + static let dangerSecondary = Color("DamusDangerSecondary") + static let dangerTertiary = Color("DamusDangerTertiary") + static let dangerQuaternary = Color("DamusDangerQuaternary") + static let dangerBorder = Color("DamusDangerBorder") + static let neutral1 = Color("DamusNeutral1") + static let neutral3 = Color("DamusNeutral3") + static let neutral6 = Color("DamusNeutral6") } From b934d66f642ec10dc9c952dde645e7706c140008 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:06:01 -0600 Subject: [PATCH 066/111] components: add lighter gradient --- damus.xcodeproj/project.pbxproj | 4 +++ .../Gradients/DamusLightGradient.swift | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 damus/Components/Gradients/DamusLightGradient.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 24b28c8dda..13e27ffc8a 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -400,6 +400,7 @@ 5C513FCC2984ACA60072348F /* QRCodeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C513FCB2984ACA60072348F /* QRCodeView.swift */; }; 5C6E1DAD2A193EC2008FC15A /* GradientButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */; }; 5C6E1DAF2A194075008FC15A /* PinkGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */; }; + 5CF2DCCE2AABE1A500984B8D /* DamusLightGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */; }; 5CF72FC229B9142F00124A13 /* ShareAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF72FC129B9142F00124A13 /* ShareAction.swift */; }; 6439E014296790CF0020672B /* ProfilePicImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6439E013296790CF0020672B /* ProfilePicImageView.swift */; }; 643EA5C8296B764E005081BB /* RelayFilterView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 643EA5C7296B764E005081BB /* RelayFilterView.swift */; }; @@ -1078,6 +1079,7 @@ 5C513FCB2984ACA60072348F /* QRCodeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QRCodeView.swift; sourceTree = ""; }; 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GradientButtonStyle.swift; sourceTree = ""; }; 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PinkGradient.swift; sourceTree = ""; }; + 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusLightGradient.swift; sourceTree = ""; }; 5CF72FC129B9142F00124A13 /* ShareAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareAction.swift; sourceTree = ""; }; 6439E013296790CF0020672B /* ProfilePicImageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilePicImageView.swift; sourceTree = ""; }; 643EA5C7296B764E005081BB /* RelayFilterView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterView.swift; sourceTree = ""; }; @@ -1735,6 +1737,7 @@ F71694F72A6983AF001F4053 /* GrayGradient.swift */, 5C0707D02A1ECB38004E7B51 /* DamusLogoGradient.swift */, 4C687C202A5F7ED00092C550 /* DamusBackground.swift */, + 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */, ); path = Gradients; sourceTree = ""; @@ -2788,6 +2791,7 @@ 4C06670E28FDEAA000038D2A /* utf8.c in Sources */, 4C3EA66D28FF782800C48A62 /* amount.c in Sources */, 4C32B9562A9AD44700DC3548 /* TableVerifier.swift in Sources */, + 5CF2DCCE2AABE1A500984B8D /* DamusLightGradient.swift in Sources */, 4C5E54062A9671F800FF6E60 /* UserStatusSheet.swift in Sources */, F71694F42A6732B7001F4053 /* GradientFollowButton.swift in Sources */, 4C3AC7A728369BA200E1F516 /* SearchHomeView.swift in Sources */, diff --git a/damus/Components/Gradients/DamusLightGradient.swift b/damus/Components/Gradients/DamusLightGradient.swift new file mode 100644 index 0000000000..c6537e1327 --- /dev/null +++ b/damus/Components/Gradients/DamusLightGradient.swift @@ -0,0 +1,30 @@ +// +// DamusLightGradient.swift +// damus +// +// Created by eric on 9/8/23. +// + +import SwiftUI + +fileprivate let damus_grad_c1 = hex_col(r: 0xd3, g: 0x2d, b: 0xc3) +fileprivate let damus_grad_c2 = hex_col(r: 0x33, g: 0xc5, b: 0xbc) +fileprivate let damus_grad = [damus_grad_c1, damus_grad_c2] + +struct DamusLightGradient: View { + var body: some View { + DamusLightGradient.gradient + .opacity(0.5) + .edgesIgnoringSafeArea([.top,.bottom]) + } + + static var gradient: LinearGradient { + LinearGradient(colors: damus_grad, startPoint: .topLeading, endPoint: .bottomTrailing) + } +} + +struct DamusLightGradient_Previews: PreviewProvider { + static var previews: some View { + DamusLightGradient() + } +} From 94ce604b9d2a787d43227263835d412c1d31cd20 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:06:31 -0600 Subject: [PATCH 067/111] components: add neutral button style component --- damus.xcodeproj/project.pbxproj | 4 ++ damus/Components/NeutralButtonStyle.swift | 49 +++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 damus/Components/NeutralButtonStyle.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 13e27ffc8a..2e64a511cd 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -400,6 +400,7 @@ 5C513FCC2984ACA60072348F /* QRCodeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C513FCB2984ACA60072348F /* QRCodeView.swift */; }; 5C6E1DAD2A193EC2008FC15A /* GradientButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */; }; 5C6E1DAF2A194075008FC15A /* PinkGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */; }; + 5CC868DD2AA29B3200FB22BA /* NeutralButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CC868DC2AA29B3200FB22BA /* NeutralButtonStyle.swift */; }; 5CF2DCCE2AABE1A500984B8D /* DamusLightGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */; }; 5CF72FC229B9142F00124A13 /* ShareAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF72FC129B9142F00124A13 /* ShareAction.swift */; }; 6439E014296790CF0020672B /* ProfilePicImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6439E013296790CF0020672B /* ProfilePicImageView.swift */; }; @@ -1079,6 +1080,7 @@ 5C513FCB2984ACA60072348F /* QRCodeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QRCodeView.swift; sourceTree = ""; }; 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GradientButtonStyle.swift; sourceTree = ""; }; 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PinkGradient.swift; sourceTree = ""; }; + 5CC868DC2AA29B3200FB22BA /* NeutralButtonStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NeutralButtonStyle.swift; sourceTree = ""; }; 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusLightGradient.swift; sourceTree = ""; }; 5CF72FC129B9142F00124A13 /* ShareAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareAction.swift; sourceTree = ""; }; 6439E013296790CF0020672B /* ProfilePicImageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilePicImageView.swift; sourceTree = ""; }; @@ -2043,6 +2045,7 @@ 4C8D00C929DF80350036AF10 /* TruncatedText.swift */, 4C28595F2A12A2BE004746F7 /* SupporterBadge.swift */, 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */, + 5CC868DC2AA29B3200FB22BA /* NeutralButtonStyle.swift */, ); path = Components; sourceTree = ""; @@ -2858,6 +2861,7 @@ 4C73C5142A4437C10062CAC0 /* ZapUserView.swift in Sources */, 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */, 4C1A9A1D29DDCF9B00516EAC /* NotificationSettingsView.swift in Sources */, + 5CC868DD2AA29B3200FB22BA /* NeutralButtonStyle.swift in Sources */, 4C75EFB528049D790006080F /* Relay.swift in Sources */, 4CEE2AF1280B216B00AB5EEF /* EventDetailView.swift in Sources */, 4CC7AAFA297F64AC00430951 /* EventMenu.swift in Sources */, diff --git a/damus/Components/NeutralButtonStyle.swift b/damus/Components/NeutralButtonStyle.swift new file mode 100644 index 0000000000..3d61b19e49 --- /dev/null +++ b/damus/Components/NeutralButtonStyle.swift @@ -0,0 +1,49 @@ +// +// NeutralButtonStyle.swift +// damus +// +// Created by eric on 9/1/23. +// + +import SwiftUI + +struct NeutralButtonStyle: ButtonStyle { + func makeBody(configuration: Self.Configuration) -> some View { + return configuration.label + .background(DamusColors.neutral1) + .cornerRadius(12) + .overlay( + RoundedRectangle(cornerRadius: 12) + .stroke(DamusColors.neutral3, lineWidth: 1) + ) + .scaleEffect(configuration.isPressed ? 0.95 : 1) + } +} + + +struct NeutralButtonStyle_Previews: PreviewProvider { + static var previews: some View { + VStack { + Button(action: { + print("dynamic size") + }) { + Text(verbatim: "Dynamic Size") + .padding() + } + .buttonStyle(NeutralButtonStyle()) + + + Button(action: { + print("infinite width") + }) { + HStack { + Text(verbatim: "Infinite Width") + .padding() + } + .frame(minWidth: 300, maxWidth: .infinity, alignment: .center) + } + .buttonStyle(NeutralButtonStyle()) + .padding() + } + } +} From 65f365189655040a8bf339bed8c9db106fb05cf3 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:13:47 -0600 Subject: [PATCH 068/111] ui: dont display globe image for free relay types --- damus/Views/Relays/RelayType.swift | 5 ----- 1 file changed, 5 deletions(-) diff --git a/damus/Views/Relays/RelayType.swift b/damus/Views/Relays/RelayType.swift index 9e8b93d34d..328d42db11 100644 --- a/damus/Views/Relays/RelayType.swift +++ b/damus/Views/Relays/RelayType.swift @@ -13,11 +13,6 @@ struct RelayType: View { var body: some View { if is_paid { Image("bitcoin-logo") - } else { - Image("globe") - .resizable() - .frame(width: 20, height: 20) - .foregroundColor(.gray) } } } From 6f35de65f93b4967caafe3279aafeb15ced2b84c Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:18:23 -0600 Subject: [PATCH 069/111] relays: add icon field to metadata --- damus/Nostr/Relay.swift | 1 + damus/Views/Relays/RelayDetailView.swift | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/damus/Nostr/Relay.swift b/damus/Nostr/Relay.swift index 92bfb588e7..c9c2ec8405 100644 --- a/damus/Nostr/Relay.swift +++ b/damus/Nostr/Relay.swift @@ -75,6 +75,7 @@ struct RelayMetadata: Codable { let version: String? let limitation: Limitations? let payments_url: String? + let icon: String? var is_paid: Bool { return limitation?.payment_required ?? false diff --git a/damus/Views/Relays/RelayDetailView.swift b/damus/Views/Relays/RelayDetailView.swift index 6f9aab293f..3662222fcb 100644 --- a/damus/Views/Relays/RelayDetailView.swift +++ b/damus/Views/Relays/RelayDetailView.swift @@ -173,7 +173,7 @@ struct RelayDetailView: View { struct RelayDetailView_Previews: PreviewProvider { static var previews: some View { - let metadata = RelayMetadata(name: "name", description: "desc", pubkey: test_pubkey, contact: "contact", supported_nips: [1,2,3], software: "software", version: "version", limitation: Limitations.empty, payments_url: "https://jb55.com") + let metadata = RelayMetadata(name: "name", description: "desc", pubkey: test_pubkey, contact: "contact", supported_nips: [1,2,3], software: "software", version: "version", limitation: Limitations.empty, payments_url: "https://jb55.com", icon: "") RelayDetailView(state: test_damus_state(), relay: "relay", nip11: metadata) } } From 7f6a702412389f596dd261e5f4374d91209e1f92 Mon Sep 17 00:00:00 2001 From: Suhail Saqan Date: Thu, 7 Sep 2023 17:57:19 -0700 Subject: [PATCH 070/111] emojis: make width dynamic and font bigger add calculateOverlayWidth to support this Closes: https://github.com/damus-io/damus/pull/1542 Signed-off-by: William Casarin --- damus/Views/ActionBar/EventActionBar.swift | 56 ++++++++++++++-------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift index f314fd62b4..f5b453a928 100644 --- a/damus/Views/ActionBar/EventActionBar.swift +++ b/damus/Views/ActionBar/EventActionBar.swift @@ -256,8 +256,8 @@ struct LikeButton: View { Group { if isReactionsVisible { ZStack { - RoundedRectangle(cornerRadius: 10) - .frame(width: 250, height: 50) + RoundedRectangle(cornerRadius: 20) + .frame(width: calculateOverlayWidth(), height: 50) .foregroundColor(DamusColors.black) .scaleEffect(Double(showReactionsBG), anchor: .topTrailing) .animation( @@ -267,9 +267,9 @@ struct LikeButton: View { .overlay( Rectangle() .foregroundColor(Color.white.opacity(0.2)) - .frame(width: 250, height: 50) + .frame(width: calculateOverlayWidth(), height: 50) .clipShape( - RoundedRectangle(cornerRadius: 10) + RoundedRectangle(cornerRadius: 20) ) ) .overlay(reactions()) @@ -287,28 +287,34 @@ struct LikeButton: View { } } } + + func calculateOverlayWidth() -> CGFloat { + let maxWidth: CGFloat = 250 + let numberOfEmojis = emojis.count + let minimumWidth: CGFloat = 75 + + if numberOfEmojis > 0 { + let emojiWidth: CGFloat = 25 + let padding: CGFloat = 15 + let buttonWidth: CGFloat = 18 + let buttonPadding: CGFloat = 20 + + let totalWidth = CGFloat(numberOfEmojis) * (emojiWidth + padding) + buttonWidth + buttonPadding + return min(maxWidth, max(minimumWidth, totalWidth)) + } else { + return minimumWidth + } + } func reactions() -> some View { HStack { - Button(action: { - withAnimation(.easeOut(duration: 0.2)) { - isReactionsVisible = false - showReactionsBG = 0 - } - showEmojis = [] - }) { - Image(systemName: "xmark.circle.fill") - .font(.body) - .foregroundColor(.gray) - } - .padding(.leading, 7.5) - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: 20) { + HStack(spacing: 15) { ForEach(emojis, id: \.self) { emoji in if let index = emojis.firstIndex(of: emoji) { let scale = index < showEmojis.count ? showEmojis[index] : 0 Text(emoji) + .font(.system(size: 25)) .scaleEffect(Double(scale)) .onTapGesture { emojiTapped(emoji) @@ -316,8 +322,20 @@ struct LikeButton: View { } } } - .padding(.trailing, 10) + .padding(.leading, 10) + } + Button(action: { + withAnimation(.easeOut(duration: 0.2)) { + isReactionsVisible = false + showReactionsBG = 0 + } + showEmojis = [] + }) { + Image(systemName: "xmark.circle.fill") + .font(.system(size: 18)) + .foregroundColor(.gray) } + .padding(.trailing, 7.5) } } From 4263b9690fff3469905679f71828994f43efb3b1 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 09:50:10 -0700 Subject: [PATCH 071/111] deps: add tldextract This is needed for the new relay view --- damus.xcodeproj/project.pbxproj | 17 +++++++++++++++++ .../xcshareddata/swiftpm/Package.resolved | 9 +++++++++ 2 files changed, 26 insertions(+) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 2e64a511cd..5bed16936c 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -401,6 +401,7 @@ 5C6E1DAD2A193EC2008FC15A /* GradientButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */; }; 5C6E1DAF2A194075008FC15A /* PinkGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */; }; 5CC868DD2AA29B3200FB22BA /* NeutralButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CC868DC2AA29B3200FB22BA /* NeutralButtonStyle.swift */; }; + 5CF2DCCA2AA3A49B00984B8D /* TLDExtract in Frameworks */ = {isa = PBXBuildFile; productRef = 5CF2DCC92AA3A49B00984B8D /* TLDExtract */; }; 5CF2DCCE2AABE1A500984B8D /* DamusLightGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */; }; 5CF72FC229B9142F00124A13 /* ShareAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF72FC129B9142F00124A13 /* ShareAction.swift */; }; 6439E014296790CF0020672B /* ProfilePicImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6439E013296790CF0020672B /* ProfilePicImageView.swift */; }; @@ -1132,6 +1133,7 @@ 4C06670428FC7EC500038D2A /* Kingfisher in Frameworks */, 4C649881286E0EE300EAE2B3 /* secp256k1 in Frameworks */, 4C27C9322A64766F007DBC75 /* MarkdownUI in Frameworks */, + 5CF2DCCA2AA3A49B00984B8D /* TLDExtract in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -2310,6 +2312,7 @@ 4C649880286E0EE300EAE2B3 /* secp256k1 */, 4C06670328FC7EC500038D2A /* Kingfisher */, 4C27C9312A64766F007DBC75 /* MarkdownUI */, + 5CF2DCC92AA3A49B00984B8D /* TLDExtract */, ); productName = damus; productReference = 4CE6DEE327F7A08100C66700 /* damus.app */; @@ -2417,6 +2420,7 @@ 4C06670228FC7EC500038D2A /* XCRemoteSwiftPackageReference "Kingfisher" */, 4CCF9AB02A1FE80B00E03CFB /* XCRemoteSwiftPackageReference "GSPlayer" */, 4C27C9302A64766F007DBC75 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, + 5CF2DCC82AA3A49B00984B8D /* XCRemoteSwiftPackageReference "TLDExtractSwift" */, ); productRefGroup = 4CE6DEE427F7A08100C66700 /* Products */; projectDirPath = ""; @@ -3424,6 +3428,14 @@ minimumVersion = 0.2.26; }; }; + 5CF2DCC82AA3A49B00984B8D /* XCRemoteSwiftPackageReference "TLDExtractSwift" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/MarcoEidinger/TLDExtractSwift"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 2.0.0; + }; + }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ @@ -3442,6 +3454,11 @@ package = 4C64987F286E0EE300EAE2B3 /* XCRemoteSwiftPackageReference "secp256k1" */; productName = secp256k1; }; + 5CF2DCC92AA3A49B00984B8D /* TLDExtract */ = { + isa = XCSwiftPackageProductDependency; + package = 5CF2DCC82AA3A49B00984B8D /* XCRemoteSwiftPackageReference "TLDExtractSwift" */; + productName = TLDExtract; + }; /* End XCSwiftPackageProductDependency section */ /* Begin XCVersionGroup section */ diff --git a/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index c8409240c6..7f3c64b5a7 100644 --- a/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -33,6 +33,15 @@ "state" : { "revision" : "76bb7971da7fbf429de1c84f1244adf657242fee" } + }, + { + "identity" : "tldextractswift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/MarcoEidinger/TLDExtractSwift", + "state" : { + "revision" : "d4cbc911ab087e34439879df7f0314075886ae21", + "version" : "2.3.5" + } } ], "version" : 2 From 7baf7e66dc4a8cdc9c7c465e2a9d82dc6d9e5c30 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:11:13 -0600 Subject: [PATCH 072/111] relays: add relay pic view for displaying relay icons --- damus.xcodeproj/project.pbxproj | 4 ++ damus/Views/Relays/RelayPicView.swift | 100 ++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 damus/Views/Relays/RelayPicView.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 5bed16936c..45283a24ed 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -402,6 +402,7 @@ 5C6E1DAF2A194075008FC15A /* PinkGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */; }; 5CC868DD2AA29B3200FB22BA /* NeutralButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CC868DC2AA29B3200FB22BA /* NeutralButtonStyle.swift */; }; 5CF2DCCA2AA3A49B00984B8D /* TLDExtract in Frameworks */ = {isa = PBXBuildFile; productRef = 5CF2DCC92AA3A49B00984B8D /* TLDExtract */; }; + 5CF2DCCC2AA3AF0B00984B8D /* RelayPicView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF2DCCB2AA3AF0B00984B8D /* RelayPicView.swift */; }; 5CF2DCCE2AABE1A500984B8D /* DamusLightGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */; }; 5CF72FC229B9142F00124A13 /* ShareAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF72FC129B9142F00124A13 /* ShareAction.swift */; }; 6439E014296790CF0020672B /* ProfilePicImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6439E013296790CF0020672B /* ProfilePicImageView.swift */; }; @@ -1082,6 +1083,7 @@ 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GradientButtonStyle.swift; sourceTree = ""; }; 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PinkGradient.swift; sourceTree = ""; }; 5CC868DC2AA29B3200FB22BA /* NeutralButtonStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NeutralButtonStyle.swift; sourceTree = ""; }; + 5CF2DCCB2AA3AF0B00984B8D /* RelayPicView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayPicView.swift; sourceTree = ""; }; 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusLightGradient.swift; sourceTree = ""; }; 5CF72FC129B9142F00124A13 /* ShareAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareAction.swift; sourceTree = ""; }; 6439E013296790CF0020672B /* ProfilePicImageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilePicImageView.swift; sourceTree = ""; }; @@ -1919,6 +1921,7 @@ 4CE8794F2996B2BD00F758CC /* RelayStatusView.swift */, 4CE879512996B68900F758CC /* RelayType.swift */, 4CDA128929E9D10C0006FA5A /* SignalView.swift */, + 5CF2DCCB2AA3AF0B00984B8D /* RelayPicView.swift */, ); path = Relays; sourceTree = ""; @@ -2667,6 +2670,7 @@ 4C32B9602A9AD44700DC3548 /* Struct.swift in Sources */, 4CC7AAF0297F11C700430951 /* SelectedEventView.swift in Sources */, 4CC7AAF8297F1CEE00430951 /* EventProfile.swift in Sources */, + 5CF2DCCC2AA3AF0B00984B8D /* RelayPicView.swift in Sources */, 4C687C242A5FA86D0092C550 /* SearchHeaderView.swift in Sources */, 64FBD06F296255C400D9D3B2 /* Theme.swift in Sources */, 4C1A9A2329DDDB8100516EAC /* IconLabel.swift in Sources */, diff --git a/damus/Views/Relays/RelayPicView.swift b/damus/Views/Relays/RelayPicView.swift new file mode 100644 index 0000000000..40c51af8d5 --- /dev/null +++ b/damus/Views/Relays/RelayPicView.swift @@ -0,0 +1,100 @@ +// +// RelayPicView.swift +// damus +// +// Created by eric on 9/2/23. +// + +import SwiftUI +import Kingfisher +import TLDExtract + +struct InnerRelayPicView: View { + let url: URL? + let size: CGFloat + let highlight: Highlight + let disable_animation: Bool + @State var failedImage: Bool = false + + var Placeholder: some View { + RoundedRectangle(cornerRadius: 15) + .frame(width: size, height: size) + .foregroundColor(DamusColors.adaptableGrey) + .overlay(RoundedRectangle(cornerRadius: 15).stroke(highlight_color(highlight), lineWidth: pfp_line_width(highlight))) + .padding(2) + } + + var body: some View { + ZStack { + Color(uiColor: .secondarySystemBackground) + + KFAnimatedImage(url) + .imageContext(.pfp, disable_animation: disable_animation) + .onFailure({ result in + failedImage = true + }) + .cancelOnDisappear(true) + .configure { view in + view.framePreloadCount = 3 + } + .placeholder { _ in + Placeholder + } + .scaledToFit() + + if failedImage { + let abbrv = String(url?.hostname?.first?.uppercased() ?? "R") + Text("\(abbrv)") + .font(.system(size: 40, weight: .bold)) + } + } + .frame(width: size, height: size) + .clipShape(RoundedRectangle(cornerRadius: 15)) + .overlay(RoundedRectangle(cornerRadius: 15).stroke(failedImage ? .gray : highlight_color(highlight), lineWidth: failedImage ? 1 : pfp_line_width(highlight))) + } +} + +struct RelayPicView: View { + let relay: String + let icon: String? + let size: CGFloat + let highlight: Highlight + let disable_animation: Bool + + init(relay: String, icon: String? = nil, size: CGFloat, highlight: Highlight, disable_animation: Bool) { + self.relay = relay + self.icon = icon + self.size = size + self.highlight = highlight + self.disable_animation = disable_animation + } + + var body: some View { + InnerRelayPicView(url: get_relay_url(relay: relay, icon: icon), size: size, highlight: highlight, disable_animation: disable_animation) + } +} + +func get_relay_url(relay: String, icon: String?) -> URL { + let extractor = TLDExtract() + var favicon = relay + "/favicon.ico" + if let parseRelay: TLDResult = extractor.parse(relay) { + favicon = "https://" + (parseRelay.rootDomain ?? relay) + "/favicon.ico" + } + let pic = icon ?? favicon + if let url = URL(string: pic) { + return url + } + return URL(string: "")! +} + +struct RelayPicView_Previews: PreviewProvider { + static var previews: some View { + VStack { + RelayPicView(relay: "wss://relay.damus.io", size: 55, highlight: .none, disable_animation: false) + RelayPicView(relay: "wss://nostr.wine", size: 55, highlight: .none, disable_animation: false) + RelayPicView(relay: "wss://nos.lol", size: 55, highlight: .none, disable_animation: false) + RelayPicView(relay: "fail", size: 55, highlight: .none, disable_animation: false) + } + } +} + From 14586b616cad45fb61cc3948c2fa9ec66ddfdb08 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 18:21:39 -0700 Subject: [PATCH 073/111] log: remove some verbose preload logs --- damus/Util/EventCache.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/damus/Util/EventCache.swift b/damus/Util/EventCache.swift index 29d3229c46..cd4458b6a2 100644 --- a/damus/Util/EventCache.swift +++ b/damus/Util/EventCache.swift @@ -365,15 +365,15 @@ func get_preload_plan(evcache: EventCache, ev: NostrEvent, our_keypair: Keypair, func preload_image(url: URL) { if ImageCache.default.isCached(forKey: url.absoluteString) { - print("Preloaded image \(url.absoluteString) found in cache") + //print("Preloaded image \(url.absoluteString) found in cache") // looks like we already have it cached. no download needed return } - print("Preloading image \(url.absoluteString)") - + //print("Preloading image \(url.absoluteString)") + KingfisherManager.shared.retrieveImage(with: Kingfisher.ImageResource(downloadURL: url)) { val in - print("Preloaded image \(url.absoluteString)") + //print("Preloaded image \(url.absoluteString)") } } @@ -391,8 +391,8 @@ func preload_event(plan: PreloadPlan, state: DamusState) async { let profiles = state.profiles let our_keypair = state.keypair - print("Preloading event \(plan.event.content)") - + //print("Preloading event \(plan.event.content)") + if artifacts == nil && plan.load_artifacts { let arts = render_note_content(ev: plan.event, profiles: profiles, keypair: our_keypair) artifacts = arts From 1b1d4bd6d1e3b53c2986dfbee4cad53b9e992d64 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 18:22:15 -0700 Subject: [PATCH 074/111] perf: use plain images for actionbar buttons The action bar is really slow to render for some reason, start removing stuff --- damus/Views/ActionBar/EventActionBar.swift | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift index f5b453a928..08fbd92828 100644 --- a/damus/Views/ActionBar/EventActionBar.swift +++ b/damus/Views/ActionBar/EventActionBar.swift @@ -149,14 +149,15 @@ struct EventActionBar: View { func EventActionButton(img: String, col: Color?, action: @escaping () -> ()) -> some View { - Button(action: action) { - Image(img) - .resizable() - .foregroundColor(col == nil ? Color.gray : col!) - .font(.footnote.weight(.medium)) - .aspectRatio(contentMode: .fit) - .frame(width: 20, height: 20) - } + Image(img) + .resizable() + .foregroundColor(col == nil ? Color.gray : col!) + .font(.footnote.weight(.medium)) + .aspectRatio(contentMode: .fit) + .frame(width: 20, height: 20) + .onTapGesture { + action() + } } struct LikeButton: View { From 76a6dbc4060caa2ad33d84a3321fc47b24252c2f Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 18:22:48 -0700 Subject: [PATCH 075/111] perf: remove unused zstack on like button --- damus/Views/ActionBar/EventActionBar.swift | 36 +++++++++------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift index 08fbd92828..891217fe3d 100644 --- a/damus/Views/ActionBar/EventActionBar.swift +++ b/damus/Views/ActionBar/EventActionBar.swift @@ -198,27 +198,6 @@ struct LikeButton: View { } var body: some View { - ZStack { - likeButton() - .accessibilityLabel(NSLocalizedString("Like", comment: "Accessibility Label for Like button")) - .rotationEffect(Angle(degrees: shouldAnimate ? rotationAngle : 0)) - .onReceive(self.timer) { _ in - shakaAnimationLogic() - } - .simultaneousGesture(longPressGesture()) - .highPriorityGesture(TapGesture().onEnded { - guard !isReactionsVisible else { return } - withAnimation(Animation.easeOut(duration: 0.15)) { - self.action(damus_state.settings.default_emoji_reaction) - shouldAnimate = true - amountOfAngleIncrease = 20.0 - } - }) - .overlay(reactionsOverlay()) - } - } - - func likeButton() -> some View { Group { if let liked_emoji { buildMaskView(for: liked_emoji) @@ -231,6 +210,21 @@ struct LikeButton: View { .foregroundColor(.gray) } } + .accessibilityLabel(NSLocalizedString("Like", comment: "Accessibility Label for Like button")) + .rotationEffect(Angle(degrees: shouldAnimate ? rotationAngle : 0)) + .onReceive(self.timer) { _ in + shakaAnimationLogic() + } + .simultaneousGesture(longPressGesture()) + .highPriorityGesture(TapGesture().onEnded { + guard !isReactionsVisible else { return } + withAnimation(Animation.easeOut(duration: 0.15)) { + self.action(damus_state.settings.default_emoji_reaction) + shouldAnimate = true + amountOfAngleIncrease = 20.0 + } + }) + .overlay(reactionsOverlay()) } func shakaAnimationLogic() { From 36acdf420ed0b9d49d55d9318e3adf9ab0ddfd27 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 18:23:30 -0700 Subject: [PATCH 076/111] perf: remove zstack on profile pictures helps a bit? I think? --- damus/Views/Profile/ProfilePicView.swift | 27 +++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/damus/Views/Profile/ProfilePicView.swift b/damus/Views/Profile/ProfilePicView.swift index a22f136f1c..6467cf9876 100644 --- a/damus/Views/Profile/ProfilePicView.swift +++ b/damus/Views/Profile/ProfilePicView.swift @@ -45,27 +45,24 @@ struct InnerProfilePicView: View { } var body: some View { - ZStack { - Color(uiColor: .systemBackground) - - KFAnimatedImage(url) - .imageContext(.pfp, disable_animation: disable_animation) - .onFailure(fallbackUrl: fallbackUrl, cacheKey: url?.absoluteString) - .cancelOnDisappear(true) - .configure { view in - view.framePreloadCount = 3 - } - .placeholder { _ in - Placeholder - } - .scaledToFill() - } + KFAnimatedImage(url) + .imageContext(.pfp, disable_animation: disable_animation) + .onFailure(fallbackUrl: fallbackUrl, cacheKey: url?.absoluteString) + .cancelOnDisappear(true) + .configure { view in + view.framePreloadCount = 3 + } + .placeholder { _ in + Placeholder + } + .scaledToFill() .frame(width: size, height: size) .clipShape(Circle()) .overlay(Circle().stroke(highlight_color(highlight), lineWidth: pfp_line_width(highlight))) } } + struct ProfilePicView: View { let pubkey: Pubkey let size: CGFloat From 5b901656f3db4762aa1021250fa0b0dfdfd3963f Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 16:11:39 -0700 Subject: [PATCH 077/111] perf: fix weird lag when switching timelines --- damus/Util/Keys.swift | 8 ++++---- damus/Views/Settings/KeySettingsView.swift | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/damus/Util/Keys.swift b/damus/Util/Keys.swift index 1fc33ba060..13313de6b2 100644 --- a/damus/Util/Keys.swift +++ b/damus/Util/Keys.swift @@ -30,8 +30,8 @@ struct FullKeypair: Equatable { struct Keypair { let pubkey: Pubkey let privkey: Privkey? - let pubkey_bech32: String - let privkey_bech32: String? + //let pubkey_bech32: String + //let privkey_bech32: String? static var empty: Keypair { Keypair(pubkey: .empty, privkey: nil) @@ -52,8 +52,8 @@ struct Keypair { init(pubkey: Pubkey, privkey: Privkey?) { self.pubkey = pubkey self.privkey = privkey - self.pubkey_bech32 = pubkey.npub - self.privkey_bech32 = privkey?.nsec + //self.pubkey_bech32 = pubkey.npub + //self.privkey_bech32 = privkey?.nsec } } diff --git a/damus/Views/Settings/KeySettingsView.swift b/damus/Views/Settings/KeySettingsView.swift index 315f83769e..22650c477a 100644 --- a/damus/Views/Settings/KeySettingsView.swift +++ b/damus/Views/Settings/KeySettingsView.swift @@ -20,7 +20,7 @@ struct KeySettingsView: View { @Environment(\.dismiss) var dismiss init(keypair: Keypair) { - _privkey = State(initialValue: keypair.privkey_bech32 ?? "") + _privkey = State(initialValue: keypair.privkey?.nsec ?? "") self.keypair = keypair } @@ -40,7 +40,7 @@ struct KeySettingsView: View { func CopyButton(is_pk: Bool) -> some View { return Button(action: { let copyKey = { - UIPasteboard.general.string = is_pk ? self.keypair.pubkey_bech32 : self.privkey + UIPasteboard.general.string = is_pk ? self.keypair.pubkey.npub : self.privkey self.privkey_copied = !is_pk self.pubkey_copied = is_pk @@ -74,14 +74,14 @@ struct KeySettingsView: View { Form { Section(NSLocalizedString("Public Account ID", comment: "Section title for the user's public account ID.")) { HStack { - Text(keypair.pubkey_bech32) - + Text(keypair.pubkey.npub) + CopyButton(is_pk: true) } .clipShape(RoundedRectangle(cornerRadius: 5)) } - if let sec = keypair.privkey_bech32 { + if let sec = keypair.privkey?.nsec { Section(NSLocalizedString("Secret Account Login Key", comment: "Section title for user's secret account login key.")) { HStack { if show_privkey == false || !has_authenticated_locally { From c13f29e98cd1092a295330832694e88d5dcf111d Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 18:21:55 -0700 Subject: [PATCH 078/111] router: hash bytes for a quick sanity check probably a no-op --- damus/Util/Router.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/damus/Util/Router.swift b/damus/Util/Router.swift index b9a31aabce..407f549bb6 100644 --- a/damus/Util/Router.swift +++ b/damus/Util/Router.swift @@ -125,10 +125,10 @@ enum Route: Hashable { switch self { case .ProfileByKey(let pubkey): hasher.combine("profilebykey") - hasher.combine(pubkey) + hasher.combine(pubkey.id.bytes) case .Profile(let profile, _): hasher.combine("profile") - hasher.combine(profile.pubkey) + hasher.combine(profile.pubkey.id.bytes) case .Followers: hasher.combine("followers") case .Relay(let relay, _): From e30d38e69fb2b0a411ed27949b9e213fd854bbd2 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 18:23:07 -0700 Subject: [PATCH 079/111] router: use tap gestures instead of nav links I was hoping this would fix something but it did not --- damus/Views/Events/EventProfile.swift | 7 +++---- damus/Views/Profile/MaybeAnonPfpView.swift | 7 ++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/damus/Views/Events/EventProfile.swift b/damus/Views/Events/EventProfile.swift index 7b2fe4ef68..fb55dd172d 100644 --- a/damus/Views/Events/EventProfile.swift +++ b/damus/Views/Events/EventProfile.swift @@ -38,11 +38,10 @@ struct EventProfile: View { var body: some View { HStack(alignment: .center) { - VStack { - NavigationLink(value: Route.ProfileByKey(pubkey: pubkey)) { - ProfilePicView(pubkey: pubkey, size: pfp_size, highlight: .none, profiles: damus_state.profiles, disable_animation: disable_animation) + ProfilePicView(pubkey: pubkey, size: pfp_size, highlight: .none, profiles: damus_state.profiles, disable_animation: disable_animation) + .onTapGesture { + damus_state.nav.push(route: .ProfileByKey(pubkey: pubkey)) } - } VStack(alignment: .leading) { EventProfileName(pubkey: pubkey, profile: profile, damus: damus_state, size: size) diff --git a/damus/Views/Profile/MaybeAnonPfpView.swift b/damus/Views/Profile/MaybeAnonPfpView.swift index 5a2400a7f8..717069fdb7 100644 --- a/damus/Views/Profile/MaybeAnonPfpView.swift +++ b/damus/Views/Profile/MaybeAnonPfpView.swift @@ -28,9 +28,10 @@ struct MaybeAnonPfpView: View { .font(.largeTitle) .frame(width: size, height: size) } else { - NavigationLink(value: Route.ProfileByKey(pubkey: pubkey)) { - ProfilePicView(pubkey: pubkey, size: size, highlight: .none, profiles: state.profiles, disable_animation: state.settings.disable_animation) - } + ProfilePicView(pubkey: pubkey, size: size, highlight: .none, profiles: state.profiles, disable_animation: state.settings.disable_animation) + .onTapGesture { + state.nav.push(route: Route.ProfileByKey(pubkey: pubkey)) + } } } } From 8dad8e67033b6bbe04bc1f11048f2ebab107b85b Mon Sep 17 00:00:00 2001 From: Jericho Hasselbush Date: Sat, 9 Sep 2023 17:08:27 -0400 Subject: [PATCH 080/111] posting: fix issue with username and multiple emojis Fixes issue where username with multiple emojis would place cursor in strange position. Now properly moves the cursor to space past the multiple emoji user name. Any amount would be great. Not a complex issue to fix! Tipjar: lnbc1pj0eddtpp5km07jgrfm47nfswqqp33ngv374gzad2hshkra7zm3l0cmpusnp3qdqqcqzzsxqyz5vqsp5rklkzj9upf32z3c3nmc9xg4pdlz5p5mp3s332ygefexf79tq8ucs9qyyssqxfh4kz3sg9zczsnj49w23aw35z87jwyx9m5su8kkyxlspyjk4ajy7vhxuw2rzw4lz8vfutfakm2rggvpzhzs9ehfus4nl683dl99f4sqgm9zkq Changelog-Fixed: Fixes issue where username with multiple emojis would place cursor in strange position. Signed-off-by: Jericho Hasselbush Signed-off-by: William Casarin --- damus/Views/Posting/UserSearch.swift | 2 +- damusTests/UserSearchAppendTests.swift | 54 ++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 damusTests/UserSearchAppendTests.swift diff --git a/damus/Views/Posting/UserSearch.swift b/damus/Views/Posting/UserSearch.swift index f39be6012b..7de3b228d8 100644 --- a/damus/Views/Posting/UserSearch.swift +++ b/damus/Views/Posting/UserSearch.swift @@ -47,7 +47,7 @@ struct UserSearch: View { tagModel.diff = appended.tag.length - wordRange.length focusWordAttributes = (nil, nil) - newCursorIndex = wordRange.location + appended.tag.string.count + newCursorIndex = wordRange.location + appended.tag.length } var body: some View { diff --git a/damusTests/UserSearchAppendTests.swift b/damusTests/UserSearchAppendTests.swift new file mode 100644 index 0000000000..897b7d5ef6 --- /dev/null +++ b/damusTests/UserSearchAppendTests.swift @@ -0,0 +1,54 @@ +// +// Created by Jericho Hasselbush on 9/9/23. +// + + +// Test fix for https://github.com/damus-io/damus/issues/1525 +// Only change in damus source is in UserSearch.swift +// UserSearch.appendUserTag + +import XCTest +@testable import damus + +final class UserSearchAppendTests: XCTestCase { + func testCursorShouldBeAtEndOfEmoji() throws { + let simpleTag = NSMutableAttributedString("@JB55") + let emojiTag = NSMutableAttributedString("@BTCapsule 🏴🧡") + let post = NSMutableAttributedString("A Post") + + var cursorIndex: Int = 0 + appendUserTag(withTag: simpleTag, post: post, word_range: .init(location: 0, length: 0), newCursorIndex: &cursorIndex, spy: simulatedCursor ) + XCTAssertEqual(cursorIndex, simpleTag.length + 1) // +1 for past end of tag + cursorIndex = 0 + appendUserTag(withTag: emojiTag, post: post, word_range: .init(location: 0, length: 0), newCursorIndex: &cursorIndex, spy: simulatedCursor) + XCTAssertEqual(cursorIndex, emojiTag.length + 1) // +1 for past end of tag + } +} + +typealias CursorSpy = (Int, NSMutableAttributedString) -> Void + +var simulatedCursor: CursorSpy = { cursorIndex, tag in + let tagWithSimulatedCursor = NSMutableAttributedString(attributedString: tag) + if tagWithSimulatedCursor.length < cursorIndex { + tagWithSimulatedCursor.append(.init(string: "|")) + } else { + tagWithSimulatedCursor.insert(.init(string: "|"), at: cursorIndex) + } + print(tagWithSimulatedCursor.string) +} + +func appendUserTag(withTag tag: NSMutableAttributedString, + post: NSMutableAttributedString, + word_range: NSRange, + newCursorIndex: inout Int, + spy: CursorSpy = { _, _ in }) { + let appended = append_user_tag(tag: tag, post: post, word_range: word_range) + + // faulty call +// newCursorIndex = word_range.location + appended.tag.string.count + + // good call + newCursorIndex = word_range.location + appended.tag.length + + spy(newCursorIndex, tag) +} From 08035945535d94c8efbd136c97a92a85361836af Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Sun, 10 Sep 2023 15:01:07 -0500 Subject: [PATCH 081/111] ui: make ImageView's tab indicator dots tappable Changelog-Changed: Make carousel tab dots tappable Signed-off-by: Bryan Montz Signed-off-by: William Casarin --- damus/Views/Images/ImageView.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/damus/Views/Images/ImageView.swift b/damus/Views/Images/ImageView.swift index 850dc583dd..a2cea82f4e 100644 --- a/damus/Views/Images/ImageView.swift +++ b/damus/Views/Images/ImageView.swift @@ -24,6 +24,9 @@ struct ImageView: View { Capsule() .fill(index == selectedIndex ? Color(UIColor.label) : Color.secondary) .frame(width: 7, height: 7) + .onTapGesture { + selectedIndex = index + } } } .padding() From 9bac83352b66da2598518dbdcd605c7d037dda2e Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Sun, 10 Sep 2023 15:01:08 -0500 Subject: [PATCH 082/111] ui: improve bottom spacing for ImageView's tab indicator dots Signed-off-by: Bryan Montz Signed-off-by: William Casarin --- damus/Views/Images/ImageView.swift | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/damus/Views/Images/ImageView.swift b/damus/Views/Images/ImageView.swift index a2cea82f4e..862666fb37 100644 --- a/damus/Views/Images/ImageView.swift +++ b/damus/Views/Images/ImageView.swift @@ -63,18 +63,20 @@ struct ImageView: View { showMenu.toggle() }) .overlay( - VStack { - if showMenu { - NavDismissBarView() - Spacer() - - if (urls.count > 1) { - tabViewIndicator + GeometryReader { geo in + VStack { + if showMenu { + NavDismissBarView() + Spacer() + + if (urls.count > 1) { + tabViewIndicator + } } } + .animation(.easeInOut, value: showMenu) + .padding(.bottom, geo.safeAreaInsets.bottom == 0 ? 12 : 0) } - .animation(.easeInOut, value: showMenu) - .padding(.bottom, Theme.safeAreaInsets?.bottom) ) } } From e3ccf95780a3ae4b1e6980a7d8bbc53edfdfbc10 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 11 Sep 2023 13:59:43 -0700 Subject: [PATCH 083/111] ui: fix padding of username next to pfp on some views Changelog-Fixed: Fix padding of username next to pfp on some views --- damus/Views/Events/EventProfile.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/damus/Views/Events/EventProfile.swift b/damus/Views/Events/EventProfile.swift index fb55dd172d..18533ee971 100644 --- a/damus/Views/Events/EventProfile.swift +++ b/damus/Views/Events/EventProfile.swift @@ -37,13 +37,13 @@ struct EventProfile: View { } var body: some View { - HStack(alignment: .center) { + HStack(alignment: .center, spacing: 10) { ProfilePicView(pubkey: pubkey, size: pfp_size, highlight: .none, profiles: damus_state.profiles, disable_animation: disable_animation) .onTapGesture { damus_state.nav.push(route: .ProfileByKey(pubkey: pubkey)) } - VStack(alignment: .leading) { + VStack(alignment: .leading, spacing: 0) { EventProfileName(pubkey: pubkey, profile: profile, damus: damus_state, size: size) UserStatusView(status: damus_state.profiles.profile_data(pubkey).status, show_general: damus_state.settings.show_general_statuses, show_music: damus_state.settings.show_music_statuses) From 6a88ca27772d967bcb5ec5589f363bfa9e6b3450 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 11 Sep 2023 14:00:10 -0700 Subject: [PATCH 084/111] relays: fix crash in new RelayPicView --- damus/Views/Relays/RelayPicView.swift | 75 ++++++++++++++++----------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/damus/Views/Relays/RelayPicView.swift b/damus/Views/Relays/RelayPicView.swift index 40c51af8d5..f28a0620c7 100644 --- a/damus/Views/Relays/RelayPicView.swift +++ b/damus/Views/Relays/RelayPicView.swift @@ -9,43 +9,55 @@ import SwiftUI import Kingfisher import TLDExtract +struct FailedRelayImage: View { + let url: URL? + + var body: some View { + let abbrv = String(url?.hostname?.first?.uppercased() ?? "R") + Text("\(abbrv)") + .font(.system(size: 40, weight: .bold)) + } +} + struct InnerRelayPicView: View { let url: URL? let size: CGFloat let highlight: Highlight let disable_animation: Bool @State var failedImage: Bool = false - - var Placeholder: some View { - RoundedRectangle(cornerRadius: 15) - .frame(width: size, height: size) - .foregroundColor(DamusColors.adaptableGrey) - .overlay(RoundedRectangle(cornerRadius: 15).stroke(highlight_color(highlight), lineWidth: pfp_line_width(highlight))) - .padding(2) + + func Placeholder(url: URL?) -> some View { + ZStack { + RoundedRectangle(cornerRadius: 15) + .frame(width: size, height: size) + .foregroundColor(DamusColors.adaptableGrey) + .overlay(RoundedRectangle(cornerRadius: 15).stroke(highlight_color(highlight), lineWidth: pfp_line_width(highlight))) + .padding(2) + + FailedRelayImage(url: url) + } } var body: some View { ZStack { Color(uiColor: .secondarySystemBackground) - KFAnimatedImage(url) - .imageContext(.pfp, disable_animation: disable_animation) - .onFailure({ result in - failedImage = true - }) - .cancelOnDisappear(true) - .configure { view in - view.framePreloadCount = 3 - } - .placeholder { _ in - Placeholder - } - .scaledToFit() - - if failedImage { - let abbrv = String(url?.hostname?.first?.uppercased() ?? "R") - Text("\(abbrv)") - .font(.system(size: 40, weight: .bold)) + if let url { + KFAnimatedImage(url) + .imageContext(.pfp, disable_animation: disable_animation) + .onFailure { _ in + failedImage = true + } + .cancelOnDisappear(true) + .configure { view in + view.framePreloadCount = 3 + } + .placeholder { _ in + Placeholder(url: url) + } + .scaledToFit() + } else { + FailedRelayImage(url: nil) } } .frame(width: size, height: size) @@ -68,23 +80,24 @@ struct RelayPicView: View { self.highlight = highlight self.disable_animation = disable_animation } + + var relay_url: URL? { + get_relay_url(relay: relay, icon: icon) + } var body: some View { - InnerRelayPicView(url: get_relay_url(relay: relay, icon: icon), size: size, highlight: highlight, disable_animation: disable_animation) + InnerRelayPicView(url: relay_url, size: size, highlight: highlight, disable_animation: disable_animation) } } -func get_relay_url(relay: String, icon: String?) -> URL { +func get_relay_url(relay: String, icon: String?) -> URL? { let extractor = TLDExtract() var favicon = relay + "/favicon.ico" if let parseRelay: TLDResult = extractor.parse(relay) { favicon = "https://" + (parseRelay.rootDomain ?? relay) + "/favicon.ico" } let pic = icon ?? favicon - if let url = URL(string: pic) { - return url - } - return URL(string: "")! + return URL(string: pic) } struct RelayPicView_Previews: PreviewProvider { From 16edc3fe1308dff4ae2677feff49f8078c21b5a5 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 11 Sep 2023 14:26:40 -0700 Subject: [PATCH 085/111] relays: bouncy edit animation --- damus/Views/Relays/RelayConfigView.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/damus/Views/Relays/RelayConfigView.swift b/damus/Views/Relays/RelayConfigView.swift index 9e9ad96460..a00ac7de4b 100644 --- a/damus/Views/Relays/RelayConfigView.swift +++ b/damus/Views/Relays/RelayConfigView.swift @@ -157,11 +157,15 @@ struct RelayConfigView: View { if state.keypair.privkey != nil { if showActionButtons { Button("Done") { - showActionButtons.toggle() + withAnimation(.bouncy) { + showActionButtons.toggle() + } } } else { Button("Edit") { - showActionButtons.toggle() + withAnimation(.bouncy) { + showActionButtons.toggle() + } } } } From 1fc5ceff3be93e0b8774c363f367f34dbcf9d151 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 13 Sep 2023 05:39:45 -0700 Subject: [PATCH 086/111] relays: fix withAnimation on older versions Maybe this is an iOS17 thing? --- damus/Views/Relays/RelayConfigView.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/damus/Views/Relays/RelayConfigView.swift b/damus/Views/Relays/RelayConfigView.swift index a00ac7de4b..7e5452b55f 100644 --- a/damus/Views/Relays/RelayConfigView.swift +++ b/damus/Views/Relays/RelayConfigView.swift @@ -157,13 +157,13 @@ struct RelayConfigView: View { if state.keypair.privkey != nil { if showActionButtons { Button("Done") { - withAnimation(.bouncy) { + withAnimation { showActionButtons.toggle() } } } else { Button("Edit") { - withAnimation(.bouncy) { + withAnimation { showActionButtons.toggle() } } From b1e0a6210987da97f0b3c66ce58ef85b6cb3c3d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=99Aquino?= Date: Wed, 13 Sep 2023 18:44:42 +0000 Subject: [PATCH 087/111] nwc: fix parsing issue with NIP-47 compliant NWC urls without double-slashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: https://github.com/damus-io/damus/issues/1547 Changelog-Fixed: Fix parsing issue with NIP-47 compliant NWC urls without double-slashes Signed-off-by: Daniel D’Aquino Reviewed-by: William Casarin Signed-off-by: William Casarin --- damus/Util/WalletConnect.swift | 10 +++++----- damusTests/WalletConnectTests.swift | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/damus/Util/WalletConnect.swift b/damus/Util/WalletConnect.swift index 57cd9766cf..7997298735 100644 --- a/damus/Util/WalletConnect.swift +++ b/damus/Util/WalletConnect.swift @@ -36,11 +36,11 @@ struct WalletConnectURL: Equatable { } init?(str: String) { - guard let url = URL(string: str), - url.scheme == "nostrwalletconnect" || url.scheme == "nostr+walletconnect", - let pkhost = url.host, - let pubkey = hex_decode_pubkey(pkhost), - let components = URLComponents(url: url, resolvingAgainstBaseURL: true), + guard let components = URLComponents(string: str), + components.scheme == "nostrwalletconnect" || components.scheme == "nostr+walletconnect", + // The line below provides flexibility for both `nostrwalletconnect://` (non-compliant, but commonly used) and `nostrwalletconnect:` (NIP-47 compliant) formats + let encoded_pubkey = components.path == "" ? components.host : components.path, + let pubkey = hex_decode_pubkey(encoded_pubkey), let items = components.queryItems, let relay = items.first(where: { qi in qi.name == "relay" })?.value, let relay_url = RelayURL(relay), diff --git a/damusTests/WalletConnectTests.swift b/damusTests/WalletConnectTests.swift index c8fe43db92..f7c31228b7 100644 --- a/damusTests/WalletConnectTests.swift +++ b/damusTests/WalletConnectTests.swift @@ -36,6 +36,12 @@ final class WalletConnectTests: XCTestCase { } func testDoesNWCParse() { + // Test an NWC url format which is not technically NIP-47 and RFC 3986 compliant, but still commonly used (by Alby, for example) + // See Github issue #1547 for details on why this URL is non-compliant + // This test URL also features: + // - `nostrwalletconnect` scheme + // - A non-url-encoded relay parameter + // - lud16 parameter let pk = Pubkey(hex: "9d088f4760422443d4699b485e2ac66e565a2f5da1198c55ddc5679458e3f67a")! let sec = Privkey(hex: "ff2eefd57196d42089e1b42acc39916d7ecac52e0625bd70597bbd5be14aff18")! let relay = "wss://relay.getalby.com/v1" @@ -51,6 +57,26 @@ final class WalletConnectTests: XCTestCase { XCTAssertEqual(url.keypair.pubkey, privkey_to_pubkey(privkey: sec)) XCTAssertEqual(url.relay.id, relay) XCTAssertEqual(url.lud16, "jb55@jb55.com") + + // Test an NWC url format which is NIP-47 and RFC 3986 compliant + // This test URL also features: + // - `nostr+walletconnect` scheme + // - A url-encoded relay parameter + // - No lud16 parameter + let pk_2 = Pubkey(hex: "b889ff5b1513b641e2a139f661a661364979c5beee91842f8f0ef42ab558e9d4")! + let sec_2 = Privkey(hex: "71a8c14c1407c113601079c4302dab36460f0ccd0ad506f1f2dc73b5100e4f3c")! + let relay_2 = "wss://relay.damus.io" + let str_2 = "nostr+walletconnect:b889ff5b1513b641e2a139f661a661364979c5beee91842f8f0ef42ab558e9d4?relay=wss%3A%2F%2Frelay.damus.io&secret=71a8c14c1407c113601079c4302dab36460f0ccd0ad506f1f2dc73b5100e4f3c" + + let url_2 = WalletConnectURL(str: str_2) + XCTAssertNotNil(url_2) + guard let url_2 else { + return + } + XCTAssertEqual(url_2.pubkey, pk_2) + XCTAssertEqual(url_2.keypair.privkey, sec_2) + XCTAssertEqual(url_2.keypair.pubkey, privkey_to_pubkey(privkey: sec_2)) + XCTAssertEqual(url_2.relay.id, relay_2) } func testNWCEphemeralRelay() { From c4a9f2fdb27c4c337d7580a0a05d44082f025049 Mon Sep 17 00:00:00 2001 From: Jericho Hasselbush Date: Wed, 13 Sep 2023 23:05:09 -0400 Subject: [PATCH 088/111] ui: hold tap to preview status URL Applied a WKWebkitView inside a .contextMenu to show preview status for URL links in user status messages. Closes: https://github.com/damus-io/damus/issues/1523 Changelog-Added: Hold tap to preview status URL Signed-off-by: Jericho Hasselbush Signed-off-by: William Casarin --- damus/Components/Status/UserStatusView.swift | 24 +++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/damus/Components/Status/UserStatusView.swift b/damus/Components/Status/UserStatusView.swift index a1444d4770..8bea043a3a 100644 --- a/damus/Components/Status/UserStatusView.swift +++ b/damus/Components/Status/UserStatusView.swift @@ -7,7 +7,7 @@ import SwiftUI import MediaPlayer - +import WebKit struct UserStatusView: View { @ObservedObject var status: UserStatusModel @@ -35,6 +35,15 @@ struct UserStatusView: View { openURL(url) } } + .contextMenu( + menuItems: { + if let url = st.url { + Button(url.absoluteString, action: { openURL(url) }) } + }, preview: { + if let url = st.url { + URLPreview(url: url) + } + }) } var body: some View { @@ -49,6 +58,19 @@ struct UserStatusView: View { } } + + struct URLPreview: UIViewRepresentable { + var url: URL + + func makeUIView(context: Context) -> WKWebView { + return WKWebView() + } + + func updateUIView(_ wkView: WKWebView, context: Context) { + let request = URLRequest(url: url) + wkView.load(request) + } + } } /* From 510432bb98a62b3509b6d152cabd6995b66521bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=99Aquino?= Date: Wed, 13 Sep 2023 21:07:52 +0000 Subject: [PATCH 089/111] ui: make blurred videos viewable by allowing blur to disappear once tapped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: https://github.com/damus-io/damus/issues/1247 Changelog-Fixed: Make blurred videos viewable by allowing blur to disappear once tapped Signed-off-by: Daniel D’Aquino Signed-off-by: William Casarin --- damus/Views/NoteContentView.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index 633a4986fc..00cb7e911c 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -26,7 +26,7 @@ struct NoteContentView: View { let damus_state: DamusState let event: NostrEvent - let show_images: Bool + @State var show_images: Bool let size: EventViewKind let preview_height: CGFloat? let options: EventViewOptions @@ -139,7 +139,9 @@ struct NoteContentView: View { ZStack { ImageCarousel(state: damus_state, evid: event.id, urls: artifacts.media) Blur() - .disabled(true) + .onTapGesture { + show_images = true + } } //.cornerRadius(10) } From 617dee3e6bd47aa94b9db5d5908e79754608b29b Mon Sep 17 00:00:00 2001 From: Jon Marrs Date: Wed, 13 Sep 2023 10:40:30 -0700 Subject: [PATCH 090/111] damus-c: remove UTF-8 punctuation from hashtags Check for UTF-8 punctuation (such as ellipsis) in addition to regular punctuation in hashtags. Closes: https://github.com/damus-io/damus/issues/1518 Closes: https://github.com/damus-io/damus/pull/1546 Signed-off-by: William Casarin --- damus-c/cursor.h | 190 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 167 insertions(+), 23 deletions(-) diff --git a/damus-c/cursor.h b/damus-c/cursor.h index 3ff5dd6289..d5b8725a00 100644 --- a/damus-c/cursor.h +++ b/damus-c/cursor.h @@ -107,6 +107,36 @@ static inline int pull_byte(struct cursor *cursor, u8 *c) return 1; } +static inline int parse_byte(struct cursor *cursor, u8 *c) +{ + if (unlikely(cursor->p >= cursor->end)) + return 0; + + *c = *cursor->p; + //cursor->p++; + + return 1; +} + +static inline int parse_char(struct cursor *cur, char c) { + if (cur->p >= cur->end) + return 0; + + if (*cur->p == c) { + cur->p++; + return 1; + } + + return 0; +} + +static inline int peek_char(struct cursor *cur, int ind) { + if ((cur->p + ind < cur->start) || (cur->p + ind >= cur->end)) + return -1; + + return *(cur->p + ind); +} + static inline int cursor_pull_c_str(struct cursor *cursor, const char **str) { *str = (const char*)cursor->p; @@ -435,12 +465,124 @@ static inline int is_whitespace(char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; } +static inline int is_underscore(char c) { + return c == '_'; +} + static inline int is_utf8_byte(u8 c) { return c & 0x80; } -static inline int is_right_boundary(char c) { - return is_whitespace(c) || ispunct(c); +static inline int parse_utf8_char(struct cursor *cursor, unsigned int *code_point, unsigned int *utf8_length) +{ + u8 first_byte; + if (!parse_byte(cursor, &first_byte)) + return 0; // Not enough data + + // Determine the number of bytes in this UTF-8 character + int remaining_bytes = 0; + if (first_byte < 0x80) { + *code_point = first_byte; + return 1; + } else if ((first_byte & 0xE0) == 0xC0) { + remaining_bytes = 1; + *utf8_length = remaining_bytes + 1; + *code_point = first_byte & 0x1F; + } else if ((first_byte & 0xF0) == 0xE0) { + remaining_bytes = 2; + *utf8_length = remaining_bytes + 1; + *code_point = first_byte & 0x0F; + } else if ((first_byte & 0xF8) == 0xF0) { + remaining_bytes = 3; + *utf8_length = remaining_bytes + 1; + *code_point = first_byte & 0x07; + } else { + remaining_bytes = 0; + *utf8_length = 1; // Assume 1 byte length for unrecognized UTF-8 characters + // TODO: We need to gracefully handle unrecognized UTF-8 characters + printf("Invalid UTF-8 byte: %x\n", *code_point); + *code_point = ((first_byte & 0xF0) << 6); // Prevent testing as punctuation + return 0; // Invalid first byte + } + + // Peek at remaining bytes + for (int i = 0; i < remaining_bytes; ++i) { + signed char next_byte; + if ((next_byte = peek_char(cursor, i+1)) == -1) { + *utf8_length = 1; + return 0; // Not enough data + } + + // Debugging lines + //printf("Cursor: %s\n", cursor->p); + //printf("Codepoint: %x\n", *code_point); + //printf("Codepoint <<6: %x\n", ((*code_point << 6) | (next_byte & 0x3F))); + //printf("Remaining bytes: %x\n", remaining_bytes); + //printf("First byte: %x\n", first_byte); + //printf("Next byte: %x\n", next_byte); + //printf("Bitwise AND result: %x\n", (next_byte & 0xC0)); + + if ((next_byte & 0xC0) != 0x80) { + *utf8_length = 1; + return 0; // Invalid byte in sequence + } + + *code_point = (*code_point << 6) | (next_byte & 0x3F); + } + + return 1; +} + +/** + * Checks if a given Unicode code point is a punctuation character + * + * @param codepoint The Unicode code point to check. @return true if the + * code point is a punctuation character, false otherwise. + */ +static inline int is_punctuation(unsigned int codepoint) { + + // Check for underscore (underscore is not treated as punctuation) + if (is_underscore(codepoint)) + return 0; + + // Check for ASCII punctuation + if (ispunct(codepoint)) + return 1; + + // Check for Unicode punctuation exceptions (punctuation allowed in hashtags) + if (codepoint == 0x301C || codepoint == 0xFF5E) // Japanese Wave Dash / Tilde + return 0; + + // Check for Unicode punctuation + // NOTE: We may need to adjust the codepoint ranges in the future, + // to include/exclude certain types of Unicode characters in hashtags. + // Unicode Blocks Reference: https://www.compart.com/en/unicode/block + return ( + // Latin-1 Supplement No-Break Space (NBSP): U+00A0 + (codepoint == 0x00A0) || + + // Latin-1 Supplement Punctuation: U+00A1 to U+00BF + (codepoint >= 0x00A1 && codepoint <= 0x00BF) || + + // General Punctuation: U+2000 to U+206F + (codepoint >= 0x2000 && codepoint <= 0x206F) || + + // Currency Symbols: U+20A0 to U+20CF + (codepoint >= 0x20A0 && codepoint <= 0x20CF) || + + // Supplemental Punctuation: U+2E00 to U+2E7F + (codepoint >= 0x2E00 && codepoint <= 0x2E7F) || + + // CJK Symbols and Punctuation: U+3000 to U+303F + (codepoint >= 0x3000 && codepoint <= 0x303F) || + + // Ideographic Description Characters: U+2FF0 to U+2FFF + (codepoint >= 0x2FF0 && codepoint <= 0x2FFF) + ); +} + +static inline int is_right_boundary(int c) { + return is_whitespace(c) || is_punctuation(c); } static inline int is_left_boundary(char c) { @@ -452,15 +594,36 @@ static inline int is_alphanumeric(char c) { } static inline int consume_until_boundary(struct cursor *cur) { - char c; + unsigned int c; + unsigned int char_length = 1; + unsigned int *utf8_char_length = &char_length; while (cur->p < cur->end) { c = *cur->p; + *utf8_char_length = 1; + + if (is_whitespace(c)) + return 1; + + // Need to check for UTF-8 characters, which can be multiple bytes long + if (is_utf8_byte(c)) { + if (!parse_utf8_char(cur, &c, utf8_char_length)) { + if (!is_right_boundary(c)){ + // TODO: We should work towards handling all UTF-8 characters. + printf("Invalid UTF-8 code point: %x\n", c); + } + } + } + if (is_right_boundary(c)) return 1; - cur->p++; + // Need to use a variable character byte length for UTF-8 (2-4 bytes) + if (cur->p + *utf8_char_length <= cur->end) + cur->p += *utf8_char_length; + else + cur->p++; } return 1; @@ -500,23 +663,4 @@ static inline int consume_until_non_alphanumeric(struct cursor *cur, int or_end) return or_end; } -static inline int parse_char(struct cursor *cur, char c) { - if (cur->p >= cur->end) - return 0; - - if (*cur->p == c) { - cur->p++; - return 1; - } - - return 0; -} - -static inline int peek_char(struct cursor *cur, int ind) { - if ((cur->p + ind < cur->start) || (cur->p + ind >= cur->end)) - return -1; - - return *(cur->p + ind); -} - #endif From aa4ecc213917a9088963e0d6ba719b54bccffbe8 Mon Sep 17 00:00:00 2001 From: Jon Marrs Date: Wed, 13 Sep 2023 10:47:29 -0700 Subject: [PATCH 091/111] test: add test cases for ASCII and UTF-8 characters in hashtags Closes: https://github.com/damus-io/damus/pull/1546 Signed-off-by: William Casarin --- damusTests/HashtagTests.swift | 498 ++++++++++++++++++++++++++++++++-- 1 file changed, 476 insertions(+), 22 deletions(-) diff --git a/damusTests/HashtagTests.swift b/damusTests/HashtagTests.swift index ffdaac67ab..5297340ac3 100644 --- a/damusTests/HashtagTests.swift +++ b/damusTests/HashtagTests.swift @@ -3,34 +3,490 @@ // damusTests // // Created by William Casarin on 2023-07-11. +// Modified by Jon Marrs on 2023-09-12. // import XCTest @testable import damus - final class HashtagTests: XCTestCase { + + // Basic hashtag tests + func testParseHashtag() { let parsed = parse_note_content(content: .content("some hashtag #bitcoin derp",nil)).blocks XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].asText, "some hashtag ") XCTAssertEqual(parsed[1].asHashtag, "bitcoin") XCTAssertEqual(parsed[2].asText, " derp") } + func testParseHashtagEnd() { + let parsed = parse_note_content(content: .content("some hashtag #bitcoin",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 2) + XCTAssertEqual(parsed[0].asText, "some hashtag ") + XCTAssertEqual(parsed[1].asHashtag, "bitcoin") + } + + //------------------------------------------------------------ + // Test ASCII + //------------------------------------------------------------ + + // Test ASCII punctuation exceptions (punctuation that is allowed in hashtags) + + // Underscores are allowed in hashtags + func testHashtagWithUnderscore() { + let parsed = parse_note_content(content: .content("the #under_score is allowed in hashtags",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "under_score") + XCTAssertEqual(parsed[2].asText, " is allowed in hashtags") + } + + // Test ASCII punctuation (not allowed in hashtags) + func testHashtagWithComma() { - let parsed = parse_note_content(content: .content("some hashtag #bitcoin, cool",nil)).blocks + let parsed = parse_note_content(content: .content("the #comma, is not allowed",nil)).blocks XCTAssertNotNil(parsed) XCTAssertEqual(parsed.count, 3) - XCTAssertEqual(parsed[0].asText, "some hashtag ") + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "comma") + XCTAssertEqual(parsed[2].asText, ", is not allowed") + } + + func testHashtagWithPeriod() { + let parsed = parse_note_content(content: .content("the #period. is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "period") + XCTAssertEqual(parsed[2].asText, ". is not allowed") + } + + func testHashtagWithQuestionMark() { + let parsed = parse_note_content(content: .content("the #question?mark is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "question") + XCTAssertEqual(parsed[2].asText, "?mark is not allowed") + } + + func testHashtagWithGraveAccent() { + let parsed = parse_note_content(content: .content("the #grave`accent is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "grave") + XCTAssertEqual(parsed[2].asText, "`accent is not allowed") + } + + func testHashtagWithTilde() { + let parsed = parse_note_content(content: .content("the #tilde~ is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "tilde") + XCTAssertEqual(parsed[2].asText, "~ is not allowed") + } + + func testHashtagWithExclamationPoint() { + let parsed = parse_note_content(content: .content("the #exclamation!point is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "exclamation") + XCTAssertEqual(parsed[2].asText, "!point is not allowed") + } + + func testHashtagWithAtSign() { + let parsed = parse_note_content(content: .content("the #at@sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "at") + XCTAssertEqual(parsed[2].asText, "@sign is not allowed") + } + + func testHashtagWithDollarSign() { + let parsed = parse_note_content(content: .content("the #dollar$sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "dollar") + XCTAssertEqual(parsed[2].asText, "$sign is not allowed") + } + + func testHashtagWithPercentSign() { + let parsed = parse_note_content(content: .content("the #percent%sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "percent") + XCTAssertEqual(parsed[2].asText, "%sign is not allowed") + } + + func testHashtagWithCaret() { + let parsed = parse_note_content(content: .content("the #caret^ is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "caret") + XCTAssertEqual(parsed[2].asText, "^ is not allowed") + } + + func testHashtagWithAmpersand() { + let parsed = parse_note_content(content: .content("the #ampersand& is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "ampersand") + XCTAssertEqual(parsed[2].asText, "& is not allowed") + } + + func testHashtagWithAsterisk() { + let parsed = parse_note_content(content: .content("the #asterisk* is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "asterisk") + XCTAssertEqual(parsed[2].asText, "* is not allowed") + } + + func testHashtagWithLeftParenthesis() { + let parsed = parse_note_content(content: .content("the #left(parenthesis is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "left") + XCTAssertEqual(parsed[2].asText, "(parenthesis is not allowed") + } + + func testHashtagWithRightParenthesis() { + let parsed = parse_note_content(content: .content("the #right)parenthesis is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "right") + XCTAssertEqual(parsed[2].asText, ")parenthesis is not allowed") + } + + func testHashtagWithDash() { + let parsed = parse_note_content(content: .content("the #dash- is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "dash") + XCTAssertEqual(parsed[2].asText, "- is not allowed") + } + + func testHashtagWithPlusSign() { + let parsed = parse_note_content(content: .content("the #plus+sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "plus") + XCTAssertEqual(parsed[2].asText, "+sign is not allowed") + } + + func testHashtagWithEqualsSign() { + let parsed = parse_note_content(content: .content("the #equals=sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "equals") + XCTAssertEqual(parsed[2].asText, "=sign is not allowed") + } + + func testHashtagWithLeftBracket() { + let parsed = parse_note_content(content: .content("the #left[bracket is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "left") + XCTAssertEqual(parsed[2].asText, "[bracket is not allowed") + } + + func testHashtagWithRightBracket() { + let parsed = parse_note_content(content: .content("the #right]bracket is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "right") + XCTAssertEqual(parsed[2].asText, "]bracket is not allowed") + } + + func testHashtagWithLeftBrace() { + let parsed = parse_note_content(content: .content("the #left{brace is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "left") + XCTAssertEqual(parsed[2].asText, "{brace is not allowed") + } + + func testHashtagWithRightBrace() { + let parsed = parse_note_content(content: .content("the #right}brace is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "right") + XCTAssertEqual(parsed[2].asText, "}brace is not allowed") + } + + func testHashtagWithBackslash() { + let parsed = parse_note_content(content: .content("the #back\\slash is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "back") + XCTAssertEqual(parsed[2].asText, "\\slash is not allowed") + } + + func testHashtagWithVerticalLine() { + let parsed = parse_note_content(content: .content("the #vertical|line is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "vertical") + XCTAssertEqual(parsed[2].asText, "|line is not allowed") + } + + func testHashtagWithSemicolon() { + let parsed = parse_note_content(content: .content("the #semicolon; is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "semicolon") + XCTAssertEqual(parsed[2].asText, "; is not allowed") + } + + func testHashtagWithColon() { + let parsed = parse_note_content(content: .content("the #colon: is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "colon") + XCTAssertEqual(parsed[2].asText, ": is not allowed") + } + + func testHashtagWithApostrophe() { + let parsed = parse_note_content(content: .content("the #apostrophe' is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "apostrophe") + XCTAssertEqual(parsed[2].asText, "' is not allowed") + } + + func testHashtagWithQuotationMark() { + let parsed = parse_note_content(content: .content("the #quotation\"mark is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "quotation") + XCTAssertEqual(parsed[2].asText, "\"mark is not allowed") + } + + func testHashtagWithLessThanSign() { + let parsed = parse_note_content(content: .content("the #lessthansign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "greaterthan") + XCTAssertEqual(parsed[2].asText, ">sign is not allowed") + } + + //------------------------------------------------------------ + // Test Unicode (UTF-8) + //------------------------------------------------------------ + + // Test UTF-8 Latin-1 Supplement Punctuation: U+00A1 to U+00BF (not allowed) + + // Test pound sign (£) (U+00A3) + func testHashtagWithPoundSign() { + let parsed = parse_note_content(content: .content("the #pound£sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "pound") + XCTAssertEqual(parsed[2].asText, "£sign is not allowed") + } + + // Test yen sign (¥) (U+00A5) + func testHashtagWithYenSign() { + let parsed = parse_note_content(content: .content("the #yen¥sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "yen") + XCTAssertEqual(parsed[2].asText, "¥sign is not allowed") + } + + // Test section sign (§) (U+00A7) + func testHashtagWithSectionSign() { + let parsed = parse_note_content(content: .content("the #section§sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "section") + XCTAssertEqual(parsed[2].asText, "§sign is not allowed") + } + + // Test plus-minus sign (±) (U+00B1) + func testHashtagWithPlusMinusSign() { + let parsed = parse_note_content(content: .content("the #plusminus±sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "plusminus") + XCTAssertEqual(parsed[2].asText, "±sign is not allowed") + } + + // Test inverted question mark (¿) (U+00BF) + func testHashtagWithInvertedQuestionMark() { + let parsed = parse_note_content(content: .content("the #invertedquestion¿mark is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "invertedquestion") + XCTAssertEqual(parsed[2].asText, "¿mark is not allowed") + } + + // Test UTF-8 Latin-1 Supplement Non-Punctuation: U+00C0 to U+00FF (allowed) + + // Test Latin small letter u with diaeresis (ü) (U+00FC) (allowed in hashtags) + func testHashtagWithAccents() { + let parsed = parse_note_content(content: .content("hello from #türkiye",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 2) + XCTAssertEqual(parsed[0].asText, "hello from ") + XCTAssertEqual(parsed[1].asHashtag, "türkiye") + } + + // Test UTF-8 General Punctuation: U+2000 to U+206F (not allowed in hashtags) + + // Test en dash (–) (U+2013) + func testHashtagWithEnDash() { + let parsed = parse_note_content(content: .content("the #en–dash is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "en") + XCTAssertEqual(parsed[2].asText, "–dash is not allowed") + } + + // Test em dash (—) (U+2014) + func testHashtagWithEmDash() { + let parsed = parse_note_content(content: .content("the #em—dash is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "em") + XCTAssertEqual(parsed[2].asText, "—dash is not allowed") + } + + // Test horizontal bar (―) (U+2015) + func testHashtagWithHorizontalBar() { + let parsed = parse_note_content(content: .content("the #horizontal―bar is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "horizontal") + XCTAssertEqual(parsed[2].asText, "―bar is not allowed") + } + + // Test horizontal ellipsis (…) (U+2026) + func testHashtagWithHorizontalEllipsis() { + let parsed = parse_note_content(content: .content("the #horizontal…ellipsis is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "horizontal") + XCTAssertEqual(parsed[2].asText, "…ellipsis is not allowed") + } + + // Test UTF-8 Currency Symbols: U+20A0 to U+20CF (not allowed in hashtags) + + // Test euro sign (€) (U+20AC) + func testHashtagWithEuroSign() { + let parsed = parse_note_content(content: .content("the #euro€sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") + XCTAssertEqual(parsed[1].asHashtag, "euro") + XCTAssertEqual(parsed[2].asText, "€sign is not allowed") + } + + // Test Bitcoin sign (₿) (U+20BF) + func testHashtagWithBitcoinSign() { + let parsed = parse_note_content(content: .content("the #bitcoin₿sign is not allowed",nil)).blocks + + XCTAssertNotNil(parsed) + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "the ") XCTAssertEqual(parsed[1].asHashtag, "bitcoin") - XCTAssertEqual(parsed[2].asText, ", cool") + XCTAssertEqual(parsed[2].asText, "₿sign is not allowed") } + // Test UTF-8 Miscellaneous Symbols: U+2600 to U+26FF (allowed in hashtags) + + // Emojis such as ☕️ (U+2615) are allowed in hashtags func testHashtagWithEmoji() { let content = "some hashtag #bitcoin☕️ cool" let parsed = parse_note_content(content: .content(content, nil)).blocks @@ -47,7 +503,10 @@ final class HashtagTests: XCTestCase { XCTAssertEqual(post_blocks[1].asHashtag, "bitcoin☕️") XCTAssertEqual(post_blocks[2].asText, " cool") } - + + // Test international Unicode (UTF-8) characters + + // Japanese: wave dash (〜) (U+301C) (allowed in hashtags) func testPowHashtag() { let content = "pow! #ぽわ〜" let parsed = parse_note_content(content: .content(content,nil)).blocks @@ -62,16 +521,9 @@ final class HashtagTests: XCTestCase { XCTAssertEqual(post_blocks[0].asText, "pow! ") XCTAssertEqual(post_blocks[1].asHashtag, "ぽわ〜") } - - func testHashtagWithAccents() { - let parsed = parse_note_content(content: .content("hello from #türkiye",nil)).blocks - - XCTAssertNotNil(parsed) - XCTAssertEqual(parsed.count, 2) - XCTAssertEqual(parsed[0].asText, "hello from ") - XCTAssertEqual(parsed[1].asHashtag, "türkiye") - } - + + // Hangul: Hangul Syllable Si (시) (U+C2DC) and + // Hangul Syllable Heom (험) (U+D5D8) (allowed in hashtags) func testHashtagWithNonLatinCharacters() { let parsed = parse_note_content(content: .content("this is a #시험 hope it works",nil)).blocks @@ -82,13 +534,15 @@ final class HashtagTests: XCTestCase { XCTAssertEqual(parsed[2].asText, " hope it works") } - func testParseHashtagEnd() { - let parsed = parse_note_content(content: .content("some hashtag #bitcoin",nil)).blocks - + // Japanese: fullwidth tilde (～) (U+FF5E) (allowed in hashtags) + func testHashtagWithFullwidthTilde() { + let parsed = parse_note_content(content: .content("pow! the fullwidth tilde #ぽわ～ is allowed in hashtags",nil)).blocks + XCTAssertNotNil(parsed) - XCTAssertEqual(parsed.count, 2) - XCTAssertEqual(parsed[0].asText, "some hashtag ") - XCTAssertEqual(parsed[1].asHashtag, "bitcoin") + XCTAssertEqual(parsed.count, 3) + XCTAssertEqual(parsed[0].asText, "pow! the fullwidth tilde ") + XCTAssertEqual(parsed[1].asHashtag, "ぽわ～") + XCTAssertEqual(parsed[2].asText, " is allowed in hashtags") } } From 01b8e43a6e4af48b591666631f588c6dc64205f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=99Aquino?= Date: Sat, 16 Sep 2023 03:32:56 +0000 Subject: [PATCH 092/111] compose: fix text wrapping issue when mentioning npub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: https://github.com/damus-io/damus/issues/1211 Changelog-Fixed: Fix text composer wrapping issue when mentioning npub Signed-off-by: Daniel D’Aquino Signed-off-by: William Casarin --- damus/Views/PostView.swift | 99 +++++++++++++++++++++------- damus/Views/Posting/UserSearch.swift | 12 +--- damus/Views/TextViewWrapper.swift | 51 ++++++++++++-- 3 files changed, 124 insertions(+), 38 deletions(-) diff --git a/damus/Views/PostView.swift b/damus/Views/PostView.swift index a484d2d42b..f82a3ffb96 100644 --- a/damus/Views/PostView.swift +++ b/damus/Views/PostView.swift @@ -14,6 +14,8 @@ enum NostrPostResult { } let POST_PLACEHOLDER = NSLocalizedString("Type your note here...", comment: "Text box prompt to ask user to type their note.") +let GHOST_CARET_VIEW_ID = "GhostCaret" +let DEBUG_SHOW_GHOST_CARET_VIEW: Bool = false class TagModel: ObservableObject { var diff = 0 @@ -54,7 +56,8 @@ struct PostView: View { @State var filtered_pubkeys: Set = [] @State var focusWordAttributes: (String?, NSRange?) = (nil, nil) @State var newCursorIndex: Int? - @State var postTextViewCanScroll: Bool = true + @State var caretRect: CGRect = CGRectNull + @State var textHeight: CGFloat? = nil @State var mediaToUpload: MediaUpload? = nil @@ -104,6 +107,16 @@ struct PostView: View { return is_post_empty || uploading_disabled } + // Returns a valid height for the text box, even when textHeight is not a number + func get_valid_text_height() -> CGFloat { + if let textHeight, textHeight.isFinite, textHeight > 0 { + return textHeight + } + else { + return 10 + } + } + var ImageButton: some View { Button(action: { attach_media = true @@ -201,11 +214,18 @@ struct PostView: View { var TextEntry: some View { ZStack(alignment: .topLeading) { - TextViewWrapper(attributedText: $post, postTextViewCanScroll: $postTextViewCanScroll, cursorIndex: newCursorIndex, getFocusWordForMention: { word, range in + TextViewWrapper(attributedText: $post, textHeight: $textHeight, cursorIndex: newCursorIndex, getFocusWordForMention: { word, range in focusWordAttributes = (word, range) self.newCursorIndex = nil }, updateCursorPosition: { newCursorIndex in self.newCursorIndex = newCursorIndex + }, onCaretRectChange: { uiView in + // When the caret position changes, we change the `caretRect` in our state, so that our ghost caret will follow our caret + if let selectedStartRange = uiView.selectedTextRange?.start { + DispatchQueue.main.async { + caretRect = uiView.caretRect(for: selectedStartRange) + } + } }) .environmentObject(tagModel) .focused($focus) @@ -213,6 +233,8 @@ struct PostView: View { .onChange(of: post) { p in post_changed(post: p, media: uploadedMedias) } + // Set a height based on the text content height, if it is available and valid + .frame(height: get_valid_text_height()) if post.string.isEmpty { Text(POST_PLACEHOLDER) @@ -292,25 +314,48 @@ struct PostView: View { } func Editor(deviceSize: GeometryProxy) -> some View { - VStack(alignment: .leading, spacing: 0) { - HStack(alignment: .top) { - ProfilePicView(pubkey: damus_state.pubkey, size: PFP_SIZE, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) - - TextEntry + HStack(alignment: .top, spacing: 0) { + if(caretRect != CGRectNull) { + GhostCaret } - .frame(height: deviceSize.size.height * multiply_factor) - .id("post") + VStack(alignment: .leading, spacing: 0) { + HStack(alignment: .top) { + ProfilePicView(pubkey: damus_state.pubkey, size: PFP_SIZE, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) + + TextEntry + } + .id("post") + + PVImageCarouselView(media: $uploadedMedias, deviceWidth: deviceSize.size.width) + .onChange(of: uploadedMedias) { media in + post_changed(post: post, media: media) + } - PVImageCarouselView(media: $uploadedMedias, deviceWidth: deviceSize.size.width) - .onChange(of: uploadedMedias) { media in - post_changed(post: post, media: media) + if case .quoting(let ev) = action { + BuilderEventView(damus: damus_state, event: ev) } - - if case .quoting(let ev) = action { - BuilderEventView(damus: damus_state, event: ev) } + .padding(.horizontal) } - .padding(.horizontal) + } + + // The GhostCaret is a vertical projection of the editor's caret that should sit beside the editor. + // The purpose of this view is create a reference point that we can scroll our ScrollView into + // This is necessary as a bridge to communicate between: + // - The UIKit-based UITextView (which has the caret position) + // - and the SwiftUI-based ScrollView/ScrollReader (where scrolling commands can only be done via the SwiftUI "ID" parameter + var GhostCaret: some View { + Rectangle() + .foregroundStyle(DEBUG_SHOW_GHOST_CARET_VIEW ? .cyan : .init(red: 0, green: 0, blue: 0, opacity: 0)) + .frame( + width: DEBUG_SHOW_GHOST_CARET_VIEW ? caretRect.width : 0, + height: caretRect.height) + // Use padding to vertically align our ghost caret with our actual text caret. + // Note: Programmatic scrolling cannot be done with the `.position` modifier. + // Experiments revealed that the scroller ignores the position modifier. + .padding(.top, caretRect.origin.y) + .id(GHOST_CARET_VIEW_ID) + .disabled(true) } func fill_target_content(target: PostTarget) { @@ -332,26 +377,36 @@ struct PostView: View { GeometryReader { (deviceSize: GeometryProxy) in VStack(alignment: .leading, spacing: 0) { let searching = get_searching_string(focusWordAttributes.0) + let searchingIsNil = searching == nil TopBar ScrollViewReader { scroller in ScrollView { - if case .replying_to(let replying_to) = self.action { - ReplyView(replying_to: replying_to, damus: damus_state, original_pubkeys: pubkeys, filtered_pubkeys: $filtered_pubkeys) + VStack(alignment: .leading) { + if case .replying_to(let replying_to) = self.action { + ReplyView(replying_to: replying_to, damus: damus_state, original_pubkeys: pubkeys, filtered_pubkeys: $filtered_pubkeys) + } + + Editor(deviceSize: deviceSize) } - - Editor(deviceSize: deviceSize) } - .frame(maxHeight: searching == nil ? .infinity : 70) + .frame(maxHeight: searching == nil ? deviceSize.size.height : 70) .onAppear { scroll_to_event(scroller: scroller, id: "post", delay: 1.0, animate: true, anchor: .top) } + // Note: The scroll commands below are specific because there seems to be quirk with ScrollReader where sending it to the exact same position twice resets its scroll position. + .onChange(of: caretRect.origin.y, perform: { newValue in + scroller.scrollTo(GHOST_CARET_VIEW_ID) + }) + .onChange(of: searchingIsNil, perform: { newValue in + scroller.scrollTo(GHOST_CARET_VIEW_ID) + }) } // This if-block observes @ for tagging if let searching { - UserSearch(damus_state: damus_state, search: searching, focusWordAttributes: $focusWordAttributes, newCursorIndex: $newCursorIndex, postTextViewCanScroll: $postTextViewCanScroll, post: $post) + UserSearch(damus_state: damus_state, search: searching, focusWordAttributes: $focusWordAttributes, newCursorIndex: $newCursorIndex, post: $post) .frame(maxHeight: .infinity) .environmentObject(tagModel) } else { diff --git a/damus/Views/Posting/UserSearch.swift b/damus/Views/Posting/UserSearch.swift index 7de3b228d8..54908a317d 100644 --- a/damus/Views/Posting/UserSearch.swift +++ b/damus/Views/Posting/UserSearch.swift @@ -21,7 +21,6 @@ struct UserSearch: View { let search: String @Binding var focusWordAttributes: (String?, NSRange?) @Binding var newCursorIndex: Int? - @Binding var postTextViewCanScroll: Bool @Binding var post: NSMutableAttributedString @EnvironmentObject var tagModel: TagModel @@ -70,12 +69,6 @@ struct UserSearch: View { .padding() } } - .onAppear() { - postTextViewCanScroll = false - } - .onDisappear() { - postTextViewCanScroll = true - } } } @@ -85,10 +78,9 @@ struct UserSearch_Previews: PreviewProvider { @State static var post: NSMutableAttributedString = NSMutableAttributedString(string: "some @jb55") @State static var word: (String?, NSRange?) = (nil, nil) @State static var newCursorIndex: Int? - @State static var postTextViewCanScroll: Bool = false - + static var previews: some View { - UserSearch(damus_state: test_damus_state(), search: search, focusWordAttributes: $word, newCursorIndex: $newCursorIndex, postTextViewCanScroll: $postTextViewCanScroll, post: $post) + UserSearch(damus_state: test_damus_state(), search: search, focusWordAttributes: $word, newCursorIndex: $newCursorIndex, post: $post) } } diff --git a/damus/Views/TextViewWrapper.swift b/damus/Views/TextViewWrapper.swift index 007c0315e4..dac5ad2270 100644 --- a/damus/Views/TextViewWrapper.swift +++ b/damus/Views/TextViewWrapper.swift @@ -7,20 +7,32 @@ import SwiftUI +// Defines how much extra bottom spacing will be applied after the text. +// This will avoid jitters when applying new lines, by ensuring it has enough space until the height is updated on the next view update cycle +let TEXT_BOX_BOTTOM_MARGIN_OFFSET: CGFloat = 30.0 + struct TextViewWrapper: UIViewRepresentable { @Binding var attributedText: NSMutableAttributedString - @Binding var postTextViewCanScroll: Bool @EnvironmentObject var tagModel: TagModel + @Binding var textHeight: CGFloat? let cursorIndex: Int? var getFocusWordForMention: ((String?, NSRange?) -> Void)? = nil let updateCursorPosition: ((Int) -> Void) + let onCaretRectChange: ((UITextView) -> Void) func makeUIView(context: Context) -> UITextView { let textView = UITextView() textView.delegate = context.coordinator - textView.isScrollEnabled = postTextViewCanScroll + + // Scroll has to be enabled. When this is disabled, the text input will overflow horizontally, even when its frame's width is limited. + textView.isScrollEnabled = true + // However, a scrolling text box inside of its parent scrollview does not provide a very good experience. We should have the textbox expand vertically + // To simulate that the text box can expand vertically, we will listen to text changes and dynamically change the text box height in response. + // Add an observer so that we can adapt the height of the text input whenever the text changes. + textView.addObserver(context.coordinator, forKeyPath: "contentSize", options: .new, context: nil) textView.showsVerticalScrollIndicator = false + TextViewWrapper.setTextProperties(textView) return textView } @@ -34,7 +46,6 @@ struct TextViewWrapper: UIViewRepresentable { } func updateUIView(_ uiView: UITextView, context: Context) { - uiView.isScrollEnabled = postTextViewCanScroll uiView.attributedText = attributedText TextViewWrapper.setTextProperties(uiView) @@ -53,24 +64,38 @@ struct TextViewWrapper: UIViewRepresentable { } func makeCoordinator() -> Coordinator { - Coordinator(attributedText: $attributedText, getFocusWordForMention: getFocusWordForMention, updateCursorPosition: updateCursorPosition) + Coordinator(attributedText: $attributedText, getFocusWordForMention: getFocusWordForMention, updateCursorPosition: updateCursorPosition, onCaretRectChange: onCaretRectChange, textHeight: $textHeight) } class Coordinator: NSObject, UITextViewDelegate { @Binding var attributedText: NSMutableAttributedString var getFocusWordForMention: ((String?, NSRange?) -> Void)? = nil let updateCursorPosition: ((Int) -> Void) - - init(attributedText: Binding, getFocusWordForMention: ((String?, NSRange?) -> Void)?, updateCursorPosition: @escaping ((Int) -> Void)) { + let onCaretRectChange: ((UITextView) -> Void) + @Binding var textHeight: CGFloat? + + init(attributedText: Binding, + getFocusWordForMention: ((String?, NSRange?) -> Void)?, + updateCursorPosition: @escaping ((Int) -> Void), + onCaretRectChange: @escaping ((UITextView) -> Void), + textHeight: Binding + ) { _attributedText = attributedText self.getFocusWordForMention = getFocusWordForMention self.updateCursorPosition = updateCursorPosition + self.onCaretRectChange = onCaretRectChange + _textHeight = textHeight } func textViewDidChange(_ textView: UITextView) { attributedText = NSMutableAttributedString(attributedString: textView.attributedText) processFocusedWordForMention(textView: textView) } + + func textViewDidChangeSelection(_ textView: UITextView) { + textView.scrollRangeToVisible(textView.selectedRange) + onCaretRectChange(textView) + } private func processFocusedWordForMention(textView: UITextView) { var val: (String?, NSRange?) = (nil, nil) @@ -158,6 +183,20 @@ struct TextViewWrapper: UIViewRepresentable { } } + override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) { + if keyPath == "contentSize", let textView = object as? UITextView { + DispatchQueue.main.async { + // Update text view height when text content size changes to fit all text content + // This is necessary to avoid having a scrolling text box combined with its parent scrolling view + self.updateTextViewHeight(textView: textView) + } + } + } + + func updateTextViewHeight(textView: UITextView) { + self.textHeight = textView.contentSize.height + TEXT_BOX_BOTTOM_MARGIN_OFFSET + } + } } From bfda0d1b74d7910a989afcb7faa7a56f2017bff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=99Aquino?= Date: Sat, 16 Sep 2023 05:37:04 +0000 Subject: [PATCH 093/111] ui: increase size of the hitbox on note ellipsis button MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changelog-Changed: Increase size of the hitbox on note ellipsis button Closes: https://github.com/damus-io/damus/issues/1454 Signed-off-by: Daniel D’Aquino Signed-off-by: William Casarin --- damus/Views/Events/EventMenu.swift | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/damus/Views/Events/EventMenu.swift b/damus/Views/Events/EventMenu.swift index cd7c430f36..ea707728c3 100644 --- a/damus/Views/Events/EventMenu.swift +++ b/damus/Views/Events/EventMenu.swift @@ -26,12 +26,19 @@ struct EventMenuContext: View { var body: some View { HStack { - Menu { - MenuItems(event: event, keypair: keypair, target_pubkey: target_pubkey, bookmarks: bookmarks, muted_threads: muted_threads, settings: settings) - } label: { - Label("", systemImage: "ellipsis") - .foregroundColor(Color.gray) - } + Label("", systemImage: "ellipsis") + .foregroundColor(Color.gray) + .contentShape(Circle()) + // Add our Menu button inside an overlay modifier to avoid affecting the rest of the layout around us. + .overlay( + Menu { + MenuItems(event: event, keypair: keypair, target_pubkey: target_pubkey, bookmarks: bookmarks, muted_threads: muted_threads, settings: settings) + } label: { + Color.clear + } + // Hitbox frame size + .frame(width: 100, height: 70) + ) } .padding([.bottom], 4) .contentShape(Rectangle()) From c4dfae9ede2b6d2892d81ee7bc5f264b32f2de95 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Fri, 8 Sep 2023 21:15:03 -0600 Subject: [PATCH 094/111] relays: update relay view to use new design Changelog-Changed: Updated relay view Closes: https://github.com/damus-io/damus/pull/1543 --- damus/Views/AddRelayView.swift | 145 +++++++++++++++-- damus/Views/Relays/RecommendedRelayView.swift | 85 ++++------ damus/Views/Relays/RelayConfigView.swift | 152 ++++++------------ damus/Views/Relays/RelayStatusView.swift | 44 ++++- damus/Views/Relays/RelayView.swift | 52 +++--- 5 files changed, 274 insertions(+), 204 deletions(-) diff --git a/damus/Views/AddRelayView.swift b/damus/Views/AddRelayView.swift index 8997437819..a992b66697 100644 --- a/damus/Views/AddRelayView.swift +++ b/damus/Views/AddRelayView.swift @@ -8,33 +8,148 @@ import SwiftUI struct AddRelayView: View { - @Binding var relay: String + let state: DamusState + @State var new_relay: String = "" + @State var relayAddErrorTitle: String? = nil + @State var relayAddErrorMessage: String? = nil + + @Environment(\.dismiss) var dismiss var body: some View { - ZStack(alignment: .leading) { - HStack{ - TextField(NSLocalizedString("wss://some.relay.com", comment: "Placeholder example for relay server address."), text: $relay) - .padding(2) - .padding(.leading, 25) + VStack { + Text("Add relay", comment: "Title text to indicate user to an add a relay.") + .font(.system(size: 20, weight: .bold)) + .padding(.vertical) + + Divider() + .padding(.bottom) + + HStack { + Label("", image: "copy2") + .onTapGesture { + if let pastedrelay = UIPasteboard.general.string { + self.new_relay = pastedrelay + } + } + TextField(NSLocalizedString("wss://some.relay.com", comment: "Placeholder example for relay server address."), text: $new_relay) .autocorrectionDisabled(true) .textInputAutocapitalization(.never) Label("", image: "close-circle") .foregroundColor(.accentColor) - .padding(.trailing, -25.0) - .opacity((relay == "") ? 0.0 : 1.0) + .opacity((new_relay == "") ? 0.0 : 1.0) .onTapGesture { - self.relay = "" + self.new_relay = "" + } + } + .padding(10) + .background(.secondary.opacity(0.2)) + .cornerRadius(10) + + if let errorMessage = relayAddErrorMessage { + VStack(spacing: 0) { + HStack(alignment: .top) { + Text(relayAddErrorTitle ?? "Error") + .bold() + .foregroundColor(DamusColors.dangerSecondary) + .padding(.leading) + Spacer() + Button(action: { + relayAddErrorTitle = nil // Clear error title + relayAddErrorMessage = nil // Clear error message + self.new_relay = "" + }, label: { + Image("close") + .frame(width: 20, height: 20) + .foregroundColor(DamusColors.dangerSecondary) + }) + .padding(.trailing) } + + Text(errorMessage) + .foregroundColor(DamusColors.dangerSecondary) + .padding(.top, 10) + } + .frame(minWidth: 300, maxWidth: .infinity, minHeight: 120, alignment: .center) + .background { + RoundedRectangle(cornerRadius: 12) + .fill(DamusColors.dangerBorder, strokeBorder: .gray.opacity(0.5), lineWidth: 1) + } } - Label("", image: "copy2") - .padding(.leading, -10) - .onTapGesture { - if let pastedrelay = UIPasteboard.general.string { - self.relay = pastedrelay + Button(action: { + if new_relay.starts(with: "wss://") == false && new_relay.starts(with: "ws://") == false { + new_relay = "wss://" + new_relay + } + + if new_relay.hasSuffix("/") { + new_relay.removeLast(); + } + + guard let url = RelayURL(new_relay), + let ev = state.contacts.event, + let keypair = state.keypair.to_full() else { + return + } + + let info = RelayInfo.rw + let descriptor = RelayDescriptor(url: url, info: info) + + do { + try state.pool.add_relay(descriptor) + relayAddErrorTitle = nil // Clear error title + relayAddErrorMessage = nil // Clear error message + } catch RelayError.RelayAlreadyExists { + relayAddErrorTitle = NSLocalizedString("Duplicate relay", comment: "Title of the duplicate relay error message.") + relayAddErrorMessage = NSLocalizedString("The relay you are trying to add is already added.\nYou're all set!", comment: "An error message that appears when the user attempts to add a relay that has already been added.") + return + } catch { + return + } + + state.pool.connect(to: [new_relay]) + + guard let new_ev = add_relay(ev: ev, keypair: keypair, current_relays: state.pool.our_descriptors, relay: new_relay, info: info) else { + return + } + + process_contact_event(state: state, ev: ev) + + state.pool.send(.event(new_ev)) + + new_relay = "" + + UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) + + dismiss() + }) { + HStack { + Text(verbatim: "Add relay") + .bold() } + .frame(minWidth: 300, maxWidth: .infinity, alignment: .center) } + .buttonStyle(GradientButtonStyle(padding: 10)) + //.disabled(!new_relay.isValidURL) <--- TODO + .padding(.vertical) + + Spacer() + } + .padding() + } +} + +// TODO +// This works sometimes, in certain cases where the relay is valid it won't allow the user to add it +// Needs improvement +extension String { + var isValidURL: Bool { + let detector = try! NSDataDetector(types: NSTextCheckingResult.CheckingType.link.rawValue) + if let match = detector.firstMatch(in: self, options: [], range: NSRange(location: 0, length: self.utf16.count)) { + // it is a link, if the match covers the whole string + return match.range.length == self.utf16.count + } else { + return false } } } @@ -43,6 +158,6 @@ struct AddRelayView_Previews: PreviewProvider { @State static var relay: String = "" static var previews: some View { - AddRelayView(relay: $relay) + AddRelayView(state: test_damus_state()) } } diff --git a/damus/Views/Relays/RecommendedRelayView.swift b/damus/Views/Relays/RecommendedRelayView.swift index dfebae351a..4fbd11d373 100644 --- a/damus/Views/Relays/RecommendedRelayView.swift +++ b/damus/Views/Relays/RecommendedRelayView.swift @@ -12,64 +12,50 @@ struct RecommendedRelayView: View { let relay: String let add_button: Bool + @ObservedObject private var model_cache: RelayModelCache + @Binding var showActionButtons: Bool init(damus: DamusState, relay: String, add_button: Bool = true, showActionButtons: Binding) { self.damus = damus self.relay = relay self.add_button = add_button + self.model_cache = damus.relay_model_cache self._showActionButtons = showActionButtons } - var body: some View { - ZStack { - HStack { - if let keypair = damus.keypair.to_full() { - if showActionButtons && add_button { - AddButton(keypair: keypair, showText: false) - } - } - - RelayType(is_paid: damus.relay_model_cache.model(with_relay_id: relay)?.metadata.is_paid ?? false) - - Text(relay).layoutPriority(1) - - if let meta = damus.relay_model_cache.model(with_relay_id: relay)?.metadata { - NavigationLink(value: Route.RelayDetail(relay: relay, metadata: meta)){ - EmptyView() - } - .opacity(0.0) - .disabled(showActionButtons) - - Spacer() - - Image("info") - .resizable() - .frame(width: 20, height: 20) - .foregroundColor(Color.accentColor) - } else { - Spacer() - - Image("question") - .resizable() - .frame(width: 20, height: 20) - .foregroundColor(.gray) - } + var recommended: [RelayDescriptor] { + let rs: [RelayDescriptor] = [] + return BOOTSTRAP_RELAYS.reduce(into: rs) { xs, x in + if damus.pool.get_relay(x) == nil, let url = RelayURL(x) { + xs.append(RelayDescriptor(url: url, info: .rw)) } } - .swipeActions { - if add_button { - if let keypair = damus.keypair.to_full() { - AddButton(keypair: keypair, showText: false) - .tint(.accentColor) + } + + var body: some View { + VStack { + let meta = model_cache.model(with_relay_id: relay)?.metadata + + RelayPicView(relay: relay, icon: meta?.icon, size: 70, highlight: .none, disable_animation: false) + if let meta = damus.relay_model_cache.model(with_relay_id: relay)?.metadata { + NavigationLink(value: Route.RelayDetail(relay: relay, metadata: meta)){ + EmptyView() } + .opacity(0.0) + .disabled(showActionButtons) + } + + HStack { + Text(meta?.name ?? relay.hostname ?? relay) + .lineLimit(1) + } + .contextMenu { + CopyAction(relay: relay) } - } - .contextMenu { - CopyAction(relay: relay) if let keypair = damus.keypair.to_full() { - AddButton(keypair: keypair, showText: true) + AddButton(keypair: keypair) } } } @@ -82,19 +68,14 @@ struct RecommendedRelayView: View { } } - func AddButton(keypair: FullKeypair, showText: Bool) -> some View { + func AddButton(keypair: FullKeypair) -> some View { Button(action: { add_action(keypair: keypair) }) { - if showText { - Text(NSLocalizedString("Connect", comment: "Button to connect to recommended relay server.")) - } - Image("plus-circle") - .resizable() - .frame(width: 20, height: 20) - .foregroundColor(.accentColor) - .padding(.leading, 5) + Text(NSLocalizedString("Add", comment: "Button to add relay server to list.")) + .padding(10) } + .buttonStyle(NeutralButtonStyle()) } func add_action(keypair: FullKeypair) { diff --git a/damus/Views/Relays/RelayConfigView.swift b/damus/Views/Relays/RelayConfigView.swift index 7e5452b55f..a5b044e6e2 100644 --- a/damus/Views/Relays/RelayConfigView.swift +++ b/damus/Views/Relays/RelayConfigView.swift @@ -9,10 +9,9 @@ import SwiftUI struct RelayConfigView: View { let state: DamusState - @State var new_relay: String = "" @State var relays: [RelayDescriptor] @State private var showActionButtons = false - @State var relayAddErrorMessage: String? = nil + @State var show_add_relay: Bool = false @Environment(\.dismiss) var dismiss @@ -41,118 +40,69 @@ struct RelayConfigView: View { } var MainContent: some View { - Form { - Section { - AddRelayView(relay: $new_relay) - } header: { - HStack { - Text(NSLocalizedString("Connect To Relay", comment: "Label for section for adding a relay server.")) - .font(.system(size: 18, weight: .heavy)) - .padding(.bottom, 5) - } - } footer: { + VStack { + Divider() + + if recommended.count > 0 { VStack { - HStack { - Spacer() - if !new_relay.isEmpty { - Button(NSLocalizedString("Cancel", comment: "Button to cancel out of view adding user inputted relay.")) { - new_relay = "" - UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) - } - .font(.system(size: 14, weight: .bold)) - .frame(width: 80, height: 30) - .foregroundColor(.white) - .background(LINEAR_GRADIENT) - .clipShape(Capsule()) - .padding(EdgeInsets(top: 15, leading: 0, bottom: 0, trailing: 0)) - - Button(NSLocalizedString("Add", comment: "Button to confirm adding user inputted relay.")) { - - if new_relay.starts(with: "wss://") == false && new_relay.starts(with: "ws://") == false { - new_relay = "wss://" + new_relay - } - - if new_relay.hasSuffix("/") { - new_relay.removeLast(); - } - - guard let url = RelayURL(new_relay), - let ev = state.contacts.event, - let keypair = state.keypair.to_full() else { - return - } - - let info = RelayInfo.rw - let descriptor = RelayDescriptor(url: url, info: info) - - do { - try state.pool.add_relay(descriptor) - relayAddErrorMessage = nil // Clear error message - } catch RelayError.RelayAlreadyExists { - relayAddErrorMessage = NSLocalizedString("This relay is already in your list", comment: "An error message that appears when the user attempts to add a relay that has already been added.") - return - } catch { - return - } - - state.pool.connect(to: [new_relay]) - - guard let new_ev = add_relay(ev: ev, keypair: keypair, current_relays: state.pool.our_descriptors, relay: new_relay, info: info) else { - return - } - - process_contact_event(state: state, ev: ev) - - state.pool.send(.event(new_ev)) - - new_relay = "" - - UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) - } - .font(.system(size: 14, weight: .bold)) - .frame(width: 80, height: 30) - .foregroundColor(.white) - .background(LINEAR_GRADIENT) - .clipShape(Capsule()) - .padding(EdgeInsets(top: 15, leading: 0, bottom: 0, trailing: 0)) + Text("Recommended relays") + .foregroundStyle(DamusLightGradient.gradient) + .padding(10) + .background { + RoundedRectangle(cornerRadius: 15) + .stroke(DamusLightGradient.gradient) } - } - if let errorMessage = relayAddErrorMessage { - HStack { - Spacer() - Text(errorMessage) - .foregroundColor(Color.red) + .padding(.vertical) + + HStack(spacing: 20) { + ForEach(recommended, id: \.url) { r in + RecommendedRelayView(damus: state, relay: r.url.id, showActionButtons: $showActionButtons) } } + .padding() } - } - - Section { - List(Array(relays), id: \.url) { relay in - RelayView(state: state, relay: relay.url.id, showActionButtons: $showActionButtons) - } - } header: { - HStack { - Text(NSLocalizedString("Connected Relays", comment: "Section title for relay servers that are connected.")) - .font(.system(size: 18, weight: .heavy)) - .padding(.bottom, 5) + .frame(minWidth: 250, maxWidth: .infinity, minHeight: 250, alignment: .center) + .background { + RoundedRectangle(cornerRadius: 12) + .fill(DamusLightGradient.gradient.opacity(0.15), strokeBorder: DamusLightGradient.gradient, lineWidth: 1) } + .padding(.horizontal) } - if recommended.count > 0 { - Section { - List(recommended, id: \.url) { r in - RecommendedRelayView(damus: state, relay: r.url.id, showActionButtons: $showActionButtons) + HStack { + Text(NSLocalizedString("My Relays", comment: "Section title for relay servers that the user is connected to.")) + .font(.system(size: 32, weight: .bold)) + + Spacer() + + Button(action: { + show_add_relay.toggle() + }) { + HStack { + Text(verbatim: "Add relay") + .padding(10) } - } header: { - Text(NSLocalizedString("Recommended Relays", comment: "Section title for recommend relay servers that could be added as part of configuration")) - .font(.system(size: 18, weight: .heavy)) - .padding(.bottom, 5) } + .buttonStyle(NeutralButtonStyle()) } + .padding(25) + + List(Array(relays), id: \.url) { relay in + RelayView(state: state, relay: relay.url.id, showActionButtons: $showActionButtons) + } + .listStyle(PlainListStyle()) } .navigationTitle(NSLocalizedString("Relays", comment: "Title of relays view")) - .navigationBarTitleDisplayMode(.large) + .navigationBarTitleDisplayMode(.inline) + .sheet(isPresented: $show_add_relay, onDismiss: { self.show_add_relay = false }) { + if #available(iOS 16.0, *) { + AddRelayView(state: state) + .presentationDetents([.height(300)]) + .presentationDragIndicator(.visible) + } else { + AddRelayView(state: state) + } + } .toolbar { if state.keypair.privkey != nil { if showActionButtons { diff --git a/damus/Views/Relays/RelayStatusView.swift b/damus/Views/Relays/RelayStatusView.swift index e3a82e173a..d8c6476f88 100644 --- a/damus/Views/Relays/RelayStatusView.swift +++ b/damus/Views/Relays/RelayStatusView.swift @@ -13,21 +13,51 @@ struct RelayStatusView: View { var body: some View { Group { if connection.isConnecting { - ProgressView() + Text("Connecting") + .font(.caption) + .frame(height: 20) + .padding(.horizontal, 10) + .foregroundColor(DamusColors.warning) + .background(DamusColors.warningQuaternary) + .cornerRadius(20) + .overlay( + RoundedRectangle(cornerRadius: 20) + .stroke(DamusColors.warningBorder, lineWidth: 1) + ) + } else if connection.isConnected { + Text("Online") + .font(.caption) + .frame(height: 20) + .padding(.horizontal, 10) + .foregroundColor(DamusColors.success) + .background(DamusColors.successQuaternary) + .cornerRadius(20) + .overlay( + RoundedRectangle(cornerRadius: 20) + .stroke(DamusColors.successBorder, lineWidth: 1) + ) } else { - Image(connection.isConnected ? "globe" : "warning.fill") - .resizable() - .foregroundColor(connection.isConnected ? .green : .red) + Text("Error") + .font(.caption) + .frame(height: 20) + .padding(.horizontal, 10) + .foregroundColor(DamusColors.danger) + .background(DamusColors.dangerQuaternary) + .border(DamusColors.dangerBorder) + .cornerRadius(20) + .overlay( + RoundedRectangle(cornerRadius: 20) + .stroke(DamusColors.dangerBorder, lineWidth: 1) + ) } } - .frame(width: 20, height: 20) - .padding(.trailing, 5) + .padding(.trailing, 20) } } struct RelayStatusView_Previews: PreviewProvider { static var previews: some View { - let connection = test_damus_state().pool.get_relay("relay")!.connection + let connection = test_damus_state().pool.get_relay("wss://relay.damus.io")!.connection RelayStatusView(connection: connection) } } diff --git a/damus/Views/Relays/RelayView.swift b/damus/Views/Relays/RelayView.swift index fc46fed16e..0b5ea52219 100644 --- a/damus/Views/Relays/RelayView.swift +++ b/damus/Views/Relays/RelayView.swift @@ -28,41 +28,35 @@ struct RelayView: View { if showActionButtons { RemoveButton(privkey: privkey, showText: false) } - else if let relay_connection { - RelayStatusView(connection: relay_connection) - } } - - RelayType(is_paid: state.relay_model_cache.model(with_relay_id: relay)?.metadata.is_paid ?? false) - - if let meta = model_cache.model(with_relay_id: relay)?.metadata { - Text(relay) - .background( - NavigationLink(value: Route.RelayDetail(relay: relay, metadata: meta), label: { - EmptyView() - }).opacity(0.0).disabled(showActionButtons) - ) - - Spacer() - Image("info") - .resizable() - .frame(width: 20, height: 20) - .foregroundColor(Color.accentColor) - } else { + let meta = model_cache.model(with_relay_id: relay)?.metadata + + RelayPicView(relay: relay, icon: meta?.icon, size: 55, highlight: .none, disable_animation: false) + + VStack(alignment: .leading) { + HStack { + Text(meta?.name ?? relay) + .font(.headline) + .padding(.bottom, 2) + RelayType(is_paid: state.relay_model_cache.model(with_relay_id: relay)?.metadata.is_paid ?? false) + } Text(relay) + .font(.subheadline) + .foregroundColor(.gray) + } + + Spacer() + + if let relay_connection { + RelayStatusView(connection: relay_connection) .background( - NavigationLink(value: Route.RelayDetail(relay: relay, metadata: nil), label: { + NavigationLink(value: Route.RelayDetail(relay: relay, metadata: meta), label: { EmptyView() - }).opacity(0.0).disabled(showActionButtons) + }) + .buttonStyle(.plain) + .disabled(showActionButtons) ) - - Spacer() - - Image("question") - .resizable() - .frame(width: 20, height: 20) - .foregroundColor(.gray) } } } From fff45499339fd264462509cc6259d06d07071c3d Mon Sep 17 00:00:00 2001 From: ericholguin Date: Sun, 10 Sep 2023 22:22:24 -0600 Subject: [PATCH 095/111] relays: remove usage of show action button binding --- damus/Views/Relays/RecommendedRelayView.swift | 86 +++++++++++++------ damus/Views/UserRelaysView.swift | 17 +--- 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/damus/Views/Relays/RecommendedRelayView.swift b/damus/Views/Relays/RecommendedRelayView.swift index 4fbd11d373..9fff458e67 100644 --- a/damus/Views/Relays/RecommendedRelayView.swift +++ b/damus/Views/Relays/RecommendedRelayView.swift @@ -11,17 +11,16 @@ struct RecommendedRelayView: View { let damus: DamusState let relay: String let add_button: Bool + let user_recommended: Bool @ObservedObject private var model_cache: RelayModelCache - @Binding var showActionButtons: Bool - - init(damus: DamusState, relay: String, add_button: Bool = true, showActionButtons: Binding) { + init(damus: DamusState, relay: String, add_button: Bool = true, user_recommended: Bool = false) { self.damus = damus self.relay = relay self.add_button = add_button + self.user_recommended = user_recommended self.model_cache = damus.relay_model_cache - self._showActionButtons = showActionButtons } var recommended: [RelayDescriptor] { @@ -34,28 +33,65 @@ struct RecommendedRelayView: View { } var body: some View { - VStack { - let meta = model_cache.model(with_relay_id: relay)?.metadata - - RelayPicView(relay: relay, icon: meta?.icon, size: 70, highlight: .none, disable_animation: false) - if let meta = damus.relay_model_cache.model(with_relay_id: relay)?.metadata { - NavigationLink(value: Route.RelayDetail(relay: relay, metadata: meta)){ - EmptyView() - } - .opacity(0.0) - .disabled(showActionButtons) - } - + let meta = model_cache.model(with_relay_id: relay)?.metadata + + if user_recommended { HStack { - Text(meta?.name ?? relay.hostname ?? relay) - .lineLimit(1) - } - .contextMenu { - CopyAction(relay: relay) + RelayPicView(relay: relay, icon: meta?.icon, size: 50, highlight: .none, disable_animation: false) + .padding(.horizontal, 5) + + VStack(alignment: .leading) { + HStack { + Text(meta?.name ?? relay.hostname ?? relay) + .font(.headline) + .padding(.bottom, 2) + + RelayType(is_paid: damus.relay_model_cache.model(with_relay_id: relay)?.metadata.is_paid ?? false) + } + + Text(relay) + .font(.subheadline) + .foregroundColor(.gray) + } + + Spacer() + + if let keypair = damus.keypair.to_full() { + VStack(alignment: .center) { + if damus.pool.get_relay(relay) == nil { + AddButton(keypair: keypair) + } else { + Image(systemName: "checkmark.circle") + .resizable() + .frame(width: 30, height: 30) + .foregroundColor(DamusColors.success) + .padding(.trailing, 10) + } + } + .padding(.horizontal, 5) + } } - - if let keypair = damus.keypair.to_full() { - AddButton(keypair: keypair) + } else { + VStack { + RelayPicView(relay: relay, icon: meta?.icon, size: 70, highlight: .none, disable_animation: false) + if let meta = damus.relay_model_cache.model(with_relay_id: relay)?.metadata { + NavigationLink(value: Route.RelayDetail(relay: relay, metadata: meta)){ + EmptyView() + } + .opacity(0.0) + } + + HStack { + Text(meta?.name ?? relay.hostname ?? relay) + .lineLimit(1) + } + .contextMenu { + CopyAction(relay: relay) + } + + if let keypair = damus.keypair.to_full() { + AddButton(keypair: keypair) + } } } } @@ -92,6 +128,6 @@ struct RecommendedRelayView: View { struct RecommendedRelayView_Previews: PreviewProvider { static var previews: some View { - RecommendedRelayView(damus: test_damus_state(), relay: "wss://relay.damus.io", showActionButtons: .constant(false)) + RecommendedRelayView(damus: test_damus_state(), relay: "wss://relay.damus.io", user_recommended: true) } } diff --git a/damus/Views/UserRelaysView.swift b/damus/Views/UserRelaysView.swift index 9071d853fb..83a5cf1711 100644 --- a/damus/Views/UserRelaysView.swift +++ b/damus/Views/UserRelaysView.swift @@ -12,7 +12,6 @@ struct UserRelaysView: View { let relays: [String] @State var relay_state: [(String, Bool)] - @State private var showAddButton = false init(state: DamusState, relays: [String]) { self.state = state @@ -29,25 +28,13 @@ struct UserRelaysView: View { var body: some View { List(relay_state, id: \.0) { (r, add) in - RecommendedRelayView(damus: state, relay: r, add_button: add, showActionButtons: $showAddButton) + RecommendedRelayView(damus: state, relay: r, add_button: add, user_recommended: true) } + .listStyle(PlainListStyle()) .onReceive(handle_notify(.relays_changed)) { _ in self.relay_state = UserRelaysView.make_relay_state(pool: state.pool, relays: self.relays) } .navigationBarTitle(NSLocalizedString("Relays", comment: "Navigation bar title that shows the list of relays for a user.")) - .toolbar{ - if state.keypair.privkey != nil { - if showAddButton { - Button(NSLocalizedString("Done", comment: "Button that, when tapped, will finish adding a different user's relays to your relay by hiding the + buttons next to the relays.")) { - showAddButton.toggle() - } - } else { - Button(NSLocalizedString("Show +", comment: "Button that, when tapped, will show + buttons next to a user's relays.")) { - showAddButton.toggle() - } - } - } - } } } From 40459e247ec5c9890629b87eb60960cf4db98a10 Mon Sep 17 00:00:00 2001 From: ericholguin Date: Sun, 10 Sep 2023 22:23:04 -0600 Subject: [PATCH 096/111] relays: user relay design --- damus/Views/Relays/RelayConfigView.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/damus/Views/Relays/RelayConfigView.swift b/damus/Views/Relays/RelayConfigView.swift index a5b044e6e2..3f692f9aa1 100644 --- a/damus/Views/Relays/RelayConfigView.swift +++ b/damus/Views/Relays/RelayConfigView.swift @@ -56,7 +56,7 @@ struct RelayConfigView: View { HStack(spacing: 20) { ForEach(recommended, id: \.url) { r in - RecommendedRelayView(damus: state, relay: r.url.id, showActionButtons: $showActionButtons) + RecommendedRelayView(damus: state, relay: r.url.id) } } .padding() From d39a3da3b728aa06ac96a2dec2f071050fa6e3f1 Mon Sep 17 00:00:00 2001 From: Suhail Saqan Date: Thu, 1 Jun 2023 14:49:56 -0500 Subject: [PATCH 097/111] util: add separate_images and separate_invoices --- damus/Nostr/NostrEvent.swift | 23 +++++++++++++++++++++++ damus/Views/FollowingView.swift | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/damus/Nostr/NostrEvent.swift b/damus/Nostr/NostrEvent.swift index 9768991eda..7be20ad40b 100644 --- a/damus/Nostr/NostrEvent.swift +++ b/damus/Nostr/NostrEvent.swift @@ -949,6 +949,29 @@ func first_eref_mention(ev: NostrEvent, keypair: Keypair) -> Mention? { return nil } +func separate_images(ev: NostrEvent, keypair: Keypair) -> [MediaUrl]? { + let urlBlocks: [URL] = ev.blocks(keypair).blocks.reduce(into: []) { urls, block in + guard case .url(let url) = block else { + return + } + if classify_url(url).is_img != nil { + urls.append(url) + } + } + let mediaUrls = urlBlocks.map { MediaUrl.image($0) } + return mediaUrls.isEmpty ? nil : mediaUrls +} + +func separate_invoices(ev: NostrEvent, keypair: Keypair) -> [Invoice]? { + let invoiceBlocks: [Invoice] = ev.blocks(keypair).blocks.reduce(into: []) { invoices, block in + guard case .invoice(let invoice) = block else { + return + } + invoices.append(invoice) + } + return invoiceBlocks.isEmpty ? nil : invoiceBlocks +} + /** Transforms a `NostrEvent` of known kind `NostrKind.like`to a human-readable emoji. If the known kind is not a `NostrKind.like`, it will return `nil`. diff --git a/damus/Views/FollowingView.swift b/damus/Views/FollowingView.swift index bdb5724eeb..2f2693f63a 100644 --- a/damus/Views/FollowingView.swift +++ b/damus/Views/FollowingView.swift @@ -93,7 +93,7 @@ struct FollowingView: View { /* struct FollowingView_Previews: PreviewProvider { static var previews: some View { - FollowingView(contact: <#NostrEvent#>, damus_state: <#DamusState#>) + FollowingView(damus_state: test_damus_state, following: test_following_model) } } */ From a88f5db10b322056a8f86e71497c2e9379631c6f Mon Sep 17 00:00:00 2001 From: William Casarin Date: Thu, 21 Sep 2023 08:47:53 -0400 Subject: [PATCH 098/111] Revert "deps: add tldextract" This reverts commit 4263b9690fff3469905679f71828994f43efb3b1. --- damus.xcodeproj/project.pbxproj | 17 ----------------- .../xcshareddata/swiftpm/Package.resolved | 9 --------- 2 files changed, 26 deletions(-) diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 45283a24ed..afc820ea71 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -401,7 +401,6 @@ 5C6E1DAD2A193EC2008FC15A /* GradientButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAC2A193EC2008FC15A /* GradientButtonStyle.swift */; }; 5C6E1DAF2A194075008FC15A /* PinkGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6E1DAE2A194075008FC15A /* PinkGradient.swift */; }; 5CC868DD2AA29B3200FB22BA /* NeutralButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CC868DC2AA29B3200FB22BA /* NeutralButtonStyle.swift */; }; - 5CF2DCCA2AA3A49B00984B8D /* TLDExtract in Frameworks */ = {isa = PBXBuildFile; productRef = 5CF2DCC92AA3A49B00984B8D /* TLDExtract */; }; 5CF2DCCC2AA3AF0B00984B8D /* RelayPicView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF2DCCB2AA3AF0B00984B8D /* RelayPicView.swift */; }; 5CF2DCCE2AABE1A500984B8D /* DamusLightGradient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF2DCCD2AABE1A500984B8D /* DamusLightGradient.swift */; }; 5CF72FC229B9142F00124A13 /* ShareAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5CF72FC129B9142F00124A13 /* ShareAction.swift */; }; @@ -1135,7 +1134,6 @@ 4C06670428FC7EC500038D2A /* Kingfisher in Frameworks */, 4C649881286E0EE300EAE2B3 /* secp256k1 in Frameworks */, 4C27C9322A64766F007DBC75 /* MarkdownUI in Frameworks */, - 5CF2DCCA2AA3A49B00984B8D /* TLDExtract in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -2315,7 +2313,6 @@ 4C649880286E0EE300EAE2B3 /* secp256k1 */, 4C06670328FC7EC500038D2A /* Kingfisher */, 4C27C9312A64766F007DBC75 /* MarkdownUI */, - 5CF2DCC92AA3A49B00984B8D /* TLDExtract */, ); productName = damus; productReference = 4CE6DEE327F7A08100C66700 /* damus.app */; @@ -2423,7 +2420,6 @@ 4C06670228FC7EC500038D2A /* XCRemoteSwiftPackageReference "Kingfisher" */, 4CCF9AB02A1FE80B00E03CFB /* XCRemoteSwiftPackageReference "GSPlayer" */, 4C27C9302A64766F007DBC75 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, - 5CF2DCC82AA3A49B00984B8D /* XCRemoteSwiftPackageReference "TLDExtractSwift" */, ); productRefGroup = 4CE6DEE427F7A08100C66700 /* Products */; projectDirPath = ""; @@ -3432,14 +3428,6 @@ minimumVersion = 0.2.26; }; }; - 5CF2DCC82AA3A49B00984B8D /* XCRemoteSwiftPackageReference "TLDExtractSwift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/MarcoEidinger/TLDExtractSwift"; - requirement = { - kind = upToNextMajorVersion; - minimumVersion = 2.0.0; - }; - }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ @@ -3458,11 +3446,6 @@ package = 4C64987F286E0EE300EAE2B3 /* XCRemoteSwiftPackageReference "secp256k1" */; productName = secp256k1; }; - 5CF2DCC92AA3A49B00984B8D /* TLDExtract */ = { - isa = XCSwiftPackageProductDependency; - package = 5CF2DCC82AA3A49B00984B8D /* XCRemoteSwiftPackageReference "TLDExtractSwift" */; - productName = TLDExtract; - }; /* End XCSwiftPackageProductDependency section */ /* Begin XCVersionGroup section */ diff --git a/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 7f3c64b5a7..c8409240c6 100644 --- a/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/damus.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -33,15 +33,6 @@ "state" : { "revision" : "76bb7971da7fbf429de1c84f1244adf657242fee" } - }, - { - "identity" : "tldextractswift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/MarcoEidinger/TLDExtractSwift", - "state" : { - "revision" : "d4cbc911ab087e34439879df7f0314075886ae21", - "version" : "2.3.5" - } } ], "version" : 2 From 305ee03b0e92b31de533b5b862602f7ab150a364 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Thu, 21 Sep 2023 09:08:07 -0400 Subject: [PATCH 099/111] relays: fix tld extraction performance issues This uses a simpler variant that doesn't require a library. It is also much faster and doesn't cause a delay when you open the relay view. Not sure why I needed to touch other parts of the code to make the build work. Probably xcode beta thing? --- damus/Views/Relays/RecommendedRelayView.swift | 4 +-- damus/Views/Relays/RelayPicView.swift | 26 +++++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/damus/Views/Relays/RecommendedRelayView.swift b/damus/Views/Relays/RecommendedRelayView.swift index 9fff458e67..6e66a39d20 100644 --- a/damus/Views/Relays/RecommendedRelayView.swift +++ b/damus/Views/Relays/RecommendedRelayView.swift @@ -42,7 +42,7 @@ struct RecommendedRelayView: View { VStack(alignment: .leading) { HStack { - Text(meta?.name ?? relay.hostname ?? relay) + Text(meta?.name ?? relay) .font(.headline) .padding(.bottom, 2) @@ -82,7 +82,7 @@ struct RecommendedRelayView: View { } HStack { - Text(meta?.name ?? relay.hostname ?? relay) + Text(meta?.name ?? relay) .lineLimit(1) } .contextMenu { diff --git a/damus/Views/Relays/RelayPicView.swift b/damus/Views/Relays/RelayPicView.swift index f28a0620c7..39beb462d7 100644 --- a/damus/Views/Relays/RelayPicView.swift +++ b/damus/Views/Relays/RelayPicView.swift @@ -7,13 +7,12 @@ import SwiftUI import Kingfisher -import TLDExtract struct FailedRelayImage: View { let url: URL? var body: some View { - let abbrv = String(url?.hostname?.first?.uppercased() ?? "R") + let abbrv = String(url?.host()?.first?.uppercased() ?? "R") Text("\(abbrv)") .font(.system(size: 40, weight: .bold)) } @@ -90,11 +89,28 @@ struct RelayPicView: View { } } +func extract_tld(_ host: String) -> String { + let parts = host.split(separator: ".") + + if parts.count >= 3 { + let last_3 = parts.suffix(3) + if parts[1] == "co" && parts[2] == "uk" { + return String(last_3.joined(separator: ".")) + } else { + return String(parts.suffix(2).joined(separator: ".")) + } + } else if parts.count == 2 { + return host + } + + return host +} + func get_relay_url(relay: String, icon: String?) -> URL? { - let extractor = TLDExtract() var favicon = relay + "/favicon.ico" - if let parseRelay: TLDResult = extractor.parse(relay) { - favicon = "https://" + (parseRelay.rootDomain ?? relay) + "/favicon.ico" + let tld = extract_tld(relay) + if tld != relay { + favicon = "https://" + tld + "/favicon.ico" } let pic = icon ?? favicon return URL(string: pic) From 49283f2bb21ebcbfd0093b872c59308ec6d7d4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=99Aquino?= Date: Mon, 18 Sep 2023 23:27:15 +0000 Subject: [PATCH 100/111] filters: add "Do not show #nsfw tagged posts" setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a setting where the user can choose to hide notes with a #nsfw hashtag. This setting was implemented to allow users to filter out adult or other unsafe content. I moved the code logic for content filtering into a new file, and defined a protocol for content filters. Although the logic is still simple, this might help in developing a flexible API in case we have more complex filtering needs in the future. I also modified the name of the "Appearance" setting to "Appearance and filters", to make it easier for users to intuitively find this setting. (Note: Re-translations of this string might be necessary) **PASS** **iOS:** - iOS 17.0 (iPhone 14 Pro) **Damus:** (This commit) **Steps:** 1. Follow another account that you control (Account B) 2. On account B, post a note saying "#test this is a test". This note should show up on the home feed. 3. On account B, post a note saying "#nsfw this is a test". This note should NOT show up on the home feed 4. Go to settings and disable the NSFW filter. Go back to the home view. The #nsfw post should now show up. 5. Close app and reopen. NSFW post should still show up (i.e. Setting should be persistent) 6. Unfollow account B 7. Close app and reopen. 8. Follow the "#grownostr" hashtag 9. Turn on the NSFW filter 10. On account B, post a note saying "#grownostr this is a test". This note should show up on the home view. 11. On account B, post a note saying "#grownostr #nsfw this is a test". This note should NOT show up. 12. Double-check the "notes and replies" tab. Note should NOT show up there either. 12. Turn off NSFW filter 13. Note from step 11 should now show up. 14. Go to Universe view and find a post with a hashtag. Remember where the post is. 14. Locally change the tag keyword from "nsfw" to that hashtag (Note: I had to test this way because my posts were not showing up in the Universe view) 15. Turn off the filter. Check post is there, in the Universe view. 16. Turn on the filter. Check post is no longer there in the Universe view. (Check the neighboring posts are the same, to make sure) 17. Bring back the code to its normal state. 18. Search for "#nsfw". Make sure that #nsfw appears (I believe this is ok, because it means the person is purposefully searching for it) Closes: https://github.com/damus-io/damus/issues/1412 Changelog-Added: Add "Do not show #nsfw tagged posts" setting Signed-off-by: Daniel D’Aquino Signed-off-by: William Casarin --- damus.xcodeproj/project.pbxproj | 4 ++ damus/ContentView.swift | 23 +++----- damus/Models/ContentFilters.swift | 58 +++++++++++++++++++ damus/Models/UserSettingsStore.swift | 3 + damus/Views/ConfigView.swift | 2 +- damus/Views/SearchHomeView.swift | 9 +++ .../Settings/AppearanceSettingsView.swift | 9 +++ 7 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 damus/Models/ContentFilters.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index afc820ea71..a02d931f55 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -423,6 +423,7 @@ BAB68BED29543FA3007BA466 /* SelectWalletView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BAB68BEC29543FA3007BA466 /* SelectWalletView.swift */; }; D2277EEA2A089BD5006C3807 /* Router.swift in Sources */ = {isa = PBXBuildFile; fileRef = D2277EE92A089BD5006C3807 /* Router.swift */; }; D71DC1EC2A9129C3006E207C /* PostViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D71DC1EB2A9129C3006E207C /* PostViewTests.swift */; }; + D723C38E2AB8D83400065664 /* ContentFilters.swift in Sources */ = {isa = PBXBuildFile; fileRef = D723C38D2AB8D83400065664 /* ContentFilters.swift */; }; D78525252A7B2EA4002FA637 /* NoteContentViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */; }; D7DEEF2F2A8C021E00E0C99F /* NostrEventTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */; }; E4FA1C032A24BB7F00482697 /* SearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */; }; @@ -1104,6 +1105,7 @@ BAB68BEC29543FA3007BA466 /* SelectWalletView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectWalletView.swift; sourceTree = ""; }; D2277EE92A089BD5006C3807 /* Router.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Router.swift; sourceTree = ""; }; D71DC1EB2A9129C3006E207C /* PostViewTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostViewTests.swift; sourceTree = ""; }; + D723C38D2AB8D83400065664 /* ContentFilters.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentFilters.swift; sourceTree = ""; }; D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NoteContentViewTests.swift; sourceTree = ""; }; D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NostrEventTests.swift; sourceTree = ""; }; E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchSettingsView.swift; sourceTree = ""; }; @@ -1287,6 +1289,7 @@ 3A23838D2A297DD200E5AA2E /* ZapButtonModel.swift */, 3A5E47C42A4A6CF400C0D090 /* Trie.swift */, 3A90B1802A4EA3AF00000D94 /* UserSearchCache.swift */, + D723C38D2AB8D83400065664 /* ContentFilters.swift */, ); path = Models; sourceTree = ""; @@ -2861,6 +2864,7 @@ 4C75EFA427FA577B0006080F /* PostView.swift in Sources */, 4C30AC7229A5677A00E2BD5A /* NotificationsView.swift in Sources */, 4C1A9A2129DDD3E100516EAC /* KeySettingsView.swift in Sources */, + D723C38E2AB8D83400065664 /* ContentFilters.swift in Sources */, 4C32B95A2A9AD44700DC3548 /* Verifiable.swift in Sources */, 4C73C5142A4437C10062CAC0 /* ZapUserView.swift in Sources */, 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */, diff --git a/damus/ContentView.swift b/damus/ContentView.swift index 8782d5835a..6e57fef734 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -56,20 +56,6 @@ enum Sheets: Identifiable { } } -enum FilterState : Int { - case posts_and_replies = 1 - case posts = 0 - - func filter(ev: NostrEvent) -> Bool { - switch self { - case .posts: - return ev.known_kind == .boost || !ev.is_reply(.empty) - case .posts_and_replies: - return true - } - } -} - struct ContentView: View { let keypair: Keypair @@ -96,6 +82,11 @@ struct ContentView: View { @StateObject var navigationCoordinator: NavigationCoordinator = NavigationCoordinator() @AppStorage("has_seen_suggested_users") private var hasSeenSuggestedUsers = false let sub_id = UUID().description + var damus_filter: DamusFilter { + get { + return DamusFilter(hide_nsfw_tagged_content: self.damus_state?.settings.hide_nsfw_tagged_content ?? true) + } + } @Environment(\.colorScheme) var colorScheme @@ -114,10 +105,10 @@ struct ContentView: View { // This is needed or else there is a bug when switching from the 3rd or 2nd tab to first. no idea why. mystery - contentTimelineView(filter: FilterState.posts.filter) + contentTimelineView(filter: damus_filter.get_filter(.posts)) .tag(FilterState.posts) .id(FilterState.posts) - contentTimelineView(filter: FilterState.posts_and_replies.filter) + contentTimelineView(filter: damus_filter.get_filter(.posts_and_replies)) .tag(FilterState.posts_and_replies) .id(FilterState.posts_and_replies) } diff --git a/damus/Models/ContentFilters.swift b/damus/Models/ContentFilters.swift new file mode 100644 index 0000000000..3aadbd41cd --- /dev/null +++ b/damus/Models/ContentFilters.swift @@ -0,0 +1,58 @@ +// +// ContentFilters.swift +// damus +// +// Created by Daniel D’Aquino on 2023-09-18. +// + +import Foundation + +protocol ContentFilter { + /// Function that implements the content filtering logic + /// - Parameter ev: The nostr event to be processed + /// - Returns: Must return `true` to show events, and return `false` to hide/filter events + func filter(ev: NostrEvent) -> Bool +} + +/// Simple filter to determine whether to show posts or all posts and replies. +enum FilterState : Int, ContentFilter { + case posts_and_replies = 1 + case posts = 0 + + func filter(ev: NostrEvent) -> Bool { + switch self { + case .posts: + return ev.known_kind == .boost || !ev.is_reply(.empty) + case .posts_and_replies: + return true + } + } +} + +/// Simple filter to determine whether to show posts with #nsfw tags +struct NSFWTagFilter: ContentFilter { + func filter(ev: NostrEvent) -> Bool { + return ev.referenced_hashtags.first(where: { t in t.hashtag == "nsfw" }) == nil + } +} + +/// Generic filter with various tweakable settings +struct DamusFilter: ContentFilter { + let hide_nsfw_tagged_content: Bool + + func filter(ev: NostrEvent) -> Bool { + if self.hide_nsfw_tagged_content { + return NSFWTagFilter().filter(ev: ev) + } + else { + return true + } + } + + func get_filter(_ filter_state: FilterState) -> ((NostrEvent) -> Bool) { + return { ev in + return filter_state.filter(ev: ev) && self.filter(ev: ev) + } + } + +} diff --git a/damus/Models/UserSettingsStore.swift b/damus/Models/UserSettingsStore.swift index 862c71a5cc..f9a4d9bf5f 100644 --- a/damus/Models/UserSettingsStore.swift +++ b/damus/Models/UserSettingsStore.swift @@ -109,6 +109,9 @@ class UserSettingsStore: ObservableObject { @Setting(key: "always_show_images", default_value: false) var always_show_images: Bool + + @Setting(key: "hide_nsfw_tagged_content", default_value: false) + var hide_nsfw_tagged_content: Bool @Setting(key: "zap_vibration", default_value: true) var zap_vibration: Bool diff --git a/damus/Views/ConfigView.swift b/damus/Views/ConfigView.swift index b77308f79f..b0d67a4a52 100644 --- a/damus/Views/ConfigView.swift +++ b/damus/Views/ConfigView.swift @@ -41,7 +41,7 @@ struct ConfigView: View { } NavigationLink(value: Route.AppearanceSettings(settings: settings)) { - IconLabel(NSLocalizedString("Appearance", comment: "Section header for text and appearance settings"), img_name: "eye", color: .red) + IconLabel(NSLocalizedString("Appearance and filters", comment: "Section header for text, appearance, and content filter settings"), img_name: "eye", color: .red) } NavigationLink(value: Route.SearchSettings(settings: settings)) { diff --git a/damus/Views/SearchHomeView.swift b/damus/Views/SearchHomeView.swift index 9b5a3e347f..2ce425326a 100644 --- a/damus/Views/SearchHomeView.swift +++ b/damus/Views/SearchHomeView.swift @@ -14,6 +14,11 @@ struct SearchHomeView: View { @StateObject var model: SearchHomeModel @State var search: String = "" @FocusState private var isFocused: Bool + var damus_filter: DamusFilter { + get { + return DamusFilter(hide_nsfw_tagged_content: self.damus_state.settings.hide_nsfw_tagged_content) + } + } let preferredLanguages = Set(Locale.preferredLanguages.map { localeToLanguage($0) }) @@ -50,6 +55,10 @@ struct SearchHomeView: View { damus: damus_state, show_friend_icon: true, filter: { ev in + if !damus_filter.filter(ev: ev) { + return false + } + if damus_state.muted_threads.isMutedThread(ev, keypair: self.damus_state.keypair) { return false } diff --git a/damus/Views/Settings/AppearanceSettingsView.swift b/damus/Views/Settings/AppearanceSettingsView.swift index 899347acbb..c1062678d7 100644 --- a/damus/Views/Settings/AppearanceSettingsView.swift +++ b/damus/Views/Settings/AppearanceSettingsView.swift @@ -85,6 +85,15 @@ struct AppearanceSettingsView: View { clear_kingfisher_cache() } } + + // MARK: - Content filters and moderation + Section( + header: Text(NSLocalizedString("Content filters", comment: "Section title for content filtering/moderation configuration.")), + footer: Text(NSLocalizedString("Notes with the #nsfw tag usually contains adult content or other \"Not safe for work\" content", comment: "Section footer clarifying what #nsfw (not safe for work) tags mean")) + ) { + Toggle(NSLocalizedString("Hide notes with #nsfw tags", comment: "Setting to hide notes with the #nsfw (not safe for work) tags"), isOn: $settings.hide_nsfw_tagged_content) + .toggleStyle(.switch) + } } From 440e37c1d396134fb3124389a06a267aec486bea Mon Sep 17 00:00:00 2001 From: William Casarin Date: Thu, 21 Sep 2023 08:30:23 -0400 Subject: [PATCH 101/111] filters: generalize ContentFilter This simplifies our content filters so that it is a bit more flexible for future additions. Fixes: 0957cc896cc8 ("Add "Do not show #nsfw tagged posts" setting") --- damus/ContentView.swift | 19 +++++++------- damus/Models/ContentFilters.swift | 42 ++++++++++++++----------------- damus/Views/SearchHomeView.swift | 10 ++++---- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/damus/ContentView.swift b/damus/ContentView.swift index 6e57fef734..d4b70eae39 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -82,12 +82,7 @@ struct ContentView: View { @StateObject var navigationCoordinator: NavigationCoordinator = NavigationCoordinator() @AppStorage("has_seen_suggested_users") private var hasSeenSuggestedUsers = false let sub_id = UUID().description - var damus_filter: DamusFilter { - get { - return DamusFilter(hide_nsfw_tagged_content: self.damus_state?.settings.hide_nsfw_tagged_content ?? true) - } - } - + @Environment(\.colorScheme) var colorScheme // connect retry timer @@ -97,7 +92,13 @@ struct ContentView: View { Text("Are you lost?", comment: "Text asking the user if they are lost in the app.") .id("what") } - + + func content_filter(_ fstate: FilterState) -> ((NostrEvent) -> Bool) { + var filters = ContentFilters.defaults(damus_state!.settings) + filters.append(fstate.filter) + return ContentFilters(filters: filters).filter + } + var PostingTimelineView: some View { VStack { ZStack { @@ -105,10 +106,10 @@ struct ContentView: View { // This is needed or else there is a bug when switching from the 3rd or 2nd tab to first. no idea why. mystery - contentTimelineView(filter: damus_filter.get_filter(.posts)) + contentTimelineView(filter: content_filter(.posts)) .tag(FilterState.posts) .id(FilterState.posts) - contentTimelineView(filter: damus_filter.get_filter(.posts_and_replies)) + contentTimelineView(filter: content_filter(.posts_and_replies)) .tag(FilterState.posts_and_replies) .id(FilterState.posts_and_replies) } diff --git a/damus/Models/ContentFilters.swift b/damus/Models/ContentFilters.swift index 3aadbd41cd..8950dce372 100644 --- a/damus/Models/ContentFilters.swift +++ b/damus/Models/ContentFilters.swift @@ -7,15 +7,9 @@ import Foundation -protocol ContentFilter { - /// Function that implements the content filtering logic - /// - Parameter ev: The nostr event to be processed - /// - Returns: Must return `true` to show events, and return `false` to hide/filter events - func filter(ev: NostrEvent) -> Bool -} /// Simple filter to determine whether to show posts or all posts and replies. -enum FilterState : Int, ContentFilter { +enum FilterState : Int { case posts_and_replies = 1 case posts = 0 @@ -30,29 +24,31 @@ enum FilterState : Int, ContentFilter { } /// Simple filter to determine whether to show posts with #nsfw tags -struct NSFWTagFilter: ContentFilter { - func filter(ev: NostrEvent) -> Bool { +func nsfw_tag_filter(ev: NostrEvent) -> Bool { return ev.referenced_hashtags.first(where: { t in t.hashtag == "nsfw" }) == nil - } } /// Generic filter with various tweakable settings -struct DamusFilter: ContentFilter { - let hide_nsfw_tagged_content: Bool - +struct ContentFilters { + var filters: [(NostrEvent) -> Bool] + func filter(ev: NostrEvent) -> Bool { - if self.hide_nsfw_tagged_content { - return NSFWTagFilter().filter(ev: ev) - } - else { - return true + for filter in filters { + if !filter(ev) { + return false + } } + + return true } - - func get_filter(_ filter_state: FilterState) -> ((NostrEvent) -> Bool) { - return { ev in - return filter_state.filter(ev: ev) && self.filter(ev: ev) +} + +extension ContentFilters { + static func defaults(_ settings: UserSettingsStore) -> [(NostrEvent) -> Bool] { + var filters = Array<(NostrEvent) -> Bool>() + if settings.hide_nsfw_tagged_content { + filters.append(nsfw_tag_filter) } + return filters } - } diff --git a/damus/Views/SearchHomeView.swift b/damus/Views/SearchHomeView.swift index 2ce425326a..2eed1664de 100644 --- a/damus/Views/SearchHomeView.swift +++ b/damus/Views/SearchHomeView.swift @@ -14,10 +14,10 @@ struct SearchHomeView: View { @StateObject var model: SearchHomeModel @State var search: String = "" @FocusState private var isFocused: Bool - var damus_filter: DamusFilter { - get { - return DamusFilter(hide_nsfw_tagged_content: self.damus_state.settings.hide_nsfw_tagged_content) - } + + var content_filter: (NostrEvent) -> Bool { + let filters = ContentFilters.defaults(self.damus_state.settings) + return ContentFilters(filters: filters).filter } let preferredLanguages = Set(Locale.preferredLanguages.map { localeToLanguage($0) }) @@ -55,7 +55,7 @@ struct SearchHomeView: View { damus: damus_state, show_friend_icon: true, filter: { ev in - if !damus_filter.filter(ev: ev) { + if !content_filter(ev) { return false } From 8586eed635afa595a6cc9143e633706e6a0f9742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=E2=80=99Aquino?= Date: Wed, 20 Sep 2023 19:01:47 +0000 Subject: [PATCH 102/111] ui: add followed hashtags to FollowingView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When users view who a certain person follows, now they will see an extra tab to see the hashtags that they follow. This new tab contains a list of followed hashtags, each of which includes an option to follow/unfollow the hashtag, as well as the ability to visit the hashtag timeline Testing **iOS:** 17.0 (iPhone 14 Pro Simulator) **Damus:** (This commit) **Test steps:** 1. Go to search view, search for a couple of hashtags: #apple, #orange, #kiwi 2. Go to the test accounts own profile via the drawer menu 3. Click on "Following". Make sure there are two tabs now. 4. Scroll down, switch tabs between "People" and "Hashtags". Make sure that scrolling and switching tabs work 5. Unfollow and follow a user. Make sure that this still works 6. Make sure that #apple, #orange, #kiwi hashtags are visible under the "Hashtags" tab 7. Unfollow "#kiwi". Check that the button label now switches from "Unfollow" to "Follow" 8. Click on "#kiwi". Make sure that it takes you to the page where posts with that hashtag appears 9. Go to @jb55's profile 10. Click on "Following" 11. Ensure that there is a "Hashtags" tab 12. Check that @jb55's followed hashtags are shown (not your own) 13. Follow one of the same hashtags as @jb55's 14. Go back to your own profile and go to your own following view again. 15. Make sure that this newly added tag is present on the list, and that #kiwi is not. Closes: https://github.com/damus-io/damus/issues/606 Changelog-Added: Add followed hashtags to your following list Signed-off-by: Daniel D’Aquino Reviewed-by: William Casarin Signed-off-by: William Casarin --- .../Components/Search/SearchHeaderView.swift | 130 ++++++++++++------ damus/Models/Contacts.swift | 5 + damus/Models/FollowingModel.swift | 4 +- damus/Nostr/Id.swift | 2 +- damus/TestData.swift | 2 + damus/Views/FollowingView.swift | 95 ++++++++++++- damus/Views/Profile/ProfileView.swift | 3 +- 7 files changed, 188 insertions(+), 53 deletions(-) diff --git a/damus/Components/Search/SearchHeaderView.swift b/damus/Components/Search/SearchHeaderView.swift index 743f096e3f..78ce111a6c 100644 --- a/damus/Components/Search/SearchHeaderView.swift +++ b/damus/Components/Search/SearchHeaderView.swift @@ -25,22 +25,11 @@ struct SearchHeaderView: View { var Icon: some View { ZStack { - Circle() - .fill(Color(red: 0xF8/255.0, green: 0xE7/255.0, blue: 0xF8/255.0)) - .frame(width: 54, height: 54) - switch described { - case .hashtag: - Text(verbatim: "#") - .font(.largeTitle.bold()) - .foregroundStyle(PinkGradient) - .mask(Text(verbatim: "#") - .font(.largeTitle.bold())) - - case .unknown: - Image(systemName: "magnifyingglass") - .font(.title.bold()) - .foregroundStyle(PinkGradient) + case .hashtag: + SingleCharacterAvatar(character: "#") + case .unknown: + SystemIconAvatar(system_name: "magnifyingglass") } } } @@ -49,32 +38,6 @@ struct SearchHeaderView: View { Text(described.description) } - func unfollow(_ hashtag: String) { - is_following = false - handle_unfollow(state: state, unfollow: FollowRef.hashtag(hashtag)) - } - - func follow(_ hashtag: String) { - is_following = true - handle_follow(state: state, follow: .hashtag(hashtag)) - } - - func FollowButton(_ ht: String) -> some View { - return Button(action: { follow(ht) }) { - Text("Follow hashtag", comment: "Button to follow a given hashtag.") - .font(.footnote.bold()) - } - .buttonStyle(GradientButtonStyle(padding: 10)) - } - - func UnfollowButton(_ ht: String) -> some View { - return Button(action: { unfollow(ht) }) { - Text("Unfollow hashtag", comment: "Button to unfollow a given hashtag.") - .font(.footnote.bold()) - } - .buttonStyle(GradientButtonStyle(padding: 10)) - } - var body: some View { HStack(alignment: .center, spacing: 30) { Icon @@ -86,9 +49,9 @@ struct SearchHeaderView: View { if state.is_privkey_user, case .hashtag(let ht) = described { if is_following { - UnfollowButton(ht) + HashtagUnfollowButton(damus_state: state, hashtag: ht, is_following: $is_following) } else { - FollowButton(ht) + HashtagFollowButton(damus_state: state, hashtag: ht, is_following: $is_following) } } } @@ -104,6 +67,87 @@ struct SearchHeaderView: View { } } +struct SystemIconAvatar: View { + let system_name: String + + var body: some View { + NonImageAvatar { + Image(systemName: system_name) + .font(.title.bold()) + } + } +} + +struct SingleCharacterAvatar: View { + let character: String + + var body: some View { + NonImageAvatar { + Text(verbatim: character) + .font(.largeTitle.bold()) + .mask(Text(verbatim: character) + .font(.largeTitle.bold())) + } + } +} + +struct NonImageAvatar: View { + let content: Content + + init(@ViewBuilder content: () -> Content) { + self.content = content() + } + + var body: some View { + ZStack { + Circle() + .fill(Color(red: 0xF8/255.0, green: 0xE7/255.0, blue: 0xF8/255.0)) + .frame(width: 54, height: 54) + + content + .foregroundStyle(PinkGradient) + } + } +} + +struct HashtagUnfollowButton: View { + let damus_state: DamusState + let hashtag: String + @Binding var is_following: Bool + + var body: some View { + return Button(action: { unfollow(hashtag) }) { + Text("Unfollow hashtag", comment: "Button to unfollow a given hashtag.") + .font(.footnote.bold()) + } + .buttonStyle(GradientButtonStyle(padding: 10)) + } + + func unfollow(_ hashtag: String) { + is_following = false + handle_unfollow(state: damus_state, unfollow: FollowRef.hashtag(hashtag)) + } +} + +struct HashtagFollowButton: View { + let damus_state: DamusState + let hashtag: String + @Binding var is_following: Bool + + var body: some View { + return Button(action: { follow(hashtag) }) { + Text("Follow hashtag", comment: "Button to follow a given hashtag.") + .font(.footnote.bold()) + } + .buttonStyle(GradientButtonStyle(padding: 10)) + } + + func follow(_ hashtag: String) { + is_following = true + handle_follow(state: damus_state, follow: .hashtag(hashtag)) + } +} + func hashtag_matches_search(desc: DescribedSearch, ref: FollowRef) -> Bool { guard case .hashtag(let follow_ht) = ref, case .hashtag(let search_ht) = desc, diff --git a/damus/Models/Contacts.swift b/damus/Models/Contacts.swift index 9c07c44c54..afb2cb94a1 100644 --- a/damus/Models/Contacts.swift +++ b/damus/Models/Contacts.swift @@ -74,6 +74,11 @@ class Contacts { guard let ev = self.event else { return Set() } return Set(ev.referenced_hashtags.map({ $0.hashtag })) } + + func follows(hashtag: Hashtag) -> Bool { + guard let ev = self.event else { return false } + return ev.referenced_hashtags.first(where: { $0 == hashtag }) != nil + } func add_friend_pubkey(_ pubkey: Pubkey) { friends.insert(pubkey) diff --git a/damus/Models/FollowingModel.swift b/damus/Models/FollowingModel.swift index 5a2306a2e6..4b70ec3bc1 100644 --- a/damus/Models/FollowingModel.swift +++ b/damus/Models/FollowingModel.swift @@ -12,12 +12,14 @@ class FollowingModel { var needs_sub: Bool = true let contacts: [Pubkey] + let hashtags: [Hashtag] let sub_id: String = UUID().description - init(damus_state: DamusState, contacts: [Pubkey]) { + init(damus_state: DamusState, contacts: [Pubkey], hashtags: [Hashtag]) { self.damus_state = damus_state self.contacts = contacts + self.hashtags = hashtags } func get_filter() -> NostrFilter { diff --git a/damus/Nostr/Id.swift b/damus/Nostr/Id.swift index 912736b5b4..f97bf7a6cd 100644 --- a/damus/Nostr/Id.swift +++ b/damus/Nostr/Id.swift @@ -65,7 +65,7 @@ struct Privkey: IdType { } -struct Hashtag: TagConvertible { +struct Hashtag: TagConvertible, Hashable { let hashtag: String static func from_tag(tag: TagSequence) -> Hashtag? { diff --git a/damus/TestData.swift b/damus/TestData.swift index 2f225ba201..16278ff4d2 100644 --- a/damus/TestData.swift +++ b/damus/TestData.swift @@ -48,6 +48,8 @@ let test_private_zap = Zap(event: test_note, invoice: test_zap_invoice, zapper: let test_pending_zap = PendingZap(amount_msat: 10000, target: .note(id: test_note.id, author: test_note.pubkey), request: .normal(test_zap_request), type: .pub, state: .external(.init(state: .fetching_invoice))) +let test_following_model = FollowingModel(damus_state: test_damus_state(), contacts: [test_pubkey, test_pubkey_2], hashtags: [Hashtag(hashtag: "grownostr"), Hashtag(hashtag: "zapathon")]) + func test_damus_state() -> DamusState { let damus = DamusState.empty diff --git a/damus/Views/FollowingView.swift b/damus/Views/FollowingView.swift index 2f2693f63a..6cb911c784 100644 --- a/damus/Views/FollowingView.swift +++ b/damus/Views/FollowingView.swift @@ -24,6 +24,53 @@ struct FollowUserView: View { } } +struct FollowHashtagView: View { + let hashtag: Hashtag + let damus_state: DamusState + @State var is_following: Bool + + init(hashtag: Hashtag, damus_state: DamusState) { + self.hashtag = hashtag + self.damus_state = damus_state + self.is_following = damus_state.contacts.follows(hashtag: hashtag) + } + + var body: some View { + HStack { + HStack { + SingleCharacterAvatar(character: "#") + + Text("#\(hashtag.hashtag)") + .bold() + } + .onTapGesture { + let search = SearchModel(state: damus_state, search: NostrFilter.init(hashtag: [hashtag.hashtag])) + damus_state.nav.push(route: Route.Search(search: search)) + } + + Spacer() + if is_following { + HashtagUnfollowButton(damus_state: damus_state, hashtag: hashtag.hashtag, is_following: $is_following) + } + else { + HashtagFollowButton(damus_state: damus_state, hashtag: hashtag.hashtag, is_following: $is_following) + } + } + .onReceive(handle_notify(.followed)) { follow in + guard case .hashtag(let ht) = follow, ht == hashtag.hashtag else { + return + } + self.is_following = true + } + .onReceive(handle_notify(.unfollowed)) { follow in + guard case .hashtag(let ht) = follow, ht == hashtag.hashtag else { + return + } + self.is_following = false + } + } +} + struct FollowersYouKnowView: View { let damus_state: DamusState let friended_followers: [Pubkey] @@ -65,21 +112,44 @@ struct FollowersView: View { } } +enum FollowingViewTabSelection: Int { + case people = 0 + case hashtags = 1 +} + struct FollowingView: View { let damus_state: DamusState let following: FollowingModel + @State var tab_selection: FollowingViewTabSelection = .people + @Environment(\.colorScheme) var colorScheme var body: some View { - ScrollView { - LazyVStack(alignment: .leading) { - ForEach(following.contacts.reversed(), id: \.self) { pk in - FollowUserView(target: .pubkey(pk), damus_state: damus_state) + TabView(selection: $tab_selection) { + ScrollView { + LazyVStack(alignment: .leading) { + ForEach(following.contacts.reversed(), id: \.self) { pk in + FollowUserView(target: .pubkey(pk), damus_state: damus_state) + } } + .padding() } - .padding() + .tag(FollowingViewTabSelection.people) + .id(FollowingViewTabSelection.people) + + ScrollView { + LazyVStack(alignment: .leading) { + ForEach(following.hashtags, id: \.self) { ht in + FollowHashtagView(hashtag: ht, damus_state: damus_state) + } + } + .padding() + } + .tag(FollowingViewTabSelection.hashtags) + .id(FollowingViewTabSelection.hashtags) } + .tabViewStyle(.page(indexDisplayMode: .never)) .onAppear { following.subscribe() } @@ -87,13 +157,24 @@ struct FollowingView: View { following.unsubscribe() } .navigationBarTitle(NSLocalizedString("Following", comment: "Navigation bar title for view that shows who a user is following.")) + .safeAreaInset(edge: .top, spacing: 0) { + VStack(spacing: 0) { + CustomPicker(selection: $tab_selection, content: { + Text("People", comment: "Label for filter for seeing only people follows.").tag(FollowingViewTabSelection.people) + Text("Hashtags", comment: "Label for filter for seeing only hashtag follows.").tag(FollowingViewTabSelection.hashtags) + }) + Divider() + .frame(height: 1) + } + .background(colorScheme == .dark ? Color.black : Color.white) + } } } -/* + struct FollowingView_Previews: PreviewProvider { static var previews: some View { FollowingView(damus_state: test_damus_state, following: test_following_model) } } - */ + diff --git a/damus/Views/Profile/ProfileView.swift b/damus/Views/Profile/ProfileView.swift index 6cab5c5e16..0f7ce1745d 100644 --- a/damus/Views/Profile/ProfileView.swift +++ b/damus/Views/Profile/ProfileView.swift @@ -370,7 +370,8 @@ struct ProfileView: View { HStack { if let contact = profile.contacts { let contacts = Array(contact.referenced_pubkeys) - let following_model = FollowingModel(damus_state: damus_state, contacts: contacts) + let hashtags = Array(contact.referenced_hashtags) + let following_model = FollowingModel(damus_state: damus_state, contacts: contacts, hashtags: hashtags) NavigationLink(value: Route.Following(following: following_model)) { HStack { let noun_text = Text(verbatim: "\(pluralizedString(key: "following_count", count: profile.following))").font(.subheadline).foregroundColor(.gray) From bb4fd7557639d6e2ebe1c25a4b5cbed66db1d7bc Mon Sep 17 00:00:00 2001 From: William Casarin Date: Mon, 28 Aug 2023 07:52:59 -0700 Subject: [PATCH 103/111] nostrdb: add profiles to nostrdb This adds profiles to nostrdb - Remove in-memory Profiles caches, nostrdb is as fast as an in-memory cache - Remove ProfileDatabase and just use nostrdb directly Changelog-Changed: Use nostrdb for profiles --- damus.xcodeproj/project.pbxproj | 39 +--- damus/Components/Status/UserStatusSheet.swift | 2 +- damus/ContentView.swift | 57 +++--- damus/Models/DamusState.swift | 12 +- damus/Models/FollowersModel.swift | 3 - damus/Models/FollowingModel.swift | 18 +- damus/Models/HomeModel.swift | 82 ++------ damus/Models/ProfileModel.swift | 2 - damus/Models/SearchHomeModel.swift | 18 +- damus/Models/ThreadModel.swift | 7 +- damus/Models/UserSearchCache.swift | 5 + damus/Models/ZapsModel.swift | 3 +- damus/Nostr/CoreData/PersistedProfile.swift | 39 ---- damus/Nostr/Nostr.swift | 123 ++++++++++-- damus/Nostr/ProfileDatabase.swift | 181 ------------------ damus/Nostr/Profiles.swift | 104 +++------- damus/Nostr/RelayConnection.swift | 8 +- damus/Nostr/RelayPool.swift | 17 +- damus/TestData.swift | 2 + damus/Util/AccountDeletion.swift | 13 +- damus/Util/EventCache.swift | 10 +- damus/Views/ActionBar/EventActionBar.swift | 2 +- damus/Views/Events/Components/EventTop.swift | 1 + damus/Views/Events/EventShell.swift | 1 + .../Views/Onboarding/SuggestedUserView.swift | 2 +- .../Onboarding/SuggestedUsersViewModel.swift | 8 +- damus/Views/Profile/EditMetadataView.swift | 20 +- damus/Views/Profile/EventProfileName.swift | 1 + damus/Views/Profile/ProfileName.swift | 3 +- damus/Views/Profile/ProfilePicView.swift | 6 +- damus/Views/Profile/ProfileView.swift | 45 ++--- damus/Views/SaveKeysView.swift | 2 +- damus/Views/Search/SearchingEventView.swift | 1 + damus/Views/SideMenuView.swift | 1 + damus/Views/Wallet/WalletView.swift | 11 +- damusTests/NostrScriptTests.swift | 2 +- damusTests/ProfileDatabaseTests.swift | 124 ------------ damusTests/UserSearchCacheTests.swift | 6 +- damusTests/WalletConnectTests.swift | 2 +- damusTests/ZapTests.swift | 4 +- nostrdb/Ndb.swift | 70 ++++++- nostrdb/Test/NdbTests.swift | 10 +- 42 files changed, 362 insertions(+), 705 deletions(-) delete mode 100644 damus/Nostr/CoreData/PersistedProfile.swift delete mode 100644 damus/Nostr/ProfileDatabase.swift delete mode 100644 damusTests/ProfileDatabaseTests.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index a02d931f55..b69ae0574e 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -355,6 +355,7 @@ 4CEE2AF5280B29E600AB5EEF /* TimeAgo.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2AF4280B29E600AB5EEF /* TimeAgo.swift */; }; 4CEE2AF7280B2DEA00AB5EEF /* ProfileName.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2AF6280B2DEA00AB5EEF /* ProfileName.swift */; }; 4CEE2B02280B39E800AB5EEF /* EventActionBar.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2B01280B39E800AB5EEF /* EventActionBar.swift */; }; + 4CEF958D2A9CE650000F901B /* verifier.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792D42A9939BD00489948 /* verifier.c */; }; 4CF0ABD42980996B00D66079 /* Report.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABD32980996B00D66079 /* Report.swift */; }; 4CF0ABD629817F5B00D66079 /* ReportView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABD529817F5B00D66079 /* ReportView.swift */; }; 4CF0ABD82981980C00D66079 /* Lists.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABD72981980C00D66079 /* Lists.swift */; }; @@ -377,10 +378,6 @@ 4CFF8F6D29CD022E008DB934 /* WideEventView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CFF8F6C29CD022E008DB934 /* WideEventView.swift */; }; 4FE60CDD295E1C5E00105A1F /* Wallet.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4FE60CDC295E1C5E00105A1F /* Wallet.swift */; }; 50088DA129E8271A008A1FDF /* WebSocket.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50088DA029E8271A008A1FDF /* WebSocket.swift */; }; - 5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */; }; - 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */; }; - 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */; }; - 501F8C5A29FF70F5001AFC1D /* ProfileDatabase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */; }; 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */; }; 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */; }; 504323A72A34915F006AE6DC /* RelayModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 504323A62A34915F006AE6DC /* RelayModel.swift */; }; @@ -1059,10 +1056,6 @@ 4CFF8F6C29CD022E008DB934 /* WideEventView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WideEventView.swift; sourceTree = ""; }; 4FE60CDC295E1C5E00105A1F /* Wallet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Wallet.swift; sourceTree = ""; }; 50088DA029E8271A008A1FDF /* WebSocket.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebSocket.swift; sourceTree = ""; }; - 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileDatabaseTests.swift; sourceTree = ""; }; - 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersistedProfile.swift; sourceTree = ""; }; - 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = Damus.xcdatamodel; sourceTree = ""; }; - 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileDatabase.swift; sourceTree = ""; }; 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorage.swift; sourceTree = ""; }; 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorageTests.swift; sourceTree = ""; }; 504323A62A34915F006AE6DC /* RelayModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayModel.swift; sourceTree = ""; }; @@ -1691,7 +1684,6 @@ 4C75EFAB28049CC80006080F /* Nostr */ = { isa = PBXGroup; children = ( - 501F8C5329FF5EE2001AFC1D /* CoreData */, 4CE6DF1527F8DEBF00C66700 /* RelayConnection.swift */, 50A60D132A28BEEE00186190 /* RelayLog.swift */, 4C75EFA527FF87A20006080F /* Nostr.swift */, @@ -1704,7 +1696,6 @@ 4C75EFBA2804A34C0006080F /* ProofOfWork.swift */, 4CEE2AEC2805B22500AB5EEF /* NostrRequest.swift */, 4CACA9DB280C38C000D9BBE8 /* Profiles.swift */, - 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */, 4C3BEFD32819DE8F00B3DE84 /* NostrKind.swift */, 4C363A8F28247A1D006E126D /* NostrLink.swift */, 50088DA029E8271A008A1FDF /* WebSocket.swift */, @@ -2137,7 +2128,6 @@ 4CB883A9297612FF00DC99E7 /* ZapTests.swift */, 4CB883AD2976FA9300DC99E7 /* FormatTests.swift */, 3A3040EC29A5CB86008A0F29 /* ReplyDescriptionTests.swift */, - 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */, 3A3040F229A91366008A0F29 /* ProfileViewTests.swift */, 3A30410029AB12AA008A0F29 /* EventGroupViewTests.swift */, 4C8D00D329E3C5D40036AF10 /* NIP19Tests.swift */, @@ -2249,15 +2239,6 @@ path = Images; sourceTree = ""; }; - 501F8C5329FF5EE2001AFC1D /* CoreData */ = { - isa = PBXGroup; - children = ( - 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */, - 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */, - ); - path = CoreData; - sourceTree = ""; - }; 7C0F392D29B57C8F0039859C /* Extensions */ = { isa = PBXGroup; children = ( @@ -2497,6 +2478,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 4CEF958D2A9CE650000F901B /* verifier.c in Sources */, 4C32B9342A9AD01A00DC3548 /* NdbProfile.swift in Sources */, 4C32B9332A99845B00DC3548 /* Ndb.swift in Sources */, 4C4793082A993E8900489948 /* refmap.c in Sources */, @@ -2624,7 +2606,6 @@ 4CD348EF29C3659D00497EB2 /* ImageUploadModel.swift in Sources */, 4C7D096E2A0AEA0400943473 /* ScannerCoordinator.swift in Sources */, 4C3BEFDC281DCE6100B3DE84 /* Liked.swift in Sources */, - 501F8C5A29FF70F5001AFC1D /* ProfileDatabase.swift in Sources */, 4CF0ABE7298444FD00D66079 /* MutedEventView.swift in Sources */, 9C83F89329A937B900136C08 /* TextViewWrapper.swift in Sources */, 4C1253502A76C5B20004F4B8 /* UnfollowedNotify.swift in Sources */, @@ -2757,7 +2738,6 @@ 4CB8838629656C8B00DC99E7 /* NIP05.swift in Sources */, 4CF0ABD82981980C00D66079 /* Lists.swift in Sources */, F71694EA2A662232001F4053 /* SuggestedUsersView.swift in Sources */, - 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */, 4C12536A2A76D3850004F4B8 /* RelaysChangedNotify.swift in Sources */, 4C30AC8029A6A53F00E2BD5A /* ProfilePicturesView.swift in Sources */, 5C6E1DAD2A193EC2008FC15A /* GradientButtonStyle.swift in Sources */, @@ -2848,7 +2828,6 @@ 4CE879552996BAB900F758CC /* RelayPaidDetail.swift in Sources */, 4C1253602A76CF890004F4B8 /* ScrollToTopNotify.swift in Sources */, 4CA3529E2A76AE67003BB08B /* FollowNotify.swift in Sources */, - 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */, 4CF0ABD42980996B00D66079 /* Report.swift in Sources */, 4C06670B28FDE64700038D2A /* damus.c in Sources */, 4C1253642A76D08F0004F4B8 /* ReportNotify.swift in Sources */, @@ -2893,7 +2872,6 @@ buildActionMask = 2147483647; files = ( 4C684A572A7FFAE6005E6031 /* UrlTests.swift in Sources */, - 5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */, 3A90B1832A4EA3C600000D94 /* UserSearchCacheTests.swift in Sources */, 4C9B0DEE2A65A75F00CBDA21 /* AttrStringTestExtensions.swift in Sources */, 4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */, @@ -3451,19 +3429,6 @@ productName = secp256k1; }; /* End XCSwiftPackageProductDependency section */ - -/* Begin XCVersionGroup section */ - 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */ = { - isa = XCVersionGroup; - children = ( - 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */, - ); - currentVersion = 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */; - path = Damus.xcdatamodeld; - sourceTree = ""; - versionGroupType = wrapper.xcdatamodel; - }; -/* End XCVersionGroup section */ }; rootObject = 4CE6DEDB27F7A08100C66700 /* Project object */; } diff --git a/damus/Components/Status/UserStatusSheet.swift b/damus/Components/Status/UserStatusSheet.swift index 9a76d5c313..00e9dba8b7 100644 --- a/damus/Components/Status/UserStatusSheet.swift +++ b/damus/Components/Status/UserStatusSheet.swift @@ -136,6 +136,6 @@ struct UserStatusSheet: View { struct UserStatusSheet_Previews: PreviewProvider { static var previews: some View { - UserStatusSheet(postbox: PostBox(pool: RelayPool()), keypair: Keypair(pubkey: .empty, privkey: nil), status: .init()) + UserStatusSheet(postbox: PostBox(pool: RelayPool(ndb: .empty)), keypair: Keypair(pubkey: .empty, privkey: nil), status: .init()) } } diff --git a/damus/ContentView.swift b/damus/ContentView.swift index d4b70eae39..9027b461d2 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -9,12 +9,6 @@ import SwiftUI import AVKit import MediaPlayer -struct TimestampedProfile { - let profile: Profile - let timestamp: UInt32 - let event: NostrEvent -} - struct ZapSheet { let target: ZapTarget let lnurl: String @@ -378,10 +372,9 @@ struct ContentView: View { invalidate_zapper_cache(pubkey: keypair.pubkey, profiles: ds.profiles, lnurl: ds.lnurls) } - profile.lud16 = lud16 - guard let ev = make_metadata_event(keypair: keypair, metadata: profile) else { - return - } + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: lud16, nip05: profile.nip05, damus_donation: profile.damus_donation, reactions: profile.reactions) + + guard let ev = make_metadata_event(keypair: keypair, metadata: prof) else { return } ds.postbox.send(ev) } .onReceive(handle_notify(.broadcast)) { ev in @@ -389,8 +382,10 @@ struct ContentView: View { return } ds.postbox.send(ev) - if let profile = ds.profiles.lookup_with_timestamp(id: ev.pubkey) { - ds.postbox.send(profile.event) + if let record = ds.profiles.lookup_with_timestamp(ev.pubkey), + let event = ds.events.lookup_by_key(record.noteKey) + { + ds.postbox.send(event) } } .onReceive(handle_notify(.unfollow)) { target in @@ -500,11 +495,10 @@ struct ContentView: View { else { return } - - profile.reactions = !hide - guard let profile_ev = make_metadata_event(keypair: keypair, metadata: profile) else { - return - } + + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: profile.damus_donation, reactions: !hide) + + guard let profile_ev = make_metadata_event(keypair: keypair, metadata: prof) else { return } damus_state.postbox.send(profile_ev) } .alert(NSLocalizedString("User muted", comment: "Alert message to indicate the user has been muted"), isPresented: $user_muted_confirm, actions: { @@ -597,7 +591,10 @@ struct ContentView: View { } func connect() { - let pool = RelayPool() + // nostrdb + let ndb = Ndb()! + + let pool = RelayPool(ndb: ndb) let model_cache = RelayModelCache() let relay_filters = RelayFilters(our_pubkey: pubkey) let bootstrap_relays = load_bootstrap_relays(pubkey: pubkey) @@ -622,13 +619,14 @@ struct ContentView: View { try? pool.add_relay(.nwc(url: nwc.relay)) } + let user_search_cache = UserSearchCache() self.damus_state = DamusState(pool: pool, keypair: keypair, likes: EventCounter(our_pubkey: pubkey), boosts: EventCounter(our_pubkey: pubkey), contacts: Contacts(our_pubkey: pubkey), - profiles: Profiles(user_search_cache: user_search_cache), + profiles: Profiles(user_search_cache: user_search_cache, ndb: ndb), dms: home.dms, previews: PreviewCache(), zaps: Zaps(our_pubkey: pubkey), @@ -637,7 +635,7 @@ struct ContentView: View { relay_filters: relay_filters, relay_model_cache: model_cache, drafts: Drafts(), - events: EventCache(), + events: EventCache(ndb: ndb), bookmarks: BookmarksManager(pubkey: pubkey), postbox: PostBox(pool: pool), bootstrap_relays: bootstrap_relays, @@ -647,7 +645,8 @@ struct ContentView: View { nav: self.navigationCoordinator, user_search_cache: user_search_cache, music: MusicController(onChange: music_changed), - video: VideoController() + video: VideoController(), + ndb: ndb ) home.damus_state = self.damus_state! @@ -817,8 +816,11 @@ func find_event_with_subid(state: DamusState, query query_: FindEvent, subid: St switch query { case .profile(let pubkey): - if let profile = state.profiles.lookup_with_timestamp(id: pubkey) { - callback(.profile(profile.profile, profile.event)) + if let record = state.profiles.lookup_with_timestamp(pubkey), + let profile = record.profile, + let event = state.events.lookup_by_key(record.noteKey) + { + callback(.profile(profile, event)) return } filter = NostrFilter(kinds: [.metadata], limit: 1, authors: [pubkey]) @@ -855,14 +857,11 @@ func find_event_with_subid(state: DamusState, query query_: FindEvent, subid: St switch query { case .profile: if ev.known_kind == .metadata { - process_metadata_event(events: state.events, our_pubkey: state.pubkey, profiles: state.profiles, ev: ev) { profile in - guard let profile else { - callback(.invalid_profile(ev)) - return - } - callback(.profile(profile, ev)) + guard let profile = state.profiles.lookup(id: ev.pubkey) else { + callback(.invalid_profile(ev)) return } + callback(.profile(profile, ev)) } case .event: callback(.event(ev)) diff --git a/damus/Models/DamusState.swift b/damus/Models/DamusState.swift index 5811235ea6..d8f5418b5d 100644 --- a/damus/Models/DamusState.swift +++ b/damus/Models/DamusState.swift @@ -34,6 +34,7 @@ struct DamusState { let user_search_cache: UserSearchCache let music: MusicController? let video: VideoController + let ndb: Ndb @discardableResult func add_zap(zap: Zapping) -> Bool { @@ -67,12 +68,12 @@ struct DamusState { let kp = Keypair(pubkey: empty_pub, privkey: nil) return DamusState.init( - pool: RelayPool(), + pool: RelayPool(ndb: .empty), keypair: Keypair(pubkey: empty_pub, privkey: empty_sec), likes: EventCounter(our_pubkey: empty_pub), boosts: EventCounter(our_pubkey: empty_pub), contacts: Contacts(our_pubkey: empty_pub), - profiles: Profiles(user_search_cache: user_search_cache), + profiles: Profiles(user_search_cache: user_search_cache, ndb: .empty), dms: DirectMessagesModel(our_pubkey: empty_pub), previews: PreviewCache(), zaps: Zaps(our_pubkey: empty_pub), @@ -81,9 +82,9 @@ struct DamusState { relay_filters: RelayFilters(our_pubkey: empty_pub), relay_model_cache: RelayModelCache(), drafts: Drafts(), - events: EventCache(), + events: EventCache(ndb: .empty), bookmarks: BookmarksManager(pubkey: empty_pub), - postbox: PostBox(pool: RelayPool()), + postbox: PostBox(pool: RelayPool(ndb: .empty)), bootstrap_relays: [], replies: ReplyCounter(our_pubkey: empty_pub), muted_threads: MutedThreadsManager(keypair: kp), @@ -91,7 +92,8 @@ struct DamusState { nav: NavigationCoordinator(), user_search_cache: user_search_cache, music: nil, - video: VideoController() + video: VideoController(), + ndb: .empty ) } } diff --git a/damus/Models/FollowersModel.swift b/damus/Models/FollowersModel.swift index c413cb695b..4faee3f03f 100644 --- a/damus/Models/FollowersModel.swift +++ b/damus/Models/FollowersModel.swift @@ -77,10 +77,7 @@ class FollowersModel: ObservableObject { if ev.known_kind == .contacts { handle_contact_event(ev) - } else if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) } - case .notice(let msg): print("followingmodel notice: \(msg)") diff --git a/damus/Models/FollowingModel.swift b/damus/Models/FollowingModel.swift index 4b70ec3bc1..45f84fe1ef 100644 --- a/damus/Models/FollowingModel.swift +++ b/damus/Models/FollowingModel.swift @@ -54,22 +54,6 @@ class FollowingModel { } func handle_event(relay_id: String, ev: NostrConnectionEvent) { - switch ev { - case .ws_event: - break - case .nostr_event(let nev): - switch nev { - case .ok: - break - case .event(_, let ev): - if ev.kind == 0 { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } - case .notice(let msg): - print("followingmodel notice: \(msg)") - case .eose: - break - } - } + // don't need to do anything here really } } diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index af7abc3e83..03e3b2942e 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -149,6 +149,7 @@ class HomeModel { } } + @MainActor func process_event(sub_id: String, relay_id: String, ev: NostrEvent) { if has_sub_id_event(sub_id: sub_id, ev_id: ev.id) { return @@ -169,7 +170,8 @@ class HomeModel { case .contacts: handle_contact_event(sub_id: sub_id, relay_id: relay_id, ev: ev) case .metadata: - handle_metadata_event(ev) + // profile metadata processing is handled by nostrdb + break case .list: handle_list_event(ev) case .boost: @@ -195,6 +197,7 @@ class HomeModel { } } + @MainActor func handle_status_event(_ ev: NostrEvent) { guard let st = UserStatus(ev: ev) else { return @@ -248,7 +251,8 @@ class HomeModel { nwc_success(state: self.damus_state, resp: resp) } } - + + @MainActor func handle_zap_event(_ ev: NostrEvent) { process_zap_event(damus_state: damus_state, ev: ev) { zapres in guard case .done(let zap) = zapres, @@ -373,7 +377,7 @@ class HomeModel { } } - + @MainActor func handle_event(relay_id: String, conn_event: NostrConnectionEvent) { switch conn_event { case .ws_event(let ev): @@ -582,10 +586,6 @@ class HomeModel { damus_state.contacts.set_mutelist(ev) } - func handle_metadata_event(_ ev: NostrEvent) { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } - func get_last_event_of_kind(relay_id: String, kind: UInt32) -> NostrEvent? { guard let m = last_event_of_kind[relay_id] else { last_event_of_kind[relay_id] = [:] @@ -791,45 +791,6 @@ func print_filters(relay_id: String?, filters groups: [[NostrFilter]]) { } */ -func process_metadata_profile(our_pubkey: Pubkey, profiles: Profiles, profile: Profile, ev: NostrEvent) { - var old_nip05: String? = nil - let mprof = profiles.lookup_with_timestamp(id: ev.pubkey) - - if let mprof { - old_nip05 = mprof.profile.nip05 - if mprof.event.created_at > ev.created_at { - // skip if we already have an newer profile - return - } - } - - if old_nip05 != profile.nip05 { - // if it's been validated before, invalidate it now - profiles.invalidate_nip05(ev.pubkey) - } - - let tprof = TimestampedProfile(profile: profile, timestamp: ev.created_at, event: ev) - profiles.add(id: ev.pubkey, profile: tprof) - - // load pfps asap - - var changed = false - - let picture = tprof.profile.picture ?? robohash(ev.pubkey) - if URL(string: picture) != nil { - changed = true - } - - let banner = tprof.profile.banner ?? "" - if URL(string: banner) != nil { - changed = true - } - - if changed { - notify(.profile_updated(pubkey: ev.pubkey, profile: profile)) - } -} - // TODO: remove this, let nostrdb handle all validation func guard_valid_event(events: EventCache, ev: NostrEvent, callback: @escaping () -> Void) { let validated = events.is_event_valid(ev.id) @@ -856,24 +817,6 @@ func guard_valid_event(events: EventCache, ev: NostrEvent, callback: @escaping ( } } -func process_metadata_event(events: EventCache, our_pubkey: Pubkey, profiles: Profiles, ev: NostrEvent, completion: ((Profile?) -> Void)? = nil) { - guard_valid_event(events: events, ev: ev) { - DispatchQueue.global(qos: .background).async { - guard let profile: Profile = decode_data(Data(ev.content.utf8)) else { - completion?(nil) - return - } - - profile.cache_lnurl() - - DispatchQueue.main.async { - process_metadata_profile(our_pubkey: our_pubkey, profiles: profiles, profile: profile, ev: ev) - completion?(profile) - } - } - } -} - func robohash(_ pk: Pubkey) -> String { return "https://robohash.org/" + pk.hex() } @@ -1394,6 +1337,7 @@ func get_zap_target_pubkey(ev: NostrEvent, events: EventCache) -> Pubkey? { return pk } +@MainActor func process_zap_event(damus_state: DamusState, ev: NostrEvent, completion: @escaping (ProcessZapResult) -> Void) { // These are zap notifications guard let ptag = get_zap_target_pubkey(ev: ev, events: damus_state.events) else { @@ -1417,17 +1361,13 @@ func process_zap_event(damus_state: DamusState, ev: NostrEvent, completion: @esc return } - guard let profile = damus_state.profiles.lookup(id: ptag) else { - completion(.failed) - return - } - - guard let lnurl = profile.lnurl else { + guard let record = damus_state.profiles.lookup_with_timestamp(ptag), + let lnurl = record.lnurl else { completion(.failed) return } - Task { + Task { [lnurl] in guard let zapper = await fetch_zapper_from_lnurl(lnurls: damus_state.lnurls, pubkey: ptag, lnurl: lnurl) else { completion(.failed) return diff --git a/damus/Models/ProfileModel.swift b/damus/Models/ProfileModel.swift index 17ceb74853..71229d3173 100644 --- a/damus/Models/ProfileModel.swift +++ b/damus/Models/ProfileModel.swift @@ -102,8 +102,6 @@ class ProfileModel: ObservableObject, Equatable { } } else if ev.known_kind == .contacts { handle_profile_contact_event(ev) - } else if ev.known_kind == .metadata { - process_metadata_event(events: damus.events, our_pubkey: damus.pubkey, profiles: damus.profiles, ev: ev) } seen_event.insert(ev.id) } diff --git a/damus/Models/SearchHomeModel.swift b/damus/Models/SearchHomeModel.swift index fcc76e1080..b655b3e397 100644 --- a/damus/Models/SearchHomeModel.swift +++ b/damus/Models/SearchHomeModel.swift @@ -139,21 +139,13 @@ func load_profiles(profiles_subid: String, relay_id: String, load: PubkeysToLoad authors: authors) damus_state.pool.subscribe_to(sub_id: profiles_subid, filters: [filter], to: [relay_id]) { sub_id, conn_ev in - let (sid, done) = handle_subid_event(pool: damus_state.pool, relay_id: relay_id, ev: conn_ev) { sub_id, ev in - guard sub_id == profiles_subid else { - return - } - - if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } - - } - - guard done && sid == profiles_subid else { + guard case .nostr_event(let ev) = conn_ev, + case .eose = ev, + sub_id == profiles_subid + else { return } - + print("done loading \(authors.count) profiles from \(relay_id)") damus_state.pool.unsubscribe(sub_id: profiles_subid, to: [relay_id]) } diff --git a/damus/Models/ThreadModel.swift b/damus/Models/ThreadModel.swift index 22b48acf39..9c44f32983 100644 --- a/damus/Models/ThreadModel.swift +++ b/damus/Models/ThreadModel.swift @@ -97,7 +97,8 @@ class ThreadModel: ObservableObject { event_map.insert(ev) objectWillChange.send() } - + + @MainActor func handle_event(relay_id: String, ev: NostrConnectionEvent) { let (sub_id, done) = handle_subid_event(pool: damus_state.pool, relay_id: relay_id, ev: ev) { sid, ev in @@ -105,9 +106,7 @@ class ThreadModel: ObservableObject { return } - if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } else if ev.known_kind == .zap { + if ev.known_kind == .zap { process_zap_event(damus_state: damus_state, ev: ev) { zap in } diff --git a/damus/Models/UserSearchCache.swift b/damus/Models/UserSearchCache.swift index 234ab82ca4..d50116905b 100644 --- a/damus/Models/UserSearchCache.swift +++ b/damus/Models/UserSearchCache.swift @@ -10,6 +10,9 @@ import Foundation /// Cache of searchable users by name, display_name, NIP-05 identifier, or own contact list petname. /// Optimized for fast searches of substrings by using a Trie. /// Optimal for performing user searches that could be initiated by typing quickly on a keyboard into a text input field. + +// TODO: replace with lmdb (the b tree should handle this just fine ?) +// we just need a name to profile index class UserSearchCache { private let trie = Trie() @@ -19,6 +22,7 @@ class UserSearchCache { } /// Computes the differences between an old profile, if it exists, and a new profile, and updates the user search cache accordingly. + @MainActor func updateProfile(id: Pubkey, profiles: Profiles, oldProfile: Profile?, newProfile: Profile) { // Remove searchable keys tied to the old profile if they differ from the new profile // to keep the trie clean without empty nodes while avoiding excessive graph searching. @@ -38,6 +42,7 @@ class UserSearchCache { } /// Adds a profile to the user search cache. + @MainActor private func addProfile(id: Pubkey, profiles: Profiles, profile: Profile) { // Searchable by name. if let name = profile.name { diff --git a/damus/Models/ZapsModel.swift b/damus/Models/ZapsModel.swift index 5558108ce0..3a9aeb34c0 100644 --- a/damus/Models/ZapsModel.swift +++ b/damus/Models/ZapsModel.swift @@ -37,7 +37,8 @@ class ZapsModel: ObservableObject { func unsubscribe() { state.pool.unsubscribe(sub_id: zaps_subid) } - + + @MainActor func handle_event(relay_id: String, conn_ev: NostrConnectionEvent) { guard case .nostr_event(let resp) = conn_ev else { return diff --git a/damus/Nostr/CoreData/PersistedProfile.swift b/damus/Nostr/CoreData/PersistedProfile.swift deleted file mode 100644 index 460b3f4ade..0000000000 --- a/damus/Nostr/CoreData/PersistedProfile.swift +++ /dev/null @@ -1,39 +0,0 @@ -// -// PersistedProfile.swift -// damus -// -// Created by Bryan Montz on 4/30/23. -// - -import Foundation -import CoreData - -@objc(PersistedProfile) -final class PersistedProfile: NSManagedObject { - @NSManaged var id: String? - @NSManaged var name: String? - @NSManaged var display_name: String? - @NSManaged var about: String? - @NSManaged var picture: String? - @NSManaged var banner: String? - @NSManaged var website: String? - @NSManaged var lud06: String? - @NSManaged var lud16: String? - @NSManaged var nip05: String? - @NSManaged var damus_donation: Int16 - @NSManaged var last_update: Date? // The date that the profile was last updated by the user - @NSManaged var network_pull_date: Date? // The date we got this profile from a relay (for staleness checking) - - func copyValues(from profile: Profile) { - name = profile.name - display_name = profile.display_name - about = profile.about - picture = profile.picture - banner = profile.banner - website = profile.website - lud06 = profile.lud06 - lud16 = profile.lud16 - nip05 = profile.nip05 - damus_donation = profile.damus_donation != nil ? Int16(profile.damus_donation!) : 0 - } -} diff --git a/damus/Nostr/Nostr.swift b/damus/Nostr/Nostr.swift index ea927b7a79..6e4ceec76e 100644 --- a/damus/Nostr/Nostr.swift +++ b/damus/Nostr/Nostr.swift @@ -7,6 +7,114 @@ import Foundation +typealias Profile = NdbProfile +//typealias ProfileRecord = NdbProfileRecord + +class ProfileRecord { + let data: NdbProfileRecord + + init(data: NdbProfileRecord) { + self.data = data + } + + var profile: Profile? { return data.profile } + var receivedAt: UInt64 { data.receivedAt } + var noteKey: UInt64 { data.noteKey } + + private var _lnurl: String? = nil + var lnurl: String? { + if let _lnurl { + return _lnurl + } + + guard let profile = data.profile, + let addr = profile.lud16 ?? profile.lud06 else { + return nil; + } + + if addr.contains("@") { + // this is a heavy op and is used a lot in views, cache it! + let addr = lnaddress_to_lnurl(addr); + self._lnurl = addr + return addr + } + + if !addr.lowercased().hasPrefix("lnurl") { + return nil + } + + return addr; + } + +} + +extension NdbProfile { + var display_name: String? { + return displayName + } + + static func displayName(profile: Profile?, pubkey: Pubkey) -> DisplayName { + return parse_display_name(profile: profile, pubkey: pubkey) + } + + var damus_donation: Int? { + return Int(damusDonation) + } + + var damus_donation_v2: Int { + return Int(damusDonationV2) + } + + var website_url: URL? { + if self.website?.trimmingCharacters(in: .whitespacesAndNewlines) == "" { + return nil + } + return self.website.flatMap { url in + let trim = url.trimmingCharacters(in: .whitespacesAndNewlines) + if !(trim.hasPrefix("http://") || trim.hasPrefix("https://")) { + return URL(string: "https://" + trim) + } + return URL(string: trim) + } + } + + init(name: String? = nil, display_name: String? = nil, about: String? = nil, picture: String? = nil, banner: String? = nil, website: String? = nil, lud06: String? = nil, lud16: String? = nil, nip05: String? = nil, damus_donation: Int? = nil, reactions: Bool = true) { + + var fbb = FlatBufferBuilder() + + let name_off = fbb.create(string: name) + let display_name_off = fbb.create(string: display_name) + let about_off = fbb.create(string: about) + let picture_off = fbb.create(string: picture) + let banner_off = fbb.create(string: banner) + let website_off = fbb.create(string: website) + let lud06_off = fbb.create(string: lud06) + let lud16_off = fbb.create(string: lud16) + let nip05_off = fbb.create(string: nip05) + + let profile_data = NdbProfile.createNdbProfile(&fbb, + nameOffset: name_off, + websiteOffset: website_off, + aboutOffset: about_off, + lud16Offset: lud16_off, + bannerOffset: banner_off, + displayNameOffset: display_name_off, + reactions: reactions, + pictureOffset: picture_off, + nip05Offset: nip05_off, + damusDonation: 0, + damusDonationV2: damus_donation.map({ Int32($0) }) ?? 0, + lud06Offset: lud06_off) + + fbb.finish(offset: profile_data) + + var buf = ByteBuffer(bytes: fbb.sizedByteArray) + let profile: Profile = try! getCheckedRoot(byteBuffer: &buf) + self = profile + } +} + +/* class Profile: Codable { var value: [String: AnyCodable] @@ -24,19 +132,6 @@ class Profile: Codable { self.damus_donation = damus_donation } - convenience init(persisted_profile: PersistedProfile) { - self.init(name: persisted_profile.name, - display_name: persisted_profile.display_name, - about: persisted_profile.about, - picture: persisted_profile.picture, - banner: persisted_profile.banner, - website: persisted_profile.website, - lud06: persisted_profile.lud06, - lud16: persisted_profile.lud16, - nip05: persisted_profile.nip05, - damus_donation: Int(persisted_profile.damus_donation)) - } - private func str(_ str: String) -> String? { return get_val(str) } @@ -200,6 +295,7 @@ class Profile: Codable { return parse_display_name(profile: profile, pubkey: pubkey) } } +*/ func make_test_profile() -> Profile { return Profile(name: "jb55", display_name: "Will", about: "Its a me", picture: "https://cdn.jb55.com/img/red-me.jpg", banner: "https://pbs.twimg.com/profile_banners/9918032/1531711830/600x200", website: "jb55.com", lud06: "jb55@jb55.com", lud16: nil, nip05: "jb55@jb55.com", damus_donation: 1) @@ -222,3 +318,4 @@ func lnaddress_to_lnurl(_ lnaddr: String) -> String? { return bech32_encode(hrp: "lnurl", Array(dat)) } + diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift deleted file mode 100644 index 6d4385f062..0000000000 --- a/damus/Nostr/ProfileDatabase.swift +++ /dev/null @@ -1,181 +0,0 @@ -// -// ProfileDatabase.swift -// damus -// -// Created by Bryan Montz on 4/30/23. -// - -import Foundation -import CoreData - -enum ProfileDatabaseError: Error { - case missing_context - case outdated_input -} - -final class ProfileDatabase { - - private let entity_name = "PersistedProfile" - private var persistent_container: NSPersistentContainer? - private var background_context: NSManagedObjectContext? - private let cache_url: URL - - /// This queue is used to synchronize access to the network_pull_date_cache dictionary, which - /// prevents data races from crashing the app. - private var queue = DispatchQueue(label: "io.damus.profile_db", - qos: .userInteractive, - attributes: .concurrent) - private var network_pull_date_cache = [Pubkey: Date]() - - init(cache_url: URL = ProfileDatabase.profile_cache_url) { - self.cache_url = cache_url - set_up() - } - - private static var profile_cache_url: URL { - (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("profiles"))! - } - - private var persistent_store_description: NSPersistentStoreDescription { - let description = NSPersistentStoreDescription(url: cache_url) - description.type = NSSQLiteStoreType - description.setOption(true as NSNumber, forKey: NSMigratePersistentStoresAutomaticallyOption) - description.setOption(true as NSNumber, forKey: NSInferMappingModelAutomaticallyOption) - description.setOption(true as NSNumber, forKey: NSSQLiteManualVacuumOption) - return description - } - - private var object_model: NSManagedObjectModel? { - guard let url = Bundle.main.url(forResource: "Damus", withExtension: "momd") else { - return nil - } - return NSManagedObjectModel(contentsOf: url) - } - - private func set_up() { - guard let object_model else { - print("⚠️ Warning: ProfileDatabase failed to load its object model") - return - } - - persistent_container = NSPersistentContainer(name: "Damus", managedObjectModel: object_model) - persistent_container?.persistentStoreDescriptions = [persistent_store_description] - persistent_container?.loadPersistentStores { _, error in - if let error { - print("WARNING: ProfileDatabase failed to load: \(error)") - } - } - - persistent_container?.viewContext.automaticallyMergesChangesFromParent = true - persistent_container?.viewContext.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) - - background_context = persistent_container?.newBackgroundContext() - background_context?.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) - } - - private func get_persisted(id: Pubkey, context: NSManagedObjectContext) -> PersistedProfile? { - let request = NSFetchRequest(entityName: entity_name) - request.predicate = NSPredicate(format: "id == %@", id.hex()) - request.fetchLimit = 1 - return try? context.fetch(request).first - } - - func get_network_pull_date(id: Pubkey) -> Date? { - var pull_date: Date? - queue.sync { - pull_date = network_pull_date_cache[id] - } - if let pull_date { - return pull_date - } - - let request = NSFetchRequest(entityName: entity_name) - request.predicate = NSPredicate(format: "id == %@", id.hex()) - request.fetchLimit = 1 - request.propertiesToFetch = ["network_pull_date"] - guard let profile = try? persistent_container?.viewContext.fetch(request).first else { - return nil - } - - queue.async(flags: .barrier) { - self.network_pull_date_cache[id] = profile.network_pull_date - } - return profile.network_pull_date - } - - // MARK: - Public - - /// Updates or inserts a new Profile into the local database. Rejects profiles whose update date - /// is older than one we already have. Database writes occur on a background context for best performance. - /// - Parameters: - /// - id: Profile id (pubkey) - /// - profile: Profile object to be stored - /// - last_update: Date that the Profile was updated - func upsert(id: Pubkey, profile: Profile, last_update: Date) async throws { - guard let context = background_context else { - throw ProfileDatabaseError.missing_context - } - - try await context.perform { - var persisted_profile: PersistedProfile? - if let profile = self.get_persisted(id: id, context: context) { - if let existing_last_update = profile.last_update, last_update < existing_last_update { - throw ProfileDatabaseError.outdated_input - } else { - persisted_profile = profile - } - } else { - persisted_profile = NSEntityDescription.insertNewObject(forEntityName: self.entity_name, into: context) as? PersistedProfile - persisted_profile?.id = id.hex() - } - persisted_profile?.copyValues(from: profile) - persisted_profile?.last_update = last_update - - let pull_date = Date.now - persisted_profile?.network_pull_date = pull_date - self.queue.async(flags: .barrier) { - self.network_pull_date_cache[id] = pull_date - } - - try context.save() - } - } - - func get(id: Pubkey) -> Profile? { - guard let container = persistent_container, - let profile = get_persisted(id: id, context: container.viewContext) else { - return nil - } - return Profile(persisted_profile: profile) - } - - var count: Int { - let request = NSFetchRequest(entityName: entity_name) - let count = try? persistent_container?.viewContext.count(for: request) - return count ?? 0 - } - - func remove_all_profiles() throws { - guard let context = background_context, let container = persistent_container else { - throw ProfileDatabaseError.missing_context - } - - queue.async(flags: .barrier) { - self.network_pull_date_cache.removeAll() - } - - let request = NSFetchRequest(entityName: entity_name) - let batch_delete_request = NSBatchDeleteRequest(fetchRequest: request) - batch_delete_request.resultType = .resultTypeObjectIDs - - let result = try container.persistentStoreCoordinator.execute(batch_delete_request, with: context) as! NSBatchDeleteResult - - // NSBatchDeleteRequest is an NSPersistentStoreRequest, which operates on disk. So now we'll manually update our in-memory context. - if let object_ids = result.result as? [NSManagedObjectID] { - let changes: [AnyHashable: Any] = [ - NSDeletedObjectsKey: object_ids - ] - NSManagedObjectContext.mergeChanges(fromRemoteContextSave: changes, into: [context]) - } - } -} diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index 9c24d612e2..ae9c01237a 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -15,72 +15,52 @@ class ValidationModel: ObservableObject { } } -class ProfileDataModel: ObservableObject { - @Published var profile: TimestampedProfile? - - init() { - self.profile = nil - } -} - class ProfileData { var status: UserStatusModel - var profile_model: ProfileDataModel var validation_model: ValidationModel var zapper: Pubkey? init() { status = .init() - profile_model = .init() validation_model = .init() zapper = nil } } class Profiles { - + private var ndb: Ndb + static let db_freshness_threshold: TimeInterval = 24 * 60 * 60 - - /// This queue is used to synchronize access to the profiles dictionary, which - /// prevents data races from crashing the app. - private var profiles_queue = DispatchQueue(label: "io.damus.profiles", - qos: .userInteractive, - attributes: .concurrent) - - private var validated_queue = DispatchQueue(label: "io.damus.profiles.validated", - qos: .userInteractive, - attributes: .concurrent) - + + @MainActor private var profiles: [Pubkey: ProfileData] = [:] + @MainActor var nip05_pubkey: [String: Pubkey] = [:] - private let database = ProfileDatabase() - let user_search_cache: UserSearchCache - init(user_search_cache: UserSearchCache) { + init(user_search_cache: UserSearchCache, ndb: Ndb) { self.user_search_cache = user_search_cache + self.ndb = ndb } - + + @MainActor func is_validated(_ pk: Pubkey) -> NIP05? { - validated_queue.sync { - self.profile_data(pk).validation_model.validated - } + self.profile_data(pk).validation_model.validated } + @MainActor func invalidate_nip05(_ pk: Pubkey) { - validated_queue.async(flags: .barrier) { - self.profile_data(pk).validation_model.validated = nil - } + self.profile_data(pk).validation_model.validated = nil } + @MainActor func set_validated(_ pk: Pubkey, nip05: NIP05?) { - validated_queue.async(flags: .barrier) { - self.profile_data(pk).validation_model.validated = nip05 - } + self.profile_data(pk).validation_model.validated = nip05 } - + + @MainActor func profile_data(_ pubkey: Pubkey) -> ProfileData { guard let data = profiles[pubkey] else { let data = ProfileData() @@ -91,60 +71,28 @@ class Profiles { return data } + @MainActor func lookup_zapper(pubkey: Pubkey) -> Pubkey? { profile_data(pubkey).zapper } - - func add(id: Pubkey, profile: TimestampedProfile) { - profiles_queue.async(flags: .barrier) { - let old_timestamped_profile = self.profile_data(id).profile_model.profile - self.profile_data(id).profile_model.profile = profile - self.user_search_cache.updateProfile(id: id, profiles: self, oldProfile: old_timestamped_profile?.profile, newProfile: profile.profile) - } - - Task { - do { - try await database.upsert(id: id, profile: profile.profile, last_update: Date(timeIntervalSince1970: TimeInterval(profile.timestamp))) - } catch { - print("⚠️ Warning: Profiles failed to save a profile: \(error)") - } - } + + func lookup_with_timestamp(_ pubkey: Pubkey) -> ProfileRecord? { + return ndb.lookup_profile(pubkey) } - + func lookup(id: Pubkey) -> Profile? { - var profile: Profile? - profiles_queue.sync { - profile = self.profile_data(id).profile_model.profile?.profile - } - return profile ?? database.get(id: id) + return ndb.lookup_profile(id)?.profile } - - func lookup_with_timestamp(id: Pubkey) -> TimestampedProfile? { - profiles_queue.sync { - return self.profile_data(id).profile_model.profile - } - } - + func has_fresh_profile(id: Pubkey) -> Bool { var profile: Profile? - profiles_queue.sync { - profile = self.profile_data(id).profile_model.profile?.profile - } - if profile != nil { - return true - } - // check memory first - return false - - // then disk - guard let pull_date = database.get_network_pull_date(id: id) else { - return false - } - return Date.now.timeIntervalSince(pull_date) < Profiles.db_freshness_threshold + guard let profile = lookup_with_timestamp(id) else { return false } + return Date.now.timeIntervalSince(Date(timeIntervalSince1970: Double(profile.receivedAt))) < Profiles.db_freshness_threshold } } +@MainActor func invalidate_zapper_cache(pubkey: Pubkey, profiles: Profiles, lnurl: LNUrls) { profiles.profile_data(pubkey).zapper = nil lnurl.endpoints.removeValue(forKey: pubkey) diff --git a/damus/Nostr/RelayConnection.swift b/damus/Nostr/RelayConnection.swift index e385f471d6..970a6c9997 100644 --- a/damus/Nostr/RelayConnection.swift +++ b/damus/Nostr/RelayConnection.swift @@ -56,12 +56,17 @@ final class RelayConnection: ObservableObject { private var subscriptionToken: AnyCancellable? private var handleEvent: (NostrConnectionEvent) -> () + private var processEvent: (WebSocketEvent) -> () private let url: RelayURL var log: RelayLog? - init(url: RelayURL, handleEvent: @escaping (NostrConnectionEvent) -> ()) { + init(url: RelayURL, + handleEvent: @escaping (NostrConnectionEvent) -> (), + processEvent: @escaping (WebSocketEvent) -> ()) + { self.url = url self.handleEvent = handleEvent + self.processEvent = processEvent } func ping() { @@ -138,6 +143,7 @@ final class RelayConnection: ObservableObject { } private func receive(event: WebSocketEvent) { + processEvent(event) switch event { case .connected: DispatchQueue.main.async { diff --git a/damus/Nostr/RelayPool.swift b/damus/Nostr/RelayPool.swift index 293e016cd9..ac99ec36b7 100644 --- a/damus/Nostr/RelayPool.swift +++ b/damus/Nostr/RelayPool.swift @@ -30,12 +30,15 @@ class RelayPool { var request_queue: [QueuedRequest] = [] var seen: Set = Set() var counts: [String: UInt64] = [:] - + var ndb: Ndb + private let network_monitor = NWPathMonitor() private let network_monitor_queue = DispatchQueue(label: "io.damus.network_monitor") private var last_network_status: NWPath.Status = .unsatisfied - init() { + init(ndb: Ndb) { + self.ndb = ndb + network_monitor.pathUpdateHandler = { [weak self] path in if (path.status == .satisfied || path.status == .requiresConnection) && self?.last_network_status != path.status { DispatchQueue.main.async { @@ -110,9 +113,15 @@ class RelayPool { if get_relay(relay_id) != nil { throw RelayError.RelayAlreadyExists } - let conn = RelayConnection(url: url) { event in + let conn = RelayConnection(url: url, handleEvent: { event in self.handle_event(relay_id: relay_id, event: event) - } + }, processEvent: { wsev in + guard case .message(let msg) = wsev, + case .string(let str) = msg + else { return } + + self.ndb.process_event(str) + }) let relay = Relay(descriptor: desc, connection: conn) self.relays.append(relay) } diff --git a/damus/TestData.swift b/damus/TestData.swift index 16278ff4d2..1eaba97a8c 100644 --- a/damus/TestData.swift +++ b/damus/TestData.swift @@ -54,9 +54,11 @@ let test_following_model = FollowingModel(damus_state: test_damus_state(), conta func test_damus_state() -> DamusState { let damus = DamusState.empty + /* let prof = Profile(name: "damus", display_name: "damus", about: "iOS app!", picture: "https://damus.io/img/logo.png", banner: "", website: "https://damus.io", lud06: nil, lud16: "jb55@sendsats.lol", nip05: "damus.io", damus_donation: nil) let tsprof = TimestampedProfile(profile: prof, timestamp: 0, event: test_note) damus.profiles.add(id: test_pubkey, profile: tsprof) + */ return damus } diff --git a/damus/Util/AccountDeletion.swift b/damus/Util/AccountDeletion.swift index faf9c06295..a36f635bf6 100644 --- a/damus/Util/AccountDeletion.swift +++ b/damus/Util/AccountDeletion.swift @@ -9,12 +9,11 @@ import Foundation func created_deleted_account_profile(keypair: FullKeypair) -> NostrEvent? { - let profile = Profile() - profile.about = "account deleted" - profile.name = "nobody" - - guard let content = encode_json(profile) else { - return nil - } + let about = "account deleted" + let name = "nobody" + + let profile = Profile(name: name, about: about) + + guard let content = encode_json(profile) else { return nil } return NostrEvent(content: content, keypair: keypair.to_keypair(), kind: 0) } diff --git a/damus/Util/EventCache.swift b/damus/Util/EventCache.swift index cd4458b6a2..6d0e638e6a 100644 --- a/damus/Util/EventCache.swift +++ b/damus/Util/EventCache.swift @@ -137,6 +137,7 @@ class EventData { } class EventCache { + private let ndb: Ndb private var events: [NoteId: NostrEvent] = [:] private var replies = ReplyMap() private var cancellable: AnyCancellable? @@ -145,7 +146,8 @@ class EventCache { //private var thread_latest: [String: Int64] - init() { + init(ndb: Ndb) { + self.ndb = ndb cancellable = NotificationCenter.default.publisher( for: UIApplication.didReceiveMemoryWarningNotification ).sink { [weak self] _ in @@ -250,7 +252,11 @@ class EventCache { insert(ev) return ev } - + + func lookup_by_key(_ key: UInt64) -> NostrEvent? { + ndb.lookup_note_by_key(key) + } + func lookup(_ evid: NoteId) -> NostrEvent? { return events[evid] } diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift index 891217fe3d..79d2e30425 100644 --- a/damus/Views/ActionBar/EventActionBar.swift +++ b/damus/Views/ActionBar/EventActionBar.swift @@ -28,7 +28,7 @@ struct EventActionBar: View { } var lnurl: String? { - damus_state.profiles.lookup(id: event.pubkey)?.lnurl + damus_state.profiles.lookup_with_timestamp(event.pubkey)?.lnurl } var show_like: Bool { diff --git a/damus/Views/Events/Components/EventTop.swift b/damus/Views/Events/Components/EventTop.swift index ac2e762fb0..1edfeecea1 100644 --- a/damus/Views/Events/Components/EventTop.swift +++ b/damus/Views/Events/Components/EventTop.swift @@ -7,6 +7,7 @@ import SwiftUI +@MainActor struct EventTop: View { let state: DamusState let event: NostrEvent diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift index 7f0bd2d6e1..55d116e428 100644 --- a/damus/Views/Events/EventShell.swift +++ b/damus/Views/Events/EventShell.swift @@ -7,6 +7,7 @@ import SwiftUI +@MainActor struct EventShell: View { let state: DamusState let event: NostrEvent diff --git a/damus/Views/Onboarding/SuggestedUserView.swift b/damus/Views/Onboarding/SuggestedUserView.swift index 43dbafe839..b30d134b8d 100644 --- a/damus/Views/Onboarding/SuggestedUserView.swift +++ b/damus/Views/Onboarding/SuggestedUserView.swift @@ -7,7 +7,7 @@ import SwiftUI -struct SuggestedUser: Codable { +struct SuggestedUser { let pubkey: Pubkey let name: String let about: String diff --git a/damus/Views/Onboarding/SuggestedUsersViewModel.swift b/damus/Views/Onboarding/SuggestedUsersViewModel.swift index 7eae7fb7c2..9c7af00045 100644 --- a/damus/Views/Onboarding/SuggestedUsersViewModel.swift +++ b/damus/Views/Onboarding/SuggestedUsersViewModel.swift @@ -86,13 +86,7 @@ class SuggestedUsersViewModel: ObservableObject { switch nev { case .event(let sub_id, let ev): - guard sub_id == self.sub_id else { - return - } - - if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } + break case .notice(let msg): print("suggested user profiles notice: \(msg)") diff --git a/damus/Views/Profile/EditMetadataView.swift b/damus/Views/Profile/EditMetadataView.swift index e9b5d0d5e0..a7c8186fe8 100644 --- a/damus/Views/Profile/EditMetadataView.swift +++ b/damus/Views/Profile/EditMetadataView.swift @@ -45,18 +45,14 @@ struct EditMetadataView: View { } func to_profile() -> Profile { - let profile = self.profile ?? Profile() - - profile.name = name - profile.display_name = display_name - profile.about = about - profile.website = website - profile.nip05 = nip05.isEmpty ? nil : nip05 - profile.picture = picture.isEmpty ? nil : picture - profile.banner = banner.isEmpty ? nil : banner - profile.lud06 = ln.contains("@") ? nil : ln - profile.lud16 = ln.contains("@") ? ln : nil - + let new_nip05 = nip05.isEmpty ? nil : nip05 + let new_picture = picture.isEmpty ? nil : picture + let new_banner = banner.isEmpty ? nil : banner + let new_lud06 = ln.contains("@") ? nil : ln + let new_lud16 = ln.contains("@") ? ln : nil + + let profile = Profile(name: name, display_name: display_name, about: about, picture: new_picture, banner: new_banner, website: website, lud06: new_lud06, lud16: new_lud16, nip05: new_nip05, damus_donation: nil) + return profile } diff --git a/damus/Views/Profile/EventProfileName.swift b/damus/Views/Profile/EventProfileName.swift index a63909fb10..9d4788987b 100644 --- a/damus/Views/Profile/EventProfileName.swift +++ b/damus/Views/Profile/EventProfileName.swift @@ -8,6 +8,7 @@ import SwiftUI /// Profile Name used when displaying an event in the timeline +@MainActor struct EventProfileName: View { let damus_state: DamusState let pubkey: Pubkey diff --git a/damus/Views/Profile/ProfileName.swift b/damus/Views/Profile/ProfileName.swift index 2a6acfd4b3..18a87b5b8b 100644 --- a/damus/Views/Profile/ProfileName.swift +++ b/damus/Views/Profile/ProfileName.swift @@ -47,7 +47,8 @@ struct ProfileName: View { var friend_type: FriendType? { return get_friend_type(contacts: damus_state.contacts, pubkey: self.pubkey) } - + + @MainActor var current_nip05: NIP05? { nip05 ?? damus_state.profiles.is_validated(pubkey) } diff --git a/damus/Views/Profile/ProfilePicView.swift b/damus/Views/Profile/ProfilePicView.swift index 6467cf9876..2a3c346b6a 100644 --- a/damus/Views/Profile/ProfilePicView.swift +++ b/damus/Views/Profile/ProfilePicView.swift @@ -105,11 +105,11 @@ func get_profile_url(picture: String?, pubkey: Pubkey, profiles: Profiles) -> UR func make_preview_profiles(_ pubkey: Pubkey) -> Profiles { let user_search_cache = UserSearchCache() - let profiles = Profiles(user_search_cache: user_search_cache) + let profiles = Profiles(user_search_cache: user_search_cache, ndb: .empty) let picture = "http://cdn.jb55.com/img/red-me.jpg" let profile = Profile(name: "jb55", display_name: "William Casarin", about: "It's me", picture: picture, banner: "", website: "https://jb55.com", lud06: nil, lud16: nil, nip05: "jb55.com", damus_donation: nil) - let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) - profiles.add(id: pubkey, profile: ts_profile) + //let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) + //profiles.add(id: pubkey, profile: ts_profile) return profiles } diff --git a/damus/Views/Profile/ProfileView.swift b/damus/Views/Profile/ProfileView.swift index 0f7ce1745d..db928e483b 100644 --- a/damus/Views/Profile/ProfileView.swift +++ b/damus/Views/Profile/ProfileView.swift @@ -216,7 +216,8 @@ struct ProfileView: View { .accentColor(DamusColors.white) } - func lnButton(lnurl: String, profile: Profile) -> some View { + func lnButton(lnurl: String, record: ProfileRecord, profile: Profile) -> some View { + let profile = record.profile! let button_img = profile.reactions == false ? "zap.fill" : "zap" return Button(action: { present_sheet(.zap(target: .profile(self.profile.pubkey), lnurl: lnurl)) @@ -235,7 +236,7 @@ struct ProfileView: View { } label: { Label(addr, image: "copy2") } - } else if let lnurl = profile.lnurl { + } else if let lnurl = record.lnurl { Button { UIPasteboard.general.string = lnurl } label: { @@ -268,14 +269,14 @@ struct ProfileView: View { .font(.footnote) } - func actionSection(profile_data: Profile?) -> some View { + func actionSection(record: ProfileRecord?) -> some View { return Group { - - if let profile = profile_data, - let lnurl = profile.lnurl, + if let record, + let profile = record.profile, + let lnurl = record.lnurl, lnurl != "" { - lnButton(lnurl: lnurl, profile: profile) + lnButton(lnurl: lnurl, record: record, profile: profile) } dmButton @@ -307,8 +308,9 @@ struct ProfileView: View { return scale < 1 ? scale : 1 } - func nameSection(profile_data: Profile?) -> some View { + func nameSection(profile_data: ProfileRecord?) -> some View { return Group { + let follows_you = profile.pubkey != damus_state.pubkey && profile.follows(pubkey: damus_state.pubkey) HStack(alignment: .center) { ProfilePicView(pubkey: profile.pubkey, size: pfp_size, highlight: .custom(imageBorderColor(), 4.0), profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) .padding(.top, -(pfp_size / 2.0)) @@ -322,48 +324,46 @@ struct ProfileView: View { } Spacer() - - let follows_you = profile.pubkey != damus_state.pubkey && profile.follows(pubkey: damus_state.pubkey) + if follows_you { followsYouBadge } - actionSection(profile_data: profile_data) + actionSection(record: profile_data) } - - ProfileNameView(pubkey: profile.pubkey, profile: profile_data, damus: damus_state) + + ProfileNameView(pubkey: profile.pubkey, profile: profile_data?.profile, damus: damus_state) } } var followersCount: some View { HStack { - if followers.count == nil { + if let followerCount = followers.count { + let nounString = pluralizedString(key: "followers_count", count: followerCount) + let nounText = Text(verbatim: nounString).font(.subheadline).foregroundColor(.gray) + Text("\(Text(verbatim: followerCount.formatted()).font(.subheadline.weight(.medium))) \(nounText)", comment: "Sentence composed of 2 variables to describe how many people are following a user. In source English, the first variable is the number of followers, and the second variable is 'Follower' or 'Followers'.") + } else { Image("download") .resizable() .frame(width: 20, height: 20) Text("Followers", comment: "Label describing followers of a user.") .font(.subheadline) .foregroundColor(.gray) - } else { - let followerCount = followers.count! - let nounString = pluralizedString(key: "followers_count", count: followerCount) - let nounText = Text(verbatim: nounString).font(.subheadline).foregroundColor(.gray) - Text("\(Text(verbatim: followerCount.formatted()).font(.subheadline.weight(.medium))) \(nounText)", comment: "Sentence composed of 2 variables to describe how many people are following a user. In source English, the first variable is the number of followers, and the second variable is 'Follower' or 'Followers'.") } } } var aboutSection: some View { VStack(alignment: .leading, spacing: 8.0) { - let profile_data = damus_state.profiles.lookup(id: profile.pubkey) + let profile_data = damus_state.profiles.lookup_with_timestamp(profile.pubkey) nameSection(profile_data: profile_data) - if let about = profile_data?.about { + if let about = profile_data?.profile?.about { AboutView(state: damus_state, about: about) } - if let url = profile_data?.website_url { + if let url = profile_data?.profile?.website_url { WebsiteLink(url: url) } @@ -514,6 +514,7 @@ extension View { } } +@MainActor func check_nip05_validity(pubkey: Pubkey, profiles: Profiles) { guard let profile = profiles.lookup(id: pubkey), let nip05 = profile.nip05, diff --git a/damus/Views/SaveKeysView.swift b/damus/Views/SaveKeysView.swift index 2425c85ad5..27d9850bd4 100644 --- a/damus/Views/SaveKeysView.swift +++ b/damus/Views/SaveKeysView.swift @@ -10,7 +10,7 @@ import Security struct SaveKeysView: View { let account: CreateAccountModel - let pool: RelayPool = RelayPool() + let pool: RelayPool = RelayPool(ndb: Ndb()!) @State var pub_copied: Bool = false @State var priv_copied: Bool = false @State var loading: Bool = false diff --git a/damus/Views/Search/SearchingEventView.swift b/damus/Views/Search/SearchingEventView.swift index 458cd9851d..038479de6d 100644 --- a/damus/Views/Search/SearchingEventView.swift +++ b/damus/Views/Search/SearchingEventView.swift @@ -20,6 +20,7 @@ enum SearchType: Equatable { case nip05(String) } +@MainActor struct SearchingEventView: View { let state: DamusState let search_type: SearchType diff --git a/damus/Views/SideMenuView.swift b/damus/Views/SideMenuView.swift index b1c420c7bb..d6fcf6305b 100644 --- a/damus/Views/SideMenuView.swift +++ b/damus/Views/SideMenuView.swift @@ -7,6 +7,7 @@ import SwiftUI +@MainActor struct SideMenuView: View { let damus_state: DamusState @Binding var isSidebarVisible: Bool diff --git a/damus/Views/Wallet/WalletView.swift b/damus/Views/Wallet/WalletView.swift index 3f33134498..3e8bc97662 100644 --- a/damus/Views/Wallet/WalletView.swift +++ b/damus/Views/Wallet/WalletView.swift @@ -168,9 +168,9 @@ struct WalletView: View { return } - profile.damus_donation = p + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: p, reactions: profile.reactions) - notify(.profile_updated(pubkey: damus_state.pubkey, profile: profile)) + notify(.profile_updated(pubkey: damus_state.pubkey, profile: prof)) } .onDisappear { guard let keypair = damus_state.keypair.to_full(), @@ -180,12 +180,11 @@ struct WalletView: View { return } - profile.damus_donation = settings.donation_percent - guard let meta = make_metadata_event(keypair: keypair, metadata: profile) else { + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: settings.donation_percent, reactions: profile.reactions) + + guard let meta = make_metadata_event(keypair: keypair, metadata: prof) else { return } - let tsprofile = TimestampedProfile(profile: profile, timestamp: meta.created_at, event: meta) - damus_state.profiles.add(id: damus_state.pubkey, profile: tsprofile) damus_state.postbox.send(meta) } } diff --git a/damusTests/NostrScriptTests.swift b/damusTests/NostrScriptTests.swift index 5386e82d41..9f903beb10 100644 --- a/damusTests/NostrScriptTests.swift +++ b/damusTests/NostrScriptTests.swift @@ -37,7 +37,7 @@ final class NostrScriptTests: XCTestCase { func test_bool_set() throws { let data = try load_bool_set_test_wasm().bytes - let pool = RelayPool() + let pool = RelayPool(ndb: .empty) let script = NostrScript(pool: pool, data: data) let pk = Pubkey(hex: "32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245")! UserSettingsStore.pubkey = pk diff --git a/damusTests/ProfileDatabaseTests.swift b/damusTests/ProfileDatabaseTests.swift deleted file mode 100644 index 77f23f6444..0000000000 --- a/damusTests/ProfileDatabaseTests.swift +++ /dev/null @@ -1,124 +0,0 @@ -// -// ProfileDatabaseTests.swift -// damusTests -// -// Created by Bryan Montz on 5/13/23. -// - -import XCTest -@testable import damus - -class ProfileDatabaseTests: XCTestCase { - - static let cache_url = (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("test-profiles"))! - let database = ProfileDatabase(cache_url: ProfileDatabaseTests.cache_url) - - override func tearDownWithError() throws { - // This method is called after the invocation of each test method in the class. - try database.remove_all_profiles() - } - - var test_profile: Profile { - Profile(name: "test-name", - display_name: "test-display-name", - about: "test-about", - picture: "test-picture", - banner: "test-banner", - website: "test-website", - lud06: "test-lud06", - lud16: "test-lud16", - nip05: "test-nip05", - damus_donation: 100) - } - - func testStoreAndRetrieveProfile() async throws { - let id = test_pubkey - - let profile = test_profile - - // make sure it's not there yet - XCTAssertNil(database.get(id: id)) - - // store the profile - try await database.upsert(id: id, profile: profile, last_update: .now) - - // read the profile out of the database - let retrievedProfile = try XCTUnwrap(database.get(id: id)) - - XCTAssertEqual(profile.name, retrievedProfile.name) - XCTAssertEqual(profile.display_name, retrievedProfile.display_name) - XCTAssertEqual(profile.about, retrievedProfile.about) - XCTAssertEqual(profile.picture, retrievedProfile.picture) - XCTAssertEqual(profile.banner, retrievedProfile.banner) - XCTAssertEqual(profile.website, retrievedProfile.website) - XCTAssertEqual(profile.lud06, retrievedProfile.lud06) - XCTAssertEqual(profile.lud16, retrievedProfile.lud16) - XCTAssertEqual(profile.nip05, retrievedProfile.nip05) - XCTAssertEqual(profile.damus_donation, retrievedProfile.damus_donation) - } - - func testRejectOutdatedProfile() async throws { - let id = test_pubkey - - // store a profile - let profile = test_profile - let profile_last_updated = Date.now - try await database.upsert(id: id, profile: profile, last_update: profile_last_updated) - - // try to store a profile with the same id but the last_update date is older than the previously stored profile - let outdatedProfile = test_profile - let outdated_last_updated = profile_last_updated.addingTimeInterval(-60) - - do { - try await database.upsert(id: id, profile: outdatedProfile, last_update: outdated_last_updated) - XCTFail("expected to throw error") - } catch let error as ProfileDatabaseError { - XCTAssertEqual(error, ProfileDatabaseError.outdated_input) - } catch { - XCTFail("not the expected error") - } - } - - func testUpdateExistingProfile() async throws { - let id = test_pubkey - - // store a profile - let profile = test_profile - let profile_last_update = Date.now - try await database.upsert(id: id, profile: profile, last_update: profile_last_update) - - // update the same profile - let updated_profile = test_profile - updated_profile.nip05 = "updated-nip05" - let updated_profile_last_update = profile_last_update.addingTimeInterval(60) - try await database.upsert(id: id, profile: updated_profile, last_update: updated_profile_last_update) - - // retrieve the profile and make sure it was updated - let retrieved_profile = database.get(id: id) - XCTAssertEqual(retrieved_profile?.nip05, "updated-nip05") - } - - func testStoreMultipleAndRemoveAllProfiles() async throws { - XCTAssertEqual(database.count, 0) - - // store a profile - let id = test_pubkey - let profile = test_profile - let profile_last_update = Date.now - try await database.upsert(id: id, profile: profile, last_update: profile_last_update) - - XCTAssertEqual(database.count, 1) - - // store another profile - let id2 = test_pubkey_2 - let profile2 = test_profile - let profile_last_update2 = Date.now - try await database.upsert(id: id2, profile: profile2, last_update: profile_last_update2) - - XCTAssertEqual(database.count, 2) - - try database.remove_all_profiles() - - XCTAssertEqual(database.count, 0) - } -} diff --git a/damusTests/UserSearchCacheTests.swift b/damusTests/UserSearchCacheTests.swift index 2b98cd73da..8a90128d32 100644 --- a/damusTests/UserSearchCacheTests.swift +++ b/damusTests/UserSearchCacheTests.swift @@ -14,6 +14,7 @@ final class UserSearchCacheTests: XCTestCase { let damusState = DamusState.empty let nip05 = "_@somedomain.com" + @MainActor override func setUpWithError() throws { keypair = try XCTUnwrap(generate_new_keypair()) @@ -24,8 +25,6 @@ final class UserSearchCacheTests: XCTestCase { damusState.profiles.set_validated(pubkey, nip05: validatedNip05) let profile = Profile(name: "tyiu", display_name: "Terry Yiu", about: nil, picture: nil, banner: nil, website: nil, lud06: nil, lud16: nil, nip05: nip05, damus_donation: nil) - let timestampedProfile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) - damusState.profiles.add(id: pubkey, profile: timestampedProfile) // Lookup to synchronize access on profiles dictionary to avoid race conditions. let _ = damusState.profiles.lookup(id: pubkey) @@ -47,6 +46,7 @@ final class UserSearchCacheTests: XCTestCase { XCTAssertEqual(damusState.user_search_cache.search(key: "i"), [keypair.pubkey]) } + @MainActor func testUpdateProfile() throws { let keypair = try XCTUnwrap(keypair) @@ -56,8 +56,6 @@ final class UserSearchCacheTests: XCTestCase { damusState.profiles.set_validated(keypair.pubkey, nip05: NIP05.parse(newNip05)) let newProfile = Profile(name: "whoami", display_name: "T-DAWG", about: nil, picture: nil, banner: nil, website: nil, lud06: nil, lud16: nil, nip05: newNip05, damus_donation: nil) - let newTimestampedProfile = TimestampedProfile(profile: newProfile, timestamp: 1000, event: test_note) - damusState.profiles.add(id: keypair.pubkey, profile: newTimestampedProfile) // Lookup to synchronize access on profiles dictionary to avoid race conditions. let _ = damusState.profiles.lookup(id: keypair.pubkey) diff --git a/damusTests/WalletConnectTests.swift b/damusTests/WalletConnectTests.swift index f7c31228b7..89d789ace3 100644 --- a/damusTests/WalletConnectTests.swift +++ b/damusTests/WalletConnectTests.swift @@ -84,7 +84,7 @@ final class WalletConnectTests: XCTestCase { let pk = "89446b900c70d62438dcf66756405eea6225ad94dc61f3856f62f9699111a9a6" let nwc = WalletConnectURL(str: "nostrwalletconnect://\(pk)?relay=ws://127.0.0.1&secret=\(sec)&lud16=jb55@jb55.com")! - let pool = RelayPool() + let pool = RelayPool(ndb: .empty) let box = PostBox(pool: pool) nwc_pay(url: nwc, pool: pool, post: box, invoice: "invoice") diff --git a/damusTests/ZapTests.swift b/damusTests/ZapTests.swift index 5208b83e5e..66975db80e 100644 --- a/damusTests/ZapTests.swift +++ b/damusTests/ZapTests.swift @@ -11,8 +11,6 @@ import XCTest final class ZapTests: XCTestCase { override func setUpWithError() throws { - let db = ProfileDatabase() - try db.remove_all_profiles() } override func tearDownWithError() throws { @@ -71,7 +69,7 @@ final class ZapTests: XCTestCase { XCTAssertEqual(zap.target, ZapTarget.profile(profile)) XCTAssertEqual(zap_notification_title(zap), "Zap") - XCTAssertEqual(zap_notification_body(profiles: Profiles(user_search_cache: UserSearchCache()), zap: zap), "You received 1k sats from 107jk7ht:2quqncxg") + XCTAssertEqual(zap_notification_body(profiles: Profiles(user_search_cache: UserSearchCache(), ndb: .empty), zap: zap), "You received 1k sats from 107jk7ht:2quqncxg") } } diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index ccefba04a7..4e236e4e6f 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -14,11 +14,28 @@ class Ndb { (FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: ""))! } + static var empty: Ndb { + Ndb(ndb: ndb_t(ndb: nil)) + } + init?() { + //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb") + //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb") + var ndb_p: OpaquePointer? = nil + let ingest_threads: Int32 = 4 + var mapsize: Int = 1024 * 1024 * 1024 * 32 + let ok = Ndb.db_path.withCString { testdir in - return ndb_init(&ndb_p, testdir, 1024 * 1024 * 1024 * 32, 4) != 0 + var ok = false + while !ok && mapsize > 1024 * 1024 * 700 { + ok = ndb_init(&ndb_p, testdir, mapsize, ingest_threads) != 0 + if !ok { + mapsize /= 2 + } + } + return ok } if !ok { @@ -28,24 +45,51 @@ class Ndb { self.ndb = ndb_t(ndb: ndb_p) } + init(ndb: ndb_t) { + self.ndb = ndb + } + + func lookup_note_by_key(_ key: UInt64) -> NdbNote? { + guard let note_p = ndb_get_note_by_key(ndb.ndb, key, nil) else { + return nil + } + return NdbNote(note: note_p, owned_size: nil) + } + func lookup_note(_ id: NoteId) -> NdbNote? { - id.id.withUnsafeBytes { bs in - guard let note_p = ndb_get_note_by_id(ndb.ndb, bs, nil) else { + id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NdbNote? in + guard let baseAddress = ptr.baseAddress, + let note_p = ndb_get_note_by_id(ndb.ndb, baseAddress, nil) else { return nil } return NdbNote(note: note_p, owned_size: nil) } } - func lookup_profile(_ pubkey: Pubkey) -> NdbProfile? { - return pubkey.id.withUnsafeBytes { pk_bytes in + func lookup_profile(_ pubkey: Pubkey) -> ProfileRecord? { + return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> ProfileRecord? in var size: Int = 0 - guard let profile_p = ndb_get_profile_by_pubkey(ndb.ndb, pk_bytes, &size) else { + + guard let baseAddress = ptr.baseAddress, + let profile_p = ndb_get_profile_by_pubkey(ndb.ndb, baseAddress, &size) + else { return nil } - let buf = ByteBuffer(assumingMemoryBound: profile_p, capacity: size) - return NdbProfile(buf, o: 0) + do { + var buf = ByteBuffer(assumingMemoryBound: profile_p, capacity: size) + let rec: NdbProfileRecord = try getDebugCheckedRoot(byteBuffer: &buf) + return ProfileRecord(data: rec) + } catch { + // Handle error appropriately + print("UNUSUAL: \(error)") + return nil + } + } + } + func process_event(_ str: String) -> Bool { + return str.withCString { cstr in + return ndb_process_event(ndb.ndb, cstr, Int32(str.utf8.count)) != 0 } } @@ -59,3 +103,13 @@ class Ndb { ndb_destroy(ndb.ndb) } } + +#if DEBUG +func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) throws -> T { + return try getCheckedRoot(byteBuffer: &byteBuffer) +} +#else +func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) throws -> T { + return try getRoot(byteBuffer: &byteBuffer) +} +#endif diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift index d54d98eee1..d7867cafe4 100644 --- a/nostrdb/Test/NdbTests.swift +++ b/nostrdb/Test/NdbTests.swift @@ -33,6 +33,11 @@ final class NdbTests: XCTestCase { } + func test_profile_creation() { + let profile = make_test_profile() + XCTAssertEqual(profile.name, "jb55") + } + func test_ndb_init() { do { @@ -54,7 +59,8 @@ final class NdbTests: XCTestCase { XCTAssertNotNil(profile) guard let profile else { return } - XCTAssertEqual(profile.name, "jb55") + XCTAssertEqual(profile.profile?.name, "jb55") + XCTAssertEqual(profile.lnurl, "fixme") } @@ -71,7 +77,7 @@ final class NdbTests: XCTestCase { XCTAssertEqual(note.id, id) XCTAssertEqual(note.pubkey, pubkey) - XCTAssertEqual(note.count, 34322) + XCTAssertEqual(note.count, 34328) XCTAssertEqual(note.kind, 3) XCTAssertEqual(note.created_at, 1689904312) From 129d3ff101da8ff29b20c99e3b8c0f1a8d4a40f6 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 6 Sep 2023 11:07:24 -0700 Subject: [PATCH 104/111] ids: introduce NoteKey These will be used to reference nostr notes from nostrdb --- damus/Types/Ids/NoteId.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/damus/Types/Ids/NoteId.swift b/damus/Types/Ids/NoteId.swift index 5acd0961c4..c76233c8ca 100644 --- a/damus/Types/Ids/NoteId.swift +++ b/damus/Types/Ids/NoteId.swift @@ -7,6 +7,8 @@ import Foundation +typealias NoteKey = UInt64 + struct NoteId: IdType, TagKey, TagConvertible { let id: Data From 9398877415f1333acd9184808064faf6d9bd4ce7 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 10:16:25 -0700 Subject: [PATCH 105/111] nostrdb/c: update to include transaction support --- nostrdb/nostrdb.c | 76 ++++++++++++++++++++++++++++------------------- nostrdb/nostrdb.h | 17 +++++++++-- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index db67aa971e..4fce06f1b6 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -207,6 +207,18 @@ struct ndb_writer_msg { }; }; +int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn) +{ + txn->ndb = ndb; + MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn; + return mdb_txn_begin(ndb->lmdb.env, NULL, 0, mdb_txn) == 0; +} + +void ndb_end_query(struct ndb_txn *txn) +{ + mdb_txn_abort(txn->mdb_txn); +} + int ndb_note_verify(void *ctx, unsigned char pubkey[32], unsigned char id[32], unsigned char sig[64]) { @@ -249,8 +261,8 @@ int ndb_get_tsid(MDB_txn *txn, struct ndb_lmdb *lmdb, enum ndb_dbs db, { MDB_val k, v; MDB_cursor *cur; - struct ndb_tsid tsid; int success = 0; + struct ndb_tsid tsid; ndb_tsid_high(&tsid, id); k.mv_data = &tsid; @@ -280,23 +292,17 @@ int ndb_get_tsid(MDB_txn *txn, struct ndb_lmdb *lmdb, enum ndb_dbs db, return success; } -static void *ndb_lookup_by_key(struct ndb *ndb, uint64_t key, +static void *ndb_lookup_by_key(struct ndb_txn *txn, uint64_t key, enum ndb_dbs store, size_t *len) { MDB_val k, v; - MDB_txn *txn; k.mv_data = &key; k.mv_size = sizeof(key); - if (mdb_txn_begin(ndb->lmdb.env, 0, 0, &txn)) { - ndb_debug("ndb_get_note_by_id: mdb_txn_begin failed\n"); - return NULL; - } - - if (mdb_get(txn, ndb->lmdb.dbs[store], &k, &v)) { + if (mdb_get(txn->mdb_txn, txn->ndb->lmdb.dbs[store], &k, &v)) { ndb_debug("ndb_get_profile_by_pubkey: mdb_get note failed\n"); - mdb_txn_abort(txn); + mdb_txn_abort(txn->mdb_txn); return NULL; } @@ -306,53 +312,63 @@ static void *ndb_lookup_by_key(struct ndb *ndb, uint64_t key, return v.mv_data; } -static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind, +static void *ndb_lookup_tsid(struct ndb_txn *txn, enum ndb_dbs ind, enum ndb_dbs store, const unsigned char *pk, - size_t *len) + size_t *len, uint64_t *primkey) { MDB_val k, v; - MDB_txn *txn; void *res = NULL; if (len) *len = 0; - if (mdb_txn_begin(ndb->lmdb.env, 0, 0, &txn)) { - ndb_debug("ndb_get_note_by_id: mdb_txn_begin failed\n"); - return NULL; - } - - if (!ndb_get_tsid(txn, &ndb->lmdb, ind, pk, &k)) { + if (!ndb_get_tsid(txn->mdb_txn, &txn->ndb->lmdb, ind, pk, &k)) { //ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n"); - goto cleanup; + return 0; } - if (mdb_get(txn, ndb->lmdb.dbs[store], &k, &v)) { + if (primkey) + *primkey = *(uint64_t*)k.mv_data; + + if (mdb_get(txn->mdb_txn, txn->ndb->lmdb.dbs[store], &k, &v)) { ndb_debug("ndb_get_profile_by_pubkey: mdb_get note failed\n"); - goto cleanup; + return 0; } res = v.mv_data; assert(((uint64_t)res % 4) == 0); if (len) *len = v.mv_size; -cleanup: - mdb_txn_abort(txn); return res; } -void *ndb_get_profile_by_pubkey(struct ndb *ndb, const unsigned char *pk, size_t *len) +void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pk, size_t *len, uint32_t *key) { - return ndb_lookup_tsid(ndb, NDB_DB_PROFILE_PK, NDB_DB_PROFILE, pk, len); + return ndb_lookup_tsid(txn, NDB_DB_PROFILE_PK, NDB_DB_PROFILE, pk, len, key); +} + +struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint32_t *key) +{ + return ndb_lookup_tsid(txn, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len, key); +} + +uint32_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id) +{ + MDB_val k; + + if (!ndb_get_tsid(txn->mdb_txn, &txn->ndb->lmdb, NDB_DB_NOTE_ID, id, &k)) + return 0; + + return *(uint32_t*)k.mv_data; } -struct ndb_note *ndb_get_note_by_id(struct ndb *ndb, const unsigned char *id, size_t *len) +struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint32_t key, size_t *len) { - return ndb_lookup_tsid(ndb, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len); + return ndb_lookup_by_key(txn, key, NDB_DB_NOTE, len); } -struct ndb_note *ndb_get_note_by_key(struct ndb *ndb, uint64_t key, size_t *len) +void *ndb_get_profile_by_key(struct ndb_txn *txn, uint32_t key, size_t *len) { - return ndb_lookup_by_key(ndb, key, NDB_DB_NOTE, len); + return ndb_lookup_by_key(txn, key, NDB_DB_PROFILE, len); } static int ndb_has_note(MDB_txn *txn, struct ndb_lmdb *lmdb, const unsigned char *id) diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index d78e34a974..f4606d59d8 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -18,10 +18,17 @@ struct ndb_json_parser; struct ndb; +// sorry, swift needs help with forward declared pointers like this struct ndb_t { struct ndb *ndb; }; +// required to keep a read +struct ndb_txn { + struct ndb *ndb; + void *mdb_txn; +}; + // To-client event types enum tce_type { NDB_TCE_EVENT = 0x1, @@ -154,9 +161,13 @@ int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[3 int ndb_init(struct ndb **ndb, const char *dbdir, size_t mapsize, int ingester_threads); int ndb_process_event(struct ndb *, const char *json, int len); int ndb_process_events(struct ndb *, const char *ldjson, size_t len); -void *ndb_get_profile_by_pubkey(struct ndb *, const unsigned char *pubkey, size_t *len); -struct ndb_note *ndb_get_note_by_id(struct ndb *, const unsigned char *id, size_t *len); -struct ndb_note *ndb_get_note_by_key(struct ndb *, uint64_t key, size_t *len); +int ndb_begin_query(struct ndb *, struct ndb_txn *); +void ndb_end_query(struct ndb_txn *); +void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey); +void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); +uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id); +struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *primkey); +struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); void ndb_destroy(struct ndb *); // BUILDER From 622a436589704bab7fda4452013275a131d3bdde Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 10:16:50 -0700 Subject: [PATCH 106/111] ndb: add NdbTxn transaction class This will be used for transactions --- damus.xcodeproj/project.pbxproj | 4 ++ nostrdb/NdbTxn.swift | 102 ++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 nostrdb/NdbTxn.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index b69ae0574e..aa2e59f9ff 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -156,6 +156,7 @@ 4C3BEFE0281DE1ED00B3DE84 /* DamusState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3BEFDF281DE1ED00B3DE84 /* DamusState.swift */; }; 4C3D52B6298DB4E6001C5831 /* ZapEvent.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3D52B5298DB4E6001C5831 /* ZapEvent.swift */; }; 4C3D52B8298DB5C6001C5831 /* TextEvent.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3D52B7298DB5C6001C5831 /* TextEvent.swift */; }; + 4C3DCC762A9FE9EC0091E592 /* NdbTxn.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3DCC752A9FC2030091E592 /* NdbTxn.swift */; }; 4C3EA63D28FF52D600C48A62 /* bolt11.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C3EA63C28FF52D600C48A62 /* bolt11.c */; }; 4C3EA64128FF553900C48A62 /* hash_u5.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C3EA64028FF553900C48A62 /* hash_u5.c */; }; 4C3EA64428FF558100C48A62 /* sha256.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C3EA64328FF558100C48A62 /* sha256.c */; }; @@ -694,6 +695,7 @@ 4C3BEFDF281DE1ED00B3DE84 /* DamusState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusState.swift; sourceTree = ""; }; 4C3D52B5298DB4E6001C5831 /* ZapEvent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ZapEvent.swift; sourceTree = ""; }; 4C3D52B7298DB5C6001C5831 /* TextEvent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TextEvent.swift; sourceTree = ""; }; + 4C3DCC752A9FC2030091E592 /* NdbTxn.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NdbTxn.swift; sourceTree = ""; }; 4C3EA63B28FF52D600C48A62 /* bolt11.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = bolt11.h; sourceTree = ""; }; 4C3EA63C28FF52D600C48A62 /* bolt11.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = bolt11.c; sourceTree = ""; }; 4C3EA63E28FF54BD00C48A62 /* short_types.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = short_types.h; sourceTree = ""; }; @@ -1816,6 +1818,7 @@ 4CDD1ADF2A6B305F001CD4DF /* NdbTagElem.swift */, 4C478E242A9932C100489948 /* Ndb.swift */, 4CDD1AE12A6B3074001CD4DF /* NdbTagsIterator.swift */, + 4C3DCC752A9FC2030091E592 /* NdbTxn.swift */, 4CE9FBB82A6B3B26007E485C /* nostrdb.c */, 4C4793032A993DB900489948 /* midl.c */, 4C4793002A993B9A00489948 /* mdb.c */, @@ -2478,6 +2481,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 4C3DCC762A9FE9EC0091E592 /* NdbTxn.swift in Sources */, 4CEF958D2A9CE650000F901B /* verifier.c in Sources */, 4C32B9342A9AD01A00DC3548 /* NdbProfile.swift in Sources */, 4C32B9332A99845B00DC3548 /* Ndb.swift in Sources */, diff --git a/nostrdb/NdbTxn.swift b/nostrdb/NdbTxn.swift new file mode 100644 index 0000000000..b053cd9121 --- /dev/null +++ b/nostrdb/NdbTxn.swift @@ -0,0 +1,102 @@ +// +// NdbTx.swift +// damus +// +// Created by William Casarin on 2023-08-30. +// + +import Foundation + +#if TXNDEBUG +fileprivate var txn_count: Int = 0 +#endif + +// Would use struct and ~Copyable but generics aren't supported well +class NdbTxn { + var txn: ndb_txn + private var val: T! + var moved: Bool + + init(ndb: Ndb, with: (NdbTxn) -> T) { + self.txn = ndb_txn() + #if TXNDEBUG + txn_count += 1 + print("opening transaction \(txn_count)") + #endif + let _ = ndb_begin_query(ndb.ndb.ndb, &self.txn) + self.moved = false + self.val = with(self) + } + + init(txn: ndb_txn, val: T) { + self.txn = txn + self.val = val + self.moved = false + } + + /// Only access temporarily! Do not store database references for longterm use. If it's a primitive type you + /// can retrieve this value with `.value` + var unsafeUnownedValue: T { + precondition(!moved) + return val + } + + deinit { + if !moved { + #if TXNDEBUG + txn_count -= 1; + print("closing transaction \(txn_count)") + #endif + ndb_end_query(&self.txn) + } + } + + // functor + func map(_ transform: (T) -> Y) -> NdbTxn { + self.moved = true + return .init(txn: self.txn, val: transform(val)) + } + + // comonad!? + // useful for moving ownership of a transaction to another value + func extend(_ with: (NdbTxn) -> Y) -> NdbTxn { + self.moved = true + return .init(txn: self.txn, val: with(self)) + } +} + +protocol OptionalType { + associatedtype Wrapped + var optional: Wrapped? { get } +} + +extension Optional: OptionalType { + typealias Wrapped = Wrapped + + var optional: Wrapped? { + return self + } +} + +extension NdbTxn where T: OptionalType { + func collect() -> NdbTxn? { + guard let unwrappedVal: T.Wrapped = val.optional else { + return nil + } + self.moved = true + return NdbTxn(txn: self.txn, val: unwrappedVal) + } +} + +extension NdbTxn where T == Bool { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == Bool? { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == Int { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == Int? { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == Double { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == Double? { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == UInt64 { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == UInt64? { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == String { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == String? { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == NoteId? { var value: T { return self.unsafeUnownedValue } } +extension NdbTxn where T == NoteId { var value: T { return self.unsafeUnownedValue } } From fc9b9f2940feb199c3ec14035f8bbe1c097a34d2 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 14:51:55 -0700 Subject: [PATCH 107/111] ndb: switch profile queries to use transactions this should ensure no crashing occurs when querying profiles --- damus/Components/NIP05Badge.swift | 3 +- damus/Components/Reposted.swift | 7 +- damus/Components/UserView.swift | 3 +- damus/ContentView.swift | 60 ++++----- damus/Models/HomeModel.swift | 11 +- damus/Models/ProfileUpdate.swift | 15 ++- damus/Nostr/Nostr.swift | 5 +- damus/Nostr/NostrResponse.swift | 2 +- damus/Nostr/Profiles.swift | 19 ++- damus/Notify/ProfileUpdatedNotify.swift | 4 +- damus/Util/DisplayName.swift | 2 +- damus/Util/EventCache.swift | 3 + damus/Util/NIP05.swift | 2 +- damus/Views/ActionBar/EventActionBar.swift | 2 +- damus/Views/BannerImageView.swift | 8 +- damus/Views/DMChatView.swift | 3 +- damus/Views/Events/Components/EventTop.swift | 3 +- .../Events/Components/ReplyDescription.swift | 17 ++- damus/Views/Events/Components/ReplyPart.swift | 6 +- damus/Views/Events/EventMenu.swift | 3 +- damus/Views/Events/EventProfile.swift | 5 +- damus/Views/Events/EventShell.swift | 4 +- damus/Views/Events/SelectedEventView.swift | 6 +- damus/Views/NoteContentView.swift | 3 +- .../Views/Notifications/EventGroupView.swift | 4 +- .../Notifications/NotificationsView.swift | 6 +- .../Views/Onboarding/SuggestedUserView.swift | 19 +-- .../Onboarding/SuggestedUsersViewModel.swift | 5 +- damus/Views/PostView.swift | 3 +- damus/Views/Posting/UserSearch.swift | 24 ++-- damus/Views/Profile/EditMetadataView.swift | 8 +- damus/Views/Profile/EventProfileName.swift | 42 ++++-- damus/Views/Profile/ProfileName.swift | 59 ++++++--- damus/Views/Profile/ProfileNameView.swift | 12 +- damus/Views/Profile/ProfilePicView.swift | 17 ++- .../Profile/ProfilePictureSelector.swift | 2 +- damus/Views/Profile/ProfileView.swift | 50 ++++---- damus/Views/QRCodeView.swift | 8 +- damus/Views/ReplyView.swift | 3 +- damus/Views/Reposts/RepostedEvent.swift | 4 +- damus/Views/Search/SearchingEventView.swift | 14 +- damus/Views/SearchResultsView.swift | 26 ++-- damus/Views/SideMenuView.swift | 3 +- damus/Views/Wallet/WalletView.swift | 11 +- damus/Views/Zaps/ZapTypePicker.swift | 3 +- nostrdb/Ndb.swift | 120 +++++++++++++++--- nostrdb/NdbNote.swift | 7 +- nostrdb/NdbTagsIterator.swift | 2 +- nostrdb/NdbTxn.swift | 2 +- nostrdb/nostrdb.c | 26 +++- nostrdb/nostrdb.h | 1 + 51 files changed, 430 insertions(+), 247 deletions(-) diff --git a/damus/Components/NIP05Badge.swift b/damus/Components/NIP05Badge.swift index 0c2552dbb1..bb11645538 100644 --- a/damus/Components/NIP05Badge.swift +++ b/damus/Components/NIP05Badge.swift @@ -45,8 +45,7 @@ struct NIP05Badge: View { } var username_matches_nip05: Bool { - guard let profile = profiles.lookup(id: pubkey), - let name = profile.name + guard let name = profiles.lookup(id: pubkey).map({ p in p?.name }).value else { return false } diff --git a/damus/Components/Reposted.swift b/damus/Components/Reposted.swift index f4e5e80f9f..f03bf5440e 100644 --- a/damus/Components/Reposted.swift +++ b/damus/Components/Reposted.swift @@ -10,13 +10,12 @@ import SwiftUI struct Reposted: View { let damus: DamusState let pubkey: Pubkey - let profile: Profile? - + var body: some View { HStack(alignment: .center) { Image("repost") .foregroundColor(Color.gray) - ProfileName(pubkey: pubkey, profile: profile, damus: damus, show_nip5_domain: false) + ProfileName(pubkey: pubkey, damus: damus, show_nip5_domain: false) .foregroundColor(Color.gray) Text("Reposted", comment: "Text indicating that the note was reposted (i.e. re-shared).") .foregroundColor(Color.gray) @@ -27,6 +26,6 @@ struct Reposted: View { struct Reposted_Previews: PreviewProvider { static var previews: some View { let test_state = test_damus_state() - Reposted(damus: test_state, pubkey: test_state.pubkey, profile: make_test_profile()) + Reposted(damus: test_state, pubkey: test_state.pubkey) } } diff --git a/damus/Components/UserView.swift b/damus/Components/UserView.swift index e59fe24bb7..b8de6c8873 100644 --- a/damus/Components/UserView.swift +++ b/damus/Components/UserView.swift @@ -37,8 +37,7 @@ struct UserView: View { ProfilePicView(pubkey: pubkey, size: PFP_SIZE, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) VStack(alignment: .leading) { - let profile = damus_state.profiles.lookup(id: pubkey) - ProfileName(pubkey: pubkey, profile: profile, damus: damus_state, show_nip5_domain: false) + ProfileName(pubkey: pubkey, damus: damus_state, show_nip5_domain: false) if let about_text { about_text .lineLimit(3) diff --git a/damus/ContentView.swift b/damus/ContentView.swift index 9027b461d2..ee1edb1476 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -359,13 +359,18 @@ struct ContentView: View { // wallet with an associated guard let ds = self.damus_state, let lud16 = nwc.lud16, - let keypair = ds.keypair.to_full(), - let profile = ds.profiles.lookup(id: ds.pubkey), - lud16 != profile.lud16 + let keypair = ds.keypair.to_full() else { return } - + + let profile_txn = ds.profiles.lookup(id: ds.pubkey) + + guard let profile = profile_txn.unsafeUnownedValue, + lud16 != profile.lud16 else { + return + } + // clear zapper cache for old lud16 if profile.lud16 != nil { // TODO: should this be somewhere else, where we process profile events!? @@ -378,15 +383,9 @@ struct ContentView: View { ds.postbox.send(ev) } .onReceive(handle_notify(.broadcast)) { ev in - guard let ds = self.damus_state else { - return - } + guard let ds = self.damus_state else { return } + ds.postbox.send(ev) - if let record = ds.profiles.lookup_with_timestamp(ev.pubkey), - let event = ds.events.lookup_by_key(record.noteKey) - { - ds.postbox.send(event) - } } .onReceive(handle_notify(.unfollow)) { target in guard let state = self.damus_state else { return } @@ -488,10 +487,12 @@ struct ContentView: View { } .onReceive(handle_notify(.onlyzaps_mode)) { hide in home.filter_events() - - guard let damus_state, - let profile = damus_state.profiles.lookup(id: damus_state.pubkey), - let keypair = damus_state.keypair.to_full() + + guard let ds = damus_state else { return } + let profile_txn = ds.profiles.lookup(id: ds.pubkey) + + guard let profile = profile_txn.unsafeUnownedValue, + let keypair = ds.keypair.to_full() else { return } @@ -499,7 +500,7 @@ struct ContentView: View { let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: profile.damus_donation, reactions: !hide) guard let profile_ev = make_metadata_event(keypair: keypair, metadata: prof) else { return } - damus_state.postbox.send(profile_ev) + ds.postbox.send(profile_ev) } .alert(NSLocalizedString("User muted", comment: "Alert message to indicate the user has been muted"), isPresented: $user_muted_confirm, actions: { Button(NSLocalizedString("Thanks!", comment: "Button to close out of alert that informs that the action to muted a user was successful.")) { @@ -507,11 +508,12 @@ struct ContentView: View { } }, message: { if let pubkey = self.muting { - let profile = damus_state!.profiles.lookup(id: pubkey) - let name = Profile.displayName(profile: profile, pubkey: pubkey).username.truncate(maxLength: 50) + let name = damus_state!.profiles.lookup(id: pubkey).map { profile in + Profile.displayName(profile: profile, pubkey: pubkey).username.truncate(maxLength: 50) + }.value Text("\(name) has been muted", comment: "Alert message that informs a user was muted.") } else { - Text("User has been muted", comment: "Alert message that informs a user was d.") + Text("User has been muted", comment: "Alert message that informs a user was muted.") } }) .alert(NSLocalizedString("Create new mutelist", comment: "Title of alert prompting the user to create a new mutelist."), isPresented: $confirm_overwrite_mutelist, actions: { @@ -567,8 +569,9 @@ struct ContentView: View { } }, message: { if let pubkey = muting { - let profile = damus_state?.profiles.lookup(id: pubkey) - let name = Profile.displayName(profile: profile, pubkey: pubkey).username.truncate(maxLength: 50) + let name = damus_state?.profiles.lookup(id: pubkey).map({ profile in + Profile.displayName(profile: profile, pubkey: pubkey).username.truncate(maxLength: 50) + }).value ?? "unknown" Text("Mute \(name)?", comment: "Alert message prompt to ask if a user should be muted.") } else { Text("Could not find user to mute...", comment: "Alert message to indicate that the muted user could not be found.") @@ -799,7 +802,7 @@ enum FindEventType { } enum FoundEvent { - case profile(Profile, NostrEvent) + case profile(Pubkey) case invalid_profile(NostrEvent) case event(NostrEvent) } @@ -816,11 +819,10 @@ func find_event_with_subid(state: DamusState, query query_: FindEvent, subid: St switch query { case .profile(let pubkey): - if let record = state.profiles.lookup_with_timestamp(pubkey), - let profile = record.profile, - let event = state.events.lookup_by_key(record.noteKey) + if let record = state.ndb.lookup_profile(pubkey).unsafeUnownedValue, + record.profile != nil { - callback(.profile(profile, event)) + callback(.profile(pubkey)) return } filter = NostrFilter(kinds: [.metadata], limit: 1, authors: [pubkey]) @@ -857,11 +859,11 @@ func find_event_with_subid(state: DamusState, query query_: FindEvent, subid: St switch query { case .profile: if ev.known_kind == .metadata { - guard let profile = state.profiles.lookup(id: ev.pubkey) else { + guard state.ndb.lookup_profile_key(ev.pubkey) != nil else { callback(.invalid_profile(ev)) return } - callback(.profile(profile, ev)) + callback(.profile(ev.pubkey)) } case .event: callback(.event(ev)) diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index 03e3b2942e..d2c2859690 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -1108,10 +1108,13 @@ func zap_notification_title(_ zap: Zap) -> String { func zap_notification_body(profiles: Profiles, zap: Zap, locale: Locale = Locale.current) -> String { let src = zap.request.ev let pk = zap.is_anon ? ANON_PUBKEY : src.pubkey - let profile = profiles.lookup(id: pk) + + let name = profiles.lookup(id: pk).map { profile in + Profile.displayName(profile: profile, pubkey: pk).displayName.truncate(maxLength: 50) + }.value + let sats = NSNumber(value: (Double(zap.invoice.amount) / 1000.0)) let formattedSats = format_msats_abbrev(zap.invoice.amount) - let name = Profile.displayName(profile: profile, pubkey: pk).displayName.truncate(maxLength: 50) if src.content.isEmpty { let format = localizedStringFormat(key: "zap_notification_no_message", locale: locale) @@ -1361,8 +1364,8 @@ func process_zap_event(damus_state: DamusState, ev: NostrEvent, completion: @esc return } - guard let record = damus_state.profiles.lookup_with_timestamp(ptag), - let lnurl = record.lnurl else { + guard let lnurl = damus_state.profiles.lookup_with_timestamp(ptag) + .map({ pr in pr?.lnurl }).value else { completion(.failed) return } diff --git a/damus/Models/ProfileUpdate.swift b/damus/Models/ProfileUpdate.swift index 7e302fba01..4f383d9463 100644 --- a/damus/Models/ProfileUpdate.swift +++ b/damus/Models/ProfileUpdate.swift @@ -8,7 +8,16 @@ import Foundation -struct ProfileUpdate { - let pubkey: Pubkey - let profile: Profile +enum ProfileUpdate { + case manual(pubkey: Pubkey, profile: Profile) + case remote(pubkey: Pubkey) + + var pubkey: Pubkey { + switch self { + case .manual(let pubkey, _): + return pubkey + case .remote(let pubkey): + return pubkey + } + } } diff --git a/damus/Nostr/Nostr.swift b/damus/Nostr/Nostr.swift index 6e4ceec76e..35d32f7752 100644 --- a/damus/Nostr/Nostr.swift +++ b/damus/Nostr/Nostr.swift @@ -8,15 +8,18 @@ import Foundation typealias Profile = NdbProfile +typealias ProfileKey = UInt64 //typealias ProfileRecord = NdbProfileRecord class ProfileRecord { let data: NdbProfileRecord - init(data: NdbProfileRecord) { + init(data: NdbProfileRecord, key: ProfileKey) { self.data = data + self.profileKey = key } + let profileKey: ProfileKey var profile: Profile? { return data.profile } var receivedAt: UInt64 { data.receivedAt } var noteKey: UInt64 { data.noteKey } diff --git a/damus/Nostr/NostrResponse.swift b/damus/Nostr/NostrResponse.swift index 218293428a..06efed8f95 100644 --- a/damus/Nostr/NostrResponse.swift +++ b/damus/Nostr/NostrResponse.swift @@ -84,7 +84,7 @@ enum NostrResponse { return nil } let new_note = note_data.assumingMemoryBound(to: ndb_note.self) - let note = NdbNote(note: new_note, owned_size: Int(len)) + let note = NdbNote(note: new_note, owned_size: Int(len), key: nil) guard let subid = sized_cstr(cstr: tce.subid, len: tce.subid_len) else { free(data) diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index ae9c01237a..adb0a61e3b 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -76,18 +76,25 @@ class Profiles { profile_data(pubkey).zapper } - func lookup_with_timestamp(_ pubkey: Pubkey) -> ProfileRecord? { + func lookup_with_timestamp(_ pubkey: Pubkey) -> NdbTxn { return ndb.lookup_profile(pubkey) } - func lookup(id: Pubkey) -> Profile? { - return ndb.lookup_profile(id)?.profile + func lookup_by_key(key: ProfileKey) -> NdbTxn { + return ndb.lookup_profile_by_key(key: key) + } + + func lookup(id: Pubkey) -> NdbTxn { + return ndb.lookup_profile(id).map({ pr in pr?.profile }) + } + + func lookup_key_by_pubkey(_ pubkey: Pubkey) -> ProfileKey? { + return ndb.lookup_profile_key(pubkey) } func has_fresh_profile(id: Pubkey) -> Bool { - var profile: Profile? - guard let profile = lookup_with_timestamp(id) else { return false } - return Date.now.timeIntervalSince(Date(timeIntervalSince1970: Double(profile.receivedAt))) < Profiles.db_freshness_threshold + guard let recv = lookup_with_timestamp(id).unsafeUnownedValue?.receivedAt else { return false } + return Date.now.timeIntervalSince(Date(timeIntervalSince1970: Double(recv))) < Profiles.db_freshness_threshold } } diff --git a/damus/Notify/ProfileUpdatedNotify.swift b/damus/Notify/ProfileUpdatedNotify.swift index b05421bfc4..4f26c9b8b9 100644 --- a/damus/Notify/ProfileUpdatedNotify.swift +++ b/damus/Notify/ProfileUpdatedNotify.swift @@ -19,7 +19,7 @@ extension NotifyHandler { } extension Notifications { - static func profile_updated(pubkey: Pubkey, profile: Profile) -> Notifications { - .init(.init(payload: ProfileUpdate(pubkey: pubkey, profile: profile))) + static func profile_updated(_ update: ProfileUpdate) -> Notifications { + .init(.init(payload: update)) } } diff --git a/damus/Util/DisplayName.swift b/damus/Util/DisplayName.swift index 4c4ec693a4..680cd077b2 100644 --- a/damus/Util/DisplayName.swift +++ b/damus/Util/DisplayName.swift @@ -7,7 +7,7 @@ import Foundation -enum DisplayName { +enum DisplayName: Equatable { case both(username: String, displayName: String) case one(String) diff --git a/damus/Util/EventCache.swift b/damus/Util/EventCache.swift index 6d0e638e6a..82bb6466a3 100644 --- a/damus/Util/EventCache.swift +++ b/damus/Util/EventCache.swift @@ -137,6 +137,7 @@ class EventData { } class EventCache { + // TODO: remove me and change code to use ndb directly private let ndb: Ndb private var events: [NoteId: NostrEvent] = [:] private var replies = ReplyMap() @@ -253,9 +254,11 @@ class EventCache { return ev } + /* func lookup_by_key(_ key: UInt64) -> NostrEvent? { ndb.lookup_note_by_key(key) } + */ func lookup(_ evid: NoteId) -> NostrEvent? { return events[evid] diff --git a/damus/Util/NIP05.swift b/damus/Util/NIP05.swift index 9432a2bb8a..1c41e4a5e9 100644 --- a/damus/Util/NIP05.swift +++ b/damus/Util/NIP05.swift @@ -7,7 +7,7 @@ import Foundation -struct NIP05 { +struct NIP05: Equatable { let username: String let host: String diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift index 79d2e30425..b08e6d1674 100644 --- a/damus/Views/ActionBar/EventActionBar.swift +++ b/damus/Views/ActionBar/EventActionBar.swift @@ -28,7 +28,7 @@ struct EventActionBar: View { } var lnurl: String? { - damus_state.profiles.lookup_with_timestamp(event.pubkey)?.lnurl + damus_state.profiles.lookup_with_timestamp(event.pubkey).map({ pr in pr?.lnurl }).value } var show_like: Bool { diff --git a/damus/Views/BannerImageView.swift b/damus/Views/BannerImageView.swift index 5c1d2e8eef..dee4e11e5d 100644 --- a/damus/Views/BannerImageView.swift +++ b/damus/Views/BannerImageView.swift @@ -81,8 +81,10 @@ struct BannerImageView: View { guard updated.pubkey == self.pubkey else { return } - - if let bannerImage = updated.profile.banner { + + let profile_txn = profiles.lookup(id: updated.pubkey) + let profile = profile_txn.unsafeUnownedValue + if let bannerImage = profile?.banner, bannerImage != self.banner { self.banner = bannerImage } } @@ -90,7 +92,7 @@ struct BannerImageView: View { } func get_banner_url(banner: String?, pubkey: Pubkey, profiles: Profiles) -> URL? { - let bannerUrlString = banner ?? profiles.lookup(id: pubkey)?.banner ?? "" + let bannerUrlString = banner ?? profiles.lookup(id: pubkey).map({ p in p?.banner }).value ?? "" if let url = URL(string: bannerUrlString) { return url } diff --git a/damus/Views/DMChatView.swift b/damus/Views/DMChatView.swift index 18bf288fc1..79a15617ae 100644 --- a/damus/Views/DMChatView.swift +++ b/damus/Views/DMChatView.swift @@ -60,12 +60,11 @@ struct DMChatView: View, KeyboardReadable { } var Header: some View { - let profile = damus_state.profiles.lookup(id: pubkey) return NavigationLink(value: Route.ProfileByKey(pubkey: pubkey)) { HStack { ProfilePicView(pubkey: pubkey, size: 24, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) - ProfileName(pubkey: pubkey, profile: profile, damus: damus_state) + ProfileName(pubkey: pubkey, damus: damus_state) } } .buttonStyle(PlainButtonStyle()) diff --git a/damus/Views/Events/Components/EventTop.swift b/damus/Views/Events/Components/EventTop.swift index 1edfeecea1..f2b5bad11b 100644 --- a/damus/Views/Events/Components/EventTop.swift +++ b/damus/Views/Events/Components/EventTop.swift @@ -22,9 +22,8 @@ struct EventTop: View { } func ProfileName(is_anon: Bool) -> some View { - let profile = state.profiles.lookup(id: self.pubkey) let pk = is_anon ? ANON_PUBKEY : self.pubkey - return EventProfileName(pubkey: pk, profile: profile, damus: state, size: .normal) + return EventProfileName(pubkey: pk, damus: state, size: .normal) } var body: some View { diff --git a/damus/Views/Events/Components/ReplyDescription.swift b/damus/Views/Events/Components/ReplyDescription.swift index 44a981467a..e6afa36f74 100644 --- a/damus/Views/Events/Components/ReplyDescription.swift +++ b/damus/Views/Events/Components/ReplyDescription.swift @@ -11,10 +11,10 @@ import SwiftUI struct ReplyDescription: View { let event: NostrEvent let replying_to: NostrEvent? - let profiles: Profiles - + let ndb: Ndb + var body: some View { - Text(verbatim: "\(reply_desc(profiles: profiles, event: event, replying_to: replying_to))") + Text(verbatim: "\(reply_desc(ndb: ndb, event: event, replying_to: replying_to))") .font(.footnote) .foregroundColor(.gray) .frame(maxWidth: .infinity, alignment: .leading) @@ -23,11 +23,11 @@ struct ReplyDescription: View { struct ReplyDescription_Previews: PreviewProvider { static var previews: some View { - ReplyDescription(event: test_note, replying_to: test_note, profiles: test_damus_state().profiles) + ReplyDescription(event: test_note, replying_to: test_note, ndb: test_damus_state().ndb) } } -func reply_desc(profiles: Profiles, event: NostrEvent, replying_to: NostrEvent?, locale: Locale = Locale.current) -> String { +func reply_desc(ndb: Ndb, event: NostrEvent, replying_to: NostrEvent?, locale: Locale = Locale.current) -> String { let desc = make_reply_description(event, replying_to: replying_to) let pubkeys = desc.pubkeys let n = desc.others @@ -38,9 +38,12 @@ func reply_desc(profiles: Profiles, event: NostrEvent, replying_to: NostrEvent?, return NSLocalizedString("Replying to self", bundle: bundle, comment: "Label to indicate that the user is replying to themself.") } + let profile_txn = NdbTxn(ndb: ndb) + let names: [String] = pubkeys.map { pk in - let prof = profiles.lookup(id: pk) - return Profile.displayName(profile: prof, pubkey: pk).username.truncate(maxLength: 50) + let prof = ndb.lookup_profile_with_txn(pk, txn: profile_txn) + + return Profile.displayName(profile: prof?.profile, pubkey: pk).username.truncate(maxLength: 50) } let uniqueNames = NSOrderedSet(array: names).array as! [String] diff --git a/damus/Views/Events/Components/ReplyPart.swift b/damus/Views/Events/Components/ReplyPart.swift index 5fc6668062..a97387e4da 100644 --- a/damus/Views/Events/Components/ReplyPart.swift +++ b/damus/Views/Events/Components/ReplyPart.swift @@ -11,7 +11,7 @@ struct ReplyPart: View { let events: EventCache let event: NostrEvent let keypair: Keypair - let profiles: Profiles + let ndb: Ndb var replying_to: NostrEvent? { guard let note_ref = event.event_refs(keypair).first(where: { evref in evref.is_direct_reply != nil })?.is_direct_reply else { @@ -24,7 +24,7 @@ struct ReplyPart: View { var body: some View { Group { if event_is_reply(event.event_refs(keypair)) { - ReplyDescription(event: event, replying_to: replying_to, profiles: profiles) + ReplyDescription(event: event, replying_to: replying_to, ndb: ndb) } else { EmptyView() } @@ -34,6 +34,6 @@ struct ReplyPart: View { struct ReplyPart_Previews: PreviewProvider { static var previews: some View { - ReplyPart(events: test_damus_state().events, event: test_note, keypair: Keypair(pubkey: .empty, privkey: nil), profiles: test_damus_state().profiles) + ReplyPart(events: test_damus_state().events, event: test_note, keypair: Keypair(pubkey: .empty, privkey: nil), ndb: test_damus_state().ndb) } } diff --git a/damus/Views/Events/EventMenu.swift b/damus/Views/Events/EventMenu.swift index ea707728c3..0c2aead794 100644 --- a/damus/Views/Events/EventMenu.swift +++ b/damus/Views/Events/EventMenu.swift @@ -52,8 +52,9 @@ struct MenuItems: View { let target_pubkey: Pubkey let bookmarks: BookmarksManager let muted_threads: MutedThreadsManager + @ObservedObject var settings: UserSettingsStore - + @State private var isBookmarked: Bool = false @State private var isMutedThread: Bool = false diff --git a/damus/Views/Events/EventProfile.swift b/damus/Views/Events/EventProfile.swift index 18533ee971..c92bbf0795 100644 --- a/damus/Views/Events/EventProfile.swift +++ b/damus/Views/Events/EventProfile.swift @@ -25,7 +25,6 @@ func eventview_pfp_size(_ size: EventViewKind) -> CGFloat { struct EventProfile: View { let damus_state: DamusState let pubkey: Pubkey - let profile: Profile? let size: EventViewKind var pfp_size: CGFloat { @@ -44,7 +43,7 @@ struct EventProfile: View { } VStack(alignment: .leading, spacing: 0) { - EventProfileName(pubkey: pubkey, profile: profile, damus: damus_state, size: size) + EventProfileName(pubkey: pubkey, damus: damus_state, size: size) UserStatusView(status: damus_state.profiles.profile_data(pubkey).status, show_general: damus_state.settings.show_general_statuses, show_music: damus_state.settings.show_music_statuses) } @@ -54,6 +53,6 @@ struct EventProfile: View { struct EventProfile_Previews: PreviewProvider { static var previews: some View { - EventProfile(damus_state: test_damus_state(), pubkey: test_note.pubkey, profile: nil, size: .normal) + EventProfile(damus_state: test_damus_state(), pubkey: test_note.pubkey, size: .normal) } } diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift index 55d116e428..01f1d34556 100644 --- a/damus/Views/Events/EventShell.swift +++ b/damus/Views/Events/EventShell.swift @@ -72,7 +72,7 @@ struct EventShell: View { UserStatusView(status: state.profiles.profile_data(pubkey).status, show_general: state.settings.show_general_statuses, show_music: state.settings.show_music_statuses) if !options.contains(.no_replying_to) { - ReplyPart(events: state.events, event: event, keypair: state.keypair, profiles: state.profiles) + ReplyPart(events: state.events, event: event, keypair: state.keypair, ndb: state.ndb) } content @@ -99,7 +99,7 @@ struct EventShell: View { VStack(alignment: .leading, spacing: 2) { EventTop(state: state, event: event, pubkey: pubkey, is_anon: is_anon) UserStatusView(status: state.profiles.profile_data(pubkey).status, show_general: state.settings.show_general_statuses, show_music: state.settings.show_music_statuses) - ReplyPart(events: state.events, event: event, keypair: state.keypair, profiles: state.profiles) + ReplyPart(events: state.events, event: event, keypair: state.keypair, ndb: state.ndb) } } .padding(.horizontal) diff --git a/damus/Views/Events/SelectedEventView.swift b/damus/Views/Events/SelectedEventView.swift index 31f73428d4..75ad2eb610 100644 --- a/damus/Views/Events/SelectedEventView.swift +++ b/damus/Views/Events/SelectedEventView.swift @@ -35,11 +35,9 @@ struct SelectedEventView: View { var body: some View { HStack(alignment: .top) { - let profile = damus.profiles.lookup(id: pubkey) - VStack(alignment: .leading) { HStack { - EventProfile(damus_state: damus, pubkey: pubkey, profile: profile, size: .normal) + EventProfile(damus_state: damus, pubkey: pubkey, size: .normal) Spacer() @@ -51,7 +49,7 @@ struct SelectedEventView: View { .lineLimit(1) if event_is_reply(event.event_refs(damus.keypair)) { - ReplyDescription(event: event, replying_to: replying_to, profiles: damus.profiles) + ReplyDescription(event: event, replying_to: replying_to, ndb: damus.ndb) .padding(.horizontal) } diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index 00cb7e911c..0c22170350 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -273,7 +273,8 @@ func mention_str(_ m: Mention, profiles: Profiles) -> CompatibleText switch m.ref { case .pubkey(let pk): let npub = bech32_pubkey(pk) - let profile = profiles.lookup(id: pk) + let profile_txn = profiles.lookup(id: pk) + let profile = profile_txn.unsafeUnownedValue let disp = Profile.displayName(profile: profile, pubkey: pk).username.truncate(maxLength: 50) var attributedString = AttributedString(stringLiteral: "@\(disp)") attributedString.link = URL(string: "damus:nostr:\(npub)") diff --git a/damus/Views/Notifications/EventGroupView.swift b/damus/Views/Notifications/EventGroupView.swift index 91f4d7a94a..b0f536a56a 100644 --- a/damus/Views/Notifications/EventGroupView.swift +++ b/damus/Views/Notifications/EventGroupView.swift @@ -69,8 +69,8 @@ func determine_reacting_to(our_pubkey: Pubkey, ev: NostrEvent?) -> ReactingTo { } func event_author_name(profiles: Profiles, pubkey: Pubkey) -> String { - let alice_prof = profiles.lookup(id: pubkey) - return Profile.displayName(profile: alice_prof, pubkey: pubkey).username.truncate(maxLength: 50) + let alice_prof_txn = profiles.lookup(id: pubkey).unsafeUnownedValue + return Profile.displayName(profile: alice_prof_txn, pubkey: pubkey).username.truncate(maxLength: 50) } func event_group_unique_pubkeys(profiles: Profiles, group: EventGroupType) -> [Pubkey] { diff --git a/damus/Views/Notifications/NotificationsView.swift b/damus/Views/Notifications/NotificationsView.swift index 76b5500e93..d33a5290c1 100644 --- a/damus/Views/Notifications/NotificationsView.swift +++ b/damus/Views/Notifications/NotificationsView.swift @@ -86,8 +86,10 @@ struct NotificationsView: View { @Environment(\.colorScheme) var colorScheme var mystery: some View { - VStack(spacing: 20) { - Text("Wake up, \(Profile.displayName(profile: state.profiles.lookup(id: state.pubkey), pubkey: state.pubkey).displayName.truncate(maxLength: 50))", comment: "Text telling the user to wake up, where the argument is their display name.") + let profile_txn = state.profiles.lookup(id: state.pubkey) + let profile = profile_txn.unsafeUnownedValue + return VStack(spacing: 20) { + Text("Wake up, \(Profile.displayName(profile: profile, pubkey: state.pubkey).displayName.truncate(maxLength: 50))", comment: "Text telling the user to wake up, where the argument is their display name.") Text("You are dreaming...", comment: "Text telling the user that they are dreaming.") } .id("what") diff --git a/damus/Views/Onboarding/SuggestedUserView.swift b/damus/Views/Onboarding/SuggestedUserView.swift index b30d134b8d..10c89fc712 100644 --- a/damus/Views/Onboarding/SuggestedUserView.swift +++ b/damus/Views/Onboarding/SuggestedUserView.swift @@ -12,14 +12,10 @@ struct SuggestedUser { let name: String let about: String let pfp: URL - let profile: Profile - init?(profile: Profile, pubkey: Pubkey) { - - guard let name = profile.name, - let about = profile.about, - let picture = profile.picture, - let pfpURL = URL(string: picture) else { + init?(name: String?, about: String?, picture: String?, pubkey: Pubkey) { + guard let name, let about, let picture, + let pfpURL = URL(string: picture) else { return nil } @@ -27,12 +23,10 @@ struct SuggestedUser { self.name = name self.about = about self.pfp = pfpURL - self.profile = profile } } struct SuggestedUserView: View { - let user: SuggestedUser let damus_state: DamusState @@ -47,7 +41,7 @@ struct SuggestedUserView: View { disable_animation: false) VStack(alignment: .leading, spacing: 4) { HStack { - ProfileName(pubkey: user.pubkey, profile: user.profile, damus: damus_state) + ProfileName(pubkey: user.pubkey, damus: damus_state) } Text(user.about) .lineLimit(3) @@ -62,9 +56,10 @@ struct SuggestedUserView: View { struct SuggestedUserView_Previews: PreviewProvider { static var previews: some View { - let profile = Profile(name: "klabo", about: "A person who likes nostr a lot and I like to tell people about myself in very long-winded ways that push the limits of UI and almost break things", picture: "https://primal.b-cdn.net/media-cache?s=m&a=1&u=https%3A%2F%2Fpbs.twimg.com%2Fprofile_images%2F1599994711430742017%2F33zLk9Wi_400x400.jpg") + let pfp = "https://primal.b-cdn.net/media-cache?s=m&a=1&u=https%3A%2F%2Fpbs.twimg.com%2Fprofile_images%2F1599994711430742017%2F33zLk9Wi_400x400.jpg" + let profile = Profile(name: "klabo", about: "A person who likes nostr a lot and I like to tell people about myself in very long-winded ways that push the limits of UI and almost break things", picture: pfp) - let user = SuggestedUser(profile: profile, pubkey: test_pubkey)! + let user = SuggestedUser(name: "klabo", about: "name", picture: "about", pubkey: test_pubkey)! List { SuggestedUserView(user: user, damus_state: test_damus_state()) } diff --git a/damus/Views/Onboarding/SuggestedUsersViewModel.swift b/damus/Views/Onboarding/SuggestedUsersViewModel.swift index 9c7af00045..b4d91c825a 100644 --- a/damus/Views/Onboarding/SuggestedUsersViewModel.swift +++ b/damus/Views/Onboarding/SuggestedUsersViewModel.swift @@ -35,8 +35,9 @@ class SuggestedUsersViewModel: ObservableObject { } func suggestedUser(pubkey: Pubkey) -> SuggestedUser? { - if let profile = damus_state.profiles.lookup(id: pubkey), - let user = SuggestedUser(profile: profile, pubkey: pubkey) { + let profile_txn = damus_state.profiles.lookup(id: pubkey) + if let profile = profile_txn.unsafeUnownedValue, + let user = SuggestedUser(name: profile.name, about: profile.about, picture: profile.picture, pubkey: pubkey) { return user } return nil diff --git a/damus/Views/PostView.swift b/damus/Views/PostView.swift index f82a3ffb96..1066bec87c 100644 --- a/damus/Views/PostView.swift +++ b/damus/Views/PostView.swift @@ -173,7 +173,8 @@ struct PostView: View { return .init(string: "") } - let profile = damus_state.profiles.lookup(id: pubkey) + let profile_txn = damus_state.profiles.lookup(id: pubkey) + let profile = profile_txn.unsafeUnownedValue return user_tag_attr_string(profile: profile, pubkey: pubkey) } diff --git a/damus/Views/Posting/UserSearch.swift b/damus/Views/Posting/UserSearch.swift index 54908a317d..66d2be41bd 100644 --- a/damus/Views/Posting/UserSearch.swift +++ b/damus/Views/Posting/UserSearch.swift @@ -7,15 +7,6 @@ import SwiftUI -struct SearchedUser: Identifiable { - let profile: Profile? - let pubkey: Pubkey - - var id: Pubkey { - return pubkey - } -} - struct UserSearch: View { let damus_state: DamusState let search: String @@ -25,13 +16,14 @@ struct UserSearch: View { @Binding var post: NSMutableAttributedString @EnvironmentObject var tagModel: TagModel - var users: [SearchedUser] { + var users: [Pubkey] { return search_profiles(profiles: damus_state.profiles, search: search) } - func on_user_tapped(user: SearchedUser) { - let pk = user.pubkey - let user_tag = user_tag_attr_string(profile: user.profile, pubkey: pk) + func on_user_tapped(pk: Pubkey) { + let profile_txn = damus_state.profiles.lookup(id: pk) + let profile = profile_txn.unsafeUnownedValue + let user_tag = user_tag_attr_string(profile: profile, pubkey: pk) appendUserTag(withTag: user_tag) } @@ -57,11 +49,11 @@ struct UserSearch: View { if users.count == 0 { EmptyUserSearchView() } else { - ForEach(users) { user in - UserView(damus_state: damus_state, pubkey: user.pubkey) + ForEach(users) { pk in + UserView(damus_state: damus_state, pubkey: pk) .contentShape(Rectangle()) .onTapGesture { - on_user_tapped(user: user) + on_user_tapped(pk: pk) } } } diff --git a/damus/Views/Profile/EditMetadataView.swift b/damus/Views/Profile/EditMetadataView.swift index a7c8186fe8..ceceb1c4f9 100644 --- a/damus/Views/Profile/EditMetadataView.swift +++ b/damus/Views/Profile/EditMetadataView.swift @@ -20,8 +20,7 @@ struct EditMetadataView: View { @State var name: String @State var ln: String @State var website: String - let profile: Profile? - + @Environment(\.dismiss) var dismiss @State var confirm_ln_address: Bool = false @@ -31,9 +30,8 @@ struct EditMetadataView: View { init(damus_state: DamusState) { self.damus_state = damus_state - let data = damus_state.profiles.lookup(id: damus_state.pubkey) - self.profile = data - + let data = damus_state.profiles.lookup(id: damus_state.pubkey).unsafeUnownedValue + _name = State(initialValue: data?.name ?? "") _display_name = State(initialValue: data?.display_name ?? "") _about = State(initialValue: data?.about ?? "") diff --git a/damus/Views/Profile/EventProfileName.swift b/damus/Views/Profile/EventProfileName.swift index 9d4788987b..a1298f394b 100644 --- a/damus/Views/Profile/EventProfileName.swift +++ b/damus/Views/Profile/EventProfileName.swift @@ -12,20 +12,19 @@ import SwiftUI struct EventProfileName: View { let damus_state: DamusState let pubkey: Pubkey - let profile: Profile? - + @State var display_name: DisplayName? @State var nip05: NIP05? @State var donation: Int? let size: EventViewKind - init(pubkey: Pubkey, profile: Profile?, damus: DamusState, size: EventViewKind = .normal) { + init(pubkey: Pubkey, damus: DamusState, size: EventViewKind = .normal) { self.damus_state = damus self.pubkey = pubkey - self.profile = profile self.size = size - self._donation = State(wrappedValue: profile?.damus_donation) + let donation = damus.ndb.lookup_profile(pubkey).map({ p in p?.profile?.damus_donation }).value + self._donation = State(wrappedValue: donation) } var friend_type: FriendType? { @@ -36,11 +35,11 @@ struct EventProfileName: View { nip05 ?? damus_state.profiles.is_validated(pubkey) } - var current_display_name: DisplayName { + func current_display_name(_ profile: Profile?) -> DisplayName { return display_name ?? Profile.displayName(profile: profile, pubkey: pubkey) } - var onlyzapper: Bool { + func onlyzapper(_ profile: Profile?) -> Bool { guard let profile else { return false } @@ -58,8 +57,10 @@ struct EventProfileName: View { } var body: some View { + let profile_txn = damus_state.profiles.lookup(id: pubkey) + let profile = profile_txn.unsafeUnownedValue HStack(spacing: 2) { - switch current_display_name { + switch current_display_name(profile) { case .one(let one): Text(one) .font(.body.weight(.bold)) @@ -84,7 +85,7 @@ struct EventProfileName: View { FriendIcon(friend: frend) } - if onlyzapper { + if onlyzapper(profile) { Image("zap-hashtag") .frame(width: 14, height: 14) } @@ -97,9 +98,24 @@ struct EventProfileName: View { if update.pubkey != pubkey { return } - display_name = Profile.displayName(profile: update.profile, pubkey: pubkey) - nip05 = damus_state.profiles.is_validated(pubkey) - donation = update.profile.damus_donation + + let profile_txn = damus_state.profiles.lookup(id: update.pubkey) + guard let profile = profile_txn.unsafeUnownedValue else { return } + + let display_name = Profile.displayName(profile: profile, pubkey: pubkey) + if display_name != self.display_name { + self.display_name = display_name + } + + let nip05 = damus_state.profiles.is_validated(pubkey) + + if self.nip05 != nip05 { + self.nip05 = nip05 + } + + if self.donation != profile.damus_donation { + donation = profile.damus_donation + } } } } @@ -107,6 +123,6 @@ struct EventProfileName: View { struct EventProfileName_Previews: PreviewProvider { static var previews: some View { - EventProfileName(pubkey: test_note.pubkey, profile: nil, damus: test_damus_state()) + EventProfileName(pubkey: test_note.pubkey, damus: test_damus_state()) } } diff --git a/damus/Views/Profile/ProfileName.swift b/damus/Views/Profile/ProfileName.swift index 18a87b5b8b..41d6a8f0c8 100644 --- a/damus/Views/Profile/ProfileName.swift +++ b/damus/Views/Profile/ProfileName.swift @@ -27,7 +27,6 @@ func get_friend_type(contacts: Contacts, pubkey: Pubkey) -> FriendType? { struct ProfileName: View { let damus_state: DamusState let pubkey: Pubkey - let profile: Profile? let prefix: String let show_nip5_domain: Bool @@ -36,9 +35,8 @@ struct ProfileName: View { @State var nip05: NIP05? @State var donation: Int? - init(pubkey: Pubkey, profile: Profile?, prefix: String = "", damus: DamusState, show_nip5_domain: Bool = true) { + init(pubkey: Pubkey, prefix: String = "", damus: DamusState, show_nip5_domain: Bool = true) { self.pubkey = pubkey - self.profile = profile self.prefix = prefix self.damus_state = damus self.show_nip5_domain = show_nip5_domain @@ -53,15 +51,15 @@ struct ProfileName: View { nip05 ?? damus_state.profiles.is_validated(pubkey) } - var current_display_name: DisplayName { + func current_display_name(profile: Profile?) -> DisplayName { return display_name ?? Profile.displayName(profile: profile, pubkey: pubkey) } - var name_choice: String { - return prefix == "@" ? current_display_name.username.truncate(maxLength: 50) : current_display_name.displayName.truncate(maxLength: 50) + func name_choice(profile: Profile?) -> String { + return prefix == "@" ? current_display_name(profile: profile).username.truncate(maxLength: 50) : current_display_name(profile: profile).displayName.truncate(maxLength: 50) } - var onlyzapper: Bool { + func onlyzapper(profile: Profile?) -> Bool { guard let profile else { return false } @@ -69,7 +67,7 @@ struct ProfileName: View { return profile.reactions == false } - var supporter: Int? { + func supporter(profile: Profile?) -> Int? { guard let profile, let donation = profile.damus_donation, donation > 0 @@ -81,21 +79,28 @@ struct ProfileName: View { } var body: some View { + let profile_txn = damus_state.profiles.lookup(id: pubkey) + let profile = profile_txn.unsafeUnownedValue + HStack(spacing: 2) { - Text(verbatim: "\(prefix)\(name_choice)") + Text(verbatim: "\(prefix)\(name_choice(profile: profile))") .font(.body) .fontWeight(prefix == "@" ? .none : .bold) + if let nip05 = current_nip05 { NIP05Badge(nip05: nip05, pubkey: pubkey, contacts: damus_state.contacts, show_domain: show_nip5_domain, profiles: damus_state.profiles) } + if let friend = friend_type, current_nip05 == nil { FriendIcon(friend: friend) } - if onlyzapper { + + if onlyzapper(profile: profile) { Image("zap-hashtag") .frame(width: 14, height: 14) } - if let supporter { + + if let supporter = supporter(profile: profile) { SupporterBadge(percent: supporter) } } @@ -103,16 +108,38 @@ struct ProfileName: View { if update.pubkey != pubkey { return } - display_name = Profile.displayName(profile: update.profile, pubkey: pubkey) - nip05 = damus_state.profiles.is_validated(pubkey) - donation = profile?.damus_donation + + var profile: Profile! + var profile_txn: NdbTxn! + + switch update { + case .remote(let pubkey): + profile_txn = damus_state.profiles.lookup(id: pubkey) + guard let prof = profile_txn.unsafeUnownedValue else { return } + profile = prof + case .manual(_, let prof): + profile = prof + } + + let display_name = Profile.displayName(profile: profile, pubkey: pubkey) + if self.display_name != display_name { + self.display_name = display_name + } + + let nip05 = damus_state.profiles.is_validated(pubkey) + if nip05 != self.nip05 { + self.nip05 = nip05 + } + + if donation != profile.damus_donation { + donation = profile.damus_donation + } } } } struct ProfileName_Previews: PreviewProvider { static var previews: some View { - ProfileName(pubkey: - test_damus_state().pubkey, profile: make_test_profile(), damus: test_damus_state()) + ProfileName(pubkey: test_damus_state().pubkey, damus: test_damus_state()) } } diff --git a/damus/Views/Profile/ProfileNameView.swift b/damus/Views/Profile/ProfileNameView.swift index 0d1c523c60..9d6efd0ea8 100644 --- a/damus/Views/Profile/ProfileNameView.swift +++ b/damus/Views/Profile/ProfileNameView.swift @@ -87,7 +87,6 @@ fileprivate struct KeyView: View { struct ProfileNameView: View { let pubkey: Pubkey - let profile: Profile? let damus: DamusState var spacing: CGFloat { 10.0 } @@ -95,10 +94,13 @@ struct ProfileNameView: View { var body: some View { Group { VStack(alignment: .leading) { + let profile_txn = self.damus.profiles.lookup(id: pubkey) + let profile = profile_txn.unsafeUnownedValue + switch Profile.displayName(profile: profile, pubkey: pubkey) { case .one: HStack(alignment: .center, spacing: spacing) { - ProfileName(pubkey: pubkey, profile: profile, damus: damus) + ProfileName(pubkey: pubkey, damus: damus) .font(.title3.weight(.bold)) } case .both(username: _, displayName: let displayName): @@ -106,7 +108,7 @@ struct ProfileNameView: View { .font(.title3.weight(.bold)) HStack(alignment: .center, spacing: spacing) { - ProfileName(pubkey: pubkey, profile: profile, prefix: "@", damus: damus) + ProfileName(pubkey: pubkey, prefix: "@", damus: damus) .font(.callout) .foregroundColor(.gray) } @@ -124,9 +126,9 @@ struct ProfileNameView: View { struct ProfileNameView_Previews: PreviewProvider { static var previews: some View { VStack { - ProfileNameView(pubkey: test_note.pubkey, profile: nil, damus: test_damus_state()) + ProfileNameView(pubkey: test_note.pubkey, damus: test_damus_state()) - ProfileNameView(pubkey: test_note.pubkey, profile: nil, damus: test_damus_state()) + ProfileNameView(pubkey: test_note.pubkey, damus: test_damus_state()) } } } diff --git a/damus/Views/Profile/ProfilePicView.swift b/damus/Views/Profile/ProfilePicView.swift index 2a3c346b6a..00184a565f 100644 --- a/damus/Views/Profile/ProfilePicView.swift +++ b/damus/Views/Profile/ProfilePicView.swift @@ -87,16 +87,25 @@ struct ProfilePicView: View { guard updated.pubkey == self.pubkey else { return } - - if let pic = updated.profile.picture { - self.picture = pic + + switch updated { + case .manual(_, let profile): + if let pic = profile.picture { + self.picture = pic + } + case .remote(pubkey: let pk): + let profile_txn = profiles.lookup(id: pk) + let profile = profile_txn.unsafeUnownedValue + if let pic = profile?.picture { + self.picture = pic + } } } } } func get_profile_url(picture: String?, pubkey: Pubkey, profiles: Profiles) -> URL { - let pic = picture ?? profiles.lookup(id: pubkey)?.picture ?? robohash(pubkey) + let pic = picture ?? profiles.lookup(id: pubkey).map({ $0?.picture }).value ?? robohash(pubkey) if let url = URL(string: pic) { return url } diff --git a/damus/Views/Profile/ProfilePictureSelector.swift b/damus/Views/Profile/ProfilePictureSelector.swift index 119c3effa3..77fc2d0f31 100644 --- a/damus/Views/Profile/ProfilePictureSelector.swift +++ b/damus/Views/Profile/ProfilePictureSelector.swift @@ -43,7 +43,7 @@ struct EditProfilePictureView: View { if let profile_url { return profile_url } else if let state = damus_state, - let picture = state.profiles.lookup(id: pubkey)?.picture { + let picture = state.profiles.lookup(id: pubkey).map({ pr in pr?.picture }).value { return URL(string: picture) } else { return profile_url ?? URL(string: robohash(pubkey)) diff --git a/damus/Views/Profile/ProfileView.swift b/damus/Views/Profile/ProfileView.swift index db928e483b..2a1339ac6d 100644 --- a/damus/Views/Profile/ProfileView.swift +++ b/damus/Views/Profile/ProfileView.swift @@ -31,11 +31,11 @@ func follow_btn_txt(_ fs: FollowState, follows_you: Bool) -> String { } } -func followedByString(_ friend_intersection: [Pubkey], profiles: Profiles, locale: Locale = Locale.current) -> String { +func followedByString(txn: NdbTxn, _ friend_intersection: [Pubkey], ndb: Ndb, locale: Locale = Locale.current) -> String { let bundle = bundleForLocale(locale: locale) - let names: [String] = friend_intersection.prefix(3).map { - let profile = profiles.lookup(id: $0) - return Profile.displayName(profile: profile, pubkey: $0).username.truncate(maxLength: 20) + let names: [String] = friend_intersection.prefix(3).map { pk in + let profile = ndb.lookup_profile_with_txn(pk, txn: txn)?.profile + return Profile.displayName(profile: profile, pubkey: pk).username.truncate(maxLength: 20) } switch friend_intersection.count { @@ -216,27 +216,29 @@ struct ProfileView: View { .accentColor(DamusColors.white) } - func lnButton(lnurl: String, record: ProfileRecord, profile: Profile) -> some View { - let profile = record.profile! - let button_img = profile.reactions == false ? "zap.fill" : "zap" - return Button(action: { + func lnButton(lnurl: String, unownedProfile: Profile?, pubkey: Pubkey) -> some View { + let reactions = unownedProfile?.reactions ?? true + let button_img = reactions ? "zap.fill" : "zap" + let lud16 = unownedProfile?.lud16 + + return Button(action: { [lnurl] in present_sheet(.zap(target: .profile(self.profile.pubkey), lnurl: lnurl)) }) { Image(button_img) .foregroundColor(button_img == "zap.fill" ? .orange : Color.primary) .profile_button_style(scheme: colorScheme) - .contextMenu { - if profile.reactions == false { + .contextMenu { [lud16, reactions, lnurl] in + if reactions == false { Text("OnlyZaps Enabled", comment: "Non-tappable text in context menu that shows up when the zap button on profile is long pressed to indicate that the user has enabled OnlyZaps, meaning that they would like to be only zapped and not accept reactions to their notes.") } - if let addr = profile.lud16 { + if let lud16 { Button { - UIPasteboard.general.string = addr + UIPasteboard.general.string = lud16 } label: { - Label(addr, image: "copy2") + Label(lud16, image: "copy2") } - } else if let lnurl = record.lnurl { + } else { Button { UIPasteboard.general.string = lnurl } label: { @@ -269,14 +271,14 @@ struct ProfileView: View { .font(.footnote) } - func actionSection(record: ProfileRecord?) -> some View { + func actionSection(record: ProfileRecord?, pubkey: Pubkey) -> some View { return Group { if let record, let profile = record.profile, let lnurl = record.lnurl, lnurl != "" { - lnButton(lnurl: lnurl, record: record, profile: profile) + lnButton(lnurl: lnurl, unownedProfile: profile, pubkey: pubkey) } dmButton @@ -311,6 +313,7 @@ struct ProfileView: View { func nameSection(profile_data: ProfileRecord?) -> some View { return Group { let follows_you = profile.pubkey != damus_state.pubkey && profile.follows(pubkey: damus_state.pubkey) + HStack(alignment: .center) { ProfilePicView(pubkey: profile.pubkey, size: pfp_size, highlight: .custom(imageBorderColor(), 4.0), profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) .padding(.top, -(pfp_size / 2.0)) @@ -329,10 +332,10 @@ struct ProfileView: View { followsYouBadge } - actionSection(record: profile_data) + actionSection(record: profile_data, pubkey: profile.pubkey) } - ProfileNameView(pubkey: profile.pubkey, profile: profile_data?.profile, damus: damus_state) + ProfileNameView(pubkey: profile.pubkey, damus: damus_state) } } @@ -355,7 +358,8 @@ struct ProfileView: View { var aboutSection: some View { VStack(alignment: .leading, spacing: 8.0) { - let profile_data = damus_state.profiles.lookup_with_timestamp(profile.pubkey) + let profile_txn = damus_state.profiles.lookup_with_timestamp(profile.pubkey) + let profile_data = profile_txn.unsafeUnownedValue nameSection(profile_data: profile_data) @@ -422,7 +426,7 @@ struct ProfileView: View { NavigationLink(value: Route.FollowersYouKnow(friendedFollowers: friended_followers, followers: followers)) { HStack { CondensedProfilePicturesView(state: damus_state, pubkeys: friended_followers, maxPictures: 3) - let followedByString = followedByString(friended_followers, profiles: damus_state.profiles) + let followedByString = followedByString(txn: profile_txn, friended_followers, ndb: damus_state.ndb) Text(followedByString) .font(.subheadline).foregroundColor(.gray) .multilineTextAlignment(.leading) @@ -516,7 +520,9 @@ extension View { @MainActor func check_nip05_validity(pubkey: Pubkey, profiles: Profiles) { - guard let profile = profiles.lookup(id: pubkey), + let profile_txn = profiles.lookup(id: pubkey) + + guard let profile = profile_txn.unsafeUnownedValue, let nip05 = profile.nip05, profiles.is_validated(pubkey) == nil else { @@ -532,7 +538,7 @@ func check_nip05_validity(pubkey: Pubkey, profiles: Profiles) { Task { @MainActor in profiles.set_validated(pubkey, nip05: validated) profiles.nip05_pubkey[nip05] = pubkey - notify(.profile_updated(pubkey: pubkey, profile: profile)) + notify(.profile_updated(.remote(pubkey: pubkey))) } } } diff --git a/damus/Views/QRCodeView.swift b/damus/Views/QRCodeView.swift index a1bdf22def..54410bcfa7 100644 --- a/damus/Views/QRCodeView.swift +++ b/damus/Views/QRCodeView.swift @@ -118,9 +118,11 @@ struct QRCodeView: View { var QRView: some View { VStack(alignment: .center) { - let profile = damus_state.profiles.lookup(id: pubkey) - - if (damus_state.profiles.lookup(id: damus_state.pubkey)?.picture) != nil { + let profile_txn = damus_state.profiles.lookup(id: pubkey) + let profile = profile_txn.unsafeUnownedValue + let our_profile = damus_state.ndb.lookup_profile_with_txn(damus_state.pubkey, txn: profile_txn) + + if our_profile?.profile?.picture != nil { ProfilePicView(pubkey: pubkey, size: 90.0, highlight: .custom(DamusColors.white, 3.0), profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) .padding(.top, 50) } else { diff --git a/damus/Views/ReplyView.swift b/damus/Views/ReplyView.swift index 916b79c52e..188993bdd4 100644 --- a/damus/Views/ReplyView.swift +++ b/damus/Views/ReplyView.swift @@ -24,10 +24,11 @@ struct ReplyView: View { var ReplyingToSection: some View { HStack { Group { + let txn = NdbTxn(ndb: damus.ndb) let names = references .map { pubkey in let pk = pubkey - let prof = damus.profiles.lookup(id: pk) + let prof = damus.ndb.lookup_profile_with_txn(pk, txn: txn)?.profile return "@" + Profile.displayName(profile: prof, pubkey: pk).username.truncate(maxLength: 50) } .joined(separator: " ") diff --git a/damus/Views/Reposts/RepostedEvent.swift b/damus/Views/Reposts/RepostedEvent.swift index edf9238547..7dda0c3436 100644 --- a/damus/Views/Reposts/RepostedEvent.swift +++ b/damus/Views/Reposts/RepostedEvent.swift @@ -15,10 +15,8 @@ struct RepostedEvent: View { var body: some View { VStack(alignment: .leading) { - let prof = damus.profiles.lookup(id: event.pubkey) - NavigationLink(value: Route.ProfileByKey(pubkey: event.pubkey)) { - Reposted(damus: damus, pubkey: event.pubkey, profile: prof) + Reposted(damus: damus, pubkey: event.pubkey) .padding(.horizontal) } .buttonStyle(PlainButtonStyle()) diff --git a/damus/Views/Search/SearchingEventView.swift b/damus/Views/Search/SearchingEventView.swift index 038479de6d..9b79eec44c 100644 --- a/damus/Views/Search/SearchingEventView.swift +++ b/damus/Views/Search/SearchingEventView.swift @@ -44,23 +44,25 @@ struct SearchingEventView: View { switch search { case .nip05(let nip05): if let pk = state.profiles.nip05_pubkey[nip05] { - if state.profiles.lookup(id: pk) != nil { + if state.profiles.lookup_key_by_pubkey(pk) != nil { self.search_state = .found_profile(pk) } } else { Task { guard let nip05 = NIP05.parse(nip05) else { - self.search_state = .not_found + Task { @MainActor in + self.search_state = .not_found + } return } guard let nip05_resp = await fetch_nip05(nip05: nip05) else { - DispatchQueue.main.async { + Task { @MainActor in self.search_state = .not_found } return } - DispatchQueue.main.async { + Task { @MainActor in guard let pk = nip05_resp.names[nip05.username] else { self.search_state = .not_found return @@ -81,11 +83,11 @@ struct SearchingEventView: View { } case .profile(let pubkey): find_event(state: state, query: .profile(pubkey: pubkey)) { res in - guard case .profile(_, let ev) = res else { + guard case .profile(let pubkey) = res else { self.search_state = .not_found return } - self.search_state = .found_profile(ev.pubkey) + self.search_state = .found_profile(pubkey) } } } diff --git a/damus/Views/SearchResultsView.swift b/damus/Views/SearchResultsView.swift index 06c7659aad..85db46a2bd 100644 --- a/damus/Views/SearchResultsView.swift +++ b/damus/Views/SearchResultsView.swift @@ -9,11 +9,11 @@ import SwiftUI struct MultiSearch { let hashtag: String - let profiles: [SearchedUser] + let profiles: [Pubkey] } enum Search: Identifiable { - case profiles([SearchedUser]) + case profiles([Pubkey]) case hashtag(String) case profile(Pubkey) case note(NoteId) @@ -49,10 +49,10 @@ struct InnerSearchResults: View { } } - func ProfilesSearch(_ results: [SearchedUser]) -> some View { + func ProfilesSearch(_ results: [Pubkey]) -> some View { return LazyVStack { - ForEach(results) { prof in - ProfileSearchResult(pk: prof.pubkey) + ForEach(results, id: \.id) { pk in + ProfileSearchResult(pk: pk) } } } @@ -171,27 +171,23 @@ func make_hashtagable(_ str: String) -> String { return String(new.filter{$0 != " "}) } -func search_profiles(profiles: Profiles, search: String) -> [SearchedUser] { +func search_profiles(profiles: Profiles, search: String) -> [Pubkey] { // Search by hex pubkey. if let pubkey = hex_decode_pubkey(search), - let profile = profiles.lookup(id: pubkey) + profiles.lookup_key_by_pubkey(pubkey) != nil { - return [SearchedUser(profile: profile, pubkey: pubkey)] + return [pubkey] } // Search by npub pubkey. if search.starts(with: "npub"), let bech32_key = decode_bech32_key(search), case Bech32Key.pub(let pk) = bech32_key, - let profile = profiles.lookup(id: pk) + profiles.lookup_key_by_pubkey(pk) != nil { - return [SearchedUser(profile: profile, pubkey: pk)] + return [pk] } let new = search.lowercased() - let matched_pubkeys = profiles.user_search_cache.search(key: new) - - return matched_pubkeys - .map { SearchedUser(profile: profiles.lookup(id: $0), pubkey: $0) } - .filter { $0.profile != nil } + return profiles.user_search_cache.search(key: new) } diff --git a/damus/Views/SideMenuView.swift b/damus/Views/SideMenuView.swift index d6fcf6305b..7b9e53d424 100644 --- a/damus/Views/SideMenuView.swift +++ b/damus/Views/SideMenuView.swift @@ -83,7 +83,8 @@ struct SideMenuView: View { } var TopProfile: some View { - let profile = damus_state.profiles.lookup(id: damus_state.pubkey) + let profile_txn = damus_state.profiles.lookup(id: damus_state.pubkey) + let profile = profile_txn.unsafeUnownedValue return VStack(alignment: .leading, spacing: verticalSpacing) { HStack { ProfilePicView(pubkey: damus_state.pubkey, size: 60, highlight: .none, profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) diff --git a/damus/Views/Wallet/WalletView.swift b/damus/Views/Wallet/WalletView.swift index 3e8bc97662..e00a2f367d 100644 --- a/damus/Views/Wallet/WalletView.swift +++ b/damus/Views/Wallet/WalletView.swift @@ -145,7 +145,7 @@ struct WalletView: View { Spacer() } - EventProfile(damus_state: damus_state, pubkey: damus_state.pubkey, profile: damus_state.profiles.lookup(id: damus_state.pubkey), size: .small) + EventProfile(damus_state: damus_state, pubkey: damus_state.pubkey, size: .small) } .padding(25) } @@ -164,17 +164,20 @@ struct WalletView: View { model.initial_percent = settings.donation_percent } .onChange(of: settings.donation_percent) { p in - guard let profile = damus_state.profiles.lookup(id: damus_state.pubkey) else { + let profile_txn = damus_state.profiles.lookup(id: damus_state.pubkey) + guard let profile = profile_txn.unsafeUnownedValue else { return } let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: p, reactions: profile.reactions) - notify(.profile_updated(pubkey: damus_state.pubkey, profile: prof)) + notify(.profile_updated(.manual(pubkey: self.damus_state.pubkey, profile: prof))) } .onDisappear { + let profile_txn = damus_state.profiles.lookup(id: damus_state.pubkey) + guard let keypair = damus_state.keypair.to_full(), - let profile = damus_state.profiles.lookup(id: damus_state.pubkey), + let profile = profile_txn.unsafeUnownedValue, model.initial_percent != profile.damus_donation else { return diff --git a/damus/Views/Zaps/ZapTypePicker.swift b/damus/Views/Zaps/ZapTypePicker.swift index a85b7e91fa..1a128b71ad 100644 --- a/damus/Views/Zaps/ZapTypePicker.swift +++ b/damus/Views/Zaps/ZapTypePicker.swift @@ -117,7 +117,8 @@ func zap_type_desc(type: ZapType, profiles: Profiles, pubkey: Pubkey) -> String case .anon: return NSLocalizedString("No one will see that you zapped", comment: "Description of anonymous zap type where the zap is sent anonymously and does not identify the user who sent it.") case .priv: - let prof = profiles.lookup(id: pubkey) + let prof_txn = profiles.lookup(id: pubkey) + let prof = prof_txn.unsafeUnownedValue let name = Profile.displayName(profile: prof, pubkey: pubkey).username.truncate(maxLength: 50) return String.localizedStringWithFormat(NSLocalizedString("private_zap_description", value: "Only '%@' will see that you zapped them", comment: "Description of private zap type where the zap is sent privately and does not identify the user to the public."), name) case .non_zap: diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index 4e236e4e6f..791c7535b1 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -49,44 +49,130 @@ class Ndb { self.ndb = ndb } - func lookup_note_by_key(_ key: UInt64) -> NdbNote? { - guard let note_p = ndb_get_note_by_key(ndb.ndb, key, nil) else { + func lookup_note_by_key_with_txn(_ key: NoteKey, txn: NdbTxn) -> NdbNote? { + guard let note_p = ndb_get_note_by_key(&txn.txn, key, nil) else { return nil } - return NdbNote(note: note_p, owned_size: nil) + return NdbNote(note: note_p, owned_size: nil, key: key) } - func lookup_note(_ id: NoteId) -> NdbNote? { - id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NdbNote? in + func lookup_note_by_key(_ key: NoteKey) -> NdbTxn { + return NdbTxn(ndb: self) { txn in + lookup_note_by_key_with_txn(key, txn: txn) + } + } + + private func lookup_profile_by_key_inner(_ key: ProfileKey, txn: NdbTxn) -> ProfileRecord? { + var size: Int = 0 + guard let profile_p = ndb_get_profile_by_key(&txn.txn, key, &size) else { + return nil + } + + return profile_flatbuf_to_record(ptr: profile_p, size: size, key: key) + } + + private func profile_flatbuf_to_record(ptr: UnsafeMutableRawPointer, size: Int, key: UInt64) -> ProfileRecord? { + do { + var buf = ByteBuffer(assumingMemoryBound: ptr, capacity: size) + let rec: NdbProfileRecord = try getDebugCheckedRoot(byteBuffer: &buf) + return ProfileRecord(data: rec, key: key) + } catch { + // Handle error appropriately + print("UNUSUAL: \(error)") + return nil + } + } + + private func lookup_note_with_txn_inner(id: NoteId, txn: NdbTxn) -> NdbNote? { + return id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NdbNote? in + var key: UInt64 = 0 guard let baseAddress = ptr.baseAddress, - let note_p = ndb_get_note_by_id(ndb.ndb, baseAddress, nil) else { + let note_p = ndb_get_note_by_id(&txn.txn, baseAddress, nil, &key) else { return nil } - return NdbNote(note: note_p, owned_size: nil) + return NdbNote(note: note_p, owned_size: nil, key: key) } } - func lookup_profile(_ pubkey: Pubkey) -> ProfileRecord? { + private func lookup_profile_with_txn_inner(pubkey: Pubkey, txn: NdbTxn) -> ProfileRecord? { return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> ProfileRecord? in var size: Int = 0 + var key: UInt64 = 0 guard let baseAddress = ptr.baseAddress, - let profile_p = ndb_get_profile_by_pubkey(ndb.ndb, baseAddress, &size) + let profile_p = ndb_get_profile_by_pubkey(&txn.txn, baseAddress, &size, &key) else { return nil } - do { - var buf = ByteBuffer(assumingMemoryBound: profile_p, capacity: size) - let rec: NdbProfileRecord = try getDebugCheckedRoot(byteBuffer: &buf) - return ProfileRecord(data: rec) - } catch { - // Handle error appropriately - print("UNUSUAL: \(error)") + return profile_flatbuf_to_record(ptr: profile_p, size: size, key: key) + } + } + + func lookup_profile_by_key_with_txn(key: ProfileKey, txn: NdbTxn) -> ProfileRecord? { + lookup_profile_by_key_inner(key, txn: txn) + } + + func lookup_profile_by_key(key: ProfileKey) -> NdbTxn { + return NdbTxn(ndb: self) { txn in + lookup_profile_by_key_inner(key, txn: txn) + } + } + + func lookup_note_with_txn(id: NoteId, txn: NdbTxn) -> NdbNote? { + lookup_note_with_txn_inner(id: id, txn: txn) + } + + func lookup_profile_key(_ pubkey: Pubkey) -> ProfileKey? { + return NdbTxn(ndb: self) { txn in + lookup_profile_key_with_txn(pubkey, txn: txn) + }.value + } + + func lookup_profile_key_with_txn(_ pubkey: Pubkey, txn: NdbTxn) -> ProfileKey? { + return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NoteKey? in + guard let p = ptr.baseAddress else { return nil } + let r = ndb_get_profilekey_by_pubkey(&txn.txn, p) + if r == 0 { return nil } + return r } } + + func lookup_note_key_with_txn(_ id: NoteId, txn: NdbTxn) -> NoteKey? { + return id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NoteKey? in + guard let p = ptr.baseAddress else { + return nil + } + let r = ndb_get_notekey_by_id(&txn.txn, p) + if r == 0 { + return nil + } + return r + } + } + + func lookup_note_key(_ id: NoteId) -> NoteKey? { + NdbTxn(ndb: self, with: { txn in lookup_note_key_with_txn(id, txn: txn) }).value + } + + func lookup_note(_ id: NoteId) -> NdbTxn { + return NdbTxn(ndb: self) { txn in + lookup_note_with_txn_inner(id: id, txn: txn) + } + } + + func lookup_profile(_ pubkey: Pubkey) -> NdbTxn { + return NdbTxn(ndb: self) { txn in + lookup_profile_with_txn_inner(pubkey: pubkey, txn: txn) + } + } + + func lookup_profile_with_txn(_ pubkey: Pubkey, txn: NdbTxn) -> ProfileRecord? { + lookup_profile_with_txn_inner(pubkey: pubkey, txn: txn) + } + func process_event(_ str: String) -> Bool { return str.withCString { cstr in return ndb_process_event(ndb.ndb, cstr, Int32(str.utf8.count)) != 0 @@ -106,7 +192,7 @@ class Ndb { #if DEBUG func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) throws -> T { - return try getCheckedRoot(byteBuffer: &byteBuffer) + return try getRoot(byteBuffer: &byteBuffer) } #else func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) throws -> T { diff --git a/nostrdb/NdbNote.swift b/nostrdb/NdbNote.swift index 5f479a23e4..b46deca395 100644 --- a/nostrdb/NdbNote.swift +++ b/nostrdb/NdbNote.swift @@ -38,6 +38,7 @@ class NdbNote: Encodable, Equatable, Hashable { // we can have owned notes, but we can also have lmdb virtual-memory mapped notes so its optional private let owned: Bool let count: Int + let key: NoteKey? let note: UnsafeMutablePointer // cached stuff (TODO: remove these) @@ -48,10 +49,11 @@ class NdbNote: Encodable, Equatable, Hashable { return NdbNote.owned_from_json_cstr(json: content_raw, json_len: content_len) }() - init(note: UnsafeMutablePointer, owned_size: Int?) { + init(note: UnsafeMutablePointer, owned_size: Int?, key: NoteKey?) { self.note = note self.owned = owned_size != nil self.count = owned_size ?? 0 + self.key = key #if DEBUG_NOTE_SIZE if let owned_size { @@ -218,6 +220,7 @@ class NdbNote: Encodable, Equatable, Hashable { } self.note = r.assumingMemoryBound(to: ndb_note.self) + self.key = nil } static func owned_from_json(json: String, bufsize: Int = 2 << 18) -> NdbNote? { @@ -245,7 +248,7 @@ class NdbNote: Encodable, Equatable, Hashable { guard let note_data = realloc(data, Int(len)) else { return nil } let new_note = note_data.assumingMemoryBound(to: ndb_note.self) - return NdbNote(note: new_note, owned_size: Int(len)) + return NdbNote(note: new_note, owned_size: Int(len), key: nil) } } diff --git a/nostrdb/NdbTagsIterator.swift b/nostrdb/NdbTagsIterator.swift index e6ecd60da7..933100b6ca 100644 --- a/nostrdb/NdbTagsIterator.swift +++ b/nostrdb/NdbTagsIterator.swift @@ -70,7 +70,7 @@ struct TagsSequence: Encodable, Sequence { } precondition(false, "sequence subscript oob") // it seems like the compiler needs this or it gets bitchy - return .init(note: .init(note: .allocate(capacity: 1), owned_size: nil), tag: .allocate(capacity: 1)) + return .init(note: .init(note: .allocate(capacity: 1), owned_size: nil, key: nil), tag: .allocate(capacity: 1)) } func makeIterator() -> TagsIterator { diff --git a/nostrdb/NdbTxn.swift b/nostrdb/NdbTxn.swift index b053cd9121..876fa80a59 100644 --- a/nostrdb/NdbTxn.swift +++ b/nostrdb/NdbTxn.swift @@ -17,7 +17,7 @@ class NdbTxn { private var val: T! var moved: Bool - init(ndb: Ndb, with: (NdbTxn) -> T) { + init(ndb: Ndb, with: (NdbTxn) -> T = { _ in () }) { self.txn = ndb_txn() #if TXNDEBUG txn_count += 1 diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index 4fce06f1b6..7e89582848 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -264,7 +264,9 @@ int ndb_get_tsid(MDB_txn *txn, struct ndb_lmdb *lmdb, enum ndb_dbs db, int success = 0; struct ndb_tsid tsid; + // position at the most recent ndb_tsid_high(&tsid, id); + k.mv_data = &tsid; k.mv_size = sizeof(tsid); @@ -341,32 +343,44 @@ static void *ndb_lookup_tsid(struct ndb_txn *txn, enum ndb_dbs ind, return res; } -void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pk, size_t *len, uint32_t *key) +void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pk, size_t *len, uint64_t *key) { return ndb_lookup_tsid(txn, NDB_DB_PROFILE_PK, NDB_DB_PROFILE, pk, len, key); } -struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint32_t *key) +struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *key) { return ndb_lookup_tsid(txn, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len, key); } -uint32_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id) +static inline uint64_t ndb_get_indexkey_by_id(struct ndb_txn *txn, + enum ndb_dbs db, + const unsigned char *id) { MDB_val k; - if (!ndb_get_tsid(txn->mdb_txn, &txn->ndb->lmdb, NDB_DB_NOTE_ID, id, &k)) + if (!ndb_get_tsid(txn->mdb_txn, &txn->ndb->lmdb, db, id, &k)) return 0; return *(uint32_t*)k.mv_data; } -struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint32_t key, size_t *len) +uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id) +{ + return ndb_get_indexkey_by_id(txn, NDB_DB_NOTE_ID, id); +} + +uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id) +{ + return ndb_get_indexkey_by_id(txn, NDB_DB_PROFILE_PK, id); +} + +struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len) { return ndb_lookup_by_key(txn, key, NDB_DB_NOTE, len); } -void *ndb_get_profile_by_key(struct ndb_txn *txn, uint32_t key, size_t *len) +void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len) { return ndb_lookup_by_key(txn, key, NDB_DB_PROFILE, len); } diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index f4606d59d8..7ee2c2c51a 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -166,6 +166,7 @@ void ndb_end_query(struct ndb_txn *); void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey); void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id); +uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id); struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *primkey); struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); void ndb_destroy(struct ndb *); From 22d635d85046ac8f2ed50f2c562ef456e0b9b25f Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 16:12:03 -0700 Subject: [PATCH 108/111] ndb: don't verify flatbuffers in release builds --- nostrdb/Ndb.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index 791c7535b1..e1a86e51e8 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -191,11 +191,11 @@ class Ndb { } #if DEBUG -func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) throws -> T { - return try getRoot(byteBuffer: &byteBuffer) +func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) throws -> T { + return getRoot(byteBuffer: &byteBuffer) } #else func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) throws -> T { - return try getRoot(byteBuffer: &byteBuffer) + return getRoot(byteBuffer: &byteBuffer) } #endif From 69c7acea76c3b931d404148ff2779278587a198c Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 10 Sep 2023 15:50:27 -0700 Subject: [PATCH 109/111] tests: add ndb support to tests stops it from crashing --- damus/Components/ImageCarousel.swift | 2 +- damus/Components/InvoiceView.swift | 2 +- damus/Components/InvoicesView.swift | 2 +- damus/Components/NIP05Badge.swift | 2 +- damus/Components/Reposted.swift | 2 +- .../Components/Search/SearchHeaderView.swift | 4 +- damus/Components/Status/UserStatusSheet.swift | 2 +- damus/Components/TranslateView.swift | 2 +- damus/Components/UserView.swift | 2 +- damus/Components/ZapButton.swift | 2 +- damus/TestData.swift | 54 ++++++++++++++++--- damus/Views/ActionBar/EventActionBar.swift | 2 +- damus/Views/ActionBar/EventDetailBar.swift | 2 +- damus/Views/ActionBar/RepostAction.swift | 2 +- damus/Views/AddRelayView.swift | 2 +- damus/Views/ConfigView.swift | 2 +- damus/Views/DMChatView.swift | 2 +- damus/Views/DMView.swift | 2 +- damus/Views/DirectMessagesView.swift | 2 +- damus/Views/EventDetailView.swift | 2 +- damus/Views/EventView.swift | 4 +- damus/Views/Events/BuilderEventView.swift | 2 +- damus/Views/Events/Components/EventTop.swift | 2 +- .../Events/Components/ReplyDescription.swift | 2 +- damus/Views/Events/Components/ReplyPart.swift | 2 +- damus/Views/Events/EventBody.swift | 2 +- damus/Views/Events/EventProfile.swift | 2 +- damus/Views/Events/EventShell.swift | 4 +- .../Events/Longform/LongformPreview.swift | 4 +- .../Views/Events/Longform/LongformView.swift | 2 +- damus/Views/Events/MutedEventView.swift | 2 +- damus/Views/Events/SelectedEventView.swift | 2 +- damus/Views/Events/TextEvent.swift | 4 +- damus/Views/Events/ZapEvent.swift | 4 +- damus/Views/Images/ImageContainerView.swift | 2 +- damus/Views/Images/ImageView.swift | 2 +- damus/Views/Muting/MutelistView.swift | 2 +- damus/Views/NoteContentView.swift | 4 +- .../Views/Notifications/EventGroupView.swift | 9 ++-- .../Notifications/NotificationItemView.swift | 2 +- .../Notifications/NotificationsView.swift | 2 +- .../Notifications/ProfilePicturesView.swift | 2 +- .../Views/Onboarding/SuggestedUserView.swift | 2 +- .../Views/Onboarding/SuggestedUsersView.swift | 2 +- damus/Views/PostView.swift | 2 +- damus/Views/Posting/UserSearch.swift | 2 +- .../CondensedProfilePicturesView.swift | 2 +- damus/Views/Profile/EditMetadataView.swift | 2 +- damus/Views/Profile/EventProfileName.swift | 2 +- damus/Views/Profile/MaybeAnonPfpView.swift | 2 +- damus/Views/Profile/ProfileEditButton.swift | 2 +- damus/Views/Profile/ProfileName.swift | 2 +- damus/Views/Profile/ProfileNameView.swift | 4 +- damus/Views/Profile/ProfilePicView.swift | 2 +- damus/Views/Profile/ProfileView.swift | 2 +- damus/Views/QRCodeView.swift | 2 +- damus/Views/Reactions/ReactionView.swift | 2 +- damus/Views/ReactionsView.swift | 2 +- damus/Views/RelayFilterView.swift | 2 +- damus/Views/Relays/RecommendedRelayView.swift | 2 +- damus/Views/Relays/RelayConfigView.swift | 2 +- damus/Views/Relays/RelayDetailView.swift | 2 +- damus/Views/Relays/RelayStatusView.swift | 2 +- damus/Views/Relays/RelayToggle.swift | 2 +- damus/Views/Relays/RelayView.swift | 2 +- damus/Views/Relays/SignalView.swift | 2 +- damus/Views/ReplyView.swift | 4 +- damus/Views/ReportView.swift | 2 +- damus/Views/Reposts/RepostView.swift | 2 +- damus/Views/Reposts/RepostedEvent.swift | 2 +- damus/Views/RepostsView.swift | 2 +- damus/Views/Search/SearchingEventView.swift | 2 +- damus/Views/SearchHomeView.swift | 2 +- damus/Views/SearchView.swift | 2 +- .../Settings/AppearanceSettingsView.swift | 2 +- damus/Views/SideMenuView.swift | 2 +- damus/Views/ThreadView.swift | 2 +- damus/Views/Timeline/InnerTimelineView.swift | 2 +- damus/Views/UserRelaysView.swift | 2 +- damus/Views/Wallet/WalletView.swift | 4 +- damus/Views/Zaps/CustomizeZapView.swift | 2 +- damus/Views/Zaps/ZapTypePicker.swift | 2 +- damus/Views/Zaps/ZapUserView.swift | 2 +- damus/Views/Zaps/ZapsView.swift | 2 +- damusTests/EventGroupViewTests.swift | 6 +-- damusTests/LongPostTests.swift | 2 +- damusTests/NoteContentViewTests.swift | 2 +- damusTests/ProfileViewTests.swift | 17 +++--- damusTests/UserSearchCacheTests.swift | 7 +-- damusTests/ZapTests.swift | 2 +- nostrdb/Ndb.swift | 14 +++-- nostrdb/Test/NdbTests.swift | 31 ++++++++--- 92 files changed, 195 insertions(+), 133 deletions(-) diff --git a/damus/Components/ImageCarousel.swift b/damus/Components/ImageCarousel.swift index 811b9ff2b8..d7586b3a70 100644 --- a/damus/Components/ImageCarousel.swift +++ b/damus/Components/ImageCarousel.swift @@ -285,7 +285,7 @@ public struct ImageFill { struct ImageCarousel_Previews: PreviewProvider { static var previews: some View { let url: MediaUrl = .image(URL(string: "https://jb55.com/red-me.jpg")!) - ImageCarousel(state: test_damus_state(), evid: test_note.id, urls: [url, url]) + ImageCarousel(state: test_damus_state, evid: test_note.id, urls: [url, url]) } } diff --git a/damus/Components/InvoiceView.swift b/damus/Components/InvoiceView.swift index ad37003069..a38f7bfce3 100644 --- a/damus/Components/InvoiceView.swift +++ b/damus/Components/InvoiceView.swift @@ -108,7 +108,7 @@ let test_invoice = Invoice(description: .description("this is a description"), a struct InvoiceView_Previews: PreviewProvider { static var previews: some View { - InvoiceView(our_pubkey: .empty, invoice: test_invoice, settings: test_damus_state().settings) + InvoiceView(our_pubkey: .empty, invoice: test_invoice, settings: test_damus_state.settings) .frame(width: 300, height: 200) } } diff --git a/damus/Components/InvoicesView.swift b/damus/Components/InvoicesView.swift index 9c587abb6d..589e21edf6 100644 --- a/damus/Components/InvoicesView.swift +++ b/damus/Components/InvoicesView.swift @@ -29,7 +29,7 @@ struct InvoicesView: View { struct InvoicesView_Previews: PreviewProvider { static var previews: some View { - InvoicesView(our_pubkey: test_note.pubkey, invoices: [Invoice.init(description: .description("description"), amount: .specific(10000), string: "invstr", expiry: 100000, payment_hash: Data(), created_at: 1000000)], settings: test_damus_state().settings) + InvoicesView(our_pubkey: test_note.pubkey, invoices: [Invoice.init(description: .description("description"), amount: .specific(10000), string: "invstr", expiry: 100000, payment_hash: Data(), created_at: 1000000)], settings: test_damus_state.settings) .frame(width: 300) } } diff --git a/damus/Components/NIP05Badge.swift b/damus/Components/NIP05Badge.swift index bb11645538..db7956af31 100644 --- a/damus/Components/NIP05Badge.swift +++ b/damus/Components/NIP05Badge.swift @@ -96,7 +96,7 @@ func use_nip05_color(pubkey: Pubkey, contacts: Contacts) -> Bool { struct NIP05Badge_Previews: PreviewProvider { static var previews: some View { - let test_state = test_damus_state() + let test_state = test_damus_state VStack { NIP05Badge(nip05: NIP05(username: "jb55", host: "jb55.com"), pubkey: test_state.pubkey, contacts: test_state.contacts, show_domain: true, profiles: test_state.profiles) diff --git a/damus/Components/Reposted.swift b/damus/Components/Reposted.swift index f03bf5440e..a3e94b0e70 100644 --- a/damus/Components/Reposted.swift +++ b/damus/Components/Reposted.swift @@ -25,7 +25,7 @@ struct Reposted: View { struct Reposted_Previews: PreviewProvider { static var previews: some View { - let test_state = test_damus_state() + let test_state = test_damus_state Reposted(damus: test_state, pubkey: test_state.pubkey) } } diff --git a/damus/Components/Search/SearchHeaderView.swift b/damus/Components/Search/SearchHeaderView.swift index 78ce111a6c..43afce374a 100644 --- a/damus/Components/Search/SearchHeaderView.swift +++ b/damus/Components/Search/SearchHeaderView.swift @@ -168,9 +168,9 @@ func is_following_hashtag(contacts: NostrEvent?, hashtag: String) -> Bool { struct SearchHeaderView_Previews: PreviewProvider { static var previews: some View { VStack(alignment: .leading) { - SearchHeaderView(state: test_damus_state(), described: .hashtag("damus")) + SearchHeaderView(state: test_damus_state, described: .hashtag("damus")) - SearchHeaderView(state: test_damus_state(), described: .unknown) + SearchHeaderView(state: test_damus_state, described: .unknown) } } } diff --git a/damus/Components/Status/UserStatusSheet.swift b/damus/Components/Status/UserStatusSheet.swift index 00e9dba8b7..8ed36a9b15 100644 --- a/damus/Components/Status/UserStatusSheet.swift +++ b/damus/Components/Status/UserStatusSheet.swift @@ -136,6 +136,6 @@ struct UserStatusSheet: View { struct UserStatusSheet_Previews: PreviewProvider { static var previews: some View { - UserStatusSheet(postbox: PostBox(pool: RelayPool(ndb: .empty)), keypair: Keypair(pubkey: .empty, privkey: nil), status: .init()) + UserStatusSheet(postbox: test_damus_state.postbox, keypair: test_keypair, status: .init()) } } diff --git a/damus/Components/TranslateView.swift b/damus/Components/TranslateView.swift index bbe7fa5f6a..c44c0f8914 100644 --- a/damus/Components/TranslateView.swift +++ b/damus/Components/TranslateView.swift @@ -120,7 +120,7 @@ extension View { struct TranslateView_Previews: PreviewProvider { static var previews: some View { - let ds = test_damus_state() + let ds = test_damus_state TranslateView(damus_state: ds, event: test_note, size: .normal) } } diff --git a/damus/Components/UserView.swift b/damus/Components/UserView.swift index b8de6c8873..c83b0c0a7f 100644 --- a/damus/Components/UserView.swift +++ b/damus/Components/UserView.swift @@ -55,6 +55,6 @@ struct UserView: View { struct UserView_Previews: PreviewProvider { static var previews: some View { - UserView(damus_state: test_damus_state(), pubkey: test_note.pubkey) + UserView(damus_state: test_damus_state, pubkey: test_note.pubkey) } } diff --git a/damus/Components/ZapButton.swift b/damus/Components/ZapButton.swift index c764fdac9a..5ad432bc55 100644 --- a/damus/Components/ZapButton.swift +++ b/damus/Components/ZapButton.swift @@ -144,7 +144,7 @@ struct ZapButton_Previews: PreviewProvider { let pending_zap = PendingZap(amount_msat: 1000, target: ZapTarget.note(id: test_note.id, author: test_note.pubkey), request: .normal(test_zap_request), type: .pub, state: .external(.init(state: .fetching_invoice))) let zaps = ZapsDataModel([.pending(pending_zap)]) - ZapButton(damus_state: test_damus_state(), target: ZapTarget.note(id: test_note.id, author: test_note.pubkey), lnurl: "lnurl", zaps: zaps) + ZapButton(damus_state: test_damus_state, target: ZapTarget.note(id: test_note.id, author: test_note.pubkey), lnurl: "lnurl", zaps: zaps) } } diff --git a/damus/TestData.swift b/damus/TestData.swift index 1eaba97a8c..b152e8d4d9 100644 --- a/damus/TestData.swift +++ b/damus/TestData.swift @@ -48,11 +48,53 @@ let test_private_zap = Zap(event: test_note, invoice: test_zap_invoice, zapper: let test_pending_zap = PendingZap(amount_msat: 10000, target: .note(id: test_note.id, author: test_note.pubkey), request: .normal(test_zap_request), type: .pub, state: .external(.init(state: .fetching_invoice))) -let test_following_model = FollowingModel(damus_state: test_damus_state(), contacts: [test_pubkey, test_pubkey_2], hashtags: [Hashtag(hashtag: "grownostr"), Hashtag(hashtag: "zapathon")]) - - -func test_damus_state() -> DamusState { - let damus = DamusState.empty +let test_following_model = FollowingModel(damus_state: test_damus_state, contacts: [test_pubkey, test_pubkey_2], hashtags: [Hashtag(hashtag: "grownostr"), Hashtag(hashtag: "zapathon")]) + + +var test_damus_state: DamusState = ({ + // Create a unique temporary directory + var tempDir: String! + do { + let fileManager = FileManager.default + let temp = fileManager.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try fileManager.createDirectory(at: temp, withIntermediateDirectories: true, attributes: nil) + tempDir = temp.absoluteString + } catch { + tempDir = "." + } + + print("opening \(tempDir!)") + let ndb = Ndb(path: tempDir)! + let our_pubkey = test_pubkey + let user_search_cache = UserSearchCache() + let pool = RelayPool(ndb: ndb) + let settings = UserSettingsStore() + let damus = DamusState(pool: pool, + keypair: test_keypair, + likes: .init(our_pubkey: our_pubkey), + boosts: .init(our_pubkey: our_pubkey), + contacts: .init(our_pubkey: our_pubkey), + profiles: .init(user_search_cache: user_search_cache, ndb: ndb), + dms: .init(our_pubkey: our_pubkey), + previews: .init(), + zaps: .init(our_pubkey: our_pubkey), + lnurls: .init(), + settings: settings, + relay_filters: .init(our_pubkey: our_pubkey), + relay_model_cache: .init(), + drafts: .init(), + events: .init(ndb: ndb), + bookmarks: .init(pubkey: our_pubkey), + postbox: .init(pool: pool), + bootstrap_relays: .init(), + replies: .init(our_pubkey: our_pubkey), + muted_threads: .init(keypair: test_keypair), + wallet: .init(settings: settings), + nav: .init(), + user_search_cache: user_search_cache, + music: .init(onChange: {_ in }), + video: .init(), + ndb: ndb) /* let prof = Profile(name: "damus", display_name: "damus", about: "iOS app!", picture: "https://damus.io/img/logo.png", banner: "", website: "https://damus.io", lud06: nil, lud16: "jb55@sendsats.lol", nip05: "damus.io", damus_donation: nil) @@ -60,7 +102,7 @@ func test_damus_state() -> DamusState { damus.profiles.add(id: test_pubkey, profile: tsprof) */ return damus -} +})() let test_wire_events = """ ["EVENT","s",{"id":"d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349","pubkey":"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245","created_at":1650049978,"kind":0,"tags":[],"content":"{\\"name\\\":\\"jb55\\",\\"picture\\":\\\"http://cdn.jb55.com/img/red-me.jpg\\",\\"about\\":\\"bitcoin, lightning and nostr dev\\",\\"nip05\\":\\"jb55.com\\"}","sig":"1315045da793c4825de1517149172bf35a6da39d91b7787afb3263721e07bc816cb898996ed8d69af05d6efcd1c926a089bd66cad870bcc361405c11ba302c51"}] diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift index b08e6d1674..3cb18a5a75 100644 --- a/damus/Views/ActionBar/EventActionBar.swift +++ b/damus/Views/ActionBar/EventActionBar.swift @@ -374,7 +374,7 @@ struct LikeButton: View { struct EventActionBar_Previews: PreviewProvider { static var previews: some View { - let ds = test_damus_state() + let ds = test_damus_state let ev = NostrEvent(content: "hi", keypair: test_keypair)! let bar = ActionBarModel.empty() diff --git a/damus/Views/ActionBar/EventDetailBar.swift b/damus/Views/ActionBar/EventDetailBar.swift index ef22210ab0..aca05e6508 100644 --- a/damus/Views/ActionBar/EventDetailBar.swift +++ b/damus/Views/ActionBar/EventDetailBar.swift @@ -56,6 +56,6 @@ struct EventDetailBar: View { struct EventDetailBar_Previews: PreviewProvider { static var previews: some View { - EventDetailBar(state: test_damus_state(), target: .empty, target_pk: .empty) + EventDetailBar(state: test_damus_state, target: .empty, target_pk: .empty) } } diff --git a/damus/Views/ActionBar/RepostAction.swift b/damus/Views/ActionBar/RepostAction.swift index dc0f051912..2ab7a5b2dc 100644 --- a/damus/Views/ActionBar/RepostAction.swift +++ b/damus/Views/ActionBar/RepostAction.swift @@ -61,6 +61,6 @@ struct RepostAction: View { struct RepostAction_Previews: PreviewProvider { static var previews: some View { - RepostAction(damus_state: test_damus_state(), event: test_note) + RepostAction(damus_state: test_damus_state, event: test_note) } } diff --git a/damus/Views/AddRelayView.swift b/damus/Views/AddRelayView.swift index a992b66697..046cf7aa88 100644 --- a/damus/Views/AddRelayView.swift +++ b/damus/Views/AddRelayView.swift @@ -158,6 +158,6 @@ struct AddRelayView_Previews: PreviewProvider { @State static var relay: String = "" static var previews: some View { - AddRelayView(state: test_damus_state()) + AddRelayView(state: test_damus_state) } } diff --git a/damus/Views/ConfigView.swift b/damus/Views/ConfigView.swift index b0d67a4a52..8ed45dd433 100644 --- a/damus/Views/ConfigView.swift +++ b/damus/Views/ConfigView.swift @@ -153,7 +153,7 @@ struct ConfigView: View { struct ConfigView_Previews: PreviewProvider { static var previews: some View { NavigationView { - ConfigView(state: test_damus_state()) + ConfigView(state: test_damus_state) } } } diff --git a/damus/Views/DMChatView.swift b/damus/Views/DMChatView.swift index 79a15617ae..234857eac1 100644 --- a/damus/Views/DMChatView.swift +++ b/damus/Views/DMChatView.swift @@ -182,7 +182,7 @@ struct DMChatView_Previews: PreviewProvider { let model = DirectMessageModel(events: [ev], our_pubkey: test_pubkey, pubkey: test_pubkey) - DMChatView(damus_state: test_damus_state(), dms: model) + DMChatView(damus_state: test_damus_state, dms: model) } } diff --git a/damus/Views/DMView.swift b/damus/Views/DMView.swift index 567b922f5d..e26ad71b91 100644 --- a/damus/Views/DMView.swift +++ b/damus/Views/DMView.swift @@ -76,6 +76,6 @@ struct DMView: View { struct DMView_Previews: PreviewProvider { static var previews: some View { let ev = NostrEvent(content: "Hey there *buddy*, want to grab some drinks later? 🍻", keypair: test_keypair, kind: 1, tags: [])! - DMView(event: ev, damus_state: test_damus_state()) + DMView(event: ev, damus_state: test_damus_state) } } diff --git a/damus/Views/DirectMessagesView.swift b/damus/Views/DirectMessagesView.swift index d6e4d0ec63..352cef7103 100644 --- a/damus/Views/DirectMessagesView.swift +++ b/damus/Views/DirectMessagesView.swift @@ -107,7 +107,7 @@ func would_filter_non_friends_from_dms(contacts: Contacts, dms: [DirectMessageMo struct DirectMessagesView_Previews: PreviewProvider { static var previews: some View { - let ds = test_damus_state() + let ds = test_damus_state DirectMessagesView(damus_state: ds, model: ds.dms, settings: ds.settings) } } diff --git a/damus/Views/EventDetailView.swift b/damus/Views/EventDetailView.swift index 3844791d02..391e64f184 100644 --- a/damus/Views/EventDetailView.swift +++ b/damus/Views/EventDetailView.swift @@ -16,7 +16,7 @@ struct EventDetailView: View { struct EventDetailView_Previews: PreviewProvider { static var previews: some View { - let _ = test_damus_state() + let _ = test_damus_state EventDetailView() } } diff --git a/damus/Views/EventView.swift b/damus/Views/EventView.swift index a3d49a7e7a..219c4cd51d 100644 --- a/damus/Views/EventView.swift +++ b/damus/Views/EventView.swift @@ -131,9 +131,9 @@ struct EventView_Previews: PreviewProvider { */ - EventView( damus: test_damus_state(), event: test_note ) + EventView( damus: test_damus_state, event: test_note ) - EventView( damus: test_damus_state(), event: test_longform_event.event, options: [.wide] ) + EventView( damus: test_damus_state, event: test_longform_event.event, options: [.wide] ) } .padding() } diff --git a/damus/Views/Events/BuilderEventView.swift b/damus/Views/Events/BuilderEventView.swift index 4e53efc7cd..1e798598a9 100644 --- a/damus/Views/Events/BuilderEventView.swift +++ b/damus/Views/Events/BuilderEventView.swift @@ -93,7 +93,7 @@ struct BuilderEventView: View { struct BuilderEventView_Previews: PreviewProvider { static var previews: some View { - BuilderEventView(damus: test_damus_state(), event_id: test_note.id) + BuilderEventView(damus: test_damus_state, event_id: test_note.id) } } diff --git a/damus/Views/Events/Components/EventTop.swift b/damus/Views/Events/Components/EventTop.swift index f2b5bad11b..bba3da04d8 100644 --- a/damus/Views/Events/Components/EventTop.swift +++ b/damus/Views/Events/Components/EventTop.swift @@ -40,6 +40,6 @@ struct EventTop: View { struct EventTop_Previews: PreviewProvider { static var previews: some View { - EventTop(state: test_damus_state(), event: test_note, pubkey: test_note.pubkey, is_anon: false) + EventTop(state: test_damus_state, event: test_note, pubkey: test_note.pubkey, is_anon: false) } } diff --git a/damus/Views/Events/Components/ReplyDescription.swift b/damus/Views/Events/Components/ReplyDescription.swift index e6afa36f74..90a64f2a80 100644 --- a/damus/Views/Events/Components/ReplyDescription.swift +++ b/damus/Views/Events/Components/ReplyDescription.swift @@ -23,7 +23,7 @@ struct ReplyDescription: View { struct ReplyDescription_Previews: PreviewProvider { static var previews: some View { - ReplyDescription(event: test_note, replying_to: test_note, ndb: test_damus_state().ndb) + ReplyDescription(event: test_note, replying_to: test_note, ndb: test_damus_state.ndb) } } diff --git a/damus/Views/Events/Components/ReplyPart.swift b/damus/Views/Events/Components/ReplyPart.swift index a97387e4da..74cf7f0ee8 100644 --- a/damus/Views/Events/Components/ReplyPart.swift +++ b/damus/Views/Events/Components/ReplyPart.swift @@ -34,6 +34,6 @@ struct ReplyPart: View { struct ReplyPart_Previews: PreviewProvider { static var previews: some View { - ReplyPart(events: test_damus_state().events, event: test_note, keypair: Keypair(pubkey: .empty, privkey: nil), ndb: test_damus_state().ndb) + ReplyPart(events: test_damus_state.events, event: test_note, keypair: Keypair(pubkey: .empty, privkey: nil), ndb: test_damus_state.ndb) } } diff --git a/damus/Views/Events/EventBody.swift b/damus/Views/Events/EventBody.swift index fe2cb72608..6cb6b105a0 100644 --- a/damus/Views/Events/EventBody.swift +++ b/damus/Views/Events/EventBody.swift @@ -43,6 +43,6 @@ struct EventBody: View { struct EventBody_Previews: PreviewProvider { static var previews: some View { - EventBody(damus_state: test_damus_state(), event: test_note, size: .normal, options: []) + EventBody(damus_state: test_damus_state, event: test_note, size: .normal, options: []) } } diff --git a/damus/Views/Events/EventProfile.swift b/damus/Views/Events/EventProfile.swift index c92bbf0795..6d8071209d 100644 --- a/damus/Views/Events/EventProfile.swift +++ b/damus/Views/Events/EventProfile.swift @@ -53,6 +53,6 @@ struct EventProfile: View { struct EventProfile_Previews: PreviewProvider { static var previews: some View { - EventProfile(damus_state: test_damus_state(), pubkey: test_note.pubkey, size: .normal) + EventProfile(damus_state: test_damus_state, pubkey: test_note.pubkey, size: .normal) } } diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift index 01f1d34556..5899b45c97 100644 --- a/damus/Views/Events/EventShell.swift +++ b/damus/Views/Events/EventShell.swift @@ -137,11 +137,11 @@ struct EventShell_Previews: PreviewProvider { static var previews: some View { VStack { - EventShell(state: test_damus_state(), event: test_note, options: [.no_action_bar]) { + EventShell(state: test_damus_state, event: test_note, options: [.no_action_bar]) { Text(verbatim: "Hello") } - EventShell(state: test_damus_state(), event: test_note, options: [.no_action_bar, .wide]) { + EventShell(state: test_damus_state, event: test_note, options: [.no_action_bar, .wide]) { Text(verbatim: "Hello") } } diff --git a/damus/Views/Events/Longform/LongformPreview.swift b/damus/Views/Events/Longform/LongformPreview.swift index 277bcd3fc7..e805d03433 100644 --- a/damus/Views/Events/Longform/LongformPreview.swift +++ b/damus/Views/Events/Longform/LongformPreview.swift @@ -87,9 +87,9 @@ struct LongformPreview: View { struct LongformPreview_Previews: PreviewProvider { static var previews: some View { VStack { - LongformPreview(state: test_damus_state(), ev: test_longform_event.event, options: []) + LongformPreview(state: test_damus_state, ev: test_longform_event.event, options: []) - LongformPreview(state: test_damus_state(), ev: test_longform_event.event, options: [.wide]) + LongformPreview(state: test_damus_state, ev: test_longform_event.event, options: [.wide]) } .frame(height: 400) } diff --git a/damus/Views/Events/Longform/LongformView.swift b/damus/Views/Events/Longform/LongformView.swift index 135953e12b..60533a3834 100644 --- a/damus/Views/Events/Longform/LongformView.swift +++ b/damus/Views/Events/Longform/LongformView.swift @@ -75,7 +75,7 @@ let test_longform_event = LongformEvent.parse(from: NostrEvent( struct LongformView_Previews: PreviewProvider { static var previews: some View { - let st = test_damus_state() + let st = test_damus_state let artifacts = render_note_content(ev: test_longform_event.event, profiles: st.profiles, keypair: Keypair(pubkey: .empty, privkey: nil)) let model = NoteArtifactsModel(state: .loaded(artifacts)) diff --git a/damus/Views/Events/MutedEventView.swift b/damus/Views/Events/MutedEventView.swift index 720a0ce832..42d7c46313 100644 --- a/damus/Views/Events/MutedEventView.swift +++ b/damus/Views/Events/MutedEventView.swift @@ -77,7 +77,7 @@ struct MutedEventView_Previews: PreviewProvider { static var previews: some View { - MutedEventView(damus_state: test_damus_state(), event: test_note, selected: false) + MutedEventView(damus_state: test_damus_state, event: test_note, selected: false) .frame(width: .infinity, height: 50) } } diff --git a/damus/Views/Events/SelectedEventView.swift b/damus/Views/Events/SelectedEventView.swift index 75ad2eb610..60e1c70a36 100644 --- a/damus/Views/Events/SelectedEventView.swift +++ b/damus/Views/Events/SelectedEventView.swift @@ -92,6 +92,6 @@ struct SelectedEventView: View { struct SelectedEventView_Previews: PreviewProvider { static var previews: some View { - SelectedEventView(damus: test_damus_state(), event: test_note, size: .selected) + SelectedEventView(damus: test_damus_state, event: test_note, size: .selected) } } diff --git a/damus/Views/Events/TextEvent.swift b/damus/Views/Events/TextEvent.swift index d37a84b34c..aeb95af49c 100644 --- a/damus/Views/Events/TextEvent.swift +++ b/damus/Views/Events/TextEvent.swift @@ -75,10 +75,10 @@ func event_is_anonymous(ev: NostrEvent) -> Bool { struct TextEvent_Previews: PreviewProvider { static var previews: some View { VStack(spacing: 20) { - TextEvent(damus: test_damus_state(), event: test_note, pubkey: test_pubkey, options: []) + TextEvent(damus: test_damus_state, event: test_note, pubkey: test_pubkey, options: []) .frame(height: 400) - TextEvent(damus: test_damus_state(), event: test_note, pubkey: test_pubkey, options: [.wide]) + TextEvent(damus: test_damus_state, event: test_note, pubkey: test_pubkey, options: [.wide]) .frame(height: 400) } } diff --git a/damus/Views/Events/ZapEvent.swift b/damus/Views/Events/ZapEvent.swift index 45fcac95da..419a3132b4 100644 --- a/damus/Views/Events/ZapEvent.swift +++ b/damus/Views/Events/ZapEvent.swift @@ -51,9 +51,9 @@ struct ZapEvent: View { struct ZapEvent_Previews: PreviewProvider { static var previews: some View { VStack { - ZapEvent(damus: test_damus_state(), zap: .zap(test_zap), is_top_zap: true) + ZapEvent(damus: test_damus_state, zap: .zap(test_zap), is_top_zap: true) - ZapEvent(damus: test_damus_state(), zap: .zap(test_private_zap), is_top_zap: false) + ZapEvent(damus: test_damus_state, zap: .zap(test_private_zap), is_top_zap: false) } } } diff --git a/damus/Views/Images/ImageContainerView.swift b/damus/Views/Images/ImageContainerView.swift index 64396ad30e..a0a740a4af 100644 --- a/damus/Views/Images/ImageContainerView.swift +++ b/damus/Views/Images/ImageContainerView.swift @@ -57,6 +57,6 @@ let test_image_url = URL(string: "https://jb55.com/red-me.jpg")! struct ImageContainerView_Previews: PreviewProvider { static var previews: some View { - ImageContainerView(video_controller: test_damus_state().video, url: .image(test_image_url), disable_animation: false) + ImageContainerView(video_controller: test_damus_state.video, url: .image(test_image_url), disable_animation: false) } } diff --git a/damus/Views/Images/ImageView.swift b/damus/Views/Images/ImageView.swift index 862666fb37..efc797d5f8 100644 --- a/damus/Views/Images/ImageView.swift +++ b/damus/Views/Images/ImageView.swift @@ -85,6 +85,6 @@ struct ImageView: View { struct ImageView_Previews: PreviewProvider { static var previews: some View { let url: MediaUrl = .image(URL(string: "https://jb55.com/red-me.jpg")!) - ImageView(video_controller: test_damus_state().video, urls: [url], disable_animation: false) + ImageView(video_controller: test_damus_state.video, urls: [url], disable_animation: false) } } diff --git a/damus/Views/Muting/MutelistView.swift b/damus/Views/Muting/MutelistView.swift index cbad407856..c914c15028 100644 --- a/damus/Views/Muting/MutelistView.swift +++ b/damus/Views/Muting/MutelistView.swift @@ -58,6 +58,6 @@ func get_mutelist_users(_ mutelist: NostrEvent?) -> Array { struct MutelistView_Previews: PreviewProvider { static var previews: some View { - MutelistView(damus_state: test_damus_state(), users: [test_note.pubkey, test_note.pubkey]) + MutelistView(damus_state: test_damus_state, users: [test_note.pubkey, test_note.pubkey]) } } diff --git a/damus/Views/NoteContentView.swift b/damus/Views/NoteContentView.swift index 0c22170350..fbc6cdad65 100644 --- a/damus/Views/NoteContentView.swift +++ b/damus/Views/NoteContentView.swift @@ -613,8 +613,8 @@ func trim_prefix(_ str: String) -> String { struct NoteContentView_Previews: PreviewProvider { static var previews: some View { - let state = test_damus_state() - let state2 = test_damus_state() + let state = test_damus_state + let state2 = test_damus_state Group { VStack { diff --git a/damus/Views/Notifications/EventGroupView.swift b/damus/Views/Notifications/EventGroupView.swift index b0f536a56a..418793a3a1 100644 --- a/damus/Views/Notifications/EventGroupView.swift +++ b/damus/Views/Notifications/EventGroupView.swift @@ -69,8 +69,9 @@ func determine_reacting_to(our_pubkey: Pubkey, ev: NostrEvent?) -> ReactingTo { } func event_author_name(profiles: Profiles, pubkey: Pubkey) -> String { - let alice_prof_txn = profiles.lookup(id: pubkey).unsafeUnownedValue - return Profile.displayName(profile: alice_prof_txn, pubkey: pubkey).username.truncate(maxLength: 50) + return profiles.lookup(id: pubkey).map({ profile in + Profile.displayName(profile: profile, pubkey: pubkey).username.truncate(maxLength: 50) + }).value } func event_group_unique_pubkeys(profiles: Profiles, group: EventGroupType) -> [Pubkey] { @@ -261,11 +262,11 @@ struct EventGroupView: View { struct EventGroupView_Previews: PreviewProvider { static var previews: some View { VStack { - EventGroupView(state: test_damus_state(), event: test_note, group: .repost(test_event_group)) + EventGroupView(state: test_damus_state, event: test_note, group: .repost(test_event_group)) .frame(height: 200) .padding() - EventGroupView(state: test_damus_state(), event: test_note, group: .reaction(test_event_group)) + EventGroupView(state: test_damus_state, event: test_note, group: .reaction(test_event_group)) .frame(height: 200) .padding() } diff --git a/damus/Views/Notifications/NotificationItemView.swift b/damus/Views/Notifications/NotificationItemView.swift index 8f06161f4e..7a52578ab1 100644 --- a/damus/Views/Notifications/NotificationItemView.swift +++ b/damus/Views/Notifications/NotificationItemView.swift @@ -88,6 +88,6 @@ let test_notification_item: NotificationItem = .repost(test_note.id, test_event_ struct NotificationItemView_Previews: PreviewProvider { static var previews: some View { - NotificationItemView(state: test_damus_state(), item: test_notification_item) + NotificationItemView(state: test_damus_state, item: test_notification_item) } } diff --git a/damus/Views/Notifications/NotificationsView.swift b/damus/Views/Notifications/NotificationsView.swift index d33a5290c1..9328ba95f8 100644 --- a/damus/Views/Notifications/NotificationsView.swift +++ b/damus/Views/Notifications/NotificationsView.swift @@ -195,7 +195,7 @@ struct NotificationsView: View { struct NotificationsView_Previews: PreviewProvider { static var previews: some View { - NotificationsView(state: test_damus_state(), notifications: NotificationsModel(), filter: NotificationFilter()) + NotificationsView(state: test_damus_state, notifications: NotificationsModel(), filter: NotificationFilter()) } } diff --git a/damus/Views/Notifications/ProfilePicturesView.swift b/damus/Views/Notifications/ProfilePicturesView.swift index 5a760bcbf9..1c274349ad 100644 --- a/damus/Views/Notifications/ProfilePicturesView.swift +++ b/damus/Views/Notifications/ProfilePicturesView.swift @@ -26,6 +26,6 @@ struct ProfilePicturesView: View { struct ProfilePicturesView_Previews: PreviewProvider { static var previews: some View { let pubkey = test_note.pubkey - ProfilePicturesView(state: test_damus_state(), pubkeys: [pubkey, pubkey]) + ProfilePicturesView(state: test_damus_state, pubkeys: [pubkey, pubkey]) } } diff --git a/damus/Views/Onboarding/SuggestedUserView.swift b/damus/Views/Onboarding/SuggestedUserView.swift index 10c89fc712..b8de6623b6 100644 --- a/damus/Views/Onboarding/SuggestedUserView.swift +++ b/damus/Views/Onboarding/SuggestedUserView.swift @@ -61,7 +61,7 @@ struct SuggestedUserView_Previews: PreviewProvider { let user = SuggestedUser(name: "klabo", about: "name", picture: "about", pubkey: test_pubkey)! List { - SuggestedUserView(user: user, damus_state: test_damus_state()) + SuggestedUserView(user: user, damus_state: test_damus_state) } } } diff --git a/damus/Views/Onboarding/SuggestedUsersView.swift b/damus/Views/Onboarding/SuggestedUsersView.swift index 4c7d6d83de..4ce268ccf5 100644 --- a/damus/Views/Onboarding/SuggestedUsersView.swift +++ b/damus/Views/Onboarding/SuggestedUsersView.swift @@ -72,6 +72,6 @@ struct SuggestedUsersSectionHeader: View { struct SuggestedUsersView_Previews: PreviewProvider { static var previews: some View { - SuggestedUsersView(model: SuggestedUsersViewModel(damus_state: test_damus_state())) + SuggestedUsersView(model: SuggestedUsersViewModel(damus_state: test_damus_state)) } } diff --git a/damus/Views/PostView.swift b/damus/Views/PostView.swift index 1066bec87c..e71612dff3 100644 --- a/damus/Views/PostView.swift +++ b/damus/Views/PostView.swift @@ -511,7 +511,7 @@ func get_searching_string(_ word: String?) -> String? { struct PostView_Previews: PreviewProvider { static var previews: some View { - PostView(action: .posting(.none), damus_state: test_damus_state()) + PostView(action: .posting(.none), damus_state: test_damus_state) } } diff --git a/damus/Views/Posting/UserSearch.swift b/damus/Views/Posting/UserSearch.swift index 66d2be41bd..de05a897c5 100644 --- a/damus/Views/Posting/UserSearch.swift +++ b/damus/Views/Posting/UserSearch.swift @@ -72,7 +72,7 @@ struct UserSearch_Previews: PreviewProvider { @State static var newCursorIndex: Int? static var previews: some View { - UserSearch(damus_state: test_damus_state(), search: search, focusWordAttributes: $word, newCursorIndex: $newCursorIndex, post: $post) + UserSearch(damus_state: test_damus_state, search: search, focusWordAttributes: $word, newCursorIndex: $newCursorIndex, post: $post) } } diff --git a/damus/Views/Profile/CondensedProfilePicturesView.swift b/damus/Views/Profile/CondensedProfilePicturesView.swift index 982e217318..ed5de758a9 100644 --- a/damus/Views/Profile/CondensedProfilePicturesView.swift +++ b/damus/Views/Profile/CondensedProfilePicturesView.swift @@ -33,6 +33,6 @@ struct CondensedProfilePicturesView: View { struct CondensedProfilePicturesView_Previews: PreviewProvider { static var previews: some View { - CondensedProfilePicturesView(state: test_damus_state(), pubkeys: [test_pubkey, test_pubkey, test_pubkey, test_pubkey], maxPictures: 3) + CondensedProfilePicturesView(state: test_damus_state, pubkeys: [test_pubkey, test_pubkey, test_pubkey, test_pubkey], maxPictures: 3) } } diff --git a/damus/Views/Profile/EditMetadataView.swift b/damus/Views/Profile/EditMetadataView.swift index ceceb1c4f9..07e3e5ce55 100644 --- a/damus/Views/Profile/EditMetadataView.swift +++ b/damus/Views/Profile/EditMetadataView.swift @@ -206,7 +206,7 @@ struct EditMetadataView: View { struct EditMetadataView_Previews: PreviewProvider { static var previews: some View { - EditMetadataView(damus_state: test_damus_state()) + EditMetadataView(damus_state: test_damus_state) } } diff --git a/damus/Views/Profile/EventProfileName.swift b/damus/Views/Profile/EventProfileName.swift index a1298f394b..bde6c2a206 100644 --- a/damus/Views/Profile/EventProfileName.swift +++ b/damus/Views/Profile/EventProfileName.swift @@ -123,6 +123,6 @@ struct EventProfileName: View { struct EventProfileName_Previews: PreviewProvider { static var previews: some View { - EventProfileName(pubkey: test_note.pubkey, damus: test_damus_state()) + EventProfileName(pubkey: test_note.pubkey, damus: test_damus_state) } } diff --git a/damus/Views/Profile/MaybeAnonPfpView.swift b/damus/Views/Profile/MaybeAnonPfpView.swift index 717069fdb7..0b4256e4d5 100644 --- a/damus/Views/Profile/MaybeAnonPfpView.swift +++ b/damus/Views/Profile/MaybeAnonPfpView.swift @@ -39,6 +39,6 @@ struct MaybeAnonPfpView: View { struct MaybeAnonPfpView_Previews: PreviewProvider { static var previews: some View { - MaybeAnonPfpView(state: test_damus_state(), is_anon: true, pubkey: ANON_PUBKEY, size: PFP_SIZE) + MaybeAnonPfpView(state: test_damus_state, is_anon: true, pubkey: ANON_PUBKEY, size: PFP_SIZE) } } diff --git a/damus/Views/Profile/ProfileEditButton.swift b/damus/Views/Profile/ProfileEditButton.swift index 19e454d9f5..6fdd29cfe4 100644 --- a/damus/Views/Profile/ProfileEditButton.swift +++ b/damus/Views/Profile/ProfileEditButton.swift @@ -42,7 +42,7 @@ struct ProfileEditButton: View { struct ProfileEditButton_Previews: PreviewProvider { static var previews: some View { Group { - ProfileEditButton(damus_state: test_damus_state()) + ProfileEditButton(damus_state: test_damus_state) } } } diff --git a/damus/Views/Profile/ProfileName.swift b/damus/Views/Profile/ProfileName.swift index 41d6a8f0c8..a351e36e7f 100644 --- a/damus/Views/Profile/ProfileName.swift +++ b/damus/Views/Profile/ProfileName.swift @@ -140,6 +140,6 @@ struct ProfileName: View { struct ProfileName_Previews: PreviewProvider { static var previews: some View { - ProfileName(pubkey: test_damus_state().pubkey, damus: test_damus_state()) + ProfileName(pubkey: test_damus_state.pubkey, damus: test_damus_state) } } diff --git a/damus/Views/Profile/ProfileNameView.swift b/damus/Views/Profile/ProfileNameView.swift index 9d6efd0ea8..eb1821d76c 100644 --- a/damus/Views/Profile/ProfileNameView.swift +++ b/damus/Views/Profile/ProfileNameView.swift @@ -126,9 +126,9 @@ struct ProfileNameView: View { struct ProfileNameView_Previews: PreviewProvider { static var previews: some View { VStack { - ProfileNameView(pubkey: test_note.pubkey, damus: test_damus_state()) + ProfileNameView(pubkey: test_note.pubkey, damus: test_damus_state) - ProfileNameView(pubkey: test_note.pubkey, damus: test_damus_state()) + ProfileNameView(pubkey: test_note.pubkey, damus: test_damus_state) } } } diff --git a/damus/Views/Profile/ProfilePicView.swift b/damus/Views/Profile/ProfilePicView.swift index 00184a565f..3d806d783f 100644 --- a/damus/Views/Profile/ProfilePicView.swift +++ b/damus/Views/Profile/ProfilePicView.swift @@ -114,7 +114,7 @@ func get_profile_url(picture: String?, pubkey: Pubkey, profiles: Profiles) -> UR func make_preview_profiles(_ pubkey: Pubkey) -> Profiles { let user_search_cache = UserSearchCache() - let profiles = Profiles(user_search_cache: user_search_cache, ndb: .empty) + let profiles = Profiles(user_search_cache: user_search_cache, ndb: test_damus_state.ndb) let picture = "http://cdn.jb55.com/img/red-me.jpg" let profile = Profile(name: "jb55", display_name: "William Casarin", about: "It's me", picture: picture, banner: "", website: "https://jb55.com", lud06: nil, lud16: nil, nip05: "jb55.com", damus_donation: nil) //let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) diff --git a/damus/Views/Profile/ProfileView.swift b/damus/Views/Profile/ProfileView.swift index 2a1339ac6d..1d767bed31 100644 --- a/damus/Views/Profile/ProfileView.swift +++ b/damus/Views/Profile/ProfileView.swift @@ -505,7 +505,7 @@ struct ProfileView: View { struct ProfileView_Previews: PreviewProvider { static var previews: some View { - let ds = test_damus_state() + let ds = test_damus_state ProfileView(damus_state: ds, pubkey: ds.pubkey) } } diff --git a/damus/Views/QRCodeView.swift b/damus/Views/QRCodeView.swift index 54410bcfa7..c8aaee9c69 100644 --- a/damus/Views/QRCodeView.swift +++ b/damus/Views/QRCodeView.swift @@ -295,7 +295,7 @@ struct QRCodeView: View { struct QRCodeView_Previews: PreviewProvider { static var previews: some View { - QRCodeView(damus_state: test_damus_state(), pubkey: test_note.pubkey) + QRCodeView(damus_state: test_damus_state, pubkey: test_note.pubkey) } } diff --git a/damus/Views/Reactions/ReactionView.swift b/damus/Views/Reactions/ReactionView.swift index 1c66631f93..4aa1152c05 100644 --- a/damus/Views/Reactions/ReactionView.swift +++ b/damus/Views/Reactions/ReactionView.swift @@ -28,6 +28,6 @@ struct ReactionView: View { struct ReactionView_Previews: PreviewProvider { static var previews: some View { - ReactionView(damus_state: test_damus_state(), reaction: NostrEvent(content: "🤙🏼", keypair: test_keypair)!) + ReactionView(damus_state: test_damus_state, reaction: NostrEvent(content: "🤙🏼", keypair: test_keypair)!) } } diff --git a/damus/Views/ReactionsView.swift b/damus/Views/ReactionsView.swift index 2af1aa089e..5d1e02f4a9 100644 --- a/damus/Views/ReactionsView.swift +++ b/damus/Views/ReactionsView.swift @@ -37,7 +37,7 @@ struct ReactionsView: View { struct ReactionsView_Previews: PreviewProvider { static var previews: some View { - let state = test_damus_state() + let state = test_damus_state ReactionsView(damus_state: state, model: ReactionsModel(state: state, target: test_note.id)) } } diff --git a/damus/Views/RelayFilterView.swift b/damus/Views/RelayFilterView.swift index 8b11162b93..02c2236370 100644 --- a/damus/Views/RelayFilterView.swift +++ b/damus/Views/RelayFilterView.swift @@ -36,6 +36,6 @@ struct RelayFilterView: View { struct RelayFilterView_Previews: PreviewProvider { static var previews: some View { - RelayFilterView(state: test_damus_state(), timeline: .search) + RelayFilterView(state: test_damus_state, timeline: .search) } } diff --git a/damus/Views/Relays/RecommendedRelayView.swift b/damus/Views/Relays/RecommendedRelayView.swift index 6e66a39d20..b5265ce281 100644 --- a/damus/Views/Relays/RecommendedRelayView.swift +++ b/damus/Views/Relays/RecommendedRelayView.swift @@ -128,6 +128,6 @@ struct RecommendedRelayView: View { struct RecommendedRelayView_Previews: PreviewProvider { static var previews: some View { - RecommendedRelayView(damus: test_damus_state(), relay: "wss://relay.damus.io", user_recommended: true) + RecommendedRelayView(damus: test_damus_state, relay: "wss://relay.damus.io", user_recommended: true) } } diff --git a/damus/Views/Relays/RelayConfigView.swift b/damus/Views/Relays/RelayConfigView.swift index 3f692f9aa1..bfa407a1b2 100644 --- a/damus/Views/Relays/RelayConfigView.swift +++ b/damus/Views/Relays/RelayConfigView.swift @@ -125,6 +125,6 @@ struct RelayConfigView: View { struct RelayConfigView_Previews: PreviewProvider { static var previews: some View { - RelayConfigView(state: test_damus_state()) + RelayConfigView(state: test_damus_state) } } diff --git a/damus/Views/Relays/RelayDetailView.swift b/damus/Views/Relays/RelayDetailView.swift index 3662222fcb..0b3023f18b 100644 --- a/damus/Views/Relays/RelayDetailView.swift +++ b/damus/Views/Relays/RelayDetailView.swift @@ -174,6 +174,6 @@ struct RelayDetailView: View { struct RelayDetailView_Previews: PreviewProvider { static var previews: some View { let metadata = RelayMetadata(name: "name", description: "desc", pubkey: test_pubkey, contact: "contact", supported_nips: [1,2,3], software: "software", version: "version", limitation: Limitations.empty, payments_url: "https://jb55.com", icon: "") - RelayDetailView(state: test_damus_state(), relay: "relay", nip11: metadata) + RelayDetailView(state: test_damus_state, relay: "relay", nip11: metadata) } } diff --git a/damus/Views/Relays/RelayStatusView.swift b/damus/Views/Relays/RelayStatusView.swift index d8c6476f88..88eda533f9 100644 --- a/damus/Views/Relays/RelayStatusView.swift +++ b/damus/Views/Relays/RelayStatusView.swift @@ -57,7 +57,7 @@ struct RelayStatusView: View { struct RelayStatusView_Previews: PreviewProvider { static var previews: some View { - let connection = test_damus_state().pool.get_relay("wss://relay.damus.io")!.connection + let connection = test_damus_state.pool.get_relay("wss://relay.damus.io")!.connection RelayStatusView(connection: connection) } } diff --git a/damus/Views/Relays/RelayToggle.swift b/damus/Views/Relays/RelayToggle.swift index a70d50ae75..22113496f4 100644 --- a/damus/Views/Relays/RelayToggle.swift +++ b/damus/Views/Relays/RelayToggle.swift @@ -42,7 +42,7 @@ struct RelayToggle: View { struct RelayToggle_Previews: PreviewProvider { static var previews: some View { - RelayToggle(state: test_damus_state(), timeline: .search, relay_id: "wss://jb55.com") + RelayToggle(state: test_damus_state, timeline: .search, relay_id: "wss://jb55.com") .padding() } } diff --git a/damus/Views/Relays/RelayView.swift b/damus/Views/Relays/RelayView.swift index 0b5ea52219..4a097bc3fc 100644 --- a/damus/Views/Relays/RelayView.swift +++ b/damus/Views/Relays/RelayView.swift @@ -117,6 +117,6 @@ struct RelayView: View { struct RelayView_Previews: PreviewProvider { static var previews: some View { - RelayView(state: test_damus_state(), relay: "wss://relay.damus.io", showActionButtons: .constant(false)) + RelayView(state: test_damus_state, relay: "wss://relay.damus.io", showActionButtons: .constant(false)) } } diff --git a/damus/Views/Relays/SignalView.swift b/damus/Views/Relays/SignalView.swift index b0f1c67bda..43a92784c5 100644 --- a/damus/Views/Relays/SignalView.swift +++ b/damus/Views/Relays/SignalView.swift @@ -29,6 +29,6 @@ struct SignalView: View { struct SignalView_Previews: PreviewProvider { static var previews: some View { - SignalView(state: test_damus_state(), signal: SignalModel(signal: 5, max_signal: 10)) + SignalView(state: test_damus_state, signal: SignalModel(signal: 5, max_signal: 10)) } } diff --git a/damus/Views/ReplyView.swift b/damus/Views/ReplyView.swift index 188993bdd4..2704216609 100644 --- a/damus/Views/ReplyView.swift +++ b/damus/Views/ReplyView.swift @@ -93,13 +93,13 @@ struct ReplyView_Previews: PreviewProvider { static var previews: some View { VStack { ReplyView(replying_to: test_note, - damus: test_damus_state(), + damus: test_damus_state, original_pubkeys: [], filtered_pubkeys: .constant([])) .frame(height: 300) ReplyView(replying_to: test_longform_event.event, - damus: test_damus_state(), + damus: test_damus_state, original_pubkeys: [], filtered_pubkeys: .constant([])) .frame(height: 300) diff --git a/damus/Views/ReportView.swift b/damus/Views/ReportView.swift index 1f7e990bfc..49a970ac5f 100644 --- a/damus/Views/ReportView.swift +++ b/damus/Views/ReportView.swift @@ -129,7 +129,7 @@ struct ReportView: View { struct ReportView_Previews: PreviewProvider { static var previews: some View { - let ds = test_damus_state() + let ds = test_damus_state VStack { ReportView(postbox: ds.postbox, target: ReportTarget.user(test_pubkey), keypair: test_keypair.to_full()!) diff --git a/damus/Views/Reposts/RepostView.swift b/damus/Views/Reposts/RepostView.swift index 11e7d8a732..6bdecc6870 100644 --- a/damus/Views/Reposts/RepostView.swift +++ b/damus/Views/Reposts/RepostView.swift @@ -18,7 +18,7 @@ struct RepostView: View { struct RepostView_Previews: PreviewProvider { static var previews: some View { - RepostView(damus_state: test_damus_state(), repost: NostrEvent(content: "", keypair: test_keypair)!) + RepostView(damus_state: test_damus_state, repost: NostrEvent(content: "", keypair: test_keypair)!) } } diff --git a/damus/Views/Reposts/RepostedEvent.swift b/damus/Views/Reposts/RepostedEvent.swift index 7dda0c3436..3ac8b4f093 100644 --- a/damus/Views/Reposts/RepostedEvent.swift +++ b/damus/Views/Reposts/RepostedEvent.swift @@ -29,6 +29,6 @@ struct RepostedEvent: View { struct RepostedEvent_Previews: PreviewProvider { static var previews: some View { - RepostedEvent(damus: test_damus_state(), event: test_note, inner_ev: test_note, options: []) + RepostedEvent(damus: test_damus_state, event: test_note, inner_ev: test_note, options: []) } } diff --git a/damus/Views/RepostsView.swift b/damus/Views/RepostsView.swift index 185ffba676..210cd70324 100644 --- a/damus/Views/RepostsView.swift +++ b/damus/Views/RepostsView.swift @@ -32,7 +32,7 @@ struct RepostsView: View { struct RepostsView_Previews: PreviewProvider { static var previews: some View { - let state = test_damus_state() + let state = test_damus_state RepostsView(damus_state: state, model: RepostsModel(state: state, target: test_note.id)) } } diff --git a/damus/Views/Search/SearchingEventView.swift b/damus/Views/Search/SearchingEventView.swift index 9b79eec44c..204d8d0005 100644 --- a/damus/Views/Search/SearchingEventView.swift +++ b/damus/Views/Search/SearchingEventView.swift @@ -126,7 +126,7 @@ struct SearchingEventView: View { struct SearchingEventView_Previews: PreviewProvider { static var previews: some View { - let state = test_damus_state() + let state = test_damus_state SearchingEventView(state: state, search_type: .event(test_note.id)) } } diff --git a/damus/Views/SearchHomeView.swift b/damus/Views/SearchHomeView.swift index 2eed1664de..e27f9d74c3 100644 --- a/damus/Views/SearchHomeView.swift +++ b/damus/Views/SearchHomeView.swift @@ -134,7 +134,7 @@ struct SearchHomeView: View { struct SearchHomeView_Previews: PreviewProvider { static var previews: some View { - let state = test_damus_state() + let state = test_damus_state SearchHomeView(damus_state: state, model: SearchHomeModel(damus_state: state)) } } diff --git a/damus/Views/SearchView.swift b/damus/Views/SearchView.swift index 5f2308c8b3..d8baf2187d 100644 --- a/damus/Views/SearchView.swift +++ b/damus/Views/SearchView.swift @@ -79,7 +79,7 @@ func describe_search(_ filter: NostrFilter) -> DescribedSearch { struct SearchView_Previews: PreviewProvider { static var previews: some View { - let test_state = test_damus_state() + let test_state = test_damus_state let filter = NostrFilter(hashtag: ["bitcoin"]) let model = SearchModel(state: test_state, search: filter) diff --git a/damus/Views/Settings/AppearanceSettingsView.swift b/damus/Views/Settings/AppearanceSettingsView.swift index c1062678d7..9e0c2d0595 100644 --- a/damus/Views/Settings/AppearanceSettingsView.swift +++ b/damus/Views/Settings/AppearanceSettingsView.swift @@ -107,6 +107,6 @@ struct AppearanceSettingsView: View { struct TextFormattingSettings_Previews: PreviewProvider { static var previews: some View { - AppearanceSettingsView(damus_state: test_damus_state(), settings: UserSettingsStore()) + AppearanceSettingsView(damus_state: test_damus_state, settings: UserSettingsStore()) } } diff --git a/damus/Views/SideMenuView.swift b/damus/Views/SideMenuView.swift index 7b9e53d424..429834ddef 100644 --- a/damus/Views/SideMenuView.swift +++ b/damus/Views/SideMenuView.swift @@ -233,7 +233,7 @@ struct SideMenuView: View { struct Previews_SideMenuView_Previews: PreviewProvider { static var previews: some View { - let ds = test_damus_state() + let ds = test_damus_state SideMenuView(damus_state: ds, isSidebarVisible: .constant(true)) } } diff --git a/damus/Views/ThreadView.swift b/damus/Views/ThreadView.swift index 1db5e31dbe..2e10501c73 100644 --- a/damus/Views/ThreadView.swift +++ b/damus/Views/ThreadView.swift @@ -103,7 +103,7 @@ struct ThreadView: View { struct ThreadView_Previews: PreviewProvider { static var previews: some View { - let state = test_damus_state() + let state = test_damus_state let thread = ThreadModel(event: test_note, damus_state: state) ThreadView(state: state, thread: thread) } diff --git a/damus/Views/Timeline/InnerTimelineView.swift b/damus/Views/Timeline/InnerTimelineView.swift index 98a986a874..4212f3dc68 100644 --- a/damus/Views/Timeline/InnerTimelineView.swift +++ b/damus/Views/Timeline/InnerTimelineView.swift @@ -69,7 +69,7 @@ struct InnerTimelineView: View { struct InnerTimelineView_Previews: PreviewProvider { static var previews: some View { - InnerTimelineView(events: test_event_holder, damus: test_damus_state(), filter: { _ in true }) + InnerTimelineView(events: test_event_holder, damus: test_damus_state, filter: { _ in true }) .frame(width: 300, height: 500) .border(Color.red) } diff --git a/damus/Views/UserRelaysView.swift b/damus/Views/UserRelaysView.swift index 83a5cf1711..336bb4d1ec 100644 --- a/damus/Views/UserRelaysView.swift +++ b/damus/Views/UserRelaysView.swift @@ -40,6 +40,6 @@ struct UserRelaysView: View { struct UserRelaysView_Previews: PreviewProvider { static var previews: some View { - UserRelaysView(state: test_damus_state(), relays: []) + UserRelaysView(state: test_damus_state, relays: []) } } diff --git a/damus/Views/Wallet/WalletView.swift b/damus/Views/Wallet/WalletView.swift index e00a2f367d..4eaf644962 100644 --- a/damus/Views/Wallet/WalletView.swift +++ b/damus/Views/Wallet/WalletView.swift @@ -194,10 +194,10 @@ struct WalletView: View { } } -let test_wallet_connect_url = WalletConnectURL(pubkey: test_pubkey, relay: .init("wss://relay.damus.io")!, keypair: test_damus_state().keypair.to_full()!, lud16: "jb55@sendsats.com") +let test_wallet_connect_url = WalletConnectURL(pubkey: test_pubkey, relay: .init("wss://relay.damus.io")!, keypair: test_damus_state.keypair.to_full()!, lud16: "jb55@sendsats.com") struct WalletView_Previews: PreviewProvider { - static let tds = test_damus_state() + static let tds = test_damus_state static var previews: some View { WalletView(damus_state: tds, model: WalletModel(state: .existing(test_wallet_connect_url), settings: tds.settings)) } diff --git a/damus/Views/Zaps/CustomizeZapView.swift b/damus/Views/Zaps/CustomizeZapView.swift index e57dea1b22..fb4cef3999 100644 --- a/damus/Views/Zaps/CustomizeZapView.swift +++ b/damus/Views/Zaps/CustomizeZapView.swift @@ -302,7 +302,7 @@ extension View { struct CustomizeZapView_Previews: PreviewProvider { static var previews: some View { - CustomizeZapView(state: test_damus_state(), target: ZapTarget.note(id: test_note.id, author: test_note.pubkey), lnurl: "") + CustomizeZapView(state: test_damus_state, target: ZapTarget.note(id: test_note.id, author: test_note.pubkey), lnurl: "") .frame(width: 400, height: 600) } } diff --git a/damus/Views/Zaps/ZapTypePicker.swift b/damus/Views/Zaps/ZapTypePicker.swift index 1a128b71ad..0369901916 100644 --- a/damus/Views/Zaps/ZapTypePicker.swift +++ b/damus/Views/Zaps/ZapTypePicker.swift @@ -105,7 +105,7 @@ struct ZapTypePicker: View { struct ZapTypePicker_Previews: PreviewProvider { @State static var zap_type: ZapType = .pub static var previews: some View { - let ds = test_damus_state() + let ds = test_damus_state ZapTypePicker(zap_type: $zap_type, settings: ds.settings, profiles: ds.profiles, pubkey: test_pubkey) } } diff --git a/damus/Views/Zaps/ZapUserView.swift b/damus/Views/Zaps/ZapUserView.swift index 465e95b991..25368531a4 100644 --- a/damus/Views/Zaps/ZapUserView.swift +++ b/damus/Views/Zaps/ZapUserView.swift @@ -23,6 +23,6 @@ struct ZapUserView: View { struct ZapUserView_Previews: PreviewProvider { static var previews: some View { - ZapUserView(state: test_damus_state(), pubkey: ANON_PUBKEY) + ZapUserView(state: test_damus_state, pubkey: ANON_PUBKEY) } } diff --git a/damus/Views/Zaps/ZapsView.swift b/damus/Views/Zaps/ZapsView.swift index fc9f8b2dda..52056a0a95 100644 --- a/damus/Views/Zaps/ZapsView.swift +++ b/damus/Views/Zaps/ZapsView.swift @@ -40,6 +40,6 @@ struct ZapsView: View { struct ZapsView_Previews: PreviewProvider { static var previews: some View { - ZapsView(state: test_damus_state(), target: .profile(test_pubkey)) + ZapsView(state: test_damus_state, target: .profile(test_pubkey)) } } diff --git a/damusTests/EventGroupViewTests.swift b/damusTests/EventGroupViewTests.swift index 1e74a5f182..799b400ea5 100644 --- a/damusTests/EventGroupViewTests.swift +++ b/damusTests/EventGroupViewTests.swift @@ -19,14 +19,14 @@ final class EventGroupViewTests: XCTestCase { } func testEventAuthorName() { - let damusState = test_damus_state() + let damusState = test_damus_state XCTAssertEqual(event_author_name(profiles: damusState.profiles, pubkey: test_pubkey), "damus") XCTAssertEqual(event_author_name(profiles: damusState.profiles, pubkey: test_pubkey_2), "1rppft3m:4qxhsgnj") XCTAssertEqual(event_author_name(profiles: damusState.profiles, pubkey: ANON_PUBKEY), "Anonymous") } func testEventGroupUniquePubkeys() { - let damusState = test_damus_state() + let damusState = test_damus_state let encodedPost = "{\"id\": \"8ba545ab96959fe0ce7db31bc10f3ac3aa5353bc4428dbf1e56a7be7062516db\",\"pubkey\": \"7e27509ccf1e297e1df164912a43406218f8bd80129424c3ef798ca3ef5c8444\",\"created_at\": 1677013417,\"kind\": 1,\"tags\": [],\"content\": \"hello\",\"sig\": \"93684f15eddf11f42afbdd81828ee9fc35350344d8650c78909099d776e9ad8d959cd5c4bff7045be3b0b255144add43d0feef97940794a1bc9c309791bebe4a\"}" @@ -49,7 +49,7 @@ final class EventGroupViewTests: XCTestCase { func testReactingToText() throws { let enUsLocale = Locale(identifier: "en-US") - let damusState = test_damus_state() + let damusState = test_damus_state let encodedPost = "{\"id\": \"8ba545ab96959fe0ce7db31bc10f3ac3aa5353bc4428dbf1e56a7be7062516db\",\"pubkey\": \"7e27509ccf1e297e1df164912a43406218f8bd80129424c3ef798ca3ef5c8444\",\"created_at\": 1677013417,\"kind\": 1,\"tags\": [],\"content\": \"hello\",\"sig\": \"93684f15eddf11f42afbdd81828ee9fc35350344d8650c78909099d776e9ad8d959cd5c4bff7045be3b0b255144add43d0feef97940794a1bc9c309791bebe4a\"}" diff --git a/damusTests/LongPostTests.swift b/damusTests/LongPostTests.swift index cad2a83cbb..f4888c9fbd 100644 --- a/damusTests/LongPostTests.swift +++ b/damusTests/LongPostTests.swift @@ -34,7 +34,7 @@ final class LongPostTests: XCTestCase { XCTAssertEqual(subid, "subid") XCTAssertTrue(ev.should_show_event) XCTAssertTrue(!ev.too_big) - XCTAssertTrue(should_show_event(keypair: test_keypair, hellthreads: test_damus_state().muted_threads, contacts: contacts, ev: ev)) + XCTAssertTrue(should_show_event(keypair: test_keypair, hellthreads: test_damus_state.muted_threads, contacts: contacts, ev: ev)) XCTAssertTrue(validate_event(ev: ev) == .ok ) } diff --git a/damusTests/NoteContentViewTests.swift b/damusTests/NoteContentViewTests.swift index a376b7c75c..e9a210d981 100644 --- a/damusTests/NoteContentViewTests.swift +++ b/damusTests/NoteContentViewTests.swift @@ -14,7 +14,7 @@ class NoteContentViewTests: XCTestCase { let note = NostrEvent(content: content, keypair: test_keypair, tags: [["t", "かっこいい"]])! let parsed: Blocks = parse_note_content(content: .init(note: note, keypair: test_keypair)) - let testState = test_damus_state() + let testState = test_damus_state let text: NoteArtifactsSeparated = render_blocks(blocks: parsed, profiles: testState.profiles) let attributedText: AttributedString = text.content.attributed diff --git a/damusTests/ProfileViewTests.swift b/damusTests/ProfileViewTests.swift index 5ef187bb37..ca3c93ddcb 100644 --- a/damusTests/ProfileViewTests.swift +++ b/damusTests/ProfileViewTests.swift @@ -21,24 +21,25 @@ final class ProfileViewTests: XCTestCase { } func testFollowedByString() throws { - let profiles = test_damus_state().profiles - let pk1 = test_pubkey let pk2 = test_pubkey_2 let pk3 = Pubkey(hex: "b42e44b555013239a0d5dcdb09ebde0857cd8a5a57efbba5a2b6ac78833cb9f0")! let pk4 = Pubkey(hex: "cc590e46363d0fa66bb27081368d01f169b8ffc7c614629d4e9eef6c88b38670")! let pk5 = Pubkey(hex: "f2aa579bb998627e04a8f553842a09446360c9d708c6141dd119c479f6ab9d29")! - XCTAssertEqual(followedByString([pk1], profiles: profiles, locale: enUsLocale), "Followed by damus") - XCTAssertEqual(followedByString([pk1, pk2], profiles: profiles, locale: enUsLocale), "Followed by damus & 1rppft3m:4qxhsgnj") - XCTAssertEqual(followedByString([pk1, pk2, pk3], profiles: profiles, locale: enUsLocale), "Followed by damus, 1rppft3m:4qxhsgnj & 1kshyfd2:cq04aze0") - XCTAssertEqual(followedByString([pk1, pk2, pk3, pk4,], profiles: profiles, locale: enUsLocale), "Followed by damus, 1rppft3m:4qxhsgnj, 1kshyfd2:cq04aze0 & 1 other") - XCTAssertEqual(followedByString([pk1, pk2, pk3, pk4, pk5], profiles: profiles, locale: enUsLocale), "Followed by damus, 1rppft3m:4qxhsgnj, 1kshyfd2:cq04aze0 & 2 others") + let ndb = Ndb(path: Ndb.db_path)! + let txn = NdbTxn(ndb: ndb) + + XCTAssertEqual(followedByString(txn: txn, [pk1], ndb: ndb, locale: enUsLocale), "Followed by damus") + XCTAssertEqual(followedByString(txn: txn, [pk1, pk2], ndb: ndb, locale: enUsLocale), "Followed by damus & 1rppft3m:4qxhsgnj") + XCTAssertEqual(followedByString(txn: txn, [pk1, pk2, pk3], ndb: ndb, locale: enUsLocale), "Followed by damus, 1rppft3m:4qxhsgnj & 1kshyfd2:cq04aze0") + XCTAssertEqual(followedByString(txn: txn, [pk1, pk2, pk3, pk4,], ndb: ndb, locale: enUsLocale), "Followed by damus, 1rppft3m:4qxhsgnj, 1kshyfd2:cq04aze0 & 1 other") + XCTAssertEqual(followedByString(txn: txn, [pk1, pk2, pk3, pk4, pk5], ndb: ndb, locale: enUsLocale), "Followed by damus, 1rppft3m:4qxhsgnj, 1kshyfd2:cq04aze0 & 2 others") let pubkeys = [pk1, pk2, pk3, pk4, pk5, pk1, pk2, pk3, pk4, pk5] Bundle.main.localizations.map { Locale(identifier: $0) }.forEach { for count in 1...10 { - XCTAssertNoThrow(followedByString(pubkeys.prefix(count).map { $0 }, profiles: profiles, locale: $0)) + XCTAssertNoThrow(followedByString(txn: txn, pubkeys.prefix(count).map { $0 }, ndb: ndb, locale: $0)) } } } diff --git a/damusTests/UserSearchCacheTests.swift b/damusTests/UserSearchCacheTests.swift index 8a90128d32..5d653ef0a7 100644 --- a/damusTests/UserSearchCacheTests.swift +++ b/damusTests/UserSearchCacheTests.swift @@ -11,7 +11,7 @@ import XCTest final class UserSearchCacheTests: XCTestCase { var keypair: FullKeypair? = nil - let damusState = DamusState.empty + let damusState = test_damus_state let nip05 = "_@somedomain.com" @MainActor @@ -55,11 +55,6 @@ final class UserSearchCacheTests: XCTestCase { damusState.profiles.set_validated(keypair.pubkey, nip05: NIP05.parse(newNip05)) - let newProfile = Profile(name: "whoami", display_name: "T-DAWG", about: nil, picture: nil, banner: nil, website: nil, lud06: nil, lud16: nil, nip05: newNip05, damus_donation: nil) - - // Lookup to synchronize access on profiles dictionary to avoid race conditions. - let _ = damusState.profiles.lookup(id: keypair.pubkey) - // Old profile attributes are removed from cache. XCTAssertEqual(damusState.user_search_cache.search(key: "tyiu"), []) XCTAssertEqual(damusState.user_search_cache.search(key: "ty"), []) diff --git a/damusTests/ZapTests.swift b/damusTests/ZapTests.swift index 66975db80e..46352dc885 100644 --- a/damusTests/ZapTests.swift +++ b/damusTests/ZapTests.swift @@ -69,7 +69,7 @@ final class ZapTests: XCTestCase { XCTAssertEqual(zap.target, ZapTarget.profile(profile)) XCTAssertEqual(zap_notification_title(zap), "Zap") - XCTAssertEqual(zap_notification_body(profiles: Profiles(user_search_cache: UserSearchCache(), ndb: .empty), zap: zap), "You received 1k sats from 107jk7ht:2quqncxg") + XCTAssertEqual(zap_notification_body(profiles: Profiles(user_search_cache: UserSearchCache(), ndb: test_damus_state.ndb), zap: zap), "You received 1k sats from 107jk7ht:2quqncxg") } } diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index e1a86e51e8..f7c677c891 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -11,14 +11,15 @@ class Ndb { let ndb: ndb_t static var db_path: String { - (FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: ""))! + let path = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString + return remove_file_prefix(path!) } static var empty: Ndb { Ndb(ndb: ndb_t(ndb: nil)) } - init?() { + init?(path: String? = nil) { //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb") //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb") @@ -27,7 +28,9 @@ class Ndb { let ingest_threads: Int32 = 4 var mapsize: Int = 1024 * 1024 * 1024 * 32 - let ok = Ndb.db_path.withCString { testdir in + let path = path.map(remove_file_prefix) ?? Ndb.db_path + + let ok = path.withCString { testdir in var ok = false while !ok && mapsize > 1024 * 1024 * 700 { ok = ndb_init(&ndb_p, testdir, mapsize, ingest_threads) != 0 @@ -199,3 +202,8 @@ func getDebugCheckedRoot(byteBuffer: inout ByteBuffer) thro return getRoot(byteBuffer: &byteBuffer) } #endif + +func remove_file_prefix(_ str: String) -> String { + return str.replacingOccurrences(of: "file://", with: "") +} + diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift index d7867cafe4..3090eacf97 100644 --- a/nostrdb/Test/NdbTests.swift +++ b/nostrdb/Test/NdbTests.swift @@ -8,12 +8,26 @@ import XCTest @testable import damus +func test_ndb_dir() -> String? { + do { + let fileManager = FileManager.default + let tempDir = fileManager.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try fileManager.createDirectory(at: tempDir, withIntermediateDirectories: true, attributes: nil) + return remove_file_prefix(tempDir.absoluteString) + } catch { + return nil + } +} + final class NdbTests: XCTestCase { + var db_dir: String = "" override func setUpWithError() throws { - // Put setup code here. This method is called before the invocation of each test method in the class. - try? FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb") - try? FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb") + guard let db = test_ndb_dir() else { + XCTFail("Could not create temp directory") + return + } + db_dir = db } override func tearDownWithError() throws { @@ -41,26 +55,27 @@ final class NdbTests: XCTestCase { func test_ndb_init() { do { - let ndb = Ndb()! + let ndb = Ndb(path: db_dir)! let ok = ndb.process_events(test_wire_events) XCTAssertTrue(ok) } do { - let ndb = Ndb()! + let ndb = Ndb(path: db_dir)! let id = NoteId(hex: "d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349")! - let note = ndb.lookup_note(id) + let txn = NdbTxn(ndb: ndb) + let note = ndb.lookup_note_with_txn(id: id, txn: txn) XCTAssertNotNil(note) guard let note else { return } let pk = Pubkey(hex: "32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245")! XCTAssertEqual(note.pubkey, pk) - let profile = ndb.lookup_profile(pk) + let profile = ndb.lookup_profile_with_txn(pk, txn: txn) XCTAssertNotNil(profile) guard let profile else { return } XCTAssertEqual(profile.profile?.name, "jb55") - XCTAssertEqual(profile.lnurl, "fixme") + XCTAssertEqual(profile.lnurl, nil) } From fafe3b4b3e4ed513d772c08fea3f744e3c6a422c Mon Sep 17 00:00:00 2001 From: William Casarin Date: Wed, 13 Sep 2023 05:40:44 -0700 Subject: [PATCH 110/111] ndb: add nostrdb migrations --- nostrdb/Ndb.swift | 2 +- nostrdb/nostrdb.c | 163 ++++++++++++++++++++++++++++++++++++++++++++-- nostrdb/nostrdb.h | 5 +- 3 files changed, 164 insertions(+), 6 deletions(-) diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index f7c677c891..e64ad62ba4 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -33,7 +33,7 @@ class Ndb { let ok = path.withCString { testdir in var ok = false while !ok && mapsize > 1024 * 1024 * 700 { - ok = ndb_init(&ndb_p, testdir, mapsize, ingest_threads) != 0 + ok = ndb_init(&ndb_p, testdir, mapsize, ingest_threads, 0) != 0 if !ok { mapsize /= 2 } diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index 7e89582848..b3f8457131 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -27,6 +27,7 @@ static const int THREAD_QUEUE_BATCH = 4096; // the maximum size of inbox queues static const int DEFAULT_QUEUE_SIZE = 1000000; +#define ndb_flag_set(flags, f) ((flags & f) == f) #define NDB_PARSED_ID (1 << 0) #define NDB_PARSED_PUBKEY (1 << 1) @@ -37,6 +38,12 @@ static const int DEFAULT_QUEUE_SIZE = 1000000; #define NDB_PARSED_TAGS (1 << 6) #define NDB_PARSED_ALL (NDB_PARSED_ID|NDB_PARSED_PUBKEY|NDB_PARSED_SIG|NDB_PARSED_CREATED_AT|NDB_PARSED_KIND|NDB_PARSED_CONTENT|NDB_PARSED_TAGS) +typedef int (*ndb_migrate_fn)(struct ndb *); + +struct ndb_migration { + ndb_migrate_fn fn; +}; + struct ndb_profile_record_builder { flatcc_builder_t *builder; void *flatbuf; @@ -61,9 +68,15 @@ enum ndb_dbs { NDB_DB_PROFILE, NDB_DB_NOTE_ID, NDB_DB_PROFILE_PK, + NDB_DB_NDB_META, NDB_DBS, }; +// keys used for storing data in the NDB metadata database (NDB_DB_NDB_META) +enum ndb_meta_key { + NDB_META_KEY_VERSION = 1 +}; + struct ndb_json_parser { const char *json; int json_len; @@ -100,6 +113,7 @@ struct ndb { struct ndb_lmdb lmdb; struct ndb_ingester ingester; struct ndb_writer writer; + int version; // lmdb environ handles, etc }; @@ -109,6 +123,49 @@ struct ndb_tsid { uint64_t timestamp; }; +// Migrations +// + +static int ndb_migrate_user_search_indices(struct ndb *ndb) +{ + return 1; +} + +static struct ndb_migration MIGRATIONS[] = { + //{ .fn = ndb_migrate_user_search_indices } +}; + + +int ndb_db_version(struct ndb *ndb) +{ + int rc; + uint64_t version, version_key; + MDB_val k, v; + MDB_txn *txn; + + version_key = NDB_META_KEY_VERSION; + k.mv_data = &version_key; + k.mv_size = sizeof(version_key); + + if ((rc = mdb_txn_begin(ndb->lmdb.env, NULL, 0, &txn))) { + fprintf(stderr, "ndb_db_version: mdb_txn_begin failed, error %d\n", rc); + return -1; + } + + if (mdb_get(txn, ndb->lmdb.dbs[NDB_DB_NDB_META], &k, &v)) { + version = -1; + } else { + if (v.mv_size != 8) { + fprintf(stderr, "run_migrations: invalid version size?"); + return 0; + } + version = *((uint64_t*)v.mv_data); + } + + mdb_txn_abort(txn); + return version; +} + /** From LMDB: Compare two items lexically */ static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) { int diff; @@ -175,6 +232,7 @@ enum ndb_writer_msgtype { NDB_WRITER_QUIT, // kill thread immediately NDB_WRITER_NOTE, // write a note to the db NDB_WRITER_PROFILE, // write a profile to the db + NDB_WRITER_DBMETA, // write ndb metadata }; struct ndb_ingester_event { @@ -199,11 +257,17 @@ struct ndb_ingester_msg { }; }; +struct ndb_writer_ndb_meta { + // these are 64 bit because I'm paranoid of db-wide alignment issues + uint64_t version; +}; + struct ndb_writer_msg { enum ndb_writer_msgtype type; union { struct ndb_writer_note note; struct ndb_writer_profile profile; + struct ndb_writer_ndb_meta ndb_meta; }; }; @@ -236,6 +300,12 @@ int ndb_note_verify(void *ctx, unsigned char pubkey[32], unsigned char id[32], return 1; } +static inline int ndb_writer_queue_msg(struct ndb_writer *writer, + struct ndb_writer_msg *msg) +{ + return prot_queue_push(&writer->inbox, msg); +} + static inline int ndb_writer_queue_msgs(struct ndb_writer *writer, struct ndb_writer_msg *msgs, int num_msgs) @@ -714,6 +784,27 @@ static uint64_t ndb_write_note(struct ndb_lmdb *lmdb, MDB_txn *txn, return note_key; } +// only to be called from the writer thread +static void ndb_write_version(struct ndb_lmdb *lmdb, MDB_txn *txn, uint64_t version) +{ + int rc; + MDB_val key, val; + uint64_t version_key; + + version_key = NDB_META_KEY_VERSION; + + key.mv_data = &version_key; + key.mv_size = sizeof(version_key); + val.mv_data = &version; + val.mv_size = sizeof(version); + + if ((rc = mdb_put(txn, lmdb->dbs[NDB_DB_NDB_META], &key, &val, 0))) { + ndb_debug("write version to ndb_meta failed: %s\n", + mdb_strerror(rc)); + return; + } +} + static void *ndb_writer_thread(void *data) { struct ndb_writer *writer = data; @@ -734,6 +825,7 @@ static void *ndb_writer_thread(void *data) switch (msg->type) { case NDB_WRITER_NOTE: any_note = 1; break; case NDB_WRITER_PROFILE: any_note = 1; break; + case NDB_WRITER_DBMETA: any_note = 1; break; case NDB_WRITER_QUIT: break; } } @@ -767,6 +859,9 @@ static void *ndb_writer_thread(void *data) case NDB_WRITER_NOTE: ndb_write_note(writer->lmdb, txn, &msg->note); break; + case NDB_WRITER_DBMETA: + ndb_write_version(writer->lmdb, txn, msg->ndb_meta.version); + break; } } @@ -962,7 +1057,7 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map } if ((rc = mdb_env_set_maxdbs(lmdb->env, NDB_DBS))) { - fprintf(stderr, "mdb_env_set_mapsize failed, error %d\n", rc); + fprintf(stderr, "mdb_env_set_maxdbs failed, error %d\n", rc); return 0; } @@ -995,6 +1090,12 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map return 0; } + // ndb metadata (db version, etc) + if ((rc = mdb_dbi_open(txn, "ndb_meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NDB_META]))) { + fprintf(stderr, "mdb_dbi_open ndb_meta failed, error %d\n", rc); + return 0; + } + // id+ts index flags unsigned int tsid_flags = MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED; @@ -1021,7 +1122,56 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map return 1; } -int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingester_threads) +static int ndb_queue_write_version(struct ndb *ndb, uint64_t version) +{ + struct ndb_writer_msg msg; + msg.type = NDB_WRITER_DBMETA; + msg.ndb_meta.version = version; + return ndb_writer_queue_msg(&ndb->writer, &msg); +} + +static int ndb_run_migrations(struct ndb *ndb) +{ + uint64_t version, latest_version, i; + + latest_version = sizeof(MIGRATIONS) / sizeof(MIGRATIONS[0]); + + if ((version = ndb_db_version(ndb)) == -1) { + version = latest_version; + + // no version found. fresh db? + if (!ndb_queue_write_version(ndb, version)) { + fprintf(stderr, "run_migrations: failed writing db version"); + return 0; + } + + return 1; + } + + if (version < latest_version) + fprintf(stderr, "nostrdb: migrating v%d -> v%d\n", + (int)version, (int)latest_version); + + for (i = version; i < latest_version; i++) { + if (!MIGRATIONS[i].fn(ndb)) { + fprintf(stderr, "run_migrations: migration v%d -> v%d failed\n", (int)i, (int)(i+1)); + return 0; + } + + if (!ndb_queue_write_version(ndb, i+1)) { + fprintf(stderr, "run_migrations: failed writing db version"); + return 0; + } + + version = i+1; + } + + ndb->version = version; + + return 1; +} + +int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingester_threads, int flags) { struct ndb *ndb; //MDB_dbi ind_id; // TODO: ind_pk, etc @@ -1036,16 +1186,21 @@ int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingest return 0; if (!ndb_writer_init(&ndb->writer, &ndb->lmdb)) { - fprintf(stderr, "ndb_writer_init failed"); + fprintf(stderr, "ndb_writer_init failed\n"); return 0; } if (!ndb_ingester_init(&ndb->ingester, &ndb->writer, ingester_threads)) { - fprintf(stderr, "failed to initialize %d ingester thread(s)", + fprintf(stderr, "failed to initialize %d ingester thread(s)\n", ingester_threads); return 0; } + if (!ndb_flag_set(flags, NDB_FLAG_NOMIGRATE) && !ndb_run_migrations(ndb)) { + fprintf(stderr, "failed to run migrations\n"); + return 0; + } + // Initialize LMDB environment and spin up threads return 1; } diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index 7ee2c2c51a..1ae76b6c79 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -7,6 +7,8 @@ #define NDB_PACKED_STR 0x1 #define NDB_PACKED_ID 0x2 +#define NDB_FLAG_NOMIGRATE (1 << 0) + //#define DEBUG 1 #ifdef DEBUG @@ -158,7 +160,8 @@ int ndb_decode_key(const char *secstr, struct ndb_keypair *keypair); int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[32], unsigned char signature[64]); // NDB -int ndb_init(struct ndb **ndb, const char *dbdir, size_t mapsize, int ingester_threads); +int ndb_init(struct ndb **ndb, const char *dbdir, size_t mapsize, int ingester_threads, int flags); +int ndb_db_version(struct ndb *ndb); int ndb_process_event(struct ndb *, const char *json, int len); int ndb_process_events(struct ndb *, const char *ldjson, size_t len); int ndb_begin_query(struct ndb *, struct ndb_txn *); From 7a85ae29ca27e763b18b1b8653c4a70cf91cb913 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Thu, 21 Sep 2023 13:19:22 -0400 Subject: [PATCH 111/111] search: switch to nostrdb profile searching Changelog-Changed: Switch to nostrdb for @'s and user search --- damus/ContentView.swift | 4 +- damus/Models/DamusState.swift | 5 +- damus/Models/HomeModel.swift | 2 - damus/Nostr/Profiles.swift | 9 +- damus/TestData.swift | 4 +- damus/Views/Posting/UserSearch.swift | 3 +- damus/Views/Profile/ProfilePicView.swift | 3 +- damus/Views/SearchResultsView.swift | 16 +- nostrdb/Ndb.swift | 26 +++ nostrdb/nostrdb.c | 270 ++++++++++++++++++++++- nostrdb/nostrdb.h | 16 ++ 11 files changed, 325 insertions(+), 33 deletions(-) diff --git a/damus/ContentView.swift b/damus/ContentView.swift index ee1edb1476..b360d0b93d 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -629,7 +629,7 @@ struct ContentView: View { likes: EventCounter(our_pubkey: pubkey), boosts: EventCounter(our_pubkey: pubkey), contacts: Contacts(our_pubkey: pubkey), - profiles: Profiles(user_search_cache: user_search_cache, ndb: ndb), + profiles: Profiles(ndb: ndb), dms: home.dms, previews: PreviewCache(), zaps: Zaps(our_pubkey: pubkey), @@ -646,7 +646,6 @@ struct ContentView: View { muted_threads: MutedThreadsManager(keypair: keypair), wallet: WalletModel(settings: settings), nav: self.navigationCoordinator, - user_search_cache: user_search_cache, music: MusicController(onChange: music_changed), video: VideoController(), ndb: ndb @@ -919,7 +918,6 @@ func handle_unfollow(state: DamusState, unfollow: FollowRef) -> Bool { switch unfollow { case .pubkey(let pk): state.contacts.remove_friend(pk) - state.user_search_cache.updateOwnContactsPetnames(id: state.pubkey, oldEvent: old_contacts, newEvent: ev) case .hashtag: // nothing to handle here really break diff --git a/damus/Models/DamusState.swift b/damus/Models/DamusState.swift index d8f5418b5d..108ee3f5e2 100644 --- a/damus/Models/DamusState.swift +++ b/damus/Models/DamusState.swift @@ -31,7 +31,6 @@ struct DamusState { let muted_threads: MutedThreadsManager let wallet: WalletModel let nav: NavigationCoordinator - let user_search_cache: UserSearchCache let music: MusicController? let video: VideoController let ndb: Ndb @@ -62,7 +61,6 @@ struct DamusState { } static var empty: DamusState { - let user_search_cache = UserSearchCache() let empty_pub: Pubkey = .empty let empty_sec: Privkey = .empty let kp = Keypair(pubkey: empty_pub, privkey: nil) @@ -73,7 +71,7 @@ struct DamusState { likes: EventCounter(our_pubkey: empty_pub), boosts: EventCounter(our_pubkey: empty_pub), contacts: Contacts(our_pubkey: empty_pub), - profiles: Profiles(user_search_cache: user_search_cache, ndb: .empty), + profiles: Profiles(ndb: .empty), dms: DirectMessagesModel(our_pubkey: empty_pub), previews: PreviewCache(), zaps: Zaps(our_pubkey: empty_pub), @@ -90,7 +88,6 @@ struct DamusState { muted_threads: MutedThreadsManager(keypair: kp), wallet: WalletModel(settings: UserSettingsStore()), nav: NavigationCoordinator(), - user_search_cache: user_search_cache, music: nil, video: VideoController(), ndb: .empty diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index d2c2859690..73afc11a52 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -734,8 +734,6 @@ func load_our_contacts(state: DamusState, m_old_ev: NostrEvent?, ev: NostrEvent) } } } - - state.user_search_cache.updateOwnContactsPetnames(id: contacts.our_pubkey, oldEvent: m_old_ev, newEvent: ev) } diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index adb0a61e3b..10dec04cbb 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -38,10 +38,7 @@ class Profiles { @MainActor var nip05_pubkey: [String: Pubkey] = [:] - let user_search_cache: UserSearchCache - - init(user_search_cache: UserSearchCache, ndb: Ndb) { - self.user_search_cache = user_search_cache + init(ndb: Ndb) { self.ndb = ndb } @@ -84,6 +81,10 @@ class Profiles { return ndb.lookup_profile_by_key(key: key) } + func search(_ query: String, limit: Int, txn: NdbTxn) -> [Pubkey] { + return ndb.search_profile(query, limit: limit, txn: txn) + } + func lookup(id: Pubkey) -> NdbTxn { return ndb.lookup_profile(id).map({ pr in pr?.profile }) } diff --git a/damus/TestData.swift b/damus/TestData.swift index b152e8d4d9..729b40c824 100644 --- a/damus/TestData.swift +++ b/damus/TestData.swift @@ -66,7 +66,6 @@ var test_damus_state: DamusState = ({ print("opening \(tempDir!)") let ndb = Ndb(path: tempDir)! let our_pubkey = test_pubkey - let user_search_cache = UserSearchCache() let pool = RelayPool(ndb: ndb) let settings = UserSettingsStore() let damus = DamusState(pool: pool, @@ -74,7 +73,7 @@ var test_damus_state: DamusState = ({ likes: .init(our_pubkey: our_pubkey), boosts: .init(our_pubkey: our_pubkey), contacts: .init(our_pubkey: our_pubkey), - profiles: .init(user_search_cache: user_search_cache, ndb: ndb), + profiles: .init(ndb: ndb), dms: .init(our_pubkey: our_pubkey), previews: .init(), zaps: .init(our_pubkey: our_pubkey), @@ -91,7 +90,6 @@ var test_damus_state: DamusState = ({ muted_threads: .init(keypair: test_keypair), wallet: .init(settings: settings), nav: .init(), - user_search_cache: user_search_cache, music: .init(onChange: {_ in }), video: .init(), ndb: ndb) diff --git a/damus/Views/Posting/UserSearch.swift b/damus/Views/Posting/UserSearch.swift index de05a897c5..a593902ecd 100644 --- a/damus/Views/Posting/UserSearch.swift +++ b/damus/Views/Posting/UserSearch.swift @@ -17,7 +17,8 @@ struct UserSearch: View { @EnvironmentObject var tagModel: TagModel var users: [Pubkey] { - return search_profiles(profiles: damus_state.profiles, search: search) + let txn = NdbTxn(ndb: damus_state.ndb) + return search_profiles(profiles: damus_state.profiles, search: search, txn: txn) } func on_user_tapped(pk: Pubkey) { diff --git a/damus/Views/Profile/ProfilePicView.swift b/damus/Views/Profile/ProfilePicView.swift index 3d806d783f..54afbbc532 100644 --- a/damus/Views/Profile/ProfilePicView.swift +++ b/damus/Views/Profile/ProfilePicView.swift @@ -113,8 +113,7 @@ func get_profile_url(picture: String?, pubkey: Pubkey, profiles: Profiles) -> UR } func make_preview_profiles(_ pubkey: Pubkey) -> Profiles { - let user_search_cache = UserSearchCache() - let profiles = Profiles(user_search_cache: user_search_cache, ndb: test_damus_state.ndb) + let profiles = Profiles(ndb: test_damus_state.ndb) let picture = "http://cdn.jb55.com/img/red-me.jpg" let profile = Profile(name: "jb55", display_name: "William Casarin", about: "It's me", picture: picture, banner: "", website: "https://jb55.com", lud06: nil, lud16: nil, nip05: "jb55.com", damus_donation: nil) //let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) diff --git a/damus/Views/SearchResultsView.swift b/damus/Views/SearchResultsView.swift index 85db46a2bd..b2748880cc 100644 --- a/damus/Views/SearchResultsView.swift +++ b/damus/Views/SearchResultsView.swift @@ -108,10 +108,12 @@ struct SearchResultsView: View { } .frame(maxHeight: .infinity) .onAppear { - self.result = search_for_string(profiles: damus_state.profiles, search) + let txn = NdbTxn.init(ndb: damus_state.ndb) + self.result = search_for_string(profiles: damus_state.profiles, search: search, txn: txn) } .onChange(of: search) { new in - self.result = search_for_string(profiles: damus_state.profiles, new) + let txn = NdbTxn.init(ndb: damus_state.ndb) + self.result = search_for_string(profiles: damus_state.profiles, search: search, txn: txn) } } } @@ -125,7 +127,7 @@ struct SearchResultsView_Previews: PreviewProvider { */ -func search_for_string(profiles: Profiles, _ new: String) -> Search? { +func search_for_string(profiles: Profiles, search new: String, txn: NdbTxn) -> Search? { guard new.count != 0 else { return nil } @@ -154,7 +156,7 @@ func search_for_string(profiles: Profiles, _ new: String) -> Search? { return .note(NoteId(decoded.data)) } - let multisearch = MultiSearch(hashtag: make_hashtagable(new), profiles: search_profiles(profiles: profiles, search: new)) + let multisearch = MultiSearch(hashtag: make_hashtagable(new), profiles: search_profiles(profiles: profiles, search: new, txn: txn)) return .multi(multisearch) } @@ -171,7 +173,7 @@ func make_hashtagable(_ str: String) -> String { return String(new.filter{$0 != " "}) } -func search_profiles(profiles: Profiles, search: String) -> [Pubkey] { +func search_profiles(profiles: Profiles, search: String, txn: NdbTxn) -> [Pubkey] { // Search by hex pubkey. if let pubkey = hex_decode_pubkey(search), profiles.lookup_key_by_pubkey(pubkey) != nil @@ -189,5 +191,7 @@ func search_profiles(profiles: Profiles, search: String) -> [Pubkey] { } let new = search.lowercased() - return profiles.user_search_cache.search(key: new) + + return profiles.search(search, limit: 10, txn: txn) } + diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index e64ad62ba4..395ae33231 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -188,6 +188,32 @@ class Ndb { } } + func search_profile(_ search: String, limit: Int, txn: NdbTxn) -> [Pubkey] { + var pks = Array() + + return search.withCString { q in + var s = ndb_search() + guard ndb_search_profile(&txn.txn, &s, q) != 0 else { + return pks + } + + defer { ndb_search_profile_end(&s) } + pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32))) + + var n = limit + while n > 0 { + guard ndb_search_profile_next(&s) != 0 else { + return pks + } + pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32))) + + n -= 1 + } + + return pks + } + } + deinit { ndb_destroy(ndb.ndb) } diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index b3f8457131..8f7bbc84fe 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -69,6 +69,7 @@ enum ndb_dbs { NDB_DB_NOTE_ID, NDB_DB_PROFILE_PK, NDB_DB_NDB_META, + NDB_DB_PROFILE_SEARCH, NDB_DBS, }; @@ -123,16 +124,149 @@ struct ndb_tsid { uint64_t timestamp; }; +static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id, + uint64_t timestamp, const char *search) +{ + memcpy(key->id, id, 32); + key->timestamp = timestamp; + strncpy(key->search, search, sizeof(key->search) - 1); + key->search[sizeof(key->search) - 1] = '\0'; +} + +static int ndb_write_profile_search_index(struct ndb_lmdb *lmdb, + MDB_txn *txn, + struct ndb_search_key *index_key, + uint64_t profile_key) +{ + int rc; + MDB_val key, val; + + key.mv_data = index_key; + key.mv_size = sizeof(*index_key); + val.mv_data = &profile_key; + val.mv_size = sizeof(profile_key); + + if ((rc = mdb_put(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], &key, &val, 0))) { + ndb_debug("ndb_write_profile_search_index failed: %s\n", + mdb_strerror(rc)); + return 0; + } + + return 1; +} + + +// map usernames and display names to profile keys for user searching +static int ndb_write_profile_search_indices(struct ndb_lmdb *lmdb, + MDB_txn *txn, + struct ndb_note *note, + uint64_t profile_key, + void *profile_root) +{ + struct ndb_search_key index; + NdbProfileRecord_table_t profile_record; + NdbProfile_table_t profile; + + profile_record = NdbProfileRecord_as_root(profile_root); + profile = NdbProfileRecord_profile_get(profile_record); + + const char *name = NdbProfile_name_get(profile); + const char *display_name = NdbProfile_display_name_get(profile); + + // words + pubkey + created + if (name) { + ndb_make_search_key(&index, note->pubkey, note->created_at, + name); + if (!ndb_write_profile_search_index(lmdb, txn, &index, + profile_key)) + return 0; + } + + if (display_name) { + // don't write the same name/display_name twice + if (name && !strcmp(display_name, name)) { + return 1; + } + ndb_make_search_key(&index, note->pubkey, note->created_at, + display_name); + if (!ndb_write_profile_search_index(lmdb, txn, &index, + profile_key)) + return 0; + } + + return 1; +} + +int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn) +{ + txn->ndb = ndb; + MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn; + return mdb_txn_begin(ndb->lmdb.env, NULL, 0, mdb_txn) == 0; +} + + // Migrations // static int ndb_migrate_user_search_indices(struct ndb *ndb) { + int rc; + MDB_cursor *cur; + MDB_val k, v; + void *profile_root; + NdbProfileRecord_table_t record; + struct ndb_txn txn; + struct ndb_note *note; + uint64_t note_key, profile_key; + size_t len; + int count; + + if (!ndb_begin_query(ndb, &txn)) { + fprintf(stderr, "ndb_migrate_user_search_indices: ndb_begin_query failed\n"); + return 0; + } + + if ((rc = mdb_cursor_open(txn.mdb_txn, ndb->lmdb.dbs[NDB_DB_PROFILE], &cur))) { + fprintf(stderr, "ndb_migrate_user_search_indices: mdb_cursor_open failed, error %d\n", rc); + return 0; + } + + count = 0; + + // loop through all profiles and write search indices + while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { + profile_root = v.mv_data; + profile_key = *((uint64_t*)k.mv_data); + record = NdbProfileRecord_as_root(profile_root); + note_key = NdbProfileRecord_note_key(record); + note = ndb_get_note_by_key(&txn, note_key, &len); + + if (note == NULL) { + fprintf(stderr, "ndb_migrate_user_search_indices: note lookup failed\n"); + return 0; + } + + if (!ndb_write_profile_search_indices(&ndb->lmdb, txn.mdb_txn, + note, profile_key, + profile_root)) { + + fprintf(stderr, "ndb_migrate_user_search_indices: ndb_write_profile_search_indices failed\n"); + return 0; + } + + count++; + } + + fprintf(stderr, "migrated %d profiles to include search indices\n", count); + + mdb_cursor_close(cur); + mdb_txn_commit(txn.mdb_txn); + return 1; } static struct ndb_migration MIGRATIONS[] = { - //{ .fn = ndb_migrate_user_search_indices } + { .fn = ndb_migrate_user_search_indices } }; @@ -187,6 +321,7 @@ static int ndb_tsid_compare(const MDB_val *a, const MDB_val *b) { struct ndb_tsid *tsa, *tsb; MDB_val a2 = *a, b2 = *b; + a2.mv_size = sizeof(tsa->id); b2.mv_size = sizeof(tsb->id); @@ -271,13 +406,6 @@ struct ndb_writer_msg { }; }; -int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn) -{ - txn->ndb = ndb; - MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn; - return mdb_txn_begin(ndb->lmdb.env, NULL, 0, mdb_txn) == 0; -} - void ndb_end_query(struct ndb_txn *txn) { mdb_txn_abort(txn->mdb_txn); @@ -674,6 +802,113 @@ static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db) return *((uint64_t*)key.mv_data); } +// make a search key meant for user queries without any other note info +static void ndb_make_search_key_low(struct ndb_search_key *key, const char *search) +{ + memset(key->id, 0, sizeof(key->id)); + key->timestamp = 0; + strncpy(key->search, search, sizeof(key->search) - 1); + key->search[sizeof(key->search) - 1] = '\0'; +} + +int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query) +{ + int rc; + struct ndb_search_key s; + MDB_val k, v; + search->cursor = NULL; + + MDB_cursor **cursor = (MDB_cursor **)&search->cursor; + + ndb_make_search_key_low(&s, query); + + k.mv_data = &s; + k.mv_size = sizeof(s); + + if ((rc = mdb_cursor_open(txn->mdb_txn, + txn->ndb->lmdb.dbs[NDB_DB_PROFILE_SEARCH], + cursor))) { + printf("search_profile: cursor opened failed: %s\n", + mdb_strerror(rc)); + return 0; + } + + // Position cursor at the next key greater than or equal to the specified key + if (mdb_cursor_get(search->cursor, &k, &v, MDB_SET_RANGE)) { + printf("search_profile: cursor get failed\n"); + goto cleanup; + } else { + search->key = k.mv_data; + assert(v.mv_size == 8); + search->profile_key = *((uint64_t*)v.mv_data); + return 1; + } + +cleanup: + mdb_cursor_close(search->cursor); + search->cursor = NULL; + return 0; +} + +void ndb_search_profile_end(struct ndb_search *search) +{ + if (search->cursor) + mdb_cursor_close(search->cursor); +} + +int ndb_search_profile_next(struct ndb_search *search) +{ + int rc; + MDB_val k, v; + unsigned char *init_id; + + init_id = search->key->id; + k.mv_data = search->key; + k.mv_size = sizeof(*search->key); + +retry: + if ((rc = mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT))) { + ndb_debug("ndb_search_profile_next: %s\n", + mdb_strerror(rc)); + return 0; + } else { + search->key = k.mv_data; + assert(v.mv_size == 8); + search->profile_key = *((uint64_t*)v.mv_data); + + // skip duplicate pubkeys + if (!memcmp(init_id, search->key->id, 32)) + goto retry; + } + + return 1; +} + +static int ndb_search_key_cmp(const MDB_val *a, const MDB_val *b) +{ + int cmp; + struct ndb_search_key *ska, *skb; + + ska = a->mv_data; + skb = b->mv_data; + + MDB_val a2 = *a; + MDB_val b2 = *b; + + a2.mv_data = ska->search; + a2.mv_size = sizeof(ska->search) + sizeof(ska->id); + + cmp = mdb_cmp_memn(&a2, &b2); + if (cmp) return cmp; + + if (ska->timestamp < skb->timestamp) + return -1; + else if (ska->timestamp > skb->timestamp) + return 1; + + return 0; +} + static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, struct ndb_writer_profile *profile, uint64_t note_key) @@ -737,6 +972,13 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, return 0; } + // write name, display_name profile search indices + if (!ndb_write_profile_search_indices(lmdb, txn, note, profile_key, + flatbuf)) { + ndb_debug("failed to write profile search indices\n"); + return 0; + } + return 1; } @@ -803,6 +1045,8 @@ static void ndb_write_version(struct ndb_lmdb *lmdb, MDB_txn *txn, uint64_t vers mdb_strerror(rc)); return; } + + fprintf(stderr, "writing version %" PRIu64 "\n", version); } static void *ndb_writer_thread(void *data) @@ -1090,6 +1334,13 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map return 0; } + // profile search db + if ((rc = mdb_dbi_open(txn, "profile_search", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_SEARCH]))) { + fprintf(stderr, "mdb_dbi_open profile_search failed, error %d\n", rc); + return 0; + } + mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], ndb_search_key_cmp); + // ndb metadata (db version, etc) if ((rc = mdb_dbi_open(txn, "ndb_meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NDB_META]))) { fprintf(stderr, "mdb_dbi_open ndb_meta failed, error %d\n", rc); @@ -1137,6 +1388,7 @@ static int ndb_run_migrations(struct ndb *ndb) latest_version = sizeof(MIGRATIONS) / sizeof(MIGRATIONS[0]); if ((version = ndb_db_version(ndb)) == -1) { + fprintf(stderr, "run_migrations: no version found, assuming new db\n"); version = latest_version; // no version found. fresh db? @@ -1146,6 +1398,8 @@ static int ndb_run_migrations(struct ndb *ndb) } return 1; + } else { + fprintf(stderr, "ndb: version %" PRIu64 " found\n", version); } if (version < latest_version) diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index 1ae76b6c79..646df21361 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -25,6 +25,19 @@ struct ndb_t { struct ndb *ndb; }; +struct ndb_search_key +{ + char search[24]; + unsigned char id[32]; + uint64_t timestamp; +}; + +struct ndb_search { + struct ndb_search_key *key; + uint64_t profile_key; + void *cursor; // MDB_cursor * +}; + // required to keep a read struct ndb_txn { struct ndb *ndb; @@ -165,6 +178,9 @@ int ndb_db_version(struct ndb *ndb); int ndb_process_event(struct ndb *, const char *json, int len); int ndb_process_events(struct ndb *, const char *ldjson, size_t len); int ndb_begin_query(struct ndb *, struct ndb_txn *); +int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query); +int ndb_search_profile_next(struct ndb_search *search); +void ndb_search_profile_end(struct ndb_search *search); void ndb_end_query(struct ndb_txn *); void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey); void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len);