fix: Store directories in the database (#16)

This change also includes a drive-by performance fix to improve the
performance when performing the initial indexing of large directories.

Folders.xcodeproj/project.pbxproj

@@ -13,6 +13,7 @@
 		D82B95842B23DC2B00C8B6FB /* FSEventsWrapper in Frameworks */ = {isa = PBXBuildFile; productRef = D82B95832B23DC2B00C8B6FB /* FSEventsWrapper */; };
 		D83312E32B76FE4E00B994B3 /* Filter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D83312E22B76FE4E00B994B3 /* Filter.swift */; };
 		D83312E72B7700DA00B994B3 /* UTType.swift in Sources */ = {isa = PBXBuildFile; fileRef = D83312E62B7700DA00B994B3 /* UTType.swift */; };
+		D83312E92B77132B00B994B3 /* Details.swift in Sources */ = {isa = PBXBuildFile; fileRef = D83312E82B77132B00B994B3 /* Details.swift */; };
 		D8472A642B2517A50070DB64 /* ShortcutItemView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8472A632B2517A50070DB64 /* ShortcutItemView.swift */; };
 		D8472A662B2517DE0070DB64 /* DirectoryWatcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8472A652B2517DE0070DB64 /* DirectoryWatcher.swift */; };
 		D8472A682B2518320070DB64 /* FixedItemSizeCollectionViewLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8472A672B2518320070DB64 /* FixedItemSizeCollectionViewLayout.swift */; };
@@ -62,6 +63,7 @@
 		D82B95802B231AD000C8B6FB /* Store.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Store.swift; sourceTree = "<group>"; };
 		D83312E22B76FE4E00B994B3 /* Filter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Filter.swift; sourceTree = "<group>"; };
 		D83312E62B7700DA00B994B3 /* UTType.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UTType.swift; sourceTree = "<group>"; };
+		D83312E82B77132B00B994B3 /* Details.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Details.swift; sourceTree = "<group>"; };
 		D8472A632B2517A50070DB64 /* ShortcutItemView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShortcutItemView.swift; sourceTree = "<group>"; };
 		D8472A652B2517DE0070DB64 /* DirectoryWatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectoryWatcher.swift; sourceTree = "<group>"; };
 		D8472A672B2518320070DB64 /* FixedItemSizeCollectionViewLayout.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FixedItemSizeCollectionViewLayout.swift; sourceTree = "<group>"; };
@@ -224,6 +226,7 @@
 			isa = PBXGroup;
 			children = (
 				D89622DA2ACD394A006F7D2E /* ApplicationModel.swift */,
+				D83312E82B77132B00B994B3 /* Details.swift */,
 				D8DDBCC32B2A0F8E003EAF4E /* PreviewItem.swift */,
 				D89622E52ACD42F2006F7D2E /* SceneModel.swift */,
 				D89622DD2ACD3B82006F7D2E /* Settings.swift */,
@@ -386,6 +389,7 @@
 				D83312E72B7700DA00B994B3 /* UTType.swift in Sources */,
 				D8472A682B2518320070DB64 /* FixedItemSizeCollectionViewLayout.swift in Sources */,
 				D8472A6E2B2518D00070DB64 /* InteractiveCollectionView.swift in Sources */,
+				D83312E92B77132B00B994B3 /* Details.swift in Sources */,
 				D83312E32B76FE4E00B994B3 /* Filter.swift in Sources */,
 				D879E3452AD27C7B00A021E0 /* SidebarItem.swift in Sources */,
 				D8472A662B2517DE0070DB64 /* DirectoryWatcher.swift in Sources */,

Folders/Extensions/FileManager.swift

@@ -23,11 +23,6 @@
 import Foundation
 import UniformTypeIdentifiers
 
-struct Details {
-    let url: URL
-    let contentType: UTType
-}
-
 extension FileManager {
 
     func files(directoryURL: URL) throws -> [Details]  {
@@ -39,21 +34,10 @@ extension FileManager {
 
         var files: [Details] = []
         for case let fileURL as URL in directoryEnumerator {
-            // Get the file metadata.
-            let isDirectory = try fileURL
-                .resourceValues(forKeys: [.isDirectoryKey])
-                .isDirectory!
-
-            // Ignore directories.
-            if isDirectory {
-                continue
-            }
-
-            // Only show images; we'll want to make this test dynamic in the future.
             guard let contentType = try fileURL.resourceValues(forKeys: [.contentTypeKey]).contentType else {
+                print("Failed to determine content type for \(fileURL).")
                 continue
             }
-
             files.append(Details(url: fileURL, contentType: contentType))
         }
 

Folders/Extensions/UTType.swift

@@ -32,7 +32,7 @@ extension UTType {
         return String(type)
     }
 
-    var subtytpe: String? {
+    var subtype: String? {
         guard let components = preferredMIMEType?.split(separator: "/"),
               components.count > 1
         else {

Folders/Models/Details.swift

@@ -0,0 +1,29 @@
+// MIT License
+//
+// Copyright (c) 2023-2024 Jason Morley
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+import Foundation
+import UniformTypeIdentifiers
+
+struct Details {
+    let url: URL
+    let contentType: UTType
+}

Folders/Utilities/DirectoryScanner.swift

@@ -63,10 +63,6 @@ class DirectoryScanner {
             case .itemClonedAtPath:
                 return
             case .itemCreated(path: let path, itemType: let itemType, eventId: _, fromUs: _):
-                // TODO: We want to support directories in the future.
-                guard itemType != .dir else {
-                    return
-                }
                 print("File created at path '\(path)'")
                 do {
                     onFileCreation(try FileManager.default.details(for: path))
@@ -94,6 +90,9 @@ class DirectoryScanner {
                 }
 
             case .itemRemoved(path: let path, itemType: let itemType, eventId: _, fromUs: _):
+
+                // TODO: Check if we receive individual deletions for files in a directory.
+
                 guard itemType != .dir else {
                     return
                 }

Folders/Utilities/DirectoryWatcher.swift

@@ -36,13 +36,22 @@ class DirectoryWatcher: NSObject, StoreObserver {
     let store: Store
     let url: URL
     let workQueue = DispatchQueue(label: "DirectoryWatcher.workQueue")
+    let filter: Filter
     var files: [URL] = []
 
     weak var delegate: DirectoryWatcherDelegate? = nil
 
     init(store: Store, url: URL) {
         self.store = store
         self.url = url
+
+        let pdf = UTType(mimeType: "application/pdf")!
+        let image = UTType(mimeType: "image/*")!
+        let video = UTType(mimeType: "video/*")!
+        filter = ParentFilter(parent: url.path) && (TypeFilter.conformsTo(pdf) ||
+                                                    TypeFilter.conformsTo(image) ||
+                                                    TypeFilter.conformsTo(video))
+
         super.init()
     }
 
@@ -56,12 +65,7 @@ class DirectoryWatcher: NSObject, StoreObserver {
                 // Get them out sorted.
                 let queryStart = Date()
                 let queryDuration = queryStart.distance(to: Date())
-
-                let pdf = UTType(mimeType: "application/pdf")!
-                let image = UTType(mimeType: "image/*")!
-                let video = UTType(mimeType: "video/*")!
-
-                let sortedFiles = try await store.files(parent: url, filter: TypeFilter.conformsTo(pdf) || TypeFilter.conformsTo(image) || TypeFilter.conformsTo(video))
+                let sortedFiles = try await store.files(filter: filter)
                 print("Query took \(queryDuration.formatted()) seconds and returned \(sortedFiles.count) files.")
 
                 DispatchQueue.main.async { [self] in
@@ -80,7 +84,7 @@ class DirectoryWatcher: NSObject, StoreObserver {
         store.remove(observer: self)
     }
 
-    func store(_ store: Store, didInsertURL url: URL) {
+    func store(_ store: Store, didInsert details: Details) {
         dispatchPrecondition(condition: .notOnQueue(.main))
 
         // Ignore updates that don't match our filter (currently just the parent URL).
@@ -89,8 +93,13 @@ class DirectoryWatcher: NSObject, StoreObserver {
         }
 
         DispatchQueue.main.async {
-            self.files.append(url)
-            self.delegate?.directoryWatcher(self, didInsertURL: url, atIndex: self.files.count - 1)
+            guard self.filter.matches(details: details) else {
+                return
+            }
+
+            // TODO: Work out where to insert this and pass this through to our observer.
+            self.files.append(details.url)
+            self.delegate?.directoryWatcher(self, didInsertURL: details.url, atIndex: self.files.count - 1)
         }
     }
 

Folders/Utilities/Filter.swift

@@ -28,7 +28,8 @@ import SQLite
 
 protocol Filter {
 
-    var filter: Expression<Bool?> { get }
+    var filter: Expression<Bool> { get }
+    func matches(details: Details) -> Bool
 
 }
 
@@ -42,10 +43,14 @@ extension Filter {
 
 struct TrueFilter: Filter {
 
-    var filter: Expression<Bool?> {
+    var filter: Expression<Bool> {
         return Expression(value: true)
     }
 
+    func matches(details: Details) -> Bool {
+        return true
+    }
+
 }
 
 struct AndFilter<A: Filter, B: Filter>: Filter {
@@ -58,10 +63,14 @@ struct AndFilter<A: Filter, B: Filter>: Filter {
         self.rhs = rhs
     }
 
-    var filter: Expression<Bool?> {
+    var filter: Expression<Bool> {
         return lhs.filter && rhs.filter
     }
 
+    func matches(details: Details) -> Bool {
+        return lhs.matches(details: details) && rhs.matches(details: details)
+    }
+
 }
 
 func &&<A: Filter, B: Filter>(lhs: A, rhs: B) -> AndFilter<A, B> {
@@ -78,10 +87,14 @@ struct OrFilter<A: Filter, B: Filter>: Filter {
         self.rhs = rhs
     }
 
-    var filter: Expression<Bool?> {
+    var filter: Expression<Bool> {
         return lhs.filter || rhs.filter
     }
 
+    func matches(details: Details) -> Bool {
+        return lhs.matches(details: details) || lhs.matches(details: details)
+    }
+
 }
 
 func ||<A: Filter, B: Filter>(lhs: A, rhs: B) -> OrFilter<A, B> {
@@ -94,19 +107,56 @@ enum TypeFilter {
 
 }
 
+struct ParentFilter: Filter {
+
+    let parent: String
+
+    init(parent: String) {
+        self.parent = parent
+    }
+
+    var filter: Expression<Bool> {
+        return Store.Schema.url.like("\(parent)%")
+    }
+
+    func matches(details: Details) -> Bool {
+        details.url.path.starts(with: parent)
+    }
+
+}
+
 extension TypeFilter: Filter {
 
-    var filter: Expression<Bool?> {
+    var filter: Expression<Bool> {
         switch self {
-        case .conformsTo(let type):
+        case .conformsTo(let contentType):
             var expression = Expression<Bool?>(value: true)
-            if let type = type.type {
+            if let type = contentType.type {
                 expression = expression && Store.Schema.type == type
             }
-            if let subtytpe = type.subtytpe, subtytpe != "*" {
-                expression = expression && Store.Schema.subtype == subtytpe
+            if let subtype = contentType.subtype, subtype != "*" {
+                expression = expression && Store.Schema.subtype == subtype
+            }
+            return expression ?? false
+        }
+    }
+
+    func matches(details: Details) -> Bool {
+        switch self {
+        case .conformsTo(let contentType):
+            guard let type = contentType.type else {
+                return true
+            }
+            guard details.contentType.type == type else {
+                return false
+            }
+            guard let subtype = contentType.subtype, subtype != "*" else {
+                return true
+            }
+            guard details.contentType.subtype == subtype else {
+                return false
             }
-            return expression
+            return true
         }
     }
 

Folders/Utilities/Store.swift

@@ -27,7 +27,7 @@ import SQLite
 
 protocol StoreObserver: NSObject {
 
-    func store(_ store: Store, didInsertURL url: URL)
+    func store(_ store: Store, didInsert details: Details)
     func store(_ store: Store, didRemoveURL url: URL)
 
 }
@@ -42,7 +42,7 @@ class Store {
         static let subtype = Expression<String?>("subtype")
     }
 
-    static let majorVersion = 3
+    static let majorVersion = 7
 
     var observers: [StoreObserver] = []
 
@@ -153,10 +153,10 @@ class Store {
                                                        Schema.url <- details.url.path,
                                                        Schema.name <- details.url.displayName,
                                                        Schema.type <- details.contentType.type,
-                                                       Schema.subtype <- details.contentType.subtytpe))
+                                                       Schema.subtype <- details.contentType.subtype))
                 for observer in self.observers {
                     DispatchQueue.global(qos: .default).async {
-                        observer.store(self, didInsertURL: details.url)
+                        observer.store(self, didInsert: details)
                     }
                 }
 
@@ -178,11 +178,9 @@ class Store {
         }
     }
 
-    func files(parent: URL, filter: Filter = TrueFilter()) async throws -> [URL] {
+    func files(filter: Filter = TrueFilter()) async throws -> [URL] {
         return try await run { [connection] in
-            print("parent = \(parent.path), filter = \(filter)")
             return try connection.prepareRowIterator(Schema.files.select(Schema.url)
-                .filter(Schema.url.like("\(parent.path)%"))
                 .filter(filter.filter)
                 .order(Schema.name.desc))
                 .map { URL(filePath: $0[Schema.url]) }

Folders/Views/GridView.swift

@@ -179,11 +179,11 @@ extension InnerGridView: DirectoryWatcherDelegate {
     }
 
     func directoryWatcher(_ directoryWatcher: DirectoryWatcher, didInsertURL url: URL, atIndex: Int) {
-        // TODO: This performs very poorly during initial update.
-        var snapshot = Snapshot()
-        snapshot.appendSections([.none])
-        snapshot.appendItems(directoryWatcher.files, toSection: Section.none)
-        dataSource.apply(snapshot, animatingDifferences: false)
+        // TODO: Insert in the correct place.
+        // TODO: We may need to rate-limit these updates.
+        var snapshot = dataSource.snapshot()
+        snapshot.appendItems([url])
+        dataSource.apply(snapshot, animatingDifferences: true)
     }
 
 }