diff --git a/lib/data/data_providers/file_system_data_provider.dart b/lib/data/data_providers/file_system_data_provider.dart index 50f784ed..f9fee53e 100644 --- a/lib/data/data_providers/file_system_data_provider.dart +++ b/lib/data/data_providers/file_system_data_provider.dart @@ -1,12 +1,7 @@ -/* this is an implementation of the data layer, based on the filesystem. -the representation of the library is a tree of directories and files, which every book is stored in a file, -and every directory is represents a category */ - import 'dart:io'; import 'dart:isolate'; import 'dart:convert'; import 'package:csv/csv.dart'; -import 'package:flutter/foundation.dart' as f; import 'package:flutter/services.dart'; import 'package:otzaria/data/data_providers/cache_provider.dart'; import 'package:otzaria/utils/docx_to_otzaria.dart'; @@ -16,56 +11,84 @@ import 'package:otzaria/models/books.dart'; import 'package:otzaria/models/library.dart'; import 'package:otzaria/models/links.dart'; -/// An implementation of the data layer based on the filesystem. -/// -/// The `FileSystemData` class represents an implementation of the data layer based on the filesystem. -/// It provides methods for accessing the library, book text, book table of contents, and links for a book. +/// A data provider that manages file system operations for the library. /// -/// The inner representation of the library is a tree of directories and files, -/// which every book is stored in a file, and every directory is represents a category. -/// The metadata is stored in a JSON file. +/// This class handles all file system related operations including: +/// - Reading and parsing book content from various file formats (txt, docx, pdf) +/// - Managing the library structure (categories and books) +/// - Handling external book data from CSV files +/// - Managing book links and metadata +/// - Providing table of contents functionality class FileSystemData { + /// Future that resolves to a mapping of book titles to their file system paths late Future> titleToPath; - Map metadata = {}; + /// Future that resolves to metadata for all books and categories + late Future>> metadata; + + /// Creates a new instance of [FileSystemData] and initializes the title to path mapping + /// and metadata FileSystemData() { - titleToPath = _updateTitleToPath(); + titleToPath = _getTitleToPath(); + metadata = _getMetadata(); } + /// Singleton instance of [FileSystemData] static FileSystemData instance = FileSystemData(); - /// Returns the library + /// Retrieves the complete library structure from the file system. + /// + /// Reads the library from the configured path and combines it with metadata + /// to create a full [Library] object containing all categories and books. Future getLibrary() async { - titleToPath = _updateTitleToPath(); - await _fetchMetadata(); + titleToPath = _getTitleToPath(); + metadata = _getMetadata(); return _getLibraryFromDirectory( - '${Settings.getValue('key-library-path') ?? '.'}${Platform.pathSeparator}אוצריא'); + '${Settings.getValue('key-library-path') ?? '.'}${Platform.pathSeparator}אוצריא', + await metadata); } - Future _getLibraryFromDirectory(String path) async { + /// Recursively builds the library structure from a directory. + /// + /// Creates a hierarchical structure of categories and books by traversing + /// the file system directory structure. + Future _getLibraryFromDirectory( + String path, Map metadata) async { + /// Recursive helper function to process directories and build category structure Future getAllCategoriesAndBooksFromDirectory( Directory dir, Category? parent) async { + final title = getTitleFromPath(dir.path); Category category = Category( - title: getTitleFromPath(dir.path), + title: title, + description: metadata[title]?['heDesc'] ?? '', + shortDescription: metadata[title]?['heShortDesc'] ?? '', + order: metadata[title]?['order'] ?? 999, subCategories: [], books: [], parent: parent); - // get the books and categories from the directory + + // Process each entity in the directory await for (FileSystemEntity entity in dir.list()) { if (entity is Directory) { + // Recursively process subdirectories as categories category.subCategories.add( await getAllCategoriesAndBooksFromDirectory( Directory(entity.path), category)); } else { + // Extract topics from the file path var topics = entity.path .split('אוצריא${Platform.pathSeparator}') .last .split(Platform.pathSeparator) .toList(); topics = topics.sublist(0, topics.length - 1); + + // Handle special case where title contains " על " if (getTitleFromPath(entity.path).contains(' על ')) { topics.add(getTitleFromPath(entity.path).split(' על ')[1]); } + + // Process PDF files if (entity.path.toLowerCase().endsWith('.pdf')) { final title = getTitleFromPath(entity.path); category.books.add( @@ -81,10 +104,11 @@ class FileSystemData { ), ); } + + // Process text and docx files if (entity.path.toLowerCase().endsWith('.txt') || entity.path.toLowerCase().endsWith('.docx')) { final title = getTitleFromPath(entity.path); - category.books.add(TextBook( title: title, author: metadata[title]?['author'], @@ -97,15 +121,17 @@ class FileSystemData { } } } + + // Sort categories and books by their order category.subCategories.sort((a, b) => a.order.compareTo(b.order)); category.books.sort((a, b) => a.order.compareTo(b.order)); return category; } - //first initialize an empty library + // Initialize empty library Library library = Library(categories: []); - //then get all the categories and books from the top directory recursively + // Process top-level directories await for (FileSystemEntity entity in Directory(path).list()) { if (entity is Directory) { library.subCategories.add(await getAllCategoriesAndBooksFromDirectory( @@ -116,21 +142,24 @@ class FileSystemData { return library; } + /// Retrieves the list of books from Otzar HaChochma Future> getOtzarBooks() { return _getOtzarBooks(); } + /// Retrieves the list of books from HebrewBooks Future> getHebrewBooks() { return _getHebrewBooks(); } + /// Internal implementation for loading Otzar HaChochma books from CSV Future> _getOtzarBooks() async { try { print('Loading Otzar HaChochma books from CSV'); final csvData = await rootBundle.loadString('assets/otzar_books.csv'); return Isolate.run(() { - // fix the line endings so that it works on all platforms + // Normalize line endings for cross-platform compatibility final normalizedCsvData = csvData.replaceAll('\r\n', '\n').replaceAll('\r', '\n'); @@ -145,7 +174,6 @@ class FileSystemData { print('Loaded ${csvTable.length} rows'); return csvTable.skip(1).map((row) { - // Skip the header row return ExternalBook( title: row[1], id: int.tryParse(row[0]) ?? -1, @@ -163,13 +191,14 @@ class FileSystemData { } } + /// Internal implementation for loading HebrewBooks from CSV Future> _getHebrewBooks() async { try { print('Loading hebrewbooks from CSV'); final csvData = await rootBundle.loadString('assets/hebrew_books.csv'); return Isolate.run(() { - // fix the line endings so that it works on all platforms + // Normalize line endings for cross-platform compatibility final normalizedCsvData = csvData.replaceAll('\r\n', '\n').replaceAll('\r', '\n'); @@ -182,10 +211,8 @@ class FileSystemData { ).convert(normalizedCsvData); print('Loaded ${csvTable.length} rows'); - // Skip the header row return csvTable.skip(1).map((row) { - // Skip the header row try { return ExternalBook( title: row[1].toString(), @@ -209,8 +236,9 @@ class FileSystemData { } } - ///the implementation of the links from app's model, based on the filesystem. - ///the links are in the folder 'links' with the name '_links.json' + /// Retrieves all links associated with a specific book. + /// + /// Links are stored in JSON files named '_links.json' in the links directory. Future> getAllLinksForBook(String title) async { try { File file = File(_getLinksPath(title)); @@ -223,50 +251,58 @@ class FileSystemData { } } - /// Retrieves the text for a book with the given title asynchronously (using Isolate). - /// supports docx files + /// Retrieves the text content of a book. + /// + /// Supports both plain text and DOCX formats. DOCX files are processed + /// using a special converter to extract their content. Future getBookText(String title) async { - return Isolate.run(() async { - String path = await _getBookPath(title); - File file = File(path); - if (path.endsWith('.docx')) { - final bytes = await file.readAsBytes(); - return docxToText(bytes, title); - } else { - return await file.readAsString(); - } - }); + final path = await _getBookPath(title); + final file = File(path); + + if (path.endsWith('.docx')) { + final bytes = await file.readAsBytes(); + return Isolate.run(() => docxToText(bytes, title)); + } else { + final content = await file.readAsString(); + return Isolate.run(() => content); + } } - /// an file system approach to get the content of a link. - /// we read the file line by line and return the content of the line with the given index. + /// Retrieves the content of a specific link within a book. + /// + /// Reads the file line by line and returns the content at the specified index. Future getLinkContent(Link link) async { String path = await _getBookPath(getTitleFromPath(link.path2)); return Isolate.run(() async => await getLineFromFile(path, link.index2)); } - /// Returns a list of all the book paths in the library directory. - List getAllBooksPathsFromDirecctory(String path) { - List paths = []; - final files = Directory(path).listSync(recursive: true); - for (var file in files) { - paths.add(file.path); - } - return paths; + /// Returns a list of all book paths in the library directory. + /// + /// This operation is performed in an isolate to prevent blocking the main thread. + static Future> getAllBooksPathsFromDirecctory( + String path) async { + return Isolate.run(() async { + List paths = []; + final files = await Directory(path).list(recursive: true).toList(); + for (var file in files) { + paths.add(file.path); + } + return paths; + }); } - /// Returns the title of the book with the given path. - -// Retrieves the table of contents for a book with the given title. - + /// Retrieves the table of contents for a book. + /// + /// Parses the book content to extract headings and create a hierarchical + /// table of contents structure. Future> getBookToc(String title) async { return _parseToc(getBookText(title)); } - ///gets a line from file in an efficient way, using a stream that is closed right - /// after the line with the given index is found. the function + /// Efficiently reads a specific line from a file. /// - ///the function gets a path to the file and an int index, and returns a Future. + /// Uses a stream to read the file line by line until the desired index + /// is reached, then closes the stream to conserve resources. Future getLineFromFile(String path, int index) async { File file = File(path); final lines = file @@ -278,16 +314,17 @@ class FileSystemData { return (await lines).last; } - /// Updates the title to path mapping using the provided library path. - Future> _updateTitleToPath() async { + /// Updates the mapping of book titles to their file system paths. + /// + /// Creates a map where keys are book titles and values are their corresponding + /// file system paths, excluding PDF files. + Future> _getTitleToPath() async { Map titleToPath = {}; if (!Settings.isInitialized) { await Settings.init(cacheProvider: HiveCache()); } final libraryPath = Settings.getValue('key-library-path'); - List paths = await Isolate.run( - () => getAllBooksPathsFromDirecctory(libraryPath), - ); + List paths = await getAllBooksPathsFromDirecctory(libraryPath); for (var path in paths) { if (path.toLowerCase().endsWith('.pdf')) continue; titleToPath[getTitleFromPath(path)] = path; @@ -295,18 +332,23 @@ class FileSystemData { return titleToPath; } - ///fetches the metadata for the books in the library from a json file using the provided library path. - Future _fetchMetadata() async { + /// Loads and parses the metadata for all books in the library. + /// + /// Reads metadata from a JSON file and creates a structured mapping of + /// book titles to their metadata information. + Future>> _getMetadata() async { String metadataString = ''; + Map> metadata = {}; try { File file = File( '${Settings.getValue('key-library-path') ?? '.'}${Platform.pathSeparator}metadata.json'); metadataString = await file.readAsString(); } catch (e) { - return; + return {}; } final tempMetadata = await Isolate.run(() => jsonDecode(metadataString) as List); + for (int i = 0; i < tempMetadata.length; i++) { final row = tempMetadata[i] as Map; metadata[row['title'].replaceAll('"', '')] = { @@ -319,8 +361,6 @@ class FileSystemData { ? [row['title'].toString()] : row['extraTitles'].map((e) => e.toString()).toList() as List, - - // get order in int even if the value is null or double 'order': row['order'] == null || row['order'] == '' ? 999 : row['order'].runtimeType == double @@ -328,65 +368,62 @@ class FileSystemData { : row['order'] as int, }; } + return metadata; } - /// Returns the path of the book with the given title. + /// Retrieves the file system path for a book with the given title. Future _getBookPath(String title) async { - // final titleToPath = await this.titleToPath; - - //return the path of the book with the given title return titleToPath[title] ?? 'error: book path not found: $title'; } - ///a function that parses the table of contents from a string, based on the heading level: for example, h1, h2, h3, etc. - ///each entry has a level and an index in the array of lines - Future> _parseToc(Future data) async { - List lines = (await data).split('\n'); - List toc = []; - Map parents = {}; // Keep track of parent nodes - - for (int i = 0; i < lines.length; i++) { - final String line = lines[i]; - if (line.startsWith('> _parseToc(Future bookContentFuture) async { + final String bookContent = await bookContentFuture; + + return Isolate.run(() { + List lines = bookContent.split('\n'); + List toc = []; + Map parents = {}; // Track parent nodes for hierarchy + + for (int i = 0; i < lines.length; i++) { + final String line = lines[i]; + if (line.startsWith('('key-library-path') ?? '.'}${Platform.pathSeparator}links${Platform.pathSeparator}${title}_links.json'; } + /// Checks if a book with the given title exists in the library. Future bookExists(String title) async { final titleToPath = await this.titleToPath; return titleToPath.keys.contains(title); diff --git a/lib/models/books.dart b/lib/models/books.dart index c2245155..182e70a2 100644 --- a/lib/models/books.dart +++ b/lib/models/books.dart @@ -83,8 +83,7 @@ class TextBook extends Book { /// /// Returns a [Future] that resolves to a [List] of [TocEntry] objects representing /// the table of contents of the book. - Future> get tableOfContents => - Isolate.run(() => data.getBookToc(title)); + Future> get tableOfContents => data.getBookToc(title); /// Retrieves all the links for the book. /// diff --git a/lib/models/library.dart b/lib/models/library.dart index 4f801076..f732bef1 100644 --- a/lib/models/library.dart +++ b/lib/models/library.dart @@ -19,16 +19,14 @@ class Category { String title; /// A description of the category, obtained from [Data.metadata]. - String get description => - FileSystemData.instance.metadata[title]?['heDesc'] ?? ''; + String description; /// A short description of the category, obtained from [Data.metadata]. - String get shortDescription => - FileSystemData.instance.metadata[title]?['heShortDesc'] ?? ''; + String shortDescription; /// The order of the category, obtained from [Data.metadata]. /// Defaults to 999 if no order is specified for this category. - int get order => FileSystemData.instance.metadata[title]?['order'] ?? 999; + int order; ///the list of sub categories that are contained in this category List subCategories; @@ -72,6 +70,9 @@ class Category { /// in this category, and [parent] is the parent category. Category({ required this.title, + required this.description, + required this.shortDescription, + required this.order, required this.subCategories, required this.books, required this.parent, @@ -91,6 +92,9 @@ class Library extends Category { Library({required List categories}) : super( title: 'ספריית אוצריא', + description: '', + shortDescription: '', + order: 0, subCategories: categories, books: [], parent: null) {