Merge pull request #9 from Ajatt-Tools/dev

Merge new changes into master

.gitignore

@@ -1,10 +1,13 @@
 \*.o
 dictpopup
 dictpopup-create
+dictpopup-cli
 cmake-build-debug/
 cmake-build-release/
+cmake-build-release-coverage/
 .cmake
 .idea
 CMakeFiles
 .gdb_history
-misc/
+misc/
+build/

CMakeLists.txt

@@ -28,20 +28,23 @@ add_definitions(${GTK3_CFLAGS_OTHER} ${NOTIFY_CFLAGS_OTHER}
         ${LIBZIP_DEFINITIONS} ${CURL_DEFINITIONS})
 set(SRCS
         src/ankiconnectc.c
-        src/database.c
         src/db.c
         src/deinflector.c
         src/dictpopup.c
-        src/jppron.c
         src/platformdep.c
         src/settings.c
-        src/util.c
-        src/pdjson.c)
+        src/util.c)
+
+set(JPPRON_SRCS
+        src/jppron/jppron.c
+        src/jppron/database.c
+        src/jppron/ajt_audio_index_parser.c
+	src/yyjson.c)
 
 # ##############################################################################
 # dictpopup
 # ##############################################################################
-add_executable(dictpopup src/frontends/gtk3popup.c ${SRCS})
+add_executable(dictpopup src/frontends/gtk3popup.c ${SRCS} ${JPPRON_SRCS})
 target_compile_definitions(dictpopup PUBLIC NOTIFICATIONS)
 target_include_directories(
         dictpopup PRIVATE ${GTK3_INCLUDE_DIRS} ${NOTIFY_INCLUDE_DIRS}
@@ -54,7 +57,9 @@ target_link_libraries(dictpopup PRIVATE ${GTK3_LIBRARIES} ${NOTIFY_LIBRARIES}
 # ##############################################################################
 # dictpopup-create
 # ##############################################################################
-add_executable(dictpopup-create src/dictpopup_create.c ${SRCS})
+add_executable(dictpopup-create src/dictpopup_create.c ${SRCS}
+	src/yyjson.c
+        src/yomichan_parser.c)
 target_include_directories(
         dictpopup-create PRIVATE ${GTK3_INCLUDE_DIRS} ${LIBZIP_INCLUDE_DIRS} include/)
 target_link_directories(dictpopup-create PRIVATE ${GTK3_LIBRARY_DIRS}
@@ -63,15 +68,15 @@ target_link_libraries(dictpopup-create PRIVATE ${GTK3_LIBRARIES} -lzip -llmdb
         ${CURL_LIBRARIES} -lmecab)
 
 # ##############################################################################
-# dictpopup-cli
+# cli
 # ##############################################################################
-add_executable(dictpopup-cli EXCLUDE_FROM_ALL src/frontends/cli.c ${SRCS})
+add_executable(cli EXCLUDE_FROM_ALL src/frontends/cli.c ${SRCS})
 target_include_directories(
-        dictpopup-cli PRIVATE ${GTK3_INCLUDE_DIRS} ${NOTIFY_INCLUDE_DIRS}
+        cli PRIVATE ${GTK3_INCLUDE_DIRS} ${NOTIFY_INCLUDE_DIRS}
         ${CURL_INCLUDE_DIRS} include/)
-target_link_directories(dictpopup-cli PRIVATE ${GTK3_LIBRARY_DIRS}
+target_link_directories(cli PRIVATE ${GTK3_LIBRARY_DIRS}
         ${NOTIFY_LIBRARY_DIRS} ${CURL_LIBRARY_DIRS})
-target_link_libraries(dictpopup-cli PRIVATE ${GTK3_LIBRARIES} ${NOTIFY_LIBRARIES}
+target_link_libraries(cli PRIVATE ${GTK3_LIBRARIES} ${NOTIFY_LIBRARIES}
         ${CURL_LIBRARIES} -lmecab -llmdb)
 
 # ##############################################################################
@@ -87,11 +92,13 @@ if (BUILD_TESTING)
             tests/ankiconnect_tests.c
             src/deinflector.c
             src/util.c
-            src/pdjson.c
+            tests/yomichan_parser_tests.c
             src/db.c
-            src/platformdep.c)
+            src/platformdep.c
+            src/yyjson.c
+            tests/ajt_audio_index_parser_tests.c)
     target_compile_definitions(c_tests PRIVATE UNIT_TEST)
-    target_include_directories(c_tests PRIVATE ${GTK3_INCLUDE_DIRS} ${CGREEN_INCLUDE_DIRS} include/ src/)
+    target_include_directories(c_tests PRIVATE ${GTK3_INCLUDE_DIRS} ${CGREEN_INCLUDE_DIRS} include/ src/ src/jppron)
     target_link_libraries(
             c_tests
             PRIVATE ${CGREEN_LIBRARIES}
@@ -102,6 +109,8 @@ if (BUILD_TESTING)
             -lmecab
             -lzip)
     add_test(NAME run_c_tests COMMAND $<TARGET_FILE:c_tests>)
+
+    #    add_test(NAME run_shell_test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/full_cycle_test.sh)
 endif ()
 
 # ##############################################################################

Makefile

@@ -27,14 +27,23 @@ ifeq ($(strip $(O_HAVEX11)),1)
 	LDLIBS += -lXfixes -lX11
 endif
 
-
-FILES = dictpopup.c util.c platformdep.c deinflector.c settings.c db.c ankiconnectc.c database.c jppron.c pdjson.c
-FILES_H = ankiconnectc.h db.h deinflector.h settings.h util.h platformdep.h database.h jppron.h pdjson.h
-SRC = $(addprefix $(SDIR)/,$(FILES))
 SRC_H = $(addprefix $(IDIR)/,$(FILES_H))
 
-FILES_CREATE = db.c pdjson.c util.c platformdep.c
-FILES_H_CREATE = db.h pdjson.h util.h buf.h platformdep.h
+SRCS = src/ankiconnectc.c \
+       src/db.c \
+       src/deinflector.c \
+       src/dictpopup.c \
+       src/platformdep.c \
+       src/settings.c \
+       src/util.c
+
+SRCS_JPPRON = src/jppron/jppron.c \
+	      src/jppron/database.c \
+	      src/jppron/ajt_audio_index_parser.c \
+	      src/yyjson.c
+
+FILES_CREATE = db.c util.c platformdep.c yomichan_parser.c yyjson.c
+FILES_H_CREATE = db.h util.h buf.h platformdep.h yyjson.h
 SRC_CREATE = $(addprefix $(SDIR)/,$(FILES_CREATE))  $(LMDB_FILES)
 SRC_H_CREATE = $(addprefix $(IDIR)/,$(FILES_H_CREATE))
 
@@ -46,14 +55,14 @@ all: CFLAGS+=$(RELEASE_CFLAGS)
 debug: $(bins)
 debug: CFLAGS+=$(DEBUG_CFLAGS)
 
-dictpopup: $(SRC) $(SRC_H) $(SDIR)/frontends/gtk3popup.c
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(NOTIF_CFLAGS) -o $@ $(SDIR)/frontends/gtk3popup.c $(SRC) $(LDLIBS) $(NOTIF_LIBS)
+dictpopup: $(SRCS) $(SRCS_JPPRON) $(SRC_H) $(SDIR)/frontends/gtk3popup.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) $(NOTIF_CFLAGS) -o $@ $(SDIR)/frontends/gtk3popup.c $(SRCS) $(SRCS_JPPRON) $(LDLIBS) $(NOTIF_LIBS)
 
 dictpopup-create: $(SRC_CREATE) $(SRC_H_CREATE) $(SDIR)/dictpopup_create.c
 	$(CC) $(CFLAGS) $(CPPFLAGS) -o $@ $(SDIR)/dictpopup_create.c $(SRC_CREATE) $(LDLIBS)
 
 cli: $(SRC) $(SRC_H) $(SDIR)/frontends/cli.c
-	$(CC) $(CFLAGS) $(DEBUG_CFLAGS) $(CPPFLAGS) -o $@ $(SDIR)/frontends/cli.c $(SRC) $(LDLIBS)
+	$(CC) $(CFLAGS) $(DEBUG_CFLAGS) $(CPPFLAGS) -o $@ $(SDIR)/frontends/cli.c $(SRCS) $(LDLIBS)
 
 deinflector: $(SRC) $(SRC_H) $(SDIR)/deinflector.c
 	$(CC) $(CFLAGS) $(DEBUG_CFLAGS) $(CPPFLAGS) -DDEINFLECTOR_MAIN -o $@ $(SDIR)/deinflector.c $(SDIR)/util.c $(LDLIBS)
@@ -94,9 +103,10 @@ ALL_C_SOURCES := $(wildcard src/*.c)
 ALL_H_SOURCES := $(wildcard include/*.h)
 
 c_analyse_targets := $(ALL_C_SOURCES:%=%-analyse)
-c_analyse_targets := $(filter-out src/pdjson.c-analyse, $(c_analyse_targets))
+c_analyse_targets := $(filter-out src/yyjson.c-analyse, $(c_analyse_targets))
 
 h_analyse_targets := $(ALL_H_SOURCES:%=%-analyse)
+h_analyse_targets := $(filter-out include/yyjson.h-analyse, $(h_analyse_targets))
 
 analyse: CFLAGS+=$(DEBUG_CFLAGS)
 analyse: $(c_analyse_targets) $(h_analyse_targets)
@@ -128,12 +138,12 @@ $(h_analyse_targets): %-analyse:
 %-shared-analyse: %
 	# cppcheck is a bit dim about unused functions/variables, leave that to
 	# clang/GCC
-	cppcheck $< -I$(IDIR) --library=gtk.cfg --library=gnu.cfg \
-		--std=c99 --quiet --inline-suppr --force \
-		--enable=all --suppress=missingIncludeSystem \
-		--suppress=unusedFunction --suppress=unmatchedSuppression \
+	cppcheck $< -I$(IDIR) --library=gtk.cfg --library=libcurl.cfg \
+		--std=c99 --quiet --inline-suppr --force --enable=all \
+		--suppress=missingIncludeSystem --suppress=unmatchedSuppression \
 		--suppress=unreadVariable --suppress=constParameterCallback \
 		--suppress=constVariablePointer --suppress=constParameterPointer \
+		--suppress=unusedFunction --suppress=*:include/yyjson.h \
 		--max-ctu-depth=32 --error-exitcode=1
 	# clang-analyzer-unix.Malloc does not understand _drop_()
 	clang-tidy $< --quiet -checks=-clang-analyzer-unix.Malloc -- -std=gnu99 -I$(IDIR) $(shell pkg-config --cflags gtk+-3.0)

TODO

@@ -1,57 +1,51 @@
 ## dictpopup
-- Fix formating for JMDict and Jitendex
+- Fix formating for JMDict (tags come after content)
+  - Just put content into separate variable and then concatenate at end
+- Fix formatting for Jitendex (Prettier lists)
 - Write a "Yomichan search"-like window application
 - Include a small default dictionary (.mdb file)
-- Add ながら deinflection
 
 Mid priority
+- Right click on '+' button -> popup menu with: "add from clipboard", ...
+- Right click on audio button -> popup menu with "copy file to clipboard", ...
 - Implement the reading label as a text view to allow for editing -> initiate new search on Enter press
 - Group terms by reading, allow for switching between different readings
 - Implement a small window to manually enter a search string instead of selecting
 - Kanjify the word in the copied sentence according to the dictionary entry
 - Remove trailing spaces from user settings
+- Need some check when looking up a hiragana conversion to prevent things like 賢人会 -> 県人会
+- Don't show circle indicator if Anki is not running / on error
+- Show a blue circle if existing cards are all new
+- Search for audio of inflected selected word (e.g. 下がらない) (optionally)
+- Switch to using GSettings
 - Write a gui config tool
   - Select dictionaries
   - Choose dictionary sort order
   - build database
   - change general config
-- Need some check when looking up a hiragana conversion to prevent things like 賢人会 -> 県人会
-- Don't show circle indicator if Anki is not running / on error
-- Search for audio of inflected selected word (e.g. 下がらない) (optionally)
-- Merge dbreader.c and dbwriter.c
 
 Low priority
+- Support pictures
 - Fix "Error obtaining window name" on recursive lookup
   - The problem is, that dictpopup creates a "child window" (?) with id one more which does not have a name
   - This affects all programs which create a child without own title name
-- Deinflector should not check for verb stem when some (non 記号 and Japanese) characters where already truncated
 - Handle "Anki not running" more gracefully
   - Disable Anki support and reenable automatically if Anki starts
 - Allow configurable keybindings
 - Allow resizing the window and remember the size on close (write to config.ini)
-- Support for pictures
 - Move all Japanese specific things into deinflector.c, choose more language agnostic function names in all other places
 - Adds tags to the Anki card, similiar to mpvacious
-- Allow canceling selecting a sentence with Esc
+- Allow canceling context selection with Esc
 - Allow changing AnkiConnect API address in config.ini
-- Switch dictionary implementation to growable buffers (buf.h)
-- Fix Makefile to only rebuild necessary parts
-- Don't call jppron through the shell
-- Refactor settings.c
-- Refactor popup.c
-- Add correct feature test macros
-- Could split Mecab conversion and kata -> hira conversion into separate configuration options
 - Skip certain lines on definition add (without selection), e.g.:
   - しず か （しづ—） [1]【静か・▽閑か】
   -（形動）〔文〕ナリ
 - Put length of longest entry into the database under LENGTH_OF_LONGEST_ENTRY. Then limit the lookup by that length
+  - Currently hard coded to 60 bytes
   - Only important for an auto scan implementation though
 
-## dictpopup-create
-- Process every dictionary in a different thread?
-  - Questionable if this makes sense, as it's already pretty fast and would make the code more complicated
-
 ## Known problems:
+- Can't pronounce 方方[かたがた]
 ### Misc
 - If there is a dictionary entry for 恭しく even though 恭しい is present in the collection, then a red dot will be shown
 ### Deinflection
@@ -69,6 +63,8 @@ freq:
 - いじらしい not present in all
 
 Possible ideas:
+- Add some way to lookup example sentences
+  - Maybe something like a menu button which opens a popup menu with: "settings", "lookup massif example sentences", ...
 - Implement a dbus service mode
   - Keep window in the background and just hide on close
   - This should allow for near instantaneous lookup (The bottleneck currently is all due to gtk + X11, i.e. starting the window)
@@ -77,8 +73,6 @@ Possible ideas:
   - But I think this won't allow moving / resizing the window
   - Another approach might be using an invisible (transparent) fullscreen window with a popover / popup menu at the mouse pointer
   - Need to look into how goldendict's move() function is implemented (Probably using Qt)
-- Show loading percentage in dictpopup-create
-- Right click on audio button -> open some menu with "copy file to clipboard", ...
 - Write a browser addon which enables using dictpopup just by pressing Shift (like yomichan)
   - It simply needs to pass the next ~50 characters starting from the current mouse position as an argument to dictpopup
   - However I strongly doubt that an extension can execute some program, even if it's a local copy

include/ajt_audio_index_parser.h

@@ -0,0 +1,17 @@
+#ifndef AUDIO_INDEX_PARSER_H
+#define AUDIO_INDEX_PARSER_H
+
+#include "util.h"
+
+typedef struct {
+    s8 origin;
+    s8 hira_reading;
+    s8 pitch_number;
+    s8 pitch_pattern;
+} fileinfo;
+
+void parse_audio_index_from_file(s8 curdir, const char *index_filepath,
+                                 void (*foreach_headword)(void *, s8, s8), void *userdata_hw,
+                                 void (*foreach_file)(void *, s8, fileinfo), void *userdata_f);
+
+#endif // AUDIO_INDEX_PARSER_H

include/ankiconnectc.h

@@ -1,6 +1,8 @@
 #ifndef ANKICONNECTC_H
 #define ANKICONNECTC_H
 
+#include "util.h"
+#include <stdbool.h>
 #include <stddef.h>
 
 typedef struct {
@@ -15,46 +17,17 @@ typedef struct {
 
 void ankicard_free(ankicard *ac);
 
-typedef struct {
-    union {
-        char *string;
-        char **stringv;
-        _Bool boolean;
-    } data;
-    _Bool ok; // Signalizes if there was an error on not. The error msg is
-              // stored in data.string
-} retval_s;
-
-/*
- * Returns: The null terminated list of decks in data.stringv
- */
-retval_s ac_get_decks(void);
-
-/*
- * Returns: A null terminated list of notetypes in data.stringv
- */
-retval_s ac_get_notetypes(void);
-
-/*
- * Search the Anki database for the entry @entry in the field @field and deck
- * @deck.
- * @deck can be null.
- * The search result will be stored in data.boolen if there was no error.
- */
-retval_s ac_search(_Bool include_suspended, char *deck, char *field, char *entry);
-
-/*
- * Performs a search as in ac_search, but in the Anki gui.
- */
-retval_s ac_gui_search(const char *deck, const char *field, const char *entry);
+bool ac_check_connection(void);
+s8 *ac_get_decks(char **error);
+s8 *ac_get_notetypes(char **error);
+int ac_check_exists(char *deck, char *field, char *entry, char **error);
+void ac_gui_search(const char *deck, const char *field, const char *entry, char **error);
+void ac_addNote(ankicard ac, char **error);
 
 /*
  * Stores the file at @path in the Anki media collection under the name
  * @filename. Doesn't overwrite existing files.
  */
-retval_s ac_store_file(const char *filename, const char *path);
-
-/* Add ankicard to anki  */
-retval_s ac_addNote(ankicard ac);
+void ac_store_file(const char *filename, const char *path, char **error);
 
 #endif

include/database.h

@@ -1,23 +1,25 @@
+#ifndef DATABASE_H
+#define DATABASE_H
+
+#include "ajt_audio_index_parser.h"
 #include "util.h"
-#include <lmdb.h>
 #include <stdbool.h>
 
-typedef struct {
-    stringbuilder_s lastval;
-    MDB_env *env;
-    MDB_dbi dbi1;
-    MDB_dbi dbi2;
-    MDB_txn *txn;
-    bool readonly;
-} database;
+typedef struct database_s database;
 
-database opendb(const char *path, bool readonly);
-void closedb(database);
+database *jdb_open(char *path, bool readonly);
+void jdb_close(database *);
 /*
  * Add to database, allowing duplicates if they are added directly after another
  */
-void addtodb1(database db, s8 key, s8 val);
-void addtodb2(database db, s8 key, s8 val);
+void jdb_add_headword_with_file(database *db, s8 headword, s8 filepath);
+void jdb_add_file_with_fileinfo(database *db, s8 filepath, fileinfo fi);
+
+_deallocator_(frees8buffer) s8 *jdb_get_files(database *db, s8 key);
+fileinfo jdb_get_fileinfo(database *db, s8 fullpath);
+
+i32 jdb_check_exists(s8 dbpath);
+
+DEFINE_DROP_FUNC(database *, jdb_close)
 
-s8 *getfiles(database db, s8 key);
-s8 getfromdb2(database db, s8 key);
+#endif // DATABASE_H