From cbc34da75bbbba383c678ce7b97f0f2ea487e409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?=
 <41315874+fumiama@users.noreply.github.com>
Date: Fri, 14 Oct 2022 10:51:00 +0800
Subject: [PATCH 01/14] feat: cmd add convert saved model to tflite

---
 .gitignore                                 |  2 ++
 deepdanbooru/__main__.py                   | 23 +++++++++++++
 deepdanbooru/commands/__init__.py          |  8 +++--
 deepdanbooru/commands/convert_to_tflite.py | 40 ++++++++++++++++++++++
 4 files changed, 70 insertions(+), 3 deletions(-)
 create mode 100644 deepdanbooru/commands/convert_to_tflite.py

diff --git a/.gitignore b/.gitignore
index e845071..9c5607b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,8 @@ __pycache__/
 test.py
 /logs
 /uploads
+/build
+/deepdanbooru.egg-info
 *.jpg
 *.jpeg
 *.png
diff --git a/deepdanbooru/__main__.py b/deepdanbooru/__main__.py
index b3988a6..9f6ec2a 100644
--- a/deepdanbooru/__main__.py
+++ b/deepdanbooru/__main__.py
@@ -196,6 +196,29 @@ def evaluate(
         verbose,
     )
 
+@main.command("conv2tflite", help="Convert saved model into tflite model.")
+@click.argument(
+    "optimizations",
+    nargs=-1,
+    type=click.STRING,
+)
+@click.option(
+    "--project-path",
+    type=click.Path(exists=True, resolve_path=True, file_okay=False, dir_okay=True),
+    help="Project path. If you want to use specific model and tags, use --model-path and --tags-path options.",
+)
+@click.option(
+    "--model-path",
+    type=click.Path(exists=True, resolve_path=True, file_okay=True, dir_okay=False),
+)
+@click.option(
+    "--save-path",
+    type=click.Path(resolve_path=True, file_okay=True, dir_okay=False),
+)
+@click.option("--verbose", default=False, is_flag=True)
+def conv2tflite(optimizations, project_path, model_path, save_path, verbose):
+    dd.commands.convert_to_tflite_from_from_saved_model(project_path, model_path, save_path, optimizations, verbose=verbose)
+
 
 if __name__ == "__main__":
     main()
diff --git a/deepdanbooru/commands/__init__.py b/deepdanbooru/commands/__init__.py
index 1c4f572..ceab469 100644
--- a/deepdanbooru/commands/__init__.py
+++ b/deepdanbooru/commands/__init__.py
@@ -1,7 +1,9 @@
+from .convert_to_tflite import convert_to_tflite_from_from_saved_model
 from .create_project import create_project
 from .download_tags import download_tags
-from .make_training_database import make_training_database
-from .train_project import train_project
 from .evaluate_project import evaluate_project
-from .grad_cam import grad_cam
 from .evaluate import evaluate, evaluate_image
+from .grad_cam import grad_cam
+from .make_training_database import make_training_database
+from .train_project import train_project
+
diff --git a/deepdanbooru/commands/convert_to_tflite.py b/deepdanbooru/commands/convert_to_tflite.py
new file mode 100644
index 0000000..db2e32f
--- /dev/null
+++ b/deepdanbooru/commands/convert_to_tflite.py
@@ -0,0 +1,40 @@
+from typing import List
+import tensorflow as tf
+import deepdanbooru as dd
+
+# optimizations: see in tf.lite.Optimize
+def convert_to_tflite_from_from_saved_model(
+    project_path: str, model_path: str, save_path: str,
+    optimizations: List[tf.lite.Optimize] = [tf.lite.Optimize.DEFAULT],
+    verbose: bool = False
+):
+    if not model_path and not project_path:
+        raise Exception("You must provide project path or model path.")
+
+    if not save_path:
+        raise Exception("You must provide a path to save tflite model.")
+
+    if model_path:
+        if verbose:
+            print(f"Loading model from {model_path} ...")
+        model = tf.keras.models.load_model(model_path)
+    else:
+        if verbose:
+            print(f"Loading model from project {project_path} ...")
+        model = dd.project.load_model_from_project(project_path)
+
+    if verbose:
+            print(f"Converting ...")
+
+    converter = tf.lite.TFLiteConverter.from_keras_model(model)
+    converter.optimizations = optimizations
+    tflite_model = converter.convert()
+
+    if verbose:
+        print(f"Saving ...")
+
+    with open(save_path, "wb") as f:
+        f.write(tflite_model)
+
+    if verbose:
+        print(f"Converted model has been saved to {save_path}")

From 5f5d36d881e311303742931a687da2ed99b28034 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?=
 <41315874+fumiama@users.noreply.github.com>
Date: Fri, 14 Oct 2022 10:55:41 +0800
Subject: [PATCH 02/14] fix optimizations

---
 deepdanbooru/__main__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepdanbooru/__main__.py b/deepdanbooru/__main__.py
index 9f6ec2a..f741d0d 100644
--- a/deepdanbooru/__main__.py
+++ b/deepdanbooru/__main__.py
@@ -217,6 +217,7 @@ def evaluate(
 )
 @click.option("--verbose", default=False, is_flag=True)
 def conv2tflite(optimizations, project_path, model_path, save_path, verbose):
+    if not optimizations: dd.commands.convert_to_tflite_from_from_saved_model(project_path, model_path, save_path, verbose=verbose)
     dd.commands.convert_to_tflite_from_from_saved_model(project_path, model_path, save_path, optimizations, verbose=verbose)
 
 

From 976fa70e61ea64581ccd4beb4ed8c75a99e6100c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?=
 <41315874+fumiama@users.noreply.github.com>
Date: Fri, 14 Oct 2022 11:09:58 +0800
Subject: [PATCH 03/14] fix optimizations

---
 deepdanbooru/__main__.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/deepdanbooru/__main__.py b/deepdanbooru/__main__.py
index f741d0d..73ea546 100644
--- a/deepdanbooru/__main__.py
+++ b/deepdanbooru/__main__.py
@@ -3,6 +3,7 @@
 import click
 
 import deepdanbooru as dd
+import tensorflow.lite as tflite
 
 __version__ = "1.0.0"
 
@@ -197,11 +198,6 @@ def evaluate(
     )
 
 @main.command("conv2tflite", help="Convert saved model into tflite model.")
-@click.argument(
-    "optimizations",
-    nargs=-1,
-    type=click.STRING,
-)
 @click.option(
     "--project-path",
     type=click.Path(exists=True, resolve_path=True, file_okay=False, dir_okay=True),
@@ -215,10 +211,16 @@ def evaluate(
     "--save-path",
     type=click.Path(resolve_path=True, file_okay=True, dir_okay=False),
 )
+@click.option("--optimize-default", default=True, is_flag=True)
+@click.option("--optimize-experimental-sparsity", default=False, is_flag=True)
 @click.option("--verbose", default=False, is_flag=True)
-def conv2tflite(optimizations, project_path, model_path, save_path, verbose):
-    if not optimizations: dd.commands.convert_to_tflite_from_from_saved_model(project_path, model_path, save_path, verbose=verbose)
-    dd.commands.convert_to_tflite_from_from_saved_model(project_path, model_path, save_path, optimizations, verbose=verbose)
+def conv2tflite(project_path, model_path, save_path, optimize_default, optimize_experimental_sparsity, verbose):
+    if not optimize_default and not optimize_experimental_sparsity:
+        raise Exception("optimization method must be specified")
+    op = []
+    if optimize_default: op = [tflite.Optimize.DEFAULT]
+    if optimize_experimental_sparsity: op.append(tflite.Optimize.EXPERIMENTAL_SPARSITY)
+    dd.commands.convert_to_tflite_from_from_saved_model(project_path, model_path, save_path, op, verbose=verbose)
 
 
 if __name__ == "__main__":

From e087e318c360e6c818d3f366bd9309429b21e252 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?=
 <41315874+fumiama@users.noreply.github.com>
Date: Fri, 14 Oct 2022 14:39:56 +0800
Subject: [PATCH 04/14] feat: set tf log level to FATAL by defalut

---
 deepdanbooru/__init__.py                   |  6 ++++++
 deepdanbooru/__main__.py                   | 10 ++++++++--
 deepdanbooru/commands/convert_to_tflite.py |  4 ++--
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/deepdanbooru/__init__.py b/deepdanbooru/__init__.py
index 6cb06ea..674fc93 100644
--- a/deepdanbooru/__init__.py
+++ b/deepdanbooru/__init__.py
@@ -1,3 +1,9 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
 import deepdanbooru.commands
 import deepdanbooru.data
 import deepdanbooru.extra
diff --git a/deepdanbooru/__main__.py b/deepdanbooru/__main__.py
index 73ea546..a5ee90a 100644
--- a/deepdanbooru/__main__.py
+++ b/deepdanbooru/__main__.py
@@ -1,7 +1,7 @@
 import sys
-
 import click
-
+import warnings
+import os
 import deepdanbooru as dd
 import tensorflow.lite as tflite
 
@@ -184,6 +184,9 @@ def evaluate(
     folder_filters,
     verbose,
 ):
+    if verbose:
+        warnings.filterwarnings("always")
+        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0'
     dd.commands.evaluate(
         target_paths,
         project_path,
@@ -215,6 +218,9 @@ def evaluate(
 @click.option("--optimize-experimental-sparsity", default=False, is_flag=True)
 @click.option("--verbose", default=False, is_flag=True)
 def conv2tflite(project_path, model_path, save_path, optimize_default, optimize_experimental_sparsity, verbose):
+    if verbose:
+        warnings.filterwarnings("always")
+        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0'
     if not optimize_default and not optimize_experimental_sparsity:
         raise Exception("optimization method must be specified")
     op = []
diff --git a/deepdanbooru/commands/convert_to_tflite.py b/deepdanbooru/commands/convert_to_tflite.py
index db2e32f..59d34f1 100644
--- a/deepdanbooru/commands/convert_to_tflite.py
+++ b/deepdanbooru/commands/convert_to_tflite.py
@@ -24,14 +24,14 @@ def convert_to_tflite_from_from_saved_model(
         model = dd.project.load_model_from_project(project_path)
 
     if verbose:
-            print(f"Converting ...")
+        print("Converting ...")
 
     converter = tf.lite.TFLiteConverter.from_keras_model(model)
     converter.optimizations = optimizations
     tflite_model = converter.convert()
 
     if verbose:
-        print(f"Saving ...")
+        print("Saving ...")
 
     with open(save_path, "wb") as f:
         f.write(tflite_model)

From 110e6ee878f77684596c32c882ca9a55f73db7a3 Mon Sep 17 00:00:00 2001
From: chavinlo <85657083+chavinlo@users.noreply.github.com>
Date: Fri, 21 Oct 2022 00:04:05 -0500
Subject: [PATCH 05/14] add txt tag save

---
 deepdanbooru/__main__.py          |  2 +-
 deepdanbooru/commands/evaluate.py | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/deepdanbooru/__main__.py b/deepdanbooru/__main__.py
index 68b3388..c30fa20 100644
--- a/deepdanbooru/__main__.py
+++ b/deepdanbooru/__main__.py
@@ -177,7 +177,7 @@ def grad_cam(project_path, target_path, output_path, threshold):
 )
 @click.option("--verbose", default=False, is_flag=True)
 def evaluate(
-    target_paths,
+    target_paths, # I guess its this one
     project_path,
     model_path,
     tags_path,
diff --git a/deepdanbooru/commands/evaluate.py b/deepdanbooru/commands/evaluate.py
index a6b6bd8..acc6f05 100644
--- a/deepdanbooru/commands/evaluate.py
+++ b/deepdanbooru/commands/evaluate.py
@@ -6,6 +6,10 @@
 
 import deepdanbooru as dd
 
+def save_txt(txt_path, list):
+    with open(txt_path, 'w') as writer:
+        for i in list:
+            writer.write(i + " , ")
 
 def evaluate_image(
     image_input: Union[str, six.BytesIO], model: Any, tags: List[str], threshold: float
@@ -30,7 +34,7 @@ def evaluate_image(
 
 
 def evaluate(
-    target_paths,
+    target_paths, #this
     project_path,
     model_path,
     tags_path,
@@ -83,8 +87,11 @@ def evaluate(
         tags = dd.project.load_tags_from_project(project_path)
 
     for image_path in target_image_paths:
-        print(f"Tags of {image_path}:")
+        print(f"Tags of {image_path}:") #yup!
+        tag_list = []
         for tag, score in evaluate_image(image_path, model, tags, threshold):
             print(f"({score:05.3f}) {tag}")
-
+            tag_list.append(tag)
+        txt_file_path = str(os.path.splitext(image_path)[0]) + ".txt"
+        save_txt(txt_file_path, tag_list)
         print()

From b0e8199a6e102af3547221e39129d254dcd037a3 Mon Sep 17 00:00:00 2001
From: chavinlo <85657083+chavinlo@users.noreply.github.com>
Date: Fri, 21 Oct 2022 00:16:50 -0500
Subject: [PATCH 06/14] add option for it and better saving format

---
 deepdanbooru/__main__.py          |  8 ++++++++
 deepdanbooru/commands/evaluate.py | 18 +++++++++++++-----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/deepdanbooru/__main__.py b/deepdanbooru/__main__.py
index c30fa20..1fec40c 100644
--- a/deepdanbooru/__main__.py
+++ b/deepdanbooru/__main__.py
@@ -170,6 +170,12 @@ def grad_cam(project_path, target_path, output_path, threshold):
     is_flag=True,
     help="If this option is enabled, TARGET_PATHS can be folder path and all images (using --folder-filters) in that folder is estimated recursively. If there are file and folder which has same name, the file is skipped and only folder is used.",
 )
+@click.option(
+    "--save-txt",
+    default=False,
+    is_flag=True,
+    help="Enable this option to save tags to a txt file with the same filename.",
+)
 @click.option(
     "--folder-filters",
     default="*.[Pp][Nn][Gg],*.[Jj][Pp][Gg],*.[Jj][Pp][Ee][Gg],*.[Gg][Ii][Ff]",
@@ -185,6 +191,7 @@ def evaluate(
     allow_gpu,
     compile_model,
     allow_folder,
+    save_txt,
     folder_filters,
     verbose,
 ):
@@ -197,6 +204,7 @@ def evaluate(
         allow_gpu,
         compile_model,
         allow_folder,
+        save_txt,
         folder_filters,
         verbose,
     )
diff --git a/deepdanbooru/commands/evaluate.py b/deepdanbooru/commands/evaluate.py
index acc6f05..02af3a7 100644
--- a/deepdanbooru/commands/evaluate.py
+++ b/deepdanbooru/commands/evaluate.py
@@ -7,9 +7,15 @@
 import deepdanbooru as dd
 
 def save_txt(txt_path, list):
+    last_index = len(list)-1
+    last_tag = list[last_index]
     with open(txt_path, 'w') as writer:
         for i in list:
-            writer.write(i + " , ")
+            if last_tag is i:
+                writer.write(i)
+                writer.close()
+            else:
+                writer.write(i + ", ")
 
 def evaluate_image(
     image_input: Union[str, six.BytesIO], model: Any, tags: List[str], threshold: float
@@ -42,6 +48,7 @@ def evaluate(
     allow_gpu,
     compile_model,
     allow_folder,
+    save_txt,
     folder_filters,
     verbose,
 ):
@@ -88,10 +95,11 @@ def evaluate(
 
     for image_path in target_image_paths:
         print(f"Tags of {image_path}:") #yup!
-        tag_list = []
+        if save_txt: tag_list = []
         for tag, score in evaluate_image(image_path, model, tags, threshold):
             print(f"({score:05.3f}) {tag}")
-            tag_list.append(tag)
-        txt_file_path = str(os.path.splitext(image_path)[0]) + ".txt"
-        save_txt(txt_file_path, tag_list)
+            if save_txt: tag_list.append(tag)
+        if save_txt:
+            txt_file_path = str(os.path.splitext(image_path)[0]) + ".txt"
+            save_txt(txt_file_path, tag_list)
         print()

From ab1382ca700483aba161ad614fb72bf92d5fbe65 Mon Sep 17 00:00:00 2001
From: chavinlo <85657083+chavinlo@users.noreply.github.com>
Date: Fri, 21 Oct 2022 00:27:17 -0500
Subject: [PATCH 07/14] silly mistake

---
 deepdanbooru/commands/evaluate.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/deepdanbooru/commands/evaluate.py b/deepdanbooru/commands/evaluate.py
index 02af3a7..94936e8 100644
--- a/deepdanbooru/commands/evaluate.py
+++ b/deepdanbooru/commands/evaluate.py
@@ -6,16 +6,17 @@
 
 import deepdanbooru as dd
 
-def save_txt(txt_path, list):
+def save_txt_file(txt_path, list):
     last_index = len(list)-1
     last_tag = list[last_index]
     with open(txt_path, 'w') as writer:
         for i in list:
-            if last_tag is i:
+            if last_tag == i:
                 writer.write(i)
                 writer.close()
             else:
                 writer.write(i + ", ")
+    print("Saved text file.")
 
 def evaluate_image(
     image_input: Union[str, six.BytesIO], model: Any, tags: List[str], threshold: float
@@ -101,5 +102,5 @@ def evaluate(
             if save_txt: tag_list.append(tag)
         if save_txt:
             txt_file_path = str(os.path.splitext(image_path)[0]) + ".txt"
-            save_txt(txt_file_path, tag_list)
+            save_txt_file(txt_file_path, tag_list)
         print()

From 57676fe8fcf482f98881e3d386e76bbd26f88c83 Mon Sep 17 00:00:00 2001
From: Kichang Kim <kichangkim@hotmail.com>
Date: Fri, 6 Jan 2023 12:56:14 +0900
Subject: [PATCH 08/14] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9d72d06..20683af 100644
--- a/README.md
+++ b/README.md
@@ -38,9 +38,9 @@ To install it with tensorflow, add `tensorflow` extra package.
 ```
 > deepdanbooru create-project [your_project_folder]
 ```
-3. Prepare tag list. If you want to use latest tags, use following command. It downloads tag from Danbooru server.
+3. Prepare tag list. If you want to use latest tags, use following command. It downloads tag from Danbooru server. (Need Danbooru account and API key)
 ```
-> deepdanbooru download-tags [your_project_folder]
+> deepdanbooru download-tags [your_project_folder] --login [your_danbooru_account] --api-key [your_danbooru_api_key]
 ```
 4. (Option) Filtering dataset. If you want to train with optional tags (rating and score), you should convert it as system tags.
 ```

From 81da0587a9cbb7e3b6173ffee4ab201f2238bfe1 Mon Sep 17 00:00:00 2001
From: Kichang Kim <kichangkim@hotmail.com>
Date: Fri, 6 Jan 2023 22:57:40 +0900
Subject: [PATCH 09/14] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 20683af..d091052 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ To install it with tensorflow, add `tensorflow` extra package.
 ```
 3. Prepare tag list. If you want to use latest tags, use following command. It downloads tag from Danbooru server. (Need Danbooru account and API key)
 ```
-> deepdanbooru download-tags [your_project_folder] --login [your_danbooru_account] --api-key [your_danbooru_api_key]
+> deepdanbooru download-tags [your_project_folder] --username [your_danbooru_account] --api-key [your_danbooru_api_key]
 ```
 4. (Option) Filtering dataset. If you want to train with optional tags (rating and score), you should convert it as system tags.
 ```

From b95ad56c50326017764607cd1f1463f8c3a22731 Mon Sep 17 00:00:00 2001
From: Kichang Kim <kichangkim@hotmail.com>
Date: Fri, 6 Jan 2023 23:05:19 +0900
Subject: [PATCH 10/14] Fix : Add User-Agent to headers when downloading tags.
 (#80)

---
 deepdanbooru/commands/download_tags.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deepdanbooru/commands/download_tags.py b/deepdanbooru/commands/download_tags.py
index 4b4e053..d3b0441 100644
--- a/deepdanbooru/commands/download_tags.py
+++ b/deepdanbooru/commands/download_tags.py
@@ -34,6 +34,7 @@ def download_category_tags(
     }
 
     request_url = "https://danbooru.donmai.us/tags.json"
+    headers={'User-Agent': 'Mozilla/5.0'}
 
     tags = set()
 
@@ -41,6 +42,7 @@ def download_category_tags(
         response = requests.get(
             request_url,
             params=parameters,
+            headers=headers,
         )
         response_json = response.json()
 

From 05eb3c39b0fae43e3caf39df801615fe79b27c2f Mon Sep 17 00:00:00 2001
From: Kichang Kim <kichangkim@hotmail.com>
Date: Sun, 22 Jan 2023 10:46:53 +0900
Subject: [PATCH 11/14] Update README.md

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d091052..4f597af 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ To install it with tensorflow, add `tensorflow` extra package.
 ```
 
 ## Dataset Structure
-DeepDanbooru uses following folder structure for input dataset. SQLite file can be any name, but must be located in same folder to `images` folder.
+DeepDanbooru uses following folder structure for input dataset. SQLite file can be any name, but must be located in same folder to `images` folder. All of image files are located in sub-folder which named first 2 characters of its filename.
 ```
 MyDataset/
 ├── images/
@@ -65,8 +65,10 @@ MyDataset/
 │   │   ├── 00000000000000000000000000000000.jpg
 │   │   ├── ...
 │   ├── 01/
+│   │   ├── 01000000000000000000000000000000.jpg
 │   │   ├── ...
 │   └── ff/
+│       ├── ff000000000000000000000000000000.jpg
 │       ├── ...
 └── my-dataset.sqlite
 ```

From ab90ef3b8d792bfccf8c4c40b3dc40bc19e82d39 Mon Sep 17 00:00:00 2001
From: Kichang Kim <kichangkim@hotmail.com>
Date: Mon, 8 Jul 2024 21:38:52 +0900
Subject: [PATCH 12/14] Fix : headers does not need anymore.

---
 deepdanbooru/commands/download_tags.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/deepdanbooru/commands/download_tags.py b/deepdanbooru/commands/download_tags.py
index d3b0441..cdcedef 100644
--- a/deepdanbooru/commands/download_tags.py
+++ b/deepdanbooru/commands/download_tags.py
@@ -34,7 +34,6 @@ def download_category_tags(
     }
 
     request_url = "https://danbooru.donmai.us/tags.json"
-    headers={'User-Agent': 'Mozilla/5.0'}
 
     tags = set()
 
@@ -42,8 +41,8 @@ def download_category_tags(
         response = requests.get(
             request_url,
             params=parameters,
-            headers=headers,
         )
+
         response_json = response.json()
 
         response_tags = [

From b96a1bb96dc7322bf04fc8d3ffbbcbc2f112b568 Mon Sep 17 00:00:00 2001
From: Kichang Kim <kichangkim@hotmail.com>
Date: Tue, 27 Aug 2024 08:25:45 +0900
Subject: [PATCH 13/14] Fix : temporary fix for keras3 (removed reset_metrics
 argument)

---
 deepdanbooru/commands/train_project.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepdanbooru/commands/train_project.py b/deepdanbooru/commands/train_project.py
index c7260b6..77a4fa6 100644
--- a/deepdanbooru/commands/train_project.py
+++ b/deepdanbooru/commands/train_project.py
@@ -231,7 +231,7 @@ def train_project(project_path, source_model):
                 sample_count = x_train.shape[0]
 
                 step_result = model.train_on_batch(
-                    x_train, y_train, reset_metrics=False
+                    x_train, y_train
                 )
 
                 used_minibatch.assign_add(1)

From 98d9315ab702e177de706825f9cee981c9afb928 Mon Sep 17 00:00:00 2001
From: Kichang Kim <kichangkim@hotmail.com>
Date: Tue, 27 Aug 2024 23:36:41 +0900
Subject: [PATCH 14/14] Update : Updated to Python 3.11, TensorFlow 2.17 and
 latest dependencies. Change : Default model serialization format is changed
 to Keras v3.

---
 README.md                              | 18 +++++++++---------
 deepdanbooru/commands/train_project.py | 12 ++++++------
 deepdanbooru/data/dataset_wrapper.py   |  8 ++++----
 deepdanbooru/project/project.py        | 12 ++++++++++--
 requirements.txt                       | 14 +++++++-------
 5 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/README.md b/README.md
index 4f597af..37e0fb7 100644
--- a/README.md
+++ b/README.md
@@ -1,19 +1,19 @@
 # DeepDanbooru
-[![Python](https://img.shields.io/badge/python-3.6-green)](https://www.python.org/doc/versions/)
+[![Python](https://img.shields.io/badge/python-3.11-green)](https://www.python.org/doc/versions/)
 [![GitHub](https://img.shields.io/github/license/KichangKim/DeepDanbooru)](https://opensource.org/licenses/MIT)
 [![Web](https://img.shields.io/badge/web%20demo-20200915-brightgreen)](http://kanotype.iptime.org:8003/deepdanbooru/)
 
 **DeepDanbooru** is anime-style girl image tag estimation system. You can estimate your images on my live demo site, [DeepDanbooru Web](http://dev.kanotype.net:8003/deepdanbooru/).
 
 ## Requirements
-DeepDanbooru is written by Python 3.7. Following packages are need to be installed.
-- tensorflow>=2.7.0
-- tensorflow-io>=2.22.0
-- Click>=7.0
-- numpy>=1.16.2
-- requests>=2.22.0
-- scikit-image>=0.15.0
-- six>=1.13.0
+DeepDanbooru is written by Python 3.11. Following packages are need to be installed.
+- Click>=8.1.7
+- numpy>=1.26.4
+- requests>=2.32.3
+- scikit-image>=0.24.0
+- six>=1.16.0
+- tensorflow>=2.17.0
+- tensorflow-io>=0.31.0
 
 Or just use `requirements.txt`.
 ```
diff --git a/deepdanbooru/commands/train_project.py b/deepdanbooru/commands/train_project.py
index 77a4fa6..a8c6e2c 100644
--- a/deepdanbooru/commands/train_project.py
+++ b/deepdanbooru/commands/train_project.py
@@ -206,7 +206,7 @@ def train_project(project_path, source_model):
                     learning_rate = learning_rate_per_epoch["learning_rate"]
         print(f"Trying to change learning rate to {learning_rate} ...")
         optimizer.learning_rate.assign(learning_rate)
-        print(f"Learning rate is changed to {optimizer.learning_rate} ...")
+        tf.print(f"Learning rate is changed to", optimizer.learning_rate, "...")
 
         while int(offset) < epoch_size:
             image_records_slice = image_records[
@@ -292,17 +292,17 @@ def train_project(project_path, source_model):
         if export_model_per_epoch == 0 or int(used_epoch) % export_model_per_epoch == 0:
             print(f"Saving model ... (per epoch {export_model_per_epoch})")
             export_path = os.path.join(
-                project_path, f"model-{model_type}.h5.e{int(used_epoch)}"
+                project_path, f"model-{model_type}.e{int(used_epoch)}.keras"
             )
-            model.save(export_path, include_optimizer=False, save_format="h5")
+            model.save(export_path, include_optimizer=False)
 
             if use_mixed_precision:
                 export_model_as_float32(
-                    model_float32, checkpoint_path, export_path + ".float32.h5"
+                    model_float32, checkpoint_path, export_path + ".float32.keras"
                 )
 
     print("Saving model ...")
-    model_path = os.path.join(project_path, f"model-{model_type}.h5")
+    model_path = os.path.join(project_path, f"model-{model_type}.keras")
 
     # tf.keras.experimental.export_saved_model throw exception now
     # see https://github.com/tensorflow/tensorflow/issues/27112
@@ -310,7 +310,7 @@ def train_project(project_path, source_model):
 
     if use_mixed_precision:
         export_model_as_float32(
-            model_float32, checkpoint_path, model_path + ".float32.h5"
+            model_float32, checkpoint_path, model_path + ".float32.keras"
         )
 
     print("Training is complete.")
diff --git a/deepdanbooru/data/dataset_wrapper.py b/deepdanbooru/data/dataset_wrapper.py
index 2bcf27c..94e9ab0 100644
--- a/deepdanbooru/data/dataset_wrapper.py
+++ b/deepdanbooru/data/dataset_wrapper.py
@@ -26,15 +26,15 @@ def __init__(
     def get_dataset(self, minibatch_size):
         dataset = tf.data.Dataset.from_tensor_slices(self.inputs)
         dataset = dataset.map(
-            self.map_load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE
+            self.map_load_image, num_parallel_calls=tf.data.AUTOTUNE
         )
-        dataset = dataset.apply(tf.data.experimental.ignore_errors())
+        dataset = dataset.ignore_errors()
         dataset = dataset.map(
             self.map_transform_image_and_label,
-            num_parallel_calls=tf.data.experimental.AUTOTUNE,
+            num_parallel_calls=tf.data.AUTOTUNE,
         )
         dataset = dataset.batch(minibatch_size)
-        dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
+        dataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)
         # dataset = dataset.apply(
         #    tf.data.experimental.prefetch_to_device('/device:GPU:0'))
 
diff --git a/deepdanbooru/project/project.py b/deepdanbooru/project/project.py
index b56a8d8..271e3cb 100644
--- a/deepdanbooru/project/project.py
+++ b/deepdanbooru/project/project.py
@@ -29,7 +29,11 @@ def load_project(project_path):
     tags = dd.data.load_tags_from_project(project_path)
 
     model_type = project_context["model"]
-    model_path = os.path.join(project_path, f"model-{model_type}.h5")
+    model_path = os.path.join(project_path, f"model-{model_type}.keras")
+
+    if not os.path.isfile(model_path):
+        model_path = os.path.join(project_path, f"model-{model_type}.h5")
+
     model = tf.keras.models.load_model(model_path)
 
     return project_context, model, tags
@@ -40,7 +44,11 @@ def load_model_from_project(project_path, compile_model=True):
     project_context = dd.io.deserialize_from_json(project_context_path)
 
     model_type = project_context["model"]
-    model_path = os.path.join(project_path, f"model-{model_type}.h5")
+    model_path = os.path.join(project_path, f"model-{model_type}.keras")
+
+    if not os.path.isfile(model_path):
+        model_path = os.path.join(project_path, f"model-{model_type}.h5")
+
     model = tf.keras.models.load_model(model_path, compile=compile_model)
 
     return model
diff --git a/requirements.txt b/requirements.txt
index d9078da..38fff21 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
-Click>=7.0
-numpy>=1.16.2
-scikit-image>=0.15.0
-tensorflow>=2.7.0
-tensorflow-io>=0.22.0
-requests>=2.22.0
-six>=1.13.0
+Click>=8.1.7
+numpy>=1.26.4
+scikit-image>=0.24.0
+tensorflow>=2.17.0
+tensorflow-io>=0.31.0
+requests>=2.32.3
+six>=1.16.0