From 4a1714b64b67df499851f2440b95dd45836c2831 Mon Sep 17 00:00:00 2001 From: jm12138 <2286040843@qq.com> Date: Wed, 11 Nov 2020 22:19:23 +0800 Subject: [PATCH 1/5] Add the export_model.py Add the export_model.py --- PaddleCV/metric_learning/export_model.py | 90 ++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 PaddleCV/metric_learning/export_model.py diff --git a/PaddleCV/metric_learning/export_model.py b/PaddleCV/metric_learning/export_model.py new file mode 100644 index 0000000000..3728071706 --- /dev/null +++ b/PaddleCV/metric_learning/export_model.py @@ -0,0 +1,90 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import argparse +import functools +import paddle +import paddle.fluid as fluid +import models +from utility import add_arguments, print_arguments, check_cuda + +parser = argparse.ArgumentParser(description=__doc__) +add_arg = functools.partial(add_arguments, argparser=parser) +# yapf: disable +add_arg('model', str, "ResNet50", "Set the network to use.") +add_arg('embedding_size', int, 0, "Embedding size.") +add_arg('image_shape', str, "3,224,224", "Input image size.") +add_arg('use_gpu', bool, True, "Whether to use GPU or not.") +add_arg('pretrained_model', str, None, "Whether to use pretrained model.") +add_arg('model_save_dir', str, 'save_inference_model', "Whether to save the inference model.") +# yapf: enable + +model_list = [m for m in dir(models) if "__" not in m] + + +def save_inference_model(args): + # parameters from arguments + model_name = args.model + pretrained_model = args.pretrained_model + model_save_dir = args.model_save_dir + image_shape = [int(m) for m in args.image_shape.split(",")] + + assert model_name in model_list, "{} is not in lists: {}".format(args.model, + model_list) + + image = fluid.data(name='image', shape=[None] + image_shape, dtype='float32') + + # model definition + model = models.__dict__[model_name]() + out = model.net(input=image, embedding_size=args.embedding_size) + + test_program = fluid.default_main_program().clone(for_test=True) + + place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) + + if pretrained_model: + def if_exist(var): + return os.path.exists(os.path.join(pretrained_model, var.name)) + + fluid.load(model_path=pretrained_model, program=test_program, executor=exe) + + print('Saving the inference model...') + + fluid.io.save_inference_model( + dirname=model_save_dir, + feeded_var_names=['image'], + target_vars=[out], + executor=exe, + params_filename='__params__' + ) + print('Finish.') + + else: + print('Please set the pretrained_model dir.') + +def main(): + paddle.enable_static() + args = parser.parse_args() + print_arguments(args) + check_cuda(args.use_gpu) + save_inference_model(args) + +if __name__ == '__main__': + main() From a8aa191813ad04af7aadc12f1621ca9ae6750d05 Mon Sep 17 00:00:00 2001 From: jm12138 <2286040843@qq.com> Date: Wed, 11 Nov 2020 22:21:40 +0800 Subject: [PATCH 2/5] Fix some bug Fix some bug --- PaddleCV/metric_learning/_ce.py | 4 ++-- PaddleCV/metric_learning/eval.py | 4 +--- PaddleCV/metric_learning/imgtool.py | 2 +- PaddleCV/metric_learning/infer.py | 6 +----- PaddleCV/metric_learning/reader.py | 4 ---- PaddleCV/metric_learning/train_elem.py | 8 ++------ PaddleCV/metric_learning/train_pair.py | 16 +++++----------- PaddleCV/metric_learning/utility.py | 7 +++++-- 8 files changed, 17 insertions(+), 34 deletions(-) diff --git a/PaddleCV/metric_learning/_ce.py b/PaddleCV/metric_learning/_ce.py index ad1d8e4b7e..ea494e5c99 100644 --- a/PaddleCV/metric_learning/_ce.py +++ b/PaddleCV/metric_learning/_ce.py @@ -2,12 +2,12 @@ import os import sys +from kpi import CostKpi, AccKpi sys.path.append(os.environ['ceroot']) -from kpi import CostKpi, DurationKpi, AccKpi # NOTE kpi.py should shared in models in some way!!!! -train_cost_kpi = CostKpi('train_cost', 0.02 0, actived=True) +train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True) test_recall_kpi = AccKpi('test_recall', 0.02, 0, actived=True) tracking_kpis = [ diff --git a/PaddleCV/metric_learning/eval.py b/PaddleCV/metric_learning/eval.py index 051739d8f4..431a797dfe 100644 --- a/PaddleCV/metric_learning/eval.py +++ b/PaddleCV/metric_learning/eval.py @@ -17,7 +17,6 @@ import os import sys -import math import time import argparse import functools @@ -110,6 +109,7 @@ def if_exist(var): def main(): + paddle.enable_static() args = parser.parse_args() print_arguments(args) check_cuda(args.use_gpu) @@ -117,6 +117,4 @@ def main(): if __name__ == '__main__': - import paddle - paddle.enable_static() main() diff --git a/PaddleCV/metric_learning/imgtool.py b/PaddleCV/metric_learning/imgtool.py index 70d5b97fe8..075870ec20 100644 --- a/PaddleCV/metric_learning/imgtool.py +++ b/PaddleCV/metric_learning/imgtool.py @@ -76,7 +76,7 @@ def crop_image(img, target_size, center): """ crop_image """ height, width = img.shape[:2] size = target_size - if center == True: + if center is True: w_start = (width - size) // 2 h_start = (height - size) // 2 else: diff --git a/PaddleCV/metric_learning/infer.py b/PaddleCV/metric_learning/infer.py index 2a9e20c526..937fe60d9e 100644 --- a/PaddleCV/metric_learning/infer.py +++ b/PaddleCV/metric_learning/infer.py @@ -17,11 +17,8 @@ import os import sys -import math -import time import argparse import functools -import numpy as np import paddle import paddle.fluid as fluid import models @@ -92,6 +89,7 @@ def if_exist(var): def main(): + paddle.enable_static() args = parser.parse_args() print_arguments(args) check_cuda(args.use_gpu) @@ -99,6 +97,4 @@ def main(): if __name__ == '__main__': - import paddle - paddle.enable_static() main() diff --git a/PaddleCV/metric_learning/reader.py b/PaddleCV/metric_learning/reader.py index 81778ea6de..fb112e5018 100644 --- a/PaddleCV/metric_learning/reader.py +++ b/PaddleCV/metric_learning/reader.py @@ -15,12 +15,8 @@ from __future__ import division from __future__ import print_function -import os -import math import random import functools -import numpy as np -import paddle from imgtool import process_image import paddle.fluid as fluid diff --git a/PaddleCV/metric_learning/train_elem.py b/PaddleCV/metric_learning/train_elem.py index 485b623c1b..299f7b3ee1 100644 --- a/PaddleCV/metric_learning/train_elem.py +++ b/PaddleCV/metric_learning/train_elem.py @@ -17,13 +17,10 @@ import os import sys -import math import time import logging import argparse import functools -import threading -import subprocess import numpy as np import paddle import paddle.fluid as fluid @@ -68,7 +65,7 @@ def optimizer_setting(params): ls = params["learning_strategy"] assert ls["name"] == "piecewise_decay", \ - "learning rate strategy must be {}, but got {}".format("piecewise_decay", lr["name"]) + "learning rate strategy must be {}, but got {}".format("piecewise_decay", ls["name"]) bd = [int(e) for e in ls["lr_steps"].split(',')] base_lr = params["lr"] @@ -302,6 +299,7 @@ def initlogging(): def main(): + paddle.enable_static() args = parser.parse_args() print_arguments(args) check_cuda(args.use_gpu) @@ -309,6 +307,4 @@ def main(): if __name__ == '__main__': - import paddle - paddle.enable_static() main() diff --git a/PaddleCV/metric_learning/train_pair.py b/PaddleCV/metric_learning/train_pair.py index 2f3ef7a0ee..e4a353a5bd 100644 --- a/PaddleCV/metric_learning/train_pair.py +++ b/PaddleCV/metric_learning/train_pair.py @@ -17,13 +17,10 @@ import os import sys -import math import time import logging import argparse import functools -import threading -import subprocess import numpy as np import paddle import paddle.fluid as fluid @@ -69,7 +66,7 @@ def optimizer_setting(params): ls = params["learning_strategy"] assert ls["name"] == "piecewise_decay", \ - "learning rate strategy must be {}, but got {}".format("piecewise_decay", lr["name"]) + "learning rate strategy must be {}, but got {}".format("piecewise_decay", ls["name"]) bd = [int(e) for e in ls["lr_steps"].split(',')] base_lr = params["lr"] @@ -153,7 +150,8 @@ def train_async(args): checkpoint = args.checkpoint pretrained_model = args.pretrained_model model_save_dir = args.model_save_dir - + if not os.path.exists(model_save_dir): + os.mkdir(model_save_dir) startup_prog = fluid.Program() train_prog = fluid.Program() tmp_prog = fluid.Program() @@ -271,10 +269,7 @@ def train_async(args): sys.stdout.flush() if iter_no % args.save_iter_step == 0 and iter_no != 0: - model_path = os.path.join(model_save_dir + '/' + model_name, - str(iter_no)) - if not os.path.isdir(model_path): - os.makedirs(model_path) + model_path = os.path.join(model_save_dir, model_name, str(iter_no)) fluid.save(program=train_prog, model_path=model_path) iter_no += 1 @@ -292,6 +287,7 @@ def initlogging(): def main(): + paddle.enable_static() args = parser.parse_args() print_arguments(args) check_cuda(args.use_gpu) @@ -299,6 +295,4 @@ def main(): if __name__ == '__main__': - import paddle - paddle.enable_static() main() diff --git a/PaddleCV/metric_learning/utility.py b/PaddleCV/metric_learning/utility.py index ada51e428b..302d2ea10d 100644 --- a/PaddleCV/metric_learning/utility.py +++ b/PaddleCV/metric_learning/utility.py @@ -18,14 +18,16 @@ from __future__ import print_function import os +import re import six import time +import shutil +import tempfile import subprocess import distutils.util import numpy as np import sys import paddle.fluid as fluid -from paddle.fluid import core import multiprocessing as mp @@ -171,10 +173,11 @@ def check_cuda(use_cuda, err = \ Please: 1. Install paddlepaddle-gpu to run your models on GPU or 2. Set use_cuda = False to run models on CPU.\n" ): try: - if use_cuda == True and fluid.is_compiled_with_cuda() == False: + if use_cuda is True and fluid.is_compiled_with_cuda() is False: print(err) sys.exit(1) except Exception as e: + print(e) pass From dc8f777635a3faf064eac276dd2a45ec835f6d92 Mon Sep 17 00:00:00 2001 From: jm12138 <2286040843@qq.com> Date: Wed, 11 Nov 2020 22:26:36 +0800 Subject: [PATCH 3/5] Clean the code Clean the code --- PaddleCV/metric_learning/models/resnet_embedding.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/PaddleCV/metric_learning/models/resnet_embedding.py b/PaddleCV/metric_learning/models/resnet_embedding.py index d08f17474f..2b0242d3ef 100644 --- a/PaddleCV/metric_learning/models/resnet_embedding.py +++ b/PaddleCV/metric_learning/models/resnet_embedding.py @@ -11,9 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import paddle import paddle.fluid as fluid -import math from paddle.fluid.param_attr import ParamAttr __all__ = ["ResNet", "ResNet50", "ResNet101", "ResNet152"] From f647e1bf9e077ac06429ca968044cb556507a020 Mon Sep 17 00:00:00 2001 From: jm12138 <2286040843@qq.com> Date: Wed, 11 Nov 2020 22:27:24 +0800 Subject: [PATCH 4/5] Clean the code Clean the code --- PaddleCV/metric_learning/losses/commonfunc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/PaddleCV/metric_learning/losses/commonfunc.py b/PaddleCV/metric_learning/losses/commonfunc.py index d7a6c7e731..2d98bf3b14 100644 --- a/PaddleCV/metric_learning/losses/commonfunc.py +++ b/PaddleCV/metric_learning/losses/commonfunc.py @@ -15,15 +15,12 @@ from __future__ import division from __future__ import print_function -import os import numpy as np -import paddle as paddle import paddle.fluid as fluid def generate_index(batch_size, samples_each_class): a = np.arange(0, batch_size * batch_size) # N*N x 1 a = a.reshape(-1, batch_size) # N x N - steps = batch_size // samples_each_class res = [] for i in range(batch_size): step = i // samples_each_class From 74913d68e0262266bd799de9f79bc4d0175d899a Mon Sep 17 00:00:00 2001 From: jm12138 <2286040843@qq.com> Date: Thu, 12 Nov 2020 00:00:08 +0800 Subject: [PATCH 5/5] Update the code. Update the code. --- PaddleCV/metric_learning/export_model.py | 2 +- PaddleCV/metric_learning/train_elem.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PaddleCV/metric_learning/export_model.py b/PaddleCV/metric_learning/export_model.py index 3728071706..d6c5c4a427 100644 --- a/PaddleCV/metric_learning/export_model.py +++ b/PaddleCV/metric_learning/export_model.py @@ -77,7 +77,7 @@ def if_exist(var): print('Finish.') else: - print('Please set the pretrained_model dir.') + print('Can\'t load the pretrained_model. Please set the true pretrained_model dir.') def main(): paddle.enable_static() diff --git a/PaddleCV/metric_learning/train_elem.py b/PaddleCV/metric_learning/train_elem.py index 299f7b3ee1..ed8162870b 100644 --- a/PaddleCV/metric_learning/train_elem.py +++ b/PaddleCV/metric_learning/train_elem.py @@ -145,6 +145,9 @@ def train_async(args): pretrained_model = args.pretrained_model model_save_dir = args.model_save_dir + if not os.path.exists(model_save_dir): + os.mkdir(model_save_dir) + startup_prog = fluid.Program() train_prog = fluid.Program() tmp_prog = fluid.Program() @@ -272,10 +275,7 @@ def train_async(args): sys.stdout.flush() if iter_no % args.save_iter_step == 0 and iter_no != 0: - model_path = os.path.join(model_save_dir + '/' + model_name, - str(iter_no)) - if not os.path.isdir(model_path): - os.makedirs(model_path) + model_path = os.path.join(model_save_dir, model_name, str(iter_no)) fluid.save(program=train_prog, model_path=model_path) iter_no += 1