Fix some bug for metric learning model, add the export_model.py and clean the code.

PaddleCV/metric_learning/_ce.py

@@ -2,12 +2,12 @@
 
 import os
 import sys
+from kpi import CostKpi, AccKpi
 sys.path.append(os.environ['ceroot'])
-from kpi import CostKpi, DurationKpi, AccKpi
 
 # NOTE kpi.py should shared in models in some way!!!!
 
-train_cost_kpi = CostKpi('train_cost', 0.02 0, actived=True)
+train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True)
 test_recall_kpi = AccKpi('test_recall', 0.02, 0, actived=True)
 
 tracking_kpis = [

PaddleCV/metric_learning/eval.py

@@ -17,7 +17,6 @@
 
 import os
 import sys
-import math
 import time
 import argparse
 import functools
@@ -110,13 +109,12 @@ def if_exist(var):
 
 
 def main():
+    paddle.enable_static()
     args = parser.parse_args()
     print_arguments(args)
     check_cuda(args.use_gpu)
     eval(args)
 
 
 if __name__ == '__main__':
-    import paddle
-    paddle.enable_static()
     main()

PaddleCV/metric_learning/export_model.py

@@ -0,0 +1,90 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import argparse
+import functools
+import paddle
+import paddle.fluid as fluid
+import models
+from utility import add_arguments, print_arguments, check_cuda
+
+parser = argparse.ArgumentParser(description=__doc__)
+add_arg = functools.partial(add_arguments, argparser=parser)
+# yapf: disable
+add_arg('model', str, "ResNet50", "Set the network to use.")
+add_arg('embedding_size', int, 0, "Embedding size.")
+add_arg('image_shape', str, "3,224,224", "Input image size.")
+add_arg('use_gpu', bool, True, "Whether to use GPU or not.")
+add_arg('pretrained_model', str, None, "Whether to use pretrained model.")
+add_arg('model_save_dir', str, 'save_inference_model', "Whether to save the inference model.")
+# yapf: enable
+
+model_list = [m for m in dir(models) if "__" not in m]
+
+
+def save_inference_model(args):
+    # parameters from arguments
+    model_name = args.model
+    pretrained_model = args.pretrained_model
+    model_save_dir = args.model_save_dir
+    image_shape = [int(m) for m in args.image_shape.split(",")]
+
+    assert model_name in model_list, "{} is not in lists: {}".format(args.model,
+                                                                     model_list)
+
+    image = fluid.data(name='image', shape=[None] + image_shape, dtype='float32')
+
+    # model definition
+    model = models.__dict__[model_name]()
+    out = model.net(input=image, embedding_size=args.embedding_size)
+
+    test_program = fluid.default_main_program().clone(for_test=True)
+
+    place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
+    exe = fluid.Executor(place)
+    exe.run(fluid.default_startup_program())
+
+    if pretrained_model:
+        def if_exist(var):
+            return os.path.exists(os.path.join(pretrained_model, var.name))
+
+        fluid.load(model_path=pretrained_model, program=test_program, executor=exe)
+
+        print('Saving the inference model...')
+
+        fluid.io.save_inference_model(
+            dirname=model_save_dir, 
+            feeded_var_names=['image'], 
+            target_vars=[out], 
+            executor=exe, 
+            params_filename='__params__'
+        )
+        print('Finish.')
+
+    else:
+        print('Can\'t load the pretrained_model. Please set the true pretrained_model dir.')
+
+def main():
+    paddle.enable_static()
+    args = parser.parse_args()
+    print_arguments(args)
+    check_cuda(args.use_gpu)
+    save_inference_model(args)
+
+if __name__ == '__main__':
+    main()

PaddleCV/metric_learning/imgtool.py

@@ -76,7 +76,7 @@ def crop_image(img, target_size, center):
     """ crop_image """
     height, width = img.shape[:2]
     size = target_size
-    if center == True:
+    if center is True:
         w_start = (width - size) // 2
         h_start = (height - size) // 2
     else:

PaddleCV/metric_learning/infer.py

@@ -17,11 +17,8 @@
 
 import os
 import sys
-import math
-import time
 import argparse
 import functools
-import numpy as np
 import paddle
 import paddle.fluid as fluid
 import models
@@ -92,13 +89,12 @@ def if_exist(var):
 
 
 def main():
+    paddle.enable_static()
     args = parser.parse_args()
     print_arguments(args)
     check_cuda(args.use_gpu)
     infer(args)
 
 
 if __name__ == '__main__':
-    import paddle
-    paddle.enable_static()
     main()

PaddleCV/metric_learning/losses/commonfunc.py

@@ -15,15 +15,12 @@
 from __future__ import division
 from __future__ import print_function
 
-import os
 import numpy as np
-import paddle as paddle
 import paddle.fluid as fluid
 
 def generate_index(batch_size, samples_each_class):
     a = np.arange(0, batch_size * batch_size) # N*N x 1
     a = a.reshape(-1, batch_size) # N x N
-    steps = batch_size // samples_each_class
     res = []
     for i in range(batch_size):
         step = i // samples_each_class

PaddleCV/metric_learning/models/resnet_embedding.py

@@ -11,9 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import paddle
 import paddle.fluid as fluid
-import math
 from paddle.fluid.param_attr import ParamAttr
 
 __all__ = ["ResNet", "ResNet50", "ResNet101", "ResNet152"]

PaddleCV/metric_learning/reader.py

@@ -15,12 +15,8 @@
 from __future__ import division
 from __future__ import print_function
 
-import os
-import math
 import random
 import functools
-import numpy as np
-import paddle
 from imgtool import process_image
 import paddle.fluid as fluid
 

PaddleCV/metric_learning/train_elem.py

@@ -17,13 +17,10 @@
 
 import os
 import sys
-import math
 import time
 import logging
 import argparse
 import functools
-import threading
-import subprocess
 import numpy as np
 import paddle
 import paddle.fluid as fluid
@@ -68,7 +65,7 @@
 def optimizer_setting(params):
     ls = params["learning_strategy"]
     assert ls["name"] == "piecewise_decay", \
-           "learning rate strategy must be {}, but got {}".format("piecewise_decay", lr["name"])
+           "learning rate strategy must be {}, but got {}".format("piecewise_decay", ls["name"])
 
     bd = [int(e) for e in ls["lr_steps"].split(',')]
     base_lr = params["lr"]
@@ -148,6 +145,9 @@ def train_async(args):
     pretrained_model = args.pretrained_model
     model_save_dir = args.model_save_dir
 
+    if not os.path.exists(model_save_dir):
+        os.mkdir(model_save_dir)
+
     startup_prog = fluid.Program()
     train_prog = fluid.Program()
     tmp_prog = fluid.Program()
@@ -275,10 +275,7 @@ def train_async(args):
                 sys.stdout.flush()
 
             if iter_no % args.save_iter_step == 0 and iter_no != 0:
-                model_path = os.path.join(model_save_dir + '/' + model_name,
-                                          str(iter_no))
-                if not os.path.isdir(model_path):
-                    os.makedirs(model_path)
+                model_path = os.path.join(model_save_dir, model_name, str(iter_no))
                 fluid.save(program=train_prog, model_path=model_path)
 
             iter_no += 1
@@ -302,13 +299,12 @@ def initlogging():
 
 
 def main():
+    paddle.enable_static()
     args = parser.parse_args()
     print_arguments(args)
     check_cuda(args.use_gpu)
     train_async(args)
 
 
 if __name__ == '__main__':
-    import paddle
-    paddle.enable_static()
     main()

PaddleCV/metric_learning/train_pair.py

@@ -17,13 +17,10 @@
 
 import os
 import sys
-import math
 import time
 import logging
 import argparse
 import functools
-import threading
-import subprocess
 import numpy as np
 import paddle
 import paddle.fluid as fluid
@@ -69,7 +66,7 @@
 def optimizer_setting(params):
     ls = params["learning_strategy"]
     assert ls["name"] == "piecewise_decay", \
-           "learning rate strategy must be {}, but got {}".format("piecewise_decay", lr["name"])
+           "learning rate strategy must be {}, but got {}".format("piecewise_decay", ls["name"])
 
     bd = [int(e) for e in ls["lr_steps"].split(',')]
     base_lr = params["lr"]
@@ -153,7 +150,8 @@ def train_async(args):
     checkpoint = args.checkpoint
     pretrained_model = args.pretrained_model
     model_save_dir = args.model_save_dir
-
+    if not os.path.exists(model_save_dir):
+        os.mkdir(model_save_dir)
     startup_prog = fluid.Program()
     train_prog = fluid.Program()
     tmp_prog = fluid.Program()
@@ -271,10 +269,7 @@ def train_async(args):
                 sys.stdout.flush()
 
             if iter_no % args.save_iter_step == 0 and iter_no != 0:
-                model_path = os.path.join(model_save_dir + '/' + model_name,
-                                          str(iter_no))
-                if not os.path.isdir(model_path):
-                    os.makedirs(model_path)
+                model_path = os.path.join(model_save_dir, model_name, str(iter_no))
                 fluid.save(program=train_prog, model_path=model_path)
 
             iter_no += 1
@@ -292,13 +287,12 @@ def initlogging():
 
 
 def main():
+    paddle.enable_static()
     args = parser.parse_args()
     print_arguments(args)
     check_cuda(args.use_gpu)
     train_async(args)
 
 
 if __name__ == '__main__':
-    import paddle
-    paddle.enable_static()
     main()

PaddleCV/metric_learning/utility.py

@@ -18,14 +18,16 @@
 from __future__ import print_function
 
 import os
+import re
 import six
 import time
+import shutil
+import tempfile
 import subprocess
 import distutils.util
 import numpy as np
 import sys
 import paddle.fluid as fluid
-from paddle.fluid import core
 import multiprocessing as mp
 
 
@@ -171,10 +173,11 @@ def check_cuda(use_cuda, err = \
     Please: 1. Install paddlepaddle-gpu to run your models on GPU or 2. Set use_cuda = False to run models on CPU.\n"
                                                                                                                      ):
     try:
-        if use_cuda == True and fluid.is_compiled_with_cuda() == False:
+        if use_cuda is True and fluid.is_compiled_with_cuda() is False:
             print(err)
             sys.exit(1)
     except Exception as e:
+        print(e)
         pass