Merge pull request #9 from thalesmg/concurrency

perf: add concurrency

Dockerfile

@@ -1,4 +1,4 @@
-FROM erikvl87/languagetool
+FROM erikvl87/languagetool:5.8-dockerupdate-2
 
 # Improving the spell checker
 # http://wiki.languagetool.org/hunspell-support

run.sh

@@ -7,9 +7,22 @@ for dict in /dicts/*.txt; do
     cat "$dict" >> org/languagetool/resource/en/hunspell/spelling.txt
 done
 
-ARGS="$*"
+SCHEMA_FILE=""
+MAX_CONCURRENCY="10"
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+      -j)
+        MAX_CONCURRENCY="$2"
+        shift 2
+        ;;
+      *)
+        SCHEMA_FILE="$1"
+        shift
+        ;;
+    esac
+done
 
-if [ -z "${ARGS}" ]; then
+if [ -z "${SCHEMA_FILE}" ]; then
     echo "Missing schema file"
     exit 1
 fi
@@ -30,6 +43,6 @@ while ! curl --fail --data "language=en-US&text=a simple test" http://localhost:
     sleep 1
 done
 
-echo "Checking $ARGS..."
+echo "Checking $SCHEMA_FILE with $MAX_CONCURRENCY processes..."
 
-emqx_schema_validate "$ARGS"
+emqx_schema_validate -j "$MAX_CONCURRENCY" "$SCHEMA_FILE"

src/emqx_schema_validate.erl

@@ -1,49 +1,60 @@
 -module(emqx_schema_validate).
 
 %% API exports
--export([main/1]).
+-export([main/1, do_spellcheck_schema/1]).
 
--define(TAB, stats_tab).
+-define(DEFAULT_MAX_CONCURRENCY, 10).
 
 %%====================================================================
 %% API functions
 %%====================================================================
 
-main(["-"]) ->
+main(Args) ->
+    main(Args, ?DEFAULT_MAX_CONCURRENCY).
+
+main(["-j", N0 | Rest], _MaxConcurrency) ->
+    N = list_to_integer(N0),
+    main(Rest, N);
+main(["-"], MaxConcurrency) ->
     Binary = iolist_to_binary(read_stdio()),
-    process_data(Binary);
-main([JsonFile]) ->
+    process_data(Binary, MaxConcurrency);
+main([JsonFile], MaxConcurrency) ->
     {ok, Binary} = file:read_file(JsonFile),
-    process_data(Binary);
-main(_) ->
+    process_data(Binary, MaxConcurrency);
+main(_, _) ->
     io:format("Usage: emqx_schema_validate <path-to-json-schema-dump>~n", []),
     erlang:halt(1).
 
 %%====================================================================
 %% Internal functions
 %%====================================================================
 
-process_data(Binary) ->
+process_data(Binary, MaxConcurrency) ->
     {ok, _} = application:ensure_all_started(emqx_schema_validate),
     langtool:start(),
     Data = jsone:decode(Binary, [{object_format, map}, {keys, atom}]),
-    spellcheck_schema(Data),
+    spellcheck_schema(Data, MaxConcurrency),
     case is_ok() of
         true -> halt(0);
         false -> halt(1)
     end.
 
-spellcheck_schema(Data) ->
-    [begin
-         do_spellcheck([FullName], Node),
-         Fields = maps:get(fields, Node, []),
-         [begin
-              FieldName = [FullName, maps:get(name, Field)],
-              do_spellcheck(FieldName, Field)
-          end || Field <- Fields]
-     end || Node = #{full_name := FullName} <- Data],
+spellcheck_schema(Data = [_ | _], MaxConcurrency) ->
+    Chunk = lists:sublist(Data, MaxConcurrency),
+    Rest = lists:sublist(Data, MaxConcurrency, length(Data)),
+    _ = rpc:pmap({?MODULE, do_spellcheck_schema}, _ExtraArgs = [], Chunk),
+    spellcheck_schema(Rest, MaxConcurrency);
+spellcheck_schema(_Data = [], _MaxConcurrency) ->
     ok.
 
+do_spellcheck_schema(Node = #{full_name := FullName}) ->
+    do_spellcheck([FullName], Node),
+    Fields = maps:get(fields, Node, []),
+    [begin
+         FieldName = [FullName, maps:get(name, Field)],
+         do_spellcheck(FieldName, Field)
+     end || Field <- Fields].
+
 %% Check spelling in any description:
 do_spellcheck(FullName, #{desc := Desc}) ->
     Resp = langtool:req(Desc),
@@ -52,17 +63,18 @@ do_spellcheck(FullName, #{desc := Desc}) ->
             ok;
         L  ->
             setfail(),
-            io:format(user, "!! '~s'~n~n", [format_name(FullName)]),
-            [io:format(user, "~s", [langtool:format_warning(I)]) || I <- L],
+            Header = io_lib:format("!! '~s'~n~n", [format_name(FullName)]),
+            Warnings = [io_lib:format("~s", [langtool:format_warning(I)]) || I <- L],
+            io:put_chars(user, [Header, Warnings]),
             ok
     end;
 %% Ignore references to structs, since the struct itself should have a description
-do_spellcheck(FullName, #{type := #{kind := <<"struct">>}}) ->
+do_spellcheck(_FullName, #{type := #{kind := <<"struct">>}}) ->
     ok;
 do_spellcheck(FullName, _) ->
     Record = hd(FullName),
     case binary:match(Record, [<<"Root Config Keys">>]) of
-       nomatch -> 
+       nomatch ->
           io:format(user, "Error: '~s' doesn't have a description~n", [format_name(FullName)]),
           setfail();
        _ -> ok
@@ -79,10 +91,10 @@ read_stdio() ->
     end.
 
 setfail() ->
-    put(?MODULE, true).
+    persistent_term:put(?MODULE, true).
 
 is_ok() ->
-    get(?MODULE) =/= true.
+    persistent_term:get(?MODULE, false) =/= true.
 
 format_name(FullName) ->
     lists:join("::", FullName).