[flow][autocomplete] introduce canonical form of contents

Summary:
An autocomplete session is a series of requests on the same token. E.g.
```
foo.|
foo.b|
foo.ba|
foo.bar|
```
Motivation for this change is to enable caching of typing artifacts within an autocomplete session.

To enable this caching this diff introduces a canonical form that can be used as a representative for all versions of contents during an autocomplete session.

The canonical form for all versions of contents in the above example is `foo.AUTO332`. Notice that we have removed the prefix `bar` before adding the sigil.

The key idea behind this concept is that the type inference process (which happens today) is identical for all of the forms:
```
foo.AUTO332
foo.bAUTO332
foo.baAUTO332
foo.barAUTO332
```

Having a single form to represent all these forms makes caching of typing artifacts for autocomplete practical.

The main parts of the implementation for this idea are:
1. Add `Autocomplete_sig.add_canonical` that returns the canonical form of the contents, and, if successful, information about the prefix and suffix of the identifier around the cursor.
2. Parse and type check the canonical form.
3. Recover the location and token under cursor for further processing in autocompleteService_js.ml. For example, convert `"AUTO332"` to `"fooAUTO332bar"`, since this is the expected form.

Changelog: [internal]

Reviewed By: SamChou19815

Differential Revision: D55273266

fbshipit-source-id: f7c99762d959509950fa0bd80199f9da91ec33a6

src/commands/commandUtils.ml

@@ -1492,6 +1492,8 @@ let make_options
       Option.value
         options_flags.long_lived_workers
         ~default:(FlowConfig.long_lived_workers flowconfig);
+    opt_autocomplete_canonical =
+      Base.Option.value (FlowConfig.autocomplete_canonical flowconfig) ~default:Options.Ac_classic;
     opt_autocomplete_mode =
       Base.Option.value
         (FlowConfig.autocomplete_mode flowconfig)

src/commands/config/flowConfig.ml

@@ -44,6 +44,7 @@ module Opts = struct
 
   type t = {
     all: bool option;
+    autocomplete_canonical: Options.autocomplete_canonical option;
     autocomplete_mode: Options.autocomplete_mode option;
     autoimports: bool option;
     autoimports_min_characters: int option;
@@ -174,6 +175,7 @@ module Opts = struct
   let default_options =
     {
       all = None;
+      autocomplete_canonical = None;
       autocomplete_mode = None;
       autoimports = None;
       autoimports_min_characters = None;
@@ -396,6 +398,15 @@ module Opts = struct
       [("typed_ast", Options.Ac_typed_ast); ("on_demand", Options.Ac_on_demand_typing)]
       (fun opts v -> Ok { opts with autocomplete_mode = Some v })
 
+  let autocomplete_canonical_parser =
+    enum
+      [
+        ("canonical", Options.Ac_canonical);
+        ("classic", Options.Ac_classic);
+        ("both", Options.Ac_both);
+      ]
+      (fun opts v -> Ok { opts with autocomplete_canonical = Some v })
+
   let casting_syntax_parser =
     enum
       [
@@ -869,6 +880,7 @@ module Opts = struct
   let parsers =
     [
       ("all", boolean (fun opts v -> Ok { opts with all = Some v }));
+      ("experimental.autocomplete_canonical", autocomplete_canonical_parser);
       ("experimental.autocomplete_mode", autocomplete_mode_parser);
       ("autoimports", boolean (fun opts v -> Ok { opts with autoimports = Some v }));
       ( "autoimports.min_characters",
@@ -1548,6 +1560,8 @@ let libs config = config.libs
 
 let all c = c.options.Opts.all
 
+let autocomplete_canonical c = c.options.Opts.autocomplete_canonical
+
 let autocomplete_mode c = c.options.Opts.autocomplete_mode
 
 let autoimports c = c.options.Opts.autoimports

src/commands/config/flowConfig.mli

@@ -66,6 +66,8 @@ val enabled_rollouts : config -> string SMap.t
 (* options *)
 val all : config -> bool option
 
+val autocomplete_canonical : config -> Options.autocomplete_canonical option
+
 val autocomplete_mode : config -> Options.autocomplete_mode option
 
 val autoimports : config -> bool option

src/common/options.ml

@@ -39,6 +39,11 @@ type autocomplete_mode =
   | Ac_typed_ast
   | Ac_on_demand_typing
 
+type autocomplete_canonical =
+  | Ac_canonical
+  | Ac_classic
+  | Ac_both (* for comparison purposes *)
+
 type format = {
   opt_bracket_spacing: bool;
   opt_single_quotes: bool;
@@ -71,6 +76,7 @@ type t = {
   opt_all: bool;
   opt_as_const: bool;
   opt_any_propagation: bool;
+  opt_autocomplete_canonical: autocomplete_canonical;
   opt_autocomplete_mode: autocomplete_mode;
   opt_autoimports: bool;
   opt_autoimports_min_characters: int;
@@ -168,6 +174,8 @@ let as_const opts = opts.opt_as_const
 
 let any_propagation opts = opts.opt_any_propagation
 
+let autocomplete_canonical opts = opts.opt_autocomplete_canonical
+
 let autocomplete_mode opts = opts.opt_autocomplete_mode
 
 let autoimports opts = opts.opt_autoimports

src/flow_dot_js.ml

@@ -360,8 +360,13 @@ let autocomplete filename content line col js_config_object :
   let filename = File_key.SourceFile filename in
   let root = File_path.dummy_path in
   let cursor_loc = Loc.cursor (Some filename) line col in
-  let (content, _) = Autocomplete_sigil.add content line col in
-  Autocomplete_js.autocomplete_set_hooks ~cursor:cursor_loc;
+  let (content, _, canon_token) =
+    Autocomplete_sigil.add_canonical (Some filename) content line col
+  in
+  let canon_cursor =
+    Base.Option.value_map ~default:cursor_loc ~f:Autocomplete_sigil.Canonical.cursor canon_token
+  in
+  Autocomplete_js.autocomplete_set_hooks ~cursor:canon_cursor;
   let r =
     match parse_content filename content with
     | Error _ -> Error "parse error"
@@ -414,6 +419,7 @@ let autocomplete filename content line col js_config_object :
           }
           None
           loc
+          canon_token
       in
       (match result with
       | AcEmpty _ -> Ok { ServerProt.Response.Completion.items = []; is_incomplete = false }

src/parser/parser_flow.ml

@@ -574,3 +574,22 @@ let jsx_pragma_expression =
 let string_is_valid_identifier_name str =
   let lexbuf = Sedlexing.Utf8.from_string str in
   Flow_lexer.is_valid_identifier_name lexbuf
+
+(**
+ * Returns the string and location of the first identifier in [input] for which
+ * [predicate] holds.
+ *)
+let find_ident ~predicate input =
+  let env = init_env ~token_sink:None ~parse_options:None None input in
+  let rec loop token =
+    match token with
+    | T_EOF -> None
+    | _ ->
+      let loc = Peek.loc env in
+      (match token with
+      | T_IDENTIFIER { value = s; _ } when predicate s -> Some (loc, s)
+      | _ ->
+        Eat.token env;
+        loop (Peek.token env))
+  in
+  loop (Peek.token env)

src/server/command_handler/commandHandler.ml

@@ -420,16 +420,26 @@ let autocomplete_on_parsed
     ~imports_min_characters
     ~imports_ranked_usage
     ~imports_ranked_usage_boost_exact_match_min_length
-    ~show_ranking_info =
+    ~show_ranking_info
+    ~canonical =
   let cursor_loc =
     let (line, column) = cursor in
     Loc.cursor (Some filename) line column
   in
-  let (contents, broader_context) =
+  (* Assuming input request at `foo.bar|`, [contents] will include `foo.AUTO332`.
+   * We will refer to this form of contents as canonical. *)
+  let (contents, broader_context, canon_token) =
     let (line, column) = cursor in
-    Autocomplete_sigil.add contents line column
+    if canonical then
+      Autocomplete_sigil.add_canonical (Some filename) contents line column
+    else
+      let (contents, broader_context) = Autocomplete_sigil.add contents line column in
+      (contents, broader_context, None)
   in
-  Autocomplete_js.autocomplete_set_hooks ~cursor:cursor_loc;
+  let canon_cursor =
+    Base.Option.value_map ~default:cursor_loc ~f:Autocomplete_sigil.Canonical.cursor canon_token
+  in
+  Autocomplete_js.autocomplete_set_hooks ~cursor:canon_cursor;
   let parse_result = Type_contents.parse_contents ~options ~profiling contents filename in
   let initial_json_props =
     let open Hh_json in
@@ -477,7 +487,7 @@ let autocomplete_on_parsed
                 show_ranking_info;
               }
             in
-            autocomplete_get_results typing ac_options trigger_character cursor_loc
+            autocomplete_get_results typing ac_options trigger_character cursor_loc canon_token
         )
       in
       Some (info, parse_errors, token_opt, ac_loc, ac_type_string, results_res)
@@ -509,7 +519,7 @@ let autocomplete
     let extra_data = json_of_skipped reason in
     (Ok response, extra_data)
   | Ok (filename, contents) ->
-    let (initial_json_props, ac_result) =
+    let autocomplete =
       autocomplete_on_parsed
         ~filename
         ~contents
@@ -525,6 +535,24 @@ let autocomplete
         ~imports_ranked_usage_boost_exact_match_min_length
         ~show_ranking_info
     in
+    let autocomplete_canonical () = autocomplete ~canonical:true in
+    let autocomplete_classic () = autocomplete ~canonical:false in
+    let (initial_json_props, ac_result) =
+      match Options.autocomplete_canonical options with
+      | Options.Ac_classic -> autocomplete_classic ()
+      | Options.Ac_canonical -> autocomplete_canonical ()
+      | Options.Ac_both ->
+        let (_, result_1) = autocomplete_canonical () in
+        let (json_2, result_2) = autocomplete_classic () in
+        let equal =
+          match (result_1, result_2) with
+          | (None, None) -> true
+          | (Some (_, _, _, _, _, r1), Some (_, _, _, _, _, r2)) ->
+            AutocompleteService_js.equal_autocomplete_service_result r1 r2
+          | (_, _) -> false
+        in
+        (("canonical_compliance", Hh_json.JSON_Bool equal) :: json_2, result_2)
+    in
     json_of_autocomplete_result initial_json_props ac_result
 
 let check_file ~options ~env ~profiling ~force file_input =

src/services/autocomplete/__tests__/autocomplete_sigil_tests.ml

@@ -14,15 +14,19 @@ let tests =
            ( "empty_line" >:: fun ctxt ->
              let contents = "// @flow\n\nfoo\n\nbar" in
              let expected = "// @flow\n\nfoo\nAUTO332\nbar" in
-             let (actual, broader_context) = Autocomplete_sigil.add contents 4 0 in
+             let (actual, broader_context, _) =
+               Autocomplete_sigil.add_canonical None contents 4 0
+             in
              assert_equal ~ctxt ~printer:(fun x -> x) expected actual;
              let expected = "foo\nAUTO332\nbar" in
              assert_equal ~ctxt ~printer:(fun x -> x) expected broader_context
            );
            ( "last_line" >:: fun ctxt ->
              let contents = "// @flow\n" in
              let expected = "// @flow\nAUTO332" in
-             let (actual, broader_context) = Autocomplete_sigil.add contents 2 0 in
+             let (actual, broader_context, _) =
+               Autocomplete_sigil.add_canonical None contents 2 0
+             in
              assert_equal ~ctxt ~printer:(fun x -> x) expected actual;
              let expected = "// @flow\nAUTO332" in
              assert_equal ~ctxt ~printer:(fun x -> x) expected broader_context

src/services/autocomplete/autocompleteService_js.ml

@@ -390,14 +390,17 @@ type 'r ac_result = {
   result: 'r;
   errors_to_log: string list;
 }
+[@@deriving eq]
 
 type 'r autocomplete_service_result_generic =
   | AcResult of 'r ac_result
   | AcEmpty of string
   | AcFatalError of string
+[@@deriving eq]
 
 type autocomplete_service_result =
   ServerProt.Response.Completion.t autocomplete_service_result_generic
+[@@deriving eq]
 
 let jsdoc_of_def_loc { loc_of_aloc; get_ast_from_shared_mem; ast; _ } def_loc =
   loc_of_aloc def_loc |> Find_documentation.jsdoc_of_getdef_loc ~ast ~get_ast_from_shared_mem
@@ -2145,10 +2148,15 @@ let string_of_autocomplete_type ac_type =
   | Ac_jsx_element _ -> "Ac_jsx_element"
   | Ac_jsx_attribute _ -> "Acjsx"
 
-let autocomplete_get_results typing ac_options trigger_character cursor =
+let autocomplete_get_results typing ac_options trigger_character cursor canon_token =
   let { layout_options; loc_of_aloc; cx; ast; available_ast; norm_genv = genv; _ } = typing in
   let open Autocomplete_js in
-  match process_location cx ~trigger_character ~cursor ~available_ast with
+  (* Since the AST that is provided to `process_location` is in canonical terms,
+   * we also need to adjust the cursor. *)
+  let canon_cursor =
+    Base.Option.value_map ~default:cursor ~f:Autocomplete_sigil.Canonical.cursor canon_token
+  in
+  match process_location cx ~trigger_character ~cursor:canon_cursor ~available_ast with
   | Error err -> (None, None, "None", AcFatalError err)
   | Ok None ->
     let result = { ServerProt.Response.Completion.items = []; is_incomplete = false } in
@@ -2158,6 +2166,17 @@ let autocomplete_get_results typing ac_options trigger_character cursor =
       AcResult { result; errors_to_log = ["Autocomplete token not found in AST"] }
     )
   | Ok (Some { tparams_rev; ac_loc; token; autocomplete_type }) ->
+    (* The remaining processing of the autocomplete result is done assuming non-
+     * canonical terms. We need to revert the canonicalization that happened
+     * earlier on. This typically means converting a canonical token ("AUTO332")
+     * to "fooAUTO332Baz", where "foo" and "Baz" are the prefix and suffix of the
+     * token around the cursor. `ac_loc` is adjusted accordingly. *)
+    let (ac_loc, token) =
+      match canon_token with
+      | None -> (ac_loc, token)
+      | Some canon ->
+        Autocomplete_sigil.Canonical.to_relative_ac_results ~canon (loc_of_aloc ac_loc) token
+    in
     (* say you're completing `foo|Baz`. the token is "fooBaz" and ac_loc points at
        "fooAUTO332Baz". if the suggestion is "fooBar", the user can choose to insert
        into the token, yielding "fooBarBaz", or they can choose to replace the token,

src/services/autocomplete/autocompleteService_js.mli

@@ -41,9 +41,13 @@ type 'r autocomplete_service_result_generic =
 type autocomplete_service_result =
   ServerProt.Response.Completion.t autocomplete_service_result_generic
 
+val equal_autocomplete_service_result :
+  autocomplete_service_result -> autocomplete_service_result -> bool
+
 val autocomplete_get_results :
   typing ->
   ac_options ->
   string option ->
   Loc.t ->
+  Autocomplete_sigil.Canonical.token option ->
   string option * ALoc.t option * string * autocomplete_service_result