Improvements for Neo4j v5 engines (#1)

* Improvements for Neo4j v5 engines

* Update workflow

.github/workflows/elixir.yml

@@ -16,20 +16,26 @@ jobs:
 
     strategy:
       matrix:
-        iex: [1.14.3]
-        otp: [24.3.4, 25.2]
+        iex: [1.15.8, 1.16.3]
+        otp: [25.3, 26.2]
+        include:
+          - iex: 1.14.5
+            otp: 24.3.4
+          - iex: 1.17.3
+            otp: 27.2
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Elixir
-        uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f
+        uses: erlef/setup-beam@v1
+        id: beam
         with:
           elixir-version: ${{ matrix.iex }}
           otp-version: ${{ matrix.otp }}
 
       - name: Restore dependencies cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
@@ -41,7 +47,7 @@ jobs:
       # Don't cache PLTs based on mix.lock hash, as Dialyzer can incrementally update even old ones
       # Cache key based on Elixir & Erlang version (also useful when running in matrix)
       - name: Restore PLT cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         id: plt_cache
         with:
           key: |

.tool-versions

@@ -1,2 +1,2 @@
-elixir 1.14.0-otp-24
-erlang 24.3.4
+elixir 1.17.3
+erlang 27.2

README.md

@@ -10,11 +10,15 @@ Most popular engine for those is Neo4j thus this library focuses on providing fu
 
 Currently only simple quering using raw Cypher queries is implemented, but there are few items on the Roadmap.
 
-### Bolt_sips
+## Existing libraries
 
-One may say "there is already a library for communication with Neo4j". They are right **BUT** first and foremost, `bolt_sips` is left unmaintained ([discussion](https://github.com/florinpatrascu/bolt_sips/issues/109)). There were few attempts to continue that, but there is still no library that would take advantage of Elixir structs, protocols and behaviours to provide robust extensibility. Secondly, `bolt_sips` is just a driver. This library purpose will be to provide complete user experience when interacting with the DB. 
+There were already few attempts to write a driver for Bolt protocol but all of them seem to be clumsy in terms of protocol logic - many things are "hardcoded" as in the docs instead of being thought out for the server's operation and coding a reusable solution. They are not taking advantage of Elixir structs, protocols and behaviours to provide robust extensibility.  
+Secondly, those libs are just a driver and this library purpose is to provide complete user experience when interacting with the DB.  
 This should be solved by building Ecto-like support for the Cypher query language.  
 
+At this point, it's worth noting that this library may not be faster than `bolt_sips` or `boltx` due to greater usage of Protocols and structs.  
+You can modify tasks from `example_app` to benchmark those on your data.
+
 ## Installation
 
 The package can be installed by adding `neo4ex` to your list of dependencies in `mix.exs`:

docker-compose.yml

@@ -1,11 +1,10 @@
-version: '3.1'
-
 services:
   graph_db:
-    image: neo4j:4.4.28-community
+    image: neo4j:5.26.0-community
     environment:
         NEO4J_AUTH: 'neo4j/letmein'
         NEO4JLABS_PLUGINS: '["apoc"]'
+        NEO4J_dbms_security_auth__minimum__password__length: 6
     ports:
       - "7474:7474"
       - "7687:7687"

example_app/config/config.exs

@@ -6,8 +6,11 @@ config :example_app, ExampleApp.Connector,
   credentials: "letmein",
   pool_size: 1
 
-config :bolt_sips, Bolt,
-  url: "bolt://localhost:7687",
-  basic_auth: [username: "neo4j", password: "letmein"],
+config :boltx, Bolt,
+  uri: "bolt://localhost:7687",
+  auth: [username: "neo4j", password: "letmein"],
+  user_agent: "boltxTest/1",
   pool_size: 1,
-  max_overflow: 0
+  max_overflow: 0,
+  prefix: :default,
+  name: Boltx

example_app/lib/example_app/application.ex

@@ -9,7 +9,7 @@ defmodule ExampleApp.Application do
   def start(_type, _args) do
     children = [
       ExampleApp.Connector,
-      {Bolt.Sips, Application.get_env(:bolt_sips, Bolt)}
+      {Boltx, Application.get_env(:boltx, Bolt)}
     ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html

example_app/lib/mix/tasks/bolt_sips_benchmark.ex → example_app/lib/mix/tasks/boltx_benchmark.ex

@@ -1,19 +1,17 @@
-defmodule Mix.Tasks.ExampleApp.BoltSipsBenchmark do
+defmodule Mix.Tasks.ExampleApp.BoltxBenchmark do
   use Mix.Task
 
   alias Neo4ex.Cypher
 
-  alias Bolt.Sips, as: Neo
-
   alias ExampleApp.Connector
 
   @requirements ["app.start"]
 
-  @shortdoc "Runs benchmark to compare with bolt_sips library"
+  @shortdoc "Runs benchmark to compare with boltx library"
   def run(_args) do
     Benchee.run(%{
       "Neo4ex" => fn -> neo4ex() end,
-      "Bolt.Sips" => fn -> bolt_sips() end
+      "Boltx" => fn -> boltx() end
     })
   end
 
@@ -22,17 +20,17 @@ defmodule Mix.Tasks.ExampleApp.BoltSipsBenchmark do
     Connector.run(%Cypher.Query{query: query, params: params})
   end
 
-  def bolt_sips() do
+  def boltx() do
     %{query: query, params: params} = customer_query()
-    Neo.query!(Neo.conn(), query, params)
+    Boltx.query!(Boltx, query, params)
   end
 
   def customer_query() do
     query = """
     MATCH (customer)
     RETURN customer, rand() as r
     ORDER BY r
-    LIMIT 10
+    LIMIT 100
     """
 
     %{query: query, params: %{}}

example_app/mix.exs

@@ -23,7 +23,7 @@ defmodule ExampleApp.MixProject do
   defp deps do
     [
       {:neo4ex, path: "../"},
-      {:bolt_sips, git: "https://github.com/florinpatrascu/bolt_sips", branch: "master"},
+      {:boltx, "~> 0.0.6"},
       {:faker, "~> 0.17.0"},
       {:jason, "~> 1.2"},
       {:benchee, "~> 1.0"}

example_app/mix.lock

@@ -1,11 +1,12 @@
 %{
-  "benchee": {:hex, :benchee, "1.2.0", "afd2f0caec06ce3a70d9c91c514c0b58114636db9d83c2dc6bfd416656618353", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "ee729e53217898b8fd30aaad3cce61973dab61574ae6f48229fe7ff42d5e4457"},
+  "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"},
   "bolt_sips": {:git, "https://github.com/florinpatrascu/bolt_sips", "b21901a46ed19b17d1c87a9ef9e56002f83f345c", [branch: "master"]},
+  "boltx": {:hex, :boltx, "0.0.6", "c6a396b1538b258e4d5ee2a94aaf8fb2c7879240efffba94b9159dbdce963790", [:mix], [{:db_connection, "~> 2.6.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "576b8f21a2021674130d04cd1fc79a4829a23d2cdf50641b3d7a00ce31b98ead"},
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
-  "db_connection": {:hex, :db_connection, "2.4.3", "3b9aac9f27347ec65b271847e6baeb4443d8474289bd18c1d6f4de655b70c94d", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c127c15b0fa6cfb32eed07465e05da6c815b032508d4ed7c116122871df73c12"},
+  "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "faker": {:hex, :faker, "0.17.0", "671019d0652f63aefd8723b72167ecdb284baf7d47ad3a82a15e9b8a6df5d1fa", [:mix], [], "hexpm", "a7d4ad84a93fd25c5f5303510753789fc2433ff241bf3b4144d3f6f291658a6a"},
-  "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
+  "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
   "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
-  "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
+  "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
 }

lib/neo4ex/bolt_protocol.ex

@@ -28,8 +28,6 @@ defmodule Neo4ex.BoltProtocol do
 
   alias Neo4ex.BoltProtocol.Structure.Message.Summary.{Success, Failure}
 
-  @user_agent "Neo4ex/#{Application.spec(:neo4ex, :vsn)}"
-
   @impl true
   def connect(opts) do
     hostname = Keyword.get(opts, :hostname)
@@ -42,7 +40,11 @@ defmodule Neo4ex.BoltProtocol do
          :ok <- hello(socket, opts) do
       {:ok, socket}
     else
-      other -> other
+      {:ok, %Failure{metadata: %{"message" => failure}}} ->
+        {:error, failure}
+
+      other ->
+        other
     end
   end
 
@@ -290,12 +292,14 @@ defmodule Neo4ex.BoltProtocol do
       end
 
     if Version.match?(bolt_version, ">= 5.1.0") do
-      hello = %Hello{extra: %Extra.Hello{user_agent: @user_agent}}
+      hello = %Hello{extra: %Extra.Hello{}}
 
       logon = %Logon{
-        scheme: scheme,
-        principal: principal,
-        credentials: credentials
+        auth: %Extra.Logon{
+          scheme: scheme,
+          principal: principal,
+          credentials: credentials
+        }
       }
 
       with(
@@ -311,7 +315,6 @@ defmodule Neo4ex.BoltProtocol do
     else
       message = %Hello{
         extra: %Extra.Hello{
-          user_agent: @user_agent,
           scheme: scheme,
           principal: principal,
           credentials: credentials

lib/neo4ex/bolt_protocol/structure.ex

@@ -82,10 +82,11 @@ defmodule Neo4ex.BoltProtocol.Structure do
   defp build_fields_list(block) do
     block
     |> Macro.prewalk([], fn
-      {:field, _, [name, opts]}, acc ->
+      {:field, _, [name | opts]}, acc ->
         opts =
-          Keyword.update(
-            opts,
+          opts
+          |> List.flatten()
+          |> Keyword.update(
             :version,
             quote(do: Version.parse_requirement!(">= 0.0.0")),
             fn requirement ->
@@ -152,6 +153,11 @@ defmodule Neo4ex.BoltProtocol.Structure do
 
   # embedded structure must be a map but we should take advantage of field versioning
   defp embedded_encoder_protocol(fields_list) do
+    # When default values contain some data from module (like values from attributes) it won't be visible inside defimpl (different module)
+    # For protocol we need only version so this is completely fine
+    fields_list =
+      Enum.map(fields_list, fn {field, opts} -> {field, [version: opts[:version]]} end)
+
     quote location: :keep do
       defimpl Neo4ex.BoltProtocol.Encoder do
         def encode(struct, bolt_version) do
@@ -169,6 +175,11 @@ defmodule Neo4ex.BoltProtocol.Structure do
   end
 
   defp encoder_protocol(fields_list) do
+    # When default values contain some data from module (like values from attributes) it won't be visible inside defimpl (different module)
+    # For protocol we need only version so this is completely fine
+    fields_list =
+      Enum.map(fields_list, fn {field, opts} -> {field, [version: opts[:version]]} end)
+
     quote location: :keep do
       defimpl Neo4ex.BoltProtocol.Encoder do
         alias Neo4ex.PackStream.{Markers, Exceptions}

lib/neo4ex/bolt_protocol/structure/message/extra/hello.ex

@@ -1,15 +1,26 @@
 defmodule Neo4ex.BoltProtocol.Structure.Message.Extra.Hello do
   use Neo4ex.BoltProtocol.Structure
 
+  @version Mix.Project.config()[:version]
+
   # can't be encoded directly, it's just helper for the Hello message
   embeded_structure do
-    field(:user_agent, default: "Neo4ex/0.1.0")
+    field(:user_agent, default: "Neo4ex/#{@version}")
+
+    field(:bolt_agent,
+      default: %{
+        product: "Neo4ex/#{@version}",
+        language: "Elixir/#{System.build_info()[:version]}"
+      },
+      version: ">= 5.3.0"
+    )
+
     field(:patch_bolt, default: ["utc"], version: ">= 4.3.0 and <= 4.4.0")
     field(:routing, default: %{}, version: ">= 4.1.0")
 
     # prior to v5.1, authentication is handled inside HELLO message
-    field(:scheme, default: "", version: "< 5.1.0")
-    field(:principal, default: "", version: "< 5.1.0")
-    field(:credentials, default: "", version: "< 5.1.0")
+    field(:scheme, version: "< 5.1.0")
+    field(:principal, version: "< 5.1.0")
+    field(:credentials, version: "< 5.1.0")
   end
 end

lib/neo4ex/bolt_protocol/structure/message/extra/logon.ex

@@ -0,0 +1,12 @@
+defmodule Neo4ex.BoltProtocol.Structure.Message.Extra.Logon do
+  use Neo4ex.BoltProtocol.Structure
+
+  # TODO: implement validation
+  # @predefined_schemes ~w(none basic bearer kerberos)
+
+  embeded_structure do
+    field(:scheme)
+    field(:principal)
+    field(:credentials)
+  end
+end

lib/neo4ex/bolt_protocol/structure/message/request/logon.ex

@@ -1,12 +1,9 @@
 defmodule Neo4ex.BoltProtocol.Structure.Message.Request.Logon do
   use Neo4ex.BoltProtocol.Structure
 
-  # TODO: implement validation
-  # @predefined_schemes ~w(none basic bearer kerberos)
+  alias Neo4ex.BoltProtocol.Structure.Message.Extra
 
   structure 0x6A do
-    field(:scheme, default: "")
-    field(:principal, default: "")
-    field(:credentials, default: "")
+    field(:auth, default: %Extra.Logon{})
   end
 end

lib/neo4ex/connector.ex

@@ -15,7 +15,7 @@ defmodule Neo4ex.Connector do
   @noop <<0::size(@chunk_size)>>
   # since 4.3 there is support for version range during negotiation
   # so "4.4.1" actually means "4.4" plus one previous version "4.3"
-  @supported_versions ["4.4.1", "4.2.0", "4.1.0", "4.0.0"]
+  @supported_versions ["5.20.20", "4.4.3", "4.2.0", "4.0.0"]
 
   defmacro __using__(otp_app: app) do
     supported_versions = @supported_versions
@@ -101,14 +101,17 @@ defmodule Neo4ex.Connector do
     end
   end
 
-  def supported_versions() do
-    Enum.flat_map(@supported_versions, fn version ->
+  defmacro supported_versions() do
+    @supported_versions
+    |> Enum.flat_map(fn version ->
       [major, minor, range] = version |> String.split(".") |> Enum.map(&String.to_integer/1)
 
       for i <- minor..(minor - range) do
         Version.parse!("#{major}.#{i}.0")
       end
     end)
+    |> Enum.uniq()
+    |> Macro.escape()
   end
 
   @doc false

lib/neo4ex/utils.ex

@@ -1,6 +1,8 @@
 defmodule Neo4ex.Utils do
   @moduledoc false
 
+  import Neo4ex.Connector, only: [supported_versions: 0]
+
   alias Neo4ex.BoltProtocol
 
   alias Neo4ex.PackStream
@@ -88,7 +90,7 @@ defmodule Neo4ex.Utils do
   end
 
   def list_valid_versions(requirement) do
-    Enum.filter(Neo4ex.Connector.supported_versions(), fn ver ->
+    Enum.filter(supported_versions(), fn ver ->
       Version.match?(ver, requirement)
     end)
   end

mix.exs

@@ -17,7 +17,7 @@ defmodule Neo4ex.MixProject do
       docs: docs(),
       test_coverage: [
         ignore_modules: [
-          Neo4ex.BoltProtocol.Structure,
+          ~r/^Neo4ex.BoltProtocol.Structure/,
           Neo4ex.PackStream.DecoderBuilder
         ],
         summary: [
@@ -40,14 +40,14 @@ defmodule Neo4ex.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:db_connection, "~> 2.4"},
+      {:db_connection, "~> 2.6.0"},
 
       # Tests
       {:mox, "~> 1.0", only: [:test]},
 
       # Linting
-      {:credo, "~> 1.6.7", only: [:dev]},
-      {:dialyxir, "~> 1.2.0", only: [:dev], runtime: false},
+      {:credo, "~> 1.6", only: [:dev]},
+      {:dialyxir, "~> 1.4", only: [:dev], runtime: false},
 
       # Documentation
       # Run with: `mix docs`