diff --git a/VERSION b/VERSION index 0ee7671d0..9919148bd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.5 \ No newline at end of file +1.10.6 \ No newline at end of file diff --git a/config/config.exs b/config/config.exs index 2cfe5d701..9460d5dd5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -28,6 +28,23 @@ config :logflare, Logflare.Source.BigQuery.Schema, updates_per_minute: 6 # Configures the endpoint config :logflare, LogflareWeb.Endpoint, + adapter: Bandit.PhoenixAdapter, + http: [ + thousand_island_options: [ + # https://cloud.google.com/load-balancing/docs/https/#timeouts_and_retries + # preserves idle keepalive connections up to load balancer max of 600s + read_timeout: 600_000, + # transport options are passed wholly to :gen_tcp + # https://github.com/mtrudel/thousand_island/blob/ae733332892b1bb2482a9cf4e97de03411fba2ad/lib/thousand_island/transports/tcp.ex#L61 + transport_options: [ + # https://www.erlang.org/doc/man/inet + # both reuseport and reuseport_lb should be provided for linux + reuseport: true, + reuseport_lb: true + # + ] + ] + ], url: [host: "localhost", scheme: "http", port: 4000], secret_key_base: "DSzZYeAgGaXlfRXPQqMOPiA8hJOYSImhnR2lO8lREOE2vWDmkGn1XWHxoCZoASlP", render_errors: [view: LogflareWeb.ErrorView, accepts: ~w(html json)], diff --git a/config/dev.exs b/config/dev.exs index f05631734..d03d13ff9 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -7,14 +7,7 @@ config :logflare, config :logflare, LogflareWeb.Endpoint, server: true, http: [ - port: System.get_env("PORT") || 4000, - transport_options: [ - max_connections: 16_384, - num_acceptors: 100, - socket_opts: [{:raw, 0xFFFF, 0x0200, <<1::32-native>>}] - ], - protocol_options: [max_keepalive: 1_000], - compress: true + port: System.get_env("PORT") || 4000 ], live_view: [ signing_salt: "eVpFFmpN+OHPrilThexLilWnF+a8zBLbCtdH/OzAayShcm1B3OHOyGiadM6qOezp" diff --git a/config/prod.exs b/config/prod.exs index a900621aa..79d2f57d9 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -5,24 +5,7 @@ config :logflare, env: :prod config :logflare, LogflareWeb.Endpoint, http: [ - port: 4000, - transport_options: [ - max_connections: 64_000, - num_acceptors: 1_000, - # for so_reuseport - socket_opts: [{:raw, 1, 15, <<1::32-native>>}] - ], - # https://blog.percy.io/tuning-nginx-behind-google-cloud-platform-http-s-load-balancer-305982ddb340 - # https://github.com/ninenines/cowboy/issues/1286#issuecomment-699643478 - protocol_options: [ - # https://ninenines.eu/docs/en/cowboy/2.8/manual/cowboy_http/ - request_timeout: 30_000, - # https://cloud.google.com/load-balancing/docs/https/#timeouts_and_retries - # must be greater than 600s - idle_timeout: 650_000, - max_keepalive: :infinity - ], - compress: true + port: 4000 ], cache_static_manifest: "priv/static/cache_manifest.json", server: true, diff --git a/mix.exs b/mix.exs index 54cf413c8..486ab10a5 100644 --- a/mix.exs +++ b/mix.exs @@ -72,13 +72,13 @@ defmodule Logflare.Mixfile do defp deps do [ # Phoenix stuff - {:phoenix, "~> 1.7"}, + {:phoenix, "~> 1.7.14"}, {:phoenix_live_view, "~> 0.18"}, {:phoenix_view, "~> 2.0"}, {:phoenix_pubsub, "~> 2.1"}, {:phoenix_ecto, "~> 4.4"}, {:phoenix_live_reload, "~> 1.4", only: :dev}, - {:bandit, ">= 0.7.7"}, + {:bandit, "~> 1.5.7"}, {:plug_crypto, "~> 1.2.2"}, {:cors_plug, "~> 2.0"}, {:plug_caisson, "~> 0.2.1"}, diff --git a/mix.lock b/mix.lock index 4dd9a060c..0108c80ec 100644 --- a/mix.lock +++ b/mix.lock @@ -1,7 +1,7 @@ %{ "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"}, "atomic_map": {:hex, :atomic_map, "0.9.3", "3c7f1302e0590164732d08ca999708efbb2cd768abf2911cf140280ce2dc499d", [:mix], [], "hexpm", "c237babf301bd2435bd85b96cffc973022b4cbb7721537059ee0dd3bb74938d2"}, - "bandit": {:hex, :bandit, "1.4.2", "a1475c8dcbffd1f43002797f99487a64c8444753ff2b282b52409e279488e1f5", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "3db8bacea631bd926cc62ccad58edfee4252d1b4c5cccbbad9825df2722b884f"}, + "bandit": {:hex, :bandit, "1.5.7", "6856b1e1df4f2b0cb3df1377eab7891bec2da6a7fd69dc78594ad3e152363a50", [:mix], [{:hpax, "~> 1.0.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "f2dd92ae87d2cbea2fa9aa1652db157b6cba6c405cb44d4f6dd87abba41371cd"}, "benchee": {:hex, :benchee, "1.3.0", "f64e3b64ad3563fa9838146ddefb2d2f94cf5b473bdfd63f5ca4d0657bf96694", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "34f4294068c11b2bd2ebf2c59aac9c7da26ffa0068afdf3419f1b176e16c5f81"}, "benchee_async": {:hex, :benchee_async, "0.1.2", "38e686bd9cdf7ae8767a08c0df6932b625b166fd97267d0437c715f9ff424bc1", [:mix], [], "hexpm", "73b0bd95173f86c61d41570adaeb151efe70c0cc273eec6501d391b35e3599dd"}, "bertex": {:hex, :bertex, "1.3.0", "0ad0df9159b5110d9d2b6654f72fbf42a54884ef43b6b651e6224c0af30ba3cb", [:mix], [], "hexpm", "0a5d5e478bb5764b7b7bae37cae1ca491200e58b089df121a2fe1c223d8ee57a"}, @@ -63,7 +63,7 @@ "gun": {:hex, :gun, "2.1.0", "b4e4cbbf3026d21981c447e9e7ca856766046eff693720ba43114d7f5de36e87", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "52fc7fc246bfc3b00e01aea1c2854c70a366348574ab50c57dfe796d24a0101d"}, "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"}, - "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, + "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, @@ -108,20 +108,20 @@ "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.1", "4a73bfa29d7780ffe33db345465919cef875034854649c37ac789eb8e8f38b21", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ee43b14e6866123a3ee1344e3c0d3d7591f4537542c2a925fcdbf46249c9b50b"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "pgoutput_decoder": {:hex, :pgoutput_decoder, "0.1.0", "d4ffae6e58a563f2e6de8a0495d9f9afbe2f4ac75d6805419cd4a0d05f414c00", [:mix], [], "hexpm", "4dbecbe4eb8de728178fd129ccba810bccafa9a8769c6714c8b7b22963081c27"}, - "phoenix": {:hex, :phoenix, "1.7.10", "02189140a61b2ce85bb633a9b6fd02dff705a5f1596869547aeb2b2b95edd729", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "cf784932e010fd736d656d7fead6a584a4498efefe5b8227e9f383bf15bb79d0"}, + "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.3", "86e9878f833829c3f66da03d75254c155d91d72a201eb56ae83482328dc7ca93", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d36c401206f3011fefd63d04e8ef626ec8791975d9d107f9a0817d426f61ac07"}, - "phoenix_html": {:hex, :phoenix_html, "3.3.3", "380b8fb45912b5638d2f1d925a3771b4516b9a78587249cabe394e0a5d579dc9", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "923ebe6fec6e2e3b3e569dfbdc6560de932cd54b000ada0208b5f45024bdd76c"}, + "phoenix_html": {:hex, :phoenix_html, "3.3.4", "42a09fc443bbc1da37e372a5c8e6755d046f22b9b11343bf885067357da21cb3", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0249d3abec3714aff3415e7ee3d9786cb325be3151e6c4b3021502c585bf53fb"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.3", "7ff51c9b6609470f681fbea20578dede0e548302b0c8bdf338b5a753a4f045bf", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "f9470a0a8bae4f56430a23d42f977b5a6205fdba6559d76f932b876bfaec652d"}, "phoenix_live_react": {:hex, :phoenix_live_react, "0.4.2", "8a37f3cccd26c3d992ffd677a7b8ecbd7746980caf560ac5d9c5efb7eb260910", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.11 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "eb0fb004afc0904510915b6b3e9458fc3144694dcd5a736d9d2feb00197becdc"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"}, "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.1", "92a37acf07afca67ac98bd326532ba8f44ad7d4bdf3e4361b03f7f02594e5ae9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "be494fd1215052729298b0e97d5c2ce8e719c00854b82cd8cf15c1cd7fcf6294"}, "phoenix_oauth2_provider": {:git, "https://github.com/Logflare/phoenix_oauth2_provider.git", "36e757920e54bffa0f603bf985aff91bb8b28f2d", [ref: "36e757920e54bffa0f603bf985aff91bb8b28f2d"]}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, - "phoenix_template": {:hex, :phoenix_template, "1.0.3", "32de561eefcefa951aead30a1f94f1b5f0379bc9e340bb5c667f65f1edfa4326", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "16f4b6588a4152f3cc057b9d0c0ba7e82ee23afa65543da535313ad8d25d8e2c"}, - "phoenix_view": {:hex, :phoenix_view, "2.0.3", "4d32c4817fce933693741deeb99ef1392619f942633dde834a5163124813aad3", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "cd34049af41be2c627df99cd4eaa71fc52a328c0c3d8e7d4aa28f880c30e7f64"}, - "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"}, + "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, + "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, + "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, "plug_caisson": {:hex, :plug_caisson, "0.2.1", "aa6a45a4f0e674459b8881d742cc0e8c7d5d0e008a29fe84dc10ab95d6fcfa74", [:mix], [{:brotli, "~> 0.3.2", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "661887bca916122c31717842fa6496c5a4d92c22b5dbef6bd6973d28188939cc"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.2", "fdadb973799ae691bf9ecad99125b16625b1c6039999da5fe544d99218e662e4", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "245d8a11ee2306094840c000e8816f0cbed69a23fc0ac2bcf8d7835ae019bb2f"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, "poison": {:hex, :poison, "5.0.0", "d2b54589ab4157bbb82ec2050757779bfed724463a544b6e20d79855a9e43b24", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "11dc6117c501b80c62a7594f941d043982a1bd05a1184280c0d9166eb4d8d3fc"}, "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"}, @@ -148,7 +148,7 @@ "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, "tesla": {:hex, :tesla, "1.9.0", "8c22db6a826e56a087eeb8cdef56889731287f53feeb3f361dec5d4c8efb6f14", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "7c240c67e855f7e63e795bf16d6b3f5115a81d1f44b7fe4eadbf656bae0fef8a"}, "test_server": {:hex, :test_server, "0.1.16", "403d6cebaa7ad1d08c0ca9475af48836c45ae0f64cda9f4e88b0f17310c5c452", [:mix], [{:bandit, ">= 1.4.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 2.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:x509, "~> 0.6", [hex: :x509, repo: "hexpm", optional: false]}], "hexpm", "f27cfc858d494e4df413a962fbe4d7aa3daf1a3824307f0b046bb8cec6ac8e42"}, - "thousand_island": {:hex, :thousand_island, "1.2.0", "4f548ae771ab5f96bc7e199f9824c0c2ce6d365f8c93f5f64dbbb33988e484bf", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "521671fea179672addb6af46455fc2a77be1edda4c0ed351633e0ef37a4b3584"}, + "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, "timber_logfmt": {:git, "https://github.com/Logflare/logfmt-elixir.git", "9766367ccb3014b47b0a621a489261011dcea769", []}, "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "tls_certificate_check": {:hex, :tls_certificate_check, "1.24.0", "d00e2887551ff8cdae4d0340d90d9fcbc4943c7b5f49d32ed4bc23aff4db9a44", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "90b25a58ee433d91c17f036d4d354bf8859a089bfda60e68a86f8eecae45ef1b"}, @@ -165,6 +165,6 @@ "uri_query": {:hex, :uri_query, "0.1.2", "ae35b83b472f3568c2c159eee3f3ccf585375d8a94fb5382db1ea3589e75c3b4", [:mix], [], "hexpm", "e3bc81816c98502c36498b9b2f239b89c71ce5eadfff7ceb2d6c0a2e6ae2ea0c"}, "warpath": {:hex, :warpath, "0.5.0", "906ca2efe1bbd07f2dafc2159897e2cb3068e567210c81c2d9858addc172df50", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a9b4f518bfce91818fd52bc8e7656259844167436f14c0c1a01a6ca7e061f671"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, - "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, + "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"}, "x509": {:hex, :x509, "0.8.8", "aaf5e58b19a36a8e2c5c5cff0ad30f64eef5d9225f0fd98fb07912ee23f7aba3", [:mix], [], "hexpm", "ccc3bff61406e5bb6a63f06d549f3dba3a1bbb456d84517efaaa210d8a33750f"}, } diff --git a/test/profiling/context_cache.exs b/test/profiling/context_cache.exs new file mode 100644 index 000000000..25d7e27a7 --- /dev/null +++ b/test/profiling/context_cache.exs @@ -0,0 +1,46 @@ +alias Logflare.Sources +alias Logflare.Users +require Phoenix.ConnTest +Mimic.copy(Broadway) +Mimic.copy(Logflare.Backends) +Mimic.copy(Logflare.Logs) +Mimic.copy(Logflare.Partners) + +Mimic.stub(Logflare.Backends, :ingest_logs, fn _, _ -> :ok end) +Mimic.stub(Logflare.Logs, :ingest_logs, fn _, _ -> :ok end) +# Mimic.stub(Broadway, :push_messages, fn _, _ -> :ok end) +ver = System.argv() |> Enum.at(0) + +v1_source = Sources.get(:"9f37d86e-e4fa-4ef2-a47e-e8d4ac1fceba") + +v2_source = Sources.get(:"94d07aab-30f5-460e-8871-eb85f4674e35") + +user = Users.get(v1_source.user_id) +str_key = inspect({:get, [user.id]}) + +Benchee.run( + %{ + "Cachex.get/3" => fn _ -> + Cachex.get(Logflare.Users.Cache, {:get, [user.id]}) + end, + "Logflare.Users.get/1" => fn _ -> + Logflare.Users.Cache.get(user.id) + end + }, + before_scenario: fn input -> + Cachex.put(Logflare.Users.Cache, {:get, [user.id]}, {:cached, user}) + Cachex.put(Logflare.Users.Cache, str_key, {:cached, user}) + {input, nil} + end, + time: 4, + memory_time: 0 +) + +# Before 090f8d93: +# Name ips average deviation median 99th % +# Cachex.get/3 1.40 M 714.86 ns ±2833.04% 625 ns 833 ns +# ContextCache.apply_fun/3 1.03 M 968.97 ns ±1686.53% 792 ns 1167 ns + +# Comparison: +# Cachex.get/3 1.40 M +# ContextCache.apply_fun/3 1.03 M - 1.36x slower +254.10 ns diff --git a/test/profiling/context_cache_cold_start.exs b/test/profiling/context_cache_cold_start.exs new file mode 100644 index 000000000..fb396c32a --- /dev/null +++ b/test/profiling/context_cache_cold_start.exs @@ -0,0 +1,54 @@ +alias Logflare.Sources +alias Logflare.Users +require Phoenix.ConnTest +Mimic.copy(Broadway) +Mimic.copy(Logflare.Backends) +Mimic.copy(Logflare.Logs) +Mimic.copy(Logflare.Partners) + +Mimic.stub(Logflare.Backends, :ingest_logs, fn _, _ -> :ok end) +Mimic.stub(Logflare.Logs, :ingest_logs, fn _, _ -> :ok end) +# Mimic.stub(Broadway, :push_messages, fn _, _ -> :ok end) +ver = System.argv() |> Enum.at(0) + +v1_source = Sources.get(:"9f37d86e-e4fa-4ef2-a47e-e8d4ac1fceba") + +v2_source = Sources.get(:"94d07aab-30f5-460e-8871-eb85f4674e35") + +user = Users.get(v1_source.user_id) + +apply_args = {:get, [user.id]} + +Benchee.run( + %{ + "Cachex.fetch/3" => fn -> + Cachex.fetch(Logflare.Users.Cache, apply_args, fn {fun, args} -> + value = apply(Logflare.Users, :get, [user.id]) + + # keys_key = {{Logflare.Users, Logflare.ContextCache.select_key(value)}, :erlang.phash2(apply_args)} + # Cachex.put(Logflare.ContextCache, keys_key, apply_args) + + {:commit, {:cached, value}} + end) + end, + "Users.Cache.get/1" => fn -> + Logflare.Users.Cache.get(user.id) + end + }, + before_each: fn input -> + Cachex.clear(Logflare.Users.Cache) + Cachex.clear(Logflare.ContextCache) + input + end, + time: 4, + memory_time: 0 +) + +# Before 090f8d93: +# Name ips average deviation median 99th % +# Users.Cache.get/1 1.67 K 598.09 μs ±69.70% 546.42 μs 1175.40 μs +# Cachex.fetch/3 1.41 K 707.75 μs ±38.63% 664.25 μs 1510.41 μs + +# Comparison: +# Users.Cache.get/1 1.67 K +# Cachex.fetch/3 1.41 K - 1.18x slower +109.67 μs diff --git a/test/profiling/ingest_http.exs b/test/profiling/ingest_http.exs new file mode 100644 index 000000000..d12b69114 --- /dev/null +++ b/test/profiling/ingest_http.exs @@ -0,0 +1,60 @@ +alias Logflare.Sources +alias Logflare.Users +require Phoenix.ConnTest +Mimic.copy(Broadway) +Mimic.copy(Logflare.Backends) +Mimic.copy(Logflare.Logs) +Mimic.copy(Logflare.Partners) + +Mimic.stub(Logflare.Backends, :ingest_logs, fn _, _ -> :ok end) +Mimic.stub(Logflare.Logs, :ingest_logs, fn _, _ -> :ok end) +# Mimic.stub(Broadway, :push_messages, fn _, _ -> :ok end) +ver = System.argv() |> Enum.at(0) + +v1_source = Sources.get(:"9f37d86e-e4fa-4ef2-a47e-e8d4ac1fceba") + +v2_source = Sources.get(:"94d07aab-30f5-460e-8871-eb85f4674e35") + +user = Users.get(v1_source.user_id) + +Benchee.run( + %{ + "pipeline" => fn input -> + Phoenix.ConnTest.build_conn() + |> Phoenix.ConnTest.dispatch( + LogflareWeb.Endpoint, + :post, + "/api/logs?source=#{input.token}&api_key=#{user.api_key}", + %{ + message: "some msg", + field: "1234", + testing: 123 + } + ) + end + }, + inputs: %{ + "v1" => v1_source, + "v2" => v2_source + }, + time: 10, + memory_time: 0 +) + +# Before 090f8d93: +# ##### With input v1 ##### +# Name ips average deviation median 99th % +# pipeline 36.05 27.74 ms ±20.18% 26.72 ms 47.35 ms + +# ##### With input v2 ##### +# Name ips average deviation median 99th % +# pipeline 37.07 26.98 ms ±21.35% 25.97 ms 55.37 ms + +# With bandit (2024-12-10): +# ##### With input v1 ##### +# Name ips average deviation median 99th % +# pipeline 44.25 22.60 ms ±14.53% 21.89 ms 34.32 ms + +# ##### With input v2 ##### +# Name ips average deviation median 99th % +# pipeline 45.08 22.18 ms ±11.26% 22.16 ms 29.60 ms diff --git a/test/profiling/ingest_http_plugs.exs b/test/profiling/ingest_http_plugs.exs new file mode 100644 index 000000000..c863b54df --- /dev/null +++ b/test/profiling/ingest_http_plugs.exs @@ -0,0 +1,135 @@ +alias Logflare.Sources +alias Logflare.Users +alias Logflare.Partners +require Phoenix.ConnTest +Mimic.copy(Broadway) +Mimic.copy(Logflare.Backends) +Mimic.copy(Logflare.Logs) +Mimic.copy(Logflare.Partners) + +Mimic.stub(Logflare.Backends, :ingest_logs, fn _, _ -> :ok end) +Mimic.stub(Logflare.Logs, :ingest_logs, fn _, _ -> :ok end) +# Mimic.stub(Broadway, :push_messages, fn _, _ -> :ok end) +ver = System.argv() |> Enum.at(0) + +v1_source = Sources.get(:"9f37d86e-e4fa-4ef2-a47e-e8d4ac1fceba") + +v2_source = Sources.get(:"94d07aab-30f5-460e-8871-eb85f4674e35") + +user = Users.get(v1_source.user_id) + +Benchee.run( + %{ + "VerifyApiAccess" => + {fn {source, conn} -> + LogflareWeb.Plugs.VerifyApiAccess.call(conn, scopes: ~w(public)) + end, + before_scenario: fn {source, conn} -> + {source, conn} + end}, + "FetchResource" => + {fn {source, conn} -> + LogflareWeb.Plugs.FetchResource.call(conn, []) + end, + before_scenario: fn {source, conn} -> + conn = + conn + |> LogflareWeb.Plugs.VerifyApiAccess.call(scopes: ~w(public)) + + {source, conn} + end}, + "VerifyResourceOwnership" => + {fn {source, conn} -> + LogflareWeb.Plugs.VerifyResourceOwnership.call(conn, []) + end, + before_scenario: fn {source, conn} -> + conn = + conn + |> LogflareWeb.Plugs.VerifyApiAccess.call(scopes: ~w(public)) + |> LogflareWeb.Plugs.FetchResource.call([]) + + {source, conn} + end}, + "SetPlanFromCache" => + {fn {source, conn} -> + LogflareWeb.Plugs.SetPlanFromCache.call(conn, []) + end, + before_scenario: fn {source, conn} -> + conn = + conn + |> LogflareWeb.Plugs.VerifyApiAccess.call(scopes: ~w(public)) + |> LogflareWeb.Plugs.FetchResource.call([]) + |> LogflareWeb.Plugs.VerifyResourceOwnership.call([]) + + {source, conn} + end}, + "RateLimiter" => + {fn {source, conn} -> + LogflareWeb.Plugs.RateLimiter.call(conn, []) + end, + before_scenario: fn {source, conn} -> + conn = + conn + |> LogflareWeb.Plugs.VerifyApiAccess.call(scopes: ~w(public)) + |> LogflareWeb.Plugs.FetchResource.call([]) + |> LogflareWeb.Plugs.VerifyResourceOwnership.call([]) + |> LogflareWeb.Plugs.SetPlanFromCache.call([]) + + {source, conn} + end}, + "BufferLimiter" => + {fn {source, conn} -> + LogflareWeb.Plugs.BufferLimiter.call(conn, []) + end, + before_scenario: fn {source, conn} -> + conn = + conn + |> LogflareWeb.Plugs.VerifyApiAccess.call(scopes: ~w(public)) + |> LogflareWeb.Plugs.FetchResource.call([]) + |> LogflareWeb.Plugs.VerifyResourceOwnership.call([]) + |> LogflareWeb.Plugs.SetPlanFromCache.call([]) + |> LogflareWeb.Plugs.RateLimiter.call([]) + + {source, conn} + end} + }, + inputs: %{ + "v1" => v1_source + # "v2" => v2_source + }, + before_scenario: fn input -> + prepared_conn = + Phoenix.ConnTest.build_conn( + :post, + "/api/logs?source=#{input.token}&api_key=#{user.api_key}", + %{ + message: "some msg", + field: "1234", + testing: 123 + } + ) + |> Plug.Conn.assign(:resource_type, :source) + + {input, prepared_conn} + end, + time: 4, + memory_time: 0 +) + +# Before 090f8d93: +# ##### With input v1 ##### +# Name ips average deviation median 99th % +# VerifyResourceOwnership 14120.42 K 0.0708 μs ±25384.39% 0.0830 μs 0.0840 μs +# FetchResource 668.87 K 1.50 μs ±1270.86% 1.33 μs 2.58 μs +# BufferLimiter 332.39 K 3.01 μs ±428.79% 2.79 μs 4.67 μs +# VerifyApiAccess 111.55 K 8.96 μs ±168.32% 7.46 μs 27.17 μs +# RateLimiter 50.82 K 19.68 μs ±23.81% 19.17 μs 26.04 μs +# SetPlanFromCache 36.09 K 27.71 μs ±22.85% 25.29 μs 40.29 μs + +# Comparison: +# VerifyResourceOwnership 14120.42 K +# FetchResource 668.87 K - 21.11x slower +1.42 μs +# BufferLimiter 332.39 K - 42.48x slower +2.94 μs +# VerifyApiAccess 111.55 K - 126.58x slower +8.89 μs +# RateLimiter 50.82 K - 277.83x slower +19.61 μs +# SetPlanFromCache 36.09 K - 391.23x slower +27.64 μs