diff --git a/config/config.exs b/config/config.exs index f71ae1f..2c18cc1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -63,6 +63,12 @@ config :logger, :console, # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason +# Use Oban for background jobs +config :munch, Oban, + engine: Oban.Engines.Basic, + queues: [default: 10], + repo: Munch.Repo + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" diff --git a/config/test.exs b/config/test.exs index f2ae508..763ed92 100644 --- a/config/test.exs +++ b/config/test.exs @@ -39,3 +39,5 @@ config :phoenix, :plug_init_mode, :runtime # Enable helpful, but potentially expensive runtime checks config :phoenix_live_view, enable_expensive_runtime_checks: true + +config :munch, Oban, testing: :inline diff --git a/lib/munch/application.ex b/lib/munch/application.ex index 12c27b4..6890eb8 100644 --- a/lib/munch/application.ex +++ b/lib/munch/application.ex @@ -10,6 +10,7 @@ defmodule Munch.Application do children = [ MunchWeb.Telemetry, Munch.Repo, + {Oban, Application.get_env(:munch, Oban)}, {DNSCluster, query: Application.get_env(:munch, :dns_cluster_query) || :ignore}, {Phoenix.PubSub, name: Munch.PubSub}, # Start a worker by calling: Munch.Worker.start_link(arg) diff --git a/lib/munch/importer.ex b/lib/munch/importer.ex new file mode 100644 index 0000000..3cbb63b --- /dev/null +++ b/lib/munch/importer.ex @@ -0,0 +1,11 @@ +defmodule Munch.Importer do + use Oban.Worker, queue: :fresh_restaurants + + alias Munch.Restaurants + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"osm_type" => osm_type, "osm_id" => osm_id}}) do + {:ok, restaurant} = Munch.Osm.nominatim_get_details(osm_type, osm_id) + Restaurants.create_restaurant(restaurant) + end +end diff --git a/lib/munch/osm.ex b/lib/munch/osm.ex index eb416a2..8f3267e 100644 --- a/lib/munch/osm.ex +++ b/lib/munch/osm.ex @@ -7,60 +7,149 @@ defmodule Munch.Osm do alias Munch.Repo alias Munch.Restaurants - def api_get_object(osm_type, osm_id) do - resp = Req.get!("https://api.openstreetmap.org/api/0.6/#{osm_type}/#{osm_id}.json") + def osm_get!(path) do + Req.get!( + url: path, + base_url: "https://api.openstreetmap.org/api/0.6", + headers: [user_agent: "Munch/0.1"] + ) + end + + def nominatim_get!(path) do + Req.get!( + url: path, + base_url: "https://nominatim.openstreetmap.org", + headers: [user_agent: "Munch/0.1"] + ) + end - decoded = Jason.decode!(resp.body) + def osm_get_object(:node, osm_id) do + resp = osm_get!("node/#{osm_id}.json") - case decoded do + case resp.body do %{"elements" => [%{"lat" => lattitude, "lon" => longitude, "tags" => tags}]} -> - %{lattitude: lattitude, longitude: longitude, tags: tags} + {:ok, %{lattitude: lattitude, longitude: longitude, tags: tags}} _ -> - raise "Invalid response from OpenStreetMaps API: #{decoded}" + {:error, "Invalid response from OpenStreetMaps API: #{resp}"} end end - def pull_restaurant(osm_type, osm_id) do - object = api_get_object(osm_type, osm_id) - %{"name" => name} = object.tags - - Repo.insert!(%Restaurants.Restaurant{ - osm_type: osm_type, - osm_id: osm_id, - name: name, - location: %Geo.Point{coordinates: {object.longitude, object.lattitude}, srid: 4326}, - note: "", - iso_country_subdivision: "", - secondary_subdivision: "" - }) + def osm_get_object(:way, _osm_id) do + {:error, "Importing ways is not implemented yet"} + end + + def osm_get_object(:relation, _osm_id) do + {:error, "Importing relations is not implemented yet"} + end + + @doc """ + Import a restaurant and queue a task to pull its details from Nominatim. + """ + def import_fresh_restaurant(osm_type, osm_id) do + with {:ok, object} <- osm_get_object(osm_type, osm_id), + {:ok, name} <- Map.fetch(object.tags, "name") do + point = %Geo.Point{coordinates: {object.longitude, object.lattitude}, srid: 4326} + + Oban.insert( + Munch.Importer.new(%{ + "osm_type" => osm_type, + "osm_id" => osm_id + }) + ) + + Repo.insert!(%Restaurants.Restaurant{ + osm_type: osm_type, + osm_id: osm_id, + name: name, + location: point, + note: "", + country: "", + iso_country_subdivision: "" + }) + end end - def copy_osm_restaurants() do - osm_restaurants = Repo.all(Munch.Osm.Restaurant) - - restaurants = - osm_restaurants - |> Enum.map(fn osm_restaurant -> - %Restaurants.Restaurant{ - osm_type: osm_restaurant.osm_type, - osm_id: osm_restaurant.osm_id, - name: osm_restaurant.tags["name"], - location: osm_restaurant.location, - note: "", - iso_country_subdivision: nil, - secondary_subdivision: nil + def nominatim_get_details(osm_type, osm_id) do + osm_type_char = + case osm_type do + :node -> "N" + :way -> "W" + :relation -> "R" + end + + resp = nominatim_get!("lookup?osm_ids=#{osm_type_char}#{osm_id}&format=jsonv2") + + case resp.body do + [ + %{ + "lat" => latitude, + "lon" => longitude, + "name" => name, + "display_name" => display_name, + "address" => + %{ + "ISO3166-2-lvl4" => iso_country_subdivision, + "country" => country + } = address } - end) - |> IO.inspect() + ] -> + {:ok, + %{ + osm_type: osm_type, + osm_id: osm_id, + name: name, + display_name: display_name, + location: %Geo.Point{ + coordinates: {String.to_float(longitude), String.to_float(latitude)}, + srid: 4326 + }, + note: "", + country: country, + iso_country_subdivision: iso_country_subdivision, + address: address + }} - restaurants - |> Enum.map(fn restaurant -> - Repo.insert!(restaurant) - end) + _ -> + {:error, "Invalid response from Nominatim API: #{resp}"} + end end - def sync_restaurant(_restaurant) do - IO.inspect("TODO: Implement sync with OSM") + @doc """ + Search for restaurants with Nominatim. + + options: + + * `:exclude` - A list of place IDs to exclude from the results. + """ + def search_restaurants(search, opts \\ []) do + exclude = Keyword.get(opts, :exclude, []) + # TODO: use AI to assemble a structured query from the search string + resp = + nominatim_get!( + "search?q=#{search}&layer=poi&exclude=#{exclude |> Enum.join(",")}&format=jsonv2" + ) + + resp.body + |> Map.new(fn x -> + case x do + %{ + "place_id" => place_id, + "lat" => lattitude, + "lon" => longitude, + "display_name" => display_name, + "osm_type" => osm_type, + "osm_id" => osm_id + } -> + {place_id, + %{ + lattitude: lattitude, + longitude: longitude, + display_name: display_name, + osm_type: String.to_atom(osm_type), + osm_id: osm_id + }} + end + end) end end diff --git a/lib/munch/osm/restaurant.ex b/lib/munch/osm/restaurant.ex deleted file mode 100644 index 7c3a33a..0000000 --- a/lib/munch/osm/restaurant.ex +++ /dev/null @@ -1,12 +0,0 @@ -defmodule Munch.Osm.Restaurant do - use Ecto.Schema - - @schema_prefix "osm" - @primary_key false - schema "restaurants" do - field :osm_type, Ecto.Enum, values: [node: "N", way: "W", relation: "R"], primary_key: true - field :osm_id, :integer, primary_key: true - field :location, Geo.PostGIS.Geometry - field :tags, :map - end -end diff --git a/lib/munch/restaurants.ex b/lib/munch/restaurants.ex index 7e5d426..5522741 100644 --- a/lib/munch/restaurants.ex +++ b/lib/munch/restaurants.ex @@ -22,13 +22,7 @@ defmodule Munch.Restaurants do end @doc """ - Search restaurants by name or address. Words are aggregated with AND. - - ## Examples - - iex> search_restaurants("Ma") - [%Restaurant{}, ...] - + Search local restaurants by name or address. Words are aggregated with AND. """ def search_restaurants(search) do words = String.split(search) diff --git a/lib/munch/restaurants/restaurant.ex b/lib/munch/restaurants/restaurant.ex index 6cfc4cf..04eedf1 100644 --- a/lib/munch/restaurants/restaurant.ex +++ b/lib/munch/restaurants/restaurant.ex @@ -8,10 +8,12 @@ defmodule Munch.Restaurants.Restaurant do field :osm_type, Ecto.Enum, values: [node: "N", way: "W", relation: "R"] field :osm_id, :integer field :name, :string + field :display_name, :string field :location, Geo.PostGIS.Geometry field :note, :string + field :country, :string field :iso_country_subdivision, :string - field :secondary_subdivision, :string + field :address, :map timestamps(type: :utc_datetime) end @@ -19,6 +21,16 @@ defmodule Munch.Restaurants.Restaurant do @doc false def changeset(restaurant, attrs) do restaurant - |> cast(attrs, [:note]) + |> cast(attrs, [ + :osm_type, + :osm_id, + :name, + :display_name, + :location, + :note, + :country, + :iso_country_subdivision, + :address + ]) end end diff --git a/lib/munch_web/live/restaurant_live/edit.ex b/lib/munch_web/live/restaurant_live/edit.ex index 7862b57..5a0ed14 100644 --- a/lib/munch_web/live/restaurant_live/edit.ex +++ b/lib/munch_web/live/restaurant_live/edit.ex @@ -59,7 +59,11 @@ defmodule MunchWeb.RestaurantLive.Edit do end def handle_event("trigger_sync", _params, socket) do - Osm.sync_restaurant(socket.assigns.restaurant) + Osm.import_fresh_restaurant( + socket.assigns.restaurant.osm_type, + socket.assigns.restaurant.osm_id + ) + {:noreply, socket} end diff --git a/lib/munch_web/live/restaurant_live/import.ex b/lib/munch_web/live/restaurant_live/import.ex new file mode 100644 index 0000000..8ce30ce --- /dev/null +++ b/lib/munch_web/live/restaurant_live/import.ex @@ -0,0 +1,54 @@ +defmodule MunchWeb.RestaurantLive.Import do + use MunchWeb, :live_view + + alias Munch.Osm + alias Munch.Restaurants + + @impl true + def render(assigns) do + ~H""" + <.header> + Add a new restaurant by search + + + <.simple_form for={@form} id="search-new-form" phx-submit="search"> + <.input field={@form[:search]} type="text" label="Search" /> + <:actions> + <.button phx-disable-with="Searching...">Search + + + + """ + end + + @impl true + def mount(_params, _session, socket) do + {:ok, socket |> assign(form: to_form(%{})) |> assign(results: %{})} + end + + @impl true + def handle_event("search", %{"search" => search}, socket) do + {:noreply, socket |> assign(results: Osm.search_restaurants(search))} + end + + def handle_event("import", %{"place_id" => place_id}, socket) do + place = socket.assigns.results[place_id] + {:ok, restaurant} = Osm.nominatim_get_details(place.osm_type, place.osm_id) + Restaurants.create_restaurant(restaurant) + {:noreply, socket} + end +end diff --git a/lib/munch_web/live/restaurant_live/new.ex b/lib/munch_web/live/restaurant_live/import_manual.ex similarity index 84% rename from lib/munch_web/live/restaurant_live/new.ex rename to lib/munch_web/live/restaurant_live/import_manual.ex index b8dc9c2..01c9522 100644 --- a/lib/munch_web/live/restaurant_live/new.ex +++ b/lib/munch_web/live/restaurant_live/import_manual.ex @@ -1,4 +1,4 @@ -defmodule MunchWeb.RestaurantLive.New do +defmodule MunchWeb.RestaurantLive.ImportManual do use MunchWeb, :live_view alias Munch.Restaurants @@ -13,9 +13,8 @@ defmodule MunchWeb.RestaurantLive.New do

- New restaurants and other changes to the OpenStreetMap map should be - imported automatically every 24 hours. You can import a restaurant - ahead of schedule by entering its osm_type and osm_id below. + If you can't find the restaurant by search, you can import a restaurant + by entering its osm_type and osm_id below.

<.simple_form for={@form} id="osm-form" phx-submit="save"> <.input @@ -52,7 +51,7 @@ defmodule MunchWeb.RestaurantLive.New do @impl true def handle_event("save", %{"osm_type" => osm_type, "osm_id" => osm_id}, socket) do - case Osm.pull_restaurant(osm_type, osm_id) do + case Osm.import_fresh_restaurant(osm_type, osm_id) do {:ok, restaurant} -> {:noreply, socket diff --git a/lib/munch_web/live/restaurant_live/index.ex b/lib/munch_web/live/restaurant_live/index.ex index 9e54103..0cc8192 100644 --- a/lib/munch_web/live/restaurant_live/index.ex +++ b/lib/munch_web/live/restaurant_live/index.ex @@ -21,7 +21,9 @@ defmodule MunchWeb.RestaurantLive.Index do row_click={fn {_id, restaurant} -> JS.navigate(~p"/restaurant/#{restaurant}") end} > <:col :let={{_id, restaurant}} label="Name"><%= restaurant.name %> - <:col :let={{_id, restaurant}} label="Address"><%= restaurant.address %> + <:col :let={{_id, restaurant}} label="Address"> + <%= restaurant.osm_type %> <%= restaurant.osm_id %> + <:action :let={{_id, restaurant}}>
<.link navigate={~p"/restaurant/#{restaurant}"}>Show diff --git a/lib/munch_web/live/restaurant_live/select_component.ex b/lib/munch_web/live/restaurant_live/select_component.ex index 1645fb2..84e8901 100644 --- a/lib/munch_web/live/restaurant_live/select_component.ex +++ b/lib/munch_web/live/restaurant_live/select_component.ex @@ -31,32 +31,33 @@ defmodule MunchWeb.RestaurantLive.SelectComponent do autofocus placeholder="Search for a restaurant" /> - + <.button phx-disable-with="Searching...">Search +
""" end - def handle_event("validate", params = %{"search" => search}, socket) do + def handle_event("save", params = %{"search" => search}, socket) do {:noreply, socket |> assign(:form, to_form(params)) @@ -72,12 +73,8 @@ defmodule MunchWeb.RestaurantLive.SelectComponent do )} end - def handle_event("save", %{"restaurant_id" => restaurant_id}, socket) do + def handle_event("select", %{"restaurant_id" => restaurant_id}, socket) do send(self(), {:restaurant_selected, socket.assigns.tag, restaurant_id}) {:noreply, socket |> assign(:form, to_form(%{})) |> assign(:restaurants, nil)} end - - def handle_event("save", _params, socket) do - {:noreply, socket} - end end diff --git a/lib/munch_web/live/restaurant_live/show.ex b/lib/munch_web/live/restaurant_live/show.ex index 2fd2b0f..55225d8 100644 --- a/lib/munch_web/live/restaurant_live/show.ex +++ b/lib/munch_web/live/restaurant_live/show.ex @@ -18,10 +18,6 @@ defmodule MunchWeb.RestaurantLive.Show do <.list> <:item title="Name"><%= @restaurant.name %> - <:item title="Country"><%= @restaurant.country %> - <:item title="City"><%= @restaurant.city %> - <:item title="Neighbourhood"><%= @restaurant.neighbourhood %> - <:item title="Address"><%= @restaurant.address %> <.back navigate={~p"/restaurants"}>Back to restaurants diff --git a/lib/munch_web/router.ex b/lib/munch_web/router.ex index 0068864..7e9f082 100644 --- a/lib/munch_web/router.ex +++ b/lib/munch_web/router.ex @@ -62,7 +62,8 @@ defmodule MunchWeb.Router do live "/users/settings/confirm_email/:token", UserLive.Settings, :confirm_email live "/users/edit", UserLive.ProfileForm, :edit - live "/restaurants/new", RestaurantLive.New, :new + live "/restaurants/new", RestaurantLive.Import, :new + live "/restaurants/new-by-id", RestaurantLive.ImportManual, :new live "/restaurant/:id/edit", RestaurantLive.Edit, :edit live "/lists/new", ListLive.Form, :new diff --git a/mix.exs b/mix.exs index 72820c2..7f190bf 100644 --- a/mix.exs +++ b/mix.exs @@ -60,7 +60,8 @@ defmodule Munch.MixProject do {:dns_cluster, "~> 0.1"}, {:bandit, "~> 1.5"}, {:geo, "~> 4.0"}, - {:geo_postgis, "~> 3.7"} + {:geo_postgis, "~> 3.7"}, + {:oban, "~> 2.17"} ] end diff --git a/mix.lock b/mix.lock index 5fe1f8d..6f662ff 100644 --- a/mix.lock +++ b/mix.lock @@ -25,6 +25,7 @@ "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "oauth2": {:hex, :oauth2, "2.1.0", "beb657f393814a3a7a8a15bd5e5776ecae341fd344df425342a3b6f1904c2989", [:mix], [{:tesla, "~> 1.5", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "8ac07f85b3307dd1acfeb0ec852f64161b22f57d0ce0c15e616a1dfc8ebe2b41"}, + "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"}, "phoenix": {:git, "https://github.com/phoenixframework/phoenix.git", "222d9aba8595f81148d454d60d2072d11a969a09", []}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"}, "phoenix_html": {:hex, :phoenix_html, "4.1.1", "4c064fd3873d12ebb1388425a8f2a19348cef56e7289e1998e2d2fa758aa982e", [:mix], [], "hexpm", "f2f2df5a72bc9a2f510b21497fd7d2b86d932ec0598f0210fed4114adc546c6f"}, diff --git a/osm/download_osm.exs b/osm/download_osm.exs deleted file mode 100644 index e496489..0000000 --- a/osm/download_osm.exs +++ /dev/null @@ -1,43 +0,0 @@ -# script for importing osm data into the database - -osm_dir = "/home/leif/Documents/munch/osm" - -IO.inspect(osm_dir) - -# change the area line to -# area["ISO3166-2"="AU-NSW"]; -# when it's ready -overpass_query = """ -area["name"="Dunedin"]; -( - nwr[amenity=restaurant](area); - nwr[amenity=fast_food](area); - nwr[amenity=cafe](area); - nwr[amenity=bar](area); - nwr[amenity=pub](area); - nwr[amenity=food_court](area); - nwr[amenity=biergarten](area); -); -out; -""" - -overpass_uri = "https://overpass-api.de/api/interpreter" - -resp = Req.post!(overpass_uri, form: [data: overpass_query]) - -File.write!(Path.join(osm_dir, "export.osm"), resp.body) - -database_config = Munch.Repo.config() - -System.cmd("osm2pgsql", [ - "--style=#{Path.join(osm_dir, "style.lua")}", - "--output=flex", - "--schema=osm", - "--database=#{database_config[:database]}", - "--user=#{database_config[:username]}", - "--host=#{database_config[:hostname]}", - "--port=#{database_config[:port]}", - Path.join(osm_dir, "export.osm") -]) - -Munch.Osm.copy_osm_restaurants() diff --git a/osm/style.lua b/osm/style.lua deleted file mode 100644 index 5a879c5..0000000 --- a/osm/style.lua +++ /dev/null @@ -1,34 +0,0 @@ -local restaurants = osm2pgsql.define_table({ - name = 'restaurants', - ids = { - type = 'any', - type_column = 'osm_type', - id_column = 'osm_id', - create_index = 'unique', - }, - columns = { - { column = 'location', type = 'point'}, - { column = 'tags', type = 'jsonb' }, - } -}) - -local function process_object(object, point) - restaurants:insert({ - location = point, - tags = object.tags, - }) -end - -function osm2pgsql.process_node(object) - process_object(object, object:as_point()) -end - -function osm2pgsql.process_way(object) - -- TODO: figure out why :as_polygon() fails on some closed ways - process_object(object, object:as_polygon():centroid()) -end - -function osm2pgsql.process_relation(object) - assert(object.tags.type == 'multipolygon') - process_object(object, object:as_multipolygon():centroid()) -end diff --git a/priv/repo/migrations/20241008001216_create_restaurants.exs b/priv/repo/migrations/20241008001216_create_restaurants.exs index 6559be8..fb30080 100644 --- a/priv/repo/migrations/20241008001216_create_restaurants.exs +++ b/priv/repo/migrations/20241008001216_create_restaurants.exs @@ -7,12 +7,15 @@ defmodule Munch.Repo.Migrations.CreateRestaurants do add :osm_type, :text add :osm_id, :bigint add :name, :text, null: false + add :display_name, :text add :location, :geometry, null: false add :note, :text + add :country, :text add :iso_country_subdivision, :text - add :secondary_subdivision, :text - + add :address, :map timestamps(type: :utc_datetime) end + + create unique_index(:restaurants, [:osm_type, :osm_id]) end end diff --git a/priv/repo/migrations/20241125054254_add_oban_jobs_table.exs b/priv/repo/migrations/20241125054254_add_oban_jobs_table.exs new file mode 100644 index 0000000..ddcbd82 --- /dev/null +++ b/priv/repo/migrations/20241125054254_add_oban_jobs_table.exs @@ -0,0 +1,11 @@ +defmodule Munch.Repo.Migrations.AddObanJobsTable do + use Ecto.Migration + + def up do + Oban.Migration.up(version: 12) + end + + def down do + Oban.Migration.down(version: 1) + end +end