From 5713670ac95aaf5539c853489be664d71411b1d8 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Fri, 1 Mar 2019 12:57:22 +0100 Subject: [PATCH 1/7] Refactor Atom feed caching Signed-off-by: Thomas Citharel Fixes Fix tests Fix tests --- lib/mobilizon/application.ex | 28 +++++- .../controllers/activity_pub_controller.ex | 4 - .../controllers/feed_controller.ex | 95 +------------------ lib/mobilizon_web/router.ex | 10 +- lib/service/feed.ex | 94 ++++++++++++++++++ mix.exs | 3 +- mix.lock | 5 +- .../controllers/feed_controller_test.exs | 43 +++++++-- 8 files changed, 169 insertions(+), 113 deletions(-) create mode 100644 lib/service/feed.ex diff --git a/lib/mobilizon/application.ex b/lib/mobilizon/application.ex index 23019a318..bfe420ce1 100644 --- a/lib/mobilizon/application.ex +++ b/lib/mobilizon/application.ex @@ -3,6 +3,7 @@ defmodule Mobilizon.Application do The Mobilizon application """ use Application + import Cachex.Spec # See https://hexdocs.pm/elixir/Application.html # for more information on OTP Applications @@ -17,7 +18,32 @@ defmodule Mobilizon.Application do supervisor(MobilizonWeb.Endpoint, []), # Start your own worker by calling: Mobilizon.Worker.start_link(arg1, arg2, arg3) # worker(Mobilizon.Worker, [arg1, arg2, arg3]), - worker(Cachex, [:mobilizon, []]), + worker( + Cachex, + [ + :feed, + [ + limit: 2500, + expiration: + expiration( + default: :timer.minutes(60), + interval: :timer.seconds(60) + ), + fallback: fallback(default: &Mobilizon.Service.Feed.create_cache/1) + ] + ], + id: :cache_feed + ), + worker( + Cachex, + [ + :json, + [ + limit: 2500 + ] + ], + id: :cache_actor + ), worker(Guardian.DB.Token.SweeperServer, []), worker(Mobilizon.Service.Federator, []) ] diff --git a/lib/mobilizon_web/controllers/activity_pub_controller.ex b/lib/mobilizon_web/controllers/activity_pub_controller.ex index 631091bcf..c7c6a9636 100644 --- a/lib/mobilizon_web/controllers/activity_pub_controller.ex +++ b/lib/mobilizon_web/controllers/activity_pub_controller.ex @@ -21,10 +21,6 @@ defmodule MobilizonWeb.ActivityPubController do "application/activity+json, application/ld+json" ] - def actor(conn, %{"name" => _name, "_format" => "atom"} = params) do - MobilizonWeb.FeedController.actor(conn, params) - end - def actor(conn, %{"name" => name}) do with %Actor{} = actor <- Actors.get_local_actor_by_name(name) do if conn |> get_req_header("accept") |> is_ap_header() do diff --git a/lib/mobilizon_web/controllers/feed_controller.ex b/lib/mobilizon_web/controllers/feed_controller.ex index 30c7b4713..66c629664 100644 --- a/lib/mobilizon_web/controllers/feed_controller.ex +++ b/lib/mobilizon_web/controllers/feed_controller.ex @@ -4,102 +4,17 @@ defmodule MobilizonWeb.FeedController do """ use MobilizonWeb, :controller - alias Mobilizon.Actors - alias Mobilizon.Actors.Actor - alias Mobilizon.Events - alias Mobilizon.Events.Event - alias Atomex.{Feed, Entry} - import MobilizonWeb.Gettext - - @version Mix.Project.config()[:version] - def version(), do: @version - - def actor(conn, %{"name" => name, "_format" => format}) when format in ["atom"] do - name = String.replace_suffix(name, ".atom", "") - + def actor(conn, %{"name" => name, "format" => "atom"}) do with {status, data} when status in [:ok, :commit] <- - Cachex.fetch(:mobilizon, "actor_" <> format <> "_" <> name, &create_cache/1) do + Cachex.fetch(:feed, "actor_" <> name) do conn |> put_resp_content_type("application/atom+xml") |> send_resp(200, data) else _err -> - send_resp(conn, 404, "Not found") - end - end - - @spec create_cache(String.t()) :: {:commit, String.t()} | {:ignore, any()} - defp create_cache(key) do - with ["actor", type, name] <- String.split(key, "_", parts: 3), - {:ok, res} <- fetch_actor_event_feed(type, name) do - {:commit, res} - else - err -> - {:ignore, err} - end - end - - @spec fetch_actor_event_feed(String.t(), String.t()) :: String.t() - defp fetch_actor_event_feed(type, name) do - with %Actor{} = actor <- Actors.get_local_actor_by_name(name), - {:ok, events, _count} <- Events.get_public_events_for_actor(actor) do - {:ok, build_actor_feed(actor, events, type)} - else - err -> - {:error, err} - end - end - - @spec build_actor_feed(Actor.t(), list(), String.t()) :: String.t() - defp build_actor_feed(%Actor{} = actor, events, type) do - display_name = Actor.display_name(actor) - - # Title uses default instance language - feed = - Feed.new( - actor.url <> ".rss", - DateTime.utc_now(), - gettext("%{actor}'s public events feed", actor: display_name) - ) - |> Feed.author(display_name, uri: actor.url) - |> Feed.link(actor.url <> "." <> type, rel: "self") - |> Feed.link(actor.url, rel: "alternate") - |> Feed.generator("Mobilizon", uri: "https://joinmobilizon.org", version: version()) - |> Feed.entries(Enum.map(events, &get_entry/1)) - - feed = if actor.avatar_url, do: Feed.icon(feed, actor.avatar_url), else: feed - - feed = - if actor.banner_url, - do: Feed.logo(feed, actor.banner_url), - else: feed - - feed - |> Feed.build() - |> Atomex.generate_document() - end - - defp get_entry(%Event{} = event) do - with {:ok, html, []} <- Earmark.as_html(event.description) do - entry = - Entry.new(event.url, event.inserted_at, event.title) - |> Entry.link(event.url, rel: "alternate", type: "text/html") - |> Entry.content({:cdata, html}, type: "html") - - entry = if event.publish_at, do: Entry.published(entry, event.publish_at), else: entry - - # Add tags - entry = - event.tags - |> Enum.map(& &1.title) - |> Enum.uniq() - |> Enum.reduce(entry, fn tag, acc -> Entry.category(acc, tag) end) - - Entry.build(entry) - else - {:error, _html, error_messages} -> - require Logger - Logger.error("Unable to produce HTML for Markdown", details: inspect(error_messages)) + conn + |> put_resp_content_type("text/html") + |> send_file(404, "priv/static/index.html") end end end diff --git a/lib/mobilizon_web/router.ex b/lib/mobilizon_web/router.ex index 129ae2cb9..e41a0cdb0 100644 --- a/lib/mobilizon_web/router.ex +++ b/lib/mobilizon_web/router.ex @@ -22,9 +22,8 @@ defmodule MobilizonWeb.Router do plug(:accepts, ["activity-json", "html"]) end - pipeline :activity_pub_rss do - plug(TrailingFormatPlug) - plug(:accepts, ["activity-json", "html", "atom"]) + pipeline :rss do + plug(:accepts, ["atom", "html"]) end pipeline :browser do @@ -57,14 +56,15 @@ defmodule MobilizonWeb.Router do end scope "/", MobilizonWeb do - pipe_through(:activity_pub_rss) + pipe_through(:rss) - get("/@:name", ActivityPubController, :actor) + get("/@:name/feed/:format", FeedController, :actor) end scope "/", MobilizonWeb do pipe_through(:activity_pub) + get("/@:name", ActivityPubController, :actor) get("/@:name/outbox", ActivityPubController, :outbox) get("/@:name/following", ActivityPubController, :following) get("/@:name/followers", ActivityPubController, :followers) diff --git a/lib/service/feed.ex b/lib/service/feed.ex new file mode 100644 index 000000000..d63611154 --- /dev/null +++ b/lib/service/feed.ex @@ -0,0 +1,94 @@ +defmodule Mobilizon.Service.Feed do + @moduledoc """ + Serve Atom Syndication Feeds + """ + + alias Mobilizon.Actors + alias Mobilizon.Actors.Actor + alias Mobilizon.Events + alias Mobilizon.Events.Event + alias Atomex.{Feed, Entry} + import MobilizonWeb.Gettext + alias MobilizonWeb.Router.Helpers, as: Routes + alias MobilizonWeb.Endpoint + + @version Mix.Project.config()[:version] + def version(), do: @version + + @spec create_cache(String.t()) :: {:commit, String.t()} | {:ignore, any()} + def create_cache("actor_" <> name) do + with {:ok, res} <- fetch_actor_event_feed(name) do + {:commit, res} + else + err -> + {:ignore, err} + end + end + + @spec fetch_actor_event_feed(String.t()) :: String.t() + defp fetch_actor_event_feed(name) do + with %Actor{} = actor <- Actors.get_local_actor_by_name(name), + {:ok, events, _count} <- Events.get_public_events_for_actor(actor) do + {:ok, build_actor_feed(actor, events)} + else + err -> + {:error, err} + end + end + + # Build an atom feed from actor and it's public events + @spec build_actor_feed(Actor.t(), list()) :: String.t() + defp build_actor_feed(%Actor{} = actor, events) do + display_name = Actor.display_name(actor) + self_url = Routes.feed_url(Endpoint, :actor, actor.preferred_username, "atom") |> URI.decode() + + # Title uses default instance language + feed = + Feed.new( + self_url, + DateTime.utc_now(), + gettext("%{actor}'s public events feed", actor: display_name) + ) + |> Feed.author(display_name, uri: actor.url) + |> Feed.link(self_url, rel: "self") + |> Feed.link(actor.url, rel: "alternate") + |> Feed.generator("Mobilizon", uri: "https://joinmobilizon.org", version: version()) + |> Feed.entries(Enum.map(events, &get_entry/1)) + + feed = if actor.avatar_url, do: Feed.icon(feed, actor.avatar_url), else: feed + + feed = + if actor.banner_url, + do: Feed.logo(feed, actor.banner_url), + else: feed + + feed + |> Feed.build() + |> Atomex.generate_document() + end + + # Create an entry for the Atom feed + @spec get_entry(Event.t()) :: any() + defp get_entry(%Event{} = event) do + with {:ok, html, []} <- Earmark.as_html(event.description) do + entry = + Entry.new(event.url, event.inserted_at, event.title) + |> Entry.link(event.url, rel: "alternate", type: "text/html") + |> Entry.content({:cdata, html}, type: "html") + + entry = if event.publish_at, do: Entry.published(entry, event.publish_at), else: entry + + # Add tags + entry = + event.tags + |> Enum.uniq() + |> Enum.reduce(entry, fn tag, acc -> Entry.category(acc, tag.slug, label: tag.title) end) + + Entry.build(entry) + else + {:error, _html, error_messages} -> + require Logger + Logger.error("Unable to produce HTML for Markdown", details: inspect(error_messages)) + end + end +end diff --git a/mix.exs b/mix.exs index 747258f5c..1a4c3e92c 100644 --- a/mix.exs +++ b/mix.exs @@ -87,7 +87,6 @@ defmodule Mobilizon.Mixfile do {:plug_cowboy, "~> 2.0"}, {:atomex, "0.3.0"}, {:cachex, "~> 3.1"}, - {:trailing_format_plug, "~> 0.0.5"}, {:earmark, "~> 1.3.1"}, # Dev and test dependencies {:phoenix_live_reload, "~> 1.2", only: :dev}, @@ -100,7 +99,7 @@ defmodule Mobilizon.Mixfile do {:exvcr, "~> 0.10", only: :test}, {:credo, "~> 1.0.0", only: [:dev, :test], runtime: false}, {:mock, "~> 0.3.0", only: :test}, - {:feeder_ex, "~> 1.1", only: :test} + {:elixir_feed_parser, "~> 2.1.0", only: :test} ] end diff --git a/mix.lock b/mix.lock index 7f87b78f2..329557d03 100644 --- a/mix.lock +++ b/mix.lock @@ -29,13 +29,14 @@ "ecto_autoslug_field": {:hex, :ecto_autoslug_field, "1.0.0", "577eed25e6d045b8d783f82c9872f97c3a84017a4feae50eaf3cf4e1334a19e2", [:mix], [{:ecto, ">= 2.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:slugger, ">= 0.2.0", [hex: :slugger, repo: "hexpm", optional: false]}], "hexpm"}, "ecto_enum": {:hex, :ecto_enum, "1.2.0", "9ead3ee04efc4cb68a50560a9d9ebb665dd697f957f1c3df8e81bf863cf7a4e9", [:mix], [{:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm"}, "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "elixir_feed_parser": {:hex, :elixir_feed_parser, "2.1.0", "bb96fb6422158dc7ad59de62ef211cc69d264acbbe63941a64a5dce97bbbc2e6", [:mix], [{:timex, "~> 3.4", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"}, "elixir_make": {:hex, :elixir_make, "0.5.2", "96a28c79f5b8d34879cd95ebc04d2a0d678cfbbd3e74c43cb63a76adf0ee8054", [:mix], [], "hexpm"}, "erlex": {:hex, :erlex, "0.2.1", "cee02918660807cbba9a7229cae9b42d1c6143b768c781fa6cee1eaf03ad860b", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"}, "ex_crypto": {:hex, :ex_crypto, "0.10.0", "af600a89b784b36613a989da6e998c1b200ff1214c3cfbaf8deca4aa2f0a1739", [:mix], [{:poison, ">= 2.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_ical": {:hex, :ex_ical, "0.2.0", "4b928b554614704016cc0c9ee226eb854da9327a1cc460457621ceacb1ac29a6", [:mix], [{:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"}, - "ex_machina": {:hex, :ex_machina, "2.2.2", "d84217a6fb7840ff771d2561b8aa6d74a0d8968e4b10ecc0d7e9890dc8fb1c6a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, + "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_unit_notifier": {:hex, :ex_unit_notifier, "0.1.4", "36a2dcab829f506e01bf17816590680dd1474407926d43e64c1263e627c364b8", [:mix], [], "hexpm"}, "exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], [], "hexpm"}, "excoveralls": {:hex, :excoveralls, "0.10.5", "7c912c4ec0715a6013647d835c87cde8154855b9b84e256bc7a63858d5f284e3", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, @@ -43,7 +44,6 @@ "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"}, "exvcr": {:hex, :exvcr, "0.10.3", "1ae3b97560430acfa88ebc737c85b2b7a9dbacd8a2b26789a19718b51ae3522c", [:mix], [{:exactor, "~> 2.2", [hex: :exactor, repo: "hexpm", optional: false]}, {:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.1", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "feeder": {:hex, :feeder, "2.2.4", "56ec535cf2f79719bc53b5c2abe5f6cf481fc01e5ae6229ab7cc829644f039ec", [:make], [], "hexpm"}, - "feeder_ex": {:hex, :feeder_ex, "1.1.0", "0be3732255cdb45dec949e0ede6852b5261c9ff173360e8274a6ac65183b2b55", [:mix], [{:feeder, "~> 2.2", [hex: :feeder, repo: "hexpm", optional: false]}], "hexpm"}, "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.12.0", "97d44903f5ca18ca85cb39aee7d9c77e98d79804bbdef56078adcf905cb2ef00", [:rebar3], [], "hexpm"}, "geo": {:hex, :geo, "3.1.0", "727e005262430d037e870ff364e65d80ca5ca21d5ac8eddd57a1ada72c3f83b0", [:mix], [], "hexpm"}, @@ -92,7 +92,6 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"}, "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, - "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.19", "7962a3997bf06303b7d1772988ede22260f3dae1bf897408ebdac2b4435f4e6a", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm"}, diff --git a/test/mobilizon_web/controllers/feed_controller_test.exs b/test/mobilizon_web/controllers/feed_controller_test.exs index 92eca1de3..c1ef12ca9 100644 --- a/test/mobilizon_web/controllers/feed_controller_test.exs +++ b/test/mobilizon_web/controllers/feed_controller_test.exs @@ -1,37 +1,64 @@ defmodule MobilizonWeb.FeedControllerTest do use MobilizonWeb.ConnCase import Mobilizon.Factory + alias MobilizonWeb.Router.Helpers, as: Routes + alias MobilizonWeb.Endpoint describe "/@:preferred_username.atom" do test "it returns an RSS representation of the actor's public events", %{conn: conn} do actor = insert(:actor) - event1 = insert(:event, organizer_actor: actor) - event2 = insert(:event, organizer_actor: actor) + tag1 = insert(:tag, title: "RSS", slug: "rss") + tag2 = insert(:tag, title: "ATOM", slug: "atom") + event1 = insert(:event, organizer_actor: actor, tags: [tag1]) + event2 = insert(:event, organizer_actor: actor, tags: [tag1, tag2]) conn = conn - |> put_req_header("accept", "application/atom+xml") - |> get("/@#{actor.preferred_username}.atom") + |> get( + Routes.feed_url(Endpoint, :actor, actor.preferred_username, "atom") + |> URI.decode() + ) assert response(conn, 200) =~ "" assert response_content_type(conn, :xml) =~ "charset=utf-8" - {:ok, feed, _} = FeederEx.parse(conn.resp_body) + {:ok, feed} = ElixirFeedParser.parse(conn.resp_body) assert feed.title == actor.preferred_username <> "'s public events feed" - Enum.each(feed.entries, fn entry -> + [entry1, entry2] = entries = feed.entries + + Enum.each(entries, fn entry -> assert entry.title in [event1.title, event2.title] end) + + assert entry1.categories == [tag2.slug, tag1.slug] + assert entry2.categories == [tag1.slug] + end + + test "it returns an RSS representation of the actor's public events with the proper accept header", + %{conn: conn} do + actor = insert(:actor) + + conn = + conn + |> put_req_header("accept", "application/atom+xml") + |> get( + Routes.feed_url(Endpoint, :actor, actor.preferred_username, "atom") + |> URI.decode() + ) + + assert response(conn, 200) =~ "" + assert response_content_type(conn, :xml) =~ "charset=utf-8" end test "it doesn't return anything for an not existing actor", %{conn: conn} do conn = conn |> put_req_header("accept", "application/atom+xml") - |> get("/@notexistent.atom") + |> get("/@notexistent/feed/atom") - assert response(conn, 404) == "Not found" + assert response(conn, 404) end end end From b3a3001e90f37048a6e528241d5962b7c53240d9 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Fri, 1 Mar 2019 12:57:39 +0100 Subject: [PATCH 2/7] Fix render/4 -> render/3 calls Signed-off-by: Thomas Citharel --- lib/mobilizon_web/controllers/fallback_controller.ex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/mobilizon_web/controllers/fallback_controller.ex b/lib/mobilizon_web/controllers/fallback_controller.ex index c36a23fc9..6817b9cb2 100644 --- a/lib/mobilizon_web/controllers/fallback_controller.ex +++ b/lib/mobilizon_web/controllers/fallback_controller.ex @@ -9,18 +9,21 @@ defmodule MobilizonWeb.FallbackController do def call(conn, {:error, %Ecto.Changeset{} = changeset}) do conn |> put_status(:unprocessable_entity) - |> render(MobilizonWeb.ChangesetView, "error.json", changeset: changeset) + |> put_view(MobilizonWeb.ChangesetView) + |> render("error.json", changeset: changeset) end def call(conn, {:error, nil}) do conn |> put_status(:unprocessable_entity) - |> render(MobilizonWeb.ErrorView, "invalid_request.json") + |> put_view(MobilizonWeb.ErrorView) + |> render("invalid_request.json") end def call(conn, {:error, :not_found}) do conn |> put_status(:not_found) - |> render(MobilizonWeb.ErrorView, :"404") + |> put_view(MobilizonWeb.ErrorView) + |> render(:"404") end end From a3ffc08e577a438a8f83ade8c98b87c25f301f06 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Fri, 1 Mar 2019 18:30:46 +0100 Subject: [PATCH 3/7] Add Activity Pub endpoints cache Signed-off-by: Thomas Citharel --- lib/mobilizon/actors/actor.ex | 4 + lib/mobilizon/application.ex | 11 ++- .../controllers/activity_pub_controller.ex | 97 +++++++++++++------ .../views/activity_pub/object_view.ex | 16 ++- lib/service/activity_pub/utils.ex | 14 ++- lib/service/feed.ex | 5 +- 6 files changed, 104 insertions(+), 43 deletions(-) diff --git a/lib/mobilizon/actors/actor.ex b/lib/mobilizon/actors/actor.ex index 54844afff..5ce7c0db1 100644 --- a/lib/mobilizon/actors/actor.ex +++ b/lib/mobilizon/actors/actor.ex @@ -377,4 +377,8 @@ defmodule Mobilizon.Actors.Actor do name -> name end end + + def clear_cache(%Actor{preferred_username: preferred_username, domain: nil}) do + Cachex.del(:activity_pub, "actor_" <> preferred_username) + end end diff --git a/lib/mobilizon/application.ex b/lib/mobilizon/application.ex index bfe420ce1..6a8c2252c 100644 --- a/lib/mobilizon/application.ex +++ b/lib/mobilizon/application.ex @@ -37,12 +37,17 @@ defmodule Mobilizon.Application do worker( Cachex, [ - :json, + :activity_pub, [ - limit: 2500 + limit: 2500, + expiration: + expiration( + default: :timer.minutes(3), + interval: :timer.seconds(15) + ) ] ], - id: :cache_actor + id: :cache_activity_pub ), worker(Guardian.DB.Token.SweeperServer, []), worker(Mobilizon.Service.Federator, []) diff --git a/lib/mobilizon_web/controllers/activity_pub_controller.ex b/lib/mobilizon_web/controllers/activity_pub_controller.ex index c7c6a9636..76670b69b 100644 --- a/lib/mobilizon_web/controllers/activity_pub_controller.ex +++ b/lib/mobilizon_web/controllers/activity_pub_controller.ex @@ -21,53 +21,96 @@ defmodule MobilizonWeb.ActivityPubController do "application/activity+json, application/ld+json" ] + @doc """ + Show an Actor's ActivityPub representation + """ + @spec actor(Plug.Conn.t(), map()) :: Plug.Conn.t() def actor(conn, %{"name" => name}) do - with %Actor{} = actor <- Actors.get_local_actor_by_name(name) do - if conn |> get_req_header("accept") |> is_ap_header() do - conn |> render_ap_actor(actor) - else - conn - |> put_resp_content_type("text/html") - |> send_file(200, "priv/static/index.html") - end + if conn |> get_req_header("accept") |> is_ap_header() do + render_cached_actor(conn, name) else - nil -> {:error, :not_found} + conn + |> put_resp_content_type("text/html") + |> send_file(200, "priv/static/index.html") end end + @spec render_cached_actor(Plug.Conn.t(), String.t()) :: Plug.Conn.t() + defp render_cached_actor(conn, name) do + case Cachex.fetch(:activity_pub, "actor_" <> name, &get_local_actor_by_name/1) do + {status, %Actor{} = actor} when status in [:ok, :commit] -> + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(ActorView.render("actor.json", %{actor: actor})) + + {:ignore, _} -> + {:error, :not_found} + end + end + + defp get_local_actor_by_name("actor_" <> name) do + case Actors.get_local_actor_by_name(name) do + nil -> {:ignore, nil} + %Actor{} = actor -> {:commit, actor} + end + end + + # Test if the request has an AP header defp is_ap_header(ap_headers) do length(@activity_pub_headers -- ap_headers) < 2 end - defp render_ap_actor(conn, %Actor{} = actor) do - conn - |> put_resp_header("content-type", "application/activity+json") - |> json(ActorView.render("actor.json", %{actor: actor})) - end - + @doc """ + Renders an Event ActivityPub's representation + """ + @spec event(Plug.Conn.t(), map()) :: Plug.Conn.t() def event(conn, %{"uuid" => uuid}) do - with %Event{} = event <- Events.get_event_full_by_uuid(uuid), - true <- event.visibility in [:public, :unlisted] do - conn - |> put_resp_header("content-type", "application/activity+json") - |> json(ObjectView.render("event.json", %{event: event |> Utils.make_event_data()})) - else - _ -> + case Cachex.fetch(:activity_pub, "event_" <> uuid, &get_event_full_by_uuid/1) do + {status, %Event{} = event} when status in [:ok, :commit] -> + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(ObjectView.render("event.json", %{event: event |> Utils.make_event_data()})) + + {:ignore, _} -> {:error, :not_found} end end + defp get_event_full_by_uuid("event_" <> uuid) do + with %Event{} = event <- Events.get_event_full_by_uuid(uuid), + true <- event.visibility in [:public, :unlisted] do + {:commit, event} + else + _ -> {:ignore, nil} + end + end + + @doc """ + Renders a Comment ActivityPub's representation + """ + @spec comment(Plug.Conn.t(), map()) :: Plug.Conn.t() def comment(conn, %{"uuid" => uuid}) do + case Cachex.fetch(:activity_pub, "comment_" <> uuid, &get_comment_full_by_uuid/1) do + {status, %Comment{} = comment} when status in [:ok, :commit] -> + conn + |> put_resp_header("content-type", "application/activity+json") + |> json( + ObjectView.render("comment.json", %{comment: comment |> Utils.make_comment_data()}) + ) + + {:ignore, _} -> + {:error, :not_found} + end + end + + defp get_comment_full_by_uuid("comment_" <> uuid) do with %Comment{} = comment <- Events.get_comment_full_from_uuid(uuid) do # Comments are always public for now # TODO : Make comments maybe restricted # true <- comment.public do - conn - |> put_resp_header("content-type", "application/activity+json") - |> json(ObjectView.render("comment.json", %{comment: comment |> Utils.make_comment_data()})) + {:commit, comment} else - _ -> - {:error, :not_found} + _ -> {:ignore, nil} end end diff --git a/lib/mobilizon_web/views/activity_pub/object_view.ex b/lib/mobilizon_web/views/activity_pub/object_view.ex index e8dcfa12a..a0372e96b 100644 --- a/lib/mobilizon_web/views/activity_pub/object_view.ex +++ b/lib/mobilizon_web/views/activity_pub/object_view.ex @@ -3,16 +3,22 @@ defmodule MobilizonWeb.ActivityPub.ObjectView do alias Mobilizon.Service.ActivityPub.Utils def render("event.json", %{event: event}) do + {:ok, html, []} = Earmark.as_html(event["summary"]) + event = %{ "type" => "Event", - "actor" => event["actor"], + "attributedTo" => event["actor"], "id" => event["id"], "name" => event["title"], "category" => event["category"], - "content" => event["summary"], - "mediaType" => "text/html" - # "published" => Timex.format!(event.inserted_at, "{ISO:Extended}"), - # "updated" => Timex.format!(event.updated_at, "{ISO:Extended}") + "content" => html, + "source" => %{ + "content" => event["summary"], + "mediaType" => "text/markdown" + }, + "mediaType" => "text/html", + "published" => event["publish_at"], + "updated" => event["updated_at"] } Map.merge(event, Utils.make_json_ld_header()) diff --git a/lib/service/activity_pub/utils.ex b/lib/service/activity_pub/utils.ex index d00a49a93..35cb8e8e4 100644 --- a/lib/service/activity_pub/utils.ex +++ b/lib/service/activity_pub/utils.ex @@ -283,16 +283,20 @@ defmodule Mobilizon.Service.ActivityPub.Utils do @spec make_event_data(Event.t(), list(String.t())) :: map() def make_event_data( - %Event{title: title, organizer_actor: actor, uuid: uuid}, + %Event{} = event, to \\ ["https://www.w3.org/ns/activitystreams#Public"] ) do %{ "type" => "Event", "to" => to, - "title" => title, - "actor" => actor.url, - "uuid" => uuid, - "id" => "#{MobilizonWeb.Endpoint.url()}/events/#{uuid}" + "title" => event.title, + "actor" => event.organizer_actor.url, + "uuid" => event.uuid, + "category" => event.category, + "summary" => event.description, + "publish_at" => (event.publish_at || event.inserted_at) |> DateTime.to_iso8601(), + "updated_at" => event.updated_at |> DateTime.to_iso8601(), + "id" => "#{MobilizonWeb.Endpoint.url()}/events/#{event.uuid}" } end diff --git a/lib/service/feed.ex b/lib/service/feed.ex index d63611154..7cfb4720e 100644 --- a/lib/service/feed.ex +++ b/lib/service/feed.ex @@ -72,11 +72,10 @@ defmodule Mobilizon.Service.Feed do defp get_entry(%Event{} = event) do with {:ok, html, []} <- Earmark.as_html(event.description) do entry = - Entry.new(event.url, event.inserted_at, event.title) + Entry.new(event.url, event.publish_at || event.inserted_at, event.title) |> Entry.link(event.url, rel: "alternate", type: "text/html") |> Entry.content({:cdata, html}, type: "html") - - entry = if event.publish_at, do: Entry.published(entry, event.publish_at), else: entry + |> Entry.published(event.publish_at || event.inserted_at) # Add tags entry = From 634157eb4b132f47339dd4c4316c934e3b182a30 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Sun, 3 Mar 2019 15:52:00 +0100 Subject: [PATCH 4/7] Bump deps Signed-off-by: Thomas Citharel --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 329557d03..aa038e615 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,5 @@ %{ - "absinthe": {:hex, :absinthe, "1.4.15", "d9655f38e8cb937acfd0fd6c2ff0e837d3666947f8a01988f8fabe0373c06f06", [:mix], [{:dataloader, "~> 1.0.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, + "absinthe": {:hex, :absinthe, "1.4.16", "0933e4d9f12652b12115d5709c0293a1bf78a22578032e9ad0dad4efee6b9eb1", [:mix], [{:dataloader, "~> 1.0.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "absinthe_ecto": {:hex, :absinthe_ecto, "0.1.3", "420b68129e79fe4571a4838904ba03e282330d335da47729ad52ffd7b8c5fcb1", [:mix], [{:absinthe, "~> 1.3.0 or ~> 1.4.0", [hex: :absinthe, repo: "hexpm", optional: false]}, {:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm"}, "absinthe_phoenix": {:hex, :absinthe_phoenix, "1.4.3", "cea34e7ebbc9a252038c1f1164878ee86bcb108905fe462be77efacda15c1e70", [:mix], [{:absinthe, "~> 1.4.0", [hex: :absinthe, repo: "hexpm", optional: false]}, {:absinthe_plug, "~> 1.4.0", [hex: :absinthe_plug, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.2", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.10.5 or ~> 2.11", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:poison, "~> 2.0 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "absinthe_plug": {:hex, :absinthe_plug, "1.4.6", "ac5d2d3d02acf52fda0f151b294017ab06e2ed1c6c15334e06aac82c94e36e08", [:mix], [{:absinthe, "~> 1.4.11", [hex: :absinthe, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.2 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, @@ -39,7 +39,7 @@ "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_unit_notifier": {:hex, :ex_unit_notifier, "0.1.4", "36a2dcab829f506e01bf17816590680dd1474407926d43e64c1263e627c364b8", [:mix], [], "hexpm"}, "exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], [], "hexpm"}, - "excoveralls": {:hex, :excoveralls, "0.10.5", "7c912c4ec0715a6013647d835c87cde8154855b9b84e256bc7a63858d5f284e3", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "excoveralls": {:hex, :excoveralls, "0.10.6", "e2b9718c9d8e3ef90bc22278c3f76c850a9f9116faf4ebe9678063310742edc2", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "exgravatar": {:hex, :exgravatar, "2.0.1", "66d595c7d63dd6bbac442c5542a724375ae29144059c6fe093e61553850aace4", [:mix], [], "hexpm"}, "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"}, "exvcr": {:hex, :exvcr, "0.10.3", "1ae3b97560430acfa88ebc737c85b2b7a9dbacd8a2b26789a19718b51ae3522c", [:mix], [{:exactor, "~> 2.2", [hex: :exactor, repo: "hexpm", optional: false]}, {:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.1", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, From 66e67aa816bbe2d1638dfdd3d7e01876fdd3063a Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Mon, 4 Mar 2019 17:20:18 +0100 Subject: [PATCH 5/7] Redirect properly to correct endpoint depending on content-type Signed-off-by: Thomas Citharel --- js/public/index.html | 1 + js/vue.config.js | 15 ++++ lib/mobilizon/actors/actor.ex | 6 ++ lib/mobilizon/actors/actors.ex | 14 +++ lib/mobilizon/events/events.ex | 20 +++++ .../controllers/activity_pub_controller.ex | 88 +++++-------------- .../controllers/page_controller.ex | 72 +++++++++++++++ lib/mobilizon_web/router.ex | 14 ++- lib/service/metadata.ex | 6 ++ lib/service/metadata/actor.ex | 25 ++++++ lib/service/metadata/comment.ex | 22 +++++ lib/service/metadata/event.ex | 24 +++++ 12 files changed, 237 insertions(+), 70 deletions(-) create mode 100644 lib/service/metadata.ex create mode 100644 lib/service/metadata/actor.ex create mode 100644 lib/service/metadata/comment.ex create mode 100644 lib/service/metadata/event.ex diff --git a/js/public/index.html b/js/public/index.html index 86e3871f2..346854e2f 100644 --- a/js/public/index.html +++ b/js/public/index.html @@ -8,6 +8,7 @@ mobilizon + diff --git a/js/vue.config.js b/js/vue.config.js index b76081558..b3363d711 100644 --- a/js/vue.config.js +++ b/js/vue.config.js @@ -19,4 +19,19 @@ module.exports = { ], }, }, + chainWebpack: config => { + config + .plugin('html') + .tap(args => { + args[0].minify = { + collapseWhitespace: true, + removeComments: false, + removeRedundantAttributes: true, + removeScriptTypeAttributes: true, + removeStyleLinkTypeAttributes: true, + useShortDoctype: true + }; + return args + }); + } }; diff --git a/lib/mobilizon/actors/actor.ex b/lib/mobilizon/actors/actor.ex index 5ce7c0db1..9cae6b29e 100644 --- a/lib/mobilizon/actors/actor.ex +++ b/lib/mobilizon/actors/actor.ex @@ -378,6 +378,12 @@ defmodule Mobilizon.Actors.Actor do end end + def display_name_and_username(%Actor{name: nil} = actor), do: actor_acct_from_actor(actor) + def display_name_and_username(%Actor{name: ""} = actor), do: actor_acct_from_actor(actor) + + def display_name_and_username(%Actor{name: name} = actor), + do: name <> " (" <> actor_acct_from_actor(actor) <> ")" + def clear_cache(%Actor{preferred_username: preferred_username, domain: nil}) do Cachex.del(:activity_pub, "actor_" <> preferred_username) end diff --git a/lib/mobilizon/actors/actors.ex b/lib/mobilizon/actors/actors.ex index 92a4e40af..181e61137 100644 --- a/lib/mobilizon/actors/actors.ex +++ b/lib/mobilizon/actors/actors.ex @@ -467,6 +467,20 @@ defmodule Mobilizon.Actors do |> Repo.preload(:organized_events) end + @doc """ + Returns a cached local actor by username + """ + @spec get_cached_local_actor_by_name(String.t()) :: + {:ok, Actor.t()} | {:commit, Actor.t()} | {:ignore, any()} + def get_cached_local_actor_by_name(name) do + Cachex.fetch(:activity_pub, "actor_" <> name, fn "actor_" <> name -> + case get_local_actor_by_name(name) do + nil -> {:ignore, nil} + %Actor{} = actor -> {:commit, actor} + end + end) + end + @doc """ Getting an actor from url, eventually creating it """ diff --git a/lib/mobilizon/events/events.ex b/lib/mobilizon/events/events.ex index 6bf3ea1ec..f9fd63770 100644 --- a/lib/mobilizon/events/events.ex +++ b/lib/mobilizon/events/events.ex @@ -154,6 +154,16 @@ defmodule Mobilizon.Events do |> Repo.one() end + def get_cached_event_full_by_uuid(uuid) do + Cachex.fetch(:activity_pub, "event_" <> uuid, fn "event_" <> uuid -> + with %Event{} = event <- get_event_full_by_uuid(uuid) do + {:commit, event} + else + _ -> {:ignore, nil} + end + end) + end + @doc """ Gets a single event, with all associations loaded. """ @@ -1018,6 +1028,16 @@ defmodule Mobilizon.Events do end end + def get_cached_comment_full_by_uuid("comment_" <> uuid) do + Cachex.fetch(:activity_pub, "comment_" <> uuid, fn "comment_" <> uuid -> + with %Comment{} = comment <- Events.get_comment_full_from_uuid(uuid) do + {:commit, comment} + else + _ -> {:ignore, nil} + end + end) + end + def get_comment_from_url(url), do: Repo.get_by(Comment, url: url) def get_comment_from_url!(url), do: Repo.get_by!(Comment, url: url) diff --git a/lib/mobilizon_web/controllers/activity_pub_controller.ex b/lib/mobilizon_web/controllers/activity_pub_controller.ex index 76670b69b..870554108 100644 --- a/lib/mobilizon_web/controllers/activity_pub_controller.ex +++ b/lib/mobilizon_web/controllers/activity_pub_controller.ex @@ -16,101 +16,55 @@ defmodule MobilizonWeb.ActivityPubController do action_fallback(:errors) - @activity_pub_headers [ - "application/activity+json", - "application/activity+json, application/ld+json" - ] - @doc """ - Show an Actor's ActivityPub representation + Renders an Actor ActivityPub's representation """ - @spec actor(Plug.Conn.t(), map()) :: Plug.Conn.t() + @spec actor(Plug.Conn.t(), String.t()) :: Plug.Conn.t() def actor(conn, %{"name" => name}) do - if conn |> get_req_header("accept") |> is_ap_header() do - render_cached_actor(conn, name) - else + with {status, %Actor{} = actor} when status in [:ok, :commit] <- + Actors.get_cached_local_actor_by_name(name) do conn - |> put_resp_content_type("text/html") - |> send_file(200, "priv/static/index.html") - end - end - - @spec render_cached_actor(Plug.Conn.t(), String.t()) :: Plug.Conn.t() - defp render_cached_actor(conn, name) do - case Cachex.fetch(:activity_pub, "actor_" <> name, &get_local_actor_by_name/1) do - {status, %Actor{} = actor} when status in [:ok, :commit] -> - conn - |> put_resp_header("content-type", "application/activity+json") - |> json(ActorView.render("actor.json", %{actor: actor})) - + |> put_resp_header("content-type", "application/activity+json") + |> json(ActorView.render("actor.json", %{actor: actor})) + else {:ignore, _} -> {:error, :not_found} end end - defp get_local_actor_by_name("actor_" <> name) do - case Actors.get_local_actor_by_name(name) do - nil -> {:ignore, nil} - %Actor{} = actor -> {:commit, actor} - end - end - - # Test if the request has an AP header - defp is_ap_header(ap_headers) do - length(@activity_pub_headers -- ap_headers) < 2 - end - @doc """ Renders an Event ActivityPub's representation """ @spec event(Plug.Conn.t(), map()) :: Plug.Conn.t() def event(conn, %{"uuid" => uuid}) do - case Cachex.fetch(:activity_pub, "event_" <> uuid, &get_event_full_by_uuid/1) do - {status, %Event{} = event} when status in [:ok, :commit] -> - conn - |> put_resp_header("content-type", "application/activity+json") - |> json(ObjectView.render("event.json", %{event: event |> Utils.make_event_data()})) - + with {status, %Event{}} = event when status in [:ok, :commit] <- + Events.get_cached_event_full_by_uuid(uuid), + true <- event.visibility in [:public, :unlisted] do + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(ObjectView.render("event.json", %{event: event |> Utils.make_event_data()})) + else {:ignore, _} -> {:error, :not_found} end end - defp get_event_full_by_uuid("event_" <> uuid) do - with %Event{} = event <- Events.get_event_full_by_uuid(uuid), - true <- event.visibility in [:public, :unlisted] do - {:commit, event} - else - _ -> {:ignore, nil} - end - end - @doc """ Renders a Comment ActivityPub's representation """ @spec comment(Plug.Conn.t(), map()) :: Plug.Conn.t() def comment(conn, %{"uuid" => uuid}) do - case Cachex.fetch(:activity_pub, "comment_" <> uuid, &get_comment_full_by_uuid/1) do - {status, %Comment{} = comment} when status in [:ok, :commit] -> - conn - |> put_resp_header("content-type", "application/activity+json") - |> json( - ObjectView.render("comment.json", %{comment: comment |> Utils.make_comment_data()}) - ) - - {:ignore, _} -> - {:error, :not_found} - end - end - - defp get_comment_full_by_uuid("comment_" <> uuid) do - with %Comment{} = comment <- Events.get_comment_full_from_uuid(uuid) do + with {status, %Comment{} = comment} when status in [:ok, :commit] <- + Events.get_cached_comment_full_by_uuid(uuid) do # Comments are always public for now # TODO : Make comments maybe restricted # true <- comment.public do - {:commit, comment} + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(ObjectView.render("comment.json", %{comment: comment |> Utils.make_comment_data()})) else - _ -> {:ignore, nil} + {:ignore, _} -> + {:error, :not_found} end end diff --git a/lib/mobilizon_web/controllers/page_controller.ex b/lib/mobilizon_web/controllers/page_controller.ex index c68f79bf6..a3e755a2e 100644 --- a/lib/mobilizon_web/controllers/page_controller.ex +++ b/lib/mobilizon_web/controllers/page_controller.ex @@ -3,6 +3,11 @@ defmodule MobilizonWeb.PageController do Controller to load our webapp """ use MobilizonWeb, :controller + alias Mobilizon.Service.Metadata + alias Mobilizon.Actors + alias Mobilizon.Actors.Actor + alias Mobilizon.Events + alias Mobilizon.Events.{Event, Comment} plug(:put_layout, false) @@ -11,4 +16,71 @@ defmodule MobilizonWeb.PageController do |> put_resp_content_type("text/html") |> send_file(200, "priv/static/index.html") end + + def actor(conn, %{"name" => name}) do + case get_format(conn) do + "html" -> + with {status, %Actor{} = actor} when status in [:ok, :commit] <- + Actors.get_cached_local_actor_by_name(name) do + render_with_meta(conn, actor) + end + + "activity+json" -> + MobilizonWeb.ActivityPubController.call(conn, :actor) + + _ -> + {:error, :not_found} + end + end + + def event(conn, %{"uuid" => uuid}) do + case get_format(conn) do + "html" -> + with {status, %Event{} = event} when status in [:ok, :commit] <- + Events.get_cached_event_full_by_uuid(uuid), + true <- event.visibility in [:public, :unlisted] do + render_with_meta(conn, event) + end + + "activity+json" -> + MobilizonWeb.ActivityPubController.call(conn, :event) + + _ -> + {:error, :not_found} + end + end + + def comment(conn, %{"uuid" => uuid}) do + case get_format(conn) do + "html" -> + with {status, %Comment{} = comment} when status in [:ok, :commit] <- + Events.get_cached_comment_full_by_uuid(uuid) do + # Comments are always public for now + # TODO : Make comments maybe restricted + # true <- comment.public do + render_with_meta(conn, comment) + end + + "activity+json" -> + MobilizonWeb.ActivityPubController.call(conn, :comment) + + _ -> + {:error, :not_found} + end + end + + # Inject OpenGraph information + defp render_with_meta(conn, object) do + {:ok, index_content} = File.read(index_file_path()) + tags = Metadata.build_tags(object) + response = String.replace(index_content, "", tags) + + conn + |> put_resp_content_type("text/html") + |> send_resp(200, response) + end + + defp index_file_path() do + Path.join(Application.app_dir(:mobilizon, "priv/static/"), "index.html") + end end diff --git a/lib/mobilizon_web/router.ex b/lib/mobilizon_web/router.ex index e41a0cdb0..73d996004 100644 --- a/lib/mobilizon_web/router.ex +++ b/lib/mobilizon_web/router.ex @@ -19,6 +19,10 @@ defmodule MobilizonWeb.Router do end pipeline :activity_pub do + plug(:accepts, ["activity-json"]) + end + + pipeline :activity_pub_and_html do plug(:accepts, ["activity-json", "html"]) end @@ -61,15 +65,19 @@ defmodule MobilizonWeb.Router do get("/@:name/feed/:format", FeedController, :actor) end + scope "/", MobilizonWeb do + pipe_through(:activity_pub_and_html) + get("/@:name", PageController, :actor) + get("/events/:uuid", PageController, :event) + get("/comments/:uuid", PageController, :comment) + end + scope "/", MobilizonWeb do pipe_through(:activity_pub) - get("/@:name", ActivityPubController, :actor) get("/@:name/outbox", ActivityPubController, :outbox) get("/@:name/following", ActivityPubController, :following) get("/@:name/followers", ActivityPubController, :followers) - get("/events/:uuid", ActivityPubController, :event) - get("/comments/:uuid", ActivityPubController, :comment) end scope "/", MobilizonWeb do diff --git a/lib/service/metadata.ex b/lib/service/metadata.ex new file mode 100644 index 000000000..bd3ba99b1 --- /dev/null +++ b/lib/service/metadata.ex @@ -0,0 +1,6 @@ +defprotocol Mobilizon.Service.Metadata do + @doc """ + Build tags + """ + def build_tags(entity) +end diff --git a/lib/service/metadata/actor.ex b/lib/service/metadata/actor.ex new file mode 100644 index 000000000..4a51e2f96 --- /dev/null +++ b/lib/service/metadata/actor.ex @@ -0,0 +1,25 @@ +defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do + alias Phoenix.HTML + alias Phoenix.HTML.Tag + alias Mobilizon.Actors.Actor + require Logger + + def build_tags(%Actor{} = actor) do + actor + |> do_build_tags() + |> Enum.map(&HTML.safe_to_string/1) + |> Enum.reduce("", fn tag, acc -> acc <> tag end) + end + + defp do_build_tags(%Actor{} = actor) do + [ + Tag.tag(:meta, property: "og:title", content: Actor.display_name_and_username(actor)), + Tag.tag(:meta, property: "og:url", content: actor.url), + Tag.tag(:meta, property: "og:description", content: actor.summary), + Tag.tag(:meta, property: "og:type", content: "profile"), + Tag.tag(:meta, property: "profile:username", content: actor.preferred_username), + Tag.tag(:meta, property: "og:image", content: actor.avatar_url), + Tag.tag(:meta, property: "twitter:card", content: "summary") + ] + end +end diff --git a/lib/service/metadata/comment.ex b/lib/service/metadata/comment.ex new file mode 100644 index 000000000..dfa504579 --- /dev/null +++ b/lib/service/metadata/comment.ex @@ -0,0 +1,22 @@ +defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Comment do + alias Phoenix.HTML + alias Phoenix.HTML.Tag + alias Mobilizon.Events.Comment + + def build_tags(%Comment{} = comment) do + comment + |> do_build_tags() + |> Enum.map(&HTML.safe_to_string/1) + |> Enum.reduce("", fn tag, acc -> acc <> tag end) + end + + defp do_build_tags(%Comment{} = comment) do + [ + Tag.tag(:meta, property: "og:title", content: comment.actor.preferred_username), + Tag.tag(:meta, property: "og:url", content: comment.url), + Tag.tag(:meta, property: "og:description", content: comment.text), + Tag.tag(:meta, property: "og:type", content: "website"), + Tag.tag(:meta, property: "twitter:card", content: "summary") + ] + end +end diff --git a/lib/service/metadata/event.ex b/lib/service/metadata/event.ex new file mode 100644 index 000000000..8a63d7b84 --- /dev/null +++ b/lib/service/metadata/event.ex @@ -0,0 +1,24 @@ +defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do + alias Phoenix.HTML + alias Phoenix.HTML.Tag + alias Mobilizon.Events.Event + + def build_tags(%Event{} = event) do + event + |> do_build_tags() + |> Enum.map(&HTML.safe_to_string/1) + |> Enum.reduce("", fn tag, acc -> acc <> tag end) + end + + defp do_build_tags(%Event{} = event) do + [ + Tag.tag(:meta, property: "og:title", content: event.title), + Tag.tag(:meta, property: "og:url", content: event.url), + Tag.tag(:meta, property: "og:description", content: event.description), + Tag.tag(:meta, property: "og:type", content: "website"), + Tag.tag(:meta, property: "og:image", content: event.thumbnail), + Tag.tag(:meta, property: "og:image", content: event.large_image), + Tag.tag(:meta, property: "twitter:card", content: "summary_large_image") + ] + end +end From 6de839dec2286f5518c098fc0a90b9b9d37840da Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Mon, 4 Mar 2019 18:38:30 +0100 Subject: [PATCH 6/7] Add JSON-LD schema Signed-off-by: Thomas Citharel --- .../views/json_ld/object_view.ex | 57 +++++++++++++++++++ lib/service/metadata/event.ex | 10 ++++ 2 files changed, 67 insertions(+) create mode 100644 lib/mobilizon_web/views/json_ld/object_view.ex diff --git a/lib/mobilizon_web/views/json_ld/object_view.ex b/lib/mobilizon_web/views/json_ld/object_view.ex new file mode 100644 index 000000000..7d2686056 --- /dev/null +++ b/lib/mobilizon_web/views/json_ld/object_view.ex @@ -0,0 +1,57 @@ +defmodule MobilizonWeb.JsonLD.ObjectView do + use MobilizonWeb, :view + + alias Mobilizon.Events.{Event, Comment} + alias Mobilizon.Actors.Actor + alias Mobilizon.Addresses.Address + alias MobilizonWeb.JsonLD.ObjectView + + def render("event.json", %{event: %Event{} = event}) do + # TODO: event.description is actually markdown! + json_ld = %{ + "@context" => "https://schema.org", + "@type" => "Event", + "name" => event.title, + "description" => event.description, + "image" => [ + event.thumbnail, + event.large_image + ], + "performer" => %{ + "@type" => + if(event.organizer_actor.type == :Group, do: "PerformingGroup", else: "Person"), + "name" => Actor.display_name(event.organizer_actor) + }, + "location" => render_one(event.physical_address, ObjectView, "place.json", as: :address) + } + + json_ld = + if event.begins_on, + do: Map.put(json_ld, "startDate", DateTime.to_iso8601(event.begins_on)), + else: json_ld + + json_ld = + if event.ends_on, + do: Map.put(json_ld, "endDate", DateTime.to_iso8601(event.ends_on)), + else: json_ld + + json_ld + end + + def render("place.json", %{address: %Address{} = address}) do + %{ + "@type" => "Place", + "name" => address.description, + "address" => %{ + "@type" => "PostalAddress", + "streetAddress" => address.streetAddress, + "addressLocality" => address.addressLocality, + "postalCode" => address.postalCode, + "addressRegion" => address.addressRegion, + "addressCountry" => address.addressCountry + } + } + end + + def render("place.json", nil), do: %{} +end diff --git a/lib/service/metadata/event.ex b/lib/service/metadata/event.ex index 8a63d7b84..17fdc56ff 100644 --- a/lib/service/metadata/event.ex +++ b/lib/service/metadata/event.ex @@ -2,14 +2,17 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do alias Phoenix.HTML alias Phoenix.HTML.Tag alias Mobilizon.Events.Event + alias MobilizonWeb.JsonLD.ObjectView def build_tags(%Event{} = event) do event |> do_build_tags() |> Enum.map(&HTML.safe_to_string/1) |> Enum.reduce("", fn tag, acc -> acc <> tag end) + |> Kernel.<>(build_json_ld_schema(event)) end + # Build OpenGraph & Twitter Tags defp do_build_tags(%Event{} = event) do [ Tag.tag(:meta, property: "og:title", content: event.title), @@ -21,4 +24,11 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do Tag.tag(:meta, property: "twitter:card", content: "summary_large_image") ] end + + # Insert JSON-LD schema by hand because Tag.content_tag wants to escape it + defp build_json_ld_schema(%Event{} = event) do + "" + end end From 9a60704ed9d43a5f5475e86fe5568fa5f9f6a7a3 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 5 Mar 2019 10:13:19 +0100 Subject: [PATCH 7/7] Add and fix tests Signed-off-by: Thomas Citharel --- lib/mobilizon/events/events.ex | 4 +-- .../controllers/activity_pub_controller.ex | 2 +- .../controllers/fallback_controller.ex | 2 +- .../controllers/page_controller.ex | 14 +++++++-- lib/mobilizon_web/views/error_view.ex | 5 ++- .../views/json_ld/object_view.ex | 2 +- .../activity_pub_controller_test.exs | 12 +++---- .../controllers/page_controller_test.exs | 31 ++++++++++++++++++- 8 files changed, 55 insertions(+), 17 deletions(-) diff --git a/lib/mobilizon/events/events.ex b/lib/mobilizon/events/events.ex index f9fd63770..1855435f3 100644 --- a/lib/mobilizon/events/events.ex +++ b/lib/mobilizon/events/events.ex @@ -1028,9 +1028,9 @@ defmodule Mobilizon.Events do end end - def get_cached_comment_full_by_uuid("comment_" <> uuid) do + def get_cached_comment_full_by_uuid(uuid) do Cachex.fetch(:activity_pub, "comment_" <> uuid, fn "comment_" <> uuid -> - with %Comment{} = comment <- Events.get_comment_full_from_uuid(uuid) do + with %Comment{} = comment <- get_comment_full_from_uuid(uuid) do {:commit, comment} else _ -> {:ignore, nil} diff --git a/lib/mobilizon_web/controllers/activity_pub_controller.ex b/lib/mobilizon_web/controllers/activity_pub_controller.ex index 870554108..2f49d16ff 100644 --- a/lib/mobilizon_web/controllers/activity_pub_controller.ex +++ b/lib/mobilizon_web/controllers/activity_pub_controller.ex @@ -37,7 +37,7 @@ defmodule MobilizonWeb.ActivityPubController do """ @spec event(Plug.Conn.t(), map()) :: Plug.Conn.t() def event(conn, %{"uuid" => uuid}) do - with {status, %Event{}} = event when status in [:ok, :commit] <- + with {status, %Event{} = event} when status in [:ok, :commit] <- Events.get_cached_event_full_by_uuid(uuid), true <- event.visibility in [:public, :unlisted] do conn diff --git a/lib/mobilizon_web/controllers/fallback_controller.ex b/lib/mobilizon_web/controllers/fallback_controller.ex index 6817b9cb2..cf85e7ed5 100644 --- a/lib/mobilizon_web/controllers/fallback_controller.ex +++ b/lib/mobilizon_web/controllers/fallback_controller.ex @@ -24,6 +24,6 @@ defmodule MobilizonWeb.FallbackController do conn |> put_status(:not_found) |> put_view(MobilizonWeb.ErrorView) - |> render(:"404") + |> render("404.html") end end diff --git a/lib/mobilizon_web/controllers/page_controller.ex b/lib/mobilizon_web/controllers/page_controller.ex index a3e755a2e..7d00df912 100644 --- a/lib/mobilizon_web/controllers/page_controller.ex +++ b/lib/mobilizon_web/controllers/page_controller.ex @@ -10,6 +10,7 @@ defmodule MobilizonWeb.PageController do alias Mobilizon.Events.{Event, Comment} plug(:put_layout, false) + action_fallback(MobilizonWeb.FallbackController) def index(conn, _params) do conn @@ -23,9 +24,12 @@ defmodule MobilizonWeb.PageController do with {status, %Actor{} = actor} when status in [:ok, :commit] <- Actors.get_cached_local_actor_by_name(name) do render_with_meta(conn, actor) + else + _ -> {:error, :not_found} end - "activity+json" -> + # "activity-json" matches "application/activity+json" inside our config + "activity-json" -> MobilizonWeb.ActivityPubController.call(conn, :actor) _ -> @@ -40,9 +44,11 @@ defmodule MobilizonWeb.PageController do Events.get_cached_event_full_by_uuid(uuid), true <- event.visibility in [:public, :unlisted] do render_with_meta(conn, event) + else + _ -> {:error, :not_found} end - "activity+json" -> + "activity-json" -> MobilizonWeb.ActivityPubController.call(conn, :event) _ -> @@ -59,9 +65,11 @@ defmodule MobilizonWeb.PageController do # TODO : Make comments maybe restricted # true <- comment.public do render_with_meta(conn, comment) + else + _ -> {:error, :not_found} end - "activity+json" -> + "activity-json" -> MobilizonWeb.ActivityPubController.call(conn, :comment) _ -> diff --git a/lib/mobilizon_web/views/error_view.ex b/lib/mobilizon_web/views/error_view.ex index 42714317e..c4d6b3ce4 100644 --- a/lib/mobilizon_web/views/error_view.ex +++ b/lib/mobilizon_web/views/error_view.ex @@ -25,7 +25,10 @@ defmodule MobilizonWeb.ErrorView do # In case no render clause matches or no # template is found, let's render it as 500 - def template_not_found(_template, assigns) do + def template_not_found(template, assigns) do + require Logger + Logger.error("Template not found") + Logger.error(inspect(template)) render("500.html", assigns) end end diff --git a/lib/mobilizon_web/views/json_ld/object_view.ex b/lib/mobilizon_web/views/json_ld/object_view.ex index 7d2686056..2c231a13a 100644 --- a/lib/mobilizon_web/views/json_ld/object_view.ex +++ b/lib/mobilizon_web/views/json_ld/object_view.ex @@ -1,7 +1,7 @@ defmodule MobilizonWeb.JsonLD.ObjectView do use MobilizonWeb, :view - alias Mobilizon.Events.{Event, Comment} + alias Mobilizon.Events.Event alias Mobilizon.Actors.Actor alias Mobilizon.Addresses.Address alias MobilizonWeb.JsonLD.ObjectView diff --git a/test/mobilizon_web/controllers/activity_pub_controller_test.exs b/test/mobilizon_web/controllers/activity_pub_controller_test.exs index 0fe234fbc..096cce389 100644 --- a/test/mobilizon_web/controllers/activity_pub_controller_test.exs +++ b/test/mobilizon_web/controllers/activity_pub_controller_test.exs @@ -13,13 +13,17 @@ defmodule MobilizonWeb.ActivityPubControllerTest do alias Mobilizon.Service.ActivityPub.Utils use ExVCR.Mock, adapter: ExVCR.Adapter.Hackney + setup do + conn = build_conn() |> put_req_header("accept", "application/activity+json") + {:ok, conn: conn} + end + describe "/@:preferred_username" do test "it returns a json representation of the actor", %{conn: conn} do actor = insert(:actor) conn = conn - |> put_req_header("accept", "application/activity+json") |> get("/@#{actor.preferred_username}") actor = Actors.get_actor!(actor.id) @@ -34,7 +38,6 @@ defmodule MobilizonWeb.ActivityPubControllerTest do conn = conn - |> put_req_header("accept", "application/activity+json") |> get("/events/#{event.uuid}") assert json_response(conn, 200) == @@ -46,7 +49,6 @@ defmodule MobilizonWeb.ActivityPubControllerTest do conn = conn - |> put_req_header("accept", "application/activity+json") |> get("/events/#{event.uuid}") assert json_response(conn, 404) @@ -59,7 +61,6 @@ defmodule MobilizonWeb.ActivityPubControllerTest do conn = conn - |> put_req_header("accept", "application/activity+json") |> get("/comments/#{comment.uuid}") assert json_response(conn, 200) == @@ -87,7 +88,6 @@ defmodule MobilizonWeb.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) - |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) assert "ok" == json_response(conn, 200) @@ -104,7 +104,6 @@ defmodule MobilizonWeb.ActivityPubControllerTest do conn = conn - |> put_req_header("accept", "application/activity+json") |> get("/@#{actor.preferred_username}/outbox") assert response(conn, 200) =~ comment.text @@ -116,7 +115,6 @@ defmodule MobilizonWeb.ActivityPubControllerTest do conn = conn - |> put_req_header("accept", "application/activity+json") |> get("/@#{actor.preferred_username}/outbox") assert response(conn, 200) =~ event.title diff --git a/test/mobilizon_web/controllers/page_controller_test.exs b/test/mobilizon_web/controllers/page_controller_test.exs index 37516df23..ef2d96d54 100644 --- a/test/mobilizon_web/controllers/page_controller_test.exs +++ b/test/mobilizon_web/controllers/page_controller_test.exs @@ -2,14 +2,43 @@ defmodule MobilizonWeb.PageControllerTest do use MobilizonWeb.ConnCase import Mobilizon.Factory + setup do + conn = build_conn() |> put_req_header("accept", "text/html") + {:ok, conn: conn} + end + test "GET /", %{conn: conn} do conn = get(conn, "/") assert html_response(conn, 200) end - test "GET /@actor", %{conn: conn} do + test "GET /@actor with existing actor", %{conn: conn} do actor = insert(:actor) conn = get(conn, "/@#{actor.preferred_username}") assert html_response(conn, 200) end + + test "GET /@actor with not existing actor", %{conn: conn} do + conn = get(conn, "/@notexisting") + assert html_response(conn, 404) + end + + test "GET /events/:uuid", %{conn: conn} do + event = insert(:event) + conn = get(conn, "/events/#{event.uuid}") + assert html_response(conn, 200) + end + + test "GET /events/:uuid with not existing event", %{conn: conn} do + conn = get(conn, "/events/not_existing_event") + assert html_response(conn, 404) + end + + test "GET /events/:uuid with event not public", %{conn: conn} do + event = insert(:event, visibility: :restricted) + conn = get(conn, "/events/#{event.uuid}") + assert html_response(conn, 404) + end + + # TODO: Comments end