From 15069b6099f9f50cb1dc2699bbbb835803a6e198 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 12 Aug 2020 14:34:19 +0200 Subject: [PATCH 1/3] Upgrade oban to 2.0 Signed-off-by: Thomas Citharel --- config/config.exs | 2 +- config/test.exs | 2 +- lib/service/workers/background.ex | 4 +- lib/service/workers/build_search.ex | 8 +- lib/service/workers/notification.ex | 21 +++-- mix.exs | 2 +- mix.lock | 2 +- .../activity_pub/transmogrifier_test.exs | 2 +- test/graphql/resolvers/person_test.exs | 4 +- test/graphql/resolvers/user_test.exs | 2 +- test/mobilizon/actors/actors_test.exs | 2 +- test/service/workers/notification_test.exs | 86 ++++++++----------- 12 files changed, 67 insertions(+), 70 deletions(-) diff --git a/config/config.exs b/config/config.exs index 6f1f25061..2c151f9aa 100644 --- a/config/config.exs +++ b/config/config.exs @@ -235,7 +235,7 @@ config :mobilizon, :anonymous, config :mobilizon, Oban, repo: Mobilizon.Storage.Repo, - prune: {:maxlen, 10_000}, + log: false, queues: [default: 10, search: 5, mailers: 10, background: 5] config :mobilizon, :rich_media, diff --git a/config/test.exs b/config/test.exs index 2c8b466f3..6fb0e2cf1 100644 --- a/config/test.exs +++ b/config/test.exs @@ -51,7 +51,7 @@ config :tesla, Mobilizon.Service.HTTP.BaseClient, adapter: Mobilizon.Service.HTT config :mobilizon, Mobilizon.Service.Geospatial, service: Mobilizon.Service.Geospatial.Mock -config :mobilizon, Oban, queues: false, prune: :disabled, crontab: false +config :mobilizon, Oban, queues: false, crontab: false, plugins: false config :mobilizon, Mobilizon.Web.Auth.Guardian, secret_key: "some secret" diff --git a/lib/service/workers/background.ex b/lib/service/workers/background.ex index d2b603d09..ad0f06055 100644 --- a/lib/service/workers/background.ex +++ b/lib/service/workers/background.ex @@ -9,7 +9,7 @@ defmodule Mobilizon.Service.Workers.Background do use Mobilizon.Service.Workers.Helper, queue: "background" @impl Oban.Worker - def perform(%{"op" => "delete_actor", "actor_id" => actor_id} = args, _job) do + def perform(%Job{args: %{"op" => "delete_actor", "actor_id" => actor_id} = args}) do with reserve_username when is_boolean(reserve_username) <- Map.get(args, "reserve_username", true), %Actor{} = actor <- Actors.get_actor(actor_id) do @@ -17,7 +17,7 @@ defmodule Mobilizon.Service.Workers.Background do end end - def perform(%{"op" => "actor_key_rotation", "actor_id" => actor_id}, _job) do + def perform(%Job{args: %{"op" => "actor_key_rotation", "actor_id" => actor_id}}) do with %Actor{} = actor <- Actors.get_actor(actor_id) do Actors.actor_key_rotation(actor) end diff --git a/lib/service/workers/build_search.ex b/lib/service/workers/build_search.ex index 85913aabd..fd94524e6 100644 --- a/lib/service/workers/build_search.ex +++ b/lib/service/workers/build_search.ex @@ -13,13 +13,17 @@ defmodule Mobilizon.Service.Workers.BuildSearch do use Mobilizon.Service.Workers.Helper, queue: "search" @impl Oban.Worker - def perform(%{"op" => "insert_search_event", "event_id" => event_id}, _job) do + def perform(%Job{ + args: %{"op" => "insert_search_event", "event_id" => event_id} + }) do with {:ok, %Event{} = event} <- Events.get_event_with_preload(event_id) do insert_search_event(event) end end - def perform(%{"op" => "update_search_event", "event_id" => event_id}, _job) do + def perform(%Job{ + args: %{"op" => "update_search_event", "event_id" => event_id} + }) do with {:ok, %Event{} = event} <- Events.get_event_with_preload(event_id) do insert_search_event(event) end diff --git a/lib/service/workers/notification.ex b/lib/service/workers/notification.ex index ec95fa987..857ede1bc 100644 --- a/lib/service/workers/notification.ex +++ b/lib/service/workers/notification.ex @@ -14,7 +14,9 @@ defmodule Mobilizon.Service.Workers.Notification do use Mobilizon.Service.Workers.Helper, queue: "mailers" @impl Oban.Worker - def perform(%{"op" => "before_event_notification", "participant_id" => participant_id}, _job) do + def perform(%Job{ + args: %{"op" => "before_event_notification", "participant_id" => participant_id} + }) do with %Participant{actor: %Actor{user_id: user_id}, event: %Event{status: :confirmed}} = participant <- Events.get_participant(participant_id), %User{email: email, locale: locale, settings: %Setting{notification_before_event: true}} <- @@ -27,7 +29,9 @@ defmodule Mobilizon.Service.Workers.Notification do end end - def perform(%{"op" => "on_day_notification", "user_id" => user_id}, _job) do + def perform(%Job{ + args: %{"op" => "on_day_notification", "user_id" => user_id} + }) do with %User{locale: locale, settings: %Setting{timezone: timezone, notification_on_day: true}} = user <- Users.get_user_with_settings!(user_id), {start, tomorrow} <- calculate_start_end(1, timezone), @@ -52,7 +56,9 @@ defmodule Mobilizon.Service.Workers.Notification do end end - def perform(%{"op" => "weekly_notification", "user_id" => user_id}, _job) do + def perform(%Job{ + args: %{"op" => "weekly_notification", "user_id" => user_id} + }) do with %User{ locale: locale, settings: %Setting{timezone: timezone, notification_each_week: true} @@ -80,14 +86,13 @@ defmodule Mobilizon.Service.Workers.Notification do end end - def perform( - %{ + def perform(%Job{ + args: %{ "op" => "pending_participation_notification", "user_id" => user_id, "event_id" => event_id - }, - _job - ) do + } + }) do with %User{} = user <- Users.get_user(user_id), {:ok, %Event{} = event} <- Events.get_event(event_id), %Page{total: total} when total > 0 <- diff --git a/mix.exs b/mix.exs index 4935b2f2a..2b8dba13d 100644 --- a/mix.exs +++ b/mix.exs @@ -116,7 +116,7 @@ defmodule Mobilizon.Mixfile do {:ex_cldr_dates_times, "~> 2.2"}, {:ex_optimizer, "~> 0.1"}, {:progress_bar, "~> 2.0"}, - {:oban, "~> 1.2.0"}, + {:oban, "~> 2.0.0"}, {:floki, "~> 0.27.0"}, {:ip_reserved, "~> 0.1.0"}, {:fast_sanitize, "~> 0.1"}, diff --git a/mix.lock b/mix.lock index dd10c0e01..83f54c213 100644 --- a/mix.lock +++ b/mix.lock @@ -100,7 +100,7 @@ "nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"}, "oauth2": {:hex, :oauth2, "2.0.0", "338382079fe16c514420fa218b0903f8ad2d4bfc0ad0c9f988867dfa246731b0", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "881b8364ac7385f9fddc7949379cbe3f7081da37233a1aa7aab844670a91e7e7"}, "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm", "9374f4302045321874cccdc57eb975893643bd69c3b22bf1312dab5f06e5788e"}, - "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"}, + "oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"}, "ordinal": {:hex, :ordinal, "0.1.0", "2f7a1a64ff4be44b8a674718bb00d1584188fe92fa2fa48b95b1e72096d74a34", [:mix], [], "hexpm", "9f3d0a50c285ac99faa9626376e11afa6fc83d42e95166768b37d176cff485a3"}, "paddle": {:hex, :paddle, "0.1.4", "3697996d79e3d771d6f7560a23e4bad1ed7b7f7fd3e784f97bc39565963b2b13", [:mix], [], "hexpm", "fc719a9e7c86f319b9f4bf413d6f0f326b0c4930d5bc6630d074598ed38e2143"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, diff --git a/test/federation/activity_pub/transmogrifier_test.exs b/test/federation/activity_pub/transmogrifier_test.exs index 3d4cdbfe5..3a0ce1328 100644 --- a/test/federation/activity_pub/transmogrifier_test.exs +++ b/test/federation/activity_pub/transmogrifier_test.exs @@ -861,7 +861,7 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do end) {:ok, _activity, _actor} = Transmogrifier.handle_incoming(data) - assert %{success: 1, failure: 0} == Oban.drain_queue(:background) + assert %{success: 1, failure: 0} == Oban.drain_queue(queue: :background) assert {:error, :actor_not_found} = Actors.get_actor_by_url(url) assert {:error, :event_not_found} = Events.get_event(event1.id) diff --git a/test/graphql/resolvers/person_test.exs b/test/graphql/resolvers/person_test.exs index 9950d5e97..7b6471f27 100644 --- a/test/graphql/resolvers/person_test.exs +++ b/test/graphql/resolvers/person_test.exs @@ -511,7 +511,7 @@ defmodule Mobilizon.GraphQL.Resolvers.PersonTest do args: %{"actor_id" => person_id, "op" => "delete_actor", "reserve_username" => true} ) - assert %{success: 1, failure: 0} == Oban.drain_queue(:background) + assert %{success: 1, failure: 0} == Oban.drain_queue(queue: :background) query = """ { @@ -708,7 +708,7 @@ defmodule Mobilizon.GraphQL.Resolvers.PersonTest do assert is_nil(res["errors"]) assert res["data"]["suspendProfile"]["id"] == to_string(remote_profile_id) - assert %{success: 1, failure: 0} == Oban.drain_queue(:background) + assert %{success: 1, failure: 0} == Oban.drain_queue(queue: :background) res = conn diff --git a/test/graphql/resolvers/user_test.exs b/test/graphql/resolvers/user_test.exs index 70637a328..24f6bc2d8 100644 --- a/test/graphql/resolvers/user_test.exs +++ b/test/graphql/resolvers/user_test.exs @@ -1381,7 +1381,7 @@ defmodule Mobilizon.GraphQL.Resolvers.UserTest do assert Users.get_user(user.id).disabled == true - assert %{success: 2, failure: 0} == Oban.drain_queue(:background) + assert %{success: 2, failure: 0} == Oban.drain_queue(queue: :background) assert_raise Ecto.NoResultsError, fn -> Events.get_event!(event_id) diff --git a/test/mobilizon/actors/actors_test.exs b/test/mobilizon/actors/actors_test.exs index b1765701c..ecba30221 100644 --- a/test/mobilizon/actors/actors_test.exs +++ b/test/mobilizon/actors/actors_test.exs @@ -370,7 +370,7 @@ defmodule Mobilizon.ActorsTest do args: %{"actor_id" => actor.id, "op" => "delete_actor", "reserve_username" => true} ) - assert %{success: 1, failure: 0} == Oban.drain_queue(:background) + assert %{success: 1, failure: 0} == Oban.drain_queue(queue: :background) assert %Actor{ name: nil, diff --git a/test/service/workers/notification_test.exs b/test/service/workers/notification_test.exs index 469e95b0d..de0a23952 100644 --- a/test/service/workers/notification_test.exs +++ b/test/service/workers/notification_test.exs @@ -32,10 +32,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do %Participant{id: participant_id} = participant = insert(:participant, role: :participant, actor: actor) - Notification.perform( - %{"op" => "before_event_notification", "participant_id" => participant_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "before_event_notification", "participant_id" => participant_id} + }) assert_delivered_email( NotificationMailer.before_event_notification( @@ -59,10 +58,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do assert {:ok, %Participant{}} = Events.delete_participant(participant) - Notification.perform( - %{"op" => "before_event_notification", "participant_id" => participant_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "before_event_notification", "participant_id" => participant_id} + }) refute_delivered_email( NotificationMailer.before_event_notification( @@ -78,10 +76,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do %Participant{id: participant_id} = participant = insert(:participant, role: :participant, event: event) - Notification.perform( - %{"op" => "before_event_notification", "participant_id" => participant_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "before_event_notification", "participant_id" => participant_id} + }) refute_delivered_email( NotificationMailer.before_event_notification( @@ -104,10 +101,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do %Participant{} = participant = insert(:participant, role: :participant, actor: actor) - Notification.perform( - %{"op" => "on_day_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "on_day_notification", "user_id" => user_id} + }) assert_delivered_email( NotificationMailer.on_day_notification( @@ -137,10 +133,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do assert {:ok, %Participant{}} = Events.delete_participant(participant) - Notification.perform( - %{"op" => "on_day_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "on_day_notification", "user_id" => user_id} + }) refute_delivered_email( NotificationMailer.on_day_notification( @@ -164,10 +159,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do %Participant{} = participant = insert(:participant, role: :participant, event: event, actor: actor) - Notification.perform( - %{"op" => "on_day_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "on_day_notification", "user_id" => user_id} + }) refute_delivered_email( NotificationMailer.on_day_notification( @@ -193,10 +187,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do acc ++ [participant] end) - Notification.perform( - %{"op" => "on_day_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "on_day_notification", "user_id" => user_id} + }) refute_delivered_email( NotificationMailer.on_day_notification( @@ -220,10 +213,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do %Participant{} = participant = insert(:participant, role: :participant, actor: actor) - Notification.perform( - %{"op" => "weekly_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "weekly_notification", "user_id" => user_id} + }) assert_delivered_email( NotificationMailer.weekly_notification( @@ -253,10 +245,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do assert {:ok, %Participant{}} = Events.delete_participant(participant) - Notification.perform( - %{"op" => "weekly_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "weekly_notification", "user_id" => user_id} + }) refute_delivered_email( NotificationMailer.weekly_notification( @@ -280,10 +271,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do %Participant{} = participant = insert(:participant, role: :participant, event: event, actor: actor) - Notification.perform( - %{"op" => "weekly_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "weekly_notification", "user_id" => user_id} + }) refute_delivered_email( NotificationMailer.weekly_notification( @@ -309,10 +299,9 @@ defmodule Mobilizon.Service.Workers.NotificationTest do acc ++ [participant] end) - Notification.perform( - %{"op" => "weekly_notification", "user_id" => user_id}, - nil - ) + Notification.perform(%Oban.Job{ + args: %{"op" => "weekly_notification", "user_id" => user_id} + }) refute_delivered_email( NotificationMailer.weekly_notification( @@ -333,14 +322,13 @@ defmodule Mobilizon.Service.Workers.NotificationTest do %Participant{} = insert(:participant, event: event, role: :not_approved) %Participant{} = insert(:participant, event: event, role: :not_approved) - Notification.perform( - %{ + Notification.perform(%Oban.Job{ + args: %{ "op" => "pending_participation_notification", "user_id" => user_id, "event_id" => event_id - }, - nil - ) + } + }) assert_delivered_email( NotificationMailer.pending_participation_notification( From 5bc7e7f9b219ed7d3f5febec4c9ca8be38d54cf5 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 12 Aug 2020 16:05:34 +0200 Subject: [PATCH 2/3] Introduce Sitemaps Signed-off-by: Thomas Citharel --- config/config.exs | 5 +- lib/mobilizon/actors/actors.ex | 12 +++- lib/mobilizon/events/events.ex | 14 ++++ lib/mobilizon/posts/posts.ex | 14 +++- lib/mobilizon/storage/ecto.ex | 2 +- lib/service/site_map.ex | 66 +++++++++++++++++++ lib/service/workers/build_site_map.ex | 12 ++++ lib/web/controllers/page_controller.ex | 12 +++- lib/web/router.ex | 29 ++++---- ...nymous_participation_confirmation.html.eex | 2 +- ...nymous_participation_confirmation.text.eex | 2 +- .../email/email_changed_new.html.eex | 2 +- .../email/email_changed_new.text.eex | 2 +- lib/web/templates/email/group_invite.html.eex | 2 +- lib/web/templates/email/group_invite.text.eex | 6 +- lib/web/templates/email/report.html.eex | 2 +- lib/web/templates/email/report.text.eex | 2 +- mix.exs | 2 + mix.lock | 3 +- test/graphql/resolvers/group_test.exs | 5 +- test/support/factory.ex | 6 +- 21 files changed, 163 insertions(+), 39 deletions(-) create mode 100644 lib/service/site_map.ex create mode 100644 lib/service/workers/build_site_map.ex diff --git a/config/config.exs b/config/config.exs index 2c151f9aa..d572d9842 100644 --- a/config/config.exs +++ b/config/config.exs @@ -236,7 +236,10 @@ config :mobilizon, :anonymous, config :mobilizon, Oban, repo: Mobilizon.Storage.Repo, log: false, - queues: [default: 10, search: 5, mailers: 10, background: 5] + queues: [default: 10, search: 5, mailers: 10, background: 5], + crontab: [ + {"@daily", Mobilizon.Service.Workers.BuildSiteMap, queue: :background} + ] config :mobilizon, :rich_media, parsers: [ diff --git a/lib/mobilizon/actors/actors.ex b/lib/mobilizon/actors/actors.ex index 5af4dabb0..b598c5d66 100644 --- a/lib/mobilizon/actors/actors.ex +++ b/lib/mobilizon/actors/actors.ex @@ -56,7 +56,6 @@ defmodule Mobilizon.Actors do :rejected ]) - @public_visibility [:public, :unlisted] @administrator_roles [:creator, :administrator] @moderator_roles [:moderator] ++ @administrator_roles @member_roles [:member] ++ @moderator_roles @@ -537,6 +536,15 @@ defmodule Mobilizon.Actors do |> Page.build_page(page, limit) end + @doc """ + Lists the groups. + """ + @spec list_groups_for_stream :: Enum.t() + def list_groups_for_stream do + groups_query() + |> Repo.stream() + end + @doc """ Returns the list of groups an actor is member of. """ @@ -1212,7 +1220,7 @@ defmodule Mobilizon.Actors do from( a in Actor, where: a.type == ^:Group, - where: a.visibility in ^@public_visibility + where: a.visibility == ^:public ) end diff --git a/lib/mobilizon/events/events.ex b/lib/mobilizon/events/events.ex index af877fb3f..6a5248ce0 100644 --- a/lib/mobilizon/events/events.ex +++ b/lib/mobilizon/events/events.ex @@ -366,6 +366,15 @@ defmodule Mobilizon.Events do |> Repo.all() end + @spec stream_events_for_sitemap :: Enum.t() + def stream_events_for_sitemap do + Event + |> filter_public_visibility() + |> filter_draft() + |> filter_local() + |> Repo.stream() + end + @doc """ Returns the list of events with the same tags. """ @@ -1583,6 +1592,11 @@ defmodule Mobilizon.Events do defp filter_future_events(query, false), do: query + @spec filter_local(Ecto.Query.t()) :: Ecto.Query.t() + defp filter_local(query) do + where(query, [q], q.local == true) + end + @spec filter_local_or_from_followed_instances_events(Ecto.Query.t()) :: Ecto.Query.t() defp filter_local_or_from_followed_instances_events(query) do from(q in query, diff --git a/lib/mobilizon/posts/posts.ex b/lib/mobilizon/posts/posts.ex index b30fae8be..7d1a7d179 100644 --- a/lib/mobilizon/posts/posts.ex +++ b/lib/mobilizon/posts/posts.ex @@ -21,6 +21,13 @@ defmodule Mobilizon.Posts do :private ]) + @spec list_posts_for_stream :: Enum.t() + def list_posts_for_stream do + Post + |> filter_public() + |> Repo.stream() + end + @doc """ Returns the list of recent posts for a group """ @@ -35,7 +42,7 @@ defmodule Mobilizon.Posts do def get_public_posts_for_group(%Actor{id: group_id}, page \\ nil, limit \\ nil) do group_id |> do_get_posts_for_group() - |> where([p], p.visibility == ^:public and not p.draft) + |> filter_public() |> Page.build_page(page, limit) end @@ -132,4 +139,9 @@ defmodule Mobilizon.Posts do where: p.post_id == ^post_id ) end + + @spec filter_public(Ecto.Query.t()) :: Ecto.Query.t() + defp filter_public(query) do + where(query, [p], p.visibility == ^:public and not p.draft) + end end diff --git a/lib/mobilizon/storage/ecto.ex b/lib/mobilizon/storage/ecto.ex index a6f8f048d..263224370 100644 --- a/lib/mobilizon/storage/ecto.ex +++ b/lib/mobilizon/storage/ecto.ex @@ -43,7 +43,7 @@ defmodule Mobilizon.Storage.Ecto do |> put_change(:id, uuid) |> put_change( :url, - apply(Routes, String.to_existing_atom("#{to_string(route)}_url"), [Endpoint, route, uuid]) + apply(Routes, String.to_existing_atom("page_url"), [Endpoint, route, uuid]) ) end end diff --git a/lib/service/site_map.ex b/lib/service/site_map.ex new file mode 100644 index 000000000..0b0570d01 --- /dev/null +++ b/lib/service/site_map.ex @@ -0,0 +1,66 @@ +defmodule Mobilizon.Service.SiteMap do + @moduledoc """ + Generates a sitemap + """ + + alias Mobilizon.{Actors, Events, Posts} + alias Mobilizon.Storage.Repo + alias Mobilizon.Web.Endpoint + alias Mobilizon.Web.Router.Helpers, as: Routes + + @default_static_frequency :monthly + + def generate_sitemap do + static_routes = [ + {Routes.page_url(Endpoint, :index, []), :daily}, + "#{Endpoint.url()}/about/instance", + "#{Endpoint.url()}/about/mobilizon", + "#{Endpoint.url()}/terms", + "#{Endpoint.url()}/privacy", + "#{Endpoint.url()}/rules", + "#{Endpoint.url()}/glossary" + ] + + config = [ + store: Sitemapper.FileStore, + store_config: [path: "priv/static"], + sitemap_url: Endpoint.url(), + gzip: false + ] + + Repo.transaction(fn -> + Events.stream_events_for_sitemap() + |> Stream.concat(Actors.list_groups_for_stream()) + |> Stream.concat(Posts.list_posts_for_stream()) + |> Stream.concat( + Enum.map(static_routes, fn route -> + {url, frequency} = + case route do + {url, frequency} -> {url, frequency} + url when is_binary(url) -> {url, @default_static_frequency} + end + + %{url: url, updated_at: nil, frequence: frequency} + end) + ) + |> Stream.map(fn %{url: url, updated_at: updated_at} = args -> + frequence = Map.get(args, :frequence, :weekly) + + %Sitemapper.URL{ + loc: url, + changefreq: frequence, + lastmod: check_date_time(updated_at) + } + end) + |> Sitemapper.generate(config) + |> Sitemapper.persist(config) + |> Sitemapper.ping(config) + |> Stream.run() + end) + end + + # Sometimes we use naive datetimes + defp check_date_time(%NaiveDateTime{} = datetime), do: DateTime.from_naive!(datetime, "Etc/UTC") + defp check_date_time(%DateTime{} = datetime), do: datetime + defp check_date_time(_), do: nil +end diff --git a/lib/service/workers/build_site_map.ex b/lib/service/workers/build_site_map.ex new file mode 100644 index 000000000..067ca8947 --- /dev/null +++ b/lib/service/workers/build_site_map.ex @@ -0,0 +1,12 @@ +defmodule Mobilizon.Service.Workers.BuildSiteMap do + @moduledoc """ + Worker to build sitemap + """ + + alias Mobilizon.Service.SiteMap + + use Oban.Worker, queue: "background" + + @impl Oban.Worker + def perform(%Job{}), do: SiteMap.generate_sitemap() +end diff --git a/lib/web/controllers/page_controller.ex b/lib/web/controllers/page_controller.ex index c57663768..5500f54fc 100644 --- a/lib/web/controllers/page_controller.ex +++ b/lib/web/controllers/page_controller.ex @@ -8,11 +8,21 @@ defmodule Mobilizon.Web.PageController do alias Mobilizon.Events.Event alias Mobilizon.Federation.ActivityPub alias Mobilizon.Tombstone - alias Mobilizon.Web.{ActivityPubController, Cache} + alias Mobilizon.Web.{ActivityPubController, Cache, PageController} plug(:put_layout, false) action_fallback(Mobilizon.Web.FallbackController) + defdelegate my_events(conn, params), to: PageController, as: :index + defdelegate create_event(conn, params), to: PageController, as: :index + defdelegate list_events(conn, params), to: PageController, as: :index + defdelegate explore_events(conn, params), to: PageController, as: :index + defdelegate edit_event(conn, params), to: PageController, as: :index + defdelegate moderation_report(conn, params), to: PageController, as: :index + defdelegate participation_email_confirmation(conn, params), to: PageController, as: :index + defdelegate user_email_validation(conn, params), to: PageController, as: :index + defdelegate my_groups(conn, params), to: PageController, as: :index + @spec index(Plug.Conn.t(), any) :: Plug.Conn.t() def index(conn, _params), do: render(conn, :index) diff --git a/lib/web/router.ex b/lib/web/router.ex index 15ec71b39..283c69aed 100644 --- a/lib/web/router.ex +++ b/lib/web/router.ex @@ -81,12 +81,12 @@ defmodule Mobilizon.Web.Router do pipe_through(:activity_pub_signature) get("/@:name", PageController, :actor) - get("/events/me", PageController, :index) + get("/events/me", PageController, :my_events) get("/events/:uuid", PageController, :event) get("/comments/:uuid", PageController, :comment) - get("/resource/:uuid", PageController, :resource, as: "resource") - get("/todo-list/:uuid", PageController, :todo_list, as: "todo_list") - get("/todo/:uuid", PageController, :todo, as: "todo") + get("/resource/:uuid", PageController, :resource) + get("/todo-list/:uuid", PageController, :todo_list) + get("/todo/:uuid", PageController, :todo) get("/@:name/todos", PageController, :todos) get("/@:name/resources", PageController, :resources) get("/@:name/posts", PageController, :posts) @@ -139,21 +139,20 @@ defmodule Mobilizon.Web.Router do pipe_through(:browser) # Because the "/events/:uuid" route caches all these, we need to force them - get("/events/create", PageController, :index) - get("/events/list", PageController, :index) - get("/events/me", PageController, :index) - get("/events/explore", PageController, :index) - get("/events/:uuid/edit", PageController, :index) + get("/events/create", PageController, :create_event) + get("/events/list", PageController, :list_events) + get("/events/me", PageController, :my_events) + get("/events/explore", PageController, :explore_events) + get("/events/:uuid/edit", PageController, :edit_event) # This is a hack to ease link generation into emails - get("/moderation/reports/:id", PageController, :index, as: "moderation_report") + get("/moderation/reports/:id", PageController, :moderation_report) - get("/participation/email/confirm/:token", PageController, :index, - as: "participation_email_confirmation" - ) + get("/participation/email/confirm/:token", PageController, :participation_email_confirmation) - get("/validate/email/:token", PageController, :index, as: "user_email_validation") - get("/groups/me", PageController, :index, as: "my_groups") + get("/validate/email/:token", PageController, :user_email_validation) + + get("/groups/me", PageController, :my_groups) get("/interact", PageController, :interact) diff --git a/lib/web/templates/email/anonymous_participation_confirmation.html.eex b/lib/web/templates/email/anonymous_participation_confirmation.html.eex index b8069d642..0dcd6ae90 100644 --- a/lib/web/templates/email/anonymous_participation_confirmation.html.eex +++ b/lib/web/templates/email/anonymous_participation_confirmation.html.eex @@ -54,7 +54,7 @@ - diff --git a/lib/web/templates/email/anonymous_participation_confirmation.text.eex b/lib/web/templates/email/anonymous_participation_confirmation.text.eex index 0ca571fdb..b5e46e1da 100644 --- a/lib/web/templates/email/anonymous_participation_confirmation.text.eex +++ b/lib/web/templates/email/anonymous_participation_confirmation.text.eex @@ -2,5 +2,5 @@ == <%= gettext "Hi there! You just registered to join this event: « %{title} ». Please confirm the e-mail address you provided:", title: @participant.event.title %> <%= gettext "If you didn't trigger this email, you may safely ignore it." %> -<%= participation_email_confirmation_url(Mobilizon.Web.Endpoint, :index, @participant.metadata.confirmation_token) %> +<%= page_url(Mobilizon.Web.Endpoint, :participation_email_confirmation, @participant.metadata.confirmation_token) %> <%= ngettext "Would you wish to cancel your attendance, visit the event page through the link above and click the « Attending » button.", "Would you wish to cancel your attendance to one or several events, visit the event pages through the links above and click the « Attending » button.", 1 %> diff --git a/lib/web/templates/email/email_changed_new.html.eex b/lib/web/templates/email/email_changed_new.html.eex index df88989bf..374d25f14 100644 --- a/lib/web/templates/email/email_changed_new.html.eex +++ b/lib/web/templates/email/email_changed_new.html.eex @@ -47,7 +47,7 @@
+ <%= gettext "Confirm my e-mail address" %>
diff --git a/lib/web/templates/email/email_changed_new.text.eex b/lib/web/templates/email/email_changed_new.text.eex index 4822d6eba..65d5c4fd6 100644 --- a/lib/web/templates/email/email_changed_new.text.eex +++ b/lib/web/templates/email/email_changed_new.text.eex @@ -1,5 +1,5 @@ <%= gettext "Confirm new email" %> == <%= gettext "Hi there! It seems like you wanted to change the email address linked to your account on %{instance}. If you still wish to do so, please click the button below to confirm the change. You will then be able to log in to %{instance} with this new email address.", %{instance: @instance_name} %> -<%= user_email_validation_url(Mobilizon.Web.Endpoint, :index, @token) %> +<%= page_url(Mobilizon.Web.Endpoint, :user_email_validation, @token) %> <%= gettext "If you didn't trigger the change yourself, please ignore this message." %> diff --git a/lib/web/templates/email/group_invite.html.eex b/lib/web/templates/email/group_invite.html.eex index 597b385f0..6b95597b9 100644 --- a/lib/web/templates/email/group_invite.html.eex +++ b/lib/web/templates/email/group_invite.html.eex @@ -55,7 +55,7 @@
- + <%= gettext "Verify your email address" %>
diff --git a/lib/web/templates/email/group_invite.text.eex b/lib/web/templates/email/group_invite.text.eex index ad2db9900..c5db561bb 100644 --- a/lib/web/templates/email/group_invite.text.eex +++ b/lib/web/templates/email/group_invite.text.eex @@ -1,9 +1,5 @@ <%= gettext "Come along!" %> - == - <%= gettext "%{inviter} just invited you to join their group %{group}", group: @group.name, inviter: @inviter.name %> - <%= gettext "To accept this invitation, head over to your groups." %> - -<%= my_groups_url(Mobilizon.Web.Endpoint, :index) %> +<%= page_url(Mobilizon.Web.Endpoint, :my_groups) %> diff --git a/lib/web/templates/email/report.html.eex b/lib/web/templates/email/report.html.eex index 75a04ea79..e883ded8b 100644 --- a/lib/web/templates/email/report.html.eex +++ b/lib/web/templates/email/report.html.eex @@ -100,7 +100,7 @@
- + <%= gettext "See my groups" %> - diff --git a/lib/web/templates/email/report.text.eex b/lib/web/templates/email/report.text.eex index 9d841acf4..f41344f2c 100644 --- a/lib/web/templates/email/report.text.eex +++ b/lib/web/templates/email/report.text.eex @@ -14,4 +14,4 @@ <%= gettext "Reason" %> <%= @report.content %> <% end %> -<%= gettext "View report:" %> <%= moderation_report_url(Mobilizon.Web.Endpoint, :index, @report.id) %> +<%= gettext "View report:" %> <%= page_url(Mobilizon.Web.Endpoint, :moderation_report, @report.id) %> diff --git a/mix.exs b/mix.exs index 2b8dba13d..95ce08a00 100644 --- a/mix.exs +++ b/mix.exs @@ -132,6 +132,8 @@ defmodule Mobilizon.Mixfile do git: "https://github.com/tcitworld/ueberauth_gitlab.git", branch: "upgrade-deps"}, {:ecto_shortuuid, "~> 0.1"}, {:tesla, "~> 1.3.0"}, + {:sitemapper, "~> 0.4.0"}, + {:xml_builder, "~> 2.1.1", override: true}, # Dev and test dependencies {:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]}, {:ex_machina, "~> 2.3", only: [:dev, :test]}, diff --git a/mix.lock b/mix.lock index 83f54c213..860307b41 100644 --- a/mix.lock +++ b/mix.lock @@ -119,6 +119,7 @@ "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, "rsa_ex": {:hex, :rsa_ex, "0.4.0", "e28dd7dc5236e156df434af0e4aa822384c8866c928e17b785d4edb7c253b558", [:mix], [], "hexpm", "40e1f08e8401da4be59a6dd0f4da30c42d5bb01703161f0208d839d97db27f4e"}, "shortuuid": {:hex, :shortuuid, "2.1.2", "14dbafdb2f6c7213fdfcc05c7572384b5051a7b1621170018ad4c05504bd96c1", [:mix], [], "hexpm", "d9b0c4f37500ea5199b6275ece872e213e9f45a015caf4aa777cec84f63ad353"}, + "sitemapper": {:hex, :sitemapper, "0.4.0", "50061503ddc306aabcb984b377415961ff49696d70cd95081b20fa2a86f18ac4", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "9bfe778635a6801e7762b185564df6a174d6016b15cbeaf50746e94ee55138c3"}, "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, "slugger": {:hex, :slugger, "0.3.0", "efc667ab99eee19a48913ccf3d038b1fb9f165fa4fbf093be898b8099e61b6ed", [:mix], [], "hexpm", "20d0ded0e712605d1eae6c5b4889581c3460d92623a930ddda91e0e609b5afba"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, @@ -139,5 +140,5 @@ "unicode_util_compat": {:hex, :unicode_util_compat, "0.5.0", "8516502659002cec19e244ebd90d312183064be95025a319a6c7e89f4bccd65b", [:rebar3], [], "hexpm", "d48d002e15f5cc105a696cf2f1bbb3fc72b4b770a184d8420c8db20da2674b38"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, "uuid": {:git, "git://github.com/botsunit/erlang-uuid", "1effbbbd200f9f5d9d5154e81b83fe8e4c3fe714", [branch: "master"]}, - "xml_builder": {:hex, :xml_builder, "2.0.0", "371ed27bb63bf0598dbaf3f0c466e5dc7d16cb4ecb68f06a67f953654062e21b", [:mix], [], "hexpm", "baeb5c8d42204bac2b856ffd50e8cda42d63b622984538d18d92733e4e790fbd"}, + "xml_builder": {:hex, :xml_builder, "2.1.2", "90cb9ad382958934c78c6ddfbe6d385a8ce147d84b61cbfa83ec93a169d0feab", [:mix], [], "hexpm", "b89046041da2fbc1d51d31493ba31b9d5fc6223c93384bf513a1a9e1df9ec081"}, } diff --git a/test/graphql/resolvers/group_test.exs b/test/graphql/resolvers/group_test.exs index 087285867..97b663f68 100644 --- a/test/graphql/resolvers/group_test.exs +++ b/test/graphql/resolvers/group_test.exs @@ -87,8 +87,9 @@ defmodule Mobilizon.Web.Resolvers.GroupTest do end describe "list groups" do - test "list_groups/3 returns all public or unlisted groups", %{conn: conn} do - group = insert(:group, visibility: :unlisted) + test "list_groups/3 returns all public groups", %{conn: conn} do + group = insert(:group, visibility: :public) + insert(:group, visibility: :unlisted) insert(:group, visibility: :private) query = """ diff --git a/test/support/factory.ex b/test/support/factory.ex index 84cf06dcb..be0581984 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -285,7 +285,7 @@ defmodule Mobilizon.Factory do title: sequence("todo list"), actor: build(:group), id: uuid, - url: Routes.todo_list_url(Endpoint, :todo_list, uuid) + url: Routes.page_url(Endpoint, :todo_list, uuid) } end @@ -299,7 +299,7 @@ defmodule Mobilizon.Factory do status: false, due_date: Timex.shift(DateTime.utc_now(), hours: 2), assigned_to: build(:actor), - url: Routes.todo_url(Endpoint, :todo, uuid), + url: Routes.page_url(Endpoint, :todo, uuid), creator: build(:actor) } end @@ -316,7 +316,7 @@ defmodule Mobilizon.Factory do actor: build(:group), creator: build(:actor), parent: nil, - url: Routes.resource_url(Endpoint, :resource, uuid), + url: Routes.page_url(Endpoint, :resource, uuid), path: "/#{title}" } end From 310e51dd13bf6494a4c6b885335ab060325a300b Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 12 Aug 2020 16:22:42 +0200 Subject: [PATCH 3/3] Add a mix task to generate a sitemap Signed-off-by: Thomas Citharel --- lib/mix/tasks/mobilizon/site_map.ex | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 lib/mix/tasks/mobilizon/site_map.ex diff --git a/lib/mix/tasks/mobilizon/site_map.ex b/lib/mix/tasks/mobilizon/site_map.ex new file mode 100644 index 000000000..ef4162892 --- /dev/null +++ b/lib/mix/tasks/mobilizon/site_map.ex @@ -0,0 +1,21 @@ +defmodule Mix.Tasks.Mobilizon.SiteMap do + @moduledoc """ + Task to generate a new Sitemap + """ + use Mix.Task + + alias Mix.Tasks.Mobilizon.Common + alias Mobilizon.Service.SiteMap + alias Mobilizon.Web.Endpoint + + @preferred_cli_env "prod" + + @shortdoc "Generates a new Sitemap" + def run(["generate"]) do + Common.start_mobilizon() + + with {:ok, :ok} <- SiteMap.generate_sitemap() do + Mix.shell().info("Sitemap saved to #{Endpoint.url()}/sitemap.xml") + end + end +end
+ <%= gettext "View report" %>