Fix issue when updating event and introduce background jobs

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel 2019-11-04 15:10:58 +01:00
parent ac758cc577
commit 5bf50057f5
17 changed files with 193 additions and 39 deletions

View File

@ -139,6 +139,11 @@ config :mobilizon, Mobilizon.Service.Geospatial.GoogleMaps,
config :mobilizon, Mobilizon.Service.Geospatial.MapQuest,
api_key: System.get_env("GEOSPATIAL_MAP_QUEST_API_KEY") || nil
config :mobilizon, Oban,
repo: Mobilizon.Storage.Repo,
prune: {:maxlen, 10_000},
queues: [default: 10, search: 20]
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

View File

@ -40,3 +40,5 @@ config :exvcr,
vcr_cassette_library_dir: "test/fixtures/vcr_cassettes"
config :mobilizon, Mobilizon.Service.Geospatial, service: Mobilizon.Service.Geospatial.Mock
config :mobilizon, Oban, queues: false, prune: :disabled

View File

@ -7,7 +7,7 @@ defmodule Mix.Tasks.Mobilizon.SetupSearch do
use Mix.Task
alias Mobilizon.Service.Search
alias Mobilizon.Service.Workers.BuildSearchWorker
alias Mobilizon.Storage.Repo
alias Mobilizon.Events.Event
import Ecto.Query
@ -30,7 +30,7 @@ defmodule Mix.Tasks.Mobilizon.SetupSearch do
end
defp insert_search_event([%Event{url: url} = event | events], nb_events) do
case Search.insert_search_event(event) do
case BuildSearchWorker.insert_search_event(event) do
{:ok, _} ->
Logger.debug("Added event #{url} to the search")

View File

@ -37,6 +37,7 @@ defmodule Mobilizon do
# supervisors
Mobilizon.Storage.Repo,
MobilizonWeb.Endpoint,
{Oban, Application.get_env(:mobilizon, Oban)},
# workers
Guardian.DB.Token.SweeperServer,
Mobilizon.Service.Federator,
@ -46,9 +47,7 @@ defmodule Mobilizon do
cachex_spec(:activity_pub, 2500, 3, 15)
]
opts = [strategy: :one_for_one, name: Mobilizon.Supervisor]
Supervisor.start_link(children, opts)
Supervisor.start_link(children, strategy: :one_for_one, name: Mobilizon.Supervisor)
end
@spec config_change(keyword, keyword, [atom]) :: :ok

View File

@ -179,6 +179,8 @@ defmodule Mobilizon.Events.Event do
defp put_tags(%Changeset{} = changeset, _), do: changeset
# We need a changeset instead of a raw struct because of slug which is generated in changeset
defp process_tag(%{id: _id} = tag), do: tag
defp process_tag(tag) do
Tag.changeset(%Tag{}, tag)
end

View File

@ -13,7 +13,7 @@ defmodule Mobilizon.Events do
alias Mobilizon.Actors.Actor
alias Mobilizon.Addresses.Address
alias Mobilizon.Service.Search
alias Mobilizon.Service.Workers.BuildSearchWorker
alias Mobilizon.Events.{
Comment,
@ -253,7 +253,9 @@ defmodule Mobilizon.Events do
def create_event(attrs \\ %{}) do
with {:ok, %{insert: %Event{} = event}} <- do_create_event(attrs),
%Event{} = event <- Repo.preload(event, @event_preloads) do
Task.start(fn -> Search.insert_search_event(event) end)
unless event.draft,
do: BuildSearchWorker.enqueue(:insert_search_event, %{"event_id" => event.id})
{:ok, event}
else
err -> err
@ -291,7 +293,7 @@ defmodule Mobilizon.Events do
We start by updating the event and then insert a first participant if the event is not a draft anymore
"""
@spec update_event(Event.t(), map) :: {:ok, Event.t()} | {:error, Changeset.t()}
def update_event(%Event{} = old_event, attrs) do
def update_event(%Event{draft: old_draft} = old_event, attrs) do
with %Changeset{changes: changes} = changeset <-
Event.update_changeset(Repo.preload(old_event, :tags), attrs),
{:ok, %{update: %Event{} = new_event}} <-
@ -301,7 +303,7 @@ defmodule Mobilizon.Events do
changeset
)
|> Multi.run(:write, fn _repo, %{update: %Event{draft: draft} = event} ->
with {:is_draft, false} <- {:is_draft, draft},
with {:was_draft, true} <- {:was_draft, old_draft == true && draft == false},
{:ok, %Participant{} = participant} <-
create_participant(
%{
@ -313,7 +315,7 @@ defmodule Mobilizon.Events do
) do
{:ok, participant}
else
{:is_draft, true} -> {:ok, nil}
{:was_draft, false} -> {:ok, nil}
err -> err
end
end)
@ -326,7 +328,8 @@ defmodule Mobilizon.Events do
changes
)
Task.start(fn -> Search.update_search_event(new_event) end)
unless new_event.draft,
do: BuildSearchWorker.enqueue(:update_search_event, %{"event_id" => new_event.id})
{:ok, Repo.preload(new_event, @event_preloads)}
end
@ -473,6 +476,16 @@ defmodule Mobilizon.Events do
|> Repo.one()
end
@doc """
Gets a tag by its title.
"""
@spec get_tag_by_title(String.t()) :: Tag.t() | nil
def get_tag_by_title(slug) do
slug
|> tag_by_title_query()
|> Repo.one()
end
@doc """
Gets an existing tag or creates the new one.
"""
@ -1351,6 +1364,11 @@ defmodule Mobilizon.Events do
from(t in Tag, where: t.slug == ^slug)
end
@spec tag_by_title_query(String.t()) :: Ecto.Query.t()
defp tag_by_title_query(title) do
from(t in Tag, where: t.title == ^title, limit: 1)
end
@spec tags_for_event_query(integer) :: Ecto.Query.t()
defp tags_for_event_query(event_id) do
from(

View File

@ -4,6 +4,7 @@ defmodule Mobilizon.Service.ActivityPub.Converter.Utils do
"""
alias Mobilizon.Actors.Actor
alias Mobilizon.Events
alias Mobilizon.Events.Tag
alias Mobilizon.Mention
alias Mobilizon.Service.ActivityPub
@ -66,7 +67,7 @@ defmodule Mobilizon.Service.ActivityPub.Converter.Utils do
defp fetch_tag(tag, acc) when is_map(tag) do
case tag["type"] do
"Hashtag" ->
acc ++ [%{title: tag}]
acc ++ [existing_tag_or_data(tag["name"])]
_err ->
acc
@ -74,7 +75,18 @@ defmodule Mobilizon.Service.ActivityPub.Converter.Utils do
end
defp fetch_tag(tag, acc) when is_bitstring(tag) do
acc ++ [%{title: tag}]
acc ++ [existing_tag_or_data(tag)]
end
defp existing_tag_or_data("#" <> tag_title) do
existing_tag_or_data(tag_title)
end
defp existing_tag_or_data(tag_title) do
case Events.get_tag_by_title(tag_title) do
%Tag{} = tag -> %{title: tag.title, id: tag.id}
nil -> %{title: tag_title}
end
end
@spec create_mention(map(), list()) :: list()

View File

@ -1,12 +1,28 @@
defmodule Mobilizon.Service.Search do
defmodule Mobilizon.Service.Workers.BuildSearchWorker do
@moduledoc """
Module to handle search service
Worker to build search results
"""
alias Mobilizon.Events
alias Mobilizon.Events.Event
alias Mobilizon.Storage.Repo
alias Ecto.Adapters.SQL
use Mobilizon.Service.Workers.WorkerHelper, queue: "search"
@impl Oban.Worker
def perform(%{"op" => "insert_search_event", "event_id" => event_id}, _job) do
with {:ok, %Event{} = event} <- Events.get_event_with_preload(event_id) do
insert_search_event(event)
end
end
def perform(%{"op" => "update_search_event", "event_id" => event_id}, _job) do
with {:ok, %Event{} = event} <- Events.get_event_with_preload(event_id) do
update_search_event(event)
end
end
def insert_search_event(%Event{} = event) do
SQL.query(
Repo,

View File

@ -0,0 +1,50 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/workers/worker_helper.ex
defmodule Mobilizon.Service.Workers.WorkerHelper do
@moduledoc """
Tools to ease dealing with workers
"""
alias Mobilizon.Config
alias Mobilizon.Service.Workers.WorkerHelper
def worker_args(queue) do
case Config.get([:workers, :retries, queue]) do
nil -> []
max_attempts -> [max_attempts: max_attempts]
end
end
def sidekiq_backoff(attempt, pow \\ 4, base_backoff \\ 15) do
backoff =
:math.pow(attempt, pow) +
base_backoff +
:rand.uniform(2 * base_backoff) * attempt
trunc(backoff)
end
defmacro __using__(opts) do
caller_module = __CALLER__.module
queue = Keyword.fetch!(opts, :queue)
quote do
# Note: `max_attempts` is intended to be overridden in `new/2` call
use Oban.Worker,
queue: unquote(queue),
max_attempts: 1
def enqueue(operation, params, worker_args \\ []) do
params = Map.merge(%{"op" => operation}, params)
queue_atom = String.to_existing_atom(unquote(queue))
worker_args = worker_args ++ WorkerHelper.worker_args(queue_atom)
unquote(caller_module)
|> apply(:new, [params, worker_args])
|> Mobilizon.Storage.Repo.insert()
end
end
end
end

View File

@ -100,6 +100,7 @@ defmodule Mobilizon.Mixfile do
{:ex_cldr_dates_times, "~> 2.0"},
{:ex_optimizer, "~> 0.1"},
{:progress_bar, "~> 2.0"},
{:oban, "~> 0.10"},
# Dev and test dependencies
{:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]},
{:ex_machina, "~> 2.3", only: [:dev, :test]},

View File

@ -85,6 +85,7 @@
"mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
"mogrify": {:hex, :mogrify, "0.7.3", "1494ee739f6e90de158dec4d4edee2d854d2f2d06a522e943f996ae176bca53d", [:mix], [], "hexpm"},
"nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"},
"oban": {:hex, :oban, "0.10.1", "c2ca0a413fb66b21dd58c7cb67fd80b01e9d79a0fbb28573f110057e7fdc3807", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
"phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
"phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},

View File

@ -0,0 +1,7 @@
defmodule Mobilizon.Storage.Repo.Migrations.AddUniqueIndexOnActorAndEventForParticipant do
use Ecto.Migration
def change do
create(unique_index(:participants, [:event_id, :actor_id]))
end
end

View File

@ -0,0 +1,13 @@
defmodule Mobilizon.Storage.Repo.Migrations.AddObanJobsTable do
use Ecto.Migration
def up do
Oban.Migrations.up()
end
# We specify `version: 1` in `down`, ensuring that we'll roll all the way back down if
# necessary, regardless of which version we've migrated `up` to.
def down do
Oban.Migrations.down(version: 1)
end
end

View File

@ -7,7 +7,7 @@ defmodule Mobilizon.EventsTest do
alias Mobilizon.Events
alias Mobilizon.Events.{Comment, Event, Participant, Session, Tag, TagRelation, Track}
alias Mobilizon.Storage.Page
alias Mobilizon.Service.Search
alias Mobilizon.Service.Workers.BuildSearchWorker
@event_valid_attrs %{
begins_on: "2010-04-17 14:00:00Z",
@ -23,7 +23,7 @@ defmodule Mobilizon.EventsTest do
setup do
actor = insert(:actor)
event = insert(:event, organizer_actor: actor, visibility: :public)
Search.insert_search_event(event)
BuildSearchWorker.insert_search_event(event)
{:ok, actor: actor, event: event}
end
@ -63,7 +63,7 @@ defmodule Mobilizon.EventsTest do
assert title == hd(Events.build_events_for_search(event.title).elements).title
%Event{} = event2 = insert(:event, title: "Special event")
Search.insert_search_event(event2)
BuildSearchWorker.insert_search_event(event2)
assert event2.title ==
Events.build_events_for_search("Special").elements |> hd() |> Map.get(:title)
@ -76,7 +76,7 @@ defmodule Mobilizon.EventsTest do
tag1 = insert(:tag, title: "coucou")
tag2 = insert(:tag, title: "hola")
%Event{} = event3 = insert(:event, title: "Nothing like it", tags: [tag1, tag2])
Search.insert_search_event(event3)
BuildSearchWorker.insert_search_event(event3)
assert event3.title ==
Events.build_events_for_search("hola").elements |> hd() |> Map.get(:title)

View File

@ -1,8 +1,10 @@
defmodule MobilizonWeb.Resolvers.EventResolverTest do
use MobilizonWeb.ConnCase
use Bamboo.Test
use Oban.Testing, repo: Mobilizon.Storage.Repo
alias Mobilizon.Events
alias MobilizonWeb.{AbsintheHelpers, Email}
alias Mobilizon.Service.Workers.BuildSearchWorker
import Mobilizon.Factory
@event %{
@ -105,6 +107,7 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
organizer_actor_id: "#{actor.id}",
category: "birthday"
) {
id,
title,
uuid
}
@ -117,6 +120,8 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
|> post("/api", AbsintheHelpers.mutation_skeleton(mutation))
assert json_response(res, 200)["data"]["createEvent"]["title"] == "come to my event"
{id, ""} = json_response(res, 200)["data"]["createEvent"]["id"] |> Integer.parse()
assert_enqueued(worker: BuildSearchWorker, args: %{event_id: id, op: :insert_search_event})
end
test "create_event/3 creates an event and escapes title and description", %{
@ -132,6 +137,7 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
begins_on: $begins_on,
organizer_actor_id: $organizer_actor_id
) {
id,
title,
description,
uuid
@ -159,6 +165,9 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
assert res["data"]["createEvent"]["description"] ==
"<b>My description</b> <img src=\"http://placekitten.com/g/200/300\" />"
{id, ""} = res["data"]["createEvent"]["id"] |> Integer.parse()
assert_enqueued(worker: BuildSearchWorker, args: %{event_id: id, op: :insert_search_event})
end
test "create_event/3 creates an event as a draft", %{conn: conn, actor: actor, user: user} do
@ -192,6 +201,12 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
event_uuid = json_response(res, 200)["data"]["createEvent"]["uuid"]
event_id = json_response(res, 200)["data"]["createEvent"]["id"]
{event_id_int, ""} = Integer.parse(event_id)
refute_enqueued(
worker: BuildSearchWorker,
args: %{event_id: event_id_int, op: :insert_search_event}
)
query = """
{
@ -275,6 +290,7 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
showEndTime: false
}
) {
id,
title,
description,
begins_on,
@ -318,6 +334,12 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
assert event["options"]["maximumAttendeeCapacity"] == 30
assert event["options"]["showRemainingAttendeeCapacity"] == true
assert event["options"]["showEndTime"] == false
{event_id_int, ""} = Integer.parse(event["id"])
assert_enqueued(
worker: BuildSearchWorker,
args: %{event_id: event_id_int, op: :insert_search_event}
)
end
test "create_event/3 creates an event with tags", %{conn: conn, actor: actor, user: user} do
@ -675,6 +697,7 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
locality: "#{address.locality}"
}
) {
id,
uuid,
url,
title,
@ -734,6 +757,13 @@ defmodule MobilizonWeb.Resolvers.EventResolverTest do
%{"slug" => "tag2-updated", "title" => "tag2_updated"}
]
{event_id_int, ""} = Integer.parse(event_res["id"])
assert_enqueued(
worker: BuildSearchWorker,
args: %{event_id: event_id_int, op: :update_search_event}
)
{:ok, new_event} = Mobilizon.Events.get_event(event.id)
assert_delivered_email(

View File

@ -474,7 +474,7 @@ defmodule MobilizonWeb.Resolvers.ParticipantResolverTest do
query = """
{
event(uuid: "#{event.uuid}") {
participants(page: 1, limit: 1, roles: "participant,moderator,administrator,creator", actorId: "#{
participants(page: 2, limit: 1, roles: "participant,moderator,administrator,creator", actorId: "#{
actor.id
}") {
role,
@ -493,11 +493,7 @@ defmodule MobilizonWeb.Resolvers.ParticipantResolverTest do
sorted_participants =
json_response(res, 200)["data"]["event"]["participants"]
|> Enum.sort_by(
&(&1
|> Map.get("actor")
|> Map.get("preferredUsername"))
)
|> Enum.filter(&(&1["role"] == "PARTICIPANT"))
assert sorted_participants == [
%{
@ -511,7 +507,7 @@ defmodule MobilizonWeb.Resolvers.ParticipantResolverTest do
query = """
{
event(uuid: "#{event.uuid}") {
participants(page: 2, limit: 1, roles: "participant,moderator,administrator,creator", actorId: "#{
participants(page: 1, limit: 1, roles: "participant,moderator,administrator,creator", actorId: "#{
actor.id
}") {
role,
@ -594,10 +590,12 @@ defmodule MobilizonWeb.Resolvers.ParticipantResolverTest do
actor_id: not_approved.id
})
rejected = insert(:actor)
Events.create_participant(%{
role: :rejected,
event_id: event.id,
actor_id: not_approved.id
actor_id: rejected.id
})
query = """

View File

@ -2,7 +2,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
use MobilizonWeb.ConnCase
alias MobilizonWeb.AbsintheHelpers
import Mobilizon.Factory
alias Mobilizon.Service.Search
alias Mobilizon.Service.Workers.BuildSearchWorker
setup %{conn: conn} do
user = insert(:user)
@ -17,7 +17,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
insert(:actor, user: user, preferred_username: "test_person")
insert(:actor, type: :Group, preferred_username: "test_group")
event = insert(:event, title: "test_event")
Search.insert_search_event(event)
BuildSearchWorker.insert_search_event(event)
query = """
{
@ -51,7 +51,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
actor = insert(:actor, user: user, preferred_username: "test_person")
insert(:actor, type: :Group, preferred_username: "test_group")
event = insert(:event, title: "test_event")
Search.insert_search_event(event)
BuildSearchWorker.insert_search_event(event)
query = """
{
@ -84,7 +84,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
insert(:actor, user: user, preferred_username: "test_person")
group = insert(:actor, type: :Group, preferred_username: "test_group")
event = insert(:event, title: "test_event")
Search.insert_search_event(event)
BuildSearchWorker.insert_search_event(event)
query = """
{
@ -118,9 +118,9 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
event1 = insert(:event, title: "Pineapple fashion week")
event2 = insert(:event, title: "I love pineAPPLE")
event3 = insert(:event, title: "Hello")
Search.insert_search_event(event1)
Search.insert_search_event(event2)
Search.insert_search_event(event3)
BuildSearchWorker.insert_search_event(event1)
BuildSearchWorker.insert_search_event(event2)
BuildSearchWorker.insert_search_event(event3)
query = """
{
@ -161,9 +161,9 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
event1 = insert(:event, title: "Pineapple fashion week")
event2 = insert(:event, title: "I love pineAPPLE")
event3 = insert(:event, title: "Hello")
Search.insert_search_event(event1)
Search.insert_search_event(event2)
Search.insert_search_event(event3)
BuildSearchWorker.insert_search_event(event1)
BuildSearchWorker.insert_search_event(event2)
BuildSearchWorker.insert_search_event(event3)
query = """
{
@ -198,7 +198,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
insert(:actor, user: user, preferred_username: "person", name: "Torréfaction du Kafé")
insert(:actor, type: :Group, preferred_username: "group", name: "Kafé group")
event = insert(:event, title: "Tour du monde des Kafés")
Search.insert_search_event(event)
BuildSearchWorker.insert_search_event(event)
# Elaborate query
query = """
@ -230,7 +230,7 @@ defmodule MobilizonWeb.Resolvers.SearchResolverTest do
insert(:actor, user: user, preferred_username: "person", name: "Torréfaction du Kafé")
group = insert(:actor, type: :Group, preferred_username: "group", name: "Kafé group")
event = insert(:event, title: "Tour du monde des Kafés")
Search.insert_search_event(event)
BuildSearchWorker.insert_search_event(event)
# Elaborate query
query = """