Save remote profiles avatars & banners locally

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

CHANGELOG.md

@@ -20,6 +20,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   * Docker
     `docker-compose exec mobilizon mobilizon_ctl maintenance.fix_unattached_media_in_body`
 
+* **Refresh remote profiles to save avatars locally**
+  Profile avatars and banners were previously only proxified and cached. Now we save them locally. Refreshing all remote actors will save profile media locally instead.
+
+  * Source install
+    `MIX_ENV=prod mix mobilizon.actors.refresh --all`
+  * Docker
+    `docker-compose exec mobilizon mobilizon_ctl actors.refresh --all`
+
 ### Added
 
 - **Add a command to clean orphan media files**. There's a `--dry-run` option to see what files would have been deleted.  

config/config.exs

@@ -81,17 +81,6 @@ config :mobilizon, Mobilizon.Web.Upload,
 
 config :mobilizon, Mobilizon.Web.Upload.Uploader.Local, uploads: "uploads"
 
-config :mobilizon, :media_proxy,
-  enabled: true,
-  proxy_opts: [
-    redirect_on_failure: false,
-    max_body_length: 25 * 1_048_576,
-    http: [
-      follow_redirect: true,
-      pool: :media
-    ]
-  ]
-
 config :mobilizon, Mobilizon.Web.Email.Mailer,
   adapter: Bamboo.SMTPAdapter,
   server: "localhost",

lib/federation/activity_pub/activity_pub.ex

@@ -766,7 +766,10 @@ defmodule Mobilizon.Federation.ActivityPub do
 
     res =
       with {:ok, %{status: 200, body: body}} <-
-             Tesla.get(url, headers: [{"Accept", "application/activity+json"}]),
+             Tesla.get(url,
+               headers: [{"Accept", "application/activity+json"}],
+               follow_redirect: true
+             ),
            :ok <- Logger.debug("response okay, now decoding json"),
            {:ok, data} <- Jason.decode(body) do
         Logger.debug("Got activity+json response at actor's endpoint, now converting data")

lib/federation/activity_pub/transmogrifier.ex

@@ -382,7 +382,7 @@ defmodule Mobilizon.Federation.ActivityPub.Transmogrifier do
       {:ok, activity, new_actor}
     else
       e ->
-        Logger.debug(inspect(e))
+        Logger.error(inspect(e))
         :error
     end
   end

lib/federation/activity_stream/converter/actor.ex

@@ -11,7 +11,9 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
   alias Mobilizon.Federation.ActivityPub.Utils
   alias Mobilizon.Federation.ActivityStream.{Converter, Convertible}
 
-  alias Mobilizon.Web.MediaProxy
+  alias Mobilizon.Service.HTTP.RemoteMediaDownloaderClient
+  alias Mobilizon.Service.RichMedia.Parser
+  alias Mobilizon.Web.Upload
 
   @behaviour Converter
 
@@ -30,18 +32,10 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
   @spec as_to_model_data(map()) :: {:ok, map()}
   def as_to_model_data(%{"type" => type} = data) when type in @allowed_types do
     avatar =
-      data["icon"]["url"] &&
+      download_picture(get_in(data, ["icon", "url"]), get_in(data, ["icon", "name"]), "avatar")
-        %{
-          "name" => data["icon"]["name"] || "avatar",
-          "url" => MediaProxy.url(data["icon"]["url"])
-        }
 
     banner =
-      data["image"]["url"] &&
+      download_picture(get_in(data, ["image", "url"]), get_in(data, ["image", "name"]), "banner")
-        %{
-          "name" => data["image"]["name"] || "banner",
-          "url" => MediaProxy.url(data["image"]["url"])
-        }
 
     %{
       url: data["id"],
@@ -140,4 +134,16 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
       })
     end
   end
+
+  @spec download_picture(String.t() | nil, String.t(), String.t()) :: map()
+  defp download_picture(nil, _name, _default_name), do: nil
+
+  defp download_picture(url, name, default_name) do
+    with {:ok, %{body: body, status: code, headers: response_headers}}
+         when code in 200..299 <- RemoteMediaDownloaderClient.get(url),
+         name <- name || Parser.get_filename_from_response(response_headers, url) || default_name,
+         {:ok, file} <- Upload.store(%{body: body, name: name}) do
+      file
+    end
+  end
 end

lib/graphql/resolvers/event.ex

@@ -10,7 +10,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
   alias Mobilizon.Users.User
 
   alias Mobilizon.GraphQL.API
-  alias Mobilizon.GraphQL.Resolvers.Person
 
   alias Mobilizon.Federation.ActivityPub.Activity
   import Mobilizon.Web.Gettext
@@ -35,7 +34,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
        ) do
     case {:has_event, Events.get_own_event_by_uuid_with_preload(uuid, user_id)} do
       {:has_event, %Event{} = event} ->
-        {:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
+        {:ok, event}
 
       {:has_event, _} ->
         {:error, :event_not_found}
@@ -51,7 +50,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
            {:has_event, Events.get_public_event_by_uuid_with_preload(uuid)},
          {:access_valid, true} <-
            {:access_valid, Map.has_key?(context, :current_user) || check_event_access(event)} do
-      {:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
+      {:ok, event}
     else
       {:has_event, _} ->
         find_private_event(parent, args, resolution)

lib/graphql/resolvers/group.ex

@@ -8,7 +8,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
   alias Mobilizon.Actors.{Actor, Member}
   alias Mobilizon.Federation.ActivityPub
   alias Mobilizon.GraphQL.API
-  alias Mobilizon.GraphQL.Resolvers.Person
   alias Mobilizon.Users.User
   alias Mobilizon.Web.Upload
   import Mobilizon.Web.Gettext
@@ -30,8 +29,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
     with {:ok, %Actor{id: group_id} = group} <-
            ActivityPub.find_or_make_group_from_nickname(name),
          {:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)},
-         {:member, true} <- {:member, Actors.is_member?(actor_id, group_id)},
+         {:member, true} <- {:member, Actors.is_member?(actor_id, group_id)} do
-         group <- Person.proxify_pictures(group) do
       {:ok, group}
     else
       {:member, false} ->
@@ -44,7 +42,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
 
   def find_group(_parent, %{preferred_username: name}, _resolution) do
     with {:ok, actor} <- ActivityPub.find_or_make_group_from_nickname(name),
-         %Actor{} = actor <- Person.proxify_pictures(actor),
          %Actor{} = actor <- restrict_fields_for_non_member_request(actor) do
       {:ok, actor}
     else

lib/graphql/resolvers/participant.ex

@@ -6,7 +6,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Events.{Event, Participant}
   alias Mobilizon.GraphQL.API.Participations
-  alias Mobilizon.GraphQL.Resolvers.Person
   alias Mobilizon.Users.User
   alias Mobilizon.Web.Email
   alias Mobilizon.Web.Email.Checker
@@ -114,7 +113,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
          %Participant{} = participant <-
            participant
            |> Map.put(:event, event)
-           |> Map.put(:actor, Person.proxify_pictures(actor)) do
+           |> Map.put(:actor, actor) do
       {:ok, participant}
     else
       {:maximum_attendee_capacity, _} ->

lib/graphql/resolvers/person.ex

@@ -15,15 +15,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
   alias Mobilizon.Federation.ActivityPub
   require Logger
 
-  alias Mobilizon.Web.{MediaProxy, Upload}
+  alias Mobilizon.Web.Upload
 
   @doc """
   Get a person
   """
   def get_person(_parent, %{id: id}, %{context: %{current_user: %User{role: role}}}) do
     with %Actor{suspended: suspended} = actor <- Actors.get_actor_with_preload(id, true),
-         true <- suspended == false or is_moderator(role),
+         true <- suspended == false or is_moderator(role) do
-         actor <- proxify_pictures(actor) do
       {:ok, actor}
     else
       _ ->
@@ -31,6 +30,8 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
     end
   end
 
+  def get_person(_parent, _args, _resolution), do: {:error, :unauthorized}
+
   @doc """
   Find a person
   """
@@ -39,8 +40,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
       }) do
     with {:ok, %Actor{id: actor_id} = actor} <-
            ActivityPub.find_or_make_actor_from_nickname(preferred_username),
-         {:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)},
+         {:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)} do
-         actor <- proxify_pictures(actor) do
       {:ok, actor}
     else
       {:own, nil} ->
@@ -120,9 +120,12 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
     args = Map.put(args, :user_id, user.id)
 
     with args <- Map.update(args, :preferred_username, "", &String.downcase/1),
-         args <- save_attached_pictures(args),
+         {:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
          {:ok, %Actor{} = new_person} <- Actors.new_person(args) do
       {:ok, new_person}
+    else
+      {:picture, {:error, :file_too_large}} ->
+        {:error, dgettext("errors", "The provided picture is too heavy")}
     end
   end
 
@@ -144,10 +147,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
     with {:find_actor, %Actor{} = actor} <-
            {:find_actor, Actors.get_actor(id)},
          {:is_owned, %Actor{}} <- User.owns_actor(user, actor.id),
-         args <- save_attached_pictures(args),
+         {:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
          {:ok, _activity, %Actor{} = actor} <- ActivityPub.update(actor, args, true) do
       {:ok, actor}
     else
+      {:picture, {:error, :file_too_large}} ->
+        {:error, dgettext("errors", "The provided picture is too heavy")}
+
       {:find_actor, nil} ->
         {:error, dgettext("errors", "Profile not found")}
 
@@ -199,18 +205,27 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
   end
 
   defp save_attached_pictures(args) do
-    Enum.reduce([:avatar, :banner], args, fn key, args ->
+    with args when is_map(args) <- save_attached_picture(args, :avatar),
-      if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
+         args when is_map(args) <- save_attached_picture(args, :banner) do
-        media = args[key][:media]
+      args
+    end
+  end
 
-        with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
+  defp save_attached_picture(args, key) do
-               Upload.store(media.file, type: key, description: media.alt) do
+    if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
-          Map.put(args, key, %{"name" => name, "url" => url, "mediaType" => content_type})
+      with media when is_map(media) <- save_picture(args[key][:media], key) do
-        end
+        Map.put(args, key, media)
-      else
-        args
       end
-    end)
+    else
+      args
+    end
+  end
+
+  defp save_picture(media, key) do
+    with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
+           Upload.store(media.file, type: key, description: media.alt) do
+      %{"name" => name, "url" => url, "mediaType" => content_type}
+    end
   end
 
   @doc """
@@ -223,10 +238,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
          {:no_actor, true} <- {:no_actor, no_actor},
          args <- Map.update(args, :preferred_username, "", &String.downcase/1),
          args <- Map.put(args, :user_id, user.id),
-         args <- save_attached_pictures(args),
+         {:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
          {:ok, %Actor{} = new_person} <- Actors.new_person(args, true) do
       {:ok, new_person}
     else
+      {:picture, {:error, :file_too_large}} ->
+        {:error, dgettext("errors", "The provided picture is too heavy")}
+
       {:error, :user_not_found} ->
         {:error, dgettext("errors", "No user with this email was found")}
 
@@ -298,12 +316,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
     end
   end
 
-  def proxify_pictures(%Actor{} = actor) do
-    actor
-    |> proxify_avatar
-    |> proxify_banner
-  end
-
   def user_for_person(%Actor{type: :Person, user_id: user_id}, _args, %{
         context: %{current_user: %User{role: role}}
       })
@@ -343,20 +355,4 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
   defp last_admin_of_a_group?(actor_id) do
     length(Actors.list_group_ids_where_last_administrator(actor_id)) > 0
   end
-
-  @spec proxify_avatar(Actor.t()) :: Actor.t()
-  defp proxify_avatar(%Actor{avatar: %{url: avatar_url} = avatar} = actor) do
-    actor |> Map.put(:avatar, avatar |> Map.put(:url, MediaProxy.url(avatar_url)))
-  end
-
-  @spec proxify_avatar(Actor.t()) :: Actor.t()
-  defp proxify_avatar(%Actor{} = actor), do: actor
-
-  @spec proxify_banner(Actor.t()) :: Actor.t()
-  defp proxify_banner(%Actor{banner: %{url: banner_url} = banner} = actor) do
-    actor |> Map.put(:banner, banner |> Map.put(:url, MediaProxy.url(banner_url)))
-  end
-
-  @spec proxify_banner(Actor.t()) :: Actor.t()
-  defp proxify_banner(%Actor{} = actor), do: actor
 end

lib/mix/tasks/mobilizon/actors/refresh.ex

@@ -73,6 +73,12 @@ defmodule Mix.Tasks.Mobilizon.Actors.Refresh do
 
       {:actor, nil} ->
         shell_error("Error: No such actor")
+
+      {:error, err} when is_binary(err) ->
+        shell_error(err)
+
+      _err ->
+        shell_error("Error while refreshing actor #{preferred_username}")
     end
   end
 

lib/mobilizon/actors/actors.ex

@@ -13,11 +13,13 @@ defmodule Mobilizon.Actors do
   alias Mobilizon.Actors.{Actor, Bot, Follower, Member}
   alias Mobilizon.Addresses.Address
   alias Mobilizon.{Crypto, Events}
+  alias Mobilizon.Events.FeedToken
   alias Mobilizon.Federation.ActivityPub
   alias Mobilizon.Medias.File
   alias Mobilizon.Service.Workers
   alias Mobilizon.Storage.{Page, Repo}
   alias Mobilizon.Users
+  alias Mobilizon.Users.User
   alias Mobilizon.Web.Email.Group
   alias Mobilizon.Web.Upload
 
@@ -215,18 +217,35 @@ defmodule Mobilizon.Actors do
   def new_person(args, default_actor \\ false) do
     args = Map.put(args, :keys, Crypto.generate_rsa_2048_private_key())
 
-    with {:ok, %Actor{id: person_id} = person} <-
+    multi =
-           %Actor{}
+      Multi.new()
-           |> Actor.registration_changeset(args)
+      |> Multi.insert(:person, Actor.registration_changeset(%Actor{}, args))
-           |> Repo.insert() do
+      |> Multi.insert(:token, fn %{person: person} ->
-      Events.create_feed_token(%{user_id: args.user_id, actor_id: person.id})
+        FeedToken.changeset(%FeedToken{}, %{
+          user_id: args.user_id,
+          actor_id: person.id,
+          token: Ecto.UUID.generate()
+        })
+      end)
 
+    multi =
       if default_actor do
         user = Users.get_user!(args.user_id)
-        Users.update_user(user, %{default_actor_id: person_id})
+
+        Multi.update(multi, :user, fn %{person: person} ->
+          User.changeset(user, %{default_actor_id: person.id})
+        end)
+      else
+        multi
       end
 
-      {:ok, person}
+    case Repo.transaction(multi) do
+      {:ok, %{person: %Actor{} = person}} ->
+        {:ok, person}
+
+      {:error, _step, err, _} ->
+        Logger.error("Error while creating a new person")
+        {:error, err}
     end
   end
 

lib/service/export/feed.ex

@@ -13,7 +13,7 @@ defmodule Mobilizon.Service.Export.Feed do
   alias Mobilizon.Storage.Page
   alias Mobilizon.Users.User
 
-  alias Mobilizon.Web.{Endpoint, MediaProxy}
+  alias Mobilizon.Web.Endpoint
   alias Mobilizon.Web.Router.Helpers, as: Routes
 
   require Logger
@@ -85,14 +85,14 @@ defmodule Mobilizon.Service.Export.Feed do
 
     feed =
       if actor.avatar do
-        feed |> Feed.icon(actor.avatar.url |> MediaProxy.url())
+        feed |> Feed.icon(actor.avatar.url)
       else
         feed
       end
 
     feed =
       if actor.banner do
-        feed |> Feed.logo(actor.banner.url |> MediaProxy.url())
+        feed |> Feed.logo(actor.banner.url)
       else
         feed
       end

lib/service/http/remote_media_downloader_client.ex

@@ -0,0 +1,22 @@
+defmodule Mobilizon.Service.HTTP.RemoteMediaDownloaderClient do
+  @moduledoc """
+  Tesla HTTP Basic Client that fetches HTML to extract metadata preview
+  """
+
+  use Tesla
+  alias Mobilizon.Config
+
+  @default_opts [
+    recv_timeout: 20_000
+  ]
+
+  adapter(Tesla.Adapter.Hackney, @default_opts)
+
+  @user_agent Config.instance_user_agent()
+
+  plug(Tesla.Middleware.FollowRedirects)
+
+  plug(Tesla.Middleware.Timeout, timeout: 10_000)
+
+  plug(Tesla.Middleware.Headers, [{"User-Agent", @user_agent}])
+end

lib/service/metadata/actor.ex

@@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
   alias Phoenix.HTML.Tag
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Web.JsonLD.ObjectView
-  alias Mobilizon.Web.MediaProxy
   import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, default_description: 1]
 
   def build_tags(_actor, _locale \\ "en")
@@ -36,7 +35,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
       tags
     else
       tags ++
-        [Tag.tag(:meta, property: "og:image", content: actor.avatar.url |> MediaProxy.url())]
+        [Tag.tag(:meta, property: "og:image", content: actor.avatar.url)]
     end
   end
 

lib/service/metadata/event.ex

@@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
   alias Phoenix.HTML.Tag
   alias Mobilizon.Events.Event
   alias Mobilizon.Web.JsonLD.ObjectView
-  alias Mobilizon.Web.MediaProxy
   import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, strip_tags: 1]
 
   def build_tags(%Event{} = event, locale \\ "en") do
@@ -28,7 +27,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
           [
             Tag.tag(:meta,
               property: "og:image",
-              content: event.picture.file.url |> MediaProxy.url()
+              content: event.picture.file.url
             )
           ]
       end

lib/service/rich_media/parser.ex

@@ -47,6 +47,14 @@ defmodule Mobilizon.Service.RichMedia.Parser do
       {:error, "Cachex error: #{inspect(e)}"}
   end
 
+  @doc """
+  Get a filename for the fetched data, using the response header or the last part of the URL
+  """
+  @spec get_filename_from_response(Enum.t(), String.t()) :: String.t() | nil
+  def get_filename_from_response(response_headers, url) do
+    get_filename_from_headers(response_headers) || get_filename_from_url(url)
+  end
+
   @spec parse_url(String.t(), Enum.t()) :: {:ok, map()} | {:error, any()}
   defp parse_url(url, options \\ []) do
     user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())

lib/web/controllers/media_proxy_controller.ex

@@ -1,50 +0,0 @@
-# Portions of this file are derived from Pleroma:
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/controller.ex
-
-defmodule Mobilizon.Web.MediaProxyController do
-  use Mobilizon.Web, :controller
-
-  alias Plug.Conn
-
-  alias Mobilizon.Config
-
-  alias Mobilizon.Web.MediaProxy
-  alias Mobilizon.Web.ReverseProxy
-
-  @default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
-
-  def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
-    with config <- Config.get([:media_proxy], []),
-         true <- Keyword.get(config, :enabled, false),
-         {:ok, url} <- MediaProxy.decode_url(sig64, url64),
-         :ok <- filename_matches(Map.has_key?(params, "filename"), conn.request_path, url) do
-      ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
-    else
-      false ->
-        send_resp(conn, 404, Conn.Status.reason_phrase(404))
-
-      {:error, :invalid_signature} ->
-        send_resp(conn, 403, Conn.Status.reason_phrase(403))
-
-      {:wrong_filename, filename} ->
-        redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
-    end
-  end
-
-  def filename_matches(has_filename, path, url) do
-    filename =
-      url
-      |> MediaProxy.filename()
-      |> URI.decode()
-
-    path = URI.decode(path)
-
-    if has_filename && filename && Path.basename(path) != filename do
-      {:wrong_filename, filename}
-    else
-      :ok
-    end
-  end
-end

lib/web/proxy/media_proxy.ex

@@ -1,91 +0,0 @@
-# Portions of this file are derived from Pleroma:
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/media_proxy.ex
-
-defmodule Mobilizon.Web.MediaProxy do
-  @moduledoc """
-  Handles proxifying media files
-  """
-
-  alias Mobilizon.Config
-
-  alias Mobilizon.Web.Endpoint
-
-  @base64_opts [padding: false]
-
-  def url(nil), do: nil
-
-  def url(""), do: nil
-
-  def url("/" <> _ = url), do: url
-
-  def url(url) do
-    config = Application.get_env(:mobilizon, :media_proxy, [])
-
-    if !Keyword.get(config, :enabled, false) or
-         String.starts_with?(url, Endpoint.url()) do
-      url
-    else
-      encode_url(url)
-    end
-  end
-
-  def encode_url(url) do
-    secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
-
-    # Must preserve `%2F` for compatibility with S3
-    # https://git.pleroma.social/pleroma/pleroma/issues/580
-    replacement = get_replacement(url, ":2F:")
-
-    # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
-    base64 =
-      url
-      |> String.replace("%2F", replacement)
-      |> URI.decode()
-      |> URI.encode()
-      |> String.replace(replacement, "%2F")
-      |> Base.url_encode64(@base64_opts)
-
-    sig = :crypto.hmac(:sha, secret, base64)
-    sig64 = sig |> Base.url_encode64(@base64_opts)
-
-    build_url(sig64, base64, filename(url))
-  end
-
-  def decode_url(sig, url) do
-    secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
-    sig = Base.url_decode64!(sig, @base64_opts)
-    local_sig = :crypto.hmac(:sha, secret, url)
-
-    if local_sig == sig do
-      {:ok, Base.url_decode64!(url, @base64_opts)}
-    else
-      {:error, :invalid_signature}
-    end
-  end
-
-  def filename(url_or_path) do
-    if path = URI.parse(url_or_path).path, do: Path.basename(path)
-  end
-
-  def build_url(sig_base64, url_base64, filename \\ nil) do
-    [
-      Config.get([:media_proxy, :base_url], Endpoint.url()),
-      "proxy",
-      sig_base64,
-      url_base64,
-      filename
-    ]
-    |> Enum.filter(fn value -> value end)
-    |> Path.join()
-  end
-
-  defp get_replacement(url, replacement) do
-    if String.contains?(url, replacement) do
-      get_replacement(url, replacement <> replacement)
-    else
-      replacement
-    end
-  end
-end

lib/web/proxy/reverse_proxy.ex

@@ -69,8 +69,6 @@ defmodule Mobilizon.Web.ReverseProxy do
 
   alias Plug.Conn
 
-  alias Mobilizon.Web.MediaProxy
-
   require Logger
 
   @type option ::
@@ -111,7 +109,7 @@ defmodule Mobilizon.Web.ReverseProxy do
     req_headers = build_req_headers(conn.req_headers, opts)
 
     opts =
-      if filename = MediaProxy.filename(url) do
+      if filename = filename(url) do
         Keyword.put_new(opts, :attachment_name, filename)
       else
         opts
@@ -388,4 +386,8 @@ defmodule Mobilizon.Web.ReverseProxy do
   defp increase_read_duration(_) do
     {:ok, :no_duration_limit, :no_duration_limit}
   end
+
+  def filename(url_or_path) do
+    if path = URI.parse(url_or_path).path, do: Path.basename(path)
+  end
 end

lib/web/router.ex

@@ -162,13 +162,6 @@ defmodule Mobilizon.Web.Router do
     post("/auth/:provider/callback", AuthController, :callback)
   end
 
-  scope "/proxy/", Mobilizon.Web do
-    pipe_through(:remote_media)
-
-    get("/:sig/:url", MediaProxyController, :remote)
-    get("/:sig/:url/:filename", MediaProxyController, :remote)
-  end
-
   if Application.fetch_env!(:mobilizon, :env) in [:dev, :e2e] do
     # If using Phoenix
     forward("/sent_emails", Bamboo.SentEmailViewerPlug)

lib/web/views/json_ld/object_view.ex

@@ -5,7 +5,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
   alias Mobilizon.Addresses.Address
   alias Mobilizon.Events.Event
   alias Mobilizon.Posts.Post
-  alias Mobilizon.Web.{Endpoint, MediaProxy}
+  alias Mobilizon.Web.Endpoint
   alias Mobilizon.Web.JsonLD.ObjectView
 
   def render("group.json", %{group: %Actor{} = group}) do
@@ -41,7 +41,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
       "image" =>
         if(event.picture,
           do: [
-            event.picture.file.url |> MediaProxy.url()
+            event.picture.file.url
           ],
           else: ["#{Endpoint.url()}/img/mobilizon_default_card.png"]
         )

mix.exs

@@ -264,7 +264,6 @@ defmodule Mobilizon.Mixfile do
         Mobilizon.Web.Plugs.UploadedMedia,
         Mobilizon.Web.FallbackController,
         Mobilizon.Web.FeedController,
-        Mobilizon.Web.MediaProxyController,
         Mobilizon.Web.PageController,
         Mobilizon.Web.ChangesetView,
         Mobilizon.Web.JsonLD.ObjectView,
@@ -295,7 +294,6 @@ defmodule Mobilizon.Mixfile do
         Mobilizon.Web.Upload.MIME,
         Mobilizon.Web.Upload.Uploader,
         Mobilizon.Web.Upload.Uploader.Local,
-        Mobilizon.Web.MediaProxy,
         Mobilizon.Web.ReverseProxy
       ],
       Geospatial: [

test/federation/activity_pub/activity_pub_test.exs

@@ -45,13 +45,13 @@ defmodule Mobilizon.Federation.ActivityPubTest do
       use_cassette "activity_pub/fetch_tcit@framapiaf.org" do
         assert {:ok,
                 %Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :public} =
-                  actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
+                  _actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
       end
 
       use_cassette "activity_pub/fetch_tcit@framapiaf.org_not_discoverable" do
         assert {:ok,
                 %Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :unlisted} =
-                  actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
+                  _actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
       end
     end
 

test/federation/activity_pub/transmogrifier_test.exs

@@ -39,7 +39,7 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
           Transmogrifier.handle_incoming(data)
 
         assert data["id"] ==
-                 "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0/activity"
+                 "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93/activity"
 
         assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
 
@@ -49,12 +49,12 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
         #               "http://localtesting.pleroma.lol/users/lain"
         #             ]
 
-        assert data["actor"] == "https://test.mobilizon.org/@Alicia"
+        assert data["actor"] == "https://mobilizon.fr/@metacartes"
 
         object = data["object"]
 
         assert object["id"] ==
-                 "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0"
+                 "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93"
 
         assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
 
@@ -63,9 +63,9 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
         #               "http://localtesting.pleroma.lol/users/lain"
         #             ]
 
-        assert object["actor"] == "https://test.mobilizon.org/@Alicia"
+        assert object["actor"] == "https://mobilizon.fr/@metacartes"
         assert object["location"]["name"] == "Locaux de Framasoft"
-        # assert object["attributedTo"] == "https://test.mobilizon.org/@Alicia"
+        # assert object["attributedTo"] == "https://mobilizon.fr/@metacartes"
 
         assert event.physical_address.street == "10 Rue Jangot"
 
@@ -84,8 +84,8 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
       %Actor{url: actor_url, id: actor_id} =
         actor =
         insert(:actor,
-          domain: "test.mobilizon.org",
+          domain: "mobilizon.fr",
-          url: "https://test.mobilizon.org/@member",
+          url: "https://mobilizon.fr/@member",
           preferred_username: "member"
         )
 

test/fixtures/mastodon-update.json

@@ -20,15 +20,15 @@
     "endpoints": {
       "sharedInbox": "https://framapiaf.org/inbox"
     },
-    "icon":{
+    "icon": {
-      "type":"Image",
+      "type": "Image",
-      "mediaType":"image/png",
+      "mediaType": "image/png",
-      "url":"https://files.mastodon.social/accounts/avatars/000/000/001/original/a285c086605e4182.png"
+      "url": "https://files.mastodon.social/accounts/avatars/000/000/001/original/d96d39a0abb45b92.jpg"
     },
-    "image":{
+    "image": {
-      "type":"Image",
+      "type": "Image",
-      "mediaType":"image/png",
+      "mediaType": "image/png",
-      "url":"https://files.mastodon.social/accounts/headers/000/000/001/original/c91b871f294ea63e.png"
+      "url": "https://files.mastodon.social/accounts/headers/000/000/001/original/c91b871f294ea63e.png"
     }
   },
   "id": "https://framapiaf.org/users/gargron#updates/1519563538",

test/fixtures/mobilizon-post-activity.json

@@ -1,96 +1,92 @@
 {
-    "@context": [
+  "@context": [
-        "https://www.w3.org/ns/activitystreams",
+    "https://www.w3.org/ns/activitystreams",
-        "https://litepub.social/litepub/context.jsonld",
+    "https://litepub.social/litepub/context.jsonld",
-        {
+    {
-            "Hashtag": "as:Hashtag",
+      "Hashtag": "as:Hashtag",
-            "category": "sc:category",
+      "category": "sc:category",
-            "ical": "http://www.w3.org/2002/12/cal/ical#",
+      "ical": "http://www.w3.org/2002/12/cal/ical#",
-            "joinMode": {
+      "joinMode": {
-                "@id": "mz:joinMode",
+        "@id": "mz:joinMode",
-                "@type": "mz:joinModeType"
+        "@type": "mz:joinModeType"
-            },
+      },
-            "joinModeType": {
+      "joinModeType": {
-                "@id": "mz:joinModeType",
+        "@id": "mz:joinModeType",
-                "@type": "rdfs:Class"
+        "@type": "rdfs:Class"
-            },
+      },
-            "maximumAttendeeCapacity": "sc:maximumAttendeeCapacity",
+      "maximumAttendeeCapacity": "sc:maximumAttendeeCapacity",
-            "mz": "https://joinmobilizon.org/ns#",
+      "mz": "https://joinmobilizon.org/ns#",
-            "repliesModerationOption": {
+      "repliesModerationOption": {
-                "@id": "mz:repliesModerationOption",
+        "@id": "mz:repliesModerationOption",
-                "@type": "mz:repliesModerationOptionType"
+        "@type": "mz:repliesModerationOptionType"
-            },
+      },
-            "repliesModerationOptionType": {
+      "repliesModerationOptionType": {
-                "@id": "mz:repliesModerationOptionType",
+        "@id": "mz:repliesModerationOptionType",
-                "@type": "rdfs:Class"
+        "@type": "rdfs:Class"
-            },
+      },
-            "sc": "http://schema.org#",
+      "sc": "http://schema.org#",
-            "uuid": "sc:identifier"
+      "uuid": "sc:identifier"
-        }
+    }
+  ],
+  "actor": "https://mobilizon.fr/@metacartes",
+  "cc": [
+    "https://framapiaf.org/users/admin/followers",
+    "https://framapiaf.org/users/tcit"
+  ],
+  "id": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93/activity",
+  "object": {
+    "attachment": [
+      {
+        "href": "https://something.org",
+        "mediaType": "text/html",
+        "name": "Another link",
+        "type": "Link"
+      },
+      {
+        "href": "https://google.com",
+        "mediaType": "text/html",
+        "name": "Website",
+        "type": "Link"
+      }
     ],
-    "actor": "https://test.mobilizon.org/@Alicia",
+    "attributedTo": "https://mobilizon.fr/@metacartes",
+    "startTime": "2018-02-12T14:08:20Z",
     "cc": [
-        "https://framapiaf.org/users/admin/followers",
+      "https://framapiaf.org/users/admin/followers",
-        "https://framapiaf.org/users/tcit"
+      "https://framapiaf.org/users/tcit"
     ],
-    "id": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0/activity",
+    "content": "<p><span class=\"h-card\"><a href=\"https://framapiaf.org/users/tcit\" class=\"u-url mention\">@<span>tcit</span></a></span></p>",
-    "object": {
+    "category": "TODO remove me",
-        "attachment": [
+    "id": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93",
-            {
+    "inReplyTo": null,
-                "href": "https://something.org",
+    "location": {
-                "mediaType": "text/html",
+      "type": "Place",
-                "name": "Another link",
+      "name": "Locaux de Framasoft",
-                "type": "Link"
+      "id": "https://event1.tcit.fr/address/eeecc11d-0030-43e8-a897-6422876372jd",
-            },
+      "address": {
-            {
+        "type": "PostalAddress",
-                "href": "https://google.com",
+        "streetAddress": "10 Rue Jangot",
-                "mediaType": "text/html",
+        "postalCode": "69007",
-                "name": "Website",
+        "addressLocality": "Lyon",
-                "type": "Link"
+        "addressRegion": "Auvergne Rhône Alpes",
-            }
+        "addressCountry": "France"
-        ],
+      }
-        "attributedTo": "https://test.mobilizon.org/@Alicia",
-        "startTime": "2018-02-12T14:08:20Z",
-        "cc": [
-            "https://framapiaf.org/users/admin/followers",
-            "https://framapiaf.org/users/tcit"
-        ],
-        "content": "<p><span class=\"h-card\"><a href=\"https://framapiaf.org/users/tcit\" class=\"u-url mention\">@<span>tcit</span></a></span></p>",
-        "category": "TODO remove me",
-        "id": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0",
-        "inReplyTo": null,
-        "location": {
-            "type": "Place",
-            "name": "Locaux de Framasoft",
-            "id": "https://event1.tcit.fr/address/eeecc11d-0030-43e8-a897-6422876372jd",
-            "address": {
-                "type": "PostalAddress",
-                "streetAddress": "10 Rue Jangot",
-                "postalCode": "69007",
-                "addressLocality": "Lyon",
-                "addressRegion": "Auvergne Rhône Alpes",
-                "addressCountry": "France"
-            }
-        },
-        "name": "My first event",
-        "published": "2018-02-12T14:08:20Z",
-        "tag": [
-            {
-                "href": "https://framapiaf.org/users/tcit",
-                "name": "@tcit@framapiaf.org",
-                "type": "Mention"
-            }
-        ],
-        "to": [
-            "https://www.w3.org/ns/activitystreams#Public"
-        ],
-        "type": "Event",
-        "url": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0",
-        "uuid": "109ccdfd-ee3e-46e1-a877-6c228763df0c"
     },
+    "name": "My first event",
     "published": "2018-02-12T14:08:20Z",
-    "to": [
+    "tag": [
-        "https://www.w3.org/ns/activitystreams#Public"
+      {
+        "href": "https://framapiaf.org/users/tcit",
+        "name": "@tcit@framapiaf.org",
+        "type": "Mention"
+      }
     ],
-    "type": "Create"
+    "to": ["https://www.w3.org/ns/activitystreams#Public"],
+    "type": "Event",
+    "url": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93",
+    "uuid": "109ccdfd-ee3e-46e1-a877-6c228763df0c"
+  },
+  "published": "2018-02-12T14:08:20Z",
+  "to": ["https://www.w3.org/ns/activitystreams#Public"],
+  "type": "Create"
 }

test/mobilizon/actors/actors_test.exs

@@ -108,7 +108,7 @@ defmodule Mobilizon.ActorsTest do
            avatar: %FileModel{name: picture_name} = _picture
          } = _actor} = ActivityPub.get_or_fetch_actor_by_url(@remote_account_url)
 
-        assert picture_name == "avatar"
+        assert picture_name == "a28c50ce5f2b13fd.jpg"
 
         %Actor{
           id: actor_found_id,
@@ -116,7 +116,7 @@ defmodule Mobilizon.ActorsTest do
         } = Actors.get_actor_by_name("#{preferred_username}@#{domain}")
 
         assert actor_found_id == actor_id
-        assert picture_name == "avatar"
+        assert picture_name == "a28c50ce5f2b13fd.jpg"
       end
     end
 

test/web/plugs/mapped_identity_to_signature_test.exs

@@ -52,7 +52,7 @@ defmodule Mobilizon.Web.Plugs.MappedSignatureToIdentityTest do
     use_cassette "activity_pub/signature/invalid_not_found" do
       conn =
         build_conn(:post, "/doesntmattter", %{"actor" => "https://framapiaf.org/users/admin"})
-        |> set_signature("http://niu.moe/users/rye")
+        |> set_signature("https://mastodon.social/users/gargron")
         |> MappedSignatureToIdentity.call(%{})
 
       assert %{valid_signature: false} == conn.assigns

test/web/proxy/media_proxy_test.exs

@@ -1,185 +0,0 @@
-# Portions of this file are derived from Pleroma:
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/test/media_proxy_test.ex
-
-defmodule Mobilizon.Web.MediaProxyTest do
-  use ExUnit.Case
-
-  import Mobilizon.Web.MediaProxy
-
-  alias Mobilizon.Config
-
-  alias Mobilizon.Web.{Endpoint, MediaProxyController}
-
-  setup do
-    enabled = Config.get([:media_proxy, :enabled])
-    on_exit(fn -> Config.put([:media_proxy, :enabled], enabled) end)
-    :ok
-  end
-
-  describe "when enabled" do
-    setup do
-      Config.put([:media_proxy, :enabled], true)
-      :ok
-    end
-
-    test "ignores invalid url" do
-      assert url(nil) == nil
-      assert url("") == nil
-    end
-
-    test "ignores relative url" do
-      assert url("/local") == "/local"
-      assert url("/") == "/"
-    end
-
-    test "ignores local url" do
-      local_url = Endpoint.url() <> "/hello"
-      local_root = Endpoint.url()
-      assert url(local_url) == local_url
-      assert url(local_root) == local_root
-    end
-
-    test "encodes and decodes URL" do
-      url = "https://pleroma.soykaf.com/static/logo.png"
-      encoded = url(url)
-
-      assert String.starts_with?(
-               encoded,
-               Config.get([:media_proxy, :base_url], Endpoint.url())
-             )
-
-      assert String.ends_with?(encoded, "/logo.png")
-
-      assert decode_result(encoded) == url
-    end
-
-    test "encodes and decodes URL without a path" do
-      url = "https://pleroma.soykaf.com"
-      encoded = url(url)
-      assert decode_result(encoded) == url
-    end
-
-    test "encodes and decodes URL without an extension" do
-      url = "https://pleroma.soykaf.com/path/"
-      encoded = url(url)
-      assert String.ends_with?(encoded, "/path")
-      assert decode_result(encoded) == url
-    end
-
-    test "encodes and decodes URL and ignores query params for the path" do
-      url = "https://pleroma.soykaf.com/static/logo.png?93939393939&bunny=true"
-      encoded = url(url)
-      assert String.ends_with?(encoded, "/logo.png")
-      assert decode_result(encoded) == url
-    end
-
-    test "ensures urls are url-encoded" do
-      assert decode_result(url("https://pleroma.social/Hello world.jpg")) ==
-               "https://pleroma.social/Hello%20world.jpg"
-
-      assert decode_result(url("https://pleroma.social/Hello%20world.jpg")) ==
-               "https://pleroma.social/Hello%20world.jpg"
-    end
-
-    test "validates signature" do
-      secret_key_base = Config.get([Endpoint, :secret_key_base])
-
-      on_exit(fn ->
-        Config.put([Endpoint, :secret_key_base], secret_key_base)
-      end)
-
-      encoded = url("https://pleroma.social")
-
-      Config.put(
-        [Endpoint, :secret_key_base],
-        "00000000000000000000000000000000000000000000000"
-      )
-
-      [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
-      assert decode_url(sig, base64) == {:error, :invalid_signature}
-    end
-
-    test "filename_matches matches url encoded paths" do
-      assert MediaProxyController.filename_matches(
-               true,
-               "/Hello%20world.jpg",
-               "http://pleroma.social/Hello world.jpg"
-             ) == :ok
-
-      assert MediaProxyController.filename_matches(
-               true,
-               "/Hello%20world.jpg",
-               "http://pleroma.social/Hello%20world.jpg"
-             ) == :ok
-    end
-
-    test "filename_matches matches non-url encoded paths" do
-      assert MediaProxyController.filename_matches(
-               true,
-               "/Hello world.jpg",
-               "http://pleroma.social/Hello%20world.jpg"
-             ) == :ok
-
-      assert MediaProxyController.filename_matches(
-               true,
-               "/Hello world.jpg",
-               "http://pleroma.social/Hello world.jpg"
-             ) == :ok
-    end
-
-    test "uses the configured base_url" do
-      base_url = Config.get([:media_proxy, :base_url])
-
-      if base_url do
-        on_exit(fn ->
-          Config.put([:media_proxy, :base_url], base_url)
-        end)
-      end
-
-      Config.put([:media_proxy, :base_url], "https://cache.pleroma.social")
-
-      url = "https://pleroma.soykaf.com/static/logo.png"
-      encoded = url(url)
-
-      assert String.starts_with?(encoded, Config.get([:media_proxy, :base_url]))
-    end
-
-    # https://git.pleroma.social/pleroma/pleroma/issues/580
-    test "encoding S3 links (must preserve `%2F`)" do
-      url =
-        "https://s3.amazonaws.com/example/test.png?X-Amz-Credential=your-access-key-id%2F20130721%2Fus-east-1%2Fs3%2Faws4_request"
-
-      encoded = url(url)
-      assert decode_result(encoded) == url
-    end
-  end
-
-  describe "when disabled" do
-    setup do
-      enabled = Config.get([:media_proxy, :enabled])
-
-      if enabled do
-        Config.put([:media_proxy, :enabled], false)
-
-        on_exit(fn ->
-          Config.put([:media_proxy, :enabled], enabled)
-          :ok
-        end)
-      end
-
-      :ok
-    end
-
-    test "does not encode remote urls" do
-      assert url("https://google.fr") == "https://google.fr"
-    end
-  end
-
-  defp decode_result(encoded) do
-    [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
-    {:ok, decoded} = decode_url(sig, base64)
-    decoded
-  end
-end