Merge branch 'save-remote-pics' into 'master'

Save remote pics

See merge request framasoft/mobilizon!763
This commit is contained in:
Thomas Citharel 2020-12-16 14:29:32 +01:00
commit 6833d79611
51 changed files with 1668 additions and 864 deletions

View File

@ -20,6 +20,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Docker
`docker-compose exec mobilizon mobilizon_ctl maintenance.fix_unattached_media_in_body`
* **Refresh remote profiles to save avatars locally**
Profile avatars and banners were previously only proxified and cached. Now we save them locally. Refreshing all remote actors will save profile media locally instead.
* Source install
`MIX_ENV=prod mix mobilizon.actors.refresh --all`
* Docker
`docker-compose exec mobilizon mobilizon_ctl actors.refresh --all`
### Added
- **Add a command to clean orphan media files**. There's a `--dry-run` option to see what files would have been deleted.

View File

@ -81,17 +81,6 @@ config :mobilizon, Mobilizon.Web.Upload,
config :mobilizon, Mobilizon.Web.Upload.Uploader.Local, uploads: "uploads"
config :mobilizon, :media_proxy,
enabled: true,
proxy_opts: [
redirect_on_failure: false,
max_body_length: 25 * 1_048_576,
http: [
follow_redirect: true,
pool: :media
]
]
config :mobilizon, Mobilizon.Web.Email.Mailer,
adapter: Bamboo.SMTPAdapter,
server: "localhost",

View File

@ -109,10 +109,14 @@
</div>
</template>
<script lang="ts">
import { Component, Prop } from "vue-property-decorator";
import { Component, Prop, Watch } from "vue-property-decorator";
import { mixins } from "vue-class-component";
import { FETCH_GROUP } from "@/graphql/group";
import { buildFileFromIMedia, readFileAsync } from "@/utils/image";
import {
buildFileFromIMedia,
buildFileVariable,
readFileAsync,
} from "@/utils/image";
import GroupMixin from "@/mixins/group";
import { PostVisibility } from "@/types/enums";
import { TAGS } from "../../graphql/tags";
@ -206,6 +210,13 @@ export default class EditPost extends mixins(GroupMixin) {
this.pictureFile = await buildFileFromIMedia(this.post.picture);
}
@Watch("post")
async updatePostPicture(oldPost: IPost, newPost: IPost): Promise<void> {
if (oldPost.picture !== newPost.picture) {
this.pictureFile = await buildFileFromIMedia(this.post.picture);
}
}
// eslint-disable-next-line consistent-return
async publish(draft: boolean): Promise<void> {
this.errors = {};
@ -292,19 +303,11 @@ export default class EditPost extends mixins(GroupMixin) {
async buildPicture(): Promise<Record<string, unknown>> {
let obj: { picture?: any } = {};
if (this.pictureFile) {
const pictureObj = {
picture: {
picture: {
name: this.pictureFile.name,
alt: `${this.actualGroup.preferredUsername}'s avatar`,
file: this.pictureFile,
},
},
};
obj = { ...pictureObj };
const pictureObj = buildFileVariable(this.pictureFile, "picture");
obj = { ...obj, ...pictureObj };
}
try {
if (this.post.picture) {
if (this.post.picture && this.pictureFile) {
const oldPictureFile = (await buildFileFromIMedia(
this.post.picture
)) as File;
@ -333,6 +336,7 @@ export default class EditPost extends mixins(GroupMixin) {
const roles = Array.isArray(givenRole) ? givenRole : [givenRole];
return (
this.person &&
this.actualGroup &&
this.person.memberships.elements.some(
({ parent: { id }, role }) =>
id === this.actualGroup.id && roles.includes(role)

View File

@ -766,7 +766,10 @@ defmodule Mobilizon.Federation.ActivityPub do
res =
with {:ok, %{status: 200, body: body}} <-
Tesla.get(url, headers: [{"Accept", "application/activity+json"}]),
Tesla.get(url,
headers: [{"Accept", "application/activity+json"}],
follow_redirect: true
),
:ok <- Logger.debug("response okay, now decoding json"),
{:ok, data} <- Jason.decode(body) do
Logger.debug("Got activity+json response at actor's endpoint, now converting data")

View File

@ -382,7 +382,7 @@ defmodule Mobilizon.Federation.ActivityPub.Transmogrifier do
{:ok, activity, new_actor}
else
e ->
Logger.debug(inspect(e))
Logger.error(inspect(e))
:error
end
end

View File

@ -11,7 +11,9 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
alias Mobilizon.Federation.ActivityPub.Utils
alias Mobilizon.Federation.ActivityStream.{Converter, Convertible}
alias Mobilizon.Web.MediaProxy
alias Mobilizon.Service.HTTP.RemoteMediaDownloaderClient
alias Mobilizon.Service.RichMedia.Parser
alias Mobilizon.Web.Upload
@behaviour Converter
@ -30,18 +32,10 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
@spec as_to_model_data(map()) :: {:ok, map()}
def as_to_model_data(%{"type" => type} = data) when type in @allowed_types do
avatar =
data["icon"]["url"] &&
%{
"name" => data["icon"]["name"] || "avatar",
"url" => MediaProxy.url(data["icon"]["url"])
}
download_picture(get_in(data, ["icon", "url"]), get_in(data, ["icon", "name"]), "avatar")
banner =
data["image"]["url"] &&
%{
"name" => data["image"]["name"] || "banner",
"url" => MediaProxy.url(data["image"]["url"])
}
download_picture(get_in(data, ["image", "url"]), get_in(data, ["image", "name"]), "banner")
%{
url: data["id"],
@ -140,4 +134,16 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
})
end
end
@spec download_picture(String.t() | nil, String.t(), String.t()) :: map()
defp download_picture(nil, _name, _default_name), do: nil
defp download_picture(url, name, default_name) do
with {:ok, %{body: body, status: code, headers: response_headers}}
when code in 200..299 <- RemoteMediaDownloaderClient.get(url),
name <- name || Parser.get_filename_from_response(response_headers, url) || default_name,
{:ok, file} <- Upload.store(%{body: body, name: name}) do
file
end
end
end

View File

@ -10,7 +10,6 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
alias Mobilizon.Addresses
alias Mobilizon.Addresses.Address
alias Mobilizon.Events.Event, as: EventModel
alias Mobilizon.Medias.Media
alias Mobilizon.Federation.ActivityStream.{Converter, Convertible}
alias Mobilizon.Federation.ActivityStream.Converter.Address, as: AddressConverter
@ -21,7 +20,8 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
fetch_tags: 1,
fetch_mentions: 1,
build_tags: 1,
maybe_fetch_actor_and_attributed_to_id: 1
maybe_fetch_actor_and_attributed_to_id: 1,
process_pictures: 2
]
require Logger
@ -34,6 +34,9 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
defdelegate model_to_as(event), to: EventConverter
end
@online_address_name "Website"
@banner_picture_name "Banner"
@doc """
Converts an AP object data to our internal data structure.
"""
@ -47,30 +50,16 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
{:tags, tags} <- {:tags, fetch_tags(object["tag"])},
{:mentions, mentions} <- {:mentions, fetch_mentions(object["tag"])},
{:visibility, visibility} <- {:visibility, get_visibility(object)},
{:options, options} <- {:options, get_options(object)} do
attachments =
object
|> Map.get("attachment", [])
|> Enum.filter(fn attachment -> Map.get(attachment, "type", "Document") == "Document" end)
picture_id =
with true <- length(attachments) > 0,
{:ok, %Media{id: picture_id}} <-
attachments
|> hd()
|> MediaConverter.find_or_create_media(actor_id) do
picture_id
else
_err ->
nil
end
{:options, options} <- {:options, get_options(object)},
[description: description, picture_id: picture_id, medias: medias] <-
process_pictures(object, actor_id) do
%{
title: object["name"],
description: object["content"],
description: description,
organizer_actor_id: actor_id,
attributed_to_id: if(is_nil(attributed_to), do: nil, else: attributed_to.id),
picture_id: picture_id,
medias: medias,
begins_on: object["startTime"],
ends_on: object["endTime"],
category: object["category"],
@ -143,6 +132,7 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
|> maybe_add_physical_address(event)
|> maybe_add_event_picture(event)
|> maybe_add_online_address(event)
|> maybe_add_inline_media(event)
end
# Get only elements that we have in EventOptions
@ -213,7 +203,7 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
"type" => "Link",
"href" => url,
"mediaType" => "text/html",
"name" => "Website"
"name" => @online_address_name
} ->
url
@ -239,7 +229,12 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
res,
"attachment",
[],
&(&1 ++ [MediaConverter.model_to_as(event.picture)])
&(&1 ++
[
event.picture
|> MediaConverter.model_to_as()
|> Map.put("name", @banner_picture_name)
])
)
end
@ -258,9 +253,21 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
"type" => "Link",
"href" => event.online_address,
"mediaType" => "text/html",
"name" => "Website"
"name" => @online_address_name
}
])
)
end
@spec maybe_add_inline_media(map(), Event.t()) :: map()
defp maybe_add_inline_media(res, event) do
medias = Enum.map(event.media, &MediaConverter.model_to_as/1)
Map.update(
res,
"attachment",
[],
&(&1 ++ medias)
)
end
end

View File

@ -9,9 +9,15 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Post do
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.Federation.ActivityPub.Utils
alias Mobilizon.Federation.ActivityStream.{Converter, Convertible}
alias Mobilizon.Federation.ActivityStream.Converter.Media, as: MediaConverter
alias Mobilizon.Posts.Post
require Logger
import Mobilizon.Federation.ActivityStream.Converter.Utils,
only: [
process_pictures: 2
]
@behaviour Converter
defimpl Convertible, for: Post do
@ -20,6 +26,8 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Post do
defdelegate model_to_as(post), to: PostConverter
end
@banner_picture_name "Banner"
@doc """
Convert an post struct to an ActivityStream representation
"""
@ -35,8 +43,11 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Post do
"name" => post.title,
"content" => post.body,
"attributedTo" => creator_url,
"published" => (post.publish_at || post.inserted_at) |> to_date()
"published" => (post.publish_at || post.inserted_at) |> to_date(),
"attachment" => []
}
|> maybe_add_post_picture(post)
|> maybe_add_inline_media(post)
end
@doc """
@ -48,15 +59,19 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Post do
%{"type" => "Article", "actor" => creator, "attributedTo" => group} = object
) do
with {:ok, %Actor{id: attributed_to_id}} <- get_actor(group),
{:ok, %Actor{id: author_id}} <- get_actor(creator) do
{:ok, %Actor{id: author_id}} <- get_actor(creator),
[description: description, picture_id: picture_id, medias: medias] <-
process_pictures(object, attributed_to_id) do
%{
title: object["name"],
body: object["content"],
body: description,
url: object["id"],
attributed_to_id: attributed_to_id,
author_id: author_id,
local: false,
publish_at: object["published"]
publish_at: object["published"],
picture_id: picture_id,
medias: medias
}
else
{:error, err} -> {:error, err}
@ -70,4 +85,34 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Post do
defp to_date(%DateTime{} = date), do: DateTime.to_iso8601(date)
defp to_date(%NaiveDateTime{} = date), do: NaiveDateTime.to_iso8601(date)
@spec maybe_add_post_picture(map(), Post.t()) :: map()
defp maybe_add_post_picture(res, post) do
if is_nil(post.picture),
do: res,
else:
Map.update(
res,
"attachment",
[],
&(&1 ++
[
post.picture
|> MediaConverter.model_to_as()
|> Map.put("name", @banner_picture_name)
])
)
end
@spec maybe_add_inline_media(map(), Post.t()) :: map()
defp maybe_add_inline_media(res, post) do
medias = Enum.map(post.media, &MediaConverter.model_to_as/1)
Map.update(
res,
"attachment",
[],
&(&1 ++ medias)
)
end
end

View File

@ -6,15 +6,19 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Utils do
alias Mobilizon.{Actors, Events}
alias Mobilizon.Actors.Actor
alias Mobilizon.Events.Tag
alias Mobilizon.Medias.Media
alias Mobilizon.Mention
alias Mobilizon.Storage.Repo
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.Federation.ActivityStream.Converter.Media, as: MediaConverter
alias Mobilizon.Web.Endpoint
require Logger
@banner_picture_name "Banner"
@spec fetch_tags([String.t()]) :: [Tag.t()]
def fetch_tags(tags) when is_list(tags) do
Logger.debug("fetching tags")
@ -169,4 +173,62 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Utils do
actor
end
end
@spec process_pictures(map(), integer()) :: Keyword.t()
def process_pictures(object, actor_id) do
attachements = Map.get(object, "attachment", [])
media_attachements = get_medias(attachements)
media_attachements_map =
media_attachements
|> Enum.map(fn media_attachement ->
{media_attachement["url"],
MediaConverter.find_or_create_media(media_attachement, actor_id)}
end)
|> Enum.reduce(%{}, fn {old_url, media}, acc ->
case media do
{:ok, %Media{} = media} ->
Map.put(acc, old_url, media)
_ ->
acc
end
end)
media_attachements_map_urls =
media_attachements_map
|> Enum.map(fn {old_url, new_media} -> {old_url, new_media.file.url} end)
|> Map.new()
picture_id =
with banner when is_map(banner) <- get_banner_picture(attachements),
{:ok, %Media{id: picture_id}} <-
MediaConverter.find_or_create_media(banner, actor_id) do
picture_id
else
_err ->
nil
end
description = replace_media_urls_in_body(object["content"], media_attachements_map_urls)
[description: description, picture_id: picture_id, medias: Map.values(media_attachements_map)]
end
defp replace_media_urls_in_body(body, media_urls),
do:
Enum.reduce(media_urls, body, fn media_url, body ->
replace_media_url_in_body(body, media_url)
end)
defp replace_media_url_in_body(body, {old_url, new_url}),
do: String.replace(body, old_url, new_url)
defp get_medias(attachments) do
Enum.filter(attachments, &(&1["type"] == "Document" && &1["name"] != @banner_picture_name))
end
defp get_banner_picture(attachments) do
Enum.find(attachments, &(&1["type"] == "Document" && &1["name"] == @banner_picture_name))
end
end

View File

@ -10,7 +10,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
alias Mobilizon.Users.User
alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Federation.ActivityPub.Activity
import Mobilizon.Web.Gettext
@ -35,7 +34,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
) do
case {:has_event, Events.get_own_event_by_uuid_with_preload(uuid, user_id)} do
{:has_event, %Event{} = event} ->
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
{:ok, event}
{:has_event, _} ->
{:error, :event_not_found}
@ -51,7 +50,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
{:has_event, Events.get_public_event_by_uuid_with_preload(uuid)},
{:access_valid, true} <-
{:access_valid, Map.has_key?(context, :current_user) || check_event_access(event)} do
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
{:ok, event}
else
{:has_event, _} ->
find_private_event(parent, args, resolution)

View File

@ -8,7 +8,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
alias Mobilizon.Actors.{Actor, Member}
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User
alias Mobilizon.Web.Upload
import Mobilizon.Web.Gettext
@ -30,8 +29,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
with {:ok, %Actor{id: group_id} = group} <-
ActivityPub.find_or_make_group_from_nickname(name),
{:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)},
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id)},
group <- Person.proxify_pictures(group) do
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id)} do
{:ok, group}
else
{:member, false} ->
@ -44,7 +42,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
def find_group(_parent, %{preferred_username: name}, _resolution) do
with {:ok, actor} <- ActivityPub.find_or_make_group_from_nickname(name),
%Actor{} = actor <- Person.proxify_pictures(actor),
%Actor{} = actor <- restrict_fields_for_non_member_request(actor) do
{:ok, actor}
else

View File

@ -6,7 +6,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
alias Mobilizon.Actors.Actor
alias Mobilizon.Events.{Event, Participant}
alias Mobilizon.GraphQL.API.Participations
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User
alias Mobilizon.Web.Email
alias Mobilizon.Web.Email.Checker
@ -114,7 +113,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
%Participant{} = participant <-
participant
|> Map.put(:event, event)
|> Map.put(:actor, Person.proxify_pictures(actor)) do
|> Map.put(:actor, actor) do
{:ok, participant}
else
{:maximum_attendee_capacity, _} ->

View File

@ -15,15 +15,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
alias Mobilizon.Federation.ActivityPub
require Logger
alias Mobilizon.Web.{MediaProxy, Upload}
alias Mobilizon.Web.Upload
@doc """
Get a person
"""
def get_person(_parent, %{id: id}, %{context: %{current_user: %User{role: role}}}) do
with %Actor{suspended: suspended} = actor <- Actors.get_actor_with_preload(id, true),
true <- suspended == false or is_moderator(role),
actor <- proxify_pictures(actor) do
true <- suspended == false or is_moderator(role) do
{:ok, actor}
else
_ ->
@ -31,6 +30,8 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
end
def get_person(_parent, _args, _resolution), do: {:error, :unauthorized}
@doc """
Find a person
"""
@ -39,8 +40,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
}) do
with {:ok, %Actor{id: actor_id} = actor} <-
ActivityPub.find_or_make_actor_from_nickname(preferred_username),
{:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)},
actor <- proxify_pictures(actor) do
{:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)} do
{:ok, actor}
else
{:own, nil} ->
@ -120,9 +120,12 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
args = Map.put(args, :user_id, user.id)
with args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args) do
{:ok, new_person}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
end
end
@ -144,10 +147,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
with {:find_actor, %Actor{} = actor} <-
{:find_actor, Actors.get_actor(id)},
{:is_owned, %Actor{}} <- User.owns_actor(user, actor.id),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, _activity, %Actor{} = actor} <- ActivityPub.update(actor, args, true) do
{:ok, actor}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:find_actor, nil} ->
{:error, dgettext("errors", "Profile not found")}
@ -199,18 +205,27 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
defp save_attached_pictures(args) do
Enum.reduce([:avatar, :banner], args, fn key, args ->
if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
media = args[key][:media]
with args when is_map(args) <- save_attached_picture(args, :avatar),
args when is_map(args) <- save_attached_picture(args, :banner) do
args
end
end
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
Upload.store(media.file, type: key, description: media.alt) do
Map.put(args, key, %{"name" => name, "url" => url, "mediaType" => content_type})
end
else
args
defp save_attached_picture(args, key) do
if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
with media when is_map(media) <- save_picture(args[key][:media], key) do
Map.put(args, key, media)
end
end)
else
args
end
end
defp save_picture(media, key) do
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
Upload.store(media.file, type: key, description: media.alt) do
%{"name" => name, "url" => url, "mediaType" => content_type}
end
end
@doc """
@ -223,10 +238,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
{:no_actor, true} <- {:no_actor, no_actor},
args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- Map.put(args, :user_id, user.id),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args, true) do
{:ok, new_person}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:error, :user_not_found} ->
{:error, dgettext("errors", "No user with this email was found")}
@ -298,12 +316,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
end
def proxify_pictures(%Actor{} = actor) do
actor
|> proxify_avatar
|> proxify_banner
end
def user_for_person(%Actor{type: :Person, user_id: user_id}, _args, %{
context: %{current_user: %User{role: role}}
})
@ -343,20 +355,4 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
defp last_admin_of_a_group?(actor_id) do
length(Actors.list_group_ids_where_last_administrator(actor_id)) > 0
end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{avatar: %{url: avatar_url} = avatar} = actor) do
actor |> Map.put(:avatar, avatar |> Map.put(:url, MediaProxy.url(avatar_url)))
end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{} = actor), do: actor
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{banner: %{url: banner_url} = banner} = actor) do
actor |> Map.put(:banner, banner |> Map.put(:url, MediaProxy.url(banner_url)))
end
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{} = actor), do: actor
end

View File

@ -73,6 +73,12 @@ defmodule Mix.Tasks.Mobilizon.Actors.Refresh do
{:actor, nil} ->
shell_error("Error: No such actor")
{:error, err} when is_binary(err) ->
shell_error(err)
_err ->
shell_error("Error while refreshing actor #{preferred_username}")
end
end

View File

@ -13,11 +13,13 @@ defmodule Mobilizon.Actors do
alias Mobilizon.Actors.{Actor, Bot, Follower, Member}
alias Mobilizon.Addresses.Address
alias Mobilizon.{Crypto, Events}
alias Mobilizon.Events.FeedToken
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.Medias.File
alias Mobilizon.Service.Workers
alias Mobilizon.Storage.{Page, Repo}
alias Mobilizon.Users
alias Mobilizon.Users.User
alias Mobilizon.Web.Email.Group
alias Mobilizon.Web.Upload
@ -215,18 +217,35 @@ defmodule Mobilizon.Actors do
def new_person(args, default_actor \\ false) do
args = Map.put(args, :keys, Crypto.generate_rsa_2048_private_key())
with {:ok, %Actor{id: person_id} = person} <-
%Actor{}
|> Actor.registration_changeset(args)
|> Repo.insert() do
Events.create_feed_token(%{user_id: args.user_id, actor_id: person.id})
multi =
Multi.new()
|> Multi.insert(:person, Actor.registration_changeset(%Actor{}, args))
|> Multi.insert(:token, fn %{person: person} ->
FeedToken.changeset(%FeedToken{}, %{
user_id: args.user_id,
actor_id: person.id,
token: Ecto.UUID.generate()
})
end)
multi =
if default_actor do
user = Users.get_user!(args.user_id)
Users.update_user(user, %{default_actor_id: person_id})
Multi.update(multi, :user, fn %{person: person} ->
User.changeset(user, %{default_actor_id: person.id})
end)
else
multi
end
{:ok, person}
case Repo.transaction(multi) do
{:ok, %{person: %Actor{} = person}} ->
{:ok, person}
{:error, _step, err, _} ->
Logger.error("Error while creating a new person")
{:error, err}
end
end

View File

@ -10,7 +10,7 @@ defmodule Mobilizon.Posts do
import Ecto.Query
require Logger
@post_preloads [:author, :attributed_to, :picture]
@post_preloads [:author, :attributed_to, :picture, :media]
import EctoEnum

View File

@ -13,7 +13,7 @@ defmodule Mobilizon.Service.Export.Feed do
alias Mobilizon.Storage.Page
alias Mobilizon.Users.User
alias Mobilizon.Web.{Endpoint, MediaProxy}
alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.Router.Helpers, as: Routes
require Logger
@ -85,14 +85,14 @@ defmodule Mobilizon.Service.Export.Feed do
feed =
if actor.avatar do
feed |> Feed.icon(actor.avatar.url |> MediaProxy.url())
feed |> Feed.icon(actor.avatar.url)
else
feed
end
feed =
if actor.banner do
feed |> Feed.logo(actor.banner.url |> MediaProxy.url())
feed |> Feed.logo(actor.banner.url)
else
feed
end

View File

@ -61,7 +61,8 @@ defmodule Mobilizon.Service.Formatter.DefaultScrubbler do
"height",
"class",
"title",
"alt"
"alt",
"data-media-id"
])
Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "mention"])

View File

@ -0,0 +1,22 @@
defmodule Mobilizon.Service.HTTP.RemoteMediaDownloaderClient do
@moduledoc """
Tesla HTTP Basic Client that fetches HTML to extract metadata preview
"""
use Tesla
alias Mobilizon.Config
@default_opts [
recv_timeout: 20_000
]
adapter(Tesla.Adapter.Hackney, @default_opts)
@user_agent Config.instance_user_agent()
plug(Tesla.Middleware.FollowRedirects)
plug(Tesla.Middleware.Timeout, timeout: 10_000)
plug(Tesla.Middleware.Headers, [{"User-Agent", @user_agent}])
end

View File

@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
alias Phoenix.HTML.Tag
alias Mobilizon.Actors.Actor
alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, default_description: 1]
def build_tags(_actor, _locale \\ "en")
@ -36,7 +35,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
tags
else
tags ++
[Tag.tag(:meta, property: "og:image", content: actor.avatar.url |> MediaProxy.url())]
[Tag.tag(:meta, property: "og:image", content: actor.avatar.url)]
end
end

View File

@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
alias Phoenix.HTML.Tag
alias Mobilizon.Events.Event
alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, strip_tags: 1]
def build_tags(%Event{} = event, locale \\ "en") do
@ -28,7 +27,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
[
Tag.tag(:meta,
property: "og:image",
content: event.picture.file.url |> MediaProxy.url()
content: event.picture.file.url
)
]
end

View File

@ -47,6 +47,14 @@ defmodule Mobilizon.Service.RichMedia.Parser do
{:error, "Cachex error: #{inspect(e)}"}
end
@doc """
Get a filename for the fetched data, using the response header or the last part of the URL
"""
@spec get_filename_from_response(Enum.t(), String.t()) :: String.t() | nil
def get_filename_from_response(response_headers, url) do
get_filename_from_headers(response_headers) || get_filename_from_url(url)
end
@spec parse_url(String.t(), Enum.t()) :: {:ok, map()} | {:error, any()}
defp parse_url(url, options \\ []) do
user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())

View File

@ -1,50 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/controller.ex
defmodule Mobilizon.Web.MediaProxyController do
use Mobilizon.Web, :controller
alias Plug.Conn
alias Mobilizon.Config
alias Mobilizon.Web.MediaProxy
alias Mobilizon.Web.ReverseProxy
@default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
with config <- Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
:ok <- filename_matches(Map.has_key?(params, "filename"), conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
false ->
send_resp(conn, 404, Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
send_resp(conn, 403, Conn.Status.reason_phrase(403))
{:wrong_filename, filename} ->
redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
end
end
def filename_matches(has_filename, path, url) do
filename =
url
|> MediaProxy.filename()
|> URI.decode()
path = URI.decode(path)
if has_filename && filename && Path.basename(path) != filename do
{:wrong_filename, filename}
else
:ok
end
end
end

View File

@ -1,91 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/media_proxy.ex
defmodule Mobilizon.Web.MediaProxy do
@moduledoc """
Handles proxifying media files
"""
alias Mobilizon.Config
alias Mobilizon.Web.Endpoint
@base64_opts [padding: false]
def url(nil), do: nil
def url(""), do: nil
def url("/" <> _ = url), do: url
def url(url) do
config = Application.get_env(:mobilizon, :media_proxy, [])
if !Keyword.get(config, :enabled, false) or
String.starts_with?(url, Endpoint.url()) do
url
else
encode_url(url)
end
end
def encode_url(url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
# Must preserve `%2F` for compatibility with S3
# https://git.pleroma.social/pleroma/pleroma/issues/580
replacement = get_replacement(url, ":2F:")
# The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
base64 =
url
|> String.replace("%2F", replacement)
|> URI.decode()
|> URI.encode()
|> String.replace(replacement, "%2F")
|> Base.url_encode64(@base64_opts)
sig = :crypto.hmac(:sha, secret, base64)
sig64 = sig |> Base.url_encode64(@base64_opts)
build_url(sig64, base64, filename(url))
end
def decode_url(sig, url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
sig = Base.url_decode64!(sig, @base64_opts)
local_sig = :crypto.hmac(:sha, secret, url)
if local_sig == sig do
{:ok, Base.url_decode64!(url, @base64_opts)}
else
{:error, :invalid_signature}
end
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
def build_url(sig_base64, url_base64, filename \\ nil) do
[
Config.get([:media_proxy, :base_url], Endpoint.url()),
"proxy",
sig_base64,
url_base64,
filename
]
|> Enum.filter(fn value -> value end)
|> Path.join()
end
defp get_replacement(url, replacement) do
if String.contains?(url, replacement) do
get_replacement(url, replacement <> replacement)
else
replacement
end
end
end

View File

@ -69,8 +69,6 @@ defmodule Mobilizon.Web.ReverseProxy do
alias Plug.Conn
alias Mobilizon.Web.MediaProxy
require Logger
@type option ::
@ -111,7 +109,7 @@ defmodule Mobilizon.Web.ReverseProxy do
req_headers = build_req_headers(conn.req_headers, opts)
opts =
if filename = MediaProxy.filename(url) do
if filename = filename(url) do
Keyword.put_new(opts, :attachment_name, filename)
else
opts
@ -388,4 +386,8 @@ defmodule Mobilizon.Web.ReverseProxy do
defp increase_read_duration(_) do
{:ok, :no_duration_limit, :no_duration_limit}
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
end

View File

@ -162,13 +162,6 @@ defmodule Mobilizon.Web.Router do
post("/auth/:provider/callback", AuthController, :callback)
end
scope "/proxy/", Mobilizon.Web do
pipe_through(:remote_media)
get("/:sig/:url", MediaProxyController, :remote)
get("/:sig/:url/:filename", MediaProxyController, :remote)
end
if Application.fetch_env!(:mobilizon, :env) in [:dev, :e2e] do
# If using Phoenix
forward("/sent_emails", Bamboo.SentEmailViewerPlug)

View File

@ -5,7 +5,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
alias Mobilizon.Addresses.Address
alias Mobilizon.Events.Event
alias Mobilizon.Posts.Post
alias Mobilizon.Web.{Endpoint, MediaProxy}
alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.JsonLD.ObjectView
def render("group.json", %{group: %Actor{} = group}) do
@ -41,7 +41,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
"image" =>
if(event.picture,
do: [
event.picture.file.url |> MediaProxy.url()
event.picture.file.url
],
else: ["#{Endpoint.url()}/img/mobilizon_default_card.png"]
)

View File

@ -8,6 +8,7 @@ defmodule Mobilizon.Web.PageView do
alias Mobilizon.Actors.Actor
alias Mobilizon.Discussions.{Comment, Discussion}
alias Mobilizon.Events.Event
alias Mobilizon.Posts.Post
alias Mobilizon.Resources.Resource
alias Mobilizon.Tombstone
@ -54,6 +55,12 @@ defmodule Mobilizon.Web.PageView do
|> Map.merge(Utils.make_json_ld_header())
end
def render("post.activity-json", %{conn: %{assigns: %{object: %Post{} = post}}}) do
post
|> Convertible.model_to_as()
|> Map.merge(Utils.make_json_ld_header())
end
def render(page, %{object: object, conn: conn} = _assigns)
when page in ["actor.html", "event.html", "comment.html", "post.html"] do
locale = get_locale(conn)

View File

@ -264,7 +264,6 @@ defmodule Mobilizon.Mixfile do
Mobilizon.Web.Plugs.UploadedMedia,
Mobilizon.Web.FallbackController,
Mobilizon.Web.FeedController,
Mobilizon.Web.MediaProxyController,
Mobilizon.Web.PageController,
Mobilizon.Web.ChangesetView,
Mobilizon.Web.JsonLD.ObjectView,
@ -295,7 +294,6 @@ defmodule Mobilizon.Mixfile do
Mobilizon.Web.Upload.MIME,
Mobilizon.Web.Upload.Uploader,
Mobilizon.Web.Upload.Uploader.Local,
Mobilizon.Web.MediaProxy,
Mobilizon.Web.ReverseProxy
],
Geospatial: [

View File

@ -45,13 +45,13 @@ defmodule Mobilizon.Federation.ActivityPubTest do
use_cassette "activity_pub/fetch_tcit@framapiaf.org" do
assert {:ok,
%Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :public} =
actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
_actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
end
use_cassette "activity_pub/fetch_tcit@framapiaf.org_not_discoverable" do
assert {:ok,
%Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :unlisted} =
actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
_actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
end
end

View File

@ -111,117 +111,110 @@ defmodule Mobilizon.Federation.ActivityPub.Transmogrifier.UpdateTest do
describe "handle incoming updates activities for group posts" do
test "it works for incoming update activities on group posts when remote actor is a moderator" do
use_cassette "activity_pub/group_post_update_activities" do
%Actor{url: remote_actor_url} =
remote_actor =
insert(:actor,
domain: "remote.domain",
url: "https://remote.domain/@remote",
preferred_username: "remote"
)
%Actor{url: remote_actor_url} =
remote_actor =
insert(:actor,
domain: "remote.domain",
url: "https://remote.domain/@remote",
preferred_username: "remote"
)
group = insert(:group)
%Member{} = member = insert(:member, actor: remote_actor, parent: group, role: :moderator)
%Post{} = post = insert(:post, attributed_to