Merge branch 'upload-and-avatar-improvements' into 'master'

Upload and avatar improvements

See merge request framasoft/mobilizon!646
This commit is contained in:
Thomas Citharel 2020-10-26 09:00:08 +01:00
commit 974620b3b3
21 changed files with 390 additions and 48 deletions

1
.gitignore vendored
View File

@ -29,6 +29,7 @@ priv/errors/*
cover/
site/
test/fixtures/image_tmp.jpg
test/fixtures/DSCN0010_tmp.jpg
test/uploads/
uploads/*
release/

View File

@ -2,7 +2,7 @@ FROM elixir:latest
LABEL maintainer="Thomas Citharel <tcit@tcit.fr>"
ENV REFRESHED_AT=2020-10-22
RUN apt-get update -yq && apt-get install -yq build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake
RUN apt-get update -yq && apt-get install -yq build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake exiftool
RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && apt-get install nodejs -yq
RUN npm install -g yarn wait-on
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

View File

@ -1,7 +1,7 @@
import { IPicture } from "@/types/picture.model";
export async function buildFileFromIPicture(obj: IPicture | null | undefined) {
if (!obj) return null;
export async function buildFileFromIPicture(obj: IPicture | null | undefined): Promise<File | null> {
if (!obj) return Promise.resolve(null);
const response = await fetch(obj.url);
const blob = await response.blob();
@ -9,7 +9,7 @@ export async function buildFileFromIPicture(obj: IPicture | null | undefined) {
return new File([blob], obj.name);
}
export function buildFileVariable<T>(file: File | null, name: string, alt?: string) {
export function buildFileVariable<T>(file: File | null, name: string, alt?: string): Record<string, unknown> {
if (!file) return {};
return {

View File

@ -27,7 +27,7 @@
<span v-else>{{ $t("I create an identity") }}</span>
</h1>
<picture-upload v-model="avatarFile" class="picture-upload" />
<picture-upload v-model="avatarFile" :defaultImageSrc="avatarUrl" class="picture-upload" />
<b-field horizontal :label="$t('Display name')">
<b-input
@ -124,7 +124,6 @@ h1 {
<script lang="ts">
import { Component, Prop, Watch } from "vue-property-decorator";
import { mixins } from "vue-class-component";
import { Route } from "vue-router";
import {
CREATE_PERSON,
CURRENT_ACTOR_CLIENT,
@ -137,7 +136,7 @@ import { IPerson, Person } from "../../../types/actor";
import PictureUpload from "../../../components/PictureUpload.vue";
import { MOBILIZON_INSTANCE_HOST } from "../../../api/_entrypoint";
import RouteName from "../../../router/name";
import { buildFileFromIPicture, buildFileVariable, readFileAsync } from "../../../utils/image";
import { buildFileVariable } from "../../../utils/image";
import { changeIdentity } from "../../../utils/auth";
import identityEditionMixin from "../../../mixins/identityEdition";
@ -185,24 +184,31 @@ export default class EditIdentity extends mixins(identityEditionMixin) {
return this.$t("Only alphanumeric characters and underscores are supported.") as string;
}
get avatarUrl(): string | null {
if (this.identity && this.identity.avatar && this.identity.avatar.url) {
return this.identity.avatar.url;
}
return null;
}
@Watch("isUpdate")
async isUpdateChanged(): Promise<void> {
this.resetFields();
}
@Watch("identityName", { immediate: true })
async onIdentityParamChanged(val: string): Promise<Route | undefined> {
async onIdentityParamChanged(val: string): Promise<void> {
// Only used when we update the identity
if (!this.isUpdate) return;
await this.redirectIfNoIdentitySelected(val);
if (!this.identityName) {
return this.$router.push({ name: "CreateIdentity" });
this.$router.push({ name: "CreateIdentity" });
}
if (this.identityName && this.identity) {
this.avatarFile = await buildFileFromIPicture(this.identity.avatar);
this.avatarFile = null;
}
}
@ -278,6 +284,7 @@ export default class EditIdentity extends mixins(identityEditionMixin) {
}
},
});
this.avatarFile = null;
this.$notifier.success(
this.$t("Identity {displayName} updated", {
@ -376,23 +383,18 @@ export default class EditIdentity extends mixins(identityEditionMixin) {
}
private async buildVariables() {
const avatarObj = buildFileVariable(
/**
* We set the avatar only if user has selected one
*/
let avatarObj: Record<string, unknown> = { avatar: null };
if (this.avatarFile) {
avatarObj = buildFileVariable(
this.avatarFile,
"avatar",
`${this.identity.preferredUsername}'s avatar`
);
}
const res = { ...this.identity, ...avatarObj };
/**
* If the avatar didn't change, no need to try reuploading it
*/
if (this.identity.avatar) {
const oldAvatarFile = (await buildFileFromIPicture(this.identity.avatar)) as File;
const oldAvatarFileContent = await readFileAsync(oldAvatarFile);
const newAvatarFileContent = await readFileAsync(this.avatarFile as File);
if (oldAvatarFileContent === newAvatarFileContent) {
res.avatar = null;
}
}
return res;
}

View File

@ -276,17 +276,19 @@ defmodule Mobilizon.Config do
end
end
@spec put([module | atom], any) :: any
def put([key], value), do: put(key, value)
def put([parent_key | keys], value) do
parent = put_in(Application.get_env(:mobilizon, parent_key), keys, value)
parent =
Application.get_env(:mobilizon, parent_key, [])
|> put_in(keys, value)
Application.put_env(:mobilizon, parent_key, parent)
end
@spec put(module | atom, any) :: any
def put(key, value), do: Application.put_env(:mobilizon, key, value)
def put(key, value) do
Application.put_env(:mobilizon, key, value)
end
@spec to_boolean(boolean | String.t()) :: boolean
defp to_boolean(boolean), do: "true" == String.downcase("#{boolean}")

25
lib/mobilizon/utils.ex Normal file
View File

@ -0,0 +1,25 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Utils do
@moduledoc """
Module that provide generic utils functions for Mobilizon
"""
@doc """
POSIX-compliant check if command is available in the system
## Examples
iex> command_available?("git")
true
iex> command_available?("wrongcmd")
false
"""
@spec command_available?(String.t()) :: boolean()
def command_available?(command) do
match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
end
end

View File

@ -13,11 +13,20 @@ defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilename do
@behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Config
alias Mobilizon.Web.Upload
def filter(upload) do
extension = List.last(String.split(upload.name, "."))
name = Config.get([__MODULE__, :text], random(extension))
{:ok, %Mobilizon.Web.Upload{upload | name: name}}
def filter(%Upload{name: name} = upload) do
extension = List.last(String.split(name, "."))
name = predefined_name(extension) || random(extension)
{:ok, :filtered, %Upload{upload | name: name}}
end
def filter(_), do: {:ok, :noop}
@spec predefined_name(String.t()) :: String.t() | nil
defp predefined_name(extension) do
with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
do: String.replace(name, "{extension}", extension)
end
defp random(extension) do

View File

@ -10,11 +10,13 @@ defmodule Mobilizon.Web.Upload.Filter.Dedupe do
@behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Web.Upload
def filter(%Upload{name: name} = upload) do
def filter(%Upload{name: name, tempfile: tempfile} = upload) do
extension = name |> String.split(".") |> List.last()
shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower)
shasum = :crypto.hash(:sha256, File.read!(tempfile)) |> Base.encode16(case: :lower)
filename = shasum <> "." <> extension
{:ok, %Upload{upload | id: shasum, path: filename}}
{:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
end
def filter(_), do: {:ok, :noop}
end

View File

@ -0,0 +1,30 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.Exiftool do
@moduledoc """
Strips GPS related EXIF tags and overwrites the file in place.
Also strips or replaces filesystem metadata e.g., timestamps.
"""
alias Mobilizon.Web.Upload
@behaviour Mobilizon.Web.Upload.Filter
@spec filter(Upload.t()) :: {:ok, any()} | {:error, String.t()}
# webp is not compatible with exiftool at this time
def filter(%Upload{content_type: "image/webp"}), do: {:ok, :noop}
def filter(%Upload{tempfile: file, content_type: "image" <> _}) do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
{_response, 0} -> {:ok, :filtered}
{error, 1} -> {:error, error}
end
rescue
_e in ErlangError ->
{:error, "exiftool command not found"}
end
def filter(_), do: {:ok, :noop}
end

View File

@ -17,7 +17,10 @@ defmodule Mobilizon.Web.Upload.Filter do
require Logger
@callback filter(Mobilizon.Web.Upload.t()) ::
:ok | {:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
{:ok, :filtered}
| {:ok, :noop}
| {:ok, :filtered, Mobilizon.Web.Upload.t()}
| {:error, any()}
@spec filter([module()], Mobilizon.Web.Upload.t()) ::
{:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
@ -28,10 +31,13 @@ defmodule Mobilizon.Web.Upload.Filter do
def filter([filter | rest], upload) do
case filter.filter(upload) do
:ok ->
{:ok, :filtered} ->
filter(rest, upload)
{:ok, upload} ->
{:ok, :filtered, upload} ->
filter(rest, upload)
{:ok, :noop} ->
filter(rest, upload)
error ->

View File

@ -15,19 +15,24 @@ defmodule Mobilizon.Web.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
@spec filter(Mobilizon.Web.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Mobilizon.Web.Upload{tempfile: file, content_type: "image" <> _}) do
filters = Config.get!([__MODULE__, :args])
do_filter(file, Config.get!([__MODULE__, :args]))
{:ok, :filtered}
rescue
_e in ErlangError ->
{:error, "mogrify command not found"}
end
def filter(_), do: {:ok, :noop}
def do_filter(file, filters) do
file
|> Mogrify.open()
|> mogrify_filter(filters)
|> Mogrify.save(in_place: true)
:ok
end
def filter(_), do: :ok
defp mogrify_filter(mogrify, nil), do: mogrify
defp mogrify_filter(mogrify, [filter | rest]) do

View File

@ -21,7 +21,7 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
case ExOptimizer.optimize(file, deps: optimizers) do
{:ok, _res} ->
:ok
{:ok, :filtered}
{:error, err} ->
require Logger
@ -30,12 +30,12 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
"Unable to optimize file #{file}. The return from the process was #{inspect(err)}"
)
:ok
{:ok, :noop}
err ->
err
{:error, err}
end
end
def filter(_), do: :ok
def filter(_), do: {:ok, :noop}
end

View File

@ -12,10 +12,12 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
alias Mobilizon.Config
@impl true
def get_file(_) do
{:ok, {:static_dir, upload_path()}}
end
@impl true
def put_file(upload) do
{path, file} = local_path(upload.path)
result_file = Path.join(path, file)
@ -27,6 +29,7 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
:ok
end
@impl true
def remove_file(path) do
with {path, file} <- local_path(path),
full_path <- Path.join(path, file),

BIN
test/fixtures/DSCN0010.jpg vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 158 KiB

View File

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilenameTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
alias Mobilizon.Config
alias Mobilizon.Web.Upload
setup do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
upload_file = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg")
}
%{upload_file: upload_file}
end
setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
test "it replaces filename on pre-defined text", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.png"
end
test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}")
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.jpg"
end
test "it replaces filename on random text", %{upload_file: upload_file} do
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert <<_::bytes-size(14)>> <> ".jpg" = name
refute name == "an… image.jpg"
end
end

View File

@ -0,0 +1,33 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.DedupeTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter.Dedupe
@shasum "590523d60d3831ec92d05cdd871078409d5780903910efec5cd35ab1b0f19d11"
test "adds shasum" do
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert {
:ok,
:filtered,
%Upload{id: @shasum, path: @shasum <> ".jpg"}
} = Dedupe.filter(upload)
end
end

View File

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.ExiftoolTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter
test "apply exiftool filter" do
assert Mobilizon.Utils.command_available?("exiftool")
File.cp!(
"test/fixtures/DSCN0010.jpg",
"test/fixtures/DSCN0010_tmp.jpg"
)
upload = %Mobilizon.Web.Upload{
name: "image_with_GPS_data.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/DSCN0010.jpg"),
tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
}
assert Filter.Exiftool.filter(upload) == {:ok, :filtered}
{exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"])
{exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"])
refute exif_original == exif_filtered
assert String.match?(exif_original, ~r/GPS/)
refute String.match?(exif_filtered, ~r/GPS/)
end
test "verify webp files are skipped" do
upload = %Upload{
name: "sample.webp",
content_type: "image/webp"
}
assert Filter.Exiftool.filter(upload) == {:ok, :noop}
end
end

View File

@ -0,0 +1,35 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.FilterTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
alias Mobilizon.Config
alias Mobilizon.Web.Upload.Filter
setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
test "applies filters" do
Config.put([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Mobilizon.Web.Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert Filter.filter([], upload) == {:ok, upload}
assert {:ok, upload} = Filter.filter([Mobilizon.Web.Upload.Filter.AnonymizeFilename], upload)
assert upload.name == "custom-file.png"
end
end

View File

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.MogrifyTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
import Mock
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter
test "apply mogrify filter" do
clear_config(Filter.Mogrify, args: [{"tint", "40"}])
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
task =
Task.async(fn ->
assert_receive {:apply_filter, {_, "tint", "40"}}, 4_000
end)
with_mock Mogrify,
open: fn _f -> %Mogrify.Image{} end,
custom: fn _m, _a -> :ok end,
custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end,
save: fn _f, _o -> :ok end do
assert Filter.Mogrify.filter(upload) == {:ok, :filtered}
end
Task.await(task)
end
end

View File

@ -79,7 +79,6 @@ defmodule Mobilizon.UploadTest do
assert data.name == "an [image.jpg"
end
@tag :skip
test "fixes incorrect content type" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")

View File

@ -0,0 +1,58 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Uploader.LocalTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Uploader.Local
describe "get_file/1" do
test "it returns path to local folder for files" do
assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
end
end
describe "put_file/1" do
test "put file to local folder" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
file_path = "local_upload/files/image.jpg"
file = %Upload{
name: "image.jpg",
content_type: "image/jpeg",
path: file_path,
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert Local.put_file(file) == :ok
assert [Local.upload_path(), file_path]
|> Path.join()
|> File.exists?()
end
end
describe "remove_file/1" do
test "removes local file" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
file_path = "local_upload/files/image.jpg"
file = %Upload{
name: "image.jpg",
content_type: "image/jpeg",
path: file_path,
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
:ok = Local.put_file(file)
local_path = Path.join([Local.upload_path(), file_path])
assert File.exists?(local_path)
Local.remove_file(file_path)
refute File.exists?(local_path)
end
end
end