Add more tests to upload filters

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel 2020-10-25 12:00:00 +01:00
parent c3bada8428
commit 4a11d4adcc
No known key found for this signature in database
GPG Key ID: A061B9DDE0CA0773
18 changed files with 364 additions and 24 deletions

1
.gitignore vendored
View File

@ -29,6 +29,7 @@ priv/errors/*
cover/ cover/
site/ site/
test/fixtures/image_tmp.jpg test/fixtures/image_tmp.jpg
test/fixtures/DSCN0010_tmp.jpg
test/uploads/ test/uploads/
uploads/* uploads/*
release/ release/

View File

@ -276,17 +276,19 @@ defmodule Mobilizon.Config do
end end
end end
@spec put([module | atom], any) :: any
def put([key], value), do: put(key, value) def put([key], value), do: put(key, value)
def put([parent_key | keys], value) do def put([parent_key | keys], value) do
parent = put_in(Application.get_env(:mobilizon, parent_key), keys, value) parent =
Application.get_env(:mobilizon, parent_key, [])
|> put_in(keys, value)
Application.put_env(:mobilizon, parent_key, parent) Application.put_env(:mobilizon, parent_key, parent)
end end
@spec put(module | atom, any) :: any def put(key, value) do
def put(key, value), do: Application.put_env(:mobilizon, key, value) Application.put_env(:mobilizon, key, value)
end
@spec to_boolean(boolean | String.t()) :: boolean @spec to_boolean(boolean | String.t()) :: boolean
defp to_boolean(boolean), do: "true" == String.downcase("#{boolean}") defp to_boolean(boolean), do: "true" == String.downcase("#{boolean}")

25
lib/mobilizon/utils.ex Normal file
View File

@ -0,0 +1,25 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Utils do
@moduledoc """
Module that provide generic utils functions for Mobilizon
"""
@doc """
POSIX-compliant check if command is available in the system
## Examples
iex> command_available?("git")
true
iex> command_available?("wrongcmd")
false
"""
@spec command_available?(String.t()) :: boolean()
def command_available?(command) do
match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
end
end

View File

@ -13,11 +13,20 @@ defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilename do
@behaviour Mobilizon.Web.Upload.Filter @behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Config alias Mobilizon.Config
alias Mobilizon.Web.Upload
def filter(upload) do def filter(%Upload{name: name} = upload) do
extension = List.last(String.split(upload.name, ".")) extension = List.last(String.split(name, "."))
name = Config.get([__MODULE__, :text], random(extension)) name = predefined_name(extension) || random(extension)
{:ok, %Mobilizon.Web.Upload{upload | name: name}} {:ok, :filtered, %Upload{upload | name: name}}
end
def filter(_), do: {:ok, :noop}
@spec predefined_name(String.t()) :: String.t() | nil
defp predefined_name(extension) do
with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
do: String.replace(name, "{extension}", extension)
end end
defp random(extension) do defp random(extension) do

View File

@ -10,11 +10,13 @@ defmodule Mobilizon.Web.Upload.Filter.Dedupe do
@behaviour Mobilizon.Web.Upload.Filter @behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Web.Upload alias Mobilizon.Web.Upload
def filter(%Upload{name: name} = upload) do def filter(%Upload{name: name, tempfile: tempfile} = upload) do
extension = name |> String.split(".") |> List.last() extension = name |> String.split(".") |> List.last()
shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower) shasum = :crypto.hash(:sha256, File.read!(tempfile)) |> Base.encode16(case: :lower)
filename = shasum <> "." <> extension filename = shasum <> "." <> extension
{:ok, %Upload{upload | id: shasum, path: filename}} {:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
end end
def filter(_), do: {:ok, :noop}
end end

View File

@ -0,0 +1,30 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.Exiftool do
@moduledoc """
Strips GPS related EXIF tags and overwrites the file in place.
Also strips or replaces filesystem metadata e.g., timestamps.
"""
alias Mobilizon.Web.Upload
@behaviour Mobilizon.Web.Upload.Filter
@spec filter(Upload.t()) :: {:ok, any()} | {:error, String.t()}
# webp is not compatible with exiftool at this time
def filter(%Upload{content_type: "image/webp"}), do: {:ok, :noop}
def filter(%Upload{tempfile: file, content_type: "image" <> _}) do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
{_response, 0} -> {:ok, :filtered}
{error, 1} -> {:error, error}
end
rescue
_e in ErlangError ->
{:error, "exiftool command not found"}
end
def filter(_), do: {:ok, :noop}
end

View File

@ -17,7 +17,10 @@ defmodule Mobilizon.Web.Upload.Filter do
require Logger require Logger
@callback filter(Mobilizon.Web.Upload.t()) :: @callback filter(Mobilizon.Web.Upload.t()) ::
:ok | {:ok, Mobilizon.Web.Upload.t()} | {:error, any()} {:ok, :filtered}
| {:ok, :noop}
| {:ok, :filtered, Mobilizon.Web.Upload.t()}
| {:error, any()}
@spec filter([module()], Mobilizon.Web.Upload.t()) :: @spec filter([module()], Mobilizon.Web.Upload.t()) ::
{:ok, Mobilizon.Web.Upload.t()} | {:error, any()} {:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
@ -28,10 +31,13 @@ defmodule Mobilizon.Web.Upload.Filter do
def filter([filter | rest], upload) do def filter([filter | rest], upload) do
case filter.filter(upload) do case filter.filter(upload) do
:ok -> {:ok, :filtered} ->
filter(rest, upload) filter(rest, upload)
{:ok, upload} -> {:ok, :filtered, upload} ->
filter(rest, upload)
{:ok, :noop} ->
filter(rest, upload) filter(rest, upload)
error -> error ->

View File

@ -15,19 +15,24 @@ defmodule Mobilizon.Web.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()] @type conversions :: conversion() | [conversion()]
@spec filter(Mobilizon.Web.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Mobilizon.Web.Upload{tempfile: file, content_type: "image" <> _}) do def filter(%Mobilizon.Web.Upload{tempfile: file, content_type: "image" <> _}) do
filters = Config.get!([__MODULE__, :args]) do_filter(file, Config.get!([__MODULE__, :args]))
{:ok, :filtered}
rescue
_e in ErlangError ->
{:error, "mogrify command not found"}
end
def filter(_), do: {:ok, :noop}
def do_filter(file, filters) do
file file
|> Mogrify.open() |> Mogrify.open()
|> mogrify_filter(filters) |> mogrify_filter(filters)
|> Mogrify.save(in_place: true) |> Mogrify.save(in_place: true)
:ok
end end
def filter(_), do: :ok
defp mogrify_filter(mogrify, nil), do: mogrify defp mogrify_filter(mogrify, nil), do: mogrify
defp mogrify_filter(mogrify, [filter | rest]) do defp mogrify_filter(mogrify, [filter | rest]) do

View File

@ -21,7 +21,7 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
case ExOptimizer.optimize(file, deps: optimizers) do case ExOptimizer.optimize(file, deps: optimizers) do
{:ok, _res} -> {:ok, _res} ->
:ok {:ok, :filtered}
{:error, err} -> {:error, err} ->
require Logger require Logger
@ -30,12 +30,12 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
"Unable to optimize file #{file}. The return from the process was #{inspect(err)}" "Unable to optimize file #{file}. The return from the process was #{inspect(err)}"
) )
:ok {:ok, :noop}
err -> err ->
err {:error, err}
end end
end end
def filter(_), do: :ok def filter(_), do: {:ok, :noop}
end end

View File

@ -12,10 +12,12 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
alias Mobilizon.Config alias Mobilizon.Config
@impl true
def get_file(_) do def get_file(_) do
{:ok, {:static_dir, upload_path()}} {:ok, {:static_dir, upload_path()}}
end end
@impl true
def put_file(upload) do def put_file(upload) do
{path, file} = local_path(upload.path) {path, file} = local_path(upload.path)
result_file = Path.join(path, file) result_file = Path.join(path, file)
@ -27,6 +29,7 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
:ok :ok
end end
@impl true
def remove_file(path) do def remove_file(path) do
with {path, file} <- local_path(path), with {path, file} <- local_path(path),
full_path <- Path.join(path, file), full_path <- Path.join(path, file),

BIN
test/fixtures/DSCN0010.jpg vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 158 KiB

View File

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilenameTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
alias Mobilizon.Config
alias Mobilizon.Web.Upload
setup do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
upload_file = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg")
}
%{upload_file: upload_file}
end
setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
test "it replaces filename on pre-defined text", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.png"
end
test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}")
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.jpg"
end
test "it replaces filename on random text", %{upload_file: upload_file} do
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert <<_::bytes-size(14)>> <> ".jpg" = name
refute name == "an… image.jpg"
end
end

View File

@ -0,0 +1,33 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.DedupeTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter.Dedupe
@shasum "590523d60d3831ec92d05cdd871078409d5780903910efec5cd35ab1b0f19d11"
test "adds shasum" do
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert {
:ok,
:filtered,
%Upload{id: @shasum, path: @shasum <> ".jpg"}
} = Dedupe.filter(upload)
end
end

View File

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.ExiftoolTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter
test "apply exiftool filter" do
assert Mobilizon.Utils.command_available?("exiftool")
File.cp!(
"test/fixtures/DSCN0010.jpg",
"test/fixtures/DSCN0010_tmp.jpg"
)
upload = %Mobilizon.Web.Upload{
name: "image_with_GPS_data.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/DSCN0010.jpg"),
tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
}
assert Filter.Exiftool.filter(upload) == {:ok, :filtered}
{exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"])
{exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"])
refute exif_original == exif_filtered
assert String.match?(exif_original, ~r/GPS/)
refute String.match?(exif_filtered, ~r/GPS/)
end
test "verify webp files are skipped" do
upload = %Upload{
name: "sample.webp",
content_type: "image/webp"
}
assert Filter.Exiftool.filter(upload) == {:ok, :noop}
end
end

View File

@ -0,0 +1,35 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.FilterTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
alias Mobilizon.Config
alias Mobilizon.Web.Upload.Filter
setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
test "applies filters" do
Config.put([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Mobilizon.Web.Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert Filter.filter([], upload) == {:ok, upload}
assert {:ok, upload} = Filter.filter([Mobilizon.Web.Upload.Filter.AnonymizeFilename], upload)
assert upload.name == "custom-file.png"
end
end

View File

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.MogrifyTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
import Mock
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter
test "apply mogrify filter" do
clear_config(Filter.Mogrify, args: [{"tint", "40"}])
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
task =
Task.async(fn ->
assert_receive {:apply_filter, {_, "tint", "40"}}, 4_000
end)
with_mock Mogrify,
open: fn _f -> %Mogrify.Image{} end,
custom: fn _m, _a -> :ok end,
custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end,
save: fn _f, _o -> :ok end do
assert Filter.Mogrify.filter(upload) == {:ok, :filtered}
end
Task.await(task)
end
end

View File

@ -79,7 +79,6 @@ defmodule Mobilizon.UploadTest do
assert data.name == "an [image.jpg" assert data.name == "an [image.jpg"
end end
@tag :skip
test "fixes incorrect content type" do test "fixes incorrect content type" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")

View File

@ -0,0 +1,58 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Uploader.LocalTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Uploader.Local
describe "get_file/1" do
test "it returns path to local folder for files" do
assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
end
end
describe "put_file/1" do
test "put file to local folder" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
file_path = "local_upload/files/image.jpg"
file = %Upload{
name: "image.jpg",
content_type: "image/jpeg",
path: file_path,
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert Local.put_file(file) == :ok
assert [Local.upload_path(), file_path]
|> Path.join()
|> File.exists?()
end
end
describe "remove_file/1" do
test "removes local file" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
file_path = "local_upload/files/image.jpg"
file = %Upload{
name: "image.jpg",
content_type: "image/jpeg",
path: file_path,
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
:ok = Local.put_file(file)
local_path = Path.join([Local.upload_path(), file_path])
assert File.exists?(local_path)
Local.remove_file(file_path)
refute File.exists?(local_path)
end
end
end