Add more tests to upload filters

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
master
Thomas Citharel 2 years ago
parent c3bada8428
commit 4a11d4adcc
No known key found for this signature in database
GPG Key ID: A061B9DDE0CA0773
  1. 1
      .gitignore
  2. 10
      lib/mobilizon/config.ex
  3. 25
      lib/mobilizon/utils.ex
  4. 17
      lib/web/upload/filter/anonymize_filename.ex
  5. 8
      lib/web/upload/filter/dedupe.ex
  6. 30
      lib/web/upload/filter/exiftool.ex
  7. 12
      lib/web/upload/filter/filter.ex
  8. 15
      lib/web/upload/filter/mogrify.ex
  9. 8
      lib/web/upload/filter/optimize.ex
  10. 3
      lib/web/upload/uploader/local.ex
  11. BIN
      test/fixtures/DSCN0010.jpg
  12. 44
      test/web/upload/filter/anonymize_filename_test.exs
  13. 33
      test/web/upload/filter/dedupe_test.exs
  14. 44
      test/web/upload/filter/exiftool_test.exs
  15. 35
      test/web/upload/filter/filter_test.exs
  16. 44
      test/web/upload/filter/mogrify_test.exs
  17. 1
      test/web/upload/upload_test.exs
  18. 58
      test/web/upload/uploader/local_test.exs

1
.gitignore vendored

@ -29,6 +29,7 @@ priv/errors/*
cover/
site/
test/fixtures/image_tmp.jpg
test/fixtures/DSCN0010_tmp.jpg
test/uploads/
uploads/*
release/

@ -276,17 +276,19 @@ defmodule Mobilizon.Config do
end
end
@spec put([module | atom], any) :: any
def put([key], value), do: put(key, value)
def put([parent_key | keys], value) do
parent = put_in(Application.get_env(:mobilizon, parent_key), keys, value)
parent =
Application.get_env(:mobilizon, parent_key, [])
|> put_in(keys, value)
Application.put_env(:mobilizon, parent_key, parent)
end
@spec put(module | atom, any) :: any
def put(key, value), do: Application.put_env(:mobilizon, key, value)
def put(key, value) do
Application.put_env(:mobilizon, key, value)
end
@spec to_boolean(boolean | String.t()) :: boolean
defp to_boolean(boolean), do: "true" == String.downcase("#{boolean}")

@ -0,0 +1,25 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Utils do
@moduledoc """
Module that provide generic utils functions for Mobilizon
"""
@doc """
POSIX-compliant check if command is available in the system
## Examples
iex> command_available?("git")
true
iex> command_available?("wrongcmd")
false
"""
@spec command_available?(String.t()) :: boolean()
def command_available?(command) do
match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
end
end

@ -13,11 +13,20 @@ defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilename do
@behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Config
alias Mobilizon.Web.Upload
def filter(upload) do
extension = List.last(String.split(upload.name, "."))
name = Config.get([__MODULE__, :text], random(extension))
{:ok, %Mobilizon.Web.Upload{upload | name: name}}
def filter(%Upload{name: name} = upload) do
extension = List.last(String.split(name, "."))
name = predefined_name(extension) || random(extension)
{:ok, :filtered, %Upload{upload | name: name}}
end
def filter(_), do: {:ok, :noop}
@spec predefined_name(String.t()) :: String.t() | nil
defp predefined_name(extension) do
with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
do: String.replace(name, "{extension}", extension)
end
defp random(extension) do

@ -10,11 +10,13 @@ defmodule Mobilizon.Web.Upload.Filter.Dedupe do
@behaviour Mobilizon.Web.Upload.Filter
alias Mobilizon.Web.Upload
def filter(%Upload{name: name} = upload) do
def filter(%Upload{name: name, tempfile: tempfile} = upload) do
extension = name |> String.split(".") |> List.last()
shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower)
shasum = :crypto.hash(:sha256, File.read!(tempfile)) |> Base.encode16(case: :lower)
filename = shasum <> "." <> extension
{:ok, %Upload{upload | id: shasum, path: filename}}
{:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
end
def filter(_), do: {:ok, :noop}
end

@ -0,0 +1,30 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.Exiftool do
@moduledoc """
Strips GPS related EXIF tags and overwrites the file in place.
Also strips or replaces filesystem metadata e.g., timestamps.
"""
alias Mobilizon.Web.Upload
@behaviour Mobilizon.Web.Upload.Filter
@spec filter(Upload.t()) :: {:ok, any()} | {:error, String.t()}
# webp is not compatible with exiftool at this time
def filter(%Upload{content_type: "image/webp"}), do: {:ok, :noop}
def filter(%Upload{tempfile: file, content_type: "image" <> _}) do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
{_response, 0} -> {:ok, :filtered}
{error, 1} -> {:error, error}
end
rescue
_e in ErlangError ->
{:error, "exiftool command not found"}
end
def filter(_), do: {:ok, :noop}
end

@ -17,7 +17,10 @@ defmodule Mobilizon.Web.Upload.Filter do
require Logger
@callback filter(Mobilizon.Web.Upload.t()) ::
:ok | {:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
{:ok, :filtered}
| {:ok, :noop}
| {:ok, :filtered, Mobilizon.Web.Upload.t()}
| {:error, any()}
@spec filter([module()], Mobilizon.Web.Upload.t()) ::
{:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
@ -28,10 +31,13 @@ defmodule Mobilizon.Web.Upload.Filter do
def filter([filter | rest], upload) do
case filter.filter(upload) do
:ok ->
{:ok, :filtered} ->
filter(rest, upload)
{:ok, upload} ->
{:ok, :filtered, upload} ->
filter(rest, upload)
{:ok, :noop} ->
filter(rest, upload)
error ->

@ -15,19 +15,24 @@ defmodule Mobilizon.Web.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
@spec filter(Mobilizon.Web.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Mobilizon.Web.Upload{tempfile: file, content_type: "image" <> _}) do
filters = Config.get!([__MODULE__, :args])
do_filter(file, Config.get!([__MODULE__, :args]))
{:ok, :filtered}
rescue
_e in ErlangError ->
{:error, "mogrify command not found"}
end
def filter(_), do: {:ok, :noop}
def do_filter(file, filters) do
file
|> Mogrify.open()
|> mogrify_filter(filters)
|> Mogrify.save(in_place: true)
:ok
end
def filter(_), do: :ok
defp mogrify_filter(mogrify, nil), do: mogrify
defp mogrify_filter(mogrify, [filter | rest]) do

@ -21,7 +21,7 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
case ExOptimizer.optimize(file, deps: optimizers) do
{:ok, _res} ->
:ok
{:ok, :filtered}
{:error, err} ->
require Logger
@ -30,12 +30,12 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
"Unable to optimize file #{file}. The return from the process was #{inspect(err)}"
)
:ok
{:ok, :noop}
err ->
err
{:error, err}
end
end
def filter(_), do: :ok
def filter(_), do: {:ok, :noop}
end

@ -12,10 +12,12 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
alias Mobilizon.Config
@impl true
def get_file(_) do
{:ok, {:static_dir, upload_path()}}
end
@impl true
def put_file(upload) do
{path, file} = local_path(upload.path)
result_file = Path.join(path, file)
@ -27,6 +29,7 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
:ok
end
@impl true
def remove_file(path) do
with {path, file} <- local_path(path),
full_path <- Path.join(path, file),

Binary file not shown.

After

Width:  |  Height:  |  Size: 158 KiB

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilenameTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
alias Mobilizon.Config
alias Mobilizon.Web.Upload
setup do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
upload_file = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg")
}
%{upload_file: upload_file}
end
setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
test "it replaces filename on pre-defined text", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.png"
end
test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}")
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.jpg"
end
test "it replaces filename on random text", %{upload_file: upload_file} do
{:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert <<_::bytes-size(14)>> <> ".jpg" = name
refute name == "an… image.jpg"
end
end

@ -0,0 +1,33 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.DedupeTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter.Dedupe
@shasum "590523d60d3831ec92d05cdd871078409d5780903910efec5cd35ab1b0f19d11"
test "adds shasum" do
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert {
:ok,
:filtered,
%Upload{id: @shasum, path: @shasum <> ".jpg"}
} = Dedupe.filter(upload)
end
end

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.ExiftoolTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter
test "apply exiftool filter" do
assert Mobilizon.Utils.command_available?("exiftool")
File.cp!(
"test/fixtures/DSCN0010.jpg",
"test/fixtures/DSCN0010_tmp.jpg"
)
upload = %Mobilizon.Web.Upload{
name: "image_with_GPS_data.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/DSCN0010.jpg"),
tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
}
assert Filter.Exiftool.filter(upload) == {:ok, :filtered}
{exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"])
{exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"])
refute exif_original == exif_filtered
assert String.match?(exif_original, ~r/GPS/)
refute String.match?(exif_filtered, ~r/GPS/)
end
test "verify webp files are skipped" do
upload = %Upload{
name: "sample.webp",
content_type: "image/webp"
}
assert Filter.Exiftool.filter(upload) == {:ok, :noop}
end
end

@ -0,0 +1,35 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.FilterTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
alias Mobilizon.Config
alias Mobilizon.Web.Upload.Filter
setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
test "applies filters" do
Config.put([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Mobilizon.Web.Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert Filter.filter([], upload) == {:ok, upload}
assert {:ok, upload} = Filter.filter([Mobilizon.Web.Upload.Filter.AnonymizeFilename], upload)
assert upload.name == "custom-file.png"
end
end

@ -0,0 +1,44 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Filter.MogrifyTest do
use Mobilizon.DataCase
use Mobilizon.Tests.Helpers
import Mock
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Filter
test "apply mogrify filter" do
clear_config(Filter.Mogrify, args: [{"tint", "40"}])
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
upload = %Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
task =
Task.async(fn ->
assert_receive {:apply_filter, {_, "tint", "40"}}, 4_000
end)
with_mock Mogrify,
open: fn _f -> %Mogrify.Image{} end,
custom: fn _m, _a -> :ok end,
custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end,
save: fn _f, _o -> :ok end do
assert Filter.Mogrify.filter(upload) == {:ok, :filtered}
end
Task.await(task)
end
end

@ -79,7 +79,6 @@ defmodule Mobilizon.UploadTest do
assert data.name == "an [image.jpg"
end
@tag :skip
test "fixes incorrect content type" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")

@ -0,0 +1,58 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Web.Upload.Uploader.LocalTest do
use Mobilizon.DataCase
alias Mobilizon.Web.Upload
alias Mobilizon.Web.Upload.Uploader.Local
describe "get_file/1" do
test "it returns path to local folder for files" do
assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
end
end
describe "put_file/1" do
test "put file to local folder" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
file_path = "local_upload/files/image.jpg"
file = %Upload{
name: "image.jpg",
content_type: "image/jpeg",
path: file_path,
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert Local.put_file(file) == :ok
assert [Local.upload_path(), file_path]
|> Path.join()
|> File.exists?()
end
end
describe "remove_file/1" do
test "removes local file" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
file_path = "local_upload/files/image.jpg"
file = %Upload{
name: "image.jpg",
content_type: "image/jpeg",
path: file_path,
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
:ok = Local.put_file(file)
local_path = Path.join([Local.upload_path(), file_path])
assert File.exists?(local_path)
Local.remove_file(file_path)
refute File.exists?(local_path)
end
end
end
Loading…
Cancel
Save