From a8e58547fdb9f718f0116d4e1e0a94d98f7cd5af Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 26 Jan 2021 10:05:01 +0100 Subject: [PATCH] Add back media proxy for resources pictures Signed-off-by: Thomas Citharel --- config/config.exs | 14 ++ lib/web/controllers/media_proxy_controller.ex | 33 ++++ lib/web/media_proxy.ex | 115 ++++++++++++ lib/web/proxy/reverse_proxy.ex | 15 +- lib/web/router.ex | 5 + test/graphql/resolvers/resource_test.exs | 9 +- .../media_proxy_controller_test.exs | 70 +++++++ test/web/media_proxy_test.exs | 176 ++++++++++++++++++ 8 files changed, 428 insertions(+), 9 deletions(-) create mode 100644 lib/web/controllers/media_proxy_controller.ex create mode 100644 lib/web/media_proxy.ex create mode 100644 test/web/controllers/media_proxy_controller_test.exs create mode 100644 test/web/media_proxy_test.exs diff --git a/config/config.exs b/config/config.exs index 413108fb8..f20c3768c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -81,6 +81,20 @@ config :mobilizon, Mobilizon.Web.Upload, config :mobilizon, Mobilizon.Web.Upload.Uploader.Local, uploads: "uploads" +config :mobilizon, :media_proxy, + enabled: true, + proxy_opts: [ + redirect_on_failure: false, + max_body_length: 25 * 1_048_576, + # Note: max_read_duration defaults to Mobilizon.Web.ReverseProxy.max_read_duration_default/1 + max_read_duration: 30_000, + http: [ + follow_redirect: true, + pool: :media + ] + ], + whitelist: [] + config :mobilizon, Mobilizon.Web.Email.Mailer, adapter: Bamboo.SMTPAdapter, server: "localhost", diff --git a/lib/web/controllers/media_proxy_controller.ex b/lib/web/controllers/media_proxy_controller.ex new file mode 100644 index 000000000..b60affacc --- /dev/null +++ b/lib/web/controllers/media_proxy_controller.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mobilizon.Web.MediaProxyController do + use Mobilizon.Web, :controller + + alias Mobilizon.Config + alias Mobilizon.Web.{MediaProxy, ReverseProxy} + alias Plug.Conn + + # sobelow_skip ["XSS.SendResp"] + def remote(conn, %{"sig" => sig64, "url" => url64}) do + with {_, true} <- {:enabled, MediaProxy.enabled?()}, + {:ok, url} <- MediaProxy.decode_url(sig64, url64), + :ok <- MediaProxy.verify_request_path_and_url(conn, url) do + ReverseProxy.call(conn, url, media_proxy_opts()) + else + {:enabled, false} -> + send_resp(conn, 404, Conn.Status.reason_phrase(404)) + + {:error, :invalid_signature} -> + send_resp(conn, 403, Conn.Status.reason_phrase(403)) + + {:wrong_filename, filename} -> + redirect(conn, external: MediaProxy.build_url(sig64, url64, filename)) + end + end + + defp media_proxy_opts do + Config.get([:media_proxy, :proxy_opts], []) + end +end diff --git a/lib/web/media_proxy.ex b/lib/web/media_proxy.ex new file mode 100644 index 000000000..eede9e72c --- /dev/null +++ b/lib/web/media_proxy.ex @@ -0,0 +1,115 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mobilizon.Web.MediaProxy do + @moduledoc """ + Module to proxify remote media + """ + alias Mobilizon.Config + alias Mobilizon.Web + + @base64_opts [padding: false] + + def url(url) when is_nil(url) or url == "", do: nil + def url("/" <> _ = url), do: url + + def url(url) do + if enabled?() and url_proxiable?(url) do + encode_url(url) + else + url + end + end + + @spec url_proxiable?(String.t()) :: boolean() + def url_proxiable?(url) do + not local?(url) + end + + def enabled?, do: Config.get([:media_proxy, :enabled], false) + + # Note: media proxy must be enabled for media preview proxy in order to load all + # non-local non-whitelisted URLs through it and be sure that body size constraint is preserved. + def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled]) + + def local?(url), do: String.starts_with?(url, Web.Endpoint.url()) + + defp base64_sig64(url) do + base64 = Base.url_encode64(url, @base64_opts) + + sig64 = + base64 + |> signed_url() + |> Base.url_encode64(@base64_opts) + + {base64, sig64} + end + + def encode_url(url) do + {base64, sig64} = base64_sig64(url) + + build_url(sig64, base64, filename(url)) + end + + def decode_url(sig, url) do + with {:ok, sig} <- Base.url_decode64(sig, @base64_opts), + signature when signature == sig <- signed_url(url) do + {:ok, Base.url_decode64!(url, @base64_opts)} + else + _ -> {:error, :invalid_signature} + end + end + + defp signed_url(url) do + :crypto.hmac(:sha, Config.get([Web.Endpoint, :secret_key_base]), url) + end + + def filename(url_or_path) do + if path = URI.parse(url_or_path).path, do: Path.basename(path) + end + + def base_url do + Web.Endpoint.url() + end + + defp proxy_url(path, sig_base64, url_base64, filename) do + [ + base_url(), + path, + sig_base64, + url_base64, + filename + ] + |> Enum.filter(& &1) + |> Path.join() + end + + def build_url(sig_base64, url_base64, filename \\ nil) do + proxy_url("proxy", sig_base64, url_base64, filename) + end + + def verify_request_path_and_url( + %Plug.Conn{params: %{"filename" => _}, request_path: request_path}, + url + ) do + verify_request_path_and_url(request_path, url) + end + + def verify_request_path_and_url(request_path, url) when is_binary(request_path) do + filename = filename(url) + + if filename && not basename_matches?(request_path, filename) do + {:wrong_filename, filename} + else + :ok + end + end + + def verify_request_path_and_url(_, _), do: :ok + + defp basename_matches?(path, filename) do + basename = Path.basename(path) + basename == filename or URI.decode(basename) == filename or URI.encode(basename) == filename + end +end diff --git a/lib/web/proxy/reverse_proxy.ex b/lib/web/proxy/reverse_proxy.ex index c3e5993f0..cfc677de8 100644 --- a/lib/web/proxy/reverse_proxy.ex +++ b/lib/web/proxy/reverse_proxy.ex @@ -4,17 +4,22 @@ # Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/reverse_proxy.ex defmodule Mobilizon.Web.ReverseProxy do - @keep_req_headers ~w(accept user-agent accept-encoding cache-control - if-modified-since if-unmodified-since if-none-match if-range range) - @resp_cache_headers ~w(etag date last-modified cache-control) - @keep_resp_headers @resp_cache_headers ++ ~w(content-type content-disposition - content-encoding content-range accept-ranges vary) + @range_headers ~w(range if-range) + @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ + ~w(if-unmodified-since if-none-match) ++ @range_headers + @resp_cache_headers ~w(etag date last-modified) + @keep_resp_headers @resp_cache_headers ++ + ~w(content-length content-type content-disposition content-encoding) ++ + ~w(content-range accept-ranges vary) @default_cache_control_header "public, max-age=1209600" @valid_resp_codes [200, 206, 304] @max_read_duration :timer.seconds(30) @max_body_length :infinity @methods ~w(GET HEAD) + def max_read_duration_default, do: @max_read_duration + def default_cache_control_header, do: @default_cache_control_header + @moduledoc """ A reverse proxy. diff --git a/lib/web/router.ex b/lib/web/router.ex index 96fafe635..a74a84bfd 100644 --- a/lib/web/router.ex +++ b/lib/web/router.ex @@ -163,6 +163,11 @@ defmodule Mobilizon.Web.Router do post("/auth/:provider/callback", AuthController, :callback) end + scope "/proxy/", Mobilizon.Web do + get("/:sig/:url", MediaProxyController, :remote) + get("/:sig/:url/:filename", MediaProxyController, :remote) + end + if Application.fetch_env!(:mobilizon, :env) in [:dev, :e2e] do # If using Phoenix forward("/sent_emails", Bamboo.SentEmailViewerPlug) diff --git a/test/graphql/resolvers/resource_test.exs b/test/graphql/resolvers/resource_test.exs index 654693920..65dc4b734 100644 --- a/test/graphql/resolvers/resource_test.exs +++ b/test/graphql/resolvers/resource_test.exs @@ -6,6 +6,7 @@ defmodule Mobilizon.GraphQL.Resolvers.ResourceTest do alias Mobilizon.Actors.{Actor, Member} alias Mobilizon.Resources.Resource alias Mobilizon.Users.User + alias Mobilizon.Web.MediaProxy alias Mobilizon.GraphQL.AbsintheHelpers @@ -405,10 +406,10 @@ defmodule Mobilizon.GraphQL.Resolvers.ResourceTest do assert is_nil(res["errors"]) assert res["data"]["createResource"]["metadata"]["faviconUrl"] == - "https://joinmobilizon.org/img/icons/favicon.png" + MediaProxy.url("https://joinmobilizon.org/img/icons/favicon.png") assert res["data"]["createResource"]["metadata"]["imageRemoteUrl"] == - "https://joinmobilizon.org/img/opengraph/home.jpg" + MediaProxy.url("https://joinmobilizon.org/img/opengraph/home.jpg") assert res["data"]["createResource"]["path"] == "/#{@resource_title}" assert res["data"]["createResource"]["resourceUrl"] == @resource_url @@ -461,10 +462,10 @@ defmodule Mobilizon.GraphQL.Resolvers.ResourceTest do assert is_nil(res["errors"]) assert res["data"]["createResource"]["metadata"]["faviconUrl"] == - "https://joinmobilizon.org/img/icons/favicon.png" + MediaProxy.url("https://joinmobilizon.org/img/icons/favicon.png") assert res["data"]["createResource"]["metadata"]["imageRemoteUrl"] == - "https://joinmobilizon.org/img/opengraph/home.jpg" + MediaProxy.url("https://joinmobilizon.org/img/opengraph/home.jpg") assert res["data"]["createResource"]["path"] == "#{parent_path}/#{@resource_title}" assert res["data"]["createResource"]["resourceUrl"] == @resource_url diff --git a/test/web/controllers/media_proxy_controller_test.exs b/test/web/controllers/media_proxy_controller_test.exs new file mode 100644 index 000000000..816ebfff3 --- /dev/null +++ b/test/web/controllers/media_proxy_controller_test.exs @@ -0,0 +1,70 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mobilizon.Web.MediaProxyControllerTest do + use Mobilizon.Web.ConnCase + use Mobilizon.Tests.Helpers + + import Mock + + alias Mobilizon.Web.MediaProxy + alias Plug.Conn + + describe "Media Proxy" do + setup do + clear_config([:media_proxy, :enabled], true) + clear_config([Mobilizon.Web.Endpoint, :secret_key_base], "00000000000") + + [url: MediaProxy.encode_url("https://google.fn/test.png")] + end + + test "it returns 404 when disabled", %{conn: conn} do + clear_config([:media_proxy, :enabled], false) + + assert %Conn{ + status: 404, + resp_body: "Not Found" + } = get(conn, "/proxy/hhgfh/eeeee") + + assert %Conn{ + status: 404, + resp_body: "Not Found" + } = get(conn, "/proxy/hhgfh/eeee/fff") + end + + test "it returns 403 for invalid signature", %{conn: conn, url: url} do + Mobilizon.Config.put([Mobilizon.Web.Endpoint, :secret_key_base], "000") + %{path: path} = URI.parse(url) + + assert %Conn{ + status: 403, + resp_body: "Forbidden" + } = get(conn, path) + + assert %Conn{ + status: 403, + resp_body: "Forbidden" + } = get(conn, "/proxy/hhgfh/eeee") + + assert %Conn{ + status: 403, + resp_body: "Forbidden" + } = get(conn, "/proxy/hhgfh/eeee/fff") + end + + test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do + invalid_url = String.replace(url, "test.png", "test-file.png") + response = get(conn, invalid_url) + assert response.status == 302 + assert redirected_to(response) == url + end + + test "it performs ReverseProxy.call with valid signature", %{conn: conn, url: url} do + with_mock Mobilizon.Web.ReverseProxy, + call: fn _conn, _url, _opts -> %Conn{status: :success} end do + assert %Conn{status: :success} = get(conn, url) + end + end + end +end diff --git a/test/web/media_proxy_test.exs b/test/web/media_proxy_test.exs new file mode 100644 index 000000000..ea5d309fd --- /dev/null +++ b/test/web/media_proxy_test.exs @@ -0,0 +1,176 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mobilizon.Web.MediaProxyTest do + use ExUnit.Case + use Mobilizon.Tests.Helpers + + alias Mobilizon.Web.{Endpoint, MediaProxy} + + defp decode_result(encoded) do + [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/") + {:ok, decoded} = MediaProxy.decode_url(sig, base64) + decoded + end + + describe "when enabled" do + setup do: clear_config([:media_proxy, :enabled], true) + + test "ignores invalid url" do + assert MediaProxy.url(nil) == nil + assert MediaProxy.url("") == nil + end + + test "ignores relative url" do + assert MediaProxy.url("/local") == "/local" + assert MediaProxy.url("/") == "/" + end + + test "ignores local url" do + local_url = Endpoint.url() <> "/hello" + local_root = Endpoint.url() + assert MediaProxy.url(local_url) == local_url + assert MediaProxy.url(local_root) == local_root + end + + test "encodes and decodes URL" do + url = "https://pleroma.soykaf.com/static/logo.png" + encoded = MediaProxy.url(url) + + assert String.starts_with?(encoded, Endpoint.url()) + + assert String.ends_with?(encoded, "/logo.png") + + assert decode_result(encoded) == url + end + + test "encodes and decodes URL without a path" do + url = "https://pleroma.soykaf.com" + encoded = MediaProxy.url(url) + assert decode_result(encoded) == url + end + + test "encodes and decodes URL without an extension" do + url = "https://pleroma.soykaf.com/path/" + encoded = MediaProxy.url(url) + assert String.ends_with?(encoded, "/path") + assert decode_result(encoded) == url + end + + test "encodes and decodes URL and ignores query params for the path" do + url = "https://pleroma.soykaf.com/static/logo.png?93939393939&bunny=true" + encoded = MediaProxy.url(url) + assert String.ends_with?(encoded, "/logo.png") + assert decode_result(encoded) == url + end + + test "validates signature" do + encoded = MediaProxy.url("https://pleroma.social") + + clear_config( + [Endpoint, :secret_key_base], + "00000000000000000000000000000000000000000000000" + ) + + [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/") + assert MediaProxy.decode_url(sig, base64) == {:error, :invalid_signature} + end + + def test_verify_request_path_and_url(request_path, url, expected_result) do + assert MediaProxy.verify_request_path_and_url(request_path, url) == expected_result + + assert MediaProxy.verify_request_path_and_url( + %Plug.Conn{ + params: %{"filename" => Path.basename(request_path)}, + request_path: request_path + }, + url + ) == expected_result + end + + test "if first arg of `verify_request_path_and_url/2` is a Plug.Conn without \"filename\" " <> + "parameter, `verify_request_path_and_url/2` returns :ok " do + assert MediaProxy.verify_request_path_and_url( + %Plug.Conn{params: %{}, request_path: "/some/path"}, + "https://instance.com/file.jpg" + ) == :ok + + assert MediaProxy.verify_request_path_and_url( + %Plug.Conn{params: %{}, request_path: "/path/to/file.jpg"}, + "https://instance.com/file.jpg" + ) == :ok + end + + test "`verify_request_path_and_url/2` preserves the encoded or decoded path" do + test_verify_request_path_and_url( + "/Hello world.jpg", + "http://pleroma.social/Hello world.jpg", + :ok + ) + + test_verify_request_path_and_url( + "/Hello%20world.jpg", + "http://pleroma.social/Hello%20world.jpg", + :ok + ) + + test_verify_request_path_and_url( + "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg", + "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg", + :ok + ) + + test_verify_request_path_and_url( + # Note: `conn.request_path` returns encoded url + "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg", + "https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg", + :ok + ) + + test_verify_request_path_and_url( + "/my%2Flong%2Furl%2F2019%2F07%2FS", + "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg", + {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"} + ) + end + + # Some sites expect ASCII encoded characters in the URL to be preserved even if + # unnecessary. + # Issues: https://git.pleroma.social/pleroma/pleroma/issues/580 + # https://git.pleroma.social/pleroma/pleroma/issues/1055 + test "preserve ASCII encoding" do + url = + "https://pleroma.com/%20/%21/%22/%23/%24/%25/%26/%27/%28/%29/%2A/%2B/%2C/%2D/%2E/%2F/%30/%31/%32/%33/%34/%35/%36/%37/%38/%39/%3A/%3B/%3C/%3D/%3E/%3F/%40/%41/%42/%43/%44/%45/%46/%47/%48/%49/%4A/%4B/%4C/%4D/%4E/%4F/%50/%51/%52/%53/%54/%55/%56/%57/%58/%59/%5A/%5B/%5C/%5D/%5E/%5F/%60/%61/%62/%63/%64/%65/%66/%67/%68/%69/%6A/%6B/%6C/%6D/%6E/%6F/%70/%71/%72/%73/%74/%75/%76/%77/%78/%79/%7A/%7B/%7C/%7D/%7E/%7F/%80/%81/%82/%83/%84/%85/%86/%87/%88/%89/%8A/%8B/%8C/%8D/%8E/%8F/%90/%91/%92/%93/%94/%95/%96/%97/%98/%99/%9A/%9B/%9C/%9D/%9E/%9F/%C2%A0/%A1/%A2/%A3/%A4/%A5/%A6/%A7/%A8/%A9/%AA/%AB/%AC/%C2%AD/%AE/%AF/%B0/%B1/%B2/%B3/%B4/%B5/%B6/%B7/%B8/%B9/%BA/%BB/%BC/%BD/%BE/%BF/%C0/%C1/%C2/%C3/%C4/%C5/%C6/%C7/%C8/%C9/%CA/%CB/%CC/%CD/%CE/%CF/%D0/%D1/%D2/%D3/%D4/%D5/%D6/%D7/%D8/%D9/%DA/%DB/%DC/%DD/%DE/%DF/%E0/%E1/%E2/%E3/%E4/%E5/%E6/%E7/%E8/%E9/%EA/%EB/%EC/%ED/%EE/%EF/%F0/%F1/%F2/%F3/%F4/%F5/%F6/%F7/%F8/%F9/%FA/%FB/%FC/%FD/%FE/%FF" + + encoded = MediaProxy.url(url) + assert decode_result(encoded) == url + end + + # This includes unsafe/reserved characters which are not interpreted as part of the URL + # and would otherwise have to be ASCII encoded. It is our role to ensure the proxied URL + # is unmodified, so we are testing these characters anyway. + test "preserve non-unicode characters per RFC3986" do + url = + "https://pleroma.com/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890-._~:/?#[]@!$&'()*+,;=|^`{}" + + encoded = MediaProxy.url(url) + assert decode_result(encoded) == url + end + + test "preserve unicode characters" do + url = "https://ko.wikipedia.org/wiki/위키백과:대문" + + encoded = MediaProxy.url(url) + assert decode_result(encoded) == url + end + end + + describe "when disabled" do + setup do: clear_config([:media_proxy, :enabled], false) + + test "does not encode remote urls" do + assert MediaProxy.url("https://google.fr") == "https://google.fr" + end + end +end