From f90089e1bf9ee1b03a23d325c54dfb239e9e35a1 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 22 May 2019 14:12:11 +0200 Subject: [PATCH] Refactor media upload Use Upload Media logic from Pleroma Backend changes for picture upload Move AS <-> Model conversion to separate module Front changes Downgrade apollo-client: https://github.com/Akryum/vue-apollo/issues/577 Signed-off-by: Thomas Citharel --- .credo.exs | 2 +- .gitignore | 2 + .graphqlconfig.yaml | 2 +- Dockerfile | 2 +- config/config.exs | 37 +- config/dev.exs | 5 +- config/prod.exs | 22 +- config/test.exs | 4 + js/package.json | 2 +- js/src/App.vue | 1 + js/src/components/Account/Identities.vue | 4 +- js/src/components/Event/EventCard.vue | 2 +- js/src/components/Group/GroupCard.vue | 2 +- js/src/components/NavBar.vue | 4 +- js/src/components/PictureUpload.vue | 29 + js/src/graphql/actor.ts | 36 +- js/src/graphql/event.ts | 57 +- js/src/graphql/feed_tokens.ts | 6 +- js/src/graphql/search.ts | 4 +- js/src/types/actor/actor.model.ts | 10 +- js/src/types/event.model.ts | 8 +- js/src/types/picture.model.ts | 16 + js/src/views/Account/MyAccount.vue | 4 +- js/src/views/Account/Profile.vue | 11 +- js/src/views/Account/Register.vue | 6 +- js/src/views/Event/Create.vue | 20 +- js/src/views/Event/Event.vue | 13 +- js/src/views/Group/Group.vue | 6 +- js/yarn.lock | 867 ++++++++++-------- lib/mobilizon/actors/actor.ex | 24 +- lib/mobilizon/actors/actors.ex | 58 +- lib/mobilizon/application.ex | 12 + lib/mobilizon/common-config.ex | 41 + lib/mobilizon/events/event.ex | 9 +- lib/mobilizon/events/events.ex | 11 +- lib/mobilizon/media.ex | 115 +++ lib/mobilizon/media/file.ex | 22 + lib/mobilizon/media/picture.ex | 21 + lib/mobilizon/mime.ex | 121 +++ lib/mobilizon_web/api/events.ex | 4 +- .../controllers/activity_pub_controller.ex | 6 +- .../controllers/media_proxy_controller.ex | 45 + .../controllers/page_controller.ex | 2 - lib/mobilizon_web/endpoint.ex | 27 +- lib/mobilizon_web/media_proxy.ex | 86 ++ lib/mobilizon_web/plugs/uploaded_media.ex | 95 ++ lib/mobilizon_web/resolvers/event.ex | 46 +- lib/mobilizon_web/resolvers/person.ex | 29 +- lib/mobilizon_web/resolvers/picture.ex | 64 ++ lib/mobilizon_web/reverse_proxy.ex | 382 ++++++++ lib/mobilizon_web/router.ex | 3 +- lib/mobilizon_web/schema.ex | 18 +- lib/mobilizon_web/schema/actor.ex | 8 +- lib/mobilizon_web/schema/actors/group.ex | 27 +- lib/mobilizon_web/schema/actors/person.ex | 39 +- lib/mobilizon_web/schema/event.ex | 35 +- lib/mobilizon_web/schema/picture.ex | 48 + lib/mobilizon_web/upload.ex | 160 ++++ lib/mobilizon_web/upload/filter.ex | 42 + .../upload/filter/anonymize_filename.ex | 28 + lib/mobilizon_web/upload/filter/dedupe.ex | 19 + lib/mobilizon_web/upload/filter/mogrify.ex | 45 + lib/mobilizon_web/upload_plug.ex | 18 - lib/mobilizon_web/uploaders/avatar.ex | 53 -- lib/mobilizon_web/uploaders/event.ex | 51 -- lib/mobilizon_web/uploaders/local.ex | 40 + lib/mobilizon_web/uploaders/uploader.ex | 73 ++ .../views/json_ld/object_view.ex | 14 +- lib/mobilizon_web/views/page_view.ex | 4 +- lib/service/activity_pub/activity_pub.ex | 22 +- lib/service/activity_pub/converter.ex | 9 + lib/service/activity_pub/converters/actor.ex | 47 + .../activity_pub/converters/comment.ex | 96 ++ lib/service/activity_pub/converters/event.ex | 70 ++ lib/service/activity_pub/utils.ex | 176 ++-- lib/service/export/feed.ex | 18 +- lib/service/formatter/formatter.ex | 2 +- lib/service/metadata/actor.ex | 9 +- lib/service/metadata/event.ex | 24 +- mix.exs | 4 +- mix.lock | 24 +- .../20190426093202_create_pictures.exs | 41 + schema.graphql | 130 ++- test/fixtures/image.jpg | Bin 0 -> 13227 bytes test/fixtures/mastodon-update.json | 25 +- .../{category_picture.png => picture.png} | Bin test/fixtures/test.txt | 1 + test/fixtures/test_tmp.txt | 1 + .../fetch_framapiaf.org_users_tcit.json | 74 +- .../fetch_framapiaf_framasoft_status.json | 138 +++ .../fetch_framasoft_framapiaf_reply.json | 271 ++++++ .../fetch_reply_to_framatube.json | 235 ++++- .../fetch_social_tcit_fr_reply.json | 156 ---- .../fetch_social_tcit_fr_status.json | 78 -- .../fetch_tcit@framapiaf.org.json | 84 +- .../mastodon-post-activity_actor_call.json | 74 +- .../actors/remote_actor_mastodon_tcit.json | 71 +- test/mobilizon/actors/actors_test.exs | 25 +- test/mobilizon/media/media_test.exs | 50 + .../activity_pub/activity_pub_test.exs | 16 +- .../activity_pub/converters/actor_test.exs | 27 + .../activity_pub/transmogrifier_test.exs | 6 - .../service/activity_pub/utils_test.exs | 2 +- .../activity_pub_controller_test.exs | 3 +- test/mobilizon_web/media_proxy_test.exs | 181 ++++ .../plugs/uploaded_media_plug_test.exs | 44 + .../resolvers/event_resolver_test.exs | 129 +++ .../resolvers/person_resolver_test.exs | 202 ++-- .../resolvers/picture_resolver_test.exs | 130 +++ .../resolvers/user_resolver_test.exs | 16 +- test/mobilizon_web/upload_test.exs | 173 ++++ test/support/data_case.ex | 17 + test/support/factory.ex | 16 + 113 files changed, 4718 insertions(+), 1328 deletions(-) create mode 100644 js/src/components/PictureUpload.vue create mode 100644 js/src/types/picture.model.ts create mode 100644 lib/mobilizon/media.ex create mode 100644 lib/mobilizon/media/file.ex create mode 100644 lib/mobilizon/media/picture.ex create mode 100644 lib/mobilizon/mime.ex create mode 100644 lib/mobilizon_web/controllers/media_proxy_controller.ex create mode 100644 lib/mobilizon_web/media_proxy.ex create mode 100644 lib/mobilizon_web/plugs/uploaded_media.ex create mode 100644 lib/mobilizon_web/resolvers/picture.ex create mode 100644 lib/mobilizon_web/reverse_proxy.ex create mode 100644 lib/mobilizon_web/schema/picture.ex create mode 100644 lib/mobilizon_web/upload.ex create mode 100644 lib/mobilizon_web/upload/filter.ex create mode 100644 lib/mobilizon_web/upload/filter/anonymize_filename.ex create mode 100644 lib/mobilizon_web/upload/filter/dedupe.ex create mode 100644 lib/mobilizon_web/upload/filter/mogrify.ex delete mode 100644 lib/mobilizon_web/upload_plug.ex delete mode 100644 lib/mobilizon_web/uploaders/avatar.ex delete mode 100644 lib/mobilizon_web/uploaders/event.ex create mode 100644 lib/mobilizon_web/uploaders/local.ex create mode 100644 lib/mobilizon_web/uploaders/uploader.ex create mode 100644 lib/service/activity_pub/converter.ex create mode 100644 lib/service/activity_pub/converters/actor.ex create mode 100644 lib/service/activity_pub/converters/comment.ex create mode 100644 lib/service/activity_pub/converters/event.ex create mode 100644 priv/repo/migrations/20190426093202_create_pictures.exs create mode 100644 test/fixtures/image.jpg rename test/fixtures/{category_picture.png => picture.png} (100%) create mode 100644 test/fixtures/test.txt create mode 100644 test/fixtures/test_tmp.txt create mode 100644 test/fixtures/vcr_cassettes/activity_pub/fetch_framapiaf_framasoft_status.json create mode 100644 test/fixtures/vcr_cassettes/activity_pub/fetch_framasoft_framapiaf_reply.json delete mode 100644 test/fixtures/vcr_cassettes/activity_pub/fetch_social_tcit_fr_reply.json delete mode 100644 test/fixtures/vcr_cassettes/activity_pub/fetch_social_tcit_fr_status.json create mode 100644 test/mobilizon/media/media_test.exs create mode 100644 test/mobilizon/service/activity_pub/converters/actor_test.exs create mode 100644 test/mobilizon_web/media_proxy_test.exs create mode 100644 test/mobilizon_web/plugs/uploaded_media_plug_test.exs create mode 100644 test/mobilizon_web/resolvers/picture_resolver_test.exs create mode 100644 test/mobilizon_web/upload_test.exs diff --git a/.credo.exs b/.credo.exs index f349c380d..83f7dd28f 100644 --- a/.credo.exs +++ b/.credo.exs @@ -100,7 +100,7 @@ # {Credo.Check.Refactor.CondStatements, []}, {Credo.Check.Refactor.CyclomaticComplexity, []}, - {Credo.Check.Refactor.FunctionArity, []}, + {Credo.Check.Refactor.FunctionArity, [max_arity: 9]}, {Credo.Check.Refactor.LongQuoteBlocks, []}, {Credo.Check.Refactor.MatchInCondition, []}, {Credo.Check.Refactor.NegatedConditionsInUnless, []}, diff --git a/.gitignore b/.gitignore index ecd29e357..da8c3986e 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ priv/data/* !priv/data/.gitkeep .vscode/ cover/ +test/fixtures/image_tmp.jpg +test/uploads/ uploads/* !uploads/.gitkeep .idea diff --git a/.graphqlconfig.yaml b/.graphqlconfig.yaml index 6058dcaba..e8956f772 100644 --- a/.graphqlconfig.yaml +++ b/.graphqlconfig.yaml @@ -4,5 +4,5 @@ projects: extensions: endpoints: dev: - url: 'http://localhost:4001/api' + url: 'http://localhost:4000/api' introspect: true diff --git a/Dockerfile b/Dockerfile index 8685ad386..a37e586dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM bitwalker/alpine-elixir:latest RUN apk add inotify-tools postgresql-client yarn -RUN apk add --no-cache make gcc libc-dev argon2 +RUN apk add --no-cache make gcc libc-dev argon2 imagemagick RUN mix local.hex --force && mix local.rebar --force diff --git a/config/config.exs b/config/config.exs index 8d65f58c9..db5c38732 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,7 +14,11 @@ config :mobilizon, :instance, description: System.get_env("MOBILIZON_INSTANCE_DESCRIPTION") || "This is a Mobilizon instance", version: "1.0.0-dev", registrations_open: System.get_env("MOBILIZON_INSTANCE_REGISTRATIONS_OPEN") || false, - repository: Mix.Project.config()[:source_url] + repository: Mix.Project.config()[:source_url], + remote_limit: 100_000, + upload_limit: 16_000_000, + avatar_upload_limit: 2_000_000, + banner_upload_limit: 4_000_000 config :mime, :types, %{ "application/activity+json" => ["activity-json"], @@ -31,6 +35,34 @@ config :mobilizon, MobilizonWeb.Endpoint, email_from: "noreply@localhost", email_to: "noreply@localhost" +# Upload configuration +config :mobilizon, MobilizonWeb.Upload, + uploader: MobilizonWeb.Uploaders.Local, + filters: [MobilizonWeb.Upload.Filter.Dedupe], + link_name: true, + proxy_remote: false, + proxy_opts: [ + redirect_on_failure: false, + max_body_length: 25 * 1_048_576, + http: [ + follow_redirect: true, + pool: :upload + ] + ] + +config :mobilizon, MobilizonWeb.Uploaders.Local, uploads: "uploads" + +config :mobilizon, :media_proxy, + enabled: false, + proxy_opts: [ + redirect_on_failure: false, + max_body_length: 25 * 1_048_576, + http: [ + follow_redirect: true, + pool: :media + ] + ] + # Configures Elixir's Logger config :logger, :console, format: "$time $metadata[$level] $message\n", @@ -62,9 +94,6 @@ config :geolix, } ] -config :arc, - storage: Arc.Storage.Local - config :phoenix, :format_encoders, json: Jason, "activity-json": Jason config :mobilizon, Mobilizon.Service.Geospatial.Nominatim, diff --git a/config/dev.exs b/config/dev.exs index 723a4a449..41fb0f730 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -8,11 +8,10 @@ use Mix.Config # with brunch.io to recompile .js and .css sources. config :mobilizon, MobilizonWeb.Endpoint, http: [ - port: System.get_env("MOBILIZON_INSTANCE_PORT") || 4001 + port: System.get_env("MOBILIZON_INSTANCE_PORT") || 4000 ], url: [ - host: System.get_env("MOBILIZON_INSTANCE_HOST") || "mobilizon.local", - port: System.get_env("MOBILIZON_INSTANCE_PORT") || 4001 + host: System.get_env("MOBILIZON_INSTANCE_HOST") || "mobilizon.local" ], debug_errors: true, code_reloader: true, diff --git a/config/prod.exs b/config/prod.exs index ad21683b1..cdbf50a39 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1,28 +1,10 @@ use Mix.Config -# For production, we often load configuration from external -# sources, such as your system environment. For this reason, -# you won't find the :http configuration below, but set inside -# MobilizonWeb.Endpoint.init/2 when load_from_system_env is -# true. Any dynamic configuration should be done there. -# -# Don't forget to configure the url host to something meaningful, -# Phoenix uses this information when generating URLs. -# -# Finally, we also include the path to a cache manifest -# containing the digested version of static files. This -# manifest is generated by the mix phx.digest task -# which you typically run after static files are built. config :mobilizon, MobilizonWeb.Endpoint, - load_from_system_env: true, + http: [:inet6, port: System.get_env("MOBILIZON_INSTANCE_PORT") || 4000], url: [ host: System.get_env("MOBILIZON_INSTANCE_HOST") || "mobilizon.me", - scheme: "https", - port: 443 - ], - http: [ - ip: {127, 0, 0, 1}, - port: System.get_env("MOBILIZON_INSTANCE_PORT") || 4000 + port: 80 ], secret_key_base: System.get_env("MOBILIZON_SECRET") || "ThisShouldBeAVeryStrongStringPleaseReplaceMe", diff --git a/config/test.exs b/config/test.exs index 9e1c31306..21432393b 100644 --- a/config/test.exs +++ b/config/test.exs @@ -33,6 +33,10 @@ config :mobilizon, Mobilizon.Repo, config :mobilizon, Mobilizon.Mailer, adapter: Bamboo.TestAdapter +config :mobilizon, MobilizonWeb.Upload, filters: [], link_name: false + +config :mobilizon, MobilizonWeb.Uploaders.Local, uploads: "test/uploads" + config :exvcr, vcr_cassette_library_dir: "test/fixtures/vcr_cassettes" diff --git a/js/package.json b/js/package.json index 68e15d393..58a014302 100644 --- a/js/package.json +++ b/js/package.json @@ -14,7 +14,7 @@ "dependencies": { "apollo-absinthe-upload-link": "^1.5.0", "apollo-cache-inmemory": "^1.5.1", - "apollo-client": "^2.5.1", + "apollo-client": "2.5.1", "apollo-link": "^1.2.11", "apollo-link-http": "^1.5.14", "apollo-link-state": "^0.4.2", diff --git a/js/src/App.vue b/js/src/App.vue index 1ec44790b..76e35f9d4 100644 --- a/js/src/App.vue +++ b/js/src/App.vue @@ -91,6 +91,7 @@ export default class App extends Vue { @import "~buefy/src/scss/components/form"; @import "~buefy/src/scss/components/modal"; @import "~buefy/src/scss/components/tag"; + @import "~buefy/src/scss/components/upload"; @import "~buefy/src/scss/utils/_all"; .router-enter-active, diff --git a/js/src/components/Account/Identities.vue b/js/src/components/Account/Identities.vue index d1337db96..c81771b1c 100644 --- a/js/src/components/Account/Identities.vue +++ b/js/src/components/Account/Identities.vue @@ -8,8 +8,8 @@
  • -
    - +
    +
    diff --git a/js/src/components/Event/EventCard.vue b/js/src/components/Event/EventCard.vue index e551cbcca..52d565029 100644 --- a/js/src/components/Event/EventCard.vue +++ b/js/src/components/Event/EventCard.vue @@ -2,7 +2,7 @@
    -
    +
    {{ tag.title }}
    diff --git a/js/src/components/Group/GroupCard.vue b/js/src/components/Group/GroupCard.vue index cb8137624..6f6531ee8 100644 --- a/js/src/components/Group/GroupCard.vue +++ b/js/src/components/Group/GroupCard.vue @@ -1,6 +1,6 @@