From 8d0d1dcbb2c53fdfaea7d70fbac25547826cff21 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Thu, 26 Nov 2020 11:41:13 +0100 Subject: [PATCH 1/2] Track usage of media files and add a job to clean them Signed-off-by: Thomas Citharel --- config/config.exs | 4 + js/src/components/Editor.vue | 18 +- js/src/components/Editor/Image.ts | 14 +- js/src/components/PictureUpload.vue | 6 +- js/src/graphql/actor.ts | 4 +- js/src/graphql/event.ts | 4 +- js/src/graphql/group.ts | 8 +- js/src/graphql/post.ts | 4 +- js/src/graphql/upload.ts | 12 +- js/src/types/actor/actor.model.ts | 10 +- js/src/types/event.model.ts | 8 +- .../{picture.model.ts => media.model.ts} | 4 +- js/src/types/post.model.ts | 4 +- js/src/utils/image.ts | 6 +- .../views/Account/children/EditIdentity.vue | 3 +- js/src/views/Event/Edit.vue | 8 +- js/src/views/Group/GroupSettings.vue | 4 +- js/src/views/Posts/Edit.vue | 8 +- lib/federation/activity_pub/utils.ex | 42 ++-- .../activity_stream/converter/event.ex | 10 +- .../activity_stream/converter/media.ex | 63 ++++++ .../activity_stream/converter/picture.ex | 63 ------ lib/graphql/api/comments.ex | 35 +++- lib/graphql/api/events.ex | 24 ++- lib/graphql/api/utils.ex | 40 +++- lib/graphql/resolvers/discussion.ex | 41 ++-- lib/graphql/resolvers/group.ex | 4 +- .../resolvers/{picture.ex => media.ex} | 97 ++++----- lib/graphql/resolvers/person.ex | 8 +- lib/graphql/resolvers/post.ex | 18 +- lib/graphql/resolvers/user.ex | 2 +- lib/graphql/schema.ex | 6 +- lib/graphql/schema/actor.ex | 4 +- lib/graphql/schema/actors/application.ex | 8 +- lib/graphql/schema/actors/group.ex | 24 +-- lib/graphql/schema/actors/person.ex | 32 +-- lib/graphql/schema/address.ex | 1 - lib/graphql/schema/event.ex | 19 +- lib/graphql/schema/media.ex | 68 +++++++ lib/graphql/schema/picture.ex | 68 ------- lib/graphql/schema/post.ex | 16 +- lib/graphql/schema/user.ex | 6 +- lib/mix/tasks/mobilizon/common.ex | 2 + lib/mix/tasks/mobilizon/maintenance.ex | 23 +++ .../fix_unattached_media_in_body.ex | 107 ++++++++++ lib/mix/tasks/mobilizon/media.ex | 23 +++ lib/mix/tasks/mobilizon/media/clean_orphan.ex | 87 +++++++++ lib/mobilizon.ex | 6 +- lib/mobilizon/actors/actor.ex | 2 +- lib/mobilizon/actors/actors.ex | 7 +- lib/mobilizon/discussions/comment.ex | 4 + lib/mobilizon/events/event.ex | 17 +- lib/mobilizon/events/events.ex | 5 +- lib/mobilizon/media/media.ex | 150 -------------- lib/mobilizon/media/picture.ex | 32 --- lib/mobilizon/{media => medias}/file.ex | 2 +- lib/mobilizon/medias/media.ex | 40 ++++ lib/mobilizon/medias/medias.ex | 184 ++++++++++++++++++ lib/mobilizon/posts/post.ex | 13 +- lib/mobilizon/posts/posts.ex | 2 +- lib/service/clean_orphan_media.ex | 60 ++++++ .../workers/clean_orphan_media_worker.ex | 31 +++ lib/web/plugs/uploaded_media.ex | 5 +- lib/web/upload/filter/optimize.ex | 2 +- mix.exs | 12 +- .../20201124094655_add_media_tables.exs | 22 +++ test/graphql/resolvers/event_test.exs | 28 +-- test/graphql/resolvers/group_test.exs | 4 +- .../{picture_test.exs => media_test.exs} | 158 +++++++-------- test/graphql/resolvers/person_test.exs | 4 +- test/mobilizon/actors/actors_test.exs | 2 +- test/mobilizon/discussions_test.exs | 3 +- test/mobilizon/events/events_test.exs | 2 +- test/mobilizon/media/media_test.exs | 41 ++-- test/mobilizon/posts_test.exs | 2 +- test/service/clean_orphan_media_test.exs | 56 ++++++ test/support/factory.ex | 15 +- test/tasks/media/clean_orphan_test.exs | 124 ++++++++++++ 78 files changed, 1405 insertions(+), 700 deletions(-) rename js/src/types/{picture.model.ts => media.model.ts} (65%) create mode 100644 lib/federation/activity_stream/converter/media.ex delete mode 100644 lib/federation/activity_stream/converter/picture.ex rename lib/graphql/resolvers/{picture.ex => media.ex} (56%) create mode 100644 lib/graphql/schema/media.ex delete mode 100644 lib/graphql/schema/picture.ex create mode 100644 lib/mix/tasks/mobilizon/maintenance.ex create mode 100644 lib/mix/tasks/mobilizon/maintenance/fix_unattached_media_in_body.ex create mode 100644 lib/mix/tasks/mobilizon/media.ex create mode 100644 lib/mix/tasks/mobilizon/media/clean_orphan.ex delete mode 100644 lib/mobilizon/media/media.ex delete mode 100644 lib/mobilizon/media/picture.ex rename lib/mobilizon/{media => medias}/file.ex (95%) create mode 100644 lib/mobilizon/medias/media.ex create mode 100644 lib/mobilizon/medias/medias.ex create mode 100644 lib/service/clean_orphan_media.ex create mode 100644 lib/service/workers/clean_orphan_media_worker.ex create mode 100644 priv/repo/migrations/20201124094655_add_media_tables.exs rename test/graphql/resolvers/{picture_test.exs => media_test.exs} (67%) create mode 100644 test/service/clean_orphan_media_test.exs create mode 100644 test/tasks/media/clean_orphan_test.exs diff --git a/config/config.exs b/config/config.exs index 6145a5c89..f9c50f8bc 100644 --- a/config/config.exs +++ b/config/config.exs @@ -28,6 +28,8 @@ config :mobilizon, :instance, upload_limit: 10_000_000, avatar_upload_limit: 2_000_000, banner_upload_limit: 4_000_000, + remove_orphan_uploads: true, + orphan_upload_grace_period_hours: 48, email_from: "noreply@localhost", email_reply_to: "noreply@localhost" @@ -250,6 +252,8 @@ config :mobilizon, Oban, crontab: [ {"@hourly", Mobilizon.Service.Workers.BuildSiteMap, queue: :background}, {"17 * * * *", Mobilizon.Service.Workers.RefreshGroups, queue: :background} + # To be activated in Mobilizon 1.2 + # {"@hourly", Mobilizon.Service.Workers.CleanOrphanMediaWorker, queue: :background} ] config :mobilizon, :rich_media, diff --git a/js/src/components/Editor.vue b/js/src/components/Editor.vue index b17fd5089..8117e5526 100644 --- a/js/src/components/Editor.vue +++ b/js/src/components/Editor.vue @@ -212,7 +212,7 @@ import { SEARCH_PERSONS } from "../graphql/search"; import { Actor, IActor, IPerson } from "../types/actor"; import Image from "./Editor/Image"; import MaxSize from "./Editor/MaxSize"; -import { UPLOAD_PICTURE } from "../graphql/upload"; +import { UPLOAD_MEDIA } from "../graphql/upload"; import { listenFileUpload } from "../utils/upload"; import { CURRENT_ACTOR_CLIENT } from "../graphql/actor"; import { IComment } from "../types/comment.model"; @@ -395,7 +395,15 @@ export default class EditorComponent extends Vue { new Image(), new MaxSize({ maxSize: this.maxSize }), ], - onUpdate: ({ getHTML }: { getHTML: Function }) => { + onUpdate: ({ + getHTML, + transaction, + getJSON, + }: { + getHTML: Function; + getJSON: Function; + transaction: unknown; + }) => { this.$emit("input", getHTML()); }, }); @@ -526,14 +534,14 @@ export default class EditorComponent extends Vue { const image = await listenFileUpload(); try { const { data } = await this.$apollo.mutate({ - mutation: UPLOAD_PICTURE, + mutation: UPLOAD_MEDIA, variables: { file: image, name: image.name, }, }); - if (data.uploadPicture && data.uploadPicture.url) { - command({ src: data.uploadPicture.url }); + if (data.uploadMedia && data.uploadMedia.url) { + command({ src: data.uploadMedia.url, "data-media-id": data.uploadMedia.id }); } } catch (error) { console.error(error); diff --git a/js/src/components/Editor/Image.ts b/js/src/components/Editor/Image.ts index 83f984a05..dccce902f 100644 --- a/js/src/components/Editor/Image.ts +++ b/js/src/components/Editor/Image.ts @@ -1,6 +1,7 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-nocheck import { Node } from "tiptap"; -import { UPLOAD_PICTURE } from "@/graphql/upload"; +import { UPLOAD_MEDIA } from "@/graphql/upload"; import apolloProvider from "@/vue-apollo"; import ApolloClient from "apollo-client"; import { NormalizedCacheObject } from "apollo-cache-inmemory"; @@ -27,16 +28,18 @@ export default class Image extends Node { title: { default: null, }, + "data-media-id": {}, }, group: "inline", draggable: true, parseDOM: [ { - tag: "img[src]", + tag: "img", getAttrs: (dom: any) => ({ src: dom.getAttribute("src"), title: dom.getAttribute("title"), alt: dom.getAttribute("alt"), + "data-media-id": dom.getAttribute("data-media-id"), }), }, ], @@ -92,13 +95,16 @@ export default class Image extends Node { try { images.forEach(async (image) => { const { data } = await client.mutate({ - mutation: UPLOAD_PICTURE, + mutation: UPLOAD_MEDIA, variables: { file: image, name: image.name, }, }); - const node = schema.nodes.image.create({ src: data.uploadPicture.url }); + const node = schema.nodes.image.create({ + src: data.uploadMedia.url, + "data-media-id": data.uploadMedia.id, + }); const transaction = view.state.tr.insert(coordinates.pos, node); view.dispatch(transaction); }); diff --git a/js/src/components/PictureUpload.vue b/js/src/components/PictureUpload.vue index 64843b69d..593304a66 100644 --- a/js/src/components/PictureUpload.vue +++ b/js/src/components/PictureUpload.vue @@ -60,14 +60,14 @@ figure.image {