From 4dff52bbe0d8fab61a6e4e65dbd4fc397bb5a513 Mon Sep 17 00:00:00 2001 From: Thomas Parisot Date: Mon, 21 Dec 2020 16:56:38 +0100 Subject: [PATCH 1/3] Release Elixir and frontend assets as part of the release --- .gitignore | 1 + .gitlab-ci.yml | 80 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2c09e1644..8791c51f5 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ erl_crash.dump # secrets files as long as you replace their contents by environment # variables. /config/*.secret.exs +/config/releases.exs /setup_db.psql diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 73ba72c52..cc6f55fd0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ stages: - test - deploy - docker + - release variables: MIX_ENV: "test" @@ -20,6 +21,11 @@ variables: MOBILIZON_DATABASE_HOST: $POSTGRES_HOST GEOLITE_CITIES_PATH: "/usr/share/GeoIP/GeoLite2-City.mmdb" MOBILIZON_INSTANCE_REGISTRATIONS_OPEN: "true" + # Release elements + PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${CI_PROJECT_NAME}/${PACKAGE_VERSION}" + ELIXIR_ASSET: "${CI_PROJECT_NAME}-standalone-${CI_COMMIT_TAG}.tar.gz" + FRONTEND_ASSET: "${CI_PROJECT_NAME}-frontend-${CI_COMMIT_TAG}.tar.gz" + cache: key: ${CI_COMMIT_REF_SLUG} @@ -81,7 +87,8 @@ exunit: - mix deps.get - MIX_ENV=test mix ecto.create - MIX_ENV=test mix ecto.migrate - dependencies: + needs: + - deps - lint-elixir script: - mix coveralls @@ -170,3 +177,74 @@ build-docker-tag: - tags variables: DOCKER_IMAGE_NAME: framasoft/mobilizon:$CI_COMMIT_TAG + +build-release-frontend: + stage: release + image: node:14-alpine + before_script: + - apk add --no-cache python build-base libwebp-tools bash imagemagick ncurses + script: + - yarn --cwd "js" install + - yarn --cwd "js" run build + artifacts: + expire_in: 1 day + paths: + - priv/static + + +build-release-elixir: + stage: release + image: elixir:alpine + before_script: + - apk add --no-cache build-base git cmake + variables: + MIX_ENV: "prod" + script: + - mix local.hex --force + - mix local.rebar --force + - mix deps.get + - cp docker/production/releases.exs ./config/ + - mix phx.digest + - mix release + needs: + - build-release-frontend + artifacts: + expire_in: 1 day + paths: + - _build/prod/rel + +build-release-upload: + stage: release + image: curlimages/curl:latest + # rules: + # - if: $CI_COMMIT_TAG + script: | + ls -R _build + tar czf /tmp/${ELIXIR_ASSET} -C _build/prod/rel mobilizon + tar czf /tmp/${FRONTEND_ASSET} -C priv/static . + ls -al /tmp/*.tar.gz + + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "_build/${ELIXIR_ASSET}" ${PACKAGE_REGISTRY_URL}/${ELIXIR_ASSET} + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "_build/${FRONTEND_ASSET}" ${PACKAGE_REGISTRY_URL}/${FRONTEND_ASSET} + artifacts: + expire_in: 1 day + when: on_success + paths: + - _build/${ELIXIR_ASSET} + - _build/${FRONTEND_ASSET} + needs: + - build-release-elixir + +build-release-create: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG + script: | + release-cli create --name "Release $CI_COMMIT_SHA" \ + --tag-name "$CI_COMMIT_TAG" \ + --assets-link "{\"name\":\"${ELIXIR_ASSET}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${ELIXIR_ASSET}\"}" \ + --assets-link "{\"name\":\"${FRONTEND_ASSET}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${FRONTEND_ASSET}\"}" + needs: + - build-release-upload + From 49acf2784dc604f18056fe6d3bbae40c8307e3e0 Mon Sep 17 00:00:00 2001 From: Thomas Parisot Date: Wed, 23 Dec 2020 00:01:25 +0100 Subject: [PATCH 2/3] Fasten release process --- .gitlab-ci.yml | 169 +++++++++++++++++++++++-------------------------- 1 file changed, 79 insertions(+), 90 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc6f55fd0..5db7d9c8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,17 @@ image: tcitworld/mobilizon-ci stages: + - install - check + - build - test + - package + - upload - deploy - - docker - - release variables: MIX_ENV: "test" + YARN_CACHE_FOLDER: "js/.yarn" # DB Variables for Postgres / Postgis POSTGRES_DB: mobilizon_test POSTGRES_USER: postgres @@ -22,107 +25,114 @@ variables: GEOLITE_CITIES_PATH: "/usr/share/GeoIP/GeoLite2-City.mmdb" MOBILIZON_INSTANCE_REGISTRATIONS_OPEN: "true" # Release elements - PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${CI_PROJECT_NAME}/${PACKAGE_VERSION}" - ELIXIR_ASSET: "${CI_PROJECT_NAME}-standalone-${CI_COMMIT_TAG}.tar.gz" - FRONTEND_ASSET: "${CI_PROJECT_NAME}-frontend-${CI_COMMIT_TAG}.tar.gz" + PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${CI_PROJECT_NAME}" cache: - key: ${CI_COMMIT_REF_SLUG} + key: "${CI_COMMIT_REF_SLUG}" paths: - ~/.cache/Cypress - - _build/ - - deps/ - - js/node_modules - cache/Cypress + - deps/ + - _build/ + - js/node_modules + - js/.yarn + +# Installed dependencies are cached across the pipeline +# So there is no need to reinstall them all the time +# It saves minutes during a pipeline build time +install: + stage: install + script: + - yarn --cwd "js" install --frozen-lockfile + - mix deps.get + - mix compile lint-elixir: stage: check script: - export EXITVALUE=0 - - mix deps.get - mix credo --strict -a || export EXITVALUE=1 - mix format --check-formatted --dry-run || export EXITVALUE=1 - exit $EXITVALUE lint-front: - image: node:14 stage: check before_script: - export EXITVALUE=0 - cd js script: - - yarn install - yarn run lint || export EXITVALUE=1 - yarn run prettier -c . || export EXITVALUE=1 - - yarn run build:assets || export EXITVALUE=1 - exit $EXITVALUE + +build-frontend: + stage: build + before_script: + - apt update + - apt install -y --no-install-recommends python build-essential webp imagemagick gifsicle jpegoptim optipng pngquant + script: + - yarn --cwd "js" run build artifacts: - expire_in: 1 day - when: on_success + expire_in: 5 days paths: - - priv/static + - priv/static + needs: + - lint-front deps: stage: check script: - export EXITVALUE=0 - - mix deps.get - mix hex.outdated || export EXITVALUE=1 - - cd js - - yarn outdated || export EXITVALUE=1 + - yarn --cwd "js" outdated || export EXITVALUE=1 - exit $EXITVALUE allow_failure: true + needs: + - install exunit: stage: test services: - name: mdillon/postgis:11 alias: postgres + variables: + MIX_ENV: test before_script: - - cd js - - yarn install - - yarn run build:assets - - cd ../ - - mix deps.get - - MIX_ENV=test mix ecto.create - - MIX_ENV=test mix ecto.migrate - needs: - - deps - - lint-elixir + - mix ecto.create + - mix ecto.migrate script: - mix coveralls jest: stage: test - before_script: - - cd js - - yarn install - dependencies: + needs: - lint-front + before_script: + - yarn add --dev jest-junit script: - - yarn run test:unit --no-color + - yarn --cwd "js" run test:unit --no-color --ci --reporters=default --reporters=jest-junit artifacts: when: always paths: - js/coverage + reports: + junit: + - js/junit.xml expire_in: 30 days + # cypress: # stage: test # services: # - name: mdillon/postgis:11 # alias: postgres +# variables: +# MIX_ENV=e2e # script: -# - mix deps.get -# - cd js -# - yarn install -# - npx cypress install # just to be sure -# - yarn run build -# - cd ../ -# - MIX_ENV=e2e mix ecto.create -# - MIX_ENV=e2e mix ecto.migrate -# - MIX_ENV=e2e mix run priv/repo/e2e.seed.exs -# - MIX_ENV=e2e mix phx.server & +# - mix ecto.create +# - mix ecto.migrate +# - mix run priv/repo/e2e.seed.exs +# - mix phx.server & # - cd js # - npx wait-on http://localhost:4000 # - if [ -z "$CYPRESS_KEY" ]; then npx cypress run; else npx cypress run --record --parallel --key $CYPRESS_KEY; fi @@ -138,13 +148,10 @@ jest: # # - mkdir public # # Mobilizon documentation is now on https://framagit.org/framasoft/joinmobilizon/documentation # # Mix docs disabled because of https://github.com/elixir-lang/ex_doc/issues/1172 -# # - mix deps.get # # - mix docs # # - mv doc public/backend -# #- cd js -# #- yarn install -# #- yarn run styleguide:build -# #- mv styleguide ../public/frontend +# #- yarn run --cwd "js" styleguide:build +# #- mv js/styleguide public/frontend # only: # - master # artifacts: @@ -153,7 +160,7 @@ jest: # - public .docker: &docker - stage: docker + stage: build cache: {} image: name: gcr.io/kaniko-project/executor:debug @@ -178,73 +185,55 @@ build-docker-tag: variables: DOCKER_IMAGE_NAME: framasoft/mobilizon:$CI_COMMIT_TAG -build-release-frontend: - stage: release - image: node:14-alpine +package-app: + stage: package before_script: - - apk add --no-cache python build-base libwebp-tools bash imagemagick ncurses - script: - - yarn --cwd "js" install - - yarn --cwd "js" run build - artifacts: - expire_in: 1 day - paths: - - priv/static - - -build-release-elixir: - stage: release - image: elixir:alpine - before_script: - - apk add --no-cache build-base git cmake + - apt update + - apt install -y --no-install-recommends build-essential git cmake variables: MIX_ENV: "prod" script: - mix local.hex --force - mix local.rebar --force - - mix deps.get - cp docker/production/releases.exs ./config/ - mix phx.digest - mix release - needs: - - build-release-frontend artifacts: - expire_in: 1 day + expire_in: 2 days paths: - _build/prod/rel -build-release-upload: - stage: release +release-upload: + stage: upload image: curlimages/curl:latest # rules: # - if: $CI_COMMIT_TAG script: | - ls -R _build - tar czf /tmp/${ELIXIR_ASSET} -C _build/prod/rel mobilizon - tar czf /tmp/${FRONTEND_ASSET} -C priv/static . - ls -al /tmp/*.tar.gz + APP_VERSION="${CI_COMMIT_TAG:-0.0.0}" + APP_ASSET="${CI_PROJECT_NAME}-standalone-${APP_VERSION}.tar.gz" - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "_build/${ELIXIR_ASSET}" ${PACKAGE_REGISTRY_URL}/${ELIXIR_ASSET} - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "_build/${FRONTEND_ASSET}" ${PACKAGE_REGISTRY_URL}/${FRONTEND_ASSET} + echo "Artifact: ${APP_ASSET}" + tar czf ${APP_ASSET} -C _build/prod/rel mobilizon + ls -al ${APP_ASSET} + + curl --silent --show-error --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "${APP_ASSET}" ${PACKAGE_REGISTRY_URL}/${APP_VERSION}/${APP_ASSET} artifacts: expire_in: 1 day when: on_success paths: - - _build/${ELIXIR_ASSET} - - _build/${FRONTEND_ASSET} - needs: - - build-release-elixir + - ${APP_ASSET} -build-release-create: - stage: release +release-create: + stage: deploy image: registry.gitlab.com/gitlab-org/release-cli:latest rules: - if: $CI_COMMIT_TAG + dependencies: [] script: | + APP_VERSION="${CI_COMMIT_TAG}" + APP_ASSET="${CI_PROJECT_NAME}-standalone-${APP_VERSION}.tar.gz" + release-cli create --name "Release $CI_COMMIT_SHA" \ --tag-name "$CI_COMMIT_TAG" \ - --assets-link "{\"name\":\"${ELIXIR_ASSET}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${ELIXIR_ASSET}\"}" \ - --assets-link "{\"name\":\"${FRONTEND_ASSET}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${FRONTEND_ASSET}\"}" - needs: - - build-release-upload + --assets-link "{\"name\":\"${APP_ASSET}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${APP_VERSION}/${APP_ASSET}\"}" From 5c720522582ce3653054fb87a46dc91ac8b91e58 Mon Sep 17 00:00:00 2001 From: Thomas Parisot Date: Wed, 13 Jan 2021 09:05:39 +0100 Subject: [PATCH 3/3] Publish Docker images when tagged on main repo --- .gitlab-ci.yml | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5db7d9c8e..c976eaa3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,10 +26,11 @@ variables: MOBILIZON_INSTANCE_REGISTRATIONS_OPEN: "true" # Release elements PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${CI_PROJECT_NAME}" + ARCH: "amd64" cache: - key: "${CI_COMMIT_REF_SLUG}" + key: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" paths: - ~/.cache/Cypress - cache/Cypress @@ -152,8 +153,8 @@ jest: # # - mv doc public/backend # #- yarn run --cwd "js" styleguide:build # #- mv js/styleguide public/frontend -# only: -# - master +# rules: +# - if: '$CI_COMMIT_BRANCH == "master"' # artifacts: # expire_in: 1 hour # paths: @@ -173,15 +174,19 @@ jest: build-docker-master: <<: *docker - only: - - schedules + rules: + - if: '$CI_PROJECT_NAMESPACE != "framasoft"' + when: never + - if: '$CI_PIPELINE_SOURCE == "schedule"' variables: DOCKER_IMAGE_NAME: framasoft/mobilizon:master build-docker-tag: <<: *docker - only: - - tags + rules: + - if: '$CI_PROJECT_NAMESPACE != "framasoft"' + when: never + - if: $CI_COMMIT_TAG variables: DOCKER_IMAGE_NAME: framasoft/mobilizon:$CI_COMMIT_TAG @@ -206,11 +211,11 @@ package-app: release-upload: stage: upload image: curlimages/curl:latest - # rules: - # - if: $CI_COMMIT_TAG + rules: + - if: $CI_COMMIT_TAG script: | - APP_VERSION="${CI_COMMIT_TAG:-0.0.0}" - APP_ASSET="${CI_PROJECT_NAME}-standalone-${APP_VERSION}.tar.gz" + APP_VERSION="${CI_COMMIT_TAG}" + APP_ASSET="${CI_PROJECT_NAME}_${APP_VERSION}_${ARCH}.tar.gz" echo "Artifact: ${APP_ASSET}" tar czf ${APP_ASSET} -C _build/prod/rel mobilizon @@ -221,7 +226,7 @@ release-upload: expire_in: 1 day when: on_success paths: - - ${APP_ASSET} + - mobilizon_*.tar.gz release-create: stage: deploy @@ -229,11 +234,12 @@ release-create: rules: - if: $CI_COMMIT_TAG dependencies: [] + cache: {} script: | APP_VERSION="${CI_COMMIT_TAG}" - APP_ASSET="${CI_PROJECT_NAME}-standalone-${APP_VERSION}.tar.gz" + APP_ASSET="${CI_PROJECT_NAME}_${APP_VERSION}_${ARCH}.tar.gz" - release-cli create --name "Release $CI_COMMIT_SHA" \ + release-cli create --name "$CI_PROJECT_TITLE v$CI_COMMIT_TAG" \ --tag-name "$CI_COMMIT_TAG" \ --assets-link "{\"name\":\"${APP_ASSET}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${APP_VERSION}/${APP_ASSET}\"}"