diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d824449c..78a5d0a7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 stages:
   - check
   - build
+  - test
   - deploy
 
 variables:
@@ -19,6 +20,8 @@ workflow:
 
 include:
   - local: .gitlab-ci/run_checks.yml
+    rules:
+      - if: $CI_COMMIT_TAG == null
   - local: .gitlab-ci/build.yml
     rules:
       - if: $CI_COMMIT_TAG == null
diff --git a/.gitlab-ci/create_release.yml b/.gitlab-ci/create_release.yml
index f2bb440a..37d1798e 100644
--- a/.gitlab-ci/create_release.yml
+++ b/.gitlab-ci/create_release.yml
@@ -1,67 +1,72 @@
-# Generate a tarball, publish it to the package registry and create a release
+# Generate a tarball, test it, then publish it to the package registry and create a release
 
 variables:
   PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/release-tarballs/${CI_COMMIT_TAG}"
   RELEASE_TARBALL: "fractal-${CI_COMMIT_TAG}.tar.xz"
   RELEASE_TARBALL_SHA: "fractal-${CI_COMMIT_TAG}.tar.xz.sha256sum"
+  METAINFO_PATH: "data/org.gnome.Fractal.metainfo.xml.in.in"
+  TARBALL_TEST_MANIFEST_PATH: ".gitlab-ci/org.gnome.Fractal.CiTest.json"
 
 release-tarball:
-  stage: deploy
+  stage: build
   image: 'quay.io/gnome_infrastructure/gnome-runtime-images:gnome-master'
   variables:
-    MANIFEST_PATH: "build-aux/org.gnome.Fractal.json"
-    APP_ID: "org.gnome.Fractal"
-    CONFIG_OPTS: "-Dprofile=default"
+    LANG: "C.UTF-8"
   tags:
     - flatpak
   script:
-    # Create flatpak manifest for release
-    - sed 's/org.gnome.Fractal.Devel/org.gnome.Fractal/g' build-aux/org.gnome.Fractal.Devel.json > ${MANIFEST_PATH}
-    - rewrite-flatpak-manifest ${MANIFEST_PATH} ${FLATPAK_MODULE} ${CONFIG_OPTS}
-
-    # Build tarball with vendored rust dependencies
+    # Create tarball with vendored rust dependencies
     - flatpak-builder --keep-build-dirs --user --disable-rofiles-fuse --stop-at=${FLATPAK_MODULE} flatpak_app --repo=repo ${BRANCH:+--default-branch=$BRANCH} ${MANIFEST_PATH}
-    # FIXME: We need --allow-dirty since the manifest is created in place
-    - echo "meson dist --allow-dirty" | flatpak-builder --disable-rofiles-fuse --build-shell=${FLATPAK_MODULE} flatpak_app ${MANIFEST_PATH}
+    - echo "meson dist --no-test" | flatpak-builder --disable-rofiles-fuse --build-shell=${FLATPAK_MODULE} flatpak_app ${MANIFEST_PATH}
     - mv .flatpak-builder/build/${FLATPAK_MODULE}/_flatpak_build/meson-dist tarball
 
-    # Upload tarball
-    - |
-      curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file tarball/${RELEASE_TARBALL} "${PACKAGE_REGISTRY_URL}/${RELEASE_TARBALL}"
-    - |
-      curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file tarball/${RELEASE_TARBALL_SHA} "${PACKAGE_REGISTRY_URL}/${RELEASE_TARBALL_SHA}"
-
-    # Extract release notes from appstream
-    # FIXME: Appstream requires the files name to be `*.metainfo.xml`
-    - cp data/${APP_ID}.metainfo.xml.in.in data/${APP_ID}.metainfo.xml
-    - appstream-util appdata-to-news "data/${APP_ID}.metainfo.xml" |
-      awk '{ x[NR] = $0 } END { for ( i=5 ; i<=NR ; i++ ) { if (x[i] ~ /^$/ && x[i+1] ~ /^Version/ && x[i+2] ~ /^=/ && x[i+3] ~ /^Release/) exit; else print x[i]; }}' >
-      release_notes.txt
-  dependencies: []
+    # Extract release notes of latest version from appstream
+    - appstreamcli metainfo-to-news --format markdown ${METAINFO_PATH} - |
+      awk '{ x[NR] = $0 } END { for ( i=5 ; i<=NR ; i++ ) { if (x[i] ~ /^$/ && x[i+1] ~ /^Version/ && x[i+2] ~ /^-/ && x[i+3] ~ /^Released:/) exit; else print x[i]; }}' >
+      release_notes.md
   artifacts:
     paths:
     - tarball
-    - release_notes.txt
+    - release_notes.md
+
+test-tarball:
+  stage: test
+  image: 'quay.io/gnome_infrastructure/gnome-runtime-images:gnome-45'
+  tags:
+    - flatpak
+  script:
+    - TARBALL_SHA=$(cut -f 1 -d " " tarball/${RELEASE_TARBALL_SHA})
+    - sed -i "s|%%TARBALL_PATH%%|${PWD}/tarball/${RELEASE_TARBALL}|g" ${TARBALL_TEST_MANIFEST_PATH}
+    - sed -i "s|%%TARBALL_SHA%%|${TARBALL_SHA}|g" ${TARBALL_TEST_MANIFEST_PATH}
+    - flatpak-builder --user --disable-rofiles-fuse --sandbox flatpak_app --repo=repo ${BRANCH:+--default-branch=$BRANCH} ${TARBALL_TEST_MANIFEST_PATH}
 
 # https://gitlab.com/gitlab-org/gitlab/-/issues/271534 The default URL needs authentication right now, which is less than optimal...
 # Get the package_files ID instead and use that as download URL in the "release" step.
-release:
+create-release:
   stage: deploy
   image: registry.gitlab.com/gitlab-org/release-cli:latest
   script:
     - apk add curl jq
+
+    # Upload tarball to package registry
+    - |
+      curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file tarball/${RELEASE_TARBALL} "${PACKAGE_REGISTRY_URL}/${RELEASE_TARBALL}"
+    - |
+      curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file tarball/${RELEASE_TARBALL_SHA} "${PACKAGE_REGISTRY_URL}/${RELEASE_TARBALL_SHA}"
+
+    # Get package IDs and build URLs
     - |
       export PACKAGE_ID=$(curl "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages?sort=desc" | jq -c .[0].id)
     - |
       export RELEASE_TARBALL_DOWNLOAD_ID=$(curl "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${PACKAGE_ID}/package_files" | jq -c .[0].id)
       export RELEASE_TARBALL_SHA_DOWNLOAD_ID=$(curl "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${PACKAGE_ID}/package_files" | jq -c .[1].id)
-
     - export RELEASE_TARBALL_URL="${CI_PROJECT_URL}/-/package_files/$RELEASE_TARBALL_DOWNLOAD_ID/download"
     - export RELEASE_TARBALL_SHA_URL="${CI_PROJECT_URL}/-/package_files/$RELEASE_TARBALL_SHA_DOWNLOAD_ID/download"
-    - cat release_notes.txt
+    - cat release_notes.md
+
+    # Create release
     - |
-      release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
+      release-cli create --name "$CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
         --assets-link "{\"name\":\"Tarball for ${CI_COMMIT_TAG}\",\"url\":\"${RELEASE_TARBALL_URL}\", \"filepath\": \"/tarball/${RELEASE_TARBALL}\"}" \
         --assets-link "{\"name\":\"Checksum for ${CI_COMMIT_TAG}\",\"url\":\"${RELEASE_TARBALL_SHA_URL}\", \"filepath\": \"/tarball/${RELEASE_TARBALL_SHA}\"}" \
-        --description release_notes.txt
-  needs: ['release-tarball']
+        --description release_notes.md
diff --git a/.gitlab-ci/org.gnome.Fractal.CiTest.json b/.gitlab-ci/org.gnome.Fractal.CiTest.json
new file mode 100644
index 00000000..4f92d565
--- /dev/null
+++ b/.gitlab-ci/org.gnome.Fractal.CiTest.json
@@ -0,0 +1,53 @@
+{
+    "app-id": "org.gnome.Fractal",
+    "runtime": "org.gnome.Platform",
+    "runtime-version": "45",
+    "sdk": "org.gnome.Sdk",
+    "sdk-extensions": [
+        "org.freedesktop.Sdk.Extension.rust-stable",
+        "org.freedesktop.Sdk.Extension.llvm16"
+    ],
+    "command": "fractal",
+    "finish-args": [
+        "--share=network",
+        "--share=ipc",
+        "--socket=fallback-x11",
+        "--socket=wayland",
+        "--socket=pulseaudio",
+        "--device=dri",
+        "--env=RUST_LOG=fractal=info,warn"
+    ],
+    "build-options": {
+        "append-ld-library-path": "/usr/lib/sdk/llvm16/lib",
+        "append-path": "/usr/lib/sdk/llvm16/bin:/usr/lib/sdk/rust-stable/bin"
+    },
+    "modules": [
+        {
+            "name": "libshumate",
+            "buildsystem": "meson",
+            "config-opts": [
+                "-Dgir=false",
+                "-Dvapi=false",
+                "-Dgtk_doc=false"
+            ],
+            "sources": [
+                {
+                    "type": "archive",
+                    "url": "https://download.gnome.org/sources/libshumate/1.1/libshumate-1.1.0.tar.xz",
+                    "sha256": "b3b7c4c2d3002f37b951c9ca99b09275d0aa82265c69f53374ce12faefd3b14e"
+                }
+            ]
+        },
+        {
+            "name": "fractal",
+            "buildsystem": "meson",
+            "sources": [
+                {
+                    "type": "archive",
+                    "path": "%%TARBALL_PATH%%",
+                    "sha256": "%%TARBALL_SHA%%"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file