Merge tag 'v4.2.0-beta3' into lets-bump-hometown-to-mastodon-4.2

.devcontainer/Dockerfile

@@ -4,10 +4,6 @@ FROM mcr.microsoft.com/devcontainers/ruby:1-3.2-bullseye
 # Install Rails
 # RUN gem install rails webdrivers
 
-# Default value to allow debug server to serve content over GitHub Codespace's port forwarding service
-# The value is a comma-separated list of allowed domains
-ENV RAILS_DEVELOPMENT_HOSTS=".githubpreview.dev,.preview.app.github.dev,.app.github.dev"
-
 ARG NODE_VERSION="16"
 RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 

.devcontainer/codespaces/devcontainer.json

@@ -0,0 +1,49 @@
+{
+  "name": "Mastodon on GitHub Codespaces",
+  "dockerComposeFile": "../docker-compose.yml",
+  "service": "app",
+  "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+
+  "features": {
+    "ghcr.io/devcontainers/features/sshd:1": {}
+  },
+
+  "runServices": ["app", "db", "redis"],
+
+  "forwardPorts": [3000, 4000],
+
+  "portsAttributes": {
+    "3000": {
+      "label": "web",
+      "onAutoForward": "notify"
+    },
+    "4000": {
+      "label": "stream",
+      "onAutoForward": "silent"
+    }
+  },
+
+  "otherPortsAttributes": {
+    "onAutoForward": "silent"
+  },
+
+  "remoteEnv": {
+    "LOCAL_DOMAIN": "${localEnv:CODESPACE_NAME}-3000.app.github.dev",
+    "LOCAL_HTTPS": "true",
+    "STREAMING_API_BASE_URL": "https://${localEnv:CODESPACE_NAME}-4000.app.github.dev",
+    "DISABLE_FORGERY_REQUEST_PROTECTION": "true",
+    "ES_ENABLED": "",
+    "LIBRE_TRANSLATE_ENDPOINT": ""
+  },
+
+  "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
+  "postCreateCommand": ".devcontainer/post-create.sh",
+  "waitFor": "postCreateCommand",
+
+  "customizations": {
+    "vscode": {
+      "settings": {},
+      "extensions": ["EditorConfig.EditorConfig", "webben.browserslist"]
+    }
+  }
+}

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "name": "Mastodon",
+  "name": "Mastodon on local machine",
   "dockerComposeFile": "docker-compose.yml",
   "service": "app",
   "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
@@ -8,13 +8,23 @@
     "ghcr.io/devcontainers/features/sshd:1": {}
   },
 
-  "runServices": ["app", "db", "redis"],
-
   "forwardPorts": [3000, 4000],
 
-  "containerEnv": {
-    "ES_ENABLED": "",
-    "LIBRE_TRANSLATE_ENDPOINT": ""
+  "portsAttributes": {
+    "3000": {
+      "label": "web",
+      "onAutoForward": "notify",
+      "requireLocalPort": true
+    },
+    "4000": {
+      "label": "stream",
+      "onAutoForward": "silent",
+      "requireLocalPort": true
+    }
+  },
+
+  "otherPortsAttributes": {
+    "onAutoForward": "silent"
   },
 
   "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",

.devcontainer/docker-compose.yml

@@ -25,6 +25,7 @@ services:
     command: sleep infinity
     ports:
       - '127.0.0.1:3000:3000'
+      - '127.0.0.1:3035:3035'
       - '127.0.0.1:4000:4000'
     networks:
       - external_network

.env.vagrant

@@ -2,3 +2,7 @@ VAGRANT=true
 LOCAL_DOMAIN=mastodon.local
 BIND=0.0.0.0
 DB_HOST=/var/run/postgresql/
+
+ES_ENABLED=true
+ES_HOST=localhost
+ES_PORT=9200

.github/workflows/build-container-image.yml

@@ -8,7 +8,9 @@ on:
         type: boolean
       push_to_images:
         type: string
-      version_suffix:
+      version_prerelease:
+        type: string
+      version_metadata:
         type: string
       flavor:
         type: string
@@ -74,8 +76,6 @@ jobs:
         if: ${{ inputs.push_to_images != '' }}
         with:
           images: ${{ inputs.push_to_images }}
-          # Only tag with latest when ran against the latest stable branch
-          # This needs to be updated after each minor version release
           flavor: ${{ inputs.flavor }}
           tags: ${{ inputs.tags }}
           labels: ${{ inputs.labels }}
@@ -83,7 +83,9 @@ jobs:
       - uses: docker/build-push-action@v4
         with:
           context: .
-          build-args: MASTODON_VERSION_SUFFIX=${{ inputs.version_suffix }}
+          build-args: |
+            MASTODON_VERSION_PRERELEASE=${{ inputs.version_prerelease }}
+            MASTODON_VERSION_METADATA=${{ inputs.version_metadata }}
           platforms: ${{ inputs.platforms }}
           provenance: false
           builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }}

.github/workflows/build-nightly.yml

@@ -16,9 +16,9 @@ jobs:
         env:
           TZ: Etc/UTC
         run: |
-          echo mastodon_version_suffix=nightly-$(date +'%Y-%m-%d')>> $GITHUB_OUTPUT
+          echo mastodon_version_prerelease=nightly.$(date +'%Y-%m-%d')>> $GITHUB_OUTPUT
     outputs:
-      suffix: ${{ steps.version_vars.outputs.mastodon_version_suffix }}
+      prerelease: ${{ steps.version_vars.outputs.mastodon_version_prerelease }}
 
   build-image:
     needs: compute-suffix
@@ -29,8 +29,7 @@ jobs:
       push_to_images: |
         tootsuite/mastodon
         ghcr.io/mastodon/mastodon
-      # The `+` is important here, result will be v4.1.2+nightly-2022-03-05
-      version_suffix: +${{ needs.compute-suffix.outputs.suffix }}
+      version_prerelease: ${{ needs.compute-suffix.outputs.prerelease }}
       labels: |
         org.opencontainers.image.description=Nightly build image used for testing purposes
       flavor: |
@@ -38,5 +37,5 @@ jobs:
       tags: |
         type=raw,value=edge
         type=raw,value=nightly
-        type=schedule,pattern=${{ needs.compute-suffix.outputs.suffix }}
+        type=schedule,pattern=${{ needs.compute-suffix.outputs.prerelease }}
     secrets: inherit

.github/workflows/build-push-pr.yml

@@ -21,9 +21,9 @@ jobs:
         uses: actions/checkout@v3
       - id: version_vars
         run: |
-          echo mastodon_version_suffix=+pr-${{ github.event.pull_request.number }}-$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
+          echo mastodon_version_metadata=pr-${{ github.event.pull_request.number }}-$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
     outputs:
-      suffix: ${{ steps.version_vars.outputs.mastodon_version_suffix }}
+      metadata: ${{ steps.version_vars.outputs.mastodon_version_metadata }}
 
   build-image:
     needs: compute-suffix
@@ -33,7 +33,7 @@ jobs:
       use_native_arm64_builder: true
       push_to_images: |
         ghcr.io/mastodon/mastodon
-      version_suffix: ${{ needs.compute-suffix.outputs.suffix }}
+      version_metadata: ${{ needs.compute-suffix.outputs.metadata }}
       flavor: |
         latest=auto
       tags: |

.github/workflows/build-releases.yml

@@ -17,6 +17,8 @@ jobs:
       push_to_images: |
         tootsuite/mastodon
         ghcr.io/mastodon/mastodon
+      # Only tag with latest when ran against the latest stable branch
+      # This needs to be updated after each minor version release
       flavor: |
         latest=${{ startsWith(github.ref, 'refs/tags/v4.1.') }}
       tags: |

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config --auto-gen-only-exclude --no-exclude-limit --no-offense-counts --no-auto-gen-timestamp`
-# using RuboCop version 1.54.2.
+# using RuboCop version 1.56.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -61,38 +61,8 @@ Lint/EmptyBlock:
     - 'spec/fabricators/access_token_fabricator.rb'
     - 'spec/fabricators/conversation_fabricator.rb'
     - 'spec/fabricators/system_key_fabricator.rb'
-    - 'spec/helpers/admin/action_logs_helper_spec.rb'
     - 'spec/lib/activitypub/adapter_spec.rb'
-    - 'spec/models/account_alias_spec.rb'
-    - 'spec/models/account_deletion_request_spec.rb'
-    - 'spec/models/account_moderation_note_spec.rb'
-    - 'spec/models/announcement_mute_spec.rb'
-    - 'spec/models/announcement_reaction_spec.rb'
-    - 'spec/models/announcement_spec.rb'
-    - 'spec/models/backup_spec.rb'
-    - 'spec/models/conversation_mute_spec.rb'
-    - 'spec/models/custom_filter_keyword_spec.rb'
-    - 'spec/models/custom_filter_spec.rb'
-    - 'spec/models/device_spec.rb'
-    - 'spec/models/encrypted_message_spec.rb'
-    - 'spec/models/featured_tag_spec.rb'
-    - 'spec/models/follow_recommendation_suppression_spec.rb'
-    - 'spec/models/list_account_spec.rb'
-    - 'spec/models/list_spec.rb'
-    - 'spec/models/login_activity_spec.rb'
-    - 'spec/models/mute_spec.rb'
-    - 'spec/models/preview_card_spec.rb'
-    - 'spec/models/preview_card_trend_spec.rb'
-    - 'spec/models/relay_spec.rb'
-    - 'spec/models/scheduled_status_spec.rb'
-    - 'spec/models/status_stat_spec.rb'
-    - 'spec/models/status_trend_spec.rb'
-    - 'spec/models/system_key_spec.rb'
-    - 'spec/models/tag_follow_spec.rb'
-    - 'spec/models/unavailable_domain_spec.rb'
-    - 'spec/models/user_invite_request_spec.rb'
     - 'spec/models/user_role_spec.rb'
-    - 'spec/models/web/setting_spec.rb'
 
 Lint/NonLocalExitFromIterator:
   Exclude:
@@ -135,7 +105,7 @@ Lint/UselessAssignment:
 
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
 Metrics/AbcSize:
-  Max: 146
+  Max: 144
 
 # Configuration parameters: CountBlocks, Max.
 Metrics/BlockNesting:
@@ -164,6 +134,19 @@ Naming/VariableNumber:
     - 'spec/models/domain_block_spec.rb'
     - 'spec/models/user_spec.rb'
 
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: SafeMultiline.
+Performance/DeletePrefix:
+  Exclude:
+    - 'app/models/featured_tag.rb'
+
+Performance/MapMethodChain:
+  Exclude:
+    - 'app/models/feed.rb'
+    - 'lib/mastodon/cli/maintenance.rb'
+    - 'spec/services/bulk_import_service_spec.rb'
+    - 'spec/services/import_service_spec.rb'
+
 RSpec/AnyInstance:
   Exclude:
     - 'spec/controllers/activitypub/inboxes_controller_spec.rb'
@@ -762,6 +745,15 @@ Style/RedundantFetchBlock:
     - 'config/initializers/paperclip.rb'
     - 'config/puma.rb'
 
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowMultipleReturnValues.
+Style/RedundantReturn:
+  Exclude:
+    - 'app/controllers/api/v1/directories_controller.rb'
+    - 'app/controllers/auth/confirmations_controller.rb'
+    - 'app/lib/ostatus/tag_manager.rb'
+    - 'app/models/form/import.rb'
+
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: ConvertCodeThatCanStartToReturnNil, AllowedMethods, MaxChainLength.
 # AllowedMethods: present?, blank?, presence, try, try!

CHANGELOG.md

@@ -8,11 +8,18 @@ The following changelog entries focus on changes visible to users, administrator
 
 ### Added
 
+- **Add full-text search of opted-in public posts and rework search operators** ([Gargron](https://github.com/mastodon/mastodon/pull/26485), [jsgoldstein](https://github.com/mastodon/mastodon/pull/26344), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26657), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26650), [jsgoldstein](https://github.com/mastodon/mastodon/pull/26659), [Gargron](https://github.com/mastodon/mastodon/pull/26660), [Gargron](https://github.com/mastodon/mastodon/pull/26663), [Gargron](https://github.com/mastodon/mastodon/pull/26688), [Gargron](https://github.com/mastodon/mastodon/pull/26689), [Gargron](https://github.com/mastodon/mastodon/pull/26686), [Gargron](https://github.com/mastodon/mastodon/pull/26687), [Gargron](https://github.com/mastodon/mastodon/pull/26692), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26697), [Gargron](https://github.com/mastodon/mastodon/pull/26699), [Gargron](https://github.com/mastodon/mastodon/pull/26701), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26710), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26739), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26754), [Gargron](https://github.com/mastodon/mastodon/pull/26662), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26755), [Gargron](https://github.com/mastodon/mastodon/pull/26781), [Gargron](https://github.com/mastodon/mastodon/pull/26782), [Gargron](https://github.com/mastodon/mastodon/pull/26760))
+  This introduces a new `public_statuses` Elasticsearch index for public posts by users who have opted in to their posts being searchable (`toot#indexable` flag).
+  This also revisits the other indexes to provide more useful indexing, and adds new search operators such as `from:me`, `before:2022-11-01`, `after:2022-11-01`, `during:2022-11-01`, `language:fr`, `has:poll`, or `in:library` (for searching only in posts you have written or interacted with).
+  Results are now ordered chronologically.
+- **Add admin notifications for new Mastodon versions** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26582))
+  This is done by querying `https://api.joinmastodon.org/update-check` every 30 minutes in a background job.
+  That URL can be changed using the `UPDATE_CHECK_URL` environment variable, and the feature outright disabled by setting that variable to an empty string (`UPDATE_CHECK_URL=`).
 - **Add “Privacy and reach” tab in profile settings** ([Gargron](https://github.com/mastodon/mastodon/pull/26484), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26508))
   This reorganized scattered privacy and reach settings to a single place, as well as improve their wording.
-- **Add display of out-of-band hashtags in the web interface** ([Gargron](https://github.com/mastodon/mastodon/pull/26492), [arbolitoloco1](https://github.com/mastodon/mastodon/pull/26497), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26506), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26525))
+- **Add display of out-of-band hashtags in the web interface** ([Gargron](https://github.com/mastodon/mastodon/pull/26492), [arbolitoloco1](https://github.com/mastodon/mastodon/pull/26497), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26506), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26525), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26606), [Gargron](https://github.com/mastodon/mastodon/pull/26666))
 - **Add role badges to the web interface** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25649), [Gargron](https://github.com/mastodon/mastodon/pull/26281))
-- **Add ability to pick domains to forward reports to using the `forward_to_domains` parameter in `POST /api/v1/reports`** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25866))
+- **Add ability to pick domains to forward reports to using the `forward_to_domains` parameter in `POST /api/v1/reports`** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25866), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26636))
   The `forward_to_domains` REST API parameter is a list of strings. If it is empty or omitted, the previous behavior is maintained.
   The `forward` parameter still needs to be set for `forward_to_domains` to be taken into account.
   The forwarded-to domains can only include that of the original author and people being replied to.
@@ -28,8 +35,15 @@ The following changelog entries focus on changes visible to users, administrator
 - **Add new onboarding flow to web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/24619), [Gargron](https://github.com/mastodon/mastodon/pull/24646), [Gargron](https://github.com/mastodon/mastodon/pull/24705), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24872), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/24883), [Gargron](https://github.com/mastodon/mastodon/pull/24954), [stevenjlm](https://github.com/mastodon/mastodon/pull/24959), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25010), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25275), [Gargron](https://github.com/mastodon/mastodon/pull/25559), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25561))
 - **Add `S3_DISABLE_CHECKSUM_MODE` environment variable for compatibility with some S3-compatible providers** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26435))
 - **Add auto-refresh of accounts we get new messages/edits of** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26510))
-- **Add Elasticsearch cluster health check and indexes mismatch check to dashboard** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26448))
-- Add support for `indexable` attribute on remote actors ([Gargron](https://github.com/mastodon/mastodon/pull/26485))
+- **Add Elasticsearch cluster health check and indexes mismatch check to dashboard** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26448), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26605), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26658))
+- Add `authorized_fetch` server setting in addition to env var ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25798))
+- Add avatar image to webfinger responses ([tvler](https://github.com/mastodon/mastodon/pull/26558))
+- Add debug logging on signature verification failure ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26637))
+- Add explicit error messages when DeepL quota is exceeded ([lutoma](https://github.com/mastodon/mastodon/pull/26704))
+- Add Elasticsearch/OpenSearch version to “Software” in admin dashboard ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26652))
+- Add `data-nosnippet` attribute to remote posts and local posts with `noindex` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26648))
+- Add support for federating `memorial` attribute ([rrgeorge](https://github.com/mastodon/mastodon/pull/26583))
+- Add Cherokee and Kalmyk to languages dropdown ([gunchleoc](https://github.com/mastodon/mastodon/pull/26012), [gunchleoc](https://github.com/mastodon/mastodon/pull/26013))
 - Add `DELETE /api/v1/profile/avatar` and `DELETE /api/v1/profile/header` to the REST API ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25124), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26573))
 - Add `ES_PRESET` option to customize numbers of shards and replicas ([Gargron](https://github.com/mastodon/mastodon/pull/26483), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26489))
   This can have a value of `single_node_cluster` (default), `small_cluster` (uses one replica) or `large_cluster` (uses one replica and a higher number of shards).
@@ -37,13 +51,13 @@ The following changelog entries focus on changes visible to users, administrator
 - Add `CACHE_BUSTER_HTTP_METHOD` environment variable ([renchap](https://github.com/mastodon/mastodon/pull/26528), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26542))
 - Add support for `DB_PASS` when using `DATABASE_URL` ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26295))
 - Add `GET /api/v1/instance/languages` to REST API ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24443))
-- Add primary key to `preview_cards_statuses` join table ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25243), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26384), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26447))
+- Add primary key to `preview_cards_statuses` join table ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25243), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26384), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26447), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26737))
 - Add client-side timeout on resend confirmation button ([Gargron](https://github.com/mastodon/mastodon/pull/26300))
 - Add published date and author to news on the explore screen in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26155))
 - Add `lang` attribute to various UI components ([c960657](https://github.com/mastodon/mastodon/pull/23869), [c960657](https://github.com/mastodon/mastodon/pull/23891), [c960657](https://github.com/mastodon/mastodon/pull/26111), [c960657](https://github.com/mastodon/mastodon/pull/26149))
 - Add stricter protocol fields validation for accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25937))
 - Add support for Azure blob storage ([mistydemeo](https://github.com/mastodon/mastodon/pull/23607), [mistydemeo](https://github.com/mastodon/mastodon/pull/26080))
-- Add toast with option to open post after publishing in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25564), [Signez](https://github.com/mastodon/mastodon/pull/25919))
+- Add toast with option to open post after publishing in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25564), [Signez](https://github.com/mastodon/mastodon/pull/25919), [Gargron](https://github.com/mastodon/mastodon/pull/26664))
 - Add canonical link tags in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25715))
 - Add button to see results for polls in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25726))
 - Add at-symbol prepended to mention span title ([forsamori](https://github.com/mastodon/mastodon/pull/25684))
@@ -93,15 +107,15 @@ The following changelog entries focus on changes visible to users, administrator
 
 ### Changed
 
-- **Change hashtags to be displayed separately when they are the last line of a post** ([renchap](https://github.com/mastodon/mastodon/pull/26499))
+- **Change hashtags to be displayed separately when they are the last line of a post** ([renchap](https://github.com/mastodon/mastodon/pull/26499), [renchap](https://github.com/mastodon/mastodon/pull/26614), [renchap](https://github.com/mastodon/mastodon/pull/26615))
 - **Change reblogs to be excluded from "Posts and replies" tab in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/26302))
-- **Change interaction modal in web interface** ([Gargron, ClearlyClaire](https://github.com/mastodon/mastodon/pull/26075), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26269), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26268), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26267), [mgmn](https://github.com/mastodon/mastodon/pull/26459))
+- **Change interaction modal in web interface** ([Gargron, ClearlyClaire](https://github.com/mastodon/mastodon/pull/26075), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26269), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26268), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26267), [mgmn](https://github.com/mastodon/mastodon/pull/26459), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26593))
 - **Change design of link previews in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/26136), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26151), [Gargron](https://github.com/mastodon/mastodon/pull/26153), [Gargron](https://github.com/mastodon/mastodon/pull/26250), [Gargron](https://github.com/mastodon/mastodon/pull/26287), [Gargron](https://github.com/mastodon/mastodon/pull/26286), [c960657](https://github.com/mastodon/mastodon/pull/26184))
 - **Change "direct message" nomenclature to "private mention" in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/24248))
 - **Change translation feature to cover Content Warnings, poll options and media descriptions** ([c960657](https://github.com/mastodon/mastodon/pull/24175), [S-H-GAMELINKS](https://github.com/mastodon/mastodon/pull/25251), [c960657](https://github.com/mastodon/mastodon/pull/26168), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26452))
 - **Change account search to match by text when opted-in** ([jsgoldstein](https://github.com/mastodon/mastodon/pull/25599), [Gargron](https://github.com/mastodon/mastodon/pull/26378))
 - **Change import feature to be clearer, less error-prone and more reliable** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21054), [mgmn](https://github.com/mastodon/mastodon/pull/24874))
-- **Change local and federated timelines to be in a single “Live feeds” column** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25641), [Gargron](https://github.com/mastodon/mastodon/pull/25683), [mgmn](https://github.com/mastodon/mastodon/pull/25694), [Plastikmensch](https://github.com/mastodon/mastodon/pull/26247))
+- **Change local and federated timelines to be tabs of a single “Live feeds” column** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25641), [Gargron](https://github.com/mastodon/mastodon/pull/25683), [mgmn](https://github.com/mastodon/mastodon/pull/25694), [Plastikmensch](https://github.com/mastodon/mastodon/pull/26247), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26633))
 - **Change user archive export to be faster and more reliable, and export `.zip` archives instead of `.tar.gz` ones** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23360), [TheEssem](https://github.com/mastodon/mastodon/pull/25034))
 - **Change `mastodon-streaming` systemd unit files to be templated** ([e-nomem](https://github.com/mastodon/mastodon/pull/24751))
 - **Change `statsd` integration to disable sidekiq metrics by default** ([mjankowski](https://github.com/mastodon/mastodon/pull/25265), [mjankowski](https://github.com/mastodon/mastodon/pull/25336), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26310))
@@ -111,6 +125,12 @@ The following changelog entries focus on changes visible to users, administrator
 - **Change replica support to native Rails adapter** ([krainboltgreene](https://github.com/mastodon/mastodon/pull/25693), [Gargron](https://github.com/mastodon/mastodon/pull/25849), [Gargron](https://github.com/mastodon/mastodon/pull/25874), [Gargron](https://github.com/mastodon/mastodon/pull/25851), [Gargron](https://github.com/mastodon/mastodon/pull/25977), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26074), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26326), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26386))
   This is a breaking change, dropping `makara` support, and requiring you to update your database configuration if you are using replicas.
   To tell Mastodon to use a read replica, you can either set the `REPLICA_DB_NAME` environment variable (along with `REPLICA_DB_USER`, `REPLICA_DB_PASS`, `REPLICA_DB_HOST`, and `REPLICA_DB_PORT`, if they differ from the primary database), or the `REPLICA_DATABASE_URL` environment variable if your configuration is based on `DATABASE_URL`.
+- Change from `node-redis` to `ioredis` for streaming ([gmemstr](https://github.com/mastodon/mastodon/pull/26581))
+- Change private statuses index to index without crutches ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26713))
+- Change video compression parameters ([Gargron](https://github.com/mastodon/mastodon/pull/26631), [Gargron](https://github.com/mastodon/mastodon/pull/26745), [Gargron](https://github.com/mastodon/mastodon/pull/26766))
+- Change admin e-mail notification settings to be their own settings group ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26596))
+- Change opacity of the delete icon in the search field to be more visible ([AntoninDelFabbro](https://github.com/mastodon/mastodon/pull/26449))
+- Change Account Search to prioritize username over display name ([jsgoldstein](https://github.com/mastodon/mastodon/pull/26623))
 - Change follow recommendation materialized view to be faster in most cases ([renchap, ClearlyClaire](https://github.com/mastodon/mastodon/pull/26545))
 - Change `robots.txt` to block GPTBot ([Foritus](https://github.com/mastodon/mastodon/pull/26396))
 - Change header of hashtag timelines in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26362), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26416))
@@ -120,9 +140,9 @@ The following changelog entries focus on changes visible to users, administrator
 - Change poll form element colors to fit with the rest of the ui ([teeerevor](https://github.com/mastodon/mastodon/pull/26139), [teeerevor](https://github.com/mastodon/mastodon/pull/26162), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26164))
 - Change 'favourite' to 'favorite' for American English ([marekr](https://github.com/mastodon/mastodon/pull/24667), [gunchleoc](https://github.com/mastodon/mastodon/pull/26009), [nabijaczleweli](https://github.com/mastodon/mastodon/pull/26109))
 - Change ActivityStreams representation of suspended accounts to not use a blank `name` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25276))
-- Change focus UI for keyboard only input ([teeerevor](https://github.com/mastodon/mastodon/pull/25935), [Gargron](https://github.com/mastodon/mastodon/pull/26125))
+- Change focus UI for keyboard only input ([teeerevor](https://github.com/mastodon/mastodon/pull/25935), [Gargron](https://github.com/mastodon/mastodon/pull/26125), [Gargron](https://github.com/mastodon/mastodon/pull/26767))
 - Change thread view to scroll to the selected post rather than the post being replied to ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24685))
-- Change links in multi-column mode so tabs are open in single-column mode ([Signez](https://github.com/mastodon/mastodon/pull/25893), [Signez](https://github.com/mastodon/mastodon/pull/26070), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25973))
+- Change links in multi-column mode so tabs are open in single-column mode ([Signez](https://github.com/mastodon/mastodon/pull/25893), [Signez](https://github.com/mastodon/mastodon/pull/26070), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25973), [Signez](https://github.com/mastodon/mastodon/pull/26019), [Signez](https://github.com/mastodon/mastodon/pull/26759))
 - Change searching with `#` to include account index ([jsgoldstein](https://github.com/mastodon/mastodon/pull/25638))
 - Change label and design of sensitive and unavailable media in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25712), [Gargron](https://github.com/mastodon/mastodon/pull/26135), [Gargron](https://github.com/mastodon/mastodon/pull/26330))
 - Change button colors to increase hover/focus contrast and consistency ([teeerevor](https://github.com/mastodon/mastodon/pull/25677), [Gargron](https://github.com/mastodon/mastodon/pull/25679))
@@ -175,6 +195,7 @@ The following changelog entries focus on changes visible to users, administrator
 - **Remove support for Ruby 2.7** ([nschonni](https://github.com/mastodon/mastodon/pull/24237))
 - **Remove clustering from streaming API** ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/24655))
 - **Remove anonymous access to the streaming API** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23989))
+- Remove `kmr` from language selection, as it was a duplicate for `ku` ([gunchleoc](https://github.com/mastodon/mastodon/pull/26014), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26787))
 - Remove 16:9 cropping from web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26132))
 - Remove back button from bookmarks, favourites and lists screens in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26126))
 - Remove display name input from sign-up form ([Gargron](https://github.com/mastodon/mastodon/pull/24704))
@@ -189,6 +210,18 @@ The following changelog entries focus on changes visible to users, administrator
 - **Fix log-in flow when involving both OAuth and external authentication** ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24073))
 - **Fix broken links in account gallery** ([c960657](https://github.com/mastodon/mastodon/pull/24218))
 - **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392))
+- **Fix migration handler not updating lists** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24808))
+- Fix invalid `Content-Type` header for WebP images ([c960657](https://github.com/mastodon/mastodon/pull/26773))
+- Fix minor inefficiencies in `tootctl search deploy` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26721))
+- Fix filter form in profiles directory overflowing instead of wrapping ([arbolitoloco1](https://github.com/mastodon/mastodon/pull/26682))
+- Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26237))
+- Fix inefficiencies in `PlainTextFormatter` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26727))
+- Fix sign up steps progress layout in right-to-left locales ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26728))
+- Fix bug with “favorited by” and “reblogged by“ view on posts only showing up to 40 items ([timothyjrogers](https://github.com/mastodon/mastodon/pull/26577), [timothyjrogers](https://github.com/mastodon/mastodon/pull/26574))
+- Fix bad search type heuristic ([Gargron](https://github.com/mastodon/mastodon/pull/26673))
+- Fix not being able to negate prefix clauses in search ([Gargron](https://github.com/mastodon/mastodon/pull/26672))
+- Fix timeout on invalid set of exclusionary parameters in `/api/v1/timelines/public` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26239))
+- Fix unexpected audio stream transcoding when uploaded video is eligible to passthrough ([yufushiro](https://github.com/mastodon/mastodon/pull/26608))
 - Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500))
 - Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409))
 - Fix adding column with default value taking longer on Postgres >= 11 ([Gargron](https://github.com/mastodon/mastodon/pull/26375))

Dockerfile

@@ -42,8 +42,8 @@ RUN apt-get update && \
 FROM node:${NODE_VERSION}
 
 # Use those args to specify your own version flags & suffixes
-ARG MASTODON_VERSION_FLAGS=""
-ARG MASTODON_VERSION_SUFFIX=""
+ARG MASTODON_VERSION_PRERELEASE=""
+ARG MASTODON_VERSION_METADATA=""
 
 ARG UID="991"
 ARG GID="991"
@@ -89,8 +89,8 @@ ENV RAILS_ENV="production" \
     NODE_ENV="production" \
     RAILS_SERVE_STATIC_FILES="true" \
     BIND="0.0.0.0" \
-    MASTODON_VERSION_FLAGS="${MASTODON_VERSION_FLAGS}" \
-    MASTODON_VERSION_SUFFIX="${MASTODON_VERSION_SUFFIX}"
+    MASTODON_VERSION_PRERELEASE="${MASTODON_VERSION_PRERELEASE}" \
+    MASTODON_VERSION_METADATA="${MASTODON_VERSION_METADATA}"
 
 # Set the run user
 USER mastodon

Gemfile

@@ -110,7 +110,7 @@ group :test do
   gem 'fuubar', '~> 2.5'
 
   # Extra RSpec extenion methods and helpers for sidekiq
-  gem 'rspec-sidekiq', '~> 3.1'
+  gem 'rspec-sidekiq', '~> 4.0'
 
   # Browser integration testing
   gem 'capybara', '~> 3.39'

Gemfile.lock

@@ -39,47 +39,47 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.7)
-      actionpack (= 7.0.7)
-      activesupport (= 7.0.7)
+    actioncable (7.0.7.2)
+      actionpack (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.7)
-      actionpack (= 7.0.7)
-      activejob (= 7.0.7)
-      activerecord (= 7.0.7)
-      activestorage (= 7.0.7)
-      activesupport (= 7.0.7)
+    actionmailbox (7.0.7.2)
+      actionpack (= 7.0.7.2)
+      activejob (= 7.0.7.2)
+      activerecord (= 7.0.7.2)
+      activestorage (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.7)
-      actionpack (= 7.0.7)
-      actionview (= 7.0.7)
-      activejob (= 7.0.7)
-      activesupport (= 7.0.7)
+    actionmailer (7.0.7.2)
+      actionpack (= 7.0.7.2)
+      actionview (= 7.0.7.2)
+      activejob (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.7)
-      actionview (= 7.0.7)
-      activesupport (= 7.0.7)
+    actionpack (7.0.7.2)
+      actionview (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.7)
-      actionpack (= 7.0.7)
-      activerecord (= 7.0.7)
-      activestorage (= 7.0.7)
-      activesupport (= 7.0.7)
+    actiontext (7.0.7.2)
+      actionpack (= 7.0.7.2)
+      activerecord (= 7.0.7.2)
+      activestorage (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.7)
-      activesupport (= 7.0.7)
+    actionview (7.0.7.2)
+      activesupport (= 7.0.7.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -89,27 +89,27 @@ GEM
       activemodel (>= 4.1, < 7.1)
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
-    activejob (7.0.7)
-      activesupport (= 7.0.7)
+    activejob (7.0.7.2)
+      activesupport (= 7.0.7.2)
       globalid (>= 0.3.6)
-    activemodel (7.0.7)
-      activesupport (= 7.0.7)
-    activerecord (7.0.7)
-      activemodel (= 7.0.7)
-      activesupport (= 7.0.7)
-    activestorage (7.0.7)
-      actionpack (= 7.0.7)
-      activejob (= 7.0.7)
-      activerecord (= 7.0.7)
-      activesupport (= 7.0.7)
+    activemodel (7.0.7.2)
+      activesupport (= 7.0.7.2)
+    activerecord (7.0.7.2)
+      activemodel (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
+    activestorage (7.0.7.2)
+      actionpack (= 7.0.7.2)
+      activejob (= 7.0.7.2)
+      activerecord (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.7)
+    activesupport (7.0.7.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.4)
+    addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
     aes_key_wrap (1.1.0)
     airbrussh (1.4.1)
@@ -124,8 +124,8 @@ GEM
     attr_required (1.0.1)
     awrence (1.2.1)
     aws-eventstream (1.2.0)
-    aws-partitions (1.793.0)
-    aws-sdk-core (3.180.3)
+    aws-partitions (1.809.0)
+    aws-sdk-core (3.181.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.5)
@@ -133,8 +133,8 @@ GEM
     aws-sdk-kms (1.71.0)
       aws-sdk-core (~> 3, >= 3.177.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.132.1)
-      aws-sdk-core (~> 3, >= 3.179.0)
+    aws-sdk-s3 (1.133.0)
+      aws-sdk-core (~> 3, >= 3.181.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.6)
     aws-sigv4 (1.6.0)
@@ -147,6 +147,7 @@ GEM
       faraday_middleware (~> 1.0, >= 1.0.0.rc1)
       net-http-persistent (~> 4.0)
       nokogiri (~> 1, >= 1.10.8)
+    base64 (0.1.1)
     bcrypt (3.1.18)
     better_errors (2.10.1)
       erubi (>= 1.0.0)
@@ -202,7 +203,7 @@ GEM
       activesupport
     cbor (0.5.9.6)
     charlock_holmes (0.7.7)
-    chewy (7.3.3)
+    chewy (7.3.4)
       activesupport (>= 5.2)
       elasticsearch (>= 7.12.0, < 7.14.0)
       elasticsearch-dsl
@@ -323,7 +324,7 @@ GEM
       ruby-progressbar (~> 1.4)
     globalid (1.1.0)
       activesupport (>= 5.0)
-    haml (6.1.1)
+    haml (6.1.2)
       temple (>= 0.8.2)
       thor
       tilt
@@ -332,7 +333,7 @@ GEM
       activesupport (>= 5.1)
       haml (>= 4.0.6)
       railties (>= 5.1)
-    haml_lint (0.49.3)
+    haml_lint (0.50.0)
       haml (>= 4.0, < 6.2)
       parallel (~> 1.10)
       rainbow
@@ -451,7 +452,7 @@ GEM
       hashie (~> 5.0)
     memory_profiler (1.0.1)
     method_source (1.0.0)
-    mime-types (3.5.0)
+    mime-types (3.5.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0808)
     mini_mime (1.1.5)
@@ -481,7 +482,7 @@ GEM
     nokogiri (1.15.4)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    oj (3.15.0)
+    oj (3.16.1)
     omniauth (2.1.1)
       hashie (>= 3.4.6)
       rack (>= 2.2.3)
@@ -518,7 +519,7 @@ GEM
     parslet (2.0.0)
     pastel (0.8.0)
       tty-color (~> 0.5)
-    pg (1.5.3)
+    pg (1.5.4)
     pghero (3.3.3)
       activerecord (>= 6)
     posix-spawn (0.3.15)
@@ -555,20 +556,20 @@ GEM
       rack
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails (7.0.7)
-      actioncable (= 7.0.7)
-      actionmailbox (= 7.0.7)
-      actionmailer (= 7.0.7)
-      actionpack (= 7.0.7)
-      actiontext (= 7.0.7)
-      actionview (= 7.0.7)
-      activejob (= 7.0.7)
-      activemodel (= 7.0.7)
-      activerecord (= 7.0.7)
-      activestorage (= 7.0.7)
-      activesupport (= 7.0.7)
+    rails (7.0.7.2)
+      actioncable (= 7.0.7.2)
+      actionmailbox (= 7.0.7.2)
+      actionmailer (= 7.0.7.2)
+      actionpack (= 7.0.7.2)
+      actiontext (= 7.0.7.2)
+      actionview (= 7.0.7.2)
+      activejob (= 7.0.7.2)
+      activemodel (= 7.0.7.2)
+      activerecord (= 7.0.7.2)
+      activestorage (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       bundler (>= 1.15.0)
-      railties (= 7.0.7)
+      railties (= 7.0.7.2)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
       actionview (>= 5.0.1.rc1)
@@ -583,9 +584,9 @@ GEM
     rails-i18n (7.0.7)
       i18n (>= 0.7, < 2)
       railties (>= 6.0.0, < 8)
-    railties (7.0.7)
-      actionpack (= 7.0.7)
-      activesupport (= 7.0.7)
+    railties (7.0.7.2)
+      actionpack (= 7.0.7.2)
+      activesupport (= 7.0.7.2)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -632,12 +633,15 @@ GEM
       rspec-expectations (~> 3.12)
       rspec-mocks (~> 3.12)
       rspec-support (~> 3.12)
-    rspec-sidekiq (3.1.0)
-      rspec-core (~> 3.0, >= 3.0.0)
-      sidekiq (>= 2.4.0)
-    rspec-support (3.12.0)
+    rspec-sidekiq (4.0.1)
+      rspec-core (~> 3.0)
+      rspec-expectations (~> 3.0)
+      rspec-mocks (~> 3.0)
+      sidekiq (>= 5, < 8)
+    rspec-support (3.12.1)
     rspec_chunked (0.6)
-    rubocop (1.54.2)
+    rubocop (1.56.2)
+      base64 (~> 0.1.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -645,7 +649,7 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.0, < 2.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.29.0)
@@ -654,14 +658,14 @@ GEM
       rubocop (~> 1.41)
     rubocop-factory_bot (2.23.1)
       rubocop (~> 1.33)
-    rubocop-performance (1.18.0)
+    rubocop-performance (1.19.0)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rails (2.20.2)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.33.0, < 2.0)
-    rubocop-rspec (2.22.0)
+    rubocop-rspec (2.23.2)
       rubocop (~> 1.33)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
@@ -727,7 +731,7 @@ GEM
       net-ssh (>= 2.8.0)
     stackprof (0.2.25)
     statsd-ruby (1.5.0)
-    stoplight (3.0.1)
+    stoplight (3.0.2)
       redlock (~> 1.0)
     strong_migrations (0.8.0)
       activerecord (>= 5.2)
@@ -791,7 +795,7 @@ GEM
     webfinger (1.2.0)
       activesupport
       httpclient (>= 2.4)
-    webmock (3.18.1)
+    webmock (3.19.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -909,7 +913,7 @@ DEPENDENCIES
   redis-namespace (~> 1.10)
   rqrcode (~> 2.2)
   rspec-rails (~> 6.0)
-  rspec-sidekiq (~> 3.1)
+  rspec-sidekiq (~> 4.0)
   rspec_chunked (~> 0.6)
   rubocop
   rubocop-capybara

Procfile.dev

@@ -1,4 +1,4 @@
 web: env PORT=3000 RAILS_ENV=development bundle exec puma -C config/puma.rb
 sidekiq: env PORT=3000 RAILS_ENV=development bundle exec sidekiq
 stream: env PORT=4000 yarn run start
-webpack: ./bin/webpack-dev-server --listen-host 0.0.0.0
+webpack: bin/webpack-dev-server

SECURITY.md

@@ -1,8 +1,11 @@
 # Security Policy
 
-If you believe you've identified a security vulnerability in Mastodon (a bug that allows something to happen that shouldn't be possible), you can reach us at <security@joinmastodon.org>.
+If you believe you've identified a security vulnerability in Mastodon (a bug that allows something to happen that shouldn't be possible), you can either:
 
-You should _not_ report such issues on GitHub or in other public spaces to give us time to publish a fix for the issue without exposing Mastodon's users to increased risk.
+- open a [Github security issue on the Mastodon project](https://github.com/mastodon/mastodon/security/advisories/new)
+- reach us at <security@joinmastodon.org>
+
+You should _not_ report such issues on public GitHub issues or in other public spaces to give us time to publish a fix for the issue without exposing Mastodon's users to increased risk.
 
 ## Scope
 

Vagrantfile

@@ -60,6 +60,37 @@ sudo usermod -a -G rvm $USER
 
 SCRIPT
 
+$provisionElasticsearch = <<SCRIPT
+# Install Elastic Search
+sudo apt install openjdk-17-jre-headless -y
+sudo wget -O /usr/share/keyrings/elasticsearch.asc https://artifacts.elastic.co/GPG-KEY-elasticsearch
+sudo sh -c 'echo "deb [signed-by=/usr/share/keyrings/elasticsearch.asc] https://artifacts.elastic.co/packages/7.x/apt stable main" > /etc/apt/sources.list.d/elastic-7.x.list'
+sudo apt update
+sudo apt install elasticsearch -y
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now elasticsearch
+
+echo 'path.data: /var/lib/elasticsearch
+path.logs: /var/log/elasticsearch
+network.host: 0.0.0.0
+http.port: 9200
+discovery.seed_hosts: ["localhost"]
+cluster.initial_master_nodes: ["node-1"]' > /etc/elasticsearch/elasticsearch.yml
+
+sudo systemctl restart elasticsearch
+
+# Install Kibana
+sudo apt install kibana -y
+sudo systemctl enable --now kibana
+
+echo 'server.host: "0.0.0.0"
+elasticsearch.hosts: ["http://localhost:9200"]' > /etc/kibana/kibana.yml
+
+sudo systemctl restart kibana
+
+SCRIPT
+
 $provisionB = <<SCRIPT
 
 source "/etc/profile.d/rvm.sh"
@@ -102,10 +133,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.provider :virtualbox do |vb|
     vb.name = "mastodon"
-    vb.customize ["modifyvm", :id, "--memory", "2048"]
-    # Increase the number of CPUs. Uncomment and adjust to
-    # increase performance
-    # vb.customize ["modifyvm", :id, "--cpus", "3"]
+    vb.customize ["modifyvm", :id, "--memory", "8192"]
+    vb.customize ["modifyvm", :id, "--cpus", "3"]
 
     # Disable VirtualBox DNS proxy to skip long-delay IPv6 resolutions.
     # https://github.com/mitchellh/vagrant/issues/1172
@@ -141,9 +170,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 3000, host: 3000
   config.vm.network :forwarded_port, guest: 4000, host: 4000
   config.vm.network :forwarded_port, guest: 8080, host: 8080
+  config.vm.network :forwarded_port, guest: 9200, host: 9200
+  config.vm.network :forwarded_port, guest: 9300, host: 9300
+  config.vm.network :forwarded_port, guest: 9243, host: 9243
+  config.vm.network :forwarded_port, guest: 5601, host: 5601
 
   # Full provisioning script, only runs on first 'vagrant up' or with 'vagrant provision'
   config.vm.provision :shell, inline: $provisionA, privileged: false, reset: true
+  # Run with elevated privileges for Elasticsearch installation
+  config.vm.provision :shell, inline: $provisionElasticsearch, privileged: true
   config.vm.provision :shell, inline: $provisionB, privileged: false
 
   config.vm.post_up_message = <<MESSAGE

app/chewy/accounts_index.rb

@@ -21,12 +21,13 @@ class AccountsIndex < Chewy::Index
 
     analyzer: {
       natural: {
-        tokenizer: 'uax_url_email',
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
@@ -62,6 +63,6 @@ class AccountsIndex < Chewy::Index
     field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
     field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
     field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
-    field(:text, type: 'text', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
+    field(:text, type: 'text', analyzer: 'verbatim', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
   end
 end

app/chewy/public_statuses_index.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class PublicStatusesIndex < Chewy::Index
+  settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
+    filter: {
+      english_stop: {
+        type: 'stop',
+        stopwords: '_english_',
+      },
+
+      english_stemmer: {
+        type: 'stemmer',
+        language: 'english',
+      },
+
+      english_possessive_stemmer: {
+        type: 'stemmer',
+        language: 'possessive_english',
+      },
+    },
+
+    analyzer: {
+      verbatim: {
+        tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
+        filter: %w(
+          lowercase
+          asciifolding
+          cjk_width
+          elision
+          english_possessive_stemmer
+          english_stop
+          english_stemmer
+        ),
+      },
+
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
+    },
+  }
+
+  index_scope ::Status.unscoped
+                      .kept
+                      .indexable
+                      .includes(:media_attachments, :preloadable_poll, :preview_cards, :tags)
+
+  root date_detection: false do
+    field(:id, type: 'long')
+    field(:account_id, type: 'long')
+    field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
+    field(:language, type: 'keyword')
+    field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
+    field(:created_at, type: 'date')
+  end
+end

app/chewy/statuses_index.rb

@@ -1,75 +1,65 @@
 # frozen_string_literal: true
 
 class StatusesIndex < Chewy::Index
-  include FormattingHelper
-
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
     filter: {
       english_stop: {
         type: 'stop',
         stopwords: '_english_',
       },
+
       english_stemmer: {
         type: 'stemmer',
         language: 'english',
       },
+
       english_possessive_stemmer: {
         type: 'stemmer',
         language: 'possessive_english',
       },
     },
+
     analyzer: {
-      content: {
+      verbatim: {
         tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
       },
+
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
     },
   }
 
-  # We do not use delete_if option here because it would call a method that we
-  # expect to be called with crutches without crutches, causing n+1 queries
-  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll)
-
-  crutch :mentions do |collection|
-    data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :favourites do |collection|
-    data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :reblogs do |collection|
-    data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :bookmarks do |collection|
-    data = ::Bookmark.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :votes do |collection|
-    data = ::PollVote.joins(:poll).where(poll: { status_id: collection.map(&:id) }).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
+  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preview_cards, :local_mentioned, :local_favorited, :local_reblogged, :local_bookmarked, :tags, preloadable_poll: :local_voters), delete_if: ->(status) { status.searchable_by.empty? }
 
   root date_detection: false do
-    field :id, type: 'long'
-    field :account_id, type: 'long'
-
-    field :text, type: 'text', value: ->(status) { status.searchable_text } do
-      field :stemmed, type: 'text', analyzer: 'content'
-    end
-
-    field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }
+    field(:id, type: 'long')
+    field(:account_id, type: 'long')
+    field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:tags, type: 'text', analyzer: 'hashtag',  value: ->(status) { status.tags.map(&:display_name) })
+    field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by })
+    field(:language, type: 'keyword')
+    field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
+    field(:created_at, type: 'date')
   end
 end

app/chewy/tags_index.rb

@@ -5,12 +5,21 @@ class TagsIndex < Chewy::Index
     analyzer: {
       content: {
         tokenizer: 'keyword',
-        filter: %w(lowercase asciifolding cjk_width),
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
       },
 
       edge_ngram: {
         tokenizer: 'edge_ngram',
-        filter: %w(lowercase asciifolding cjk_width),
+        filter: %w(
+          lowercase
+          asciifolding
+          cjk_width
+        ),
       },
     },
 
@@ -30,12 +39,9 @@ class TagsIndex < Chewy::Index
   end
 
   root date_detection: false do
-    field :name, type: 'text', analyzer: 'content' do
-      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
-    end
-
-    field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
-    field :usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts }
-    field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    field(:name, type: 'text', analyzer: 'content', value: :display_name) { field(:edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content') }
+    field(:reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? })
+    field(:usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts })
+    field(:last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at })
   end
 end

app/controllers/admin/software_updates_controller.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Admin
+  class SoftwareUpdatesController < BaseController
+    before_action :check_enabled!
+
+    def index
+      authorize :software_update, :index?
+      @software_updates = SoftwareUpdate.all.sort_by(&:gem_version)
+    end
+
+    private
+
+    def check_enabled!
+      not_found unless SoftwareUpdate.check_enabled?
+    end
+  end
+end

app/controllers/api/v1/accounts/credentials_controller.rb

@@ -30,6 +30,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
       :bot,
       :discoverable,
       :hide_collections,
+      :indexable,
       fields_attributes: [:name, :value]
     )
   end

app/controllers/api/v1/statuses/translations_controller.rb

@@ -8,7 +8,15 @@ class Api::V1::Statuses::TranslationsController < Api::BaseController
   before_action :set_translation
 
   rescue_from TranslationService::NotConfiguredError, with: :not_found
-  rescue_from TranslationService::UnexpectedResponseError, TranslationService::QuotaExceededError, TranslationService::TooManyRequestsError, with: :service_unavailable
+  rescue_from TranslationService::UnexpectedResponseError, with: :service_unavailable
+
+  rescue_from TranslationService::QuotaExceededError do
+    render json: { error: I18n.t('translation.errors.quota_exceeded') }, status: 503
+  end
+
+  rescue_from TranslationService::TooManyRequestsError do
+    render json: { error: I18n.t('translation.errors.too_many_requests') }, status: 503
+  end
 
   def create
     render json: @translation, serializer: REST::TranslationSerializer

app/controllers/application_controller.rb

@@ -11,6 +11,7 @@ class ApplicationController < ActionController::Base
   include CacheConcern
   include DomainControlHelper
   include DatabaseHelper
+  include AuthorizedFetchHelper
 
   helper_method :current_account
   helper_method :current_session
@@ -51,10 +52,6 @@ class ApplicationController < ActionController::Base
 
   private
 
-  def authorized_fetch_mode?
-    ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.limited_federation_mode
-  end
-
   def public_fetch_mode?
     !authorized_fetch_mode?
   end

app/controllers/concerns/signature_verification.rb

@@ -119,6 +119,8 @@ module SignatureVerification
   private
 
   def fail_with!(message, **options)
+    Rails.logger.warn { "Signature verification failed: #{message}" }
+
     @signature_verification_failure_reason = { error: message }.merge(options)
     @signed_request_actor = nil
   end

app/controllers/settings/privacy_controller.rb

@@ -18,7 +18,7 @@ class Settings::PrivacyController < Settings::BaseController
   private
 
   def account_params
-    params.require(:account).permit(:discoverable, :unlocked, :show_collections, settings: UserSettings.keys)
+    params.require(:account).permit(:discoverable, :unlocked, :indexable, :show_collections, settings: UserSettings.keys)
   end
 
   def set_account

app/helpers/authorized_fetch_helper.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module AuthorizedFetchHelper
+  def authorized_fetch_mode?
+    ENV.fetch('AUTHORIZED_FETCH') { Setting.authorized_fetch } == 'true' || Rails.configuration.x.limited_federation_mode
+  end
+
+  def authorized_fetch_overridden?
+    ENV.key?('AUTHORIZED_FETCH') || Rails.configuration.x.limited_federation_mode
+  end
+end

app/helpers/context_helper.rb

@@ -21,6 +21,7 @@ module ContextHelper
     blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
     discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
     indexable: { 'toot' => 'http://joinmastodon.org/ns#', 'indexable' => 'toot:indexable' },
+    memorial: { 'toot' => 'http://joinmastodon.org/ns#', 'memorial' => 'toot:memorial' },
     voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
     olm: {
       'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId',

app/helpers/languages_helper.rb

@@ -188,11 +188,11 @@ module LanguagesHelper
 
   ISO_639_3 = {
     ast: ['Asturian', 'Asturianu'].freeze,
+    chr: ['Cherokee', 'ᏣᎳᎩ ᎦᏬᏂᎯᏍᏗ'].freeze,
     ckb: ['Sorani (Kurdish)', 'سۆرانی'].freeze,
     cnr: ['Montenegrin', 'crnogorski'].freeze,
     jbo: ['Lojban', 'la .lojban.'].freeze,
     kab: ['Kabyle', 'Taqbaylit'].freeze,
-    kmr: ['Kurmanji (Kurdish)', 'Kurmancî'].freeze,
     ldn: ['Láadan', 'Láadan'].freeze,
     lfn: ['Lingua Franca Nova', 'lingua franca nova'].freeze,
     sco: ['Scots', 'Scots'].freeze,
@@ -200,6 +200,7 @@ module LanguagesHelper
     smj: ['Lule Sami', 'Julevsámegiella'].freeze,
     szl: ['Silesian', 'ślůnsko godka'].freeze,
     tok: ['Toki Pona', 'toki pona'].freeze,
+    xal: ['Kalmyk', 'Хальмг келн'].freeze,
     zba: ['Balaibalan', 'باليبلن'].freeze,
     zgh: ['Standard Moroccan Tamazight', 'ⵜⴰⵎⴰⵣⵉⵖⵜ'].freeze,
   }.freeze

app/javascript/mastodon/actions/compose.js

@@ -85,6 +85,7 @@ const messages = defineMessages({
   uploadErrorPoll:  { id: 'upload_error.poll', defaultMessage: 'File upload not allowed with polls.' },
   open: { id: 'compose.published.open', defaultMessage: 'Open' },
   published: { id: 'compose.published.body', defaultMessage: 'Post published.' },
+  saved: { id: 'compose.saved.body', defaultMessage: 'Post saved.' },
 });
 
 export const ensureComposeIsVisible = (getState, routerHistory) => {
@@ -246,7 +247,7 @@ export function submitCompose(routerHistory) {
       }
 
       dispatch(showAlert({
-        message: messages.published,
+        message: statusId === null ? messages.published : messages.saved,
         action: messages.open,
         dismissAfter: 10000,
         onClick: () => routerHistory.push(`/@${response.data.account.username}/${response.data.id}`),

app/javascript/mastodon/actions/interactions.js

@@ -1,11 +1,16 @@
-import api from '../api';
+import api, { getLinks } from '../api';
 
+import { fetchRelationships } from './accounts';
 import { importFetchedAccounts, importFetchedStatus } from './importer';
 
 export const REBLOG_REQUEST = 'REBLOG_REQUEST';
 export const REBLOG_SUCCESS = 'REBLOG_SUCCESS';
 export const REBLOG_FAIL    = 'REBLOG_FAIL';
 
+export const REBLOGS_EXPAND_REQUEST = 'REBLOGS_EXPAND_REQUEST';
+export const REBLOGS_EXPAND_SUCCESS = 'REBLOGS_EXPAND_SUCCESS';
+export const REBLOGS_EXPAND_FAIL = 'REBLOGS_EXPAND_FAIL';
+
 export const FAVOURITE_REQUEST = 'FAVOURITE_REQUEST';
 export const FAVOURITE_SUCCESS = 'FAVOURITE_SUCCESS';
 export const FAVOURITE_FAIL    = 'FAVOURITE_FAIL';
@@ -26,6 +31,10 @@ export const FAVOURITES_FETCH_REQUEST = 'FAVOURITES_FETCH_REQUEST';
 export const FAVOURITES_FETCH_SUCCESS = 'FAVOURITES_FETCH_SUCCESS';
 export const FAVOURITES_FETCH_FAIL    = 'FAVOURITES_FETCH_FAIL';
 
+export const FAVOURITES_EXPAND_REQUEST = 'FAVOURITES_EXPAND_REQUEST';
+export const FAVOURITES_EXPAND_SUCCESS = 'FAVOURITES_EXPAND_SUCCESS';
+export const FAVOURITES_EXPAND_FAIL = 'FAVOURITES_EXPAND_FAIL';
+
 export const PIN_REQUEST = 'PIN_REQUEST';
 export const PIN_SUCCESS = 'PIN_SUCCESS';
 export const PIN_FAIL    = 'PIN_FAIL';
@@ -273,8 +282,10 @@ export function fetchReblogs(id) {
     dispatch(fetchReblogsRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/reblogged_by`).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedAccounts(response.data));
-      dispatch(fetchReblogsSuccess(id, response.data));
+      dispatch(fetchReblogsSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
       dispatch(fetchReblogsFail(id, error));
     });
@@ -288,17 +299,62 @@ export function fetchReblogsRequest(id) {
   };
 }
 
-export function fetchReblogsSuccess(id, accounts) {
+export function fetchReblogsSuccess(id, accounts, next) {
   return {
     type: REBLOGS_FETCH_SUCCESS,
     id,
     accounts,
+    next,
   };
 }
 
 export function fetchReblogsFail(id, error) {
   return {
     type: REBLOGS_FETCH_FAIL,
+    id,
+    error,
+  };
+}
+
+export function expandReblogs(id) {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['user_lists', 'reblogged_by', id, 'next']);
+    if (url === null) {
+      return;
+    }
+
+    dispatch(expandReblogsRequest(id));
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+
+      dispatch(importFetchedAccounts(response.data));
+      dispatch(expandReblogsSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
+    }).catch(error => dispatch(expandReblogsFail(id, error)));
+  };
+}
+
+export function expandReblogsRequest(id) {
+  return {
+    type: REBLOGS_EXPAND_REQUEST,
+    id,
+  };
+}
+
+export function expandReblogsSuccess(id, accounts, next) {
+  return {
+    type: REBLOGS_EXPAND_SUCCESS,
+    id,
+    accounts,
+    next,
+  };
+}
+
+export function expandReblogsFail(id, error) {
+  return {
+    type: REBLOGS_EXPAND_FAIL,
+    id,
     error,
   };
 }
@@ -308,8 +364,10 @@ export function fetchFavourites(id) {
     dispatch(fetchFavouritesRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/favourited_by`).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedAccounts(response.data));
-      dispatch(fetchFavouritesSuccess(id, response.data));
+      dispatch(fetchFavouritesSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
       dispatch(fetchFavouritesFail(id, error));
     });
@@ -323,17 +381,62 @@ export function fetchFavouritesRequest(id) {
   };
 }
 
-export function fetchFavouritesSuccess(id, accounts) {
+export function fetchFavouritesSuccess(id, accounts, next) {
   return {
     type: FAVOURITES_FETCH_SUCCESS,
     id,
     accounts,
+    next,
   };
 }
 
 export function fetchFavouritesFail(id, error) {
   return {
     type: FAVOURITES_FETCH_FAIL,
+    id,
+    error,
+  };
+}
+
+export function expandFavourites(id) {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['user_lists', 'favourited_by', id, 'next']);
+    if (url === null) {
+      return;
+    }
+
+    dispatch(expandFavouritesRequest(id));
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+
+      dispatch(importFetchedAccounts(response.data));
+      dispatch(expandFavouritesSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
+    }).catch(error => dispatch(expandFavouritesFail(id, error)));
+  };
+}
+
+export function expandFavouritesRequest(id) {
+  return {
+    type: FAVOURITES_EXPAND_REQUEST,
+    id,
+  };
+}
+
+export function expandFavouritesSuccess(id, accounts, next) {
+  return {
+    type: FAVOURITES_EXPAND_SUCCESS,
+    id,
+    accounts,
+    next,
+  };
+}
+
+export function expandFavouritesFail(id, error) {
+  return {
+    type: FAVOURITES_EXPAND_FAIL,
+    id,
     error,
   };
 }

app/javascript/mastodon/components/__tests__/hashtag_bar.tsx

@@ -105,6 +105,21 @@ describe('computeHashtagBarForStatus', () => {
     );
   });
 
+  it('handles server-side normalized tags with accentuated characters', () => {
+    const status = createStatus(
+      '<p>Text</p><p><a href="test">#éaa</a> <a href="test">#Éaa</a></p>',
+      ['eaa'], // The server may normalize the hashtags in the `tags` attribute
+    );
+
+    const { hashtagsInBar, statusContentProps } =
+      computeHashtagBarForStatus(status);
+
+    expect(hashtagsInBar).toEqual(['Éaa']);
+    expect(statusContentProps.statusContent).toMatchInlineSnapshot(
+      `"<p>Text</p>"`,
+    );
+  });
+
   it('does not display in bar a hashtag in content with a case difference', () => {
     const status = createStatus(
       '<p>Text <a href="test">#Éaa</a></p><p><a href="test">#éaa</a></p>',

app/javascript/mastodon/components/column.jsx

@@ -16,7 +16,19 @@ export default class Column extends PureComponent {
   };
 
   scrollTop () {
-    const scrollable = this.props.bindToDocument ? document.scrollingElement : this.node.querySelector('.scrollable');
+    let scrollable = null;
+
+    if (this.props.bindToDocument) {
+      scrollable = document.scrollingElement;
+    } else {
+      scrollable = this.node.querySelector('.scrollable');
+
+      // Some columns have nested `.scrollable` containers, with the outer one
+      // being a wrapper while the actual scrollable content is deeper.
+      if (scrollable.classList.contains('scrollable--flex')) {
+        scrollable = scrollable?.querySelector('.scrollable') || scrollable;
+      }
+   }
 
     if (!scrollable) {
       return;

app/javascript/mastodon/components/hashtag_bar.tsx

@@ -10,8 +10,8 @@ import { groupBy, minBy } from 'lodash';
 
 import { getStatusContent } from './status_content';
 
-// About two lines on desktop
-const VISIBLE_HASHTAGS = 7;
+// Fit on a single line on desktop
+const VISIBLE_HASHTAGS = 3;
 
 // Those types are not correct, they need to be replaced once this part of the state is typed
 export type TagLike = Record<{ name: string }>;
@@ -23,8 +23,9 @@ export type StatusLike = Record<{
 }>;
 
 function normalizeHashtag(hashtag: string) {
-  if (hashtag && hashtag.startsWith('#')) return hashtag.slice(1);
-  else return hashtag;
+  return (
+    hashtag && hashtag.startsWith('#') ? hashtag.slice(1) : hashtag
+  ).normalize('NFKC');
 }
 
 function isNodeLinkHashtag(element: Node): element is HTMLLinkElement {
@@ -70,9 +71,16 @@ function uniqueHashtagsWithCaseHandling(hashtags: string[]) {
 }
 
 // Create the collator once, this is much more efficient
-const collator = new Intl.Collator(undefined, { sensitivity: 'accent' });
+const collator = new Intl.Collator(undefined, {
+  sensitivity: 'base', // we use this to emulate the ASCII folding done on the server-side, hopefuly more efficiently
+});
+
 function localeAwareInclude(collection: string[], value: string) {
-  return collection.find((item) => collator.compare(item, value) === 0);
+  const normalizedValue = value.normalize('NFKC');
+
+  return !!collection.find(
+    (item) => collator.compare(item.normalize('NFKC'), normalizedValue) === 0,
+  );
 }
 
 // We use an intermediate function here to make it easier to test
@@ -121,11 +129,13 @@ export function computeHashtagBarForStatus(status: StatusLike): {
   // try to see if the last line is only hashtags
   let onlyHashtags = true;
 
+  const normalizedTagNames = tagNames.map((tag) => tag.normalize('NFKC'));
+
   Array.from(lastChild.childNodes).forEach((node) => {
     if (isNodeLinkHashtag(node) && node.textContent) {
       const normalized = normalizeHashtag(node.textContent);
 
-      if (!localeAwareInclude(tagNames, normalized)) {
+      if (!localeAwareInclude(normalizedTagNames, normalized)) {
         // stop here, this is not a real hashtag, so consider it as text
         onlyHashtags = false;
         return;
@@ -140,12 +150,14 @@ export function computeHashtagBarForStatus(status: StatusLike): {
     }
   });
 
-  const hashtagsInBar = tagNames.filter(
-    (tag) =>
-      // the tag does not appear at all in the status content, it is an out-of-band tag
-      !localeAwareInclude(contentHashtags, tag) &&
-      !localeAwareInclude(lastLineHashtags, tag),
-  );
+  const hashtagsInBar = tagNames.filter((tag) => {
+    const normalizedTag = tag.normalize('NFKC');
+    // the tag does not appear at all in the status content, it is an out-of-band tag
+    return (
+      !localeAwareInclude(contentHashtags, normalizedTag) &&
+      !localeAwareInclude(lastLineHashtags, normalizedTag)
+    );
+  });
 
   const isOnlyOneLine = contentWithoutLastLine.content.childElementCount === 0;
   const hasMedia = status.get('media_attachments').size > 0;
@@ -198,13 +210,13 @@ const HashtagBar: React.FC<{
 
   const revealedHashtags = expanded
     ? hashtags
-    : hashtags.slice(0, VISIBLE_HASHTAGS - 1);
+    : hashtags.slice(0, VISIBLE_HASHTAGS);
 
   return (
     <div className='hashtag-bar'>
       {revealedHashtags.map((hashtag) => (
         <Link key={hashtag} to={`/tags/${hashtag}`}>
-          #{hashtag}
+          #<span>{hashtag}</span>
         </Link>
       ))}
 

app/javascript/mastodon/components/status.jsx

@@ -546,10 +546,11 @@ class Status extends ImmutablePureComponent {
     const visibilityIcon = visibilityIconInfo[status.get('visibility')];
 
     const {statusContentProps, hashtagBar} = getHashtagBarForStatus(status);
+    const expanded = !status.get('hidden')
 
     return (
       <HotKeys handlers={handlers}>
-        <div className={classNames('status__wrapper', `status__wrapper-type-${status.get('activity_pub_type') || 'none'}` , `status__wrapper-${status.get('visibility')}`, { 'status__wrapper-reply': !!status.get('in_reply_to_id'), unread, focusable: !this.props.muted })} tabIndex={this.props.muted ? null : 0} data-featured={featured ? 'true' : null} aria-label={textForScreenReader(intl, status, rebloggedByText)} ref={this.handleRef}>
+        <div className={classNames('status__wrapper', `status__wrapper-type-${status.get('activity_pub_type') || 'none'}` , `status__wrapper-${status.get('visibility')}`, { 'status__wrapper-reply': !!status.get('in_reply_to_id'), unread, focusable: !this.props.muted })} tabIndex={this.props.muted ? null : 0} data-featured={featured ? 'true' : null} aria-label={textForScreenReader(intl, status, rebloggedByText)} ref={this.handleRef} data-nosnippe={status.getIn(['account', 'noindex'], true) || undefined}>
           {prepend}
 
           <div className={classNames('status', `status-${status.get('visibility')}`, { 'status-reply': !!status.get('in_reply_to_id'), 'status--in-thread': !!rootId, 'status--first-in-thread': previousId && (!connectUp || connectToRoot), muted: this.props.muted })} data-id={status.get('id')}>
@@ -574,7 +575,7 @@ class Status extends ImmutablePureComponent {
             <StatusContent
               status={status}
               onClick={this.handleClick}
-              expanded={!status.get('hidden')}
+              expanded={expanded}
               onExpandedToggle={this.handleExpandedToggle}
               onTranslate={this.handleTranslate}
               collapsible
@@ -584,7 +585,7 @@ class Status extends ImmutablePureComponent {
 
             {status.get('activity_pub_type') === 'Article' ? null : media}
 
-            {hashtagBar}
+            {expanded && hashtagBar}
 
             <StatusActionBar scrollKey={scrollKey} status={status} account={account} onFilter={matchedFilters ? this.handleFilterClick : null} {...other} />
           </div>

app/javascript/mastodon/features/compose/components/search.jsx

@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { PureComponent } from 'react';
 
-import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import { defineMessages, injectIntl, FormattedMessage, FormattedList } from 'react-intl';
 
 import classNames from 'classnames';
 
@@ -45,6 +45,16 @@ class Search extends PureComponent {
     options: [],
   };
 
+  defaultOptions = [
+    { label: <><mark>has:</mark> <FormattedList type='disjunction' value={['media', 'poll', 'embed']} /></>, action: e => { e.preventDefault(); this._insertText('has:') } },
+    { label: <><mark>is:</mark> <FormattedList type='disjunction' value={['reply', 'sensitive']} /></>, action: e => { e.preventDefault(); this._insertText('is:') } },
+    { label: <><mark>language:</mark> <FormattedMessage id='search_popout.language_code' defaultMessage='ISO language code' /></>, action: e => { e.preventDefault(); this._insertText('language:') } },
+    { label: <><mark>from:</mark> <FormattedMessage id='search_popout.user' defaultMessage='user' /></>, action: e => { e.preventDefault(); this._insertText('from:') } },
+    { label: <><mark>before:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('before:') } },
+    { label: <><mark>during:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('during:') } },
+    { label: <><mark>after:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('after:') } },
+  ];
+
   setRef = c => {
     this.searchForm = c;
   };
@@ -70,7 +80,7 @@ class Search extends PureComponent {
 
   handleKeyDown = (e) => {
     const { selectedOption } = this.state;
-    const options = this._getOptions();
+    const options = searchEnabled ? this._getOptions().concat(this.defaultOptions) : this._getOptions();
 
     switch(e.key) {
     case 'Escape':
@@ -100,11 +110,9 @@ class Search extends PureComponent {
       if (selectedOption === -1) {
         this._submit();
       } else if (options.length > 0) {
-        options[selectedOption].action();
+        options[selectedOption].action(e);
       }
 
-      this._unfocus();
-
       break;
     case 'Delete':
       if (selectedOption > -1 && options.length > 0) {
@@ -147,6 +155,7 @@ class Search extends PureComponent {
 
     router.history.push(`/tags/${query}`);
     onClickSearchResult(query, 'hashtag');
+    this._unfocus();
   };
 
   handleAccountClick = () => {
@@ -157,6 +166,7 @@ class Search extends PureComponent {
 
     router.history.push(`/@${query}`);
     onClickSearchResult(query, 'account');
+    this._unfocus();
   };
 
   handleURLClick = () => {
@@ -164,6 +174,7 @@ class Search extends PureComponent {
     const { value, onOpenURL } = this.props;
 
     onOpenURL(value, router.history);
+    this._unfocus();
   };
 
   handleStatusSearch = () => {
@@ -182,6 +193,8 @@ class Search extends PureComponent {
     } else if (search.get('type') === 'hashtag') {
       router.history.push(`/tags/${search.get('q')}`);
     }
+
+    this._unfocus();
   };
 
   handleForgetRecentSearchClick = search => {
@@ -194,6 +207,18 @@ class Search extends PureComponent {
     document.querySelector('.ui').parentElement.focus();
   }
 
+  _insertText (text) {
+    const { value, onChange } = this.props;
+
+    if (value === '') {
+      onChange(text);
+    } else if (value[value.length - 1] === ' ') {
+      onChange(`${value}${text}`);
+    } else {
+      onChange(`${value} ${text}`);
+    }
+  }
+
   _submit (type) {
     const { onSubmit, openInRoute } = this.props;
     const { router } = this.context;
@@ -203,6 +228,8 @@ class Search extends PureComponent {
     if (openInRoute) {
       router.history.push('/search');
     }
+
+    this._unfocus();
   }
 
   _getOptions () {
@@ -325,6 +352,20 @@ class Search extends PureComponent {
               </div>
             </>
           )}
+
+          {searchEnabled && (
+            <>
+              <h4><FormattedMessage id='search_popout.options' defaultMessage='Search options' /></h4>
+
+              <div className='search__popout__menu'>
+                {this.defaultOptions.map(({ key, label, action }, i) => (
+                  <button key={key} onMouseDown={action} className={classNames('search__popout__menu__item', { selected: selectedOption === (options.length + i) })}>
+                    {label}
+                  </button>
+                ))}
+              </div>
+            </>
+          )}
         </div>
       </div>
     );

app/javascript/mastodon/features/explore/results.jsx

@@ -108,10 +108,10 @@ class Results extends PureComponent {
     return (
       <>
         <div className='account__section-headline'>
-          <button onClick={this.handleSelectAll} className={type === 'all' && 'active'}><FormattedMessage id='search_results.all' defaultMessage='All' /></button>
-          <button onClick={this.handleSelectAccounts} className={type === 'accounts' && 'active'}><FormattedMessage id='search_results.accounts' defaultMessage='Profiles' /></button>
-          <button onClick={this.handleSelectHashtags} className={type === 'hashtags' && 'active'}><FormattedMessage id='search_results.hashtags' defaultMessage='Hashtags' /></button>
-          <button onClick={this.handleSelectStatuses} className={type === 'statuses' && 'active'}><FormattedMessage id='search_results.statuses' defaultMessage='Posts' /></button>
+          <button onClick={this.handleSelectAll} className={type === 'all' ? 'active' : undefined}><FormattedMessage id='search_results.all' defaultMessage='All' /></button>
+          <button onClick={this.handleSelectAccounts} className={type === 'accounts' ? 'active' : undefined}><FormattedMessage id='search_results.accounts' defaultMessage='Profiles' /></button>
+          <button onClick={this.handleSelectHashtags} className={type === 'hashtags' ? 'active' : undefined}><FormattedMessage id='search_results.hashtags' defaultMessage='Hashtags' /></button>
+          <button onClick={this.handleSelectStatuses} className={type === 'statuses' ? 'active' : undefined}><FormattedMessage id='search_results.statuses' defaultMessage='Posts' /></button>
         </div>
 
         <div className='explore__search-results'>

app/javascript/mastodon/features/favourites/index.jsx

@@ -8,7 +8,9 @@ import ImmutablePropTypes from 'react-immutable-proptypes';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { connect } from 'react-redux';
 
-import { fetchFavourites } from 'mastodon/actions/interactions';
+import { debounce } from 'lodash';
+
+import { fetchFavourites, expandFavourites } from 'mastodon/actions/interactions';
 import ColumnHeader from 'mastodon/components/column_header';
 import { Icon }  from 'mastodon/components/icon';
 import { LoadingIndicator } from 'mastodon/components/loading_indicator';
@@ -21,7 +23,9 @@ const messages = defineMessages({
 });
 
 const mapStateToProps = (state, props) => ({
-  accountIds: state.getIn(['user_lists', 'favourited_by', props.params.statusId]),
+  accountIds: state.getIn(['user_lists', 'favourited_by', props.params.statusId, 'items']),
+  hasMore: !!state.getIn(['user_lists', 'favourited_by', props.params.statusId, 'next']),
+  isLoading: state.getIn(['user_lists', 'favourited_by', props.params.statusId, 'isLoading'], true),
 });
 
 class Favourites extends ImmutablePureComponent {
@@ -30,6 +34,8 @@ class Favourites extends ImmutablePureComponent {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
     accountIds: ImmutablePropTypes.list,
+    hasMore: PropTypes.bool,
+    isLoading: PropTypes.bool,
     multiColumn: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
@@ -40,18 +46,16 @@ class Favourites extends ImmutablePureComponent {
     }
   }
 
-  UNSAFE_componentWillReceiveProps (nextProps) {
-    if (nextProps.params.statusId !== this.props.params.statusId && nextProps.params.statusId) {
-      this.props.dispatch(fetchFavourites(nextProps.params.statusId));
-    }
-  }
-
   handleRefresh = () => {
     this.props.dispatch(fetchFavourites(this.props.params.statusId));
   };
 
+  handleLoadMore = debounce(() => {
+    this.props.dispatch(expandFavourites(this.props.params.statusId));
+  }, 300, { leading: true });
+
   render () {
-    const { intl, accountIds, multiColumn } = this.props;
+    const { intl, accountIds, hasMore, isLoading, multiColumn } = this.props;
 
     if (!accountIds) {
       return (
@@ -75,6 +79,9 @@ class Favourites extends ImmutablePureComponent {
 
         <ScrollableList
           scrollKey='favourites'
+          onLoadMore={this.handleLoadMore}
+          hasMore={hasMore}
+          isLoading={isLoading}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/home_timeline/components/critical_update_banner.tsx

@@ -0,0 +1,26 @@
+import { FormattedMessage } from 'react-intl';
+
+export const CriticalUpdateBanner = () => (
+  <div className='warning-banner'>
+    <div className='warning-banner__message'>
+      <h1>
+        <FormattedMessage
+          id='home.pending_critical_update.title'
+          defaultMessage='Critical security update available!'
+        />
+      </h1>
+      <p>
+        <FormattedMessage
+          id='home.pending_critical_update.body'
+          defaultMessage='Please update your Mastodon server as soon as possible!'
+        />{' '}
+        <a href='/admin/software_updates'>
+          <FormattedMessage
+            id='home.pending_critical_update.link'
+            defaultMessage='See updates'
+          />
+        </a>
+      </p>
+    </div>
+  </div>
+);

app/javascript/mastodon/features/home_timeline/index.jsx

@@ -14,7 +14,7 @@ import { fetchAnnouncements, toggleShowAnnouncements } from 'mastodon/actions/an
 import { IconWithBadge } from 'mastodon/components/icon_with_badge';
 import { NotSignedInIndicator } from 'mastodon/components/not_signed_in_indicator';
 import AnnouncementsContainer from 'mastodon/features/getting_started/containers/announcements_container';
-import { me } from 'mastodon/initial_state';
+import { me, criticalUpdatesPending } from 'mastodon/initial_state';
 
 import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
 import { expandHomeTimeline } from '../../actions/timelines';
@@ -23,6 +23,7 @@ import ColumnHeader from '../../components/column_header';
 import StatusListContainer from '../ui/containers/status_list_container';
 
 import { ColumnSettings } from './components/column_settings';
+import { CriticalUpdateBanner } from './components/critical_update_banner';
 import { ExplorePrompt } from './components/explore_prompt';
 
 const messages = defineMessages({
@@ -156,8 +157,9 @@ class HomeTimeline extends PureComponent {
     const { intl, hasUnread, columnId, multiColumn, tooSlow, hasAnnouncements, unreadAnnouncements, showAnnouncements } = this.props;
     const pinned = !!columnId;
     const { signedIn } = this.context.identity;
+    const banners = [];
 
-    let announcementsButton, banner;
+    let announcementsButton;
 
     if (hasAnnouncements) {
       announcementsButton = (
@@ -173,8 +175,12 @@ class HomeTimeline extends PureComponent {
       );
     }
 
+    if (criticalUpdatesPending) {
+      banners.push(<CriticalUpdateBanner key='critical-update-banner' />);
+    }
+
     if (tooSlow) {
-      banner = <ExplorePrompt />;
+      banners.push(<ExplorePrompt key='explore-prompt' />);
     }
 
     return (
@@ -196,7 +202,7 @@ class HomeTimeline extends PureComponent {
 
         {signedIn ? (
           <StatusListContainer
-            prepend={banner}
+            prepend={banners}
             alwaysPrepend
             trackScroll={!pinned}
             scrollKey={`home_timeline-${columnId}`}

app/javascript/mastodon/features/reblogs/index.jsx

@@ -8,9 +8,11 @@ import ImmutablePropTypes from 'react-immutable-proptypes';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { connect } from 'react-redux';
 
+import { debounce } from 'lodash';
+
 import { Icon }  from 'mastodon/components/icon';
 
-import { fetchReblogs } from '../../actions/interactions';
+import { fetchReblogs, expandReblogs } from '../../actions/interactions';
 import ColumnHeader from '../../components/column_header';
 import { LoadingIndicator } from '../../components/loading_indicator';
 import ScrollableList from '../../components/scrollable_list';
@@ -22,7 +24,9 @@ const messages = defineMessages({
 });
 
 const mapStateToProps = (state, props) => ({
-  accountIds: state.getIn(['user_lists', 'reblogged_by', props.params.statusId]),
+  accountIds: state.getIn(['user_lists', 'reblogged_by', props.params.statusId, 'items']),
+  hasMore: !!state.getIn(['user_lists', 'reblogged_by', props.params.statusId, 'next']),
+  isLoading: state.getIn(['user_lists', 'reblogged_by', props.params.statusId, 'isLoading'], true),
 });
 
 class Reblogs extends ImmutablePureComponent {
@@ -31,6 +35,8 @@ class Reblogs extends ImmutablePureComponent {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
     accountIds: ImmutablePropTypes.list,
+    hasMore: PropTypes.bool,
+    isLoading: PropTypes.bool,
     multiColumn: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
@@ -39,20 +45,18 @@ class Reblogs extends ImmutablePureComponent {
     if (!this.props.accountIds) {
       this.props.dispatch(fetchReblogs(this.props.params.statusId));
     }
-  }
-
-  UNSAFE_componentWillReceiveProps(nextProps) {
-    if (nextProps.params.statusId !== this.props.params.statusId && nextProps.params.statusId) {
-      this.props.dispatch(fetchReblogs(nextProps.params.statusId));
-    }
-  }
+  };
 
   handleRefresh = () => {
     this.props.dispatch(fetchReblogs(this.props.params.statusId));
   };
 
+  handleLoadMore = debounce(() => {
+    this.props.dispatch(expandReblogs(this.props.params.statusId));
+  }, 300, { leading: true });
+
   render () {
-    const { intl, accountIds, multiColumn } = this.props;
+    const { intl, accountIds, hasMore, isLoading, multiColumn } = this.props;
 
     if (!accountIds) {
       return (
@@ -76,6 +80,9 @@ class Reblogs extends ImmutablePureComponent {
 
         <ScrollableList
           scrollKey='reblogs'
+          onLoadMore={this.handleLoadMore}
+          hasMore={hasMore}
+          isLoading={isLoading}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/status/components/detailed_status.jsx

@@ -299,6 +299,7 @@ class DetailedStatus extends ImmutablePureComponent {
     }
 
     const {statusContentProps, hashtagBar} = getHashtagBarForStatus(status);
+    const expanded = !status.get('hidden')
 
     return (
       <div style={outerStyle}>
@@ -324,7 +325,7 @@ class DetailedStatus extends ImmutablePureComponent {
 
           {status.get('activity_pub_type') === 'Article' ? null : media}
 
-          {hashtagBar}
+          {expanded && hashtagBar}
 
           <div className='detailed-status__meta'>
             <a className='detailed-status__datetime' href={status.get('url')} target='_blank' rel='noopener noreferrer'>

app/javascript/mastodon/features/status/index.jsx

@@ -568,7 +568,7 @@ class Status extends ImmutablePureComponent {
         onMoveUp={this.handleMoveUp}
         onMoveDown={this.handleMoveDown}
         contextType='thread'
-        previousId={i > 0 && list.get(i - 1)}
+        previousId={i > 0 ? list.get(i - 1) : undefined}
         nextId={list.get(i + 1) || (ancestors && statusId)}
         rootId={statusId}
       />

app/javascript/mastodon/features/ui/components/modal_root.jsx

@@ -115,7 +115,10 @@ export default class ModalRoot extends PureComponent {
         {visible && (
           <>
             <BundleContainer fetchComponent={MODAL_COMPONENTS[type]} loading={this.renderLoading(type)} error={this.renderError} renderDelay={200}>
-              {(SpecificComponent) => <SpecificComponent {...props} onChangeBackgroundColor={this.setBackgroundColor} onClose={this.handleClose} ref={this.setModalRef} />}
+              {(SpecificComponent) => {
+                const ref = typeof SpecificComponent !== 'function' ? this.setModalRef : undefined;
+                return <SpecificComponent {...props} onChangeBackgroundColor={this.setBackgroundColor} onClose={this.handleClose} ref={ref} />
+              }}
             </BundleContainer>
 
             <Helmet>

app/javascript/mastodon/features/ui/components/navigation_panel.jsx

@@ -31,6 +31,7 @@ const messages = defineMessages({
   about: { id: 'navigation_bar.about', defaultMessage: 'About' },
   search: { id: 'navigation_bar.search', defaultMessage: 'Search' },
   advancedInterface: { id: 'navigation_bar.advanced_interface', defaultMessage: 'Open in advanced web interface' },
+  openedInClassicInterface: { id: 'navigation_bar.opened_in_classic_interface', defaultMessage: 'Posts, accounts, and other specific pages are opened by default in the classic web interface.' },
 });
 
 class NavigationPanel extends Component {
@@ -57,12 +58,17 @@ class NavigationPanel extends Component {
         <div className='navigation-panel__logo'>
           <Link to='/' className='column-link column-link--logo'><WordmarkLogo /></Link>
 
-          {transientSingleColumn && (
-            <a href={`/deck${location.pathname}`} className='button button--block'>
-              {intl.formatMessage(messages.advancedInterface)}
-            </a>
+          {transientSingleColumn ? (
+            <div class='switch-to-advanced'>
+              {intl.formatMessage(messages.openedInClassicInterface)}
+              {" "}
+              <a href={`/deck${location.pathname}`} class='switch-to-advanced__toggle'>
+                {intl.formatMessage(messages.advancedInterface)}
+              </a>
+            </div>
+          ) : (
+            <hr />
           )}
-          <hr />
         </div>
 
         {signedIn && (

app/javascript/mastodon/features/ui/components/report_modal.jsx

@@ -62,7 +62,7 @@ class ReportModal extends ImmutablePureComponent {
     dispatch(submitReport({
       account_id: accountId,
       status_ids: selectedStatusIds.toArray(),
-      selected_domains: selectedDomains.toArray(),
+      forward_to_domains: selectedDomains.toArray(),
       comment,
       forward: selectedDomains.size > 0,
       category,

app/javascript/mastodon/initial_state.js

@@ -87,6 +87,7 @@
  * @typedef InitialState
  * @property {Record<string, Account>} accounts
  * @property {InitialStateLanguage[]} languages
+ * @property {boolean=} critical_updates_pending
  * @property {InitialStateMeta} meta
  */
 
@@ -143,6 +144,7 @@ export const useBlurhash = getMeta('use_blurhash');
 export const usePendingItems = getMeta('use_pending_items');
 export const version = getMeta('version');
 export const languages = initialState?.languages;
+export const criticalUpdatesPending = initialState?.critical_updates_pending;
 // @ts-expect-error
 export const statusPageUrl = getMeta('status_page_url');
 export const sso_redirect = getMeta('sso_redirect');

app/javascript/mastodon/locales/en.json

@@ -137,6 +137,7 @@
   "compose.language.search": "Search languages...",
   "compose.published.body": "Post published.",
   "compose.published.open": "Open",
+  "compose.saved.body": "Post saved.",
   "compose_form.direct_message_warning_learn_more": "Learn more",
   "compose_form.encryption_warning": "Posts on Mastodon are not end-to-end encrypted. Do not share any sensitive information over Mastodon.",
   "compose_form.hashtag_warning": "This post won't be listed under any hashtag as it is not public. Only public posts can be searched by hashtag.",
@@ -314,6 +315,9 @@
   "home.explore_prompt.body": "Your home feed will have a mix of posts from the hashtags you've chosen to follow, the people you've chosen to follow, and the posts they boost. If that feels too quiet, you may want to:",
   "home.explore_prompt.title": "This is your home base within Mastodon.",
   "home.hide_announcements": "Hide announcements",
+  "home.pending_critical_update.body": "Please update your Mastodon server as soon as possible!",
+  "home.pending_critical_update.link": "See updates",
+  "home.pending_critical_update.title": "Critical security update available!",
   "home.show_announcements": "Show announcements",
   "interaction_modal.description.favourite": "With an account on Mastodon, you can favorite this post to let the author know you appreciate it and save it for later.",
   "interaction_modal.description.follow": "With an account on Mastodon, you can follow {name} to receive their posts in your home feed.",
@@ -417,6 +421,7 @@
   "navigation_bar.logout": "Logout",
   "navigation_bar.menu": "Menu",
   "navigation_bar.mutes": "Muted users",
+  "navigation_bar.opened_in_classic_interface": "Posts, accounts, and other specific pages are opened by default in the classic web interface.",
   "navigation_bar.personal": "Personal",
   "navigation_bar.pins": "Pinned posts",
   "navigation_bar.preferences": "Preferences",
@@ -593,8 +598,12 @@
   "search.quick_action.open_url": "Open URL in Mastodon",
   "search.quick_action.status_search": "Posts matching {x}",
   "search.search_or_paste": "Search or paste URL",
+  "search_popout.language_code": "ISO language code",
+  "search_popout.options": "Search options",
   "search_popout.quick_actions": "Quick actions",
   "search_popout.recent": "Recent searches",
+  "search_popout.specific_date": "specific date",
+  "search_popout.user": "user",
   "search_results.accounts": "Profiles",
   "search_results.all": "All",
   "search_results.hashtags": "Hashtags",

app/javascript/mastodon/locales/fr.json

@@ -414,6 +414,7 @@
   "navigation_bar.lists": "Listes",
   "navigation_bar.logout": "Déconnexion",
   "navigation_bar.mutes": "Comptes masqués",
+  "navigation_bar.opened_in_classic_interface": "Les messages, les comptes et les pages spécifiques sont ouvertes dans l’interface classique.",
   "navigation_bar.personal": "Personnel",
   "navigation_bar.pins": "Messages épinglés",
   "navigation_bar.preferences": "Préférences",

app/javascript/mastodon/reducers/user_lists.js

@@ -45,8 +45,18 @@ import {
   BLOCKS_EXPAND_FAIL,
 } from '../actions/blocks';
 import {
+  REBLOGS_FETCH_REQUEST,
   REBLOGS_FETCH_SUCCESS,
+  REBLOGS_FETCH_FAIL,
+  REBLOGS_EXPAND_REQUEST,
+  REBLOGS_EXPAND_SUCCESS,
+  REBLOGS_EXPAND_FAIL,
+  FAVOURITES_FETCH_REQUEST,
   FAVOURITES_FETCH_SUCCESS,
+  FAVOURITES_FETCH_FAIL,
+  FAVOURITES_EXPAND_REQUEST,
+  FAVOURITES_EXPAND_SUCCESS,
+  FAVOURITES_EXPAND_FAIL,
 } from '../actions/interactions';
 import {
   MUTES_FETCH_REQUEST,
@@ -134,9 +144,25 @@ export default function userLists(state = initialState, action) {
   case FOLLOWING_EXPAND_FAIL:
     return state.setIn(['following', action.id, 'isLoading'], false);
   case REBLOGS_FETCH_SUCCESS:
-    return state.setIn(['reblogged_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
+    return normalizeList(state, ['reblogged_by', action.id], action.accounts, action.next);
+  case REBLOGS_EXPAND_SUCCESS:
+    return appendToList(state, ['reblogged_by', action.id], action.accounts, action.next);
+  case REBLOGS_FETCH_REQUEST:
+  case REBLOGS_EXPAND_REQUEST:
+    return state.setIn(['reblogged_by', action.id, 'isLoading'], true);
+  case REBLOGS_FETCH_FAIL:
+  case REBLOGS_EXPAND_FAIL:
+    return state.setIn(['reblogged_by', action.id, 'isLoading'], false);
   case FAVOURITES_FETCH_SUCCESS:
-    return state.setIn(['favourited_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
+    return normalizeList(state, ['favourited_by', action.id], action.accounts, action.next);
+  case FAVOURITES_EXPAND_SUCCESS:
+    return appendToList(state, ['favourited_by', action.id], action.accounts, action.next);
+  case FAVOURITES_FETCH_REQUEST:
+  case FAVOURITES_EXPAND_REQUEST:
+    return state.setIn(['favourited_by', action.id, 'isLoading'], true);
+  case FAVOURITES_FETCH_FAIL:
+  case FAVOURITES_EXPAND_FAIL:
+    return state.setIn(['favourited_by', action.id, 'isLoading'], false);
   case NOTIFICATIONS_UPDATE:
     return action.notification.type === 'follow_request' ? normalizeFollowRequest(state, action.notification) : state;
   case FOLLOW_REQUESTS_FETCH_SUCCESS:

app/javascript/mastodon/test_setup.js

@@ -1 +1 @@
-import '@testing-library/jest-dom/extend-expect';
+import '@testing-library/jest-dom';

app/javascript/packs/public.jsx

@@ -7,8 +7,6 @@ import { defineMessages } from 'react-intl';
 
 import { delegate }  from '@rails/ujs';
 import axios from 'axios';
-import escapeTextContentForBrowser from 'escape-html';
-import { createBrowserHistory }  from 'history';
 import { throttle } from 'lodash';
 
 import { start } from '../mastodon/common';
@@ -48,23 +46,6 @@ window.addEventListener('message', e => {
 function loaded() {
   const { messages: localeData } = getLocale();
 
-  const scrollToDetailedStatus = () => {
-    const history = createBrowserHistory();
-    const detailedStatuses = document.querySelectorAll('.public-layout .detailed-status');
-    const location = history.location;
-
-    if (detailedStatuses.length === 1 && (!location.state || !location.state.scrolledToDetailedStatus)) {
-      detailedStatuses[0].scrollIntoView();
-      history.replace(location.pathname, { ...location.state, scrolledToDetailedStatus: true });
-    }
-  };
-
-  const getEmojiAnimationHandler = (swapTo) => {
-    return ({ target }) => {
-      target.src = target.getAttribute(swapTo);
-    };
-  };
-
   const locale = document.documentElement.lang;
 
   const dateTimeFormat = new Intl.DateTimeFormat(locale, {
@@ -158,27 +139,21 @@ function loaded() {
         const root = createRoot(content);
         root.render(<MediaContainer locale={locale} components={reactComponents} />);
         document.body.appendChild(content);
-        scrollToDetailedStatus();
       })
       .catch(error => {
         console.error(error);
-        scrollToDetailedStatus();
       });
-  } else {
-    scrollToDetailedStatus();
   }
 
-  delegate(document, '#user_account_attributes_username', 'input', throttle(() => {
-    const username = document.getElementById('user_account_attributes_username');
-
-    if (username.value && username.value.length > 0) {
-      axios.get('/api/v1/accounts/lookup', { params: { acct: username.value } }).then(() => {
-        username.setCustomValidity(formatMessage(messages.usernameTaken));
+  delegate(document, '#user_account_attributes_username', 'input', throttle(({ target }) => {
+    if (target.value && target.value.length > 0) {
+      axios.get('/api/v1/accounts/lookup', { params: { acct: target.value } }).then(() => {
+        target.setCustomValidity(formatMessage(messages.usernameTaken));
       }).catch(() => {
-        username.setCustomValidity('');
+        target.setCustomValidity('');
       });
     } else {
-      username.setCustomValidity('');
+      target.setCustomValidity('');
     }
   }, 500, { leading: false, trailing: true }));
 
@@ -196,9 +171,6 @@ function loaded() {
     }
   });
 
-  delegate(document, '.custom-emoji', 'mouseover', getEmojiAnimationHandler('data-original'));
-  delegate(document, '.custom-emoji', 'mouseout', getEmojiAnimationHandler('data-static'));
-
   delegate(document, '.status__content__spoiler-link', 'click', function() {
     const statusEl = this.parentNode.parentNode;
 
@@ -220,17 +192,6 @@ function loaded() {
   });
 }
 
-delegate(document, '#account_display_name', 'input', ({ target }) => {
-  const name = document.querySelector('.card .display-name strong');
-  if (name) {
-    if (target.value) {
-      name.innerHTML = emojify(escapeTextContentForBrowser(target.value));
-    } else {
-      name.textContent = target.dataset.default;
-    }
-  }
-});
-
 delegate(document, '#edit_profile input[type=file]', 'change', ({ target }) => {
   const avatar = document.getElementById(target.id + '-preview');
   const [file] = target.files || [];
@@ -239,33 +200,6 @@ delegate(document, '#edit_profile input[type=file]', 'change', ({ target }) => {
   avatar.src = url;
 });
 
-const getProfileAvatarAnimationHandler = (swapTo) => {
-  //animate avatar gifs on the profile page when moused over
-  return ({ target }) => {
-    const swapSrc = target.getAttribute(swapTo);
-    //only change the img source if autoplay is off and the image src is actually different
-    if(target.getAttribute('data-autoplay') !== 'true' && target.src !== swapSrc) {
-      target.src = swapSrc;
-    }
-  };
-};
-
-delegate(document, 'img#profile_page_avatar', 'mouseover', getProfileAvatarAnimationHandler('data-original'));
-
-delegate(document, 'img#profile_page_avatar', 'mouseout', getProfileAvatarAnimationHandler('data-static'));
-
-delegate(document, '#account_locked', 'change', ({ target }) => {
-  const lock = document.querySelector('.card .display-name i');
-
-  if (lock) {
-    if (target.checked) {
-      delete lock.dataset.hidden;
-    } else {
-      lock.dataset.hidden = 'true';
-    }
-  }
-});
-
 delegate(document, '.input-copy input', 'click', ({ target }) => {
   target.focus();
   target.select();
@@ -325,6 +259,9 @@ delegate(document, '.sidebar__toggle__icon', 'keydown', e => {
   }
 });
 
+delegate(document, '.custom-emoji', 'mouseover', ({ target }) => target.src = target.getAttribute('data-original'));
+delegate(document, '.custom-emoji', 'mouseout', ({ target }) => target.src = target.getAttribute('data-static'));
+
 // Empty the honeypot fields in JS in case something like an extension
 // automatically filled them.
 delegate(document, '#registration_new_user,#new_user', 'submit', () => {

app/javascript/styles/mastodon/accounts.scss

@@ -188,6 +188,7 @@
 }
 
 .information-badge,
+.simple_form .overridden,
 .simple_form .recommended,
 .simple_form .not_recommended {
   display: inline-block;
@@ -204,6 +205,7 @@
 }
 
 .information-badge,
+.simple_form .overridden,
 .simple_form .recommended,
 .simple_form .not_recommended {
   background-color: rgba($ui-secondary-color, 0.1);

app/javascript/styles/mastodon/admin.scss

@@ -143,6 +143,11 @@ $content-width: 840px;
         }
       }
 
+      .warning a {
+        color: $gold-star;
+        font-weight: 700;
+      }
+
       .simple-navigation-active-leaf a {
         color: $primary-text-color;
         background-color: $ui-highlight-color;

app/javascript/styles/mastodon/components.scss

@@ -228,20 +228,20 @@
       color: lighten($lighter-text-color, 7%);
       background-color: transparent;
     }
+  }
 
-    &.active {
-      color: $highlight-text-color;
+  &.active {
+    color: $highlight-text-color;
 
-      &:hover,
-      &:active,
-      &:focus {
-        color: $highlight-text-color;
-        background-color: transparent;
-      }
+    &:hover,
+    &:active,
+    &:focus {
+      color: $highlight-text-color;
+      background-color: transparent;
+    }
 
-      &.disabled {
-        color: lighten($highlight-text-color, 13%);
-      }
+    &.disabled {
+      color: lighten($highlight-text-color, 13%);
     }
   }
 
@@ -2381,6 +2381,7 @@ $ui-header-height: 55px;
 
   .filter-form {
     display: flex;
+    flex-wrap: wrap;
   }
 
   .autosuggest-textarea__textarea {
@@ -3270,6 +3271,22 @@ $ui-header-height: 55px;
   border-color: $ui-highlight-color;
 }
 
+.switch-to-advanced {
+  color: $light-text-color;
+  background-color: $ui-base-color;
+  padding: 15px;
+  border-radius: 4px;
+  margin-top: 4px;
+  margin-bottom: 12px;
+  font-size: 13px;
+  line-height: 18px;
+
+  .switch-to-advanced__toggle {
+    color: $ui-button-tertiary-color;
+    font-weight: bold;
+  }
+}
+
 .column-link {
   background: lighten($ui-base-color, 8%);
   color: $primary-text-color;
@@ -4991,6 +5008,12 @@ a.status-card {
     }
 
     &__menu {
+      margin-bottom: 20px;
+
+      &:last-child {
+        margin-bottom: 0;
+      }
+
       &__message {
         color: $dark-text-color;
         padding: 0 10px;
@@ -5125,6 +5148,7 @@ a.status-card {
 
     &.active {
       transform: rotate(90deg);
+      opacity: 1;
     }
 
     &:hover {
@@ -8539,6 +8563,44 @@ noscript {
     }
   }
 
+  &__choices {
+    display: flex;
+    gap: 40px;
+
+    &__choice {
+      flex: 1;
+      box-sizing: border-box;
+
+      h3 {
+        margin-bottom: 20px;
+      }
+
+      p {
+        color: $darker-text-color;
+        margin-bottom: 20px;
+        font-size: 15px;
+      }
+
+      .button {
+        margin-bottom: 10px;
+
+        &:last-child {
+          margin-bottom: 0;
+        }
+      }
+    }
+  }
+
+  @media screen and (max-width: $no-gap-breakpoint - 1px) {
+    &__choices {
+      flex-direction: column;
+
+      &__choice {
+        margin-top: 40px;
+      }
+    }
+  }
+
   .link-button {
     font-size: inherit;
     display: inline;
@@ -8798,7 +8860,8 @@ noscript {
   }
 }
 
-.dismissable-banner {
+.dismissable-banner,
+.warning-banner {
   position: relative;
   margin: 10px;
   margin-bottom: 5px;
@@ -8876,6 +8939,21 @@ noscript {
   }
 }
 
+.warning-banner {
+  border: 1px solid $warning-red;
+  background: rgba($warning-red, 0.15);
+
+  &__message {
+    h1 {
+      color: $warning-red;
+    }
+
+    a {
+      color: $primary-text-color;
+    }
+  }
+}
+
 .image {
   position: relative;
   overflow: hidden;
@@ -9263,20 +9341,24 @@ noscript {
   display: flex;
   flex-wrap: wrap;
   font-size: 14px;
+  line-height: 18px;
   gap: 4px;
+  color: $darker-text-color;
 
   a {
     display: inline-flex;
-    border-radius: 4px;
-    background: rgba($highlight-text-color, 0.2);
-    color: $highlight-text-color;
-    padding: 0.4em 0.6em;
+    color: inherit;
     text-decoration: none;
 
-    &:hover,
-    &:focus,
-    &:active {
-      background: rgba($highlight-text-color, 0.3);
+    &:hover span {
+      text-decoration: underline;
     }
   }
+
+  .link-button {
+    color: inherit;
+    font-size: inherit;
+    line-height: inherit;
+    padding: 0;
+  }
 }

app/javascript/styles/mastodon/forms.scss

@@ -103,6 +103,7 @@ code {
         }
       }
 
+      .overridden,
       .recommended,
       .not_recommended {
         position: absolute;
@@ -1185,14 +1186,14 @@ code {
   }
 
   li:first-child .label {
-    left: auto;
     inset-inline-start: 0;
+    inset-inline-end: auto;
     text-align: start;
     transform: none;
   }
 
   li:last-child .label {
-    left: auto;
+    inset-inline-start: auto;
     inset-inline-end: 0;
     text-align: end;
     transform: none;

app/javascript/styles/mastodon/tables.scss

@@ -12,6 +12,11 @@
     border-top: 1px solid $ui-base-color;
     text-align: start;
     background: darken($ui-base-color, 4%);
+
+    &.critical {
+      font-weight: 700;
+      color: $gold-star;
+    }
   }
 
   & > thead > tr > th {

app/lib/admin/metrics/dimension/software_versions_dimension.rb

@@ -10,7 +10,7 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim
   protected
 
   def perform_query
-    [mastodon_version, ruby_version, postgresql_version, redis_version]
+    [mastodon_version, ruby_version, postgresql_version, redis_version, elasticsearch_version].compact
   end
 
   def mastodon_version
@@ -57,6 +57,22 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim
     }
   end
 
+  def elasticsearch_version
+    return unless Chewy.enabled?
+
+    client_info = Chewy.client.info
+    version = client_info.dig('version', 'number')
+
+    {
+      key: 'elasticsearch',
+      human_key: client_info.dig('version', 'distribution') == 'opensearch' ? 'OpenSearch' : 'Elasticsearch',
+      value: version,
+      human_value: version,
+    }
+  rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
+    nil
+  end
+
   def redis_info
     @redis_info ||= if redis.is_a?(Redis::Namespace)
                       redis.redis.info

app/lib/admin/system_check.rb

@@ -2,6 +2,7 @@
 
 class Admin::SystemCheck
   ACTIVE_CHECKS = [
+    Admin::SystemCheck::SoftwareVersionCheck,
     Admin::SystemCheck::MediaPrivacyCheck,
     Admin::SystemCheck::DatabaseSchemaCheck,
     Admin::SystemCheck::SidekiqProcessCheck,

app/lib/admin/system_check/elasticsearch_check.rb

@@ -6,6 +6,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
     AccountsIndex,
     TagsIndex,
     StatusesIndex,
+    PublicStatusesIndex,
   ].freeze
 
   def skip?
@@ -41,7 +42,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
       Admin::SystemCheck::Message.new(:elasticsearch_health_red)
     elsif cluster_health['number_of_nodes'] < 2 && es_preset != 'single_node_cluster'
       Admin::SystemCheck::Message.new(:elasticsearch_preset_single_node, nil, 'https://docs.joinmastodon.org/admin/optional/elasticsearch/#scaling')
-    elsif Chewy.client.indices.get_settings['chewy_specifications'].dig('settings', 'index', 'number_of_replicas')&.to_i&.positive? && es_preset == 'single_node_cluster'
+    elsif Chewy.client.indices.get_settings[Chewy::Stash::Specification.index_name]&.dig('settings', 'index', 'number_of_replicas')&.to_i&.positive? && es_preset == 'single_node_cluster'
       Admin::SystemCheck::Message.new(:elasticsearch_reset_chewy)
     elsif cluster_health['status'] == 'yellow'
       Admin::SystemCheck::Message.new(:elasticsearch_health_yellow)
@@ -85,7 +86,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
 
   def mismatched_indexes
     @mismatched_indexes ||= INDEXES.filter_map do |klass|
-      klass.index_name if Chewy.client.indices.get_mapping[klass.index_name]&.deep_symbolize_keys != klass.mappings_hash
+      klass.base_name if Chewy.client.indices.get_mapping[klass.index_name]&.deep_symbolize_keys != klass.mappings_hash
     end
   end
 

app/lib/admin/system_check/software_version_check.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class Admin::SystemCheck::SoftwareVersionCheck < Admin::SystemCheck::BaseCheck
+  include RoutingHelper
+
+  def skip?
+    !current_user.can?(:view_devops) || !SoftwareUpdate.check_enabled?
+  end
+
+  def pass?
+    software_updates.empty?
+  end
+
+  def message
+    if software_updates.any?(&:urgent?)
+      Admin::SystemCheck::Message.new(:software_version_critical_check, nil, admin_software_updates_path, true)
+    else
+      Admin::SystemCheck::Message.new(:software_version_patch_check, nil, admin_software_updates_path)
+    end
+  end
+
+  private
+
+  def software_updates
+    @software_updates ||= SoftwareUpdate.pending_to_a.filter { |update| update.urgent? || update.patch_type? }
+  end
+end

app/lib/importer/accounts_index_importer.rb

@@ -4,10 +4,10 @@ class Importer::AccountsIndexImporter < Importer::BaseImporter
   def import!
     scope.includes(:account_stat).find_in_batches(batch_size: @batch_size) do |tmp|
       in_work_unit(tmp) do |accounts|
-        bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: accounts).bulk_body
+        bulk = build_bulk_body(accounts)
 
-        indexed = bulk.count { |entry| entry[:index] }
-        deleted = bulk.count { |entry| entry[:delete] }
+        indexed = bulk.size
+        deleted = 0
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/importer/base_importer.rb

@@ -70,8 +70,16 @@ class Importer::BaseImporter
 
   protected
 
-  def in_work_unit(*args, &block)
-    work_unit = Concurrent::Promises.future_on(@executor, *args, &block)
+  def build_bulk_body(to_import)
+    # Specialize `Chewy::Index::Import::BulkBuilder#bulk_body` to avoid a few
+    # inefficiencies, as none of our fields or join fields and we do not need
+    # `BulkBuilder`'s versatility.
+    crutches = Chewy::Index::Crutch::Crutches.new index, to_import
+    to_import.map { |object| { index: { _id: object.id, data: index.compose(object, crutches, fields: []) } } }
+  end
+
+  def in_work_unit(...)
+    work_unit = Concurrent::Promises.future_on(@executor, ...)
 
     work_unit.on_fulfillment!(&@on_progress)
     work_unit.on_rejection!(&@on_failure)

app/lib/importer/instances_index_importer.rb

@@ -4,10 +4,10 @@ class Importer::InstancesIndexImporter < Importer::BaseImporter
   def import!
     index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp|
       in_work_unit(tmp) do |instances|
-        bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: instances).bulk_body
+        bulk = build_bulk_body(instances)
 
-        indexed = bulk.count { |entry| entry[:index] }
-        deleted = bulk.count { |entry| entry[:delete] }
+        indexed = bulk.size
+        deleted = 0
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/importer/public_statuses_index_importer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Importer::PublicStatusesIndexImporter < Importer::BaseImporter
+  def import!
+    scope.select(:id).find_in_batches(batch_size: @batch_size) do |batch|
+      in_work_unit(batch.pluck(:id)) do |status_ids|
+        bulk = ActiveRecord::Base.connection_pool.with_connection do
+          build_bulk_body(index.adapter.default_scope.where(id: status_ids))
+        end
+
+        indexed = bulk.size
+        deleted = 0
+
+        Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
+
+        [indexed, deleted]
+      end
+    end
+
+    wait!
+  end
+
+  private
+
+  def index
+    PublicStatusesIndex
+  end
+
+  def scope
+    Status.indexable
+  end
+end

app/lib/importer/statuses_index_importer.rb

@@ -13,32 +13,25 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
 
       scope.find_in_batches(batch_size: @batch_size) do |tmp|
         in_work_unit(tmp.map(&:status_id)) do |status_ids|
-          bulk = ActiveRecord::Base.connection_pool.with_connection do
-            Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body
-          end
-
-          indexed = 0
           deleted = 0
 
-          # We can't use the delete_if proc to do the filtering because delete_if
-          # is called before rendering the data and we need to filter based
-          # on the results of the filter, so this filtering happens here instead
-          bulk.map! do |entry|
-            new_entry = if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank?
-                          { delete: entry[:index].except(:data) }
-                        else
-                          entry
-                        end
-
-            if new_entry[:index]
-              indexed += 1
-            else
-              deleted += 1
+          bulk = ActiveRecord::Base.connection_pool.with_connection do
+            to_index = index.adapter.default_scope.where(id: status_ids)
+            crutches = Chewy::Index::Crutch::Crutches.new index, to_index
+            to_index.map do |object|
+              # This is unlikely to happen, but the post may have been
+              # un-interacted with since it was queued for indexing
+              if object.searchable_by.empty?
+                deleted += 1
+                { delete: { _id: object.id } }
+              else
+                { index: { _id: object.id, data: index.compose(object, crutches, fields: []) } }
+              end
             end
-
-            new_entry
           end
 
+          indexed = bulk.size - deleted
+
           Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 
           [indexed, deleted]

app/lib/importer/tags_index_importer.rb

@@ -4,10 +4,10 @@ class Importer::TagsIndexImporter < Importer::BaseImporter
   def import!
     index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp|
       in_work_unit(tmp) do |tags|
-        bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: tags).bulk_body
+        bulk = build_bulk_body(tags)
 
-        indexed = bulk.count { |entry| entry[:index] }
-        deleted = bulk.count { |entry| entry[:delete] }
+        indexed = bulk.size
+        deleted = 0
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/plain_text_formatter.rb

@@ -2,6 +2,7 @@
 
 class PlainTextFormatter
   include ActionView::Helpers::TextHelper
+
   NEWLINE_TAGS_RE = %r{(<br />|<br>|</p>)+}
 
   attr_reader :text, :local
@@ -17,7 +18,10 @@ class PlainTextFormatter
     if local?
       text
     else
-      html_entities.decode(strip_tags(insert_newlines)).chomp
+      node = Nokogiri::HTML.fragment(insert_newlines)
+      # Elements that are entirely removed with our Sanitize config
+      node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
+      node.text.chomp
     end
   end
 
@@ -26,8 +30,4 @@ class PlainTextFormatter
   def insert_newlines
     text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
   end
-
-  def html_entities
-    HTMLEntities.new
-  end
 end

app/lib/search_query_parser.rb

@@ -6,10 +6,10 @@ class SearchQueryParser < Parslet::Parser
   rule(:colon)     { str(':') }
   rule(:space)     { match('\s').repeat(1) }
   rule(:operator)  { (str('+') | str('-')).as(:operator) }
-  rule(:prefix)    { (term >> colon).as(:prefix) }
+  rule(:prefix)    { term >> colon }
   rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
   rule(:phrase)    { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
-  rule(:clause)    { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
+  rule(:clause)    { (operator.maybe >> prefix.maybe.as(:prefix) >> (phrase | term | shortcode)).as(:clause) | prefix.as(:clause) | quote.as(:junk) }
   rule(:query)     { (clause >> space.maybe).repeat.as(:query) }
   root(:query)
 end

app/lib/search_query_transformer.rb

@@ -1,58 +1,109 @@
 # frozen_string_literal: true
 
 class SearchQueryTransformer < Parslet::Transform
+  SUPPORTED_PREFIXES = %w(
+    has
+    is
+    language
+    from
+    before
+    after
+    during
+    in
+  ).freeze
+
   class Query
-    attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses
+    def initialize(clauses, options = {})
+      raise ArgumentError if options[:current_account].nil?
+
+      @clauses = clauses
+      @options = options
 
-    def initialize(clauses)
-      grouped = clauses.chunk(&:operator).to_h
-      @should_clauses = grouped.fetch(:should, [])
-      @must_not_clauses = grouped.fetch(:must_not, [])
-      @must_clauses = grouped.fetch(:must, [])
-      @filter_clauses = grouped.fetch(:filter, [])
+      flags_from_clauses!
     end
 
-    def apply(search)
-      should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
-      must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
-      must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
-      filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) }
-      search.query.minimum_should_match(1)
+    def request
+      search = Chewy::Search::Request.new(*indexes).filter(default_filter)
+
+      must_clauses.each { |clause| search = search.query.must(clause.to_query) }
+      must_not_clauses.each { |clause| search = search.query.must_not(clause.to_query) }
+      filter_clauses.each { |clause| search = search.filter(**clause.to_query) }
+
+      search
     end
 
     private
 
-    def clause_to_query(clause)
-      case clause
-      when TermClause
-        { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
-      when PhraseClause
-        { match_phrase: { text: { query: clause.phrase } } }
-      else
-        raise "Unexpected clause type: #{clause}"
-      end
+    def clauses_by_operator
+      @clauses_by_operator ||= @clauses.compact.chunk(&:operator).to_h
+    end
+
+    def flags_from_clauses!
+      @flags = clauses_by_operator.fetch(:flag, []).to_h { |clause| [clause.prefix, clause.term] }
+    end
+
+    def must_clauses
+      clauses_by_operator.fetch(:must, [])
+    end
+
+    def must_not_clauses
+      clauses_by_operator.fetch(:must_not, [])
     end
 
-    def clause_to_filter(clause)
-      case clause
-      when PrefixClause
-        { term: { clause.filter => clause.term } }
+    def filter_clauses
+      clauses_by_operator.fetch(:filter, [])
+    end
+
+    def indexes
+      case @flags['in']
+      when 'library'
+        [StatusesIndex]
       else
-        raise "Unexpected clause type: #{clause}"
+        [PublicStatusesIndex, StatusesIndex]
       end
     end
+
+    def default_filter
+      {
+        bool: {
+          should: [
+            {
+              term: {
+                _index: PublicStatusesIndex.index_name,
+              },
+            },
+            {
+              bool: {
+                must: [
+                  {
+                    term: {
+                      _index: StatusesIndex.index_name,
+                    },
+                  },
+                  {
+                    term: {
+                      searchable_by: @options[:current_account].id,
+                    },
+                  },
+                ],
+              },
+            },
+          ],
+
+          minimum_should_match: 1,
+        },
+      }
+    end
   end
 
   class Operator
     class << self
       def symbol(str)
         case str
-        when '+'
+        when '+', nil
           :must
         when '-'
           :must_not
-        when nil
-          :should
         else
           raise "Unknown operator: #{str}"
         end
@@ -61,61 +112,140 @@ class SearchQueryTransformer < Parslet::Transform
   end
 
   class TermClause
-    attr_reader :prefix, :operator, :term
+    attr_reader :operator, :term
 
-    def initialize(prefix, operator, term)
-      @prefix = prefix
+    def initialize(operator, term)
       @operator = Operator.symbol(operator)
       @term = term
     end
+
+    def to_query
+      if @term.start_with?('#')
+        { match: { tags: { query: @term } } }
+      else
+        { multi_match: { type: 'most_fields', query: @term, fields: ['text', 'text.stemmed'], operator: 'and' } }
+      end
+    end
   end
 
   class PhraseClause
-    attr_reader :prefix, :operator, :phrase
+    attr_reader :operator, :phrase
 
-    def initialize(prefix, operator, phrase)
-      @prefix = prefix
+    def initialize(operator, phrase)
       @operator = Operator.symbol(operator)
       @phrase = phrase
     end
+
+    def to_query
+      { match_phrase: { text: { query: @phrase } } }
+    end
   end
 
   class PrefixClause
-    attr_reader :filter, :operator, :term
+    attr_reader :operator, :prefix, :term
 
-    def initialize(prefix, term)
+    def initialize(prefix, operator, term, options = {})
+      @prefix = prefix
+      @negated = operator == '-'
+      @options = options
       @operator = :filter
+
       case prefix
+      when 'has', 'is'
+        @filter = :properties
+        @type = :term
+        @term = term
+      when 'language'
+        @filter = :language
+        @type = :term
+        @term = language_code_from_term(term)
       when 'from'
         @filter = :account_id
+        @type = :term
+        @term = account_id_from_term(term)
+      when 'before'
+        @filter = :created_at
+        @type = :range
+        @term = { lt: term, time_zone: @options[:current_account]&.user_time_zone.presence || 'UTC' }
+      when 'after'
+        @filter = :created_at
+        @type = :range
+        @term = { gt: term, time_zone: @options[:current_account]&.user_time_zone.presence || 'UTC' }
+      when 'during'
+        @filter = :created_at
+        @type = :range
+        @term = { gte: term, lte: term, time_zone: @options[:current_account]&.user_time_zone.presence || 'UTC' }
+      when 'in'
+        @operator = :flag
+        @term = term
+      else
+        raise "Unknown prefix: #{prefix}"
+      end
+    end
 
-        username, domain = term.gsub(/\A@/, '').split('@')
-        domain           = nil if TagManager.instance.local_domain?(domain)
-        account          = Account.find_remote!(username, domain)
-
-        @term = account.id
+    def to_query
+      if @negated
+        { bool: { must_not: { @type => { @filter => @term } } } }
       else
-        raise Mastodon::SyntaxError
+        { @type => { @filter => @term } }
       end
     end
+
+    private
+
+    def account_id_from_term(term)
+      return @options[:current_account]&.id || -1 if term == 'me'
+
+      username, domain = term.gsub(/\A@/, '').split('@')
+      domain = nil if TagManager.instance.local_domain?(domain)
+      account = Account.find_remote(username, domain)
+
+      # If the account is not found, we want to return empty results, so return
+      # an ID that does not exist
+      account&.id || -1
+    end
+
+    def language_code_from_term(term)
+      language_code = term
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      language_code = term.downcase
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      language_code = term.split(/[_-]/).first.downcase
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      term
+    end
   end
 
   rule(clause: subtree(:clause)) do
     prefix   = clause[:prefix][:term].to_s if clause[:prefix]
     operator = clause[:operator]&.to_s
 
-    if clause[:prefix]
-      PrefixClause.new(prefix, clause[:term].to_s)
+    if clause[:prefix] && SUPPORTED_PREFIXES.include?(prefix)
+      PrefixClause.new(prefix, operator, clause[:term].to_s, current_account: current_account)
+    elsif clause[:prefix]
+      TermClause.new(operator, "#{prefix} #{clause[:term]}")
     elsif clause[:term]
-      TermClause.new(prefix, operator, clause[:term].to_s)
+      TermClause.new(operator, clause[:term].to_s)
     elsif clause[:shortcode]
-      TermClause.new(prefix, operator, ":#{clause[:term]}:")
+      TermClause.new(operator, ":#{clause[:term]}:")
     elsif clause[:phrase]
-      PhraseClause.new(prefix, operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s)
+      PhraseClause.new(operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s)
     else
       raise "Unexpected clause type: #{clause}"
     end
   end
 
-  rule(query: sequence(:clauses)) { Query.new(clauses) }
+  rule(junk: subtree(:junk)) do
+    nil
+  end
+
+  rule(query: sequence(:clauses)) do
+    Query.new(clauses, current_account: current_account)
+  end
 end

app/lib/vacuum/statuses_vacuum.rb

@@ -20,7 +20,10 @@ class Vacuum::StatusesVacuum
       statuses.direct_visibility
               .includes(mentions: :account)
               .find_each(&:unlink_from_conversations!)
-      remove_from_search_index(statuses.ids) if Chewy.enabled?
+      if Chewy.enabled?
+        remove_from_index(statuses.ids, 'chewy:queue:StatusesIndex')
+        remove_from_index(statuses.ids, 'chewy:queue:PublicStatusesIndex')
+      end
 
       # Foreign keys take care of most associated records for us.
       # Media attachments will be orphaned.
@@ -38,7 +41,7 @@ class Vacuum::StatusesVacuum
     Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false)
   end
 
-  def remove_from_search_index(status_ids)
-    with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', status_ids) }
+  def remove_from_index(status_ids, index)
+    with_redis { |redis| redis.sadd(index, status_ids) }
   end
 end

app/mailers/admin_mailer.rb

@@ -45,6 +45,22 @@ class AdminMailer < ApplicationMailer
     end
   end
 
+  def new_software_updates
+    locale_for_account(@me) do
+      mail subject: default_i18n_subject(instance: @instance)
+    end
+  end
+
+  def new_critical_software_updates
+    headers['Priority'] = 'urgent'
+    headers['X-Priority'] = '1'
+    headers['Importance'] = 'high'
+
+    locale_for_account(@me) do
+      mail subject: default_i18n_subject(instance: @instance)
+    end
+  end
+
   private
 
   def process_params

app/models/account.rb

@@ -82,6 +82,7 @@ class Account < ApplicationRecord
   include DomainMaterializable
   include AccountMerging
   include AccountSearch
+  include AccountStatusesSearch
 
   enum protocol: { ostatus: 0, activitypub: 1 }
   enum suspension_origin: { local: 0, remote: 1 }, _prefix: true

app/models/concerns/account_statuses_search.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module AccountStatusesSearch
+  extend ActiveSupport::Concern
+
+  included do
+    after_update_commit :enqueue_update_public_statuses_index, if: :saved_change_to_indexable?
+    after_destroy_commit :enqueue_remove_from_public_statuses_index, if: :indexable?
+  end
+
+  def enqueue_update_public_statuses_index
+    if indexable?
+      enqueue_add_to_public_statuses_index
+    else
+      enqueue_remove_from_public_statuses_index
+    end
+  end
+
+  def enqueue_add_to_public_statuses_index
+    return unless Chewy.enabled?
+
+    AddToPublicStatusesIndexWorker.perform_async(id)
+  end
+
+  def enqueue_remove_from_public_statuses_index
+    return unless Chewy.enabled?
+
+    RemoveFromPublicStatusesIndexWorker.perform_async(id)
+  end
+
+  def add_to_public_statuses_index!
+    return unless Chewy.enabled?
+
+    statuses.without_reblogs.where(visibility: :public).find_in_batches do |batch|
+      PublicStatusesIndex.import(batch)
+    end
+  end
+
+  def remove_from_public_statuses_index!
+    return unless Chewy.enabled?
+
+    PublicStatusesIndex.filter(term: { account_id: id }).delete_all
+  end
+end

app/models/concerns/status_search_concern.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module StatusSearchConcern
+  extend ActiveSupport::Concern
+
+  included do
+    scope :indexable, -> { without_reblogs.where(visibility: :public).joins(:account).where(account: { indexable: true }) }
+  end
+
+  def searchable_by
+    @searchable_by ||= begin
+      ids = []
+
+      ids << account_id if local?
+
+      ids += local_mentioned.pluck(:id)
+      ids += local_favorited.pluck(:id)
+      ids += local_reblogged.pluck(:id)
+      ids += local_bookmarked.pluck(:id)
+      ids += preloadable_poll.local_voters.pluck(:id) if preloadable_poll.present?
+
+      ids.uniq
+    end
+  end
+
+  def searchable_text
+    [
+      spoiler_text,
+      FormattingHelper.extract_status_plain_text(self),
+      preloadable_poll&.options&.join("\n\n"),
+      ordered_media_attachments.map(&:description).join("\n\n"),
+    ].compact.join("\n\n")
+  end
+
+  def searchable_properties
+    [].tap do |properties|
+      properties << 'image' if ordered_media_attachments.any?(&:image?)
+      properties << 'video' if ordered_media_attachments.any?(&:video?)
+      properties << 'audio' if ordered_media_attachments.any?(&:audio?)
+      properties << 'media' if with_media?
+      properties << 'poll' if with_poll?
+      properties << 'link' if with_preview_card?
+      properties << 'embed' if preview_cards.any?(&:video?)
+      properties << 'sensitive' if sensitive?
+      properties << 'reply' if reply?
+    end
+  end
+end

app/models/form/admin_settings.rb

@@ -3,6 +3,8 @@
 class Form::AdminSettings
   include ActiveModel::Model
 
+  include AuthorizedFetchHelper
+
   KEYS = %i(
     site_contact_username
     site_contact_email
@@ -36,6 +38,7 @@ class Form::AdminSettings
     backups_retention_period
     status_page_url
     captcha_enabled
+    authorized_fetch
   ).freeze
 
   INTEGER_KEYS = %i(
@@ -58,6 +61,7 @@ class Form::AdminSettings
     default_federation
     require_invite_text
     captcha_enabled
+    authorized_fetch
   ).freeze
 
   UPLOAD_KEYS = %i(
@@ -65,6 +69,10 @@ class Form::AdminSettings
     mascot
   ).freeze
 
+  OVERRIDEN_SETTINGS = {
+    authorized_fetch: :authorized_fetch_mode?,
+  }.freeze
+
   attr_accessor(*KEYS)
 
   validates :registrations_mode, inclusion: { in: %w(open approved none) }, if: -> { defined?(@registrations_mode) }
@@ -84,6 +92,8 @@ class Form::AdminSettings
 
       stored_value = if UPLOAD_KEYS.include?(key)
                        SiteUpload.where(var: key).first_or_initialize(var: key)
+                     elsif OVERRIDEN_SETTINGS.include?(key)
+                       public_send(OVERRIDEN_SETTINGS[key])
                      else
                        Setting.public_send(key)
                      end

app/models/media_attachment.rb

@@ -44,6 +44,7 @@ class MediaAttachment < ApplicationRecord
 
   MAX_VIDEO_MATRIX_LIMIT = 8_294_400 # 3840x2160px
   MAX_VIDEO_FRAME_RATE   = 120
+  MAX_VIDEO_FRAMES       = 36_000 # Approx. 5 minutes at 120 fps
 
   IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp .heic .heif .avif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
@@ -98,17 +99,15 @@ class MediaAttachment < ApplicationRecord
     convert_options: {
       output: {
         'loglevel' => 'fatal',
-        'movflags' => 'faststart',
-        'pix_fmt' => 'yuv420p',
-        'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
-        'vsync' => 'cfr',
+        'preset' => 'veryfast',
+        'movflags' => 'faststart', # Move metadata to start of file so playback can begin before download finishes
+        'pix_fmt' => 'yuv420p', # Ensure color space for cross-browser compatibility
+        'vf' => 'crop=floor(iw/2)*2:floor(ih/2)*2', # h264 requires width and height to be even. Crop instead of scale to avoid blurring
         'c:v' => 'h264',
-        'maxrate' => '1300K',
-        'bufsize' => '1300K',
-        'b:v' => '1300K',
-        'frames:v' => 60 * 60 * 3,
-        'crf' => 18,
+        'c:a' => 'aac',
+        'b:a' => '192k',
         'map_metadata' => '-1',
+        'frames:v' => MAX_VIDEO_FRAMES,
       }.freeze,
     }.freeze,
   }.freeze
@@ -135,7 +134,7 @@ class MediaAttachment < ApplicationRecord
       convert_options: {
         output: {
           'loglevel' => 'fatal',
-          :vf => 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
+          :vf => 'scale=\'min(640\, iw):min(640\, ih)\':force_original_aspect_ratio=decrease',
         }.freeze,
       }.freeze,
       format: 'png',

app/models/poll.rb

@@ -28,6 +28,7 @@ class Poll < ApplicationRecord
 
   has_many :votes, class_name: 'PollVote', inverse_of: :poll, dependent: :delete_all
   has_many :voters, -> { group('accounts.id') }, through: :votes, class_name: 'Account', source: :account
+  has_many :local_voters, -> { group('accounts.id').merge(Account.local) }, through: :votes, class_name: 'Account', source: :account
 
   has_many :notifications, as: :activity, dependent: :destroy
 

app/models/public_feed.rb

@@ -49,7 +49,7 @@ class PublicFeed
   end
 
   def local_only?
-    options[:local]
+    options[:local] && !options[:remote]
   end
 
   def without_local_only?
@@ -57,7 +57,7 @@ class PublicFeed
   end
 
   def remote_only?
-    options[:remote]
+    options[:remote] && !options[:local]
   end
 
   def account?

app/models/software_update.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: software_updates
+#
+#  id            :bigint(8)        not null, primary key
+#  version       :string           not null
+#  urgent        :boolean          default(FALSE), not null
+#  type          :integer          default("patch"), not null
+#  release_notes :string           default(""), not null
+#  created_at    :datetime         not null
+#  updated_at    :datetime         not null
+#
+
+class SoftwareUpdate < ApplicationRecord
+  self.inheritance_column = nil
+
+  enum type: { patch: 0, minor: 1, major: 2 }, _suffix: :type
+
+  def gem_version
+    Gem::Version.new(version)
+  end
+
+  class << self
+    def check_enabled?
+      ENV['UPDATE_CHECK_URL'] != ''
+    end
+
+    def pending_to_a
+      return [] unless check_enabled?
+
+      all.to_a.filter { |update| update.gem_version > Mastodon::Version.gem_version }
+    end
+
+    def urgent_pending?
+      pending_to_a.any?(&:urgent?)
+    end
+  end
+end

app/models/status.rb

@@ -39,6 +39,7 @@ class Status < ApplicationRecord
   include StatusSnapshotConcern
   include RateLimitable
   include StatusSafeReblogInsert
+  include StatusSearchConcern
 
   rate_limit by: :account, family: :statuses
 
@@ -49,6 +50,7 @@ class Status < ApplicationRecord
   attr_accessor :override_timestamps
 
   update_index('statuses', :proper)
+  update_index('public_statuses', :proper)
 
   enum visibility: { public: 0, unlisted: 1, private: 2, direct: 3, limited: 4 }, _suffix: :visibility
 
@@ -72,6 +74,12 @@ class Status < ApplicationRecord
   has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status
   has_many :media_attachments, dependent: :nullify
 
+  # Those associations are used for the private search index
+  has_many :local_mentioned, -> { merge(Account.local) }, through: :active_mentions, source: :account
+  has_many :local_favorited, -> { merge(Account.local) }, through: :favourites, source: :account
+  has_many :local_reblogged, -> { merge(Account.local) }, through: :reblogs, source: :account
+  has_many :local_bookmarked, -> { merge(Account.local) }, through: :bookmarks, source: :account
+
   has_and_belongs_to_many :tags
   has_and_belongs_to_many :preview_cards
 
@@ -170,37 +178,6 @@ class Status < ApplicationRecord
     "v3:#{super}"
   end
 
-  def searchable_by(preloaded = nil)
-    ids = []
-
-    ids << account_id if local?
-
-    if preloaded.nil?
-      ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id)
-      ids += favourites.joins(:account).merge(Account.local).pluck(:account_id)
-      ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id)
-      ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id)
-      ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present?
-    else
-      ids += preloaded.mentions[id] || []
-      ids += preloaded.favourites[id] || []
-      ids += preloaded.reblogs[id] || []
-      ids += preloaded.bookmarks[id] || []
-      ids += preloaded.votes[id] || []
-    end
-
-    ids.uniq
-  end
-
-  def searchable_text
-    [
-      spoiler_text,
-      FormattingHelper.extract_status_plain_text(self),
-      preloadable_poll ? preloadable_poll.options.join("\n\n") : nil,
-      ordered_media_attachments.map(&:description).join("\n\n"),
-    ].compact.join("\n\n")
-  end
-
   def to_log_human_identifier
     account.acct
   end
@@ -279,6 +256,10 @@ class Status < ApplicationRecord
     preview_cards.any?
   end
 
+  def with_poll?
+    preloadable_poll.present?
+  end
+
   def non_sensitive_with_media?
     !sensitive? && with_media?
   end

app/models/user_settings.rb

@@ -44,6 +44,7 @@ class UserSettings
     setting :pending_account, default: true
     setting :trends, default: true
     setting :appeal, default: true
+    setting :software_updates, default: 'critical', in: %w(none critical patch all)
   end
 
   namespace :interactions do

app/policies/software_update_policy.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class SoftwareUpdatePolicy < ApplicationPolicy
+  def index?
+    role.can?(:view_devops)
+  end
+end

app/presenters/initial_state_presenter.rb

@@ -3,9 +3,13 @@
 class InitialStatePresenter < ActiveModelSerializers::Model
   attributes :settings, :push_subscription, :token,
              :current_account, :admin, :owner, :text, :visibility,
-             :disabled_account, :moved_to_account
+             :disabled_account, :moved_to_account, :critical_updates_pending
 
   def role
     current_account&.user_role
   end
+
+  def critical_updates_pending
+    role&.can?(:view_devops) && SoftwareUpdate.urgent_pending?
+  end
 end

app/serializers/activitypub/actor_serializer.rb

@@ -7,13 +7,14 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   context :security
 
   context_extensions :manually_approves_followers, :featured, :also_known_as,
-                     :moved_to, :property_value, :discoverable, :olm, :suspended
+                     :moved_to, :property_value, :discoverable, :olm, :suspended,
+                     :memorial, :indexable
 
   attributes :id, :type, :following, :followers,
              :inbox, :outbox, :featured, :featured_tags,
              :preferred_username, :name, :summary,
              :url, :manually_approves_followers,
-             :discoverable, :published
+             :discoverable, :indexable, :published, :memorial
 
   has_one :public_key, serializer: ActivityPub::PublicKeySerializer
 
@@ -98,6 +99,10 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
     object.suspended? ? false : (object.discoverable || false)
   end
 
+  def indexable
+    object.suspended? ? false : (object.indexable || false)
+  end
+
   def name
     object.suspended? ? object.username : (object.display_name.presence || object.username)
   end

app/serializers/initial_state_serializer.rb

@@ -7,6 +7,8 @@ class InitialStateSerializer < ActiveModel::Serializer
              :media_attachments, :settings,
              :max_toot_chars, :languages
 
+  attribute :critical_updates_pending, if: -> { object&.role&.can?(:view_devops) && SoftwareUpdate.check_enabled? }
+
   has_one :push_subscription, serializer: REST::WebPushSubscriptionSerializer
   has_one :role, serializer: REST::RoleSerializer
 

app/serializers/webfinger_serializer.rb

@@ -18,18 +18,31 @@ class WebfingerSerializer < ActiveModel::Serializer
   end
 
   def links
-    if object.instance_actor?
-      [
-        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: about_more_url(instance_actor: true) },
-        { rel: 'self', type: 'application/activity+json', href: instance_actor_url },
-        { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
-      ]
-    else
-      [
-        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
-        { rel: 'self', type: 'application/activity+json', href: account_url(object) },
-        { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
-      ]
+    [
+      { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: profile_page_href },
+      { rel: 'self', type: 'application/activity+json', href: self_href },
+      { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
+    ].tap do |x|
+      x << { rel: 'http://webfinger.net/rel/avatar', type: object.avatar.content_type, href: full_asset_url(object.avatar_original_url) } if show_avatar?
     end
   end
+
+  private
+
+  def show_avatar?
+    media_present = object.avatar.present? && object.avatar.content_type.present?
+
+    # Show avatar only if an instance shows profiles to logged out users
+    allowed_by_config = ENV['DISALLOW_UNAUTHENTICATED_API_ACCESS'] != 'true' && !Rails.configuration.x.limited_federation_mode
+
+    media_present && allowed_by_config
+  end
+
+  def profile_page_href
+    object.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(object)
+  end
+
+  def self_href
+    object.instance_actor? ? instance_actor_url : account_url(object)
+  end
 end

app/services/account_search_service.rb

@@ -124,7 +124,7 @@ class AccountSearchService < BaseService
         multi_match: {
           query: @query,
           type: 'bool_prefix',
-          fields: %w(username username.* display_name display_name.*),
+          fields: %w(username^2 username.*^2 display_name display_name.*),
         },
       }
     end

app/services/activitypub/process_account_service.rb

@@ -116,6 +116,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.also_known_as           = as_array(@json['alsoKnownAs'] || []).map { |item| value_or_id(item) }
     @account.discoverable            = @json['discoverable'] || false
     @account.indexable               = @json['indexable'] || false
+    @account.memorial                = @json['memorial'] || false
   end
 
   def set_fetchable_key!

app/services/batched_remove_status_service.rb

@@ -35,7 +35,10 @@ class BatchedRemoveStatusService < BaseService
 
     # Since we skipped all callbacks, we also need to manually
     # deindex the statuses
-    Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) if Chewy.enabled?
+    if Chewy.enabled?
+      Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs)
+      Chewy.strategy.current.update(PublicStatusesIndex, statuses_and_reblogs)
+    end
 
     return if options[:skip_side_effects]
 

app/services/concerns/payloadable.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Payloadable
+  include AuthorizedFetchHelper
+
   # @param [ActiveModelSerializers::Model] record
   # @param [ActiveModelSerializers::Serializer] serializer
   # @param [Hash] options
@@ -23,6 +25,6 @@ module Payloadable
   end
 
   def signing_enabled?
-    ENV['AUTHORIZED_FETCH'] != 'true' && !Rails.configuration.x.limited_federation_mode
+    !authorized_fetch_mode?
   end
 end

app/services/search_service.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
 class SearchService < BaseService
+  QUOTE_EQUIVALENT_CHARACTERS = /[“”„«»「」『』《》]/
+
   def call(query, account, limit, options = {})
-    @query     = query&.strip
+    @query     = query&.strip&.gsub(QUOTE_EQUIVALENT_CHARACTERS, '"')
     @account   = account
     @options   = options
     @limit     = limit.to_i
@@ -17,7 +19,7 @@ class SearchService < BaseService
         results.merge!(url_resource_results) unless url_resource.nil? || @offset.positive? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
       elsif @query.present?
         results[:accounts] = perform_accounts_search! if account_searchable?
-        results[:statuses] = perform_statuses_search! if full_text_searchable?
+        results[:statuses] = perform_statuses_search! if status_searchable?
         results[:hashtags] = perform_hashtags_search! if hashtag_searchable?
       end
     end
@@ -39,25 +41,15 @@ class SearchService < BaseService
   end
 
   def perform_statuses_search!
-    definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id }))
-
-    definition = definition.filter(term: { account_id: @options[:account_id] }) if @options[:account_id].present?
-
-    if @options[:min_id].present? || @options[:max_id].present?
-      range      = {}
-      range[:gt] = @options[:min_id].to_i if @options[:min_id].present?
-      range[:lt] = @options[:max_id].to_i if @options[:max_id].present?
-      definition = definition.filter(range: { id: range })
-    end
-
-    results             = definition.limit(@limit).offset(@offset).objects.compact
-    account_ids         = results.map(&:account_id)
-    account_domains     = results.map(&:account_domain)
-    preloaded_relations = @account.relations_map(account_ids, account_domains)
-
-    results.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
-  rescue Faraday::ConnectionFailed, Parslet::ParseFailed
-    []
+    StatusesSearchService.new.call(
+      @query,
+      @account,
+      limit: @limit,
+      offset: @offset,
+      account_id: @options[:account_id],
+      min_id: @options[:min_id],
+      max_id: @options[:max_id]
+    )
   end
 
   def perform_hashtags_search!
@@ -89,18 +81,16 @@ class SearchService < BaseService
     url_resource.class.name.downcase.pluralize.to_sym
   end
 
-  def full_text_searchable?
-    return false unless Chewy.enabled?
-
-    statuses_search? && !@account.nil? && !(@query.include?('@') && !@query.include?(' '))
+  def status_searchable?
+    Chewy.enabled? && status_search? && @account.present?
   end
 
   def account_searchable?
-    account_search? && !(@query.include?('@') && @query.include?(' '))
+    account_search?
   end
 
   def hashtag_searchable?
-    hashtag_search? && !@query.include?('@')
+    hashtag_search?
   end
 
   def account_search?
@@ -111,11 +101,7 @@ class SearchService < BaseService
     @options[:type].blank? || @options[:type] == 'hashtags'
   end
 
-  def statuses_search?
+  def status_search?
     @options[:type].blank? || @options[:type] == 'statuses'
   end
-
-  def parsed_query
-    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
-  end
 end

app/services/software_update_check_service.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class SoftwareUpdateCheckService < BaseService
+  def call
+    clean_outdated_updates!
+    return unless SoftwareUpdate.check_enabled?
+
+    process_update_notices!(fetch_update_notices)
+  end
+
+  private
+
+  def clean_outdated_updates!
+    SoftwareUpdate.find_each do |software_update|
+      software_update.delete if Mastodon::Version.gem_version >= software_update.gem_version
+    rescue ArgumentError
+      software_update.delete
+    end
+  end
+
+  def fetch_update_notices
+    Request.new(:get, "#{api_url}?version=#{version}").add_headers('Accept' => 'application/json', 'User-Agent' => 'Mastodon update checker').perform do |res|
+      return Oj.load(res.body_with_limit, mode: :strict) if res.code == 200
+    end
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Oj::ParseError
+    nil
+  end
+
+  def api_url
+    ENV.fetch('UPDATE_CHECK_URL', 'https://api.joinmastodon.org/update-check')
+  end
+
+  def version
+    @version ||= Mastodon::Version.to_s.split('+')[0]
+  end
+
+  def process_update_notices!(update_notices)
+    return if update_notices.blank? || update_notices['updatesAvailable'].blank?
+
+    # Clear notices that are not listed by the update server anymore
+    SoftwareUpdate.where.not(version: update_notices['updatesAvailable'].pluck('version')).delete_all
+
+    # Check if any of the notices is new, and issue notifications
+    known_versions = SoftwareUpdate.where(version: update_notices['updatesAvailable'].pluck('version')).pluck(:version)
+    new_update_notices = update_notices['updatesAvailable'].filter { |notice| known_versions.exclude?(notice['version']) }
+    return if new_update_notices.blank?
+
+    new_updates = new_update_notices.map do |notice|
+      SoftwareUpdate.create!(version: notice['version'], urgent: notice['urgent'], type: notice['type'], release_notes: notice['releaseNotes'])
+    end
+
+    notify_devops!(new_updates)
+  end
+
+  def should_notify_user?(user, urgent_version, patch_version)
+    case user.settings['notification_emails.software_updates']
+    when 'none'
+      false
+    when 'critical'
+      urgent_version
+    when 'patch'
+      urgent_version || patch_version
+    when 'all'
+      true
+    end
+  end
+
+  def notify_devops!(new_updates)
+    has_new_urgent_version = new_updates.any?(&:urgent?)
+    has_new_patch_version  = new_updates.any?(&:patch_type?)
+
+    User.those_who_can(:view_devops).includes(:account).find_each do |user|
+      next unless should_notify_user?(user, has_new_urgent_version, has_new_patch_version)
+
+      if has_new_urgent_version
+        AdminMailer.with(recipient: user.account).new_critical_software_updates.deliver_later
+      else
+        AdminMailer.with(recipient: user.account).new_software_updates.deliver_later
+      end
+    end
+  end
+end

app/services/statuses_search_service.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class StatusesSearchService < BaseService
+  def call(query, account = nil, options = {})
+    @query   = query&.strip
+    @account = account
+    @options = options
+    @limit   = options[:limit].to_i
+    @offset  = options[:offset].to_i
+
+    status_search_results
+  end
+
+  private
+
+  def status_search_results
+    request             = parsed_query.request
+    results             = request.collapse(field: :id).order(id: { order: :desc }).limit(@limit).offset(@offset).objects.compact
+    account_ids         = results.map(&:account_id)
+    account_domains     = results.map(&:account_domain)
+    preloaded_relations = @account.relations_map(account_ids, account_domains)
+
+    results.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
+  rescue Faraday::ConnectionFailed, Parslet::ParseFailed
+    []
+  end
+
+  def parsed_query
+    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query), current_account: @account)
+  end
+end

app/views/admin/accounts/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.accounts.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = form_tag admin_accounts_url, method: 'GET', class: 'simple_form' do
   .filters
     .filter-subset.filter-subset--with-select

app/views/admin/action_logs/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.action_logs.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = form_tag admin_action_logs_url, method: 'GET', class: 'simple_form' do
   = hidden_field_tag :target_account_id, params[:target_account_id] if params[:target_account_id].present?
 

app/views/admin/announcements/edit.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = simple_form_for @announcement, url: admin_announcement_path(@announcement), html: { novalidate: false } do |f|
   = render 'shared/error_messages', object: @announcement
 

app/views/admin/announcements/new.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = simple_form_for @announcement, url: admin_announcements_path, html: { novalidate: false } do |f|
   = render 'shared/error_messages', object: @announcement
 

app/views/admin/custom_emojis/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.custom_emojis.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - if can?(:create, :custom_emoji)
   - content_for :heading_actions do
     = link_to t('admin.custom_emojis.upload'), new_admin_custom_emoji_path, class: 'button'