Merge latest changes from 'stable-4.2'

2 years ago · c72697c2d2
17 changed files with 206 additions and 89 deletions
--- a/1
+++ b/1
@ -65,7 +65,6 @@ gem 'nsa'
 gem 'oj', '~> 3.14'
 gem 'ox', '~> 2.14'
 gem 'parslet'
-gem 'posix-spawn'
 gem 'public_suffix', '~> 5.0'
 gem 'pundit', '~> 2.3'
 gem 'premailer-rails'
--- a/Gemfile.lock
+++ b/Gemfile.lock
@ -28,47 +28,47 @@ GIT
 GEM
  remote: https://rubygems.org/
  specs:
-    actioncable (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actioncable (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      nio4r (~> 2.0)
      websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionmailbox (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activestorage (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      mail (>= 2.7.1)
      net-imap
      net-pop
      net-smtp
-    actionmailer (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      actionview (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionmailer (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      actionview (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      mail (~> 2.5, >= 2.5.4)
      net-imap
      net-pop
      net-smtp
      rails-dom-testing (~> 2.0)
-    actionpack (7.0.8.1)
-      actionview (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionpack (7.0.8.4)
+      actionview (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      rack (~> 2.0, >= 2.2.4)
      rack-test (>= 0.6.3)
      rails-dom-testing (~> 2.0)
      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actiontext (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activestorage (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      globalid (>= 0.6.0)
      nokogiri (>= 1.8.5)
-    actionview (7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionview (7.0.8.4)
+      activesupport (= 7.0.8.4)
      builder (~> 3.1)
      erubi (~> 1.4)
      rails-dom-testing (~> 2.0)
@ -78,22 +78,22 @@ GEM
      activemodel (>= 4.1, < 7.1)
      case_transform (>= 0.2)
      jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
-    activejob (7.0.8.1)
-      activesupport (= 7.0.8.1)
+    activejob (7.0.8.4)
+      activesupport (= 7.0.8.4)
      globalid (>= 0.3.6)
-    activemodel (7.0.8.1)
-      activesupport (= 7.0.8.1)
-    activerecord (7.0.8.1)
-      activemodel (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
-    activestorage (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    activemodel (7.0.8.4)
+      activesupport (= 7.0.8.4)
+    activerecord (7.0.8.4)
+      activemodel (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
+    activestorage (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      marcel (~> 1.0)
      mini_mime (>= 1.1.0)
-    activesupport (7.0.8.1)
+    activesupport (7.0.8.4)
      concurrent-ruby (~> 1.0, >= 1.0.2)
      i18n (>= 1.6, < 2)
      minitest (>= 5.1)
@ -191,8 +191,8 @@ GEM
      xpath (~> 3.2)
    case_transform (0.2)
      activesupport
-    cbor (0.5.9.6)
-    charlock_holmes (0.7.7)
+    cbor (0.5.9.8)
+    charlock_holmes (0.7.8)
    chewy (7.3.4)
      activesupport (>= 5.2)
      elasticsearch (>= 7.12.0, < 7.14.0)
@ -350,7 +350,7 @@ GEM
    httplog (1.6.2)
      rack (>= 2.0)
      rainbow (>= 2.0.0)
-    i18n (1.14.1)
+    i18n (1.14.5)
      concurrent-ruby (~> 1.0)
    i18n-tasks (1.0.12)
      activesupport (>= 4.0.2)
@ -432,7 +432,7 @@ GEM
      net-imap
      net-pop
      net-smtp
-    marcel (1.0.2)
+    marcel (1.0.4)
    mario-redis-lock (1.2.1)
      redis (>= 3.0.5)
    matrix (0.4.2)
@ -446,7 +446,7 @@ GEM
      mime-types-data (~> 3.2015)
    mime-types-data (3.2023.0808)
    mini_mime (1.1.5)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.7)
    minitest (5.19.0)
    msgpack (1.7.1)
    multi_json (1.15.0)
@ -468,8 +468,8 @@ GEM
    net-smtp (0.3.4)
      net-protocol
    net-ssh (7.1.0)
-    nio4r (2.7.0)
-    nokogiri (1.16.5)
+    nio4r (2.7.3)
+    nokogiri (1.16.6)
      mini_portile2 (~> 2.8.2)
      racc (~> 1.4)
    nsa (0.3.0)
@ -517,7 +517,6 @@ GEM
    pg (1.5.5)
    pghero (3.3.4)
      activerecord (>= 6)
-    posix-spawn (0.3.15)
    premailer (1.21.0)
      addressable
      css_parser (>= 1.12.0)
@ -534,7 +533,7 @@ GEM
      activesupport (>= 3.0.0)
    raabro (1.4.0)
    racc (1.7.3)
-    rack (2.2.8.1)
+    rack (2.2.9)
    rack-attack (6.7.0)
      rack (>= 1.0, < 4)
    rack-cors (2.0.2)
@ -551,20 +550,20 @@ GEM
      rack
    rack-test (2.1.0)
      rack (>= 1.3)
-    rails (7.0.8.1)
-      actioncable (= 7.0.8.1)
-      actionmailbox (= 7.0.8.1)
-      actionmailer (= 7.0.8.1)
-      actionpack (= 7.0.8.1)
-      actiontext (= 7.0.8.1)
-      actionview (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activemodel (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    rails (7.0.8.4)
+      actioncable (= 7.0.8.4)
+      actionmailbox (= 7.0.8.4)
+      actionmailer (= 7.0.8.4)
+      actionpack (= 7.0.8.4)
+      actiontext (= 7.0.8.4)
+      actionview (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activemodel (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activestorage (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      bundler (>= 1.15.0)
-      railties (= 7.0.8.1)
+      railties (= 7.0.8.4)
    rails-controller-testing (1.0.5)
      actionpack (>= 5.0.1.rc1)
      actionview (>= 5.0.1.rc1)
@ -579,9 +578,9 @@ GEM
    rails-i18n (7.0.7)
      i18n (>= 0.7, < 2)
      railties (>= 6.0.0, < 8)
-    railties (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    railties (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
      method_source
      rake (>= 12.2)
      thor (~> 1.0)
@ -744,7 +743,7 @@ GEM
    terrapin (0.6.0)
      climate_control (>= 0.0.3, < 1.0)
    test-prof (1.2.3)
-    thor (1.3.0)
+    thor (1.3.1)
    tilt (2.2.0)
    timeout (0.4.1)
    tpm-key_attestation (0.12.0)
@ -810,7 +809,7 @@ GEM
    xorcist (1.1.3)
    xpath (3.2.0)
      nokogiri (~> 1.8)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.16)

 PLATFORMS
  ruby
@ -892,7 +891,6 @@ DEPENDENCIES
  parslet
  pg (~> 1.5)
  pghero
-  posix-spawn
  premailer-rails
  private_address_check (~> 0.5)
  public_suffix (~> 5.0)
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@ -265,16 +265,21 @@ class LinkDetailsExtractor
  end

  def document
-    @document ||= Nokogiri::HTML(@html, nil, encoding)
+    @document ||= detect_encoding_and_parse_document
  end

-  def encoding
-    @encoding ||= begin
-      guess = detector.detect(@html, @html_charset)
-      guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  def detect_encoding_and_parse_document
+    [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
+      document = Nokogiri::HTML(@html, nil, encoding)
+      return document if document.to_s.valid_encoding?
    end
  end

+  def detect_encoding
+    guess = detector.detect(@html, @html_charset)
+    guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  end
+
  def detector
    @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
      detector.strip_tags = true
--- a/app/lib/search_query_transformer.rb
+++ b/app/lib/search_query_transformer.rb
@ -223,7 +223,7 @@ class SearchQueryTransformer < Parslet::Transform
  end

  rule(clause: subtree(:clause)) do
-    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    prefix   = clause[:prefix][:term].to_s.downcase if clause[:prefix]
    operator = clause[:operator]&.to_s
    term     = clause[:phrase] ? clause[:phrase].map { |term| term[:term].to_s }.join(' ') : clause[:term].to_s

--- a/app/models/status_edit.rb
+++ b/app/models/status_edit.rb
@ -42,7 +42,7 @@ class StatusEdit < ApplicationRecord
  default_scope { order(id: :asc) }

  delegate :local?, :application, :edited?, :edited_at,
-           :discarded?, :visibility, to: :status
+           :discarded?, :visibility, :language, to: :status

  def emojis
    return @emojis if defined?(@emojis)
--- a/app/serializers/manifest_serializer.rb
+++ b/app/serializers/manifest_serializer.rb
@ -16,11 +16,18 @@ class ManifestSerializer < ActiveModel::Serializer
    512
  ).freeze

-  attributes :name, :short_name,
+  attributes :id, :name, :short_name,
             :icons, :theme_color, :background_color,
             :display, :start_url, :scope,
             :share_target, :shortcuts

+  def id
+    # This is set to `/home` because that was the old value of `start_url` and
+    # thus the fallback ID computed by Chrome:
+    # https://developer.chrome.com/blog/pwa-manifest-id/
+    '/home'
+  end
+
  def name
    object.title
  end
@ -53,7 +60,7 @@ class ManifestSerializer < ActiveModel::Serializer
  end

  def start_url
-    '/home'
+    '/'
  end

  def scope
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@ -19,8 +19,8 @@ class BackupService < BaseService

  def build_outbox_json!(file)
    skeleton = serialize(collection_presenter, ActivityPub::CollectionSerializer)
-    skeleton[:@context] = full_context
-    skeleton[:orderedItems] = ['!PLACEHOLDER!']
+    skeleton['@context'] = full_context
+    skeleton['orderedItems'] = ['!PLACEHOLDER!']
    skeleton = Oj.dump(skeleton)
    prepend, append = skeleton.split('"!PLACEHOLDER!"')
    add_comma = false
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@ -15,6 +15,9 @@ class FetchLinkCardService < BaseService
    )
  }iox

+  # URL size limit to safely store in PosgreSQL's unique indexes
+  BYTESIZE_LIMIT = 2692
+
  def call(status)
    @status       = status
    @original_url = parse_urls
@ -85,7 +88,7 @@ class FetchLinkCardService < BaseService

  def bad_url?(uri)
    # Avoid local instance URLs and invalid URLs
-    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
+    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT
  end

  def mention_link?(anchor)
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@ -188,7 +188,7 @@ class PostStatusService < BaseService
  end

  def scheduled_in_the_past?
-    @scheduled_at.present? && @scheduled_at <= Time.now.utc + MIN_SCHEDULE_OFFSET
+    @scheduled_at.present? && @scheduled_at <= Time.now.utc
  end

  def bump_potential_friendship!
--- a/config/initializers/statsd.rb
+++ b/config/initializers/statsd.rb
@ -3,13 +3,17 @@
 if ENV['STATSD_ADDR'].present?
  host, port = ENV['STATSD_ADDR'].split(':')

-  statsd = Statsd.new(host, port)
-  statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }
+  begin
+    statsd = Statsd.new(host, port)
+    statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }

-  NSA.inform_statsd(statsd) do |informant|
-    informant.collect(:action_controller, :web)
-    informant.collect(:active_record, :db)
-    informant.collect(:active_support_cache, :cache)
-    informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true'
+    NSA.inform_statsd(statsd) do |informant|
+      informant.collect(:action_controller, :web)
+      informant.collect(:active_record, :db)
+      informant.collect(:active_support_cache, :cache)
+      informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true'
+    end
+  rescue
+    Rails.logger.warn("statsd address #{ENV['STATSD_ADDR']} not reachable, proceeding without statsd")
  end
 end
--- a/package.json
+++ b/package.json
@ -226,5 +226,6 @@
    "Capfile|Gemfile|*.{rb,ruby,ru,rake}": "bundle exec rubocop --force-exclusion -a",
    "*.{js,jsx,ts,tsx}": "eslint --fix",
    "*.{css,scss}": "stylelint --fix"
-  }
+  },
+  "packageManager": "yarn@1.22.22"
 }
--- a/spec/controllers/admin/statuses_controller_spec.rb
+++ b/spec/controllers/admin/statuses_controller_spec.rb
@ -43,6 +43,11 @@ describe Admin::StatusesController do

  describe 'GET #show' do
    before do
+      status.media_attachments << Fabricate(:media_attachment, type: :image, account: status.account)
+      status.save!
+      status.snapshot!(at_time: status.created_at, rate_limit: false)
+      status.update!(text: 'Hello, this is an edited post')
+      status.snapshot!(rate_limit: false)
      get :show, params: { account_id: account.id, id: status.id }
    end

--- a/spec/controllers/api/v1/statuses_controller_spec.rb
+++ b/spec/controllers/api/v1/statuses_controller_spec.rb
@ -182,6 +182,46 @@ RSpec.describe Api::V1::StatusesController do
          expect(response.headers['X-RateLimit-Remaining']).to eq '0'
        end
      end
+
+      context 'with missing thread' do
+        subject { post :create, params: params }
+
+        let(:params) { { status: 'Hello world', in_reply_to_id: 0 } }
+
+        it 'returns http not found' do
+          subject
+
+          expect(response).to have_http_status(404)
+        end
+      end
+
+      context 'when scheduling a status' do
+        subject { post :create, params: params }
+
+        let(:params) { { status: 'Hello world', scheduled_at: 10.minutes.from_now } }
+        let(:account) { user.account }
+
+        it 'returns HTTP 200' do
+          subject
+
+          expect(response).to have_http_status(200)
+        end
+
+        it 'creates a scheduled status' do
+          expect { subject }.to change { account.scheduled_statuses.count }.from(0).to(1)
+        end
+
+        context 'when the scheduling time is less than 5 minutes' do
+          let(:params) { { status: 'Hello world', scheduled_at: 4.minutes.from_now } }
+
+          it 'does not create a scheduled status', :aggregate_failures do
+            subject
+
+            expect(response).to have_http_status(422)
+            expect(account.scheduled_statuses).to be_empty
+          end
+        end
+      end
    end

    describe 'DELETE #destroy' do
--- a/spec/fixtures/requests/low_confidence_latin1.txt
+++ b/spec/fixtures/requests/low_confidence_latin1.txt
@ -0,0 +1,17 @@
+HTTP/1.1 200 OK
+server: nginx
+date: Thu, 13 Jun 2024 14:33:13 GMT
+content-type: text/html; charset=ISO-8859-1
+content-length: 158
+accept-ranges: bytes
+
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Tofu á l'orange</title>
+</head>
+<body>
+  <h2>Tofu á l'orange</h2>
+</body>
+</html>
--- a/spec/services/backup_service_spec.rb
+++ b/spec/services/backup_service_spec.rb
@ -55,9 +55,11 @@ RSpec.describe BackupService, type: :service do
  end

  def expect_outbox_export
-    json = export_json(:outbox)
+    body = export_json_raw(:outbox)
+    json = Oj.load(body)

    aggregate_failures do
+      expect(body.scan('@context').count).to eq 1
      expect(json['@context']).to_not be_nil
      expect(json['type']).to eq 'OrderedCollection'
      expect(json['totalItems']).to eq 2
@ -85,8 +87,12 @@ RSpec.describe BackupService, type: :service do
    end
  end

+  def export_json_raw(type)
+    read_zip_file(backup, "#{type}.json")
+  end
+
  def export_json(type)
-    Oj.load(read_zip_file(backup, "#{type}.json"))
+    Oj.load(export_json_raw(type))
  end

  def include_create_item(status)
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@ -26,6 +26,7 @@ RSpec.describe FetchLinkCardService, type: :service do
    stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt'))
    stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
    stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
+    stub_request(:get, 'http://example.com/low_confidence_latin1').to_return(request_fixture('low_confidence_latin1.txt'))

    Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache

@ -148,6 +149,14 @@ RSpec.describe FetchLinkCardService, type: :service do
      end
    end

+    context 'with a URL of a page in ISO-8859-1 encoding, that charlock_holmes cannot detect' do
+      let(:status) { Fabricate(:status, text: 'Check out http://example.com/low_confidence_latin1') }
+
+      it 'decodes the HTML' do
+        expect(status.preview_card.title).to eq("Tofu á l'orange")
+      end
+    end
+
    context 'with a Japanese path URL' do
      let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }

@ -184,6 +193,19 @@ RSpec.describe FetchLinkCardService, type: :service do
      end
    end

+    context 'with an URL too long for PostgreSQL unique indexes' do
+      let(:url) { "http://example.com/#{'a' * 2674}" }
+      let(:status) { Fabricate(:status, text: url) }
+
+      it 'does not fetch the URL' do
+        expect(a_request(:get, url)).to_not have_been_made
+      end
+
+      it 'does not create a preview card' do
+        expect(status.preview_card).to be_nil
+      end
+    end
+
    context 'with a URL of a page with oEmbed support' do
      let(:html) { '<!doctype html><title>Hello world</title><link rel="alternate" type="application/json+oembed" href="http://example.com/oembed?url=http://example.com/html">' }
      let(:status) { Fabricate(:status, text: 'http://example.com/html') }
--- a/spec/services/post_status_service_spec.rb
+++ b/spec/services/post_status_service_spec.rb
@ -61,6 +61,16 @@ RSpec.describe PostStatusService, type: :service do
      status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future)
      expect(status2.id).to eq status1.id
    end
+
+    context 'when scheduled_at is less than min offset' do
+      let(:invalid_scheduled_time) { 4.minutes.from_now }
+
+      it 'raises invalid record error' do
+        expect do
+          subject.call(account, text: 'Hi future!', scheduled_at: invalid_scheduled_time)
+        end.to raise_error(ActiveRecord::RecordInvalid)
+      end
+    end
  end

  it 'creates response to the original status of boost' do