Improve various queries against account domains (#25126)

app/models/account.rb

@@ -119,7 +119,7 @@ class Account < ApplicationRecord
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
   scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
-  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches("%.#{domain}"))) }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: Instance.by_domain_and_subdomain(domain).select(:domain)) }
   scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) }
   scope :not_domain_blocked_by_account, ->(account) { where(arel_table[:domain].eq(nil).or(arel_table[:domain].not_in(account.excluded_from_timeline_domains))) }
 

app/models/instance.rb

@@ -22,6 +22,7 @@ class Instance < ApplicationRecord
   end
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+  scope :by_domain_and_subdomain, ->(domain) { where("reverse('.' || domain) LIKE reverse(?)", "%.#{domain}") }
 
   def self.refresh
     Scenic.database.refresh_materialized_view(table_name, concurrently: true, cascade: false)

db/migrate/20230524190515_add_index_accounts_on_domain_and_id.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddIndexAccountsOnDomainAndId < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :accounts, [:domain, :id], name: :index_accounts_on_domain_and_id, algorithm: :concurrently
+  end
+end

db/migrate/20230524192812_fix_account_domain_casing.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class FixAccountDomainCasing < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured do
+      execute 'UPDATE accounts SET domain = lower(domain) WHERE domain IS NOT NULL AND domain != lower(domain)'
+    end
+  end
+
+  def down; end
+end

db/migrate/20230524194155_add_index_instances_on_reverse_domain.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddIndexInstancesOnReverseDomain < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :instances, "reverse('.' || domain), domain", name: :index_instances_on_reverse_domain, algorithm: :concurrently
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_03_30_155710) do
+ActiveRecord::Schema.define(version: 2023_05_24_194155) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -188,6 +188,7 @@ ActiveRecord::Schema.define(version: 2023_03_30_155710) do
     t.datetime "requested_review_at"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
+    t.index ["domain", "id"], name: "index_accounts_on_domain_and_id"
     t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id", where: "(moved_to_account_id IS NOT NULL)"
     t.index ["uri"], name: "index_accounts_on_uri"
     t.index ["url"], name: "index_accounts_on_url", opclass: :text_pattern_ops, where: "(url IS NOT NULL)"
@@ -1283,6 +1284,7 @@ ActiveRecord::Schema.define(version: 2023_03_30_155710) do
      FROM (domain_allows
        LEFT JOIN domain_counts ON (((domain_counts.domain)::text = (domain_allows.domain)::text)));
   SQL
+  add_index "instances", "reverse(('.'::text || (domain)::text)), domain", name: "index_instances_on_reverse_domain"
   add_index "instances", ["domain"], name: "index_instances_on_domain", unique: true
 
   create_view "user_ips", sql_definition: <<-SQL

lib/tasks/tests.rake

@@ -58,6 +58,11 @@ namespace :tests do
         puts 'User settings not kept as expected'
         exit(1)
       end
+
+      unless Account.find_remote('bob', 'ActivityPub.com').domain == 'activitypub.com'
+        puts 'Account domains not properly normalized'
+        exit(1)
+      end
     end
 
     desc 'Populate the database with test data for 2.4.3'
@@ -160,7 +165,7 @@ namespace :tests do
         INSERT INTO "accounts"
           (id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url)
         VALUES
-          (6, 'bob', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(),
+          (6, 'bob', 'ActivityPub.com', NULL, #{remote_public_key_ap}, now(), now(),
            1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers');
 
         INSERT INTO "accounts"