Add end-to-end (system) tests (#25461)

.github/workflows/test-ruby.yml

@@ -153,3 +153,100 @@ jobs:
         run: './bin/rails db:create db:schema:load db:seed'
 
       - run: bundle exec rake rspec_chunked
+
+  test-e2e:
+    name: End to End testing
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: true
+      RAILS_ENV: test
+      BUNDLE_WITH: test
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '3.0'
+          - '3.1'
+          - '.ruby-version'
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/download-artifact@v3
+        with:
+          path: './public'
+          name: ${{ github.sha }}
+
+      - name: Update package index
+        run: sudo apt-get update
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+          node-version-file: '.nvmrc'
+
+      - name: Install native Ruby dependencies
+        run: sudo apt-get install -y libicu-dev libidn11-dev
+
+      - name: Install additional system dependencies
+        run: sudo apt-get install -y ffmpeg imagemagick
+
+      - name: Set up bundler cache
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          bundler-cache: true
+
+      - run: yarn --frozen-lockfile
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bundle exec rake spec:system
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: e2e-logs-${{ matrix.ruby-version }}
+          path: log/
+
+      - name: Archive test screenshots
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: e2e-screenshots
+          path: tmp/screenshots/

Gemfile

@@ -113,6 +113,10 @@ group :test do
 
   # Browser integration testing
   gem 'capybara', '~> 3.39'
+  gem 'selenium-webdriver'
+
+  # Used to reset the database between system tests
+  gem 'database_cleaner-active_record'
 
   # Used to mock environment variables
   gem 'climate_control', '~> 0.2'

Gemfile.lock

@@ -199,6 +199,10 @@ GEM
     crass (1.0.6)
     css_parser (1.14.0)
       addressable
+    database_cleaner-active_record (2.1.0)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
     date (3.3.3)
     debug_inspector (1.1.0)
     devise (4.9.2)
@@ -656,6 +660,10 @@ GEM
     scenic (1.7.0)
       activerecord (>= 4.0.0)
       railties (>= 4.0.0)
+    selenium-webdriver (4.9.1)
+      rexml (~> 3.2, >= 3.2.5)
+      rubyzip (>= 1.2.2, < 3.0)
+      websocket (~> 1.0)
     semantic_range (3.0.0)
     sidekiq (6.5.9)
       connection_pool (>= 2.2.5, < 3)
@@ -768,6 +776,7 @@ GEM
       rack-proxy (>= 0.6.1)
       railties (>= 5.2)
       semantic_range (>= 2.3.0)
+    websocket (1.2.9)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -804,6 +813,7 @@ DEPENDENCIES
   color_diff (~> 0.1)
   concurrent-ruby
   connection_pool
+  database_cleaner-active_record
   devise (~> 4.9)
   devise-two-factor (~> 4.1)
   devise_pam_authenticatable2 (~> 9.2)
@@ -885,6 +895,7 @@ DEPENDENCIES
   rubyzip (~> 2.3)
   sanitize (~> 6.0)
   scenic (~> 1.7)
+  selenium-webdriver
   sidekiq (~> 6.5)
   sidekiq-bulk (~> 0.2.0)
   sidekiq-scheduler (~> 5.0)

config/application.rb

@@ -199,7 +199,7 @@ module Mastodon
     # We use our own middleware for this
     config.public_file_server.enabled = false
 
-    config.middleware.use PublicFileServerMiddleware if Rails.env.development? || ENV['RAILS_SERVE_STATIC_FILES'] == 'true'
+    config.middleware.use PublicFileServerMiddleware if Rails.env.development? || Rails.env.test? || ENV['RAILS_SERVE_STATIC_FILES'] == 'true'
     config.middleware.use Rack::Attack
     config.middleware.use Mastodon::RackMiddleware
 

config/webpack/tests.js

@@ -5,5 +5,5 @@ const { merge } = require('webpack-merge');
 const sharedConfig = require('./shared');
 
 module.exports = merge(sharedConfig, {
-  mode: 'development',
+  mode: 'production',
 });

lib/tasks/spec.rake

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+if Rake::Task.task_defined?('spec:system')
+  namespace :spec do
+    task :enable_system_specs do # rubocop:disable Rails/RakeEnvironment
+      ENV['RUN_SYSTEM_SPECS'] = 'true'
+    end
+  end
+
+  Rake::Task['spec:system'].enhance ['spec:enable_system_specs']
+end

spec/rails_helper.rb

@@ -1,6 +1,14 @@
 # frozen_string_literal: true
 
 ENV['RAILS_ENV'] ||= 'test'
+
+# This needs to be defined before Rails is initialized
+RUN_SYSTEM_SPECS = ENV.fetch('RUN_SYSTEM_SPECS', false)
+
+if RUN_SYSTEM_SPECS
+  STREAMING_PORT = ENV.fetch('TEST_STREAMING_PORT', '4020')
+  ENV['STREAMING_API_BASE_URL'] = "http://localhost:#{STREAMING_PORT}"
+end
 require File.expand_path('../config/environment', __dir__)
 
 abort('The Rails environment is running in production mode!') if Rails.env.production?
@@ -15,10 +23,14 @@ require 'chewy/rspec'
 Dir[Rails.root.join('spec', 'support', '**', '*.rb')].each { |f| require f }
 
 ActiveRecord::Migration.maintain_test_schema!
-WebMock.disable_net_connect!(allow: Chewy.settings[:host])
+WebMock.disable_net_connect!(allow: Chewy.settings[:host], allow_localhost: RUN_SYSTEM_SPECS)
 Sidekiq::Testing.inline!
 Sidekiq.logger = nil
 
+# System tests config
+DatabaseCleaner.strategy = [:deletion]
+streaming_server_manager = StreamingServerManager.new
+
 Devise::Test::ControllerHelpers.module_eval do
   alias_method :original_sign_in, :sign_in
 
@@ -56,6 +68,8 @@ module SignedRequestHelpers
 end
 
 RSpec.configure do |config|
+  # This is set before running spec:system, see lib/tasks/tests.rake
+  config.filter_run_excluding type: :system unless RUN_SYSTEM_SPECS
   config.fixture_path = Rails.root.join('spec', 'fixtures')
   config.use_transactional_fixtures = true
   config.order = 'random'
@@ -83,8 +97,7 @@ RSpec.configure do |config|
   end
 
   config.before :each, type: :feature do
-    https = ENV['LOCAL_HTTPS'] == 'true'
+    Capybara.current_driver = :rack_test
-    Capybara.app_host = "http#{https ? 's' : ''}://#{ENV.fetch('LOCAL_DOMAIN')}"
   end
 
   config.before :each, type: :controller do
@@ -95,6 +108,35 @@ RSpec.configure do |config|
     stub_jsonld_contexts!
   end
 
+  config.before :suite do
+    if RUN_SYSTEM_SPECS
+      Webpacker.compile
+      streaming_server_manager.start(port: STREAMING_PORT)
+    end
+  end
+
+  config.after :suite do
+    streaming_server_manager.stop
+  end
+
+  config.around :each, type: :system do |example|
+    # driven_by :selenium, using: :chrome, screen_size: [1600, 1200]
+    driven_by :selenium, using: :headless_chrome, screen_size: [1600, 1200]
+
+    # The streaming server needs access to the database
+    # but with use_transactional_tests every transaction
+    # is rolled-back, so the streaming server never sees the data
+    # So we disable this feature for system tests, and use DatabaseCleaner to clean
+    # the database tables between each test
+    self.use_transactional_tests = false
+
+    DatabaseCleaner.cleaning do
+      example.run
+    end
+
+    self.use_transactional_tests = true
+  end
+
   config.before(:each) do |example|
     unless example.metadata[:paperclip_processing]
       allow_any_instance_of(Paperclip::Attachment).to receive(:post_process).and_return(true) # rubocop:disable RSpec/AnyInstance

spec/spec_helper.rb

@@ -52,3 +52,80 @@ def expect_push_bulk_to_match(klass, matcher)
     'args' => matcher,
   }))
 end
+
+class StreamingServerManager
+  @running_thread = nil
+
+  def initialize
+    at_exit { stop }
+  end
+
+  def start(port: 4020)
+    return if @running_thread
+
+    queue = Queue.new
+
+    @queue = queue
+
+    @running_thread = Thread.new do
+      Open3.popen2e(
+        {
+          'REDIS_NAMESPACE' => ENV.fetch('REDIS_NAMESPACE'),
+          'DB_NAME' => "#{ENV.fetch('DB_NAME', 'mastodon')}_test#{ENV.fetch('TEST_ENV_NUMBER', '')}",
+          'RAILS_ENV' => ENV.fetch('RAILS_ENV', 'test'),
+          'NODE_ENV' => ENV.fetch('STREAMING_NODE_ENV', 'development'),
+          'PORT' => port.to_s,
+        },
+        'node index.js', # must not call yarn here, otherwise it will fail because yarn does not send signals to its child process
+        chdir: Rails.root.join('streaming')
+      ) do |_stdin, stdout_err, process_thread|
+        status = :starting
+
+        # Spawn a thread to listen on streaming server output
+        output_thread = Thread.new do
+          stdout_err.each_line do |line|
+            Rails.logger.info "Streaming server: #{line}"
+
+            if status == :starting && line.match('Streaming API now listening on')
+              status = :started
+              @queue.enq 'started'
+            end
+          end
+        end
+
+        # And another thread to listen on commands from the main thread
+        loop do
+          msg = queue.pop
+
+          case msg
+          when 'stop'
+            # we need to properly stop the reading thread
+            output_thread.kill
+
+            # Then stop the node process
+            Process.kill('KILL', process_thread.pid)
+
+            # And we stop ourselves
+            @running_thread.kill
+          end
+        end
+      end
+    end
+
+    # wait for 10 seconds for the streaming server to start
+    Timeout.timeout(10) do
+      loop do
+        break if @queue.pop == 'started'
+      end
+    end
+  end
+
+  def stop
+    return unless @running_thread
+
+    @queue.enq 'stop'
+
+    # Wait for the thread to end
+    @running_thread.join
+  end
+end

spec/support/stories/profile_stories.rb

@@ -9,6 +9,8 @@ module ProfileStories
       email: email, password: password, confirmed_at: confirmed_at,
       account: Fabricate(:account, username: 'bob')
     )
+
+    Web::Setting.where(user: bob).first_or_initialize(user: bob).update!(data: { introductionVersion: 201812160442020 }) if finished_onboarding # rubocop:disable Style/NumericLiterals
   end
 
   def as_a_logged_in_user
@@ -42,4 +44,8 @@ module ProfileStories
   def password
     @password ||= 'password'
   end
+
+  def finished_onboarding
+    @finished_onboarding || false
+  end
 end

spec/system/new_statuses_spec.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe 'NewStatuses' do
+  include ProfileStories
+
+  subject { page }
+
+  let(:email)               { 'test@example.com' }
+  let(:password)            { 'password' }
+  let(:confirmed_at)        { Time.zone.now }
+  let(:finished_onboarding) { true }
+
+  before do
+    as_a_logged_in_user
+    visit root_path
+  end
+
+  it 'can be posted' do
+    expect(subject).to have_css('div.app-holder')
+
+    status_text = 'This is a new status!'
+
+    within('.compose-form') do
+      fill_in "What's on your mind?", with: status_text
+      click_on 'Publish!'
+    end
+
+    expect(subject).to have_selector('.status__content__text', text: status_text)
+  end
+
+  it 'can be posted again' do
+    expect(subject).to have_css('div.app-holder')
+
+    status_text = 'This is a second status!'
+
+    within('.compose-form') do
+      fill_in "What's on your mind?", with: status_text
+      click_on 'Publish!'
+    end
+
+    expect(subject).to have_selector('.status__content__text', text: status_text)
+  end
+end