diff --git a/Gemfile b/Gemfile index f839e40aa53..058f906a64f 100644 --- a/Gemfile +++ b/Gemfile @@ -233,3 +233,5 @@ gem 'hcaptcha', '~> 7.1' gem 'mail', '~> 2.8' gem 'base58', '~> 0.2.3' + +gem 'bloom_fit', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index 4bab1e66277..0a3e55c11f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -133,6 +133,8 @@ GEM bindata (2.5.1) binding_of_caller (2.0.0) debug_inspector (>= 1.2.0) + bloom_fit (1.2.0) + msgpack (~> 1.0) blurhash (0.1.8) bootsnap (1.26.0) msgpack (~> 1.5) @@ -946,6 +948,7 @@ DEPENDENCIES base58 (~> 0.2.3) better_errors (~> 2.9) binding_of_caller + bloom_fit (~> 1.2) blurhash (~> 0.1) bootsnap brakeman (~> 8.0) diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 0deeb76e3ea..a08bd1d744a 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -6,10 +6,11 @@ class AccountsController < ApplicationController include AccountControllerConcern include SignatureAuthentication + include Redisable - vary_by -> { public_fetch_mode? ? 'Accept, Accept-Language, Cookie' : 'Accept, Accept-Language, Cookie, Signature' } + vary_by -> { actors_require_signature? ? 'Accept, Accept-Language, Cookie, Signature' : 'Accept, Accept-Language, Cookie' } - before_action :require_account_signature!, if: -> { request.format == :json && authorized_fetch_mode? } + before_action :require_account_signature!, if: -> { request.format == :json && actors_require_signature? } skip_around_action :set_locale, if: -> { [:json, :rss].include?(request.format&.to_sym) } skip_before_action :require_functional!, unless: :limited_federation_mode? @@ -31,7 +32,10 @@ class AccountsController < ApplicationController end format.json do - expires_in 3.minutes, public: !(authorized_fetch_mode? && signed_request_account.present?) + expires_in 3.minutes, public: !(actors_require_signature? && signed_request_account.present?) + + record_reach! + render_with_cache json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter end end @@ -39,6 +43,10 @@ class AccountsController < ApplicationController private + def record_reach! + AccountReachFilter.record_reach_for(@account.id, signed_request_account&.preferred_inbox_url) + end + def filtered_statuses default_statuses.tap do |statuses| statuses.merge!(hashtag_scope) if tag_requested? diff --git a/app/helpers/admin/settings/discovery_helper.rb b/app/helpers/admin/settings/discovery_helper.rb index 0aa4d4368f3..c7197cecc5e 100644 --- a/app/helpers/admin/settings/discovery_helper.rb +++ b/app/helpers/admin/settings/discovery_helper.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true module Admin::Settings::DiscoveryHelper - def discovery_warning_hint_text + def authorized_fetch_warning_hint_text authorized_fetch_overridden? ? t('admin.settings.security.authorized_fetch_overridden_hint') : nil end - def discovery_hint_text + def authorized_fetch_hint_text t('admin.settings.security.authorized_fetch_hint') end - def discovery_recommended_value + def authorized_fetch_recommended_value authorized_fetch_overridden? ? :overridden : nil end end diff --git a/app/helpers/authorized_fetch_helper.rb b/app/helpers/authorized_fetch_helper.rb index edcbf23c49c..ac2f3eb9298 100644 --- a/app/helpers/authorized_fetch_helper.rb +++ b/app/helpers/authorized_fetch_helper.rb @@ -1,8 +1,23 @@ # frozen_string_literal: true module AuthorizedFetchHelper + def authorized_fetch_mode + case ENV.fetch('AUTHORIZED_FETCH') { Setting.authorized_fetch } + when true, 'true', 'all' + 'all' + when false, 'false', 'none' + 'none' + else + 'actors' + end + end + def authorized_fetch_mode? - ENV.fetch('AUTHORIZED_FETCH') { Setting.authorized_fetch && 'true' } == 'true' || Rails.configuration.x.mastodon.limited_federation_mode + authorized_fetch_mode == 'all' || Rails.configuration.x.mastodon.limited_federation_mode + end + + def actors_require_signature? + Mastodon::Feature.bloom_filters_enabled? || %w(actors all).include?(authorized_fetch_mode) || Rails.configuration.x.mastodon.limited_federation_mode end def authorized_fetch_overridden? diff --git a/app/models/account.rb b/app/models/account.rb index 31a5deadc82..d3b03696e6b 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -109,6 +109,7 @@ class Account < ApplicationRecord include DomainNormalizable include Paginable include Reviewable + include AuthorizedFetchHelper enum :protocol, { ostatus: 0, activitypub: 1 } enum :suspension_origin, { local: 0, remote: 1 }, prefix: true @@ -122,6 +123,7 @@ class Account < ApplicationRecord # Remote user validations validates :uri, presence: true, exclusion: { in: [''] }, uniqueness: true, unless: :local?, on: :create + validates :inbox_url, presence: true, if: -> { !local? && (new_record? || will_save_change_to_inbox_url?) } # Local user validations validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: USERNAME_LENGTH_LIMIT }, if: -> { local? && will_save_change_to_username? && !actor_type_application? } @@ -531,6 +533,7 @@ class Account < ApplicationRecord end before_validation :prepare_contents, if: :local? + before_create :prepare_reach_filter, if: :local? before_create :generate_keys before_destroy :clean_feed_manager @@ -549,6 +552,12 @@ class Account < ApplicationRecord private + def prepare_reach_filter + return if instance_actor? + + build_reach_filter if actors_require_signature? + end + def prepare_contents display_name&.strip! note&.strip! diff --git a/app/models/account_reach_filter.rb b/app/models/account_reach_filter.rb new file mode 100644 index 00000000000..2656f88b00b --- /dev/null +++ b/app/models/account_reach_filter.rb @@ -0,0 +1,179 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: account_reach_filters +# +# id :bigint(8) not null, primary key +# bloom_filter :binary +# salt :string not null +# saturated :boolean default(FALSE), not null +# created_at :datetime not null +# updated_at :datetime not null +# account_id :bigint(8) not null +# +class AccountReachFilter < ApplicationRecord + include Redisable + include Lockable + + belongs_to :account + + after_initialize :set_salt + + # This class uses bloom filters to (optionally) keep track of which server is aware of an account. + # Once an account with an `AccountReachFilter` has federated at least once, `bloom_filter` will + # hold a bloom filter used to approximate the set of remote domains it has federated to. + # + # Before different accounts have widely different federation patterns, we start with a small bloom + # filter and add new larger ones if needed, until the account has federated so much that we consider + # the reach filter “saturated”, meaning the underlying bloom filter is not worth keeping. + # + # We use the same target positive rate for all individual bloom filters, so the more bloom filters + # an account reach filter has, the more the total error rate will increase. + # + # For instance, with a false positive of 0.01%, on a server that knows about 100_000 different domains, + # an account reach filter with 5 underlying bloom filters will cause around 50 false positives. + + # Ideal false positive rate + TARGET_FALSE_POSITIVE_RATE = 0.0001 + + # Tolerated false positive rate for largest-size bloom filters, after which a filter is considered saturated + TARGET_SATURATION_FALSE_POSITIVE_RATE = 0.75 + + # Target capacity for bloom filters of individual sizes + BLOOM_FILTER_TARGET_CAPACITIES = [50, 100, 350, 1_000, 1_500, 5_000].freeze + + # Batch size for processing queued additions + BATCH_SIZE = 500 + DEBOUNCE_DELAY = 5.minutes + + class BloomFilterSerializer + def self.load(value) + return [] if value.nil? + + value = MessagePack.unpack(value) + + # A previous version of the code stored a single bloom filter + value = [value] unless value.first.is_a?(Array) + + value.map do |marshal| + BloomFit.allocate.tap { |bf| bf.marshal_load(marshal) } + end + end + + def self.dump(value) + return nil if value.empty? + + MessagePack.pack(value.map(&:marshal_dump)) + end + end + + # An earlier version used the name `bloom_filter`, but it can hold multiple + alias_attribute :bloom_filters, :bloom_filter + serialize :bloom_filters, coder: BloomFilterSerializer + + class << self + include Redisable + include AuthorizedFetchHelper + + # This is a class method rather than an instance method because we want to avoid + # loading the database row: the `bloom_filters` column can be quite large + def record_reach_for(account_id, inbox_url) + # Remove reach filter if the we can't ensure we're going to handle every request + return AccountReachFilter.where(account_id: account_id).delete_all unless actors_require_signature? + + # It's not ideal, but we have no way to tell the remote account about it… + return if inbox_url.blank? + + account_reach_filter_id = AccountReachFilter.where(account_id: account_id, saturated: false).pick(:id) + return if account_reach_filter_id.nil? + + with_redis do |redis| + redis.sadd("account_reach:#{account_reach_filter_id}:to_add", Addressable::URI.parse(inbox_url).normalized_host) + end + + UpdateAccountReachWorker.perform_in(DEBOUNCE_DELAY, account_reach_filter_id) + end + end + + def add(*hosts) + return if saturated || hosts.empty? + + bloom_filters << BloomFit.new(capacity: BLOOM_FILTER_TARGET_CAPACITIES.first, false_positive_rate: TARGET_FALSE_POSITIVE_RATE) if bloom_filters.empty? + + next_filter_class = bloom_filters.size + threshold = ((next_filter_class.nil? ? TARGET_SATURATION_FALSE_POSITIVE_RATE : TARGET_FALSE_POSITIVE_RATE)**(1.0 / bloom_filters.last.k)) * bloom_filters.last.m + + hosts.each do |host| + next if include?(host) + + bloom_filters.last.add("#{salt}:#{host}") + next if bloom_filters.last.set_bits < threshold + + # We have reached the threshold after which false-positives are too frequent for us. + # Either update the filter or mark it as saturated. + if next_filter_class < BLOOM_FILTER_TARGET_CAPACITIES.size + bloom_filters << BloomFit.new(capacity: BLOOM_FILTER_TARGET_CAPACITIES[next_filter_class], false_positive_rate: TARGET_FALSE_POSITIVE_RATE) + + next_filter_class = bloom_filter.size + threshold = ((next_filter_class.nil? ? TARGET_SATURATION_FALSE_POSITIVE_RATE : TARGET_FALSE_POSITIVE_RATE)**(1.0 / bloom_filters.last.k)) * bloom_filters.last.m + else + update!(saturated: true, bloom_filter: nil) + + break + end + end + end + + def include?(host) + return true if saturated + + bloom_filters.any? { |filter| filter.include?("#{salt}:#{host}") } + end + + def filter_inboxes(inboxes) + return inboxes if destroyed? || saturated? + + process_queued_additions_with_lock! + + return inboxes if saturated? + return [] if bloom_filters.all?(&:blank?) + + inboxes.filter do |url| + include?(Addressable::URI.parse(url).normalized_host) + rescue + true + end + end + + def process_queued_additions! + with_redis do |redis| + reload + loop do + domains = redis.spop("account_reach:#{id}:to_add", BATCH_SIZE) + break if domains.blank? + + add(*domains) + end + end + + save! + end + + private + + def process_queued_additions_with_lock! + return unless persisted? + raise 'AccountReachFilter changes need to be performed with a lock' if changed? + + with_redis_lock("consolidate_account_reach_filter:#{id}", autorelease: 5.minutes) do + reload + + process_queued_additions! + end + end + + def set_salt + self.salt ||= SecureRandom.alphanumeric(4) + end +end diff --git a/app/models/concerns/account/associations.rb b/app/models/concerns/account/associations.rb index 1a3d5f68456..82c1b8a8862 100644 --- a/app/models/concerns/account/associations.rb +++ b/app/models/concerns/account/associations.rb @@ -78,5 +78,8 @@ module Account::Associations # BulkImport records owned by account has_many :bulk_imports, inverse_of: :account, dependent: :delete_all + + # Bloom filter for reach (servers knowing about that actor) + has_one :reach_filter, class_name: 'AccountReachFilter', inverse_of: :account, dependent: :destroy end end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index 5e97151438f..427044192fa 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -83,10 +83,6 @@ class Form::AdminSettings custom_css ).freeze - OVERRIDEN_SETTINGS = { - authorized_fetch: :authorized_fetch_mode?, - }.freeze - UPLOAD_MIME_TYPES = %w(image/jpeg image/png image/gif image/webp).freeze DESCRIPTION_LIMIT = 200 @@ -95,6 +91,7 @@ class Form::AdminSettings FEED_ACCESS_MODES = %w(public authenticated disabled).freeze ALTERNATE_FEED_ACCESS_MODES = %w(public authenticated).freeze LANDING_PAGE = %w(trends overview local_feed about).freeze + AUTHORIZED_FETCH_MODES = %w(none actors all).freeze attr_accessor(*KEYS) @@ -122,8 +119,6 @@ class Form::AdminSettings stored_value = if UPLOAD_KEYS.include?(key) SiteUpload.where(var: key).first_or_initialize(var: key) - elsif OVERRIDEN_SETTINGS.include?(key) - public_send(OVERRIDEN_SETTINGS[key]) else Setting.public_send(key) end @@ -141,6 +136,10 @@ class Form::AdminSettings end end + def authorized_fetch + authorized_fetch_mode + end + def save # NOTE: Annoyingly, files are processed and can error out before # validations are called, and `valid?` clears errors… diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index ff5bbea966c..0d534e86449 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -159,6 +159,9 @@ class DeleteAccountService < BaseService purge_feeds! purge_other_associations! + # This needs to happen *after* delivery of `Delete` activities is scheduled + @account.reach_filter&.destroy + @account.destroy unless keep_account_record? end @@ -307,7 +310,9 @@ class DeleteAccountService < BaseService end def low_priority_delivery_inboxes - Account.inboxes - delivery_inboxes + inboxes = Account.inboxes - delivery_inboxes + inboxes = @account.reach_filter.filter_inboxes(inboxes) if @account.reach_filter.present? + inboxes end def reported_status_ids diff --git a/app/views/admin/settings/discovery/show.html.haml b/app/views/admin/settings/discovery/show.html.haml index cf30a2e310c..eabace0417c 100644 --- a/app/views/admin/settings/discovery/show.html.haml +++ b/app/views/admin/settings/discovery/show.html.haml @@ -89,13 +89,16 @@ .fields-group = f.input :authorized_fetch, - as: :boolean, + as: :radio_buttons, + collection: f.object.class::AUTHORIZED_FETCH_MODES, disabled: authorized_fetch_overridden?, - hint: discovery_hint_text, + hint: authorized_fetch_hint_text, label: t('admin.settings.security.authorized_fetch'), - recommended: discovery_recommended_value, - warning_hint: discovery_warning_hint_text, - wrapper: :with_label + include_blank: false, + label_method: ->(mode) { safe_join([I18n.t(mode, scope: 'admin.settings.security.authorized_fetch_modes'), content_tag(:span, I18n.t(mode, scope: 'admin.settings.security.authorized_fetch_hints'), class: 'hint')]) }, + recommended: authorized_fetch_recommended_value, + warning_hint: authorized_fetch_warning_hint_text, + wrapper: :with_block_label %h2= t('admin.settings.discovery.follow_recommendations') diff --git a/app/workers/scheduler/self_destruct_scheduler.rb b/app/workers/scheduler/self_destruct_scheduler.rb index 05bca4f540b..2d4068a7b90 100644 --- a/app/workers/scheduler/self_destruct_scheduler.rb +++ b/app/workers/scheduler/self_destruct_scheduler.rb @@ -59,7 +59,10 @@ class Scheduler::SelfDestructScheduler .sign!(account) .to_json - ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| + # We most probably don't need to send deletion notices to every server + filtered_inboxes = account.reach_filter.present? ? account.reach_filter.filter_inboxes(inboxes) : inboxes + + ActivityPub::DeliveryWorker.push_bulk(filtered_inboxes, limit: 1_000) do |inbox_url| [json, account.id, inbox_url] end diff --git a/app/workers/update_account_reach_worker.rb b/app/workers/update_account_reach_worker.rb new file mode 100644 index 00000000000..5234fa89c42 --- /dev/null +++ b/app/workers/update_account_reach_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class UpdateAccountReachWorker + include Sidekiq::Worker + include Redisable + include Lockable + + # This job is using `until_executing` rather than `until_executed` on purpose: + # with `until_executed`, there exists a time window during which the worker + # would still hold a lock while not looking at new items in + # `account_reach:id:to_add`, so there would be no guarantee those items would + # be processed. `until_executing` ensures anything added while the lock is held + # will be processed by the worker. + sidekiq_options queue: 'ingress', lock: :until_executing, retry: 5 + + def perform(account_reach_filter_id) + # Since we are using `until_executing` rather than `until_executed`, lock + # the whole process to avoid race conditions. + # Jobs should be very short (far below a second under normal load), so + # 5 minutes for auto-release sounds like plenty enough to account for + # high load situations. + with_redis_lock("consolidate_account_reach_filter:#{account_reach_filter_id}", autorelease: 5.minutes) do + filter = AccountReachFilter.find_by(id: account_reach_filter_id) + return if filter.nil? + + filter.process_queued_additions! + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 5e31ea1b172..928a0618103 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1022,7 +1022,15 @@ en: warning_hint: We recommend using “Approval required for sign up” unless you are confident your moderation team can handle spam and malicious registrations in a timely fashion. security: authorized_fetch: Require authentication from federated servers - authorized_fetch_hint: Requiring authentication from federated servers enables stricter enforcement of both user-level and server-level blocks. However, this comes at the cost of a performance penalty, reduces the reach of your replies, and may introduce compatibility issues with some federated services. In addition, this will not prevent dedicated actors from fetching your public posts and accounts. + authorized_fetch_hint: Decide when remote servers are required to authenticate when fetching content from this server. + authorized_fetch_hints: + actors: Requiring authentication for profiles enables tracking of which server has ever discovered each of your local accounts, which can greatly reduce the number of unnecessary requests performed when notifying servers of an account deletion. This option comes at a slight performance cost when serving profiles due to said tracking and the inability for the reverse-proxy to cache responses for these requests. This might also introduce compatibility issues with some federated services. + all: In addition to the benefits and drawbacks of requiring authentication for profiles, unconditionally requiring authentication from federated servers enables stricter enforcement of both user-level and server-level blocks, but and also prevents caching of most requests. It will also reduce the reach of your replies, as reply forwarding will be disabled. + none: Requiring authentication only when strictly necessary allows your reverse proxy to cache more requests. This option might get removed in the future. Only use it if the other options cause issues. + authorized_fetch_modes: + actors: For profiles and private content + all: For every request + none: For private content only authorized_fetch_overridden_hint: You are currently unable to change this setting because it is overridden by an environment variable. federation_authentication: Federation authentication enforcement title: Server settings diff --git a/db/migrate/20260820142255_create_account_reach_filters.rb b/db/migrate/20260820142255_create_account_reach_filters.rb new file mode 100644 index 00000000000..32de799c1ae --- /dev/null +++ b/db/migrate/20260820142255_create_account_reach_filters.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class CreateAccountReachFilters < ActiveRecord::Migration[8.1] + def change + create_table :account_reach_filters do |t| + t.references :account, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true } + t.string :salt, null: false + t.boolean :saturated, null: false, default: false + t.binary :bloom_filter + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 579bff9dd37..21c6bb11965 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_08_12_154114) do +ActiveRecord::Schema[8.1].define(version: 2026_08_20_142255) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -90,6 +90,16 @@ ActiveRecord::Schema[8.1].define(version: 2026_08_12_154114) do t.index ["target_account_id"], name: "index_account_pins_on_target_account_id" end + create_table "account_reach_filters", force: :cascade do |t| + t.bigint "account_id", null: false + t.binary "bloom_filter" + t.datetime "created_at", null: false + t.string "salt", null: false + t.boolean "saturated", default: false, null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_account_reach_filters_on_account_id", unique: true + end + create_table "account_relationship_severance_events", force: :cascade do |t| t.bigint "account_id", null: false t.datetime "created_at", null: false @@ -1492,6 +1502,7 @@ ActiveRecord::Schema[8.1].define(version: 2026_08_12_154114) do add_foreign_key "account_notes", "accounts", on_delete: :cascade add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade add_foreign_key "account_pins", "accounts", on_delete: :cascade + add_foreign_key "account_reach_filters", "accounts", on_delete: :cascade add_foreign_key "account_relationship_severance_events", "accounts", on_delete: :cascade add_foreign_key "account_relationship_severance_events", "relationship_severance_events", on_delete: :cascade add_foreign_key "account_stats", "accounts", on_delete: :cascade diff --git a/spec/fabricators/account_fabricator.rb b/spec/fabricators/account_fabricator.rb index 6a5218cee98..7e1e1b4349e 100644 --- a/spec/fabricators/account_fabricator.rb +++ b/spec/fabricators/account_fabricator.rb @@ -12,6 +12,7 @@ Fabricator(:account) do silenced_at { |attrs| attrs[:silenced] ? Time.now.utc : nil } requested_deletion_at { |attrs| attrs[:requested_deletion] ? Time.now.utc : nil } user { |attrs| attrs[:domain].nil? ? Fabricate.build(:user, account: nil) : nil } + inbox_url { |attrs| attrs[:domain].nil? ? '' : "https://#{attrs[:domain]}/users/#{attrs[:username]}/inbox" } uri { |attrs| attrs[:domain].nil? ? nil : "https://#{attrs[:domain]}/users/#{attrs[:username]}" } discoverable true indexable true diff --git a/spec/fabricators/account_reach_filter_fabricator.rb b/spec/fabricators/account_reach_filter_fabricator.rb new file mode 100644 index 00000000000..2b7c5ec7e46 --- /dev/null +++ b/spec/fabricators/account_reach_filter_fabricator.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Fabricator(:account_reach_filter) do + account do + # Awful hack but ensures we have the right reach filter regardless of configuration + Fabricate(:account).tap { |account| account.reach_filter&.destroy } + end + + salt { SecureRandom.alphanumeric(4) } +end diff --git a/spec/models/account_reach_filter_spec.rb b/spec/models/account_reach_filter_spec.rb new file mode 100644 index 00000000000..9b87873dc4f --- /dev/null +++ b/spec/models/account_reach_filter_spec.rb @@ -0,0 +1,208 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe AccountReachFilter do + describe 'basic functionality' do + let(:filter) { Fabricate(:account_reach_filter) } + + it 'allows correct membership tests' do + filter.add('mastodon.social') + expect(filter.include?('mastodon.social')).to be true + expect(filter.include?('mastodon.online')).to be false + end + + it 'allows correct membership tests after save/reload' do + filter.add('mastodon.social') + filter.save! + filter.reload + expect(filter.include?('mastodon.social')).to be true + expect(filter.include?('mastodon.online')).to be false + end + end + + describe '#add' do + let(:filter) { Fabricate(:account_reach_filter) } + + context 'with a single argument' do + it 'allows correct membership tests' do + filter.add('mastodon.social') + + expect(filter.include?('mastodon.social')).to be true + expect(filter.include?('mastodon.online')).to be false + expect(filter.include?('example.com')).to be false + end + end + + context 'with multiple arguments' do + it 'allows correct membership tests' do + filter.add('mastodon.social', 'mastodon.online') + + expect(filter.include?('mastodon.social')).to be true + expect(filter.include?('mastodon.online')).to be true + expect(filter.include?('example.com')).to be false + end + end + + context 'when saturated' do + before { filter.update!(saturated: true) } + + it 'keeps the filter saturated' do + expect { filter.add('mastodon.social') } + .to_not change(filter, :saturated) + end + end + + context 'when adding to a filter that needs upgrading' do + before do + stub_const('AccountReachFilter::BLOOM_FILTER_TARGET_CAPACITIES', [3, 10_000]) + filter.add('mastodon.social') + filter.save! + end + + it 'upgrades the filter and keeps functionality', :aggregate_failures do + expect do + filter.add('mastodon.online', 'example.com', 'joinmastodon.org') + filter.save! + end + .to(change { filter.bloom_filters.size }) + + expect(%w(mastodon.social mastodon.online example.com joinmastodon.org evil.com unknown.com).filter { |value| filter.include?(value) }) + .to contain_exactly('mastodon.social', 'mastodon.online', 'example.com', 'joinmastodon.org') + end + + context 'when adding to a filter so much that it saturates' do + before do + stub_const('AccountReachFilter::BLOOM_FILTER_TARGET_CAPACITIES', [2, 5]) + filter.add('mastodon.social') + filter.save! + end + + it 'upgrades the filter and keeps functionality', :aggregate_failures do + expect do + filter.add(*Array.new(100) { |i| "https://test#{i}.example.com" }) + filter.save! + end + .to(change(filter, :saturated).to(true)) + end + end + end + end + + describe '#include?' do + let(:filter) { Fabricate(:account_reach_filter) } + + context 'with a saturated filter' do + before { filter.update!(saturated: true) } + + it 'always returns true' do + expect(%w(mastodon.social mastodon.online example.com joinmastodon.org).all? { |value| filter.include?(value) }) + .to be true + end + end + end + + describe '#filter_inboxes' do + let(:inboxes) do + %w(https://example.com/inbox https://mastodon.social/inbox https://mastodon.online/inbox) + end + + context 'when the filter is empty' do + let(:filter) { Fabricate(:account_reach_filter) } + + it 'returns an empty array' do + expect(filter.filter_inboxes(inboxes)) + .to be_empty + end + end + + context 'when the filter has items' do + let(:filter) { Fabricate(:account_reach_filter) } + + before do + filter.add('mastodon.social') + filter.save! + end + + it 'returns the correct inboxes' do + expect(filter.filter_inboxes(inboxes)) + .to contain_exactly('https://mastodon.social/inbox') + end + end + + context 'when the filter is saturated' do + let(:filter) { Fabricate(:account_reach_filter, saturated: true) } + + it 'returns all inboxes' do + expect(filter.filter_inboxes(inboxes)) + .to eq inboxes + end + end + end + + describe '.record_reach_for' do + subject { described_class.record_reach_for(account.id, inbox_url) } + + let(:inbox_url) { 'https://mastodon.social/inbox' } + + context 'when signatures are required for actors' do + before { Setting.authorized_fetch = true } + + context 'with an account that has a reach filter' do + let(:account) { Fabricate(:account) } + let(:reach_filter) { account.reach_filter } + + it 'keeps the filter and schedules UpdateAccountReachWorker' do + expect { subject } + .to not_change(described_class, :count) + .and enqueue_sidekiq_job(UpdateAccountReachWorker).with(reach_filter.id) + + expect(described_class.exists?(id: reach_filter.id)).to be true + end + end + + context 'with an account that does not have a reach filter' do + let(:account) { Fabricate(:account) } + + before { described_class.where(account_id: account.id).delete_all } + + it 'does not create a filter nor schedule UpdateAccountReachWorker' do + expect { subject } + .to_not enqueue_sidekiq_job(UpdateAccountReachWorker) + + expect(described_class.exists?(account_id: account.id)).to be false + end + end + end + + context 'when signatures are not required for actors' do + before { Setting.authorized_fetch = false } + + context 'with an account that has a reach filter' do + let!(:account) { Fabricate(:account_reach_filter).account } + + it 'deletes the filter and does not enqueue any job' do + expect { subject } + .to change(described_class, :count).by(-1) + + expect(UpdateAccountReachWorker).to_not have_enqueued_sidekiq_job + + expect(described_class.exists?(account_id: account.id)).to be false + end + end + + context 'with an account that does not have a reach filter' do + let(:account) { Fabricate(:account) } + + before { described_class.where(account_id: account.id).delete_all } + + it 'does not create any filter nor schedules UpdateAccountReachWorker' do + expect { subject } + .to_not enqueue_sidekiq_job(UpdateAccountReachWorker) + + expect(described_class.exists?(account_id: account.id)).to be false + end + end + end + end +end diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index 70d349d13ad..a1735af6c37 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -745,12 +745,12 @@ RSpec.describe Account do context 'when is remote' do it 'does not generate keys' do key = OpenSSL::PKey::RSA.new(1024).public_key - account = described_class.create!(domain: 'remote', uri: 'https://remote/actor', username: 'remote_user_with_public', public_key: key.to_pem) + account = described_class.create!(domain: 'remote', uri: 'https://remote/actor', inbox_url: 'https://remote/actor/inbox', username: 'remote_user_with_public', public_key: key.to_pem) expect(account.keypair.keypair.params).to eq key.params end it 'normalizes domain' do - account = described_class.create!(domain: 'にゃん', uri: 'https://xn--r9j5b5b/actor', username: 'remote_user_with_idn_domain') + account = described_class.create!(domain: 'にゃん', uri: 'https://xn--r9j5b5b/actor', inbox_url: 'https://xn--r9j5b5b/actor/inbox', username: 'remote_user_with_idn_domain') expect(account.domain).to eq 'xn--r9j5b5b' end end diff --git a/spec/requests/accounts_spec.rb b/spec/requests/accounts_spec.rb index 26f8f577b09..79b490e5b7f 100644 --- a/spec/requests/accounts_spec.rb +++ b/spec/requests/accounts_spec.rb @@ -4,13 +4,26 @@ require 'rails_helper' RSpec.describe 'Accounts show response' do let(:account) { Fabricate(:account) } + let(:authorized_fetch_mode) { 'false' } + + around do |example| + ClimateControl.modify AUTHORIZED_FETCH: authorized_fetch_mode.to_s do + example.run + end + end context 'with numeric-based identifiers' do context 'with JSON format' do - it 'returns http success' do + before do + account.build_reach_filter + account.save! + end + + it 'returns http success and removes reach filter' do get "/ap/users/#{account.id}", headers: { 'ACCEPT' => 'application/json' } expect(response).to have_http_status(200) + expect(account.reload.reach_filter.present?).to be false end end @@ -146,12 +159,6 @@ RSpec.describe 'Accounts show response' do let(:authorized_fetch_mode) { false } let(:headers) { { 'ACCEPT' => 'application/json' } } - around do |example| - ClimateControl.modify AUTHORIZED_FETCH: authorized_fetch_mode.to_s do - example.run - end - end - context 'with a normal account in a JSON request' do before do get short_account_path(username: account.username), headers: headers @@ -198,14 +205,17 @@ RSpec.describe 'Accounts show response' do end end - context 'with signature' do - let(:remote_account) { Fabricate(:account, domain: 'example.com') } + context 'with signature', :inline_jobs do + let(:remote_account) { Fabricate(:account, domain: 'example.com', inbox_url: 'https://example.com/inbox') } before do + account.build_reach_filter + account.save! + get short_account_path(username: account.username), headers: headers, sign_with: remote_account end - it 'returns a JSON version of the account', :aggregate_failures do + it 'returns a JSON version of the account and removes reach filter', :aggregate_failures do expect(response) .to have_http_status(200) .and have_cacheable_headers.with_vary('Accept, Accept-Language, Cookie') @@ -214,12 +224,14 @@ RSpec.describe 'Accounts show response' do ) expect(response.parsed_body).to include(:id, :type, :preferredUsername, :inbox, :publicKey, :name, :summary) + + expect(account.reload.reach_filter.present?).to be false end context 'with authorized fetch mode' do let(:authorized_fetch_mode) { true } - it 'returns a private signature JSON version of the account', :aggregate_failures do + it 'returns a private signature JSON version of the account and updates the reach filter', :aggregate_failures do expect(response) .to have_http_status(200) .and have_attributes( @@ -230,6 +242,9 @@ RSpec.describe 'Accounts show response' do expect(response.headers['Vary']).to include 'Signature' expect(response.parsed_body).to include(:id, :type, :preferredUsername, :inbox, :publicKey, :name, :summary) + + expect(account.reach_filter.reload.include?('example.com')).to be true + expect(account.reach_filter.reload.include?('bad.com')).to be false end end end diff --git a/spec/services/delete_account_service_spec.rb b/spec/services/delete_account_service_spec.rb index d0861ac6e3e..925ae1af5e2 100644 --- a/spec/services/delete_account_service_spec.rb +++ b/spec/services/delete_account_service_spec.rb @@ -81,20 +81,42 @@ RSpec.describe DeleteAccountService do before do stub_request(:post, remote_alice.inbox_url).to_return(status: 201) stub_request(:post, remote_bob.inbox_url).to_return(status: 201) + stub_request(:post, remote_eve.inbox_url).to_return(status: 201) end let!(:remote_alice) { Fabricate(:account, inbox_url: 'https://alice.com/inbox', domain: 'alice.com', protocol: :activitypub) } - let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', domain: 'bob.com', protocol: :activitypub) } + let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', domain: 'bob.com', protocol: :activitypub) } + let!(:remote_eve) { Fabricate(:account, inbox_url: 'https://eve.com/inbox', domain: 'eve.com', protocol: :activitypub) } it_behaves_like 'common behavior' do let(:account) { Fabricate(:account) } let(:local_follower) { Fabricate(:account) } let!(:collection) { Fabricate(:collection, account:) } # rubocop:disable RSpec/LetSetup - it 'sends a delete actor activity to all known inboxes' do - subject - expect(a_request(:post, remote_alice.inbox_url)).to have_been_made.once - expect(a_request(:post, remote_bob.inbox_url)).to have_been_made.once + context 'without a reach filter' do + before { account.reach_filter&.destroy } + + it 'sends a delete actor activity to all known inboxes' do + subject + expect(a_request(:post, remote_alice.inbox_url)).to have_been_made.once + expect(a_request(:post, remote_bob.inbox_url)).to have_been_made.once + expect(a_request(:post, remote_eve.inbox_url)).to have_been_made.once + end + end + + context 'with a reach filter' do + before do + account.build_reach_filter + account.reach_filter.add('alice.com') + account.reach_filter.add('bob.com') + end + + it 'sends a delete actor activity to inboxes matching the reach filter' do + subject + expect(a_request(:post, remote_alice.inbox_url)).to have_been_made.once + expect(a_request(:post, remote_bob.inbox_url)).to have_been_made.once + expect(a_request(:post, remote_eve.inbox_url)).to_not have_been_made + end end end end diff --git a/spec/workers/scheduler/self_destruct_scheduler_spec.rb b/spec/workers/scheduler/self_destruct_scheduler_spec.rb index bf9c989fcf4..23dc1171968 100644 --- a/spec/workers/scheduler/self_destruct_scheduler_spec.rb +++ b/spec/workers/scheduler/self_destruct_scheduler_spec.rb @@ -40,6 +40,7 @@ RSpec.describe Scheduler::SelfDestructScheduler do context 'when sidekiq is operational' do let!(:other_account) { Fabricate :account, inbox_url: 'https://host.example/inbox', domain: 'host.example', protocol: :activitypub } + let!(:another_account) { Fabricate :account, inbox_url: 'https://another-host.example/inbox', domain: 'another-host.example', protocol: :activitypub } it 'deletes local non-deleted accounts' do worker.perform @@ -47,17 +48,44 @@ RSpec.describe Scheduler::SelfDestructScheduler do expect(account.reload.requested_deletion_at).to_not be_nil end - it 'deletes local accounts marked for deletion' do - account.update(requested_deletion_at: 10.days.ago) - deletion_request = Fabricate(:account_deletion_request, account: account) + context 'without a reach filter' do + before { account.reach_filter&.destroy } - worker.perform + it 'deletes local accounts marked for deletion' do + account.update(requested_deletion_at: 10.days.ago) + deletion_request = Fabricate(:account_deletion_request, account: account) - expect(ActivityPub::DeliveryWorker) - .to have_enqueued_sidekiq_job(match_json_values(type: 'Delete', signature: be_present), account.id, other_account.inbox_url) + worker.perform - expect(account.reload.requested_deletion_at).to be > 1.day.ago - expect { deletion_request.reload }.to raise_error(ActiveRecord::RecordNotFound) + expect(ActivityPub::DeliveryWorker) + .to have_enqueued_sidekiq_job(match_json_values(type: 'Delete', signature: be_present), account.id, other_account.inbox_url) + .and have_enqueued_sidekiq_job(match_json_values(type: 'Delete', signature: be_present), account.id, another_account.inbox_url) + + expect(account.reload.requested_deletion_at).to be > 1.day.ago + expect { deletion_request.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context 'with a reach filter' do + before do + account.reach_filter ||= account.create_reach_filter + + account.reach_filter.add('host.example') + account.reach_filter.save! + end + + it 'deletes local accounts marked for deletion' do + account.update(requested_deletion_at: 10.days.ago) + deletion_request = Fabricate(:account_deletion_request, account: account) + + worker.perform + + expect(ActivityPub::DeliveryWorker) + .to have_enqueued_sidekiq_job(match_json_values(type: 'Delete', signature: be_present), account.id, other_account.inbox_url) + + expect(account.reload.requested_deletion_at).to be > 1.day.ago + expect { deletion_request.reload }.to raise_error(ActiveRecord::RecordNotFound) + end end end end diff --git a/spec/workers/update_account_reach_worker_spec.rb b/spec/workers/update_account_reach_worker_spec.rb new file mode 100644 index 00000000000..17a62f42c2f --- /dev/null +++ b/spec/workers/update_account_reach_worker_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe UpdateAccountReachWorker do + subject { described_class.new } + + describe 'perform' do + let(:account_reach_filter) { Fabricate(:account_reach_filter) } + + before do + 100.times { |i| redis.sadd("account_reach:#{account_reach_filter.id}:to_add", "test-domain-#{i}.org") } + end + + it 'consolidates pending additions' do + subject.perform(account_reach_filter.id) + account_reach_filter.reload + + 100.times { |i| expect(account_reach_filter.include?("test-domain-#{i}.org")).to be true } + + expect(account_reach_filter.include?('unknwon-domain.org')).to be false + end + end +end