From 0986a34d2d2bbb773e47f019f4183f00228150dd Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 26 May 2026 16:46:05 +0200 Subject: [PATCH 01/13] add json validation to import model --- app/models/form/import.rb | 38 +++++++++++++++++++--- app/views/settings/imports/index.html.haml | 2 +- spec/fixtures/files/custom_filters.json | 32 ++++++++++++++++++ spec/models/form/import_spec.rb | 11 +++++++ 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 spec/fixtures/files/custom_filters.json diff --git a/app/models/form/import.rb b/app/models/form/import.rb index 3cc4af064ff..986d33c2b38 100644 --- a/app/models/form/import.rb +++ b/app/models/form/import.rb @@ -55,6 +55,8 @@ class Form::Import :bookmarks elsif file_name_matches?('lists') :lists + elsif file_name_matches?('custom_filters') + :custom_filters end end @@ -150,9 +152,34 @@ class Form::Import end end + def file_type_is_json? + data.content_type == 'application/json' + end + + def data_from_json + @data_from_json ||= JSON.parse(data.read)['custom_filters'].map(&:deep_symbolize_keys) + end + + def allowed_json_key? + type.to_sym.in?(%i(custom_filters)) + end + def validate_data return if data.nil? return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT + + if file_type_is_json? + validate_json_data + else + validate_csv_data + end + rescue CSV::MalformedCSVError => e + errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message)) + rescue EmptyFileError + errors.add(:data, I18n.t('imports.errors.empty')) + end + + def validate_csv_data return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless default_csv_headers.all? { |header| csv_data.headers.include?(header) } errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT @@ -163,9 +190,12 @@ class Form::Import limit -= current_account.following_count unless overwrite errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if csv_row_count > limit end - rescue CSV::MalformedCSVError => e - errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message)) - rescue EmptyFileError - errors.add(:data, I18n.t('imports.errors.empty')) + end + + def validate_json_data + return unless allowed_json_key? + + errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if data_from_json.count > ROWS_PROCESSING_LIMIT + errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless allowed_json_key? end end diff --git a/app/views/settings/imports/index.html.haml b/app/views/settings/imports/index.html.haml index e5195b74cc8..027724391bc 100644 --- a/app/views/settings/imports/index.html.haml +++ b/app/views/settings/imports/index.html.haml @@ -5,7 +5,7 @@ .field-group = f.input :type, as: :grouped_select, - collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking) }, + collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking custom_filters) }, group_label_method: ->(group) { I18n.t("imports.type_groups.#{group.first}") }, group_method: :last, hint: t('imports.preface'), diff --git a/spec/fixtures/files/custom_filters.json b/spec/fixtures/files/custom_filters.json new file mode 100644 index 00000000000..25d91cc7478 --- /dev/null +++ b/spec/fixtures/files/custom_filters.json @@ -0,0 +1,32 @@ +{ + "custom_filters": [ + { + "title": "dfjswa", + "expire_at": null, + "context": ["home"], + "action": "warn", + "keywords_attributes": [{ "keyword": "dvshja", "whole_word": true }], + "statuses": [] + }, + { + "title": "filter with a phrase as title", + "expire_at": null, + "context": ["home", "notifications", "public"], + "action": "warn", + "keywords_attributes": [ + { "keyword": "more words let's see ", "whole_word": true } + ], + "statuses": [] + }, + { + "title": "how do I add a status to a filter?", + "expire_at": null, + "context": ["public", "account"], + "action": "warn", + "keywords_attributes": [ + { "keyword": "will this be a status?", "whole_word": true } + ], + "statuses": [] + } + ] +} diff --git a/spec/models/form/import_spec.rb b/spec/models/form/import_spec.rb index d682e13ecb9..e61ba1f4e9d 100644 --- a/spec/models/form/import_spec.rb +++ b/spec/models/form/import_spec.rb @@ -43,6 +43,17 @@ RSpec.describe Form::Import do end end + context 'when the import type is custom_filters' do + let(:data) { fixture_file_upload(import_file, content_type) } + let(:import_file) { File.open('spec/fixtures/files/custom_filters.json') } + let(:import_type) { 'custom_filters' } + let(:content_type) { 'application/json' } + + it 'passes validation' do + expect(subject).to be_valid + end + end + context 'when the file too large' do let(:import_type) { 'following' } let(:import_file) { 'imports.txt' } From 99a1dd20f0c922a37dc73c499f51b70edfefa66c Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 2 Jun 2026 14:11:39 +0200 Subject: [PATCH 02/13] add specs --- .../settings/imports_controller_spec.rb | 57 ++++++++++++++++ spec/fixtures/files/custom_filters.json | 2 +- spec/fixtures/files/empty.json | 0 spec/models/form/import_spec.rb | 65 ++++++++++++++++++- spec/requests/settings/imports_spec.rb | 22 ++++++- 5 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 spec/fixtures/files/empty.json diff --git a/spec/controllers/settings/imports_controller_spec.rb b/spec/controllers/settings/imports_controller_spec.rb index a8fa9fdd35a..d147dda08c7 100644 --- a/spec/controllers/settings/imports_controller_spec.rb +++ b/spec/controllers/settings/imports_controller_spec.rb @@ -169,6 +169,39 @@ RSpec.describe Settings::ImportsController do it_behaves_like 'export failed rows', 'following_accounts_failures.csv', "Account address,Show boosts,Notify on new posts,Languages\nfoo@bar,true,false,\nuser@bar,false,true,\"fr, de\"\n" end + context 'with custom filters' do + subject { get :failures, params: { id: bulk_import.id }, format: :json } + + let(:import_type) { 'custom_filters' } + let(:rows) do + [ + { + 'title' => 'random title', + 'expire_at' => nil, + 'context' => ['public', 'account'], + 'action' => 'warn', + 'keywords_attributes' => [{ + 'keyword' => 'all them keywords', + 'whole_word' => true, + }, { + 'keyword' => 'more keywords even', + 'whole_word' => true, + }], + 'statuses' => ['status'], + }, + ] + end + let(:bulk_import) { Fabricate(:bulk_import, account: user.account, type: import_type, state: :finished) } + + before do + rows.each { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) } + bulk_import.update(total_items: bulk_import.rows.count, processed_items: bulk_import.rows.count, imported_items: 0) + end + + it_behaves_like 'export failed rows', 'custom_filters_failures.json', + '{"custom_filters":[{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expire_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]},{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expire_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]}]}' # rubocop:disable Layout/LineLength + end + context 'with blocks' do let(:import_type) { 'blocking' } @@ -291,5 +324,29 @@ RSpec.describe Settings::ImportsController do it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'merge' it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'overwrite' + + context 'with custom filter' do + subject { post :create, params: { form_import: { type: 'custom_filters', mode: mode, data: data } } } + + describe 'successful import' do + let(:data) { fixture_file_upload('custom_filters.json', 'application/json') } + let(:mode) { 'merge' } + + it 'creates an unconfirmed bulk_import with expected type and redirects', :aggregate_failures do + expect { subject }.to change { user.account.bulk_imports.pluck(:state, :type) }.from([]).to([['unconfirmed', 'custom_filters']]) + expect(response).to redirect_to(settings_import_path(user.account.bulk_imports.first)) + end + end + + describe 'failing import' do + let(:mode) { 'merge' } + let(:data) { fixture_file_upload('empty.json', 'application/json') } + + it 'does not creates an unconfirmed bulk_import', :aggregate_failures do + expect { subject }.to_not(change { user.account.bulk_imports.count }) + expect(response.body).to include('field_with_errors') + end + end + end end end diff --git a/spec/fixtures/files/custom_filters.json b/spec/fixtures/files/custom_filters.json index 25d91cc7478..75596dfed12 100644 --- a/spec/fixtures/files/custom_filters.json +++ b/spec/fixtures/files/custom_filters.json @@ -24,7 +24,7 @@ "context": ["public", "account"], "action": "warn", "keywords_attributes": [ - { "keyword": "will this be a status?", "whole_word": true } + { "keyword": "something something", "whole_word": true } ], "statuses": [] } diff --git a/spec/fixtures/files/empty.json b/spec/fixtures/files/empty.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/spec/models/form/import_spec.rb b/spec/models/form/import_spec.rb index e61ba1f4e9d..676278361f2 100644 --- a/spec/models/form/import_spec.rb +++ b/spec/models/form/import_spec.rb @@ -43,7 +43,7 @@ RSpec.describe Form::Import do end end - context 'when the import type is custom_filters' do + describe 'when the import type is custom_filters' do let(:data) { fixture_file_upload(import_file, content_type) } let(:import_file) { File.open('spec/fixtures/files/custom_filters.json') } let(:import_type) { 'custom_filters' } @@ -269,6 +269,69 @@ RSpec.describe Form::Import do end end + describe 'when importing json' do + let(:import_type) { 'custom_filters' } + let(:data) { fixture_file_upload('custom_filters.json', 'application/json') } + let(:import_mode) { 'merge' } + let(:expected_rows) do + [ + { + 'title' => 'dfjswa', + 'expire_at' => nil, + 'context' => ['home'], + 'action' => 'warn', + 'keywords_attributes' => [{ 'keyword' => 'dvshja', 'whole_word' => true }], + 'statuses' => [], + }, + { + 'title' => 'filter with a phrase as title', + 'expire_at' => nil, + 'context' => %w(home notifications public), + 'action' => 'warn', + 'keywords_attributes' => [ + { 'keyword' => "more words let's see ", 'whole_word' => true }, + ], + 'statuses' => [], + }, + { + 'title' => 'how do I add a status to a filter?', + 'expire_at' => nil, + 'context' => ['public', 'account'], + 'action' => 'warn', + 'keywords_attributes' => [ + { 'keyword' => 'something something', 'whole_word' => true }, + ], + 'statuses' => [], + }, + ] + end + + before do + subject.save + end + + context 'with a BulkImport' do + let(:bulk_import) { account.bulk_imports.first } + + it 'creates a bulk import with correct values' do + expect(bulk_import) + .to be_present + .and have_attributes( + type: eq(subject.type), + original_filename: eq(subject.data.original_filename), + likely_mismatched?: eq(subject.likely_mismatched_json?), + overwrite?: eq(!!subject.overwrite), # rubocop:disable Style/DoubleNegation + processed_items: eq(0), + imported_items: eq(0), + total_items: eq(bulk_import.rows.count), + state_unconfirmed?: be(true) + ) + expect(bulk_import.rows.pluck(:data)) + .to match_array(expected_rows) + end + end + end + it_behaves_like('on successful import', 'following', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } }) it_behaves_like('on successful import', 'following', 'overwrite', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } }) it_behaves_like('on successful import', 'blocking', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } }) diff --git a/spec/requests/settings/imports_spec.rb b/spec/requests/settings/imports_spec.rb index e2051e015f4..b3777a11c38 100644 --- a/spec/requests/settings/imports_spec.rb +++ b/spec/requests/settings/imports_spec.rb @@ -4,7 +4,10 @@ require 'rails_helper' RSpec.describe 'Settings Imports' do describe 'POST /settings/imports' do - before { sign_in Fabricate(:user) } + let(:user) { Fabricate(:user) } + let(:account) { user.account } + + before { sign_in user } it 'gracefully handles invalid nested params' do post settings_imports_path(form_import: 'invalid') @@ -12,5 +15,22 @@ RSpec.describe 'Settings Imports' do expect(response) .to have_http_status(400) end + + describe 'with JSON' do + subject { post settings_imports_path, params: { form_import: { type: 'custom_filters', mode: 'merge', data: data } } } + + let!(:data) { fixture_file_upload('custom_filters.json', 'application/json') } + let(:confirm) { post confirm_settings_import_path(id: user.account.bulk_imports.last.id) } + + it 'redirects to confirm_settings_import_path' do + subject + expect(response).to have_http_status(302) + expect(response).to redirect_to(settings_import_path(id: user.account.bulk_imports.last.id)) + expect(user.account.bulk_imports.last.state).to eq('unconfirmed') + confirm + expect(response).to have_http_status(302) + expect(user.account.bulk_imports.last.state).to eq('scheduled') + end + end end end From b5143e8b3bfc0f36ae8153f1203362213fdaf01d Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 2 Jun 2026 16:26:42 +0200 Subject: [PATCH 03/13] add import controller action --- app/controllers/settings/imports_controller.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/controllers/settings/imports_controller.rb b/app/controllers/settings/imports_controller.rb index be1699315f6..a8dc31b7c74 100644 --- a/app/controllers/settings/imports_controller.rb +++ b/app/controllers/settings/imports_controller.rb @@ -13,6 +13,7 @@ class Settings::ImportsController < Settings::BaseController domain_blocking: 'blocked_domains_failures.csv', bookmarks: 'bookmarks_failures.csv', lists: 'lists_failures.csv', + custom_filters: 'custom_filters_failures.json', }.freeze TYPE_TO_HEADERS_MAP = { @@ -61,6 +62,21 @@ class Settings::ImportsController < Settings::BaseController send_data export_data, filename: filename end + + format.json do + filename = TYPE_TO_FILENAME_MAP[@bulk_import.type.to_sym] + + data_collection = { custom_filters: [] } + @bulk_import.rows.find_each do |row| + case @bulk_import.type.to_sym + when :custom_filters + data_collection[:custom_filters] << row.data + end + end + export_data = JSON.generate(data_collection) + + send_data export_data, filename: filename + end end end From 10bf6583ece4bb581045bed8ec925e63b0a0d76c Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 2 Jun 2026 16:28:05 +0200 Subject: [PATCH 04/13] add json handling to form import controller --- app/models/bulk_import.rb | 1 + app/models/form/import.rb | 51 ++++++++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/app/models/bulk_import.rb b/app/models/bulk_import.rb index 79f47d11ddc..617cab9ea3b 100644 --- a/app/models/bulk_import.rb +++ b/app/models/bulk_import.rb @@ -34,6 +34,7 @@ class BulkImport < ApplicationRecord domain_blocking: 3, bookmarks: 4, lists: 5, + custom_filters: 6, } enum :state, { diff --git a/app/models/form/import.rb b/app/models/form/import.rb index 986d33c2b38..0b818669261 100644 --- a/app/models/form/import.rb +++ b/app/models/form/import.rb @@ -55,23 +55,34 @@ class Form::Import :bookmarks elsif file_name_matches?('lists') :lists - elsif file_name_matches?('custom_filters') - :custom_filters end end + def guessed_type_json + :custom_filters if file_name_matches?('custom_filters') + end + # Whether the uploaded CSV file seems to correspond to a different import type than the one selected def likely_mismatched? guessed_type.present? && guessed_type != type.to_sym end + def likely_mismatched_json? + guessed_type_json.present? && guessed_type_json != type.to_sym + end + def save return false unless valid? ApplicationRecord.transaction do now = Time.now.utc - @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?) - nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length + if content_type_is_json? + @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched_json?) + nb_items = BulkImportRow.insert_all(json_data.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length + else + @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?) + nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length + end @bulk_import.update(total_items: nb_items) end end @@ -152,29 +163,19 @@ class Form::Import end end - def file_type_is_json? - data.content_type == 'application/json' - end - - def data_from_json - @data_from_json ||= JSON.parse(data.read)['custom_filters'].map(&:deep_symbolize_keys) - end - - def allowed_json_key? - type.to_sym.in?(%i(custom_filters)) - end - def validate_data return if data.nil? return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT - if file_type_is_json? + if content_type_is_json? validate_json_data else validate_csv_data end rescue CSV::MalformedCSVError => e errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message)) + rescue JSON::ParserError => e + errors.add(:data, I18n.t('imports.errors.invalid_json_file', error: e.message)) rescue EmptyFileError errors.add(:data, I18n.t('imports.errors.empty')) end @@ -193,9 +194,19 @@ class Form::Import end def validate_json_data - return unless allowed_json_key? - - errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if data_from_json.count > ROWS_PROCESSING_LIMIT + errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if json_data.count > ROWS_PROCESSING_LIMIT errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless allowed_json_key? end + + def content_type_is_json? + data.content_type == 'application/json' + end + + def json_data + @json_data ||= JSON.parse(data.read)['custom_filters'].map(&:deep_symbolize_keys) + end + + def allowed_json_key? + type.to_sym.in?(%i(custom_filters)) + end end From 0aaae39db48c5ad2f4b8465a3497c3768fac2702 Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 2 Jun 2026 16:29:12 +0200 Subject: [PATCH 05/13] add translation key --- config/locales/en-GB.yml | 1 + config/locales/en.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/config/locales/en-GB.yml b/config/locales/en-GB.yml index 9d535e3af02..bc08197b95f 100644 --- a/config/locales/en-GB.yml +++ b/config/locales/en-GB.yml @@ -1629,6 +1629,7 @@ en-GB: types: blocking: Blocking list bookmarks: Bookmarks + custom_filters: Custom filters domain_blocking: Domain blocking list following: Following list lists: Lists diff --git a/config/locales/en.yml b/config/locales/en.yml index 95631bd86b9..04fa5b1d687 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1665,6 +1665,7 @@ en: empty: Empty CSV file incompatible_type: Incompatible with the selected import type invalid_csv_file: 'Invalid CSV file. Error: %{error}' + invalid_json_file: 'Invalid JSON file. Error: %{error}' over_rows_processing_limit: contains more than %{count} rows too_large: File is too large failures: Failures From 9cf7455a922dd4a25c3385d6e42245bc33dabd60 Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 2 Jun 2026 18:02:53 +0200 Subject: [PATCH 06/13] add service spec --- spec/services/bulk_import_service_spec.rb | 50 +++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/spec/services/bulk_import_service_spec.rb b/spec/services/bulk_import_service_spec.rb index 9adbd522dca..14fcc2fc83d 100644 --- a/spec/services/bulk_import_service_spec.rb +++ b/spec/services/bulk_import_service_spec.rb @@ -292,6 +292,56 @@ RSpec.describe BulkImportService do end end + context 'when importing custom_filters' do + let(:import_type) { 'custom_filters' } + let(:overwrite) { false } + + let!(:rows) do + [{ + 'title' => 'foo', + 'expire_at' => nil, + 'context' => ['home', 'notifications'], + 'action' => 'warn', + 'keywords_attributes' => [{ + 'keyword' => 'discourse', + 'whole_word' => true, + }, { + 'keyword' => 'something', + 'whole_word' => false, + }], + 'statuses' => ['Lorem ipsum dolor sit amet'], + }, { + 'title' => 'bar', + 'expire_at' => nil, + 'context' => ['notifications'], + 'action' => 'warn', + 'keywords_attributes' => [{ + 'keyword' => 'discourse', + 'whole_word' => true, + }, { + 'keyword' => 'something', + 'whole_word' => false, + }], + 'statuses' => ['something something'], + }].map { |data| import.rows.create!(data: data) } + end + + it 'enqueues workers for the expected rows and updates bookmarks after worker run' do + subject.call(import) + + expect(row_worker_job_args) + .to match_array(rows.map(&:id)) + + stub_fetch_remote_and_drain_workers + expect(account.custom_filters.map(&:title)) + .to eq(['foo', 'bar']) + expect(account.custom_filters.first.statuses.count) + .to eq(1) + expect(account.custom_filters.first.keywords.count) + .to eq(2) + end + end + context 'when importing bookmarks with overwrite' do let(:import_type) { 'bookmarks' } let(:overwrite) { true } From a3850c421093d603fd9c57be72409d5d571c8c0f Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 2 Jun 2026 18:05:53 +0200 Subject: [PATCH 07/13] add to bulk import service --- app/services/bulk_import_service.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/app/services/bulk_import_service.rb b/app/services/bulk_import_service.rb index 8e0864a07f4..2736f7d0bab 100644 --- a/app/services/bulk_import_service.rb +++ b/app/services/bulk_import_service.rb @@ -18,6 +18,8 @@ class BulkImportService < BaseService import_bookmarks! when :lists import_lists! + when :custom_filters + import_custom_filters! end @import.update!(state: :finished, finished_at: Time.now.utc) if @import.processing_complete? @@ -182,4 +184,27 @@ class BulkImportService < BaseService [row.id] end end + + def import_custom_filters! + rows = @import.rows.to_a + + included_custom_filters = rows.map(&:data).uniq + + @account.custom_filters.where.not(title: included_custom_filters).destroy_all if @import.overwrite? + + included_custom_filters.each do |filter| + filter_object = @account.custom_filters.find_or_create_by!(title: filter['title'], context: filter['context']) + filter['keywords_attributes'].each do |keyword| + filter_object.keywords.find_or_create_by!(keyword: keyword['keyword'], whole_word: keyword['whole_word']) + end + filter['statuses'].each do |status| + status = @account.statuses.find_or_create_by!(text: status) + filter_object.statuses.find_or_create_by!(status_id: status.id) + end + end + + Import::RowWorker.push_bulk(rows) do |row| + [row.id] + end + end end From 93db81d0c6efdf195ab698f0d1f2a1af32db976e Mon Sep 17 00:00:00 2001 From: Pia B Date: Wed, 3 Jun 2026 18:06:08 +0200 Subject: [PATCH 08/13] refactor import service --- app/services/bulk_import_service.rb | 59 ++++++--- spec/services/bulk_import_service_spec.rb | 150 ++++++++++++++-------- 2 files changed, 143 insertions(+), 66 deletions(-) diff --git a/app/services/bulk_import_service.rb b/app/services/bulk_import_service.rb index 2736f7d0bab..9c30dd705e1 100644 --- a/app/services/bulk_import_service.rb +++ b/app/services/bulk_import_service.rb @@ -186,25 +186,52 @@ class BulkImportService < BaseService end def import_custom_filters! - rows = @import.rows.to_a + rows_by_title = @import.rows.index_by { |row| row.data['title'] }.to_h + included_custom_filters = rows_by_title.values.pluck('data') + custom_filters_hash = included_custom_filters.map { |fil| fil.except('statuses', 'keywords_attributes') } + custom_filters_hash.map { |fil| fil['account_id'] = @account.id } - included_custom_filters = rows.map(&:data).uniq + @account.custom_filters.where.not(title: rows_by_title.keys).destroy_all if @import.overwrite? + CustomFilter.insert_all(custom_filters_hash) + import_custom_filter_keywords_and_statuses(rows_by_title) - @account.custom_filters.where.not(title: included_custom_filters).destroy_all if @import.overwrite? - - included_custom_filters.each do |filter| - filter_object = @account.custom_filters.find_or_create_by!(title: filter['title'], context: filter['context']) - filter['keywords_attributes'].each do |keyword| - filter_object.keywords.find_or_create_by!(keyword: keyword['keyword'], whole_word: keyword['whole_word']) - end - filter['statuses'].each do |status| - status = @account.statuses.find_or_create_by!(text: status) - filter_object.statuses.find_or_create_by!(status_id: status.id) - end - end - - Import::RowWorker.push_bulk(rows) do |row| + Import::RowWorker.push_bulk(rows_by_title.values) do |row| [row.id] end end + + def import_custom_filter_keywords_and_statuses(rows_by_title) + statuses_to_insert = [] + keywords_to_insert = [] + @account.custom_filters.find_each do |filter| + row = rows_by_title[filter.title] + next if row.nil? + + keywords = row.data['keywords_attributes'] + statuses = row.data['statuses'] + + keywords.each do |keyword| + next if keyword.blank? + + keywords_to_insert << { + keyword: keyword['keyword'], + whole_word: keyword['whole_word'], + custom_filter_id: filter.id, + } + end + + status_ids = Status.where(uri: statuses).ids + next if status_ids.blank? + + status_ids.each do |id| + statuses_to_insert << { + custom_filter_id: filter.id, + status_id: id, + } + end + end + + CustomFilterKeyword.insert_all(keywords_to_insert) if keywords_to_insert.any? + CustomFilterStatus.insert_all(statuses_to_insert) if statuses_to_insert.any? + end end diff --git a/spec/services/bulk_import_service_spec.rb b/spec/services/bulk_import_service_spec.rb index 14fcc2fc83d..4822e40d572 100644 --- a/spec/services/bulk_import_service_spec.rb +++ b/spec/services/bulk_import_service_spec.rb @@ -292,56 +292,6 @@ RSpec.describe BulkImportService do end end - context 'when importing custom_filters' do - let(:import_type) { 'custom_filters' } - let(:overwrite) { false } - - let!(:rows) do - [{ - 'title' => 'foo', - 'expire_at' => nil, - 'context' => ['home', 'notifications'], - 'action' => 'warn', - 'keywords_attributes' => [{ - 'keyword' => 'discourse', - 'whole_word' => true, - }, { - 'keyword' => 'something', - 'whole_word' => false, - }], - 'statuses' => ['Lorem ipsum dolor sit amet'], - }, { - 'title' => 'bar', - 'expire_at' => nil, - 'context' => ['notifications'], - 'action' => 'warn', - 'keywords_attributes' => [{ - 'keyword' => 'discourse', - 'whole_word' => true, - }, { - 'keyword' => 'something', - 'whole_word' => false, - }], - 'statuses' => ['something something'], - }].map { |data| import.rows.create!(data: data) } - end - - it 'enqueues workers for the expected rows and updates bookmarks after worker run' do - subject.call(import) - - expect(row_worker_job_args) - .to match_array(rows.map(&:id)) - - stub_fetch_remote_and_drain_workers - expect(account.custom_filters.map(&:title)) - .to eq(['foo', 'bar']) - expect(account.custom_filters.first.statuses.count) - .to eq(1) - expect(account.custom_filters.first.keywords.count) - .to eq(2) - end - end - context 'when importing bookmarks with overwrite' do let(:import_type) { 'bookmarks' } let(:overwrite) { true } @@ -379,6 +329,106 @@ RSpec.describe BulkImportService do end end + context 'when importing custom_filters' do + let(:import_type) { 'custom_filters' } + let!(:rows) do + [{ + 'title' => 'baz', + 'expires_at' => nil, + 'context' => ['home', 'notifications'], + 'action' => 'warn', + 'keywords_attributes' => [{ + 'keyword' => 'discourse', + 'whole_word' => true, + }, { + 'keyword' => 'something', + 'whole_word' => false, + }], + 'statuses' => ['http://localhost:3000/ap/users/116646814515254858/statuses/116681350935935708'], + }, { + 'title' => 'buzz', + 'expires_at' => nil, + 'context' => ['notifications'], + 'action' => 'warn', + 'keywords_attributes' => [{ + 'keyword' => 'discourse', + 'whole_word' => true, + }, { + 'keyword' => 'something', + 'whole_word' => false, + }], + 'statuses' => [ActivityPub::TagManager.instance.uri_for(status)], + }].map { |data| import.rows.create!(data: data) } + end + let(:overwrite) { false } + let(:status) { Fabricate(:status, account: account, text: 'something something') } + let(:filter) { Fabricate(:custom_filter, account: account, title: 'a mazing title') } + let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter, status: status) } + + before do + status_filter + end + + it 'enqueues workers for the expected rows and updates filters, keywords and statuses after worker run' do + subject.call(import) + expect(row_worker_job_args).to match_array(rows.map(&:id)) + stub_fetch_remote_and_drain_workers + expect(account.custom_filters.count).to eq(3) + expect(account.custom_filters.order(:phrase).last.statuses.count).to eq(1) + expect(account.custom_filters.last.keywords.count).to eq(2) + end + end + + context 'when importing custom_filters with overwrite' do + let(:import_type) { 'custom_filters' } + let!(:rows) do + [{ + 'title' => 'baz', + 'expires_at' => nil, + 'context' => ['home', 'notifications'], + 'action' => 'warn', + 'keywords_attributes' => [{ + 'keyword' => 'discourse', + 'whole_word' => true, + }, { + 'keyword' => 'something', + 'whole_word' => false, + }], + 'statuses' => ['http://localhost:3000/ap/users/116646814515254858/statuses/116681350935935708'], + }, { + 'title' => 'buzz', + 'expires_at' => nil, + 'context' => ['notifications'], + 'action' => 'warn', + 'keywords_attributes' => [{ + 'keyword' => 'discourse', + 'whole_word' => true, + }, { + 'keyword' => 'something', + 'whole_word' => false, + }], + 'statuses' => [ActivityPub::TagManager.instance.uri_for(status)], + }].map { |data| import.rows.create!(data: data) } + end + let(:overwrite) { true } + let(:status) { Fabricate(:status, text: 'something something') } + let(:filter) { Fabricate(:custom_filter, account: account, title: 'a mazing title') } + let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter, status: status) } + + before do + status_filter + end + + it 'enqueues workers for the expected rows and updates filters, keywords and statuses after worker run' do + subject.call(import) + expect(row_worker_job_args).to match_array(rows.map(&:id)) + stub_fetch_remote_and_drain_workers + expect(account.custom_filters.count).to eq(2) + expect(account.custom_filters.order(:phrase).last.statuses.count).to eq(1) + expect(account.custom_filters.last.keywords.count).to eq(2) + end + end + def row_worker_job_args Import::RowWorker .jobs From 469997b36cec7e51d0aff4b578c1ca80cb70415a Mon Sep 17 00:00:00 2001 From: Pia B Date: Fri, 5 Jun 2026 14:21:15 +0200 Subject: [PATCH 09/13] fix typo --- spec/controllers/settings/imports_controller_spec.rb | 4 ++-- spec/fixtures/files/custom_filters.json | 6 +++--- spec/models/form/import_spec.rb | 6 +++--- spec/requests/settings/imports_spec.rb | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/controllers/settings/imports_controller_spec.rb b/spec/controllers/settings/imports_controller_spec.rb index d147dda08c7..cc89f4517d8 100644 --- a/spec/controllers/settings/imports_controller_spec.rb +++ b/spec/controllers/settings/imports_controller_spec.rb @@ -177,7 +177,7 @@ RSpec.describe Settings::ImportsController do [ { 'title' => 'random title', - 'expire_at' => nil, + 'expires_at' => nil, 'context' => ['public', 'account'], 'action' => 'warn', 'keywords_attributes' => [{ @@ -199,7 +199,7 @@ RSpec.describe Settings::ImportsController do end it_behaves_like 'export failed rows', 'custom_filters_failures.json', - '{"custom_filters":[{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expire_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]},{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expire_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]}]}' # rubocop:disable Layout/LineLength + '{"custom_filters":[{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expires_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]},{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expires_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]}]}' # rubocop:disable Layout/LineLength end context 'with blocks' do diff --git a/spec/fixtures/files/custom_filters.json b/spec/fixtures/files/custom_filters.json index 75596dfed12..2107982f0eb 100644 --- a/spec/fixtures/files/custom_filters.json +++ b/spec/fixtures/files/custom_filters.json @@ -2,7 +2,7 @@ "custom_filters": [ { "title": "dfjswa", - "expire_at": null, + "expires_at": null, "context": ["home"], "action": "warn", "keywords_attributes": [{ "keyword": "dvshja", "whole_word": true }], @@ -10,7 +10,7 @@ }, { "title": "filter with a phrase as title", - "expire_at": null, + "expires_at": null, "context": ["home", "notifications", "public"], "action": "warn", "keywords_attributes": [ @@ -20,7 +20,7 @@ }, { "title": "how do I add a status to a filter?", - "expire_at": null, + "expires_at": null, "context": ["public", "account"], "action": "warn", "keywords_attributes": [ diff --git a/spec/models/form/import_spec.rb b/spec/models/form/import_spec.rb index 676278361f2..7d7b4f37628 100644 --- a/spec/models/form/import_spec.rb +++ b/spec/models/form/import_spec.rb @@ -277,7 +277,7 @@ RSpec.describe Form::Import do [ { 'title' => 'dfjswa', - 'expire_at' => nil, + 'expires_at' => nil, 'context' => ['home'], 'action' => 'warn', 'keywords_attributes' => [{ 'keyword' => 'dvshja', 'whole_word' => true }], @@ -285,7 +285,7 @@ RSpec.describe Form::Import do }, { 'title' => 'filter with a phrase as title', - 'expire_at' => nil, + 'expires_at' => nil, 'context' => %w(home notifications public), 'action' => 'warn', 'keywords_attributes' => [ @@ -295,7 +295,7 @@ RSpec.describe Form::Import do }, { 'title' => 'how do I add a status to a filter?', - 'expire_at' => nil, + 'expires_at' => nil, 'context' => ['public', 'account'], 'action' => 'warn', 'keywords_attributes' => [ diff --git a/spec/requests/settings/imports_spec.rb b/spec/requests/settings/imports_spec.rb index b3777a11c38..6ea702f06f5 100644 --- a/spec/requests/settings/imports_spec.rb +++ b/spec/requests/settings/imports_spec.rb @@ -25,7 +25,7 @@ RSpec.describe 'Settings Imports' do it 'redirects to confirm_settings_import_path' do subject expect(response).to have_http_status(302) - expect(response).to redirect_to(settings_import_path(id: user.account.bulk_imports.last.id)) + .and redirect_to(settings_import_path(id: user.account.bulk_imports.last.id)) expect(user.account.bulk_imports.last.state).to eq('unconfirmed') confirm expect(response).to have_http_status(302) From 0d3625ddca1e6e46af7a0f99a886c05e6aa9fccf Mon Sep 17 00:00:00 2001 From: Pia B Date: Fri, 5 Jun 2026 17:38:07 +0200 Subject: [PATCH 10/13] refactor bulk import --- app/services/bulk_import_row_service.rb | 9 +++++ app/services/bulk_import_service.rb | 47 +++---------------------- 2 files changed, 13 insertions(+), 43 deletions(-) diff --git a/app/services/bulk_import_row_service.rb b/app/services/bulk_import_row_service.rb index ac5080f0ba4..05a3046ca77 100644 --- a/app/services/bulk_import_row_service.rb +++ b/app/services/bulk_import_row_service.rb @@ -42,6 +42,15 @@ class BulkImportRowService FollowService.new.call(@account, @target_account) unless @account.id == @target_account.id list.accounts << @target_account + when :custom_filters + filter = @account.custom_filters.find_or_initialize_by(title: @data['title'], context: @data['context']) + filter.keywords = @data['keywords_attributes'].map { |keyword| CustomFilterKeyword.new(keyword: keyword['keyword'], whole_word: keyword['whole_word']) } + filter.action = @data['action'].to_sym + filter.expires_at = @data['expires_at'] + status_ids = Status.where(uri: @data['statuses']).ids + filter.statuses = status_ids.map { |status| CustomFilterStatus.new(status_id: status) } if status_ids.any? + + filter.save! end true diff --git a/app/services/bulk_import_service.rb b/app/services/bulk_import_service.rb index 9c30dd705e1..2cabeda2363 100644 --- a/app/services/bulk_import_service.rb +++ b/app/services/bulk_import_service.rb @@ -186,52 +186,13 @@ class BulkImportService < BaseService end def import_custom_filters! - rows_by_title = @import.rows.index_by { |row| row.data['title'] }.to_h - included_custom_filters = rows_by_title.values.pluck('data') - custom_filters_hash = included_custom_filters.map { |fil| fil.except('statuses', 'keywords_attributes') } - custom_filters_hash.map { |fil| fil['account_id'] = @account.id } + rows = @import.rows.to_a + titles = rows.map { |row| row.data['title'] } - @account.custom_filters.where.not(title: rows_by_title.keys).destroy_all if @import.overwrite? - CustomFilter.insert_all(custom_filters_hash) - import_custom_filter_keywords_and_statuses(rows_by_title) + @account.custom_filters.where.not(title: titles).destroy_all if @import.overwrite? - Import::RowWorker.push_bulk(rows_by_title.values) do |row| + Import::RowWorker.push_bulk(rows) do |row| [row.id] end end - - def import_custom_filter_keywords_and_statuses(rows_by_title) - statuses_to_insert = [] - keywords_to_insert = [] - @account.custom_filters.find_each do |filter| - row = rows_by_title[filter.title] - next if row.nil? - - keywords = row.data['keywords_attributes'] - statuses = row.data['statuses'] - - keywords.each do |keyword| - next if keyword.blank? - - keywords_to_insert << { - keyword: keyword['keyword'], - whole_word: keyword['whole_word'], - custom_filter_id: filter.id, - } - end - - status_ids = Status.where(uri: statuses).ids - next if status_ids.blank? - - status_ids.each do |id| - statuses_to_insert << { - custom_filter_id: filter.id, - status_id: id, - } - end - end - - CustomFilterKeyword.insert_all(keywords_to_insert) if keywords_to_insert.any? - CustomFilterStatus.insert_all(statuses_to_insert) if statuses_to_insert.any? - end end From e3dc59627969b31c56052d28082cbffd4df9bf1f Mon Sep 17 00:00:00 2001 From: Pia B Date: Mon, 8 Jun 2026 11:42:54 +0200 Subject: [PATCH 11/13] set failure route for respective format --- app/views/settings/imports/index.html.haml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/views/settings/imports/index.html.haml b/app/views/settings/imports/index.html.haml index 027724391bc..478a26b9c97 100644 --- a/app/views/settings/imports/index.html.haml +++ b/app/views/settings/imports/index.html.haml @@ -61,5 +61,9 @@ %td - if import.failure_count.positive? - = link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do - = import.failure_count + - if import.type == 'custom_filters' + = link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :json) do + = import.failure_count + - else + = link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do + = import.failure_count From 73e4650540c9b28017125d5323b278746a32fa52 Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 9 Jun 2026 10:19:54 +0200 Subject: [PATCH 12/13] handle delete all --- app/services/bulk_import_row_service.rb | 3 +-- app/services/bulk_import_service.rb | 4 ++-- spec/services/bulk_import_service_spec.rb | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/services/bulk_import_row_service.rb b/app/services/bulk_import_row_service.rb index 05a3046ca77..e6ccc7c2fb9 100644 --- a/app/services/bulk_import_row_service.rb +++ b/app/services/bulk_import_row_service.rb @@ -43,13 +43,12 @@ class BulkImportRowService list.accounts << @target_account when :custom_filters - filter = @account.custom_filters.find_or_initialize_by(title: @data['title'], context: @data['context']) + filter = @account.custom_filters.create!(title: @data['title'], context: @data['context']) filter.keywords = @data['keywords_attributes'].map { |keyword| CustomFilterKeyword.new(keyword: keyword['keyword'], whole_word: keyword['whole_word']) } filter.action = @data['action'].to_sym filter.expires_at = @data['expires_at'] status_ids = Status.where(uri: @data['statuses']).ids filter.statuses = status_ids.map { |status| CustomFilterStatus.new(status_id: status) } if status_ids.any? - filter.save! end diff --git a/app/services/bulk_import_service.rb b/app/services/bulk_import_service.rb index 2cabeda2363..6190f983c83 100644 --- a/app/services/bulk_import_service.rb +++ b/app/services/bulk_import_service.rb @@ -187,9 +187,9 @@ class BulkImportService < BaseService def import_custom_filters! rows = @import.rows.to_a - titles = rows.map { |row| row.data['title'] } + rows.map { |row| row.data['title'] } - @account.custom_filters.where.not(title: titles).destroy_all if @import.overwrite? + @account.custom_filters.destroy_all if @import.overwrite? Import::RowWorker.push_bulk(rows) do |row| [row.id] diff --git a/spec/services/bulk_import_service_spec.rb b/spec/services/bulk_import_service_spec.rb index 4822e40d572..bd90952efe1 100644 --- a/spec/services/bulk_import_service_spec.rb +++ b/spec/services/bulk_import_service_spec.rb @@ -425,7 +425,7 @@ RSpec.describe BulkImportService do stub_fetch_remote_and_drain_workers expect(account.custom_filters.count).to eq(2) expect(account.custom_filters.order(:phrase).last.statuses.count).to eq(1) - expect(account.custom_filters.last.keywords.count).to eq(2) + expect(account.custom_filters.order(:phrase).last.keywords.count).to eq(2) end end From f5f31ea9e6b908f3cc08a98db1dfa6211188ea1a Mon Sep 17 00:00:00 2001 From: Pia B Date: Tue, 9 Jun 2026 12:38:44 +0200 Subject: [PATCH 13/13] check for key in json instead of filename --- app/models/form/import.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/models/form/import.rb b/app/models/form/import.rb index 0b818669261..a7df46f47eb 100644 --- a/app/models/form/import.rb +++ b/app/models/form/import.rb @@ -59,7 +59,7 @@ class Form::Import end def guessed_type_json - :custom_filters if file_name_matches?('custom_filters') + :custom_filters if parse_json.keys.any?('custom_filters') end # Whether the uploaded CSV file seems to correspond to a different import type than the one selected @@ -195,7 +195,7 @@ class Form::Import def validate_json_data errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if json_data.count > ROWS_PROCESSING_LIMIT - errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless allowed_json_key? + errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless allowed_type_for_json? end def content_type_is_json? @@ -203,10 +203,14 @@ class Form::Import end def json_data - @json_data ||= JSON.parse(data.read)['custom_filters'].map(&:deep_symbolize_keys) + parse_json['custom_filters'].map(&:deep_symbolize_keys) end - def allowed_json_key? + def parse_json + @parse_json ||= JSON.parse(data.read) + end + + def allowed_type_for_json? type.to_sym.in?(%i(custom_filters)) end end