Leahnaya/mastodon
1
0
Fork 0
mirror of https://github.com/mastodon/mastodon.git synced 2026-09-12 07:26:11 -05:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge f5f31ea9e6 into fef5a501cc

Browse Source
This commit is contained in:
Pia B.
2026-06-09 12:39:00 +02:00
committed by GitHub
parent fef5a501cc f5f31ea9e6
commit a0abc817af
14 changed files with 382 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/controllers/settings/imports_controller.rb
View File

@@ -13,6 +13,7 @@ class Settings::ImportsController < Settings::BaseController
domain_blocking: 'blocked_domains_failures.csv',
bookmarks: 'bookmarks_failures.csv',
lists: 'lists_failures.csv',
custom_filters: 'custom_filters_failures.json',
}.freeze
TYPE_TO_HEADERS_MAP = {
@@ -61,6 +62,21 @@ class Settings::ImportsController < Settings::BaseController
send_data export_data, filename: filename
end
format.json do
filename = TYPE_TO_FILENAME_MAP[@bulk_import.type.to_sym]
data_collection = { custom_filters: [] }
@bulk_import.rows.find_each do |row|
case @bulk_import.type.to_sym
when :custom_filters
data_collection[:custom_filters] << row.data
end
end
export_data = JSON.generate(data_collection)
send_data export_data, filename: filename
end
end
end

1
app/models/bulk_import.rb
View File

@@ -34,6 +34,7 @@ class BulkImport < ApplicationRecord
domain_blocking: 3,
bookmarks: 4,
lists: 5,
custom_filters: 6,
}
enum :state, {

57
app/models/form/import.rb
View File

@@ -58,18 +58,31 @@ class Form::Import
end
end
def guessed_type_json
:custom_filters if parse_json.keys.any?('custom_filters')
end
# Whether the uploaded CSV file seems to correspond to a different import type than the one selected
def likely_mismatched?
guessed_type.present? && guessed_type != type.to_sym
end
def likely_mismatched_json?
guessed_type_json.present? && guessed_type_json != type.to_sym
end
def save
return false unless valid?
ApplicationRecord.transaction do
now = Time.now.utc
@bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
if content_type_is_json?
@bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched_json?)
nb_items = BulkImportRow.insert_all(json_data.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
else
@bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
end
@bulk_import.update(total_items: nb_items)
end
end
@@ -153,6 +166,21 @@ class Form::Import
def validate_data
return if data.nil?
return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT
if content_type_is_json?
validate_json_data
else
validate_csv_data
end
rescue CSV::MalformedCSVError => e
errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
rescue JSON::ParserError => e
errors.add(:data, I18n.t('imports.errors.invalid_json_file', error: e.message))
rescue EmptyFileError
errors.add(:data, I18n.t('imports.errors.empty'))
end
def validate_csv_data
return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless default_csv_headers.all? { |header| csv_data.headers.include?(header) }
errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT
@@ -163,9 +191,26 @@ class Form::Import
limit -= current_account.following_count unless overwrite
errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if csv_row_count > limit
end
rescue CSV::MalformedCSVError => e
errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
rescue EmptyFileError
errors.add(:data, I18n.t('imports.errors.empty'))
end
def validate_json_data
errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if json_data.count > ROWS_PROCESSING_LIMIT
errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless allowed_type_for_json?
end
def content_type_is_json?
data.content_type == 'application/json'
end
def json_data
parse_json['custom_filters'].map(&:deep_symbolize_keys)
end
def parse_json
@parse_json ||= JSON.parse(data.read)
end
def allowed_type_for_json?
type.to_sym.in?(%i(custom_filters))
end
end

8
app/services/bulk_import_row_service.rb
View File

@@ -42,6 +42,14 @@ class BulkImportRowService
FollowService.new.call(@account, @target_account) unless @account.id == @target_account.id
list.accounts << @target_account
when :custom_filters
filter = @account.custom_filters.create!(title: @data['title'], context: @data['context'])
filter.keywords = @data['keywords_attributes'].map { |keyword| CustomFilterKeyword.new(keyword: keyword['keyword'], whole_word: keyword['whole_word']) }
filter.action = @data['action'].to_sym
filter.expires_at = @data['expires_at']
status_ids = Status.where(uri: @data['statuses']).ids
filter.statuses = status_ids.map { |status| CustomFilterStatus.new(status_id: status) } if status_ids.any?
filter.save!
end
true

13
app/services/bulk_import_service.rb
View File

@@ -18,6 +18,8 @@ class BulkImportService < BaseService
import_bookmarks!
when :lists
import_lists!
when :custom_filters
import_custom_filters!
end
@import.update!(state: :finished, finished_at: Time.now.utc) if @import.processing_complete?
@@ -182,4 +184,15 @@ class BulkImportService < BaseService
[row.id]
end
end
def import_custom_filters!
rows = @import.rows.to_a
rows.map { |row| row.data['title'] }
@account.custom_filters.destroy_all if @import.overwrite?
Import::RowWorker.push_bulk(rows) do |row|
[row.id]
end
end
end

10
app/views/settings/imports/index.html.haml
View File

@@ -5,7 +5,7 @@
.field-group
= f.input :type,
as: :grouped_select,
collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking) },
collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking custom_filters) },
group_label_method: ->(group) { I18n.t("imports.type_groups.#{group.first}") },
group_method: :last,
hint: t('imports.preface'),
@@ -61,5 +61,9 @@
%td
- if import.failure_count.positive?
= link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do
= import.failure_count
- if import.type == 'custom_filters'
= link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :json) do
= import.failure_count
- else
= link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do
= import.failure_count

1
config/locales/en-GB.yml
View File

@@ -1629,6 +1629,7 @@ en-GB:
types:
blocking: Blocking list
bookmarks: Bookmarks
custom_filters: Custom filters
domain_blocking: Domain blocking list
following: Following list
lists: Lists

1
config/locales/en.yml
View File

@@ -1665,6 +1665,7 @@ en:
empty: Empty CSV file
incompatible_type: Incompatible with the selected import type
invalid_csv_file: 'Invalid CSV file. Error: %{error}'
invalid_json_file: 'Invalid JSON file. Error: %{error}'
over_rows_processing_limit: contains more than %{count} rows
too_large: File is too large
failures: Failures

57
spec/controllers/settings/imports_controller_spec.rb
View File

@@ -169,6 +169,39 @@ RSpec.describe Settings::ImportsController do
it_behaves_like 'export failed rows', 'following_accounts_failures.csv', "Account address,Show boosts,Notify on new posts,Languages\nfoo@bar,true,false,\nuser@bar,false,true,\"fr, de\"\n"
end
context 'with custom filters' do
subject { get :failures, params: { id: bulk_import.id }, format: :json }
let(:import_type) { 'custom_filters' }
let(:rows) do
[
{
'title' => 'random title',
'expires_at' => nil,
'context' => ['public', 'account'],
'action' => 'warn',
'keywords_attributes' => [{
'keyword' => 'all them keywords',
'whole_word' => true,
}, {
'keyword' => 'more keywords even',
'whole_word' => true,
}],
'statuses' => ['status'],
},
]
end
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, type: import_type, state: :finished) }
before do
rows.each { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) }
bulk_import.update(total_items: bulk_import.rows.count, processed_items: bulk_import.rows.count, imported_items: 0)
end
it_behaves_like 'export failed rows', 'custom_filters_failures.json',
'{"custom_filters":[{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expires_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]},{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expires_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]}]}' # rubocop:disable Layout/LineLength
end
context 'with blocks' do
let(:import_type) { 'blocking' }
@@ -291,5 +324,29 @@ RSpec.describe Settings::ImportsController do
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'merge'
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'overwrite'
context 'with custom filter' do
subject { post :create, params: { form_import: { type: 'custom_filters', mode: mode, data: data } } }
describe 'successful import' do
let(:data) { fixture_file_upload('custom_filters.json', 'application/json') }
let(:mode) { 'merge' }
it 'creates an unconfirmed bulk_import with expected type and redirects', :aggregate_failures do
expect { subject }.to change { user.account.bulk_imports.pluck(:state, :type) }.from([]).to([['unconfirmed', 'custom_filters']])
expect(response).to redirect_to(settings_import_path(user.account.bulk_imports.first))
end
end
describe 'failing import' do
let(:mode) { 'merge' }
let(:data) { fixture_file_upload('empty.json', 'application/json') }
it 'does not creates an unconfirmed bulk_import', :aggregate_failures do
expect { subject }.to_not(change { user.account.bulk_imports.count })
expect(response.body).to include('field_with_errors')
end
end
end
end
end

32
spec/fixtures/files/custom_filters.json vendored Normal file
View File

@@ -0,0 +1,32 @@
{
"custom_filters": [
{
"title": "dfjswa",
"expires_at": null,
"context": ["home"],
"action": "warn",
"keywords_attributes": [{ "keyword": "dvshja", "whole_word": true }],
"statuses": []
},
{
"title": "filter with a phrase as title",
"expires_at": null,
"context": ["home", "notifications", "public"],
"action": "warn",
"keywords_attributes": [
{ "keyword": "more words let's see ", "whole_word": true }
],
"statuses": []
},
{
"title": "how do I add a status to a filter?",
"expires_at": null,
"context": ["public", "account"],
"action": "warn",
"keywords_attributes": [
{ "keyword": "something something", "whole_word": true }
],
"statuses": []
}
]
}

0
spec/fixtures/files/empty.json vendored Normal file
View File

74
spec/models/form/import_spec.rb
View File

@@ -43,6 +43,17 @@ RSpec.describe Form::Import do
end
end
describe 'when the import type is custom_filters' do
let(:data) { fixture_file_upload(import_file, content_type) }
let(:import_file) { File.open('spec/fixtures/files/custom_filters.json') }
let(:import_type) { 'custom_filters' }
let(:content_type) { 'application/json' }
it 'passes validation' do
expect(subject).to be_valid
end
end
context 'when the file too large' do
let(:import_type) { 'following' }
let(:import_file) { 'imports.txt' }
@@ -258,6 +269,69 @@ RSpec.describe Form::Import do
end
end
describe 'when importing json' do
let(:import_type) { 'custom_filters' }
let(:data) { fixture_file_upload('custom_filters.json', 'application/json') }
let(:import_mode) { 'merge' }
let(:expected_rows) do
[
{
'title' => 'dfjswa',
'expires_at' => nil,
'context' => ['home'],
'action' => 'warn',
'keywords_attributes' => [{ 'keyword' => 'dvshja', 'whole_word' => true }],
'statuses' => [],
},
{
'title' => 'filter with a phrase as title',
'expires_at' => nil,
'context' => %w(home notifications public),
'action' => 'warn',
'keywords_attributes' => [
{ 'keyword' => "more words let's see ", 'whole_word' => true },
],
'statuses' => [],
},
{
'title' => 'how do I add a status to a filter?',
'expires_at' => nil,
'context' => ['public', 'account'],
'action' => 'warn',
'keywords_attributes' => [
{ 'keyword' => 'something something', 'whole_word' => true },
],
'statuses' => [],
},
]
end
before do
subject.save
end
context 'with a BulkImport' do
let(:bulk_import) { account.bulk_imports.first }
it 'creates a bulk import with correct values' do
expect(bulk_import)
.to be_present
.and have_attributes(
type: eq(subject.type),
original_filename: eq(subject.data.original_filename),
likely_mismatched?: eq(subject.likely_mismatched_json?),
overwrite?: eq(!!subject.overwrite), # rubocop:disable Style/DoubleNegation
processed_items: eq(0),
imported_items: eq(0),
total_items: eq(bulk_import.rows.count),
state_unconfirmed?: be(true)
)
expect(bulk_import.rows.pluck(:data))
.to match_array(expected_rows)
end
end
end
it_behaves_like('on successful import', 'following', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
it_behaves_like('on successful import', 'following', 'overwrite', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
it_behaves_like('on successful import', 'blocking', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })

22
spec/requests/settings/imports_spec.rb
View File

@@ -4,7 +4,10 @@ require 'rails_helper'
RSpec.describe 'Settings Imports' do
describe 'POST /settings/imports' do
before { sign_in Fabricate(:user) }
let(:user) { Fabricate(:user) }
let(:account) { user.account }
before { sign_in user }
it 'gracefully handles invalid nested params' do
post settings_imports_path(form_import: 'invalid')
@@ -12,5 +15,22 @@ RSpec.describe 'Settings Imports' do
expect(response)
.to have_http_status(400)
end
describe 'with JSON' do
subject { post settings_imports_path, params: { form_import: { type: 'custom_filters', mode: 'merge', data: data } } }
let!(:data) { fixture_file_upload('custom_filters.json', 'application/json') }
let(:confirm) { post confirm_settings_import_path(id: user.account.bulk_imports.last.id) }
it 'redirects to confirm_settings_import_path' do
subject
expect(response).to have_http_status(302)
.and redirect_to(settings_import_path(id: user.account.bulk_imports.last.id))
expect(user.account.bulk_imports.last.state).to eq('unconfirmed')
confirm
expect(response).to have_http_status(302)
expect(user.account.bulk_imports.last.state).to eq('scheduled')
end
end
end
end

100
spec/services/bulk_import_service_spec.rb
View File

@@ -329,6 +329,106 @@ RSpec.describe BulkImportService do
end
end
context 'when importing custom_filters' do
let(:import_type) { 'custom_filters' }
let!(:rows) do
[{
'title' => 'baz',
'expires_at' => nil,
'context' => ['home', 'notifications'],
'action' => 'warn',
'keywords_attributes' => [{
'keyword' => 'discourse',
'whole_word' => true,
}, {
'keyword' => 'something',
'whole_word' => false,
}],
'statuses' => ['http://localhost:3000/ap/users/116646814515254858/statuses/116681350935935708'],
}, {
'title' => 'buzz',
'expires_at' => nil,
'context' => ['notifications'],
'action' => 'warn',
'keywords_attributes' => [{
'keyword' => 'discourse',
'whole_word' => true,
}, {
'keyword' => 'something',
'whole_word' => false,
}],
'statuses' => [ActivityPub::TagManager.instance.uri_for(status)],
}].map { |data| import.rows.create!(data: data) }
end
let(:overwrite) { false }
let(:status) { Fabricate(:status, account: account, text: 'something something') }
let(:filter) { Fabricate(:custom_filter, account: account, title: 'a mazing title') }
let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter, status: status) }
before do
status_filter
end
it 'enqueues workers for the expected rows and updates filters, keywords and statuses after worker run' do
subject.call(import)
expect(row_worker_job_args).to match_array(rows.map(&:id))
stub_fetch_remote_and_drain_workers
expect(account.custom_filters.count).to eq(3)
expect(account.custom_filters.order(:phrase).last.statuses.count).to eq(1)
expect(account.custom_filters.last.keywords.count).to eq(2)
end
end
context 'when importing custom_filters with overwrite' do
let(:import_type) { 'custom_filters' }
let!(:rows) do
[{
'title' => 'baz',
'expires_at' => nil,
'context' => ['home', 'notifications'],
'action' => 'warn',
'keywords_attributes' => [{
'keyword' => 'discourse',
'whole_word' => true,
}, {
'keyword' => 'something',
'whole_word' => false,
}],
'statuses' => ['http://localhost:3000/ap/users/116646814515254858/statuses/116681350935935708'],
}, {
'title' => 'buzz',
'expires_at' => nil,
'context' => ['notifications'],
'action' => 'warn',
'keywords_attributes' => [{
'keyword' => 'discourse',
'whole_word' => true,
}, {
'keyword' => 'something',
'whole_word' => false,
}],
'statuses' => [ActivityPub::TagManager.instance.uri_for(status)],
}].map { |data| import.rows.create!(data: data) }
end
let(:overwrite) { true }
let(:status) { Fabricate(:status, text: 'something something') }
let(:filter) { Fabricate(:custom_filter, account: account, title: 'a mazing title') }
let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter, status: status) }
before do
status_filter
end
it 'enqueues workers for the expected rows and updates filters, keywords and statuses after worker run' do
subject.call(import)
expect(row_worker_job_args).to match_array(rows.map(&:id))
stub_fetch_remote_and_drain_workers
expect(account.custom_filters.count).to eq(2)
expect(account.custom_filters.order(:phrase).last.statuses.count).to eq(1)
expect(account.custom_filters.order(:phrase).last.keywords.count).to eq(2)
end
end
def row_worker_job_args
Import::RowWorker
.jobs