diff --git a/app/controllers/concerns/dedupable.rb b/app/controllers/concerns/dedupable.rb
index e2f5c552d4..c72e9a34cf 100644
--- a/app/controllers/concerns/dedupable.rb
+++ b/app/controllers/concerns/dedupable.rb
@@ -3,13 +3,14 @@
module Dedupable
extend ActiveSupport::Concern
+ CandidateGroup = Struct.new(:label, :records, :reasons, keyword_init: true)
+
def dedupe_index
authorize!
config = dedupe_config
mc = config[:model_class]
- groups = mc.all.group_by { |r| r.name.to_s.strip.downcase }
- @possible_duplicates = groups.select { |_name, records| records.size > 1 }
+ @possible_duplicate_groups = dedupe_candidate_groups(config)
@records_for_select = mc.order(:name).map { |r| [ r.name, r.id ] }
@dedupe = build_dedupe_vars(config)
@@ -38,9 +39,11 @@ def dedupe_preview
return redirect_to url_for(action: :dedupe_index),
alert: "#{mc.model_name.human} not found (ID: #{missing.join(', ')})."
end
- join_assoc, join_incl = dedupe_primary_join(mc)
- @delete_items = @record_to_delete.public_send(join_assoc).includes(join_incl)
- @keep_items = @record_to_keep.public_send(join_assoc).includes(join_incl)
+ deduper = ModelDeduper.new(model_class: mc)
+ @reassignment_delete = deduper.reassignment_preview(@record_to_delete)
+ @reassignment_keep = deduper.reassignment_preview(@record_to_keep)
+ @lost_references = deduper.lost_references(@record_to_delete)
+ @unhandled_references = deduper.unhandled_references(@record_to_delete)
@dedupe = build_dedupe_vars(config)
render "dedupes/preview"
@@ -76,23 +79,38 @@ def dedupe_perform
record_to_delete = mc.find(params["#{mn}_to_delete_id"])
record_to_keep = mc.find(params["#{mn}_to_keep_id"])
+ unhandled = ModelDeduper.new(model_class: mc).unhandled_references(record_to_delete)
+ if unhandled.any?
+ tables = unhandled.map { |ref| ref[:table] }.uniq.join(", ")
+ return redirect_to url_for(action: :dedupe_index),
+ alert: "Can't merge: #{tables} still reference this #{mc.model_name.human.downcase} and the deduper doesn't reassign them. A developer needs to teach ModelDeduper about them before merging."
+ end
+
keep_param_key = "#{mn}_to_keep"
if params[keep_param_key].present?
editable = mc.column_names - %w[id created_at updated_at legacy_id]
record_to_keep.update!(params.require(keep_param_key).permit(editable))
end
+ # Combine values that must survive the merge rather than be replaced wholesale
+ # (e.g. an org's FileMaker codes), so the kept record keeps both records' links.
+ if (hook = config[:merge_keeper])
+ hook.call(record_to_keep, record_to_delete)
+ record_to_keep.save!
+ end
+
+ deduper = ModelDeduper.new(model_class: mc, logger: Rails.logger, dry_run: false, min_usage: 0)
+
if respond_to?(:track_event, true)
track_event("dedupe.#{mn}", {
resource_type: mc.name,
resource_id: record_to_keep.id,
deleted_record: record_to_delete.attributes,
kept_record: { id: record_to_keep.id, name: record_to_keep.name },
- associations_moved: record_to_delete.public_send(dedupe_primary_join(mc).first).count
+ associations_moved: deduper.reassignment_counts(record_to_delete).values.sum
})
end
- deduper = ModelDeduper.new(model_class: mc, logger: Rails.logger, dry_run: false, min_usage: 0)
deduper.merge(record_to_keep, record_to_delete)
label = mc.model_name.human.pluralize
@@ -116,27 +134,23 @@ def dedupe_config
raise NotImplementedError, "#{self.class} must implement #dedupe_config"
end
- # Returns [association_name, includes_name] for the primary polymorphic join.
- # e.g. [:categorizable_items, :categorizable]
- def dedupe_primary_join(mc)
- assoc = mc.reflect_on_all_associations(:has_many).find do |a|
- next if a.options[:through]
- begin
- a.klass.reflect_on_all_associations(:belongs_to).any?(&:polymorphic?)
- rescue NameError
- false
- end
- end
- raise "No polymorphic join found for #{mc.name}" unless assoc
-
- poly = assoc.klass.reflect_on_all_associations(:belongs_to).find(&:polymorphic?)
- [ assoc.name, poly.name ]
+ # Candidate duplicate groups for the index. A config may supply a
+ # :candidate_finder (a callable returning objects that respond to #label,
+ # #records, and #reasons โ e.g. OrganizationServices::DuplicateFinder); without
+ # one, fall back to exact normalized-name grouping.
+ def dedupe_candidate_groups(config)
+ finder = config[:candidate_finder]
+ return Array(finder.call) if finder
+
+ config[:model_class].all
+ .group_by { |record| record.name.to_s.strip.downcase }
+ .select { |_name, records| records.size > 1 }
+ .map { |name, records| CandidateGroup.new(label: name, records: records, reasons: []) }
end
def build_dedupe_vars(config)
mc = config[:model_class]
mn = mc.model_name.singular
- join_assoc, join_incl = dedupe_primary_join(mc)
opts = config[:belongs_to_options]
{
@@ -147,9 +161,8 @@ def build_dedupe_vars(config)
delete_id_param: "#{mn}_to_delete_id",
keep_id_param: "#{mn}_to_keep_id",
keep_param_key: "#{mn}_to_keep".to_sym,
- item_type_col: "#{join_incl}_type".to_sym,
- item_id_col: "#{join_incl}_id".to_sym,
- join_association: join_assoc,
+ editable_columns: config[:editable_columns],
+ union_columns: Array(config[:union_columns]).map(&:to_s),
belongs_to_options: opts.is_a?(Proc) ? opts.call : (opts || {}),
record_extras: config[:record_extras]
}
diff --git a/app/controllers/organizations_controller.rb b/app/controllers/organizations_controller.rb
index 8c234862dc..af88bcfdaf 100644
--- a/app/controllers/organizations_controller.rb
+++ b/app/controllers/organizations_controller.rb
@@ -1,5 +1,5 @@
class OrganizationsController < ApplicationController
- include AhoyTracking, TagAssignable
+ include AhoyTracking, TagAssignable, Dedupable
before_action :set_organization, only: [ :show, :edit, :update, :destroy, :populations_served ]
def index
@@ -283,6 +283,31 @@ def organization_params
)
end
+ def dedupe_config
+ {
+ model_class: Organization,
+ domain: :organizations,
+ candidate_finder: -> { OrganizationServices::DuplicateFinder.new.groups },
+ editable_columns: %w[
+ name filemaker_code email website_url agency_type description
+ mission_vision_values notes organization_status_id organization_obligation_id
+ ],
+ union_columns: %w[filemaker_code],
+ merge_keeper: ->(keep, delete) {
+ keep.filemaker_code = Organization.join_filemaker_codes(keep.filemaker_code, delete.filemaker_code)
+ },
+ belongs_to_options: -> {
+ {
+ "organization_status_id" => OrganizationStatus.order(:name),
+ "organization_obligation_id" => OrganizationObligation.order(:name)
+ }
+ },
+ record_extras: ->(org) {
+ [ org.filemaker_code.presence && "FileMaker #{org.filemaker_code}", org.program_location ].compact.join(" ยท ").presence
+ }
+ }
+ end
+
def find_duplicate_organizations(name)
return [] if name.blank?
diff --git a/app/helpers/admin_cards_helper.rb b/app/helpers/admin_cards_helper.rb
index 1ab9b302ae..3eb00b3572 100644
--- a/app/helpers/admin_cards_helper.rb
+++ b/app/helpers/admin_cards_helper.rb
@@ -66,7 +66,10 @@ def reference_cards
# -----------------------------
def deprecated_data_cards
[
- custom_card("Organization statuses", organization_statuses_path, icon: "๐งฎ", color: :emerald, intensity: 100)
+ custom_card("Organization statuses", organization_statuses_path, icon: "๐งฎ", color: :emerald, intensity: 100),
+ custom_card("Dedupe organizations", dedupe_index_organizations_path, icon: "๐งน", color: DomainTheme.color_for(:organizations), intensity: 100),
+ custom_card("Dedupe categories", dedupe_index_categories_path, icon: "๐งน", color: DomainTheme.color_for(:categories), intensity: 100),
+ custom_card("Dedupe sectors", dedupe_index_sectors_path, icon: "๐งน", color: DomainTheme.color_for(:sectors), intensity: 100)
]
end
diff --git a/app/models/organization.rb b/app/models/organization.rb
index 3c5ec6cd53..03f48833cc 100644
--- a/app/models/organization.rb
+++ b/app/models/organization.rb
@@ -285,6 +285,23 @@ def rollup_cache_version
remote_searchable_by :name
+ # FileMaker is the source of truth, and a record can carry more than one
+ # FileMaker code (e.g. when two orgs that each mapped to a FileMaker record are
+ # merged). The `filemaker_code` column holds them as a trimmed, comma-separated
+ # list; these are the seam for reading and combining them.
+ def filemaker_codes
+ Organization.split_filemaker_codes(filemaker_code)
+ end
+
+ def self.split_filemaker_codes(value)
+ value.to_s.split(",").map(&:strip).reject(&:blank?).uniq
+ end
+
+ # A single normalized column value from any mix of code strings/lists.
+ def self.join_filemaker_codes(*values)
+ values.flatten.flat_map { |value| split_filemaker_codes(value) }.uniq.sort.join(", ").presence
+ end
+
# Returns the website as a clickable, scheme-qualified URL โ prepending
# https:// to a bare domain like "awbw.org" โ or nil when the value is blank or
# not a usable web address. Drives the external links on the org profile and
diff --git a/app/services/model_deduper.rb b/app/services/model_deduper.rb
index dda226325b..66582306e6 100644
--- a/app/services/model_deduper.rb
+++ b/app/services/model_deduper.rb
@@ -36,39 +36,250 @@ def merge(record_to_keep, record_to_delete)
merge_duplicate(record_to_keep, record_to_delete, usage_counts)
end
+ # A { human label => count } of the records that would be reassigned off this
+ # record on merge, skipping empty associations. Drives the preview so an admin
+ # sees exactly what moves (affiliations, event registrations, reports, โฆ).
+ def reassignment_counts(record)
+ reassignable_joins.each_with_object({}) do |join, counts|
+ count = join[:join_class].where(join[:foreign_key] => record.id).count
+ next if count.zero?
+
+ counts[join_label(join[:join_class])] = count
+ end
+ end
+
+ # Like #reassignment_counts, but also a few sample record names per association
+ # so the preview can show *what* moves, not just how many. Each entry is
+ # { label:, count:, names: [ up to SAMPLE_SIZE display strings ] }.
+ SAMPLE_SIZE = 3
+
+ def reassignment_preview(record)
+ reassignable_joins.filter_map do |join|
+ scope = join[:join_class].where(join[:foreign_key] => record.id)
+ count = scope.count
+ next if count.zero?
+
+ {
+ label: join_label(join[:join_class]),
+ count: count,
+ names: scope.limit(SAMPLE_SIZE).map { |related| record_display(related) }
+ }
+ end.sort_by { |entry| entry[:label] }
+ end
+
+ # A best-effort human name for an associated record: its own name/title, else a
+ # named parent (e.g. an affiliation's person), else its id.
+ def record_display(record)
+ own = display_string(record)
+ return own if own
+
+ record.class.reflect_on_all_associations(:belongs_to).each do |assoc|
+ next if assoc.polymorphic?
+
+ related = record.try(assoc.name)
+ parent = display_string(related)
+ return parent if parent
+ end
+ "##{record.id}"
+ end
+
+ def display_string(record)
+ return unless record
+
+ %i[name title full_name].filter_map { |method| record.try(method).presence }.first
+ end
+
+ # Polymorphic references the merge repoints to the kept record (type stays, id
+ # moves) so analytics and audit history follow the survivor instead of orphaning.
+ # table => [ type_column, id_column ].
+ REASSIGNED_POLYMORPHIC_REFERENCES = {
+ "ahoy_events" => %w[resource_type resource_id],
+ "versions" => %w[item_type item_id]
+ }.freeze
+
+ # References that are lost with the deleted record (its own attached files purge)
+ # rather than moved โ surfaced on the preview so an admin sees the loss.
+ LOST_POLYMORPHIC_REFERENCES = { "active_storage_attachments" => %w[record_type record_id] }.freeze
+
+ # Billing links between orgs need a deliberate decision (rubyforgood/awbw#2378);
+ # skipped here so they neither block nor silently move.
+ DEFERRED_REFERENCE_TABLES = %w[
+ pay_customers pay_merchants pay_subscriptions pay_charges pay_payment_methods pay_webhooks
+ ].freeze
+
+ # Framework internals that purge with the record and aren't worth surfacing.
+ IGNORED_REFERENCE_TABLES = %w[
+ action_text_rich_texts action_text_mentions ckeditor_assets
+ active_storage_blobs active_storage_variant_records ahoy_visits
+ ].freeze
+
+ HANDLED_ELSEWHERE_TABLES = (
+ REASSIGNED_POLYMORPHIC_REFERENCES.keys + LOST_POLYMORPHIC_REFERENCES.keys +
+ DEFERRED_REFERENCE_TABLES + IGNORED_REFERENCE_TABLES
+ ).freeze
+
+ # Attached files (e.g. the logo) that are deleted with the record, not moved.
+ # Each entry is { label:, count: } โ drives the preview's "will be lost" note.
+ def lost_references(record)
+ LOST_POLYMORPHIC_REFERENCES.filter_map do |table, (type_column, id_column)|
+ next unless reference_table_present?(table)
+
+ klass = model_for_table(table)
+ next unless klass.column_names.include?(id_column)
+
+ count = klass.where(type_column => model_class.polymorphic_name, id_column => record.id).count
+ next if count.zero?
+
+ { label: "Attached files (e.g. logo)", count: count }
+ end
+ end
+
+ # Defensive safeguard: tables that still reference `record` but that the merge
+ # would NOT reassign โ e.g. a new association added to the model that this flow
+ # doesn't yet account for. Data-driven (it checks the actual record) and
+ # independent of any `dependent:` option, so a future association can't silently
+ # orphan rows or break referential integrity. Each entry is { table:, column: }.
+ def unhandled_references(record)
+ covered = reassignable_joins.map { |join| [ join[:join_class].table_name, join[:foreign_key].to_s ] }.to_set
+ connection = model_class.connection
+ convention_fk = "#{model_class.model_name.singular}_id"
+ polymorphic_name = model_class.polymorphic_name
+
+ connection.tables.flat_map do |table|
+ next [] if table == model_class.table_name || HANDLED_ELSEWHERE_TABLES.include?(table)
+
+ column_names = connection.columns(table).map(&:name)
+ gaps = []
+
+ if column_names.include?(convention_fk) && !covered.include?([ table, convention_fk ]) &&
+ model_for_table(table).where(convention_fk => record.id).exists?
+ gaps << { table: table, column: convention_fk }
+ end
+
+ column_names.each do |column|
+ next unless column.end_with?("_type")
+
+ id_column = column.sub(/_type\z/, "_id")
+ next unless column_names.include?(id_column)
+ next if covered.include?([ table, id_column ])
+ next unless model_for_table(table).where(column => polymorphic_name, id_column => record.id).exists?
+
+ gaps << { table: table, column: id_column }
+ end
+
+ gaps.uniq
+ end.uniq
+ end
+
private
- # Auto-detect all polymorphic join associations on the model.
- # Returns an array of hashes, each with :join_class, :foreign_key,
- # :polymorphic_type_column, and :polymorphic_id_column.
- def polymorphic_joins
- @polymorphic_joins ||= model_class.reflect_on_all_associations(:has_many).filter_map do |assoc|
- next if assoc.options[:through] # skip has_many :through
+ # Every association that references this model by a foreign key โ whether a
+ # plain FK child (affiliations, reports), a polymorphic child (addresses via
+ # as: :addressable), or a tagging join (categorizable_items). Each is reassigned
+ # from the duplicate to the kept record on merge. Reflection-driven, so any
+ # FK-based model works with no bespoke config.
+ #
+ # Sources are unioned: this model's own has_many/:as declarations, plus a scan
+ # of every model for a non-polymorphic belongs_to pointing back here โ so a
+ # child with an organization_id but no inverse has_many (payments, stories, โฆ)
+ # is still reassigned rather than orphaned or blocked by its FK constraint.
+ #
+ # Collisions are resolved by DB *unique indexes*, not by guessing: the columns
+ # of a unique index that includes the FK are the scope in which only one row may
+ # exist (e.g. [category_id, categorizable_type, categorizable_id] โ
+ # [category_id, categorizable_type]), so a row the kept record already has is
+ # deleted instead of violating the index.
+ def reassignable_joins
+ @reassignable_joins ||= begin
+ seen = Set.new
+ (has_many_joins + belongs_to_joins).select do |join|
+ seen.add?([ join[:join_class].table_name, join[:foreign_key] ])
+ end
+ end
+ end
+
+ def has_many_joins
+ model_class.reflect_on_all_associations(:has_many).filter_map do |assoc|
+ next if assoc.options[:through]
begin
join_klass = assoc.klass
rescue NameError
next
end
- poly = join_klass.reflect_on_all_associations(:belongs_to).find(&:polymorphic?)
- next unless poly
+ fk = assoc.foreign_key.to_s
+ next unless join_klass.column_names.include?(fk)
- {
- join_class: join_klass,
- foreign_key: assoc.foreign_key.to_sym,
- polymorphic_type_column: poly.foreign_type.to_sym,
- polymorphic_id_column: poly.foreign_key.to_sym
- }
+ join_for(join_klass, fk)
end
end
+ # Schema-driven: every DB foreign key pointing at this model's table, so a child
+ # with an organization_id but no inverse has_many (payments, stories, โฆ) is still
+ # reassigned. Reading the schema avoids eager-loading every model (which would
+ # touch databases not configured in every environment).
+ def belongs_to_joins
+ connection = model_class.connection
+ connection.tables.flat_map do |table|
+ next [] if table == model_class.table_name
+
+ connection.foreign_keys(table).filter_map do |fk_def|
+ next unless fk_def.to_table == model_class.table_name
+
+ join_for(model_for_table(table), fk_def.column.to_s)
+ end
+ end
+ rescue NotImplementedError
+ []
+ end
+
+ # The model backing a table, or an anonymous one bound to it. Guards against a
+ # name that resolves to a class mapped elsewhere (e.g. an STI child of another
+ # table), which would reassign the wrong table.
+ def model_for_table(table)
+ klass = table.classify.constantize
+ return klass if klass.table_name == table
+
+ anonymous_model_for(table)
+ rescue NameError
+ anonymous_model_for(table)
+ end
+
+ def anonymous_model_for(table)
+ klass = Class.new(ApplicationRecord)
+ klass.table_name = table
+ klass
+ end
+
+ def join_for(join_klass, fk)
+ {
+ join_class: join_klass,
+ foreign_key: fk.to_sym,
+ natural_key: natural_key_columns(join_klass, fk)
+ }
+ end
+
+ # Columns (other than the FK) of a unique index that includes the FK.
+ def natural_key_columns(join_klass, fk)
+ join_klass.connection.indexes(join_klass.table_name)
+ .select(&:unique)
+ .map { |index| Array(index.columns) }
+ .select { |cols| cols.include?(fk) }
+ .map { |cols| (cols - [ fk ]).map(&:to_sym) }
+ .find(&:present?) || []
+ end
+
def model_label
model_class.name.underscore.humanize.downcase
end
+ def join_label(join_class)
+ (join_class.name || join_class.table_name.classify).underscore.humanize.pluralize
+ end
+
def usage_counts
counts = Hash.new(0)
- polymorphic_joins.each do |join|
+ reassignable_joins.each do |join|
join[:join_class].where(join[:foreign_key] => model_class.pluck(:id))
.group(join[:foreign_key]).count
.each { |id, count| counts[id] += count }
@@ -104,37 +315,57 @@ def merge_duplicate(primary, dupe, usage)
return if dry_run
ActiveRecord::Base.transaction do
- polymorphic_joins.each do |join|
+ reassignable_joins.each do |join|
merge_join(primary, dupe, join)
end
- dupe.destroy!
+ reassign_polymorphic_references(primary, dupe)
+
+ dupe.reload.destroy!
logger.info " deleted #{model_label} #{dupe.id}"
end
end
+ # Repoint analytics/audit rows (ahoy events, versions) from the dupe to the kept
+ # record so its history survives the merge. Type column stays; only the id moves.
+ def reassign_polymorphic_references(primary, dupe)
+ REASSIGNED_POLYMORPHIC_REFERENCES.each do |table, (type_column, id_column)|
+ next unless reference_table_present?(table)
+
+ klass = model_for_table(table)
+ next unless klass.column_names.include?(id_column)
+
+ moved = klass.where(type_column => model_class.polymorphic_name, id_column => dupe.id)
+ .update_all(id_column => primary.id)
+ logger.info " moved #{moved} #{table} to primary" if moved > 0
+ end
+ end
+
+ def reference_table_present?(table)
+ model_class.connection.data_source_exists?(table)
+ end
+
def merge_join(primary, dupe, join)
jc = join[:join_class]
fk = join[:foreign_key]
- type_col = join[:polymorphic_type_column]
- id_col = join[:polymorphic_id_column]
+ natural_key = join[:natural_key]
- existing_taggings = jc
- .where(fk => primary.id)
- .pluck(type_col, id_col)
- .map { |type, id| "#{type}_#{id}" }
- .to_set
-
- items_to_move = jc.where(fk => dupe.id)
+ if natural_key.empty?
+ moved = jc.where(fk => dupe.id).update_all(fk => primary.id)
+ logger.info " moved #{moved} #{jc.name} to primary" if moved > 0
+ return
+ end
- items_to_move.find_each do |item|
- tagging_key = "#{item.public_send(type_col)}_#{item.public_send(id_col)}"
+ existing = jc.where(fk => primary.id).pluck(*natural_key).map { |values| Array(values) }.to_set
- if existing_taggings.include?(tagging_key)
+ jc.where(fk => dupe.id).find_each do |item|
+ key = natural_key.map { |col| item.public_send(col) }
+ if existing.include?(key)
item.destroy!
logger.info " deleted duplicate #{jc.name} #{item.id} (primary already has it)"
else
item.update!(fk => primary.id)
+ existing << key
logger.info " moved #{jc.name} #{item.id} to primary"
end
end
diff --git a/app/services/organization_services/duplicate_finder.rb b/app/services/organization_services/duplicate_finder.rb
new file mode 100644
index 0000000000..2c70829282
--- /dev/null
+++ b/app/services/organization_services/duplicate_finder.rb
@@ -0,0 +1,118 @@
+require "set"
+
+module OrganizationServices
+ # Surfaces likely-duplicate organizations for the deduper's candidate list.
+ # Clusters organizations by several signals โ normalized name, name minus a
+ # legal suffix (Inc/LLC/โฆ), FileMaker code, and shared address โ then annotates
+ # each cluster with why it was flagged. FileMaker is the source of truth, so a
+ # cluster whose members carry different FileMaker codes is surfaced as a
+ # conflict for an admin to resolve, never silently treated as a clean merge.
+ class DuplicateFinder
+ LEGAL_SUFFIX = /\b(?:incorporated|inc|llc|l\.l\.c|corporation|corp|company|co|limited|ltd)\b\.?/i
+
+ Group = Struct.new(:key, :label, :records, :reasons, keyword_init: true)
+
+ def initialize(scope = Organization.all)
+ @organizations = scope.includes(:addresses).to_a
+ end
+
+ def groups
+ union = UnionFind.new(@organizations.map(&:id))
+ cluster(union) { |org| [ normalized_name(org).presence ].compact }
+ cluster(union) { |org| [ suffixless_name(org).presence ].compact }
+ cluster(union) { |org| org.filemaker_codes }
+ cluster(union) { |org| address_keys(org) }
+
+ by_id = @organizations.index_by(&:id)
+ union.components.filter_map do |ids|
+ next if ids.size < 2
+ records = ids.map { |id| by_id[id] }.sort_by(&:id)
+ Group.new(
+ key: records.map(&:id).join("-"),
+ label: records.first.name.to_s,
+ records: records,
+ reasons: reasons_for(records)
+ )
+ end.sort_by { |group| group.label.to_s.downcase }
+ end
+
+ private
+
+ def cluster(union)
+ buckets = Hash.new { |hash, key| hash[key] = [] }
+ @organizations.each do |org|
+ yield(org).each { |key| buckets[key] << org.id }
+ end
+ buckets.each_value { |ids| union.union_all(ids) if ids.size > 1 }
+ end
+
+ def reasons_for(records)
+ reasons = []
+ normalized = records.map { |org| normalized_name(org) }.uniq
+ reasons << "Same name" if normalized.size == 1
+
+ suffixless = records.map { |org| suffixless_name(org) }.reject(&:blank?).uniq
+ reasons << "Same name aside from Inc/LLC/etc." if normalized.size > 1 && suffixless.size == 1
+
+ reasons.concat(filemaker_reasons(records))
+
+ keys = records.flat_map { |org| address_keys(org) }
+ reasons << "Shared address" if keys.tally.any? { |_key, count| count > 1 }
+ reasons
+ end
+
+ def filemaker_reasons(records)
+ code_sets = records.map(&:filemaker_codes)
+ present = code_sets.flatten.uniq.sort
+ return [] if present.empty?
+ return [ "โ Multiple FileMaker codes (#{present.join(", ")}) โ merging keeps all of them" ] if present.size > 1
+ return [ "FileMaker code on only one record" ] if code_sets.any?(&:empty?)
+
+ [ "Same FileMaker code (#{present.first})" ]
+ end
+
+ def normalized_name(org)
+ org.name.to_s.downcase.gsub(/[^a-z0-9 ]/, " ").squish
+ end
+
+ def suffixless_name(org)
+ normalized_name(org).gsub(LEGAL_SUFFIX, "").squish
+ end
+
+ def address_keys(org)
+ org.addresses.reject(&:inactive?).filter_map do |address|
+ parts = [ address.street_address, address.city, address.zip_code ].map { |part| part.to_s.strip.downcase }
+ next if parts.all?(&:blank?)
+ parts.join("|")
+ end
+ end
+
+ # Minimal disjoint-set: clusters ids connected by any shared signal into
+ # components, so an org linked by name to one dupe and by address to another
+ # lands in a single group.
+ class UnionFind
+ def initialize(ids)
+ @parent = ids.to_h { |id| [ id, id ] }
+ end
+
+ def union_all(ids)
+ ids.each { |id| union(ids.first, id) }
+ end
+
+ def components
+ @parent.keys.group_by { |id| find(id) }.values
+ end
+
+ private
+
+ def find(id)
+ id = @parent[id] while @parent[id] != id
+ id
+ end
+
+ def union(a, b)
+ @parent[find(a)] = find(b)
+ end
+ end
+ end
+end
diff --git a/app/views/dedupes/_preview.html.erb b/app/views/dedupes/_preview.html.erb
index 532e5840d2..d5b1b38ca0 100644
--- a/app/views/dedupes/_preview.html.erb
+++ b/app/views/dedupes/_preview.html.erb
@@ -1,155 +1,116 @@
-<%# Shared dedupe preview partial
+<%# Shared dedupe preview partial.
Required locals:
- domain: Symbol (:categories or :sectors)
- record_to_delete: The record that will be deleted
- record_to_keep: The record that will be kept
- delete_items: Associated items of the record to delete
- keep_items: Associated items of the record to keep
- item_type_col: Polymorphic type column (e.g. :categorizable_type)
- item_id_col: Polymorphic id column (e.g. :categorizable_id)
- model_label: Human label (e.g. "Category")
- keep_param_key: Param key for keep fields (e.g. :category_to_keep)
- form_builder: The parent form builder (f)
- belongs_to_options: Hash of { column_name => collection } for select fields
+ domain: Symbol for DomainTheme (e.g. :organizations)
+ record_to_delete: The record that will be deleted
+ record_to_keep: The record that will be kept
+ reassignment_delete: { human label => count } of records moving off the deleted record
+ reassignment_keep: { human label => count } the kept record already has
+ model_label: Human label (e.g. "Organization")
+ keep_param_key: Param key for keep fields (e.g. :organization_to_keep)
+ editable_columns: Optional allowlist of column names to show/edit; nil shows all
+ union_columns: Column names whose values are combined into the keeper (not replaced)
+ form_builder: The parent form builder (f)
+ belongs_to_options: Hash of { column_name => collection } for select fields
%>
<%
- color = DomainTheme.color_for(domain)
+ border = DomainTheme.border_class_for(domain, intensity: 200)
+ header_bg = DomainTheme.bg_class_for(domain, intensity: 50)
model_class = record_to_delete.class
+ union_columns ||= []
skip_columns = %w[id created_at updated_at legacy_id]
- columns = model_class.columns.reject { |c| skip_columns.include?(c.name) }
-
- # Resolve belongs_to associations for _id columns
+ columns =
+ if editable_columns.present?
+ editable_columns.filter_map { |name| model_class.columns_hash[name.to_s] }
+ else
+ model_class.columns.reject { |c| skip_columns.include?(c.name) }
+ end
bt_associations = model_class.reflect_on_all_associations(:belongs_to).index_by { |a| a.foreign_key.to_s }
belongs_to_options ||= {}
- assoc_name = item_type_col.to_s.sub(/_type$/, '').to_sym
%>
-
-
-
-
+
+
- <%= model_label %> to delete: <%= record_to_delete.name %>
+ <%= model_label %> to delete: <%= record_to_delete.name %>
+ The <%= @dedupe[:model_label].downcase %> on the left is deleted; its records
+ move to the one on the right. Edit the keeper's fields before merging.
+
+ These reference this <%= @dedupe[:model_label].downcase %> but the deduper wouldn't move them, so merging
+ could orphan data or break referential integrity:
- Danger: This action cannot be undone.
- "<%= @record_to_delete.name %>" (ID: <%= @record_to_delete.id %>) will be permanently deleted
- and its <%= @delete_items.count %> associations moved to
- "<%= @record_to_keep.name %>" (ID: <%= @record_to_keep.id %>).
+
+ <% if @lost_references.any? %>
+
+
+
+
+
Deleted, not moved
+
+ These belong to "<%= @record_to_delete.name %>" and are removed with it โ they are
+ not transferred to "<%= @record_to_keep.name %>":
+ This can't be undone.
+ "<%= @record_to_delete.name %>" (ID <%= @record_to_delete.id %>) will be deleted and
+ <%= moved_total %> associated
+ <%= "record".pluralize(moved_total) %> moved to
+ "<%= @record_to_keep.name %>" (ID <%= @record_to_keep.id %>).
+
- <% if allowed_to?(:new?, Organization) %>
- <%= link_to "New Organization",
- new_organization_path,
- class: "admin-only bg-blue-100 btn btn-primary-outline" %>
- <% end %>
+
+ <% if allowed_to?(:manage?, Organization) %>
+ <%= link_to "Dedupe", dedupe_index_organizations_path,
+ class: "text-sm #{eyebrow_link_class} px-2 py-1" %>
+ <% end %>
+ <% if allowed_to?(:new?, Organization) %>
+ <%= link_to "New Organization",
+ new_organization_path,
+ class: "admin-only bg-blue-100 btn btn-primary-outline" %>
+ <% end %>
+
diff --git a/config/features.yml b/config/features.yml
index e31dfff8d1..783e1a7bd0 100644
--- a/config/features.yml
+++ b/config/features.yml
@@ -2262,3 +2262,29 @@
pro_tips:
- "Only mutations show: opening or printing a record never clutters its log."
- "Deleting a child or attachment is recorded too โ the entry keeps the name even after the file is gone."
+
+- name: "Merge duplicate organizations"
+ area: reporting
+ display_status: admin_facing
+ released_on: 2026-08-25
+ pr_number: 2374
+ action_path: "/organizations/dedupe_index"
+ summary: >-
+ Admins can now merge two duplicate organizations into one. The deduper
+ suggests likely duplicates (by name, name minus Inc/LLC, FileMaker code, and
+ shared address), then previews exactly what will move before merging.
+ description: >-
+ Reach it from the Dedupe link on the Organizations page. It clusters likely
+ duplicates and flags why each was grouped โ same name, a shared FileMaker
+ code, or a shared address. FileMaker is the source of truth, and a record can
+ carry more than one code (a trimmed, comma-separated list); when two records
+ have different codes, merging keeps all of them so no FileMaker link is lost.
+ The preview shows a side-by-side of both records with the kept record's key
+ fields editable, plus a summary of every associated record (affiliations,
+ event registrations, reports, payments, workshop logs, tags, and more) that
+ moves to the kept organization. The merge reassigns all of them and deletes
+ the duplicate. It can't be undone. Admin-only.
+ pro_tips:
+ - "Edit the kept organization's fields right in the preview before you merge."
+ - "Use Swap if you picked the wrong record to keep."
+ - "Both records' FileMaker codes are kept on the merged organization โ a field flagged 'Kept on merge' is combined, not dropped."
diff --git a/config/routes.rb b/config/routes.rb
index 7f05bcfcd9..0296f01110 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -277,6 +277,10 @@
resources :organizations do
collection do
get :check_duplicates
+ get :dedupe_index
+ get :dedupe_preview
+ post :dedupe_perform
+ patch :dedupe_update_keep
end
member do
get :populations_served
diff --git a/spec/models/organization_spec.rb b/spec/models/organization_spec.rb
index 67ecf9948d..13fb7b8801 100644
--- a/spec/models/organization_spec.rb
+++ b/spec/models/organization_spec.rb
@@ -422,4 +422,24 @@ def version = Organization.find(organization.id).rollup_cache_version
expect(Organization.awbw).to be_nil
end
end
+
+ describe "FileMaker codes" do
+ it "parses a trimmed, de-duplicated list from the column" do
+ org = build(:organization, filemaker_code: " FM1 , FM2,FM1 ")
+
+ expect(org.filemaker_codes).to eq(%w[FM1 FM2])
+ end
+
+ it "is empty when the column is blank" do
+ expect(build(:organization, filemaker_code: nil).filemaker_codes).to eq([])
+ end
+
+ it "combines mixed strings and lists into one sorted normalized value" do
+ expect(Organization.join_filemaker_codes("FM2, FM1", " FM3 ", nil, "FM1")).to eq("FM1, FM2, FM3")
+ end
+
+ it "returns nil when there is nothing to combine" do
+ expect(Organization.join_filemaker_codes(nil, "")).to be_nil
+ end
+ end
end
diff --git a/spec/requests/admin/home_spec.rb b/spec/requests/admin/home_spec.rb
index 54e7def6da..20cda01c67 100644
--- a/spec/requests/admin/home_spec.rb
+++ b/spec/requests/admin/home_spec.rb
@@ -18,4 +18,15 @@
expect(response.body).to include("Form submissions")
expect(response.body).to include(%(href="#{form_submissions_path}"))
end
+
+ it "links to the record dedupers under deprecated data" do
+ get "/admin"
+
+ expect(response.body).to include("Dedupe organizations", "Dedupe categories", "Dedupe sectors")
+ expect(response.body).to include(
+ %(href="#{dedupe_index_organizations_path}"),
+ %(href="#{dedupe_index_categories_path}"),
+ %(href="#{dedupe_index_sectors_path}")
+ )
+ end
end
diff --git a/spec/requests/dedupable_spec.rb b/spec/requests/dedupable_spec.rb
index 5fe06cb260..656dbf9252 100644
--- a/spec/requests/dedupable_spec.rb
+++ b/spec/requests/dedupable_spec.rb
@@ -303,4 +303,103 @@
end
end
end
+
+ # ============================================================
+ # ORGANIZATIONS โ FK-based model with a custom candidate finder
+ # ============================================================
+
+ describe "Organizations" do
+ before { sign_in admin }
+
+ describe "GET dedupe_index" do
+ it "renders and surfaces candidate groups from the duplicate finder" do
+ create(:organization, name: "Hope Center")
+ create(:organization, name: "hope center")
+
+ get dedupe_index_organizations_path
+
+ expect(response).to have_http_status(:ok)
+ expect(response.body).to include("Same name")
+ end
+ end
+
+ describe "GET dedupe_preview" do
+ let!(:keep) { create(:organization, name: "Keep Org") }
+ let!(:delete_rec) { create(:organization, name: "Delete Org") }
+ before { create(:affiliation, organization: delete_rec) }
+
+ it "renders the preview with a reassignment summary and curated fields" do
+ get dedupe_preview_organizations_path(
+ organization_to_keep_id: keep.id,
+ organization_to_delete_id: delete_rec.id
+ )
+
+ expect(response).to have_http_status(:ok)
+ expect(response.body).to include("Keep Org", "Delete Org")
+ expect(response.body).to include("Records that will move", "Affiliations")
+ expect(response.body).to include("Website url")
+ expect(response.body).not_to include("Profile show age ranges")
+ end
+ end
+
+ describe "POST dedupe_perform" do
+ let!(:keep) { create(:organization, name: "Keeper Org") }
+ let!(:delete_rec) { create(:organization, name: "Duplicate Org") }
+ let!(:report) { create(:report, organization: delete_rec) }
+
+ it "merges, reassigns FK associations, and deletes the duplicate" do
+ expect {
+ post dedupe_perform_organizations_path, params: {
+ organization_to_delete_id: delete_rec.id,
+ organization_to_keep_id: keep.id
+ }
+ }.to change(Organization, :count).by(-1)
+
+ expect(response).to redirect_to(organizations_path)
+ expect(Organization.exists?(delete_rec.id)).to be false
+ expect(report.reload.organization_id).to eq(keep.id)
+ end
+
+ it "applies keep-field edits before merging" do
+ post dedupe_perform_organizations_path, params: {
+ organization_to_delete_id: delete_rec.id,
+ organization_to_keep_id: keep.id,
+ organization_to_keep: { name: "Canonical Org" }
+ }
+
+ expect(keep.reload.name).to eq("Canonical Org")
+ end
+
+ it "blocks the merge when an association would be orphaned" do
+ create(:other_response, promotable: delete_rec)
+
+ get dedupe_preview_organizations_path(
+ organization_to_delete_id: delete_rec.id,
+ organization_to_keep_id: keep.id
+ )
+ expect(response.body).to include("Merge blocked")
+
+ expect {
+ post dedupe_perform_organizations_path, params: {
+ organization_to_delete_id: delete_rec.id,
+ organization_to_keep_id: keep.id
+ }
+ }.not_to change(Organization, :count)
+ expect(response).to redirect_to(dedupe_index_organizations_path)
+ expect(Organization.exists?(delete_rec.id)).to be true
+ end
+
+ it "combines both organizations' FileMaker codes onto the keeper" do
+ keep.update!(filemaker_code: "FM-100")
+ delete_rec.update!(filemaker_code: "FM-273")
+
+ post dedupe_perform_organizations_path, params: {
+ organization_to_delete_id: delete_rec.id,
+ organization_to_keep_id: keep.id
+ }
+
+ expect(keep.reload.filemaker_code).to eq("FM-100, FM-273")
+ end
+ end
+ end
end
diff --git a/spec/services/model_deduper_spec.rb b/spec/services/model_deduper_spec.rb
index 4f0fb422ee..1ce987bf40 100644
--- a/spec/services/model_deduper_spec.rb
+++ b/spec/services/model_deduper_spec.rb
@@ -117,4 +117,139 @@
join_factory: :sectorable_item,
join_fk: :sector_id
end
+
+ # Organizations are referenced by plain organization_id foreign keys rather than
+ # polymorphic joins, so they exercise the reflection-driven FK reassignment pass.
+ describe "foreign-key reassignment (Organization)" do
+ subject(:service) do
+ described_class.new(model_class: Organization, logger: logger, dry_run: false)
+ end
+
+ let!(:keep) { create(:organization, name: "Keep Org") }
+ let!(:dupe) { create(:organization, name: "Dupe Org") }
+
+ it "reassigns plain FK associations to the kept organization" do
+ report = create(:report, organization: dupe)
+ service.merge(keep, dupe)
+ expect(report.reload.organization_id).to eq(keep.id)
+ end
+
+ it "reassigns restrict_with_error associations and still deletes the duplicate" do
+ affiliation = create(:affiliation, organization: dupe)
+
+ expect { service.merge(keep, dupe) }
+ .to change { Organization.exists?(dupe.id) }.from(true).to(false)
+ expect(affiliation.reload.organization_id).to eq(keep.id)
+ end
+
+ it "moves a non-colliding event registration link to the kept organization" do
+ registration = create(:event_registration)
+ link = create(:event_registration_organization, organization: dupe, event_registration: registration)
+
+ service.merge(keep, dupe)
+ expect(link.reload.organization_id).to eq(keep.id)
+ end
+
+ it "collapses a natural-key collision instead of violating the unique index" do
+ registration = create(:event_registration)
+ create(:event_registration_organization, organization: keep, event_registration: registration)
+ colliding = create(:event_registration_organization, organization: dupe, event_registration: registration)
+
+ expect { service.merge(keep, dupe) }
+ .to change(EventRegistrationOrganization, :count).by(-1)
+ expect(EventRegistrationOrganization.exists?(colliding.id)).to be false
+ expect(EventRegistrationOrganization.where(organization_id: keep.id, event_registration_id: registration.id).count).to eq(1)
+ end
+
+ it "reassigns polymorphic joins alongside FK associations" do
+ bookmark = create(:bookmark, bookmarkable: dupe)
+ service.merge(keep, dupe)
+ expect(bookmark.reload.bookmarkable).to eq(keep)
+ end
+
+ # Payment/Story reference organizations by FK but Organization declares no
+ # inverse has_many, so they are only reachable via the belongs_to scan. Their
+ # DB foreign keys would otherwise block the destroy.
+ it "reassigns FK children that have no inverse has_many on the model" do
+ news = create(:community_news, organization: dupe)
+ story = create(:story, organization: dupe)
+
+ expect { service.merge(keep, dupe) }.not_to raise_error
+ expect(news.reload.organization_id).to eq(keep.id)
+ expect(story.reload.organization_id).to eq(keep.id)
+ end
+ end
+
+ describe "#unhandled_references (coverage safeguard)" do
+ subject(:service) { described_class.new(model_class: Organization) }
+
+ let(:org) { create(:organization, name: "Solo Org") }
+
+ it "is empty when every reference is reassignable" do
+ create(:affiliation, organization: org)
+ create(:report, organization: org)
+
+ expect(service.unhandled_references(org)).to eq([])
+ end
+
+ it "flags a polymorphic reference the deduper does not reassign" do
+ # OtherResponse#owner is covered, but #promotable is not โ a promotable
+ # pointing at the org would be orphaned on merge.
+ create(:other_response, promotable: org)
+
+ expect(service.unhandled_references(org))
+ .to include(a_hash_including(table: "other_responses", column: "promotable_id"))
+ end
+
+ it "ignores gem-managed infrastructure that references the org" do
+ blob = ActiveStorage::Blob.create_before_direct_upload!(
+ filename: "logo.png", byte_size: 1, checksum: "x", content_type: "image/png"
+ )
+ ActiveStorage::Attachment.create!(name: "logo", record: org, blob: blob)
+
+ tables = service.unhandled_references(org).map { |ref| ref[:table] }
+ expect(tables).not_to include("active_storage_attachments")
+ end
+ end
+
+ describe "analytics/audit reassignment" do
+ subject(:service) { described_class.new(model_class: Organization, logger: logger, dry_run: false) }
+
+ let!(:keep) { create(:organization, name: "Keep Org") }
+ let!(:dupe) { create(:organization, name: "Dupe Org") }
+ let(:ahoy_events) { Class.new(ActiveRecord::Base) { self.table_name = "ahoy_events" } }
+ let(:versions) { Class.new(ActiveRecord::Base) { self.table_name = "versions" } }
+
+ it "repoints ahoy events to the kept organization" do
+ event = ahoy_events.create!(name: "view", resource_type: "Organization", resource_id: dupe.id, time: Time.current)
+
+ service.merge(keep, dupe)
+ expect(event.reload.resource_id).to eq(keep.id)
+ end
+
+ it "repoints paper_trail versions to the kept organization" do
+ version = versions.create!(event: "update", item_type: "Organization", item_id: dupe.id)
+
+ service.merge(keep, dupe)
+ expect(version.reload.item_id).to eq(keep.id)
+ end
+ end
+
+ describe "#lost_references" do
+ subject(:service) { described_class.new(model_class: Organization) }
+
+ it "reports attached files that are deleted with the record" do
+ org = create(:organization)
+ blob = ActiveStorage::Blob.create_before_direct_upload!(
+ filename: "logo.png", byte_size: 1, checksum: "x", content_type: "image/png"
+ )
+ ActiveStorage::Attachment.create!(name: "logo", record: org, blob: blob)
+
+ expect(service.lost_references(org)).to include(a_hash_including(count: 1))
+ end
+
+ it "is empty when the record has no attached files" do
+ expect(service.lost_references(create(:organization))).to eq([])
+ end
+ end
end
diff --git a/spec/services/organization_services/duplicate_finder_spec.rb b/spec/services/organization_services/duplicate_finder_spec.rb
new file mode 100644
index 0000000000..cb7b4c7d8c
--- /dev/null
+++ b/spec/services/organization_services/duplicate_finder_spec.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe OrganizationServices::DuplicateFinder do
+ def org(attrs = {})
+ create(:organization, attrs)
+ end
+
+ describe "#groups" do
+ it "groups organizations with the same normalized name" do
+ a = org(name: "Hope Center")
+ b = org(name: "hope center")
+ org(name: "Unrelated")
+
+ groups = described_class.new.groups
+
+ expect(groups.size).to eq(1)
+ expect(groups.first.records).to contain_exactly(a, b)
+ expect(groups.first.reasons).to include("Same name")
+ end
+
+ it "groups names that match once a legal suffix is removed" do
+ a = org(name: "Bright Futures Inc")
+ b = org(name: "Bright Futures LLC")
+
+ group = described_class.new.groups.first
+
+ expect(group.records).to contain_exactly(a, b)
+ expect(group.reasons).to include("Same name aside from Inc/LLC/etc.")
+ end
+
+ it "flags a FileMaker code present on only one record" do
+ org(name: "Sunrise", filemaker_code: "FM123")
+ org(name: "Sunrise", filemaker_code: nil)
+
+ expect(described_class.new.groups.first.reasons).to include("FileMaker code on only one record")
+ end
+
+ it "surfaces multiple FileMaker codes as a keep-all conflict" do
+ org(name: "Rivertown", filemaker_code: "FM1")
+ org(name: "Rivertown", filemaker_code: "FM2")
+
+ expect(described_class.new.groups.first.reasons)
+ .to include(a_string_matching(/Multiple FileMaker codes.*merging keeps all/))
+ end
+
+ it "groups records that share one code in a comma-separated list" do
+ a = org(name: "Lakeside A", filemaker_code: "FM1, FM2")
+ b = org(name: "Lakeside B", filemaker_code: " FM2 ")
+
+ group = described_class.new.groups.first
+
+ expect(group.records).to contain_exactly(a, b)
+ expect(group.reasons).to include(a_string_matching(/Multiple FileMaker codes \(FM1, FM2\)/))
+ end
+
+ it "groups organizations that share an address" do
+ a = org(name: "Alpha")
+ b = org(name: "Beta")
+ create(:address, addressable: a, street_address: "1 Main St", city: "Townsville", zip_code: "10001")
+ create(:address, addressable: b, street_address: "1 Main St", city: "Townsville", zip_code: "10001")
+
+ group = described_class.new.groups.first
+
+ expect(group.records).to contain_exactly(a, b)
+ expect(group.reasons).to include("Shared address")
+ end
+
+ it "returns nothing when there are no duplicates" do
+ org(name: "Only One")
+
+ expect(described_class.new.groups).to be_empty
+ end
+ end
+end