From 6b424274b08454d7b8cfd6a4794ae342c2eeed06 Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 09:56:49 -0400 Subject: [PATCH 1/9] Add organization deduper on the existing dedup framework (festi-inspired) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend ModelDeduper to reassign every FK reference to a model (not just polymorphic tagging joins), unioning declared has_many/:as associations with a schema-driven scan of DB foreign keys so FK children without an inverse has_many (payments, stories, monthly_reports, …) are reassigned rather than orphaned or blocked by their FK constraint. Collisions are resolved from DB unique indexes. Add OrganizationServices::DuplicateFinder (name, legal-suffix, FileMaker-code, and address clustering, with FileMaker treated as source of truth) and wire OrganizationsController into the Dedupable concern with a custom candidate finder. The index now renders annotated candidate groups with per-pair preview links. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controllers/concerns/dedupable.rb | 19 ++- app/controllers/organizations_controller.rb | 19 ++- app/services/model_deduper.rb | 129 ++++++++++++++---- .../organization_services/duplicate_finder.rb | 119 ++++++++++++++++ app/views/dedupes/index.html.erb | 52 +++++-- config/routes.rb | 4 + spec/requests/dedupable_spec.rb | 65 +++++++++ spec/services/model_deduper_spec.rb | 62 +++++++++ .../duplicate_finder_spec.rb | 66 +++++++++ 9 files changed, 488 insertions(+), 47 deletions(-) create mode 100644 app/services/organization_services/duplicate_finder.rb create mode 100644 spec/services/organization_services/duplicate_finder_spec.rb diff --git a/app/controllers/concerns/dedupable.rb b/app/controllers/concerns/dedupable.rb index e2f5c552d4..1563593954 100644 --- a/app/controllers/concerns/dedupable.rb +++ b/app/controllers/concerns/dedupable.rb @@ -3,13 +3,14 @@ module Dedupable extend ActiveSupport::Concern + CandidateGroup = Struct.new(:label, :records, :reasons, keyword_init: true) + def dedupe_index authorize! config = dedupe_config mc = config[:model_class] - groups = mc.all.group_by { |r| r.name.to_s.strip.downcase } - @possible_duplicates = groups.select { |_name, records| records.size > 1 } + @possible_duplicate_groups = dedupe_candidate_groups(config) @records_for_select = mc.order(:name).map { |r| [ r.name, r.id ] } @dedupe = build_dedupe_vars(config) @@ -116,6 +117,20 @@ def dedupe_config raise NotImplementedError, "#{self.class} must implement #dedupe_config" end + # Candidate duplicate groups for the index. A config may supply a + # :candidate_finder (a callable returning objects that respond to #label, + # #records, and #reasons — e.g. OrganizationServices::DuplicateFinder); without + # one, fall back to exact normalized-name grouping. + def dedupe_candidate_groups(config) + finder = config[:candidate_finder] + return Array(finder.call) if finder + + config[:model_class].all + .group_by { |record| record.name.to_s.strip.downcase } + .select { |_name, records| records.size > 1 } + .map { |name, records| CandidateGroup.new(label: name, records: records, reasons: []) } + end + # Returns [association_name, includes_name] for the primary polymorphic join. # e.g. [:categorizable_items, :categorizable] def dedupe_primary_join(mc) diff --git a/app/controllers/organizations_controller.rb b/app/controllers/organizations_controller.rb index 8c234862dc..5675d5f9e6 100644 --- a/app/controllers/organizations_controller.rb +++ b/app/controllers/organizations_controller.rb @@ -1,5 +1,5 @@ class OrganizationsController < ApplicationController - include AhoyTracking, TagAssignable + include AhoyTracking, TagAssignable, Dedupable before_action :set_organization, only: [ :show, :edit, :update, :destroy, :populations_served ] def index @@ -283,6 +283,23 @@ def organization_params ) end + def dedupe_config + { + model_class: Organization, + domain: :organizations, + candidate_finder: -> { OrganizationServices::DuplicateFinder.new.groups }, + belongs_to_options: -> { + { + "organization_status_id" => OrganizationStatus.order(:name), + "organization_obligation_id" => OrganizationObligation.order(:name) + } + }, + record_extras: ->(org) { + [ org.filemaker_code.presence && "FileMaker #{org.filemaker_code}", org.program_location ].compact.join(", ").presence + } + } + end + def find_duplicate_organizations(name) return [] if name.blank? diff --git a/app/services/model_deduper.rb b/app/services/model_deduper.rb index dda226325b..421b664b3a 100644 --- a/app/services/model_deduper.rb +++ b/app/services/model_deduper.rb @@ -38,37 +38,109 @@ def merge(record_to_keep, record_to_delete) private - # Auto-detect all polymorphic join associations on the model. - # Returns an array of hashes, each with :join_class, :foreign_key, - # :polymorphic_type_column, and :polymorphic_id_column. - def polymorphic_joins - @polymorphic_joins ||= model_class.reflect_on_all_associations(:has_many).filter_map do |assoc| - next if assoc.options[:through] # skip has_many :through + # Every association that references this model by a foreign key — whether a + # plain FK child (affiliations, reports), a polymorphic child (addresses via + # as: :addressable), or a tagging join (categorizable_items). Each is reassigned + # from the duplicate to the kept record on merge. Reflection-driven, so any + # FK-based model works with no bespoke config. + # + # Sources are unioned: this model's own has_many/:as declarations, plus a scan + # of every model for a non-polymorphic belongs_to pointing back here — so a + # child with an organization_id but no inverse has_many (payments, stories, …) + # is still reassigned rather than orphaned or blocked by its FK constraint. + # + # Collisions are resolved by DB *unique indexes*, not by guessing: the columns + # of a unique index that includes the FK are the scope in which only one row may + # exist (e.g. [category_id, categorizable_type, categorizable_id] → + # [category_id, categorizable_type]), so a row the kept record already has is + # deleted instead of violating the index. + def reassignable_joins + @reassignable_joins ||= begin + seen = Set.new + (has_many_joins + belongs_to_joins).select do |join| + seen.add?([ join[:join_class].table_name, join[:foreign_key] ]) + end + end + end + + def has_many_joins + model_class.reflect_on_all_associations(:has_many).filter_map do |assoc| + next if assoc.options[:through] begin join_klass = assoc.klass rescue NameError next end - poly = join_klass.reflect_on_all_associations(:belongs_to).find(&:polymorphic?) - next unless poly + fk = assoc.foreign_key.to_s + next unless join_klass.column_names.include?(fk) - { - join_class: join_klass, - foreign_key: assoc.foreign_key.to_sym, - polymorphic_type_column: poly.foreign_type.to_sym, - polymorphic_id_column: poly.foreign_key.to_sym - } + join_for(join_klass, fk) end end + # Schema-driven: every DB foreign key pointing at this model's table, so a child + # with an organization_id but no inverse has_many (payments, stories, …) is still + # reassigned. Reading the schema avoids eager-loading every model (which would + # touch databases not configured in every environment). + def belongs_to_joins + connection = model_class.connection + connection.tables.flat_map do |table| + next [] if table == model_class.table_name + + connection.foreign_keys(table).filter_map do |fk_def| + next unless fk_def.to_table == model_class.table_name + + join_for(model_for_table(table), fk_def.column.to_s) + end + end + rescue NotImplementedError + [] + end + + # The model backing a table, or an anonymous one bound to it. Guards against a + # name that resolves to a class mapped elsewhere (e.g. an STI child of another + # table), which would reassign the wrong table. + def model_for_table(table) + klass = table.classify.constantize + return klass if klass.table_name == table + + anonymous_model_for(table) + rescue NameError + anonymous_model_for(table) + end + + def anonymous_model_for(table) + klass = Class.new(ApplicationRecord) + klass.table_name = table + klass + end + + def join_for(join_klass, fk) + { + join_class: join_klass, + foreign_key: fk.to_sym, + natural_key: natural_key_columns(join_klass, fk) + } + end + + # Columns (other than the FK) of a unique index that includes the FK. + def natural_key_columns(join_klass, fk) + join_klass.connection.indexes(join_klass.table_name) + .select(&:unique) + .map { |index| Array(index.columns) } + .select { |cols| cols.include?(fk) } + .map { |cols| (cols - [ fk ]).map(&:to_sym) } + .find(&:present?) || [] + end + def model_label model_class.name.underscore.humanize.downcase end def usage_counts counts = Hash.new(0) - polymorphic_joins.each do |join| + reassignable_joins.each do |join| join[:join_class].where(join[:foreign_key] => model_class.pluck(:id)) .group(join[:foreign_key]).count .each { |id, count| counts[id] += count } @@ -104,11 +176,11 @@ def merge_duplicate(primary, dupe, usage) return if dry_run ActiveRecord::Base.transaction do - polymorphic_joins.each do |join| + reassignable_joins.each do |join| merge_join(primary, dupe, join) end - dupe.destroy! + dupe.reload.destroy! logger.info " deleted #{model_label} #{dupe.id}" end end @@ -116,25 +188,24 @@ def merge_duplicate(primary, dupe, usage) def merge_join(primary, dupe, join) jc = join[:join_class] fk = join[:foreign_key] - type_col = join[:polymorphic_type_column] - id_col = join[:polymorphic_id_column] + natural_key = join[:natural_key] - existing_taggings = jc - .where(fk => primary.id) - .pluck(type_col, id_col) - .map { |type, id| "#{type}_#{id}" } - .to_set - - items_to_move = jc.where(fk => dupe.id) + if natural_key.empty? + moved = jc.where(fk => dupe.id).update_all(fk => primary.id) + logger.info " moved #{moved} #{jc.name} to primary" if moved > 0 + return + end - items_to_move.find_each do |item| - tagging_key = "#{item.public_send(type_col)}_#{item.public_send(id_col)}" + existing = jc.where(fk => primary.id).pluck(*natural_key).map { |values| Array(values) }.to_set - if existing_taggings.include?(tagging_key) + jc.where(fk => dupe.id).find_each do |item| + key = natural_key.map { |col| item.public_send(col) } + if existing.include?(key) item.destroy! logger.info " deleted duplicate #{jc.name} #{item.id} (primary already has it)" else item.update!(fk => primary.id) + existing << key logger.info " moved #{jc.name} #{item.id} to primary" end end diff --git a/app/services/organization_services/duplicate_finder.rb b/app/services/organization_services/duplicate_finder.rb new file mode 100644 index 0000000000..de596fde8a --- /dev/null +++ b/app/services/organization_services/duplicate_finder.rb @@ -0,0 +1,119 @@ +require "set" + +module OrganizationServices + # Surfaces likely-duplicate organizations for the deduper's candidate list. + # Clusters organizations by several signals — normalized name, name minus a + # legal suffix (Inc/LLC/…), FileMaker code, and shared address — then annotates + # each cluster with why it was flagged. FileMaker is the source of truth, so a + # cluster whose members carry different FileMaker codes is surfaced as a + # conflict for an admin to resolve, never silently treated as a clean merge. + class DuplicateFinder + SOURCE_OF_TRUTH_FIELD = :filemaker_code + LEGAL_SUFFIX = /\b(?:incorporated|inc|llc|l\.l\.c|corporation|corp|company|co|limited|ltd)\b\.?/i + + Group = Struct.new(:key, :label, :records, :reasons, keyword_init: true) + + def initialize(scope = Organization.all) + @organizations = scope.includes(:addresses).to_a + end + + def groups + union = UnionFind.new(@organizations.map(&:id)) + cluster(union) { |org| [ normalized_name(org).presence ].compact } + cluster(union) { |org| [ suffixless_name(org).presence ].compact } + cluster(union) { |org| [ org.public_send(SOURCE_OF_TRUTH_FIELD).presence ].compact } + cluster(union) { |org| address_keys(org) } + + by_id = @organizations.index_by(&:id) + union.components.filter_map do |ids| + next if ids.size < 2 + records = ids.map { |id| by_id[id] }.sort_by(&:id) + Group.new( + key: records.map(&:id).join("-"), + label: records.first.name.to_s, + records: records, + reasons: reasons_for(records) + ) + end.sort_by { |group| group.label.to_s.downcase } + end + + private + + def cluster(union) + buckets = Hash.new { |hash, key| hash[key] = [] } + @organizations.each do |org| + yield(org).each { |key| buckets[key] << org.id } + end + buckets.each_value { |ids| union.union_all(ids) if ids.size > 1 } + end + + def reasons_for(records) + reasons = [] + normalized = records.map { |org| normalized_name(org) }.uniq + reasons << "Same name" if normalized.size == 1 + + suffixless = records.map { |org| suffixless_name(org) }.reject(&:blank?).uniq + reasons << "Same name aside from Inc/LLC/etc." if normalized.size > 1 && suffixless.size == 1 + + reasons.concat(filemaker_reasons(records)) + + keys = records.flat_map { |org| address_keys(org) } + reasons << "Shared address" if keys.tally.any? { |_key, count| count > 1 } + reasons + end + + def filemaker_reasons(records) + codes = records.map { |org| org.public_send(SOURCE_OF_TRUTH_FIELD).presence } + present = codes.compact.uniq + return [ "⚠ Different FileMaker codes (#{present.join(", ")}) — choose which to keep" ] if present.size > 1 + return [] if present.empty? + return [ "FileMaker code on only one record" ] if codes.any?(&:blank?) + + [ "Same FileMaker code (#{present.first})" ] + end + + def normalized_name(org) + org.name.to_s.downcase.gsub(/[^a-z0-9 ]/, " ").squish + end + + def suffixless_name(org) + normalized_name(org).gsub(LEGAL_SUFFIX, "").squish + end + + def address_keys(org) + org.addresses.reject(&:inactive?).filter_map do |address| + parts = [ address.street_address, address.city, address.zip_code ].map { |part| part.to_s.strip.downcase } + next if parts.all?(&:blank?) + parts.join("|") + end + end + + # Minimal disjoint-set: clusters ids connected by any shared signal into + # components, so an org linked by name to one dupe and by address to another + # lands in a single group. + class UnionFind + def initialize(ids) + @parent = ids.to_h { |id| [ id, id ] } + end + + def union_all(ids) + ids.each { |id| union(ids.first, id) } + end + + def components + @parent.keys.group_by { |id| find(id) }.values + end + + private + + def find(id) + id = @parent[id] while @parent[id] != id + id + end + + def union(a, b) + @parent[find(a)] = find(b) + end + end + end +end diff --git a/app/views/dedupes/index.html.erb b/app/views/dedupes/index.html.erb index a23e87b428..46b3dc8cee 100644 --- a/app/views/dedupes/index.html.erb +++ b/app/views/dedupes/index.html.erb @@ -61,27 +61,49 @@ <% end %> - <% if @possible_duplicates.any? %> + <% if @possible_duplicate_groups.any? %>

- Possible Duplicate Groups (<%= @possible_duplicates.count %>) + Possible Duplicate Groups (<%= @possible_duplicate_groups.count %>)

- <% @possible_duplicates.each do |normalized_name, records| %> + <% @possible_duplicate_groups.each do |group| %> + <% keep_record = group.records.first %>
-

- "<%= normalized_name %>" (<%= records.count %> <%= @dedupe[:model_label_plural].downcase %>) -

-
    - <% records.each do |record| %> -
  • - <%= record.name %> - - (ID: <%= record.id %>, - <% if @dedupe[:record_extras] %><%= @dedupe[:record_extras].call(record) %>, <% end %> - <%= record.published? ? 'Published' : 'Unpublished' %>, - <%= record.public_send(@dedupe[:join_association]).count %> taggings) +
    +

    + "<%= group.label %>" (<%= group.records.count %> <%= @dedupe[:model_label_plural].downcase %>) +

    +
    + <% if group.reasons.any? %> +
    + <% group.reasons.each do |reason| %> + + <%= reason %> + <% end %> +
    + <% end %> +
      + <% group.records.each do |record| %> +
    • + + <%= record.name %> + + (ID: <%= record.id %>, + <% if @dedupe[:record_extras] %><%= @dedupe[:record_extras].call(record) %>, <% end %> + <%= record.published? ? 'Published' : 'Unpublished' %>, + <%= record.public_send(@dedupe[:join_association]).count %> taggings) + + + <% if record != keep_record %> + <%= link_to "Preview merge →", + url_for(action: :dedupe_preview, + @dedupe[:delete_id_param] => record.id, + @dedupe[:keep_id_param] => keep_record.id), + data: { turbo: false }, + class: "text-sm #{DomainTheme.text_class_for(@dedupe[:domain], intensity: 700)} hover:underline whitespace-nowrap" %> + <% end %>
    • <% end %>
    diff --git a/config/routes.rb b/config/routes.rb index 7f05bcfcd9..0296f01110 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -277,6 +277,10 @@ resources :organizations do collection do get :check_duplicates + get :dedupe_index + get :dedupe_preview + post :dedupe_perform + patch :dedupe_update_keep end member do get :populations_served diff --git a/spec/requests/dedupable_spec.rb b/spec/requests/dedupable_spec.rb index 5fe06cb260..4d3ca4d586 100644 --- a/spec/requests/dedupable_spec.rb +++ b/spec/requests/dedupable_spec.rb @@ -303,4 +303,69 @@ end end end + + # ============================================================ + # ORGANIZATIONS — FK-based model with a custom candidate finder + # ============================================================ + + describe "Organizations" do + before { sign_in admin } + + describe "GET dedupe_index" do + it "renders and surfaces candidate groups from the duplicate finder" do + create(:organization, name: "Hope Center") + create(:organization, name: "hope center") + + get dedupe_index_organizations_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Same name") + end + end + + describe "GET dedupe_preview" do + let!(:keep) { create(:organization, name: "Keep Org") } + let!(:delete_rec) { create(:organization, name: "Delete Org") } + + it "renders the preview page" do + get dedupe_preview_organizations_path( + organization_to_keep_id: keep.id, + organization_to_delete_id: delete_rec.id + ) + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Keep Org") + expect(response.body).to include("Delete Org") + end + end + + describe "POST dedupe_perform" do + let!(:keep) { create(:organization, name: "Keeper Org") } + let!(:delete_rec) { create(:organization, name: "Duplicate Org") } + let!(:report) { create(:report, organization: delete_rec) } + + it "merges, reassigns FK associations, and deletes the duplicate" do + expect { + post dedupe_perform_organizations_path, params: { + organization_to_delete_id: delete_rec.id, + organization_to_keep_id: keep.id + } + }.to change(Organization, :count).by(-1) + + expect(response).to redirect_to(organizations_path) + expect(Organization.exists?(delete_rec.id)).to be false + expect(report.reload.organization_id).to eq(keep.id) + end + + it "applies keep-field edits before merging" do + post dedupe_perform_organizations_path, params: { + organization_to_delete_id: delete_rec.id, + organization_to_keep_id: keep.id, + organization_to_keep: { name: "Canonical Org" } + } + + expect(keep.reload.name).to eq("Canonical Org") + end + end + end end diff --git a/spec/services/model_deduper_spec.rb b/spec/services/model_deduper_spec.rb index 4f0fb422ee..33be8418f6 100644 --- a/spec/services/model_deduper_spec.rb +++ b/spec/services/model_deduper_spec.rb @@ -117,4 +117,66 @@ join_factory: :sectorable_item, join_fk: :sector_id end + + # Organizations are referenced by plain organization_id foreign keys rather than + # polymorphic joins, so they exercise the reflection-driven FK reassignment pass. + describe "foreign-key reassignment (Organization)" do + subject(:service) do + described_class.new(model_class: Organization, logger: logger, dry_run: false) + end + + let!(:keep) { create(:organization, name: "Keep Org") } + let!(:dupe) { create(:organization, name: "Dupe Org") } + + it "reassigns plain FK associations to the kept organization" do + report = create(:report, organization: dupe) + service.merge(keep, dupe) + expect(report.reload.organization_id).to eq(keep.id) + end + + it "reassigns restrict_with_error associations and still deletes the duplicate" do + affiliation = create(:affiliation, organization: dupe) + + expect { service.merge(keep, dupe) } + .to change { Organization.exists?(dupe.id) }.from(true).to(false) + expect(affiliation.reload.organization_id).to eq(keep.id) + end + + it "moves a non-colliding event registration link to the kept organization" do + registration = create(:event_registration) + link = create(:event_registration_organization, organization: dupe, event_registration: registration) + + service.merge(keep, dupe) + expect(link.reload.organization_id).to eq(keep.id) + end + + it "collapses a natural-key collision instead of violating the unique index" do + registration = create(:event_registration) + create(:event_registration_organization, organization: keep, event_registration: registration) + colliding = create(:event_registration_organization, organization: dupe, event_registration: registration) + + expect { service.merge(keep, dupe) } + .to change(EventRegistrationOrganization, :count).by(-1) + expect(EventRegistrationOrganization.exists?(colliding.id)).to be false + expect(EventRegistrationOrganization.where(organization_id: keep.id, event_registration_id: registration.id).count).to eq(1) + end + + it "reassigns polymorphic joins alongside FK associations" do + bookmark = create(:bookmark, bookmarkable: dupe) + service.merge(keep, dupe) + expect(bookmark.reload.bookmarkable).to eq(keep) + end + + # Payment/Story reference organizations by FK but Organization declares no + # inverse has_many, so they are only reachable via the belongs_to scan. Their + # DB foreign keys would otherwise block the destroy. + it "reassigns FK children that have no inverse has_many on the model" do + news = create(:community_news, organization: dupe) + story = create(:story, organization: dupe) + + expect { service.merge(keep, dupe) }.not_to raise_error + expect(news.reload.organization_id).to eq(keep.id) + expect(story.reload.organization_id).to eq(keep.id) + end + end end diff --git a/spec/services/organization_services/duplicate_finder_spec.rb b/spec/services/organization_services/duplicate_finder_spec.rb new file mode 100644 index 0000000000..aea6844556 --- /dev/null +++ b/spec/services/organization_services/duplicate_finder_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe OrganizationServices::DuplicateFinder do + def org(attrs = {}) + create(:organization, attrs) + end + + describe "#groups" do + it "groups organizations with the same normalized name" do + a = org(name: "Hope Center") + b = org(name: "hope center") + org(name: "Unrelated") + + groups = described_class.new.groups + + expect(groups.size).to eq(1) + expect(groups.first.records).to contain_exactly(a, b) + expect(groups.first.reasons).to include("Same name") + end + + it "groups names that match once a legal suffix is removed" do + a = org(name: "Bright Futures Inc") + b = org(name: "Bright Futures LLC") + + group = described_class.new.groups.first + + expect(group.records).to contain_exactly(a, b) + expect(group.reasons).to include("Same name aside from Inc/LLC/etc.") + end + + it "flags a FileMaker code present on only one record" do + org(name: "Sunrise", filemaker_code: "FM123") + org(name: "Sunrise", filemaker_code: nil) + + expect(described_class.new.groups.first.reasons).to include("FileMaker code on only one record") + end + + it "surfaces conflicting FileMaker codes as a choose-which conflict" do + org(name: "Rivertown", filemaker_code: "FM1") + org(name: "Rivertown", filemaker_code: "FM2") + + expect(described_class.new.groups.first.reasons) + .to include(a_string_matching(/Different FileMaker codes/)) + end + + it "groups organizations that share an address" do + a = org(name: "Alpha") + b = org(name: "Beta") + create(:address, addressable: a, street_address: "1 Main St", city: "Townsville", zip_code: "10001") + create(:address, addressable: b, street_address: "1 Main St", city: "Townsville", zip_code: "10001") + + group = described_class.new.groups.first + + expect(group.records).to contain_exactly(a, b) + expect(group.reasons).to include("Shared address") + end + + it "returns nothing when there are no duplicates" do + org(name: "Only One") + + expect(described_class.new.groups).to be_empty + end + end +end From 03f66c3e6acff556221274fd3939d4917c3b7917 Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 10:05:06 -0400 Subject: [PATCH 2/9] Second pass: bring the deduper UI into awbw style Replace the generic festi-style preview with awbw conventions: DomainTheme colors, sentence-case copy, btn classes, eyebrow nav, and the standard blue form-focus. The preview now shows a meaningful reassignment summary (counts of each association that moves) instead of a single polymorphic join's items, and orgs edit only a curated set of fields rather than every column. Index groups gain reason chips, a suggested-keeper badge, and per-pair preview links. Add a Dedupe entry point on the Organizations page and a Features & tips entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controllers/concerns/dedupable.rb | 33 +-- app/controllers/organizations_controller.rb | 6 +- app/services/model_deduper.rb | 16 ++ app/views/dedupes/_preview.html.erb | 173 ++++++-------- .../dedupes/_reassignment_summary.html.erb | 15 ++ app/views/dedupes/index.html.erb | 211 ++++++++---------- app/views/dedupes/preview.html.erb | 123 +++++----- app/views/organizations/index.html.erb | 20 +- config/features.yml | 25 +++ spec/requests/dedupable_spec.rb | 9 +- 10 files changed, 305 insertions(+), 326 deletions(-) create mode 100644 app/views/dedupes/_reassignment_summary.html.erb diff --git a/app/controllers/concerns/dedupable.rb b/app/controllers/concerns/dedupable.rb index 1563593954..8c61e006bf 100644 --- a/app/controllers/concerns/dedupable.rb +++ b/app/controllers/concerns/dedupable.rb @@ -39,9 +39,9 @@ def dedupe_preview return redirect_to url_for(action: :dedupe_index), alert: "#{mc.model_name.human} not found (ID: #{missing.join(', ')})." end - join_assoc, join_incl = dedupe_primary_join(mc) - @delete_items = @record_to_delete.public_send(join_assoc).includes(join_incl) - @keep_items = @record_to_keep.public_send(join_assoc).includes(join_incl) + deduper = ModelDeduper.new(model_class: mc) + @reassignment_delete = deduper.reassignment_counts(@record_to_delete) + @reassignment_keep = deduper.reassignment_counts(@record_to_keep) @dedupe = build_dedupe_vars(config) render "dedupes/preview" @@ -83,17 +83,18 @@ def dedupe_perform record_to_keep.update!(params.require(keep_param_key).permit(editable)) end + deduper = ModelDeduper.new(model_class: mc, logger: Rails.logger, dry_run: false, min_usage: 0) + if respond_to?(:track_event, true) track_event("dedupe.#{mn}", { resource_type: mc.name, resource_id: record_to_keep.id, deleted_record: record_to_delete.attributes, kept_record: { id: record_to_keep.id, name: record_to_keep.name }, - associations_moved: record_to_delete.public_send(dedupe_primary_join(mc).first).count + associations_moved: deduper.reassignment_counts(record_to_delete).values.sum }) end - deduper = ModelDeduper.new(model_class: mc, logger: Rails.logger, dry_run: false, min_usage: 0) deduper.merge(record_to_keep, record_to_delete) label = mc.model_name.human.pluralize @@ -131,27 +132,9 @@ def dedupe_candidate_groups(config) .map { |name, records| CandidateGroup.new(label: name, records: records, reasons: []) } end - # Returns [association_name, includes_name] for the primary polymorphic join. - # e.g. [:categorizable_items, :categorizable] - def dedupe_primary_join(mc) - assoc = mc.reflect_on_all_associations(:has_many).find do |a| - next if a.options[:through] - begin - a.klass.reflect_on_all_associations(:belongs_to).any?(&:polymorphic?) - rescue NameError - false - end - end - raise "No polymorphic join found for #{mc.name}" unless assoc - - poly = assoc.klass.reflect_on_all_associations(:belongs_to).find(&:polymorphic?) - [ assoc.name, poly.name ] - end - def build_dedupe_vars(config) mc = config[:model_class] mn = mc.model_name.singular - join_assoc, join_incl = dedupe_primary_join(mc) opts = config[:belongs_to_options] { @@ -162,9 +145,7 @@ def build_dedupe_vars(config) delete_id_param: "#{mn}_to_delete_id", keep_id_param: "#{mn}_to_keep_id", keep_param_key: "#{mn}_to_keep".to_sym, - item_type_col: "#{join_incl}_type".to_sym, - item_id_col: "#{join_incl}_id".to_sym, - join_association: join_assoc, + editable_columns: config[:editable_columns], belongs_to_options: opts.is_a?(Proc) ? opts.call : (opts || {}), record_extras: config[:record_extras] } diff --git a/app/controllers/organizations_controller.rb b/app/controllers/organizations_controller.rb index 5675d5f9e6..5251bb6d5d 100644 --- a/app/controllers/organizations_controller.rb +++ b/app/controllers/organizations_controller.rb @@ -288,6 +288,10 @@ def dedupe_config model_class: Organization, domain: :organizations, candidate_finder: -> { OrganizationServices::DuplicateFinder.new.groups }, + editable_columns: %w[ + name filemaker_code email website_url agency_type description + mission_vision_values notes organization_status_id organization_obligation_id + ], belongs_to_options: -> { { "organization_status_id" => OrganizationStatus.order(:name), @@ -295,7 +299,7 @@ def dedupe_config } }, record_extras: ->(org) { - [ org.filemaker_code.presence && "FileMaker #{org.filemaker_code}", org.program_location ].compact.join(", ").presence + [ org.filemaker_code.presence && "FileMaker #{org.filemaker_code}", org.program_location ].compact.join(" · ").presence } } end diff --git a/app/services/model_deduper.rb b/app/services/model_deduper.rb index 421b664b3a..f7160095b7 100644 --- a/app/services/model_deduper.rb +++ b/app/services/model_deduper.rb @@ -36,6 +36,18 @@ def merge(record_to_keep, record_to_delete) merge_duplicate(record_to_keep, record_to_delete, usage_counts) end + # A { human label => count } of the records that would be reassigned off this + # record on merge, skipping empty associations. Drives the preview so an admin + # sees exactly what moves (affiliations, event registrations, reports, …). + def reassignment_counts(record) + reassignable_joins.each_with_object({}) do |join, counts| + count = join[:join_class].where(join[:foreign_key] => record.id).count + next if count.zero? + + counts[join_label(join[:join_class])] = count + end + end + private # Every association that references this model by a foreign key — whether a @@ -138,6 +150,10 @@ def model_label model_class.name.underscore.humanize.downcase end + def join_label(join_class) + (join_class.name || join_class.table_name.classify).underscore.humanize.pluralize + end + def usage_counts counts = Hash.new(0) reassignable_joins.each do |join| diff --git a/app/views/dedupes/_preview.html.erb b/app/views/dedupes/_preview.html.erb index 532e5840d2..0145889324 100644 --- a/app/views/dedupes/_preview.html.erb +++ b/app/views/dedupes/_preview.html.erb @@ -1,155 +1,110 @@ -<%# Shared dedupe preview partial +<%# Shared dedupe preview partial. Required locals: - domain: Symbol (:categories or :sectors) - record_to_delete: The record that will be deleted - record_to_keep: The record that will be kept - delete_items: Associated items of the record to delete - keep_items: Associated items of the record to keep - item_type_col: Polymorphic type column (e.g. :categorizable_type) - item_id_col: Polymorphic id column (e.g. :categorizable_id) - model_label: Human label (e.g. "Category") - keep_param_key: Param key for keep fields (e.g. :category_to_keep) - form_builder: The parent form builder (f) - belongs_to_options: Hash of { column_name => collection } for select fields + domain: Symbol for DomainTheme (e.g. :organizations) + record_to_delete: The record that will be deleted + record_to_keep: The record that will be kept + reassignment_delete: { human label => count } of records moving off the deleted record + reassignment_keep: { human label => count } the kept record already has + model_label: Human label (e.g. "Organization") + keep_param_key: Param key for keep fields (e.g. :organization_to_keep) + editable_columns: Optional allowlist of column names to show/edit; nil shows all + form_builder: The parent form builder (f) + belongs_to_options: Hash of { column_name => collection } for select fields %> <% - color = DomainTheme.color_for(domain) + border = DomainTheme.border_class_for(domain, intensity: 200) + header_bg = DomainTheme.bg_class_for(domain, intensity: 50) model_class = record_to_delete.class skip_columns = %w[id created_at updated_at legacy_id] - columns = model_class.columns.reject { |c| skip_columns.include?(c.name) } - - # Resolve belongs_to associations for _id columns + columns = + if editable_columns.present? + editable_columns.filter_map { |name| model_class.columns_hash[name.to_s] } + else + model_class.columns.reject { |c| skip_columns.include?(c.name) } + end bt_associations = model_class.reflect_on_all_associations(:belongs_to).index_by { |a| a.foreign_key.to_s } belongs_to_options ||= {} - assoc_name = item_type_col.to_s.sub(/_type$/, '').to_sym %> - -
    - -
    + +
    - <%= model_label %> to delete: <%= record_to_delete.name %> + <%= model_label %> to delete: <%= record_to_delete.name %>
    -
    - - <%= model_label %> to keep: <%= record_to_keep.name %> +
    + + <%= model_label %> to keep: <%= record_to_keep.name %>
    -
    - ID - <%= record_to_delete.id %> +
    + ID <%= record_to_delete.id %>
    -
    - ID - <%= record_to_keep.id %> +
    + ID <%= record_to_keep.id %>
    - + <%= form_builder.fields_for keep_param_key, record_to_keep do |keep_f| %> <% columns.each do |col| %> - <% delete_val = record_to_delete.public_send(col.name) %> - <% keep_val = record_to_keep.public_send(col.name) %> - <% col_assoc = bt_associations[col.name] %> <% - if col_assoc - label = col.name.sub(/_id$/, '').titleize - display_val = record_to_delete.public_send(col_assoc.name)&.try(:name) || '(none)' - elsif col.type == :boolean - label = col.name.titleize - display_val = delete_val ? 'Yes' : 'No' - elsif col.type == :datetime - label = col.name.titleize - display_val = delete_val&.strftime("%Y-%m-%d %H:%M") || 'N/A' - else - label = col.name.titleize - display_val = delete_val - end + delete_val = record_to_delete.public_send(col.name) + keep_val = record_to_keep.public_send(col.name) + col_assoc = bt_associations[col.name] + label = col.name.humanize + display_val = + if col_assoc + record_to_delete.public_send(col_assoc.name)&.try(:name) || "(none)" + elsif col.type == :boolean + delete_val ? "Yes" : "No" + elsif col.type == :datetime + delete_val&.strftime("%Y-%m-%d %H:%M") || "—" + else + delete_val.presence || "—" + end is_different = delete_val != keep_val %> -
    - <%= label %> - <%= display_val %> +
    + <%= label %> + "><%= display_val %> <% if is_different %> - DIFFERENT + Will be dropped <% end %>
    -
    +
    + <% if col.type == :boolean %> <% elsif col_assoc && belongs_to_options[col.name].present? %> - - <%= keep_f.collection_select col.name.to_sym, belongs_to_options[col.name], :id, :name, {}, class: "ml-2 px-2 py-1 border border-gray-200 rounded text-gray-900 focus:ring-2 focus:ring-#{color}-500 focus:border-transparent" %> - <% elsif col.type == :integer %> - - <%= keep_f.number_field col.name.to_sym, class: "ml-2 w-20 px-2 py-1 border border-gray-200 rounded text-gray-900 focus:ring-2 focus:ring-#{color}-500 focus:border-transparent" %> + <%= keep_f.collection_select col.name.to_sym, belongs_to_options[col.name], :id, :name, {}, class: "w-full px-2 py-1 border border-gray-300 rounded text-gray-900 focus:border-blue-500 focus:ring-blue-200" %> + <% elsif col.type == :integer && col_assoc.nil? %> + <%= keep_f.number_field col.name.to_sym, class: "w-32 px-2 py-1 border border-gray-300 rounded text-gray-900 focus:border-blue-500 focus:ring-blue-200" %> + <% elsif col.type == :text %> + <%= keep_f.text_area col.name.to_sym, rows: 2, class: "w-full px-2 py-1 border border-gray-300 rounded text-gray-900 focus:border-blue-500 focus:ring-blue-200" %> + <% elsif col_assoc.nil? %> + <%= keep_f.text_field col.name.to_sym, class: "w-full px-2 py-1 border border-gray-300 rounded text-gray-900 #{"font-semibold" if col.name == "name"} focus:border-blue-500 focus:ring-blue-200" %> <% else %> - - <%= keep_f.text_field col.name.to_sym, class: "ml-2 px-2 py-1 border border-gray-200 rounded text-gray-900 #{col.name == 'name' ? 'font-semibold' : ''} focus:ring-2 focus:ring-#{color}-500 focus:border-transparent" %> + <%= record_to_keep.public_send(col_assoc.name)&.try(:name) || "(none)" %> <% end %>
    <% end %> <% end %> - -
    - Created - <%= record_to_delete.created_at&.strftime("%Y-%m-%d %H:%M") || 'N/A' %> -
    -
    - Created - <%= record_to_keep.created_at&.strftime("%Y-%m-%d %H:%M") || 'N/A' %> -
    - - -
    - Associated Records - <%= delete_items.count %> -
    -
    - Associated Records - <%= keep_items.count %> -
    - - -
    - <% if delete_items.any? %> -

    Tagged items

    -
      - <% delete_items.sort_by { |item| [item.public_send(item_type_col), (item.public_send(assoc_name).try(:name) || item.public_send(assoc_name).try(:title) || "").to_s.downcase] }.each do |item| %> -
    • - • <%= item.public_send(item_type_col) %>: - <%= item.public_send(assoc_name).try(:name) || item.public_send(assoc_name).try(:title) || "ID #{item.public_send(item_id_col)}" %> -
    • - <% end %> -
    - <% end %> -
    -
    - <% if keep_items.any? %> -

    Tagged items

    -
      - <% keep_items.sort_by { |item| [item.public_send(item_type_col), (item.public_send(assoc_name).try(:name) || item.public_send(assoc_name).try(:title) || "").to_s.downcase] }.each do |item| %> -
    • - • <%= item.public_send(item_type_col) %>: - <%= item.public_send(assoc_name).try(:name) || item.public_send(assoc_name).try(:title) || "ID #{item.public_send(item_id_col)}" %> -
    • - <% end %> -
    - <% end %> -
    + + <%= render "dedupes/reassignment_summary", border: border, title: "Records that will move", counts: reassignment_delete, empty: "Nothing to move" %> + <%= render "dedupes/reassignment_summary", border: border, title: "Records already here", counts: reassignment_keep, empty: "None yet" %>
    diff --git a/app/views/dedupes/_reassignment_summary.html.erb b/app/views/dedupes/_reassignment_summary.html.erb new file mode 100644 index 0000000000..6b8ecb13c3 --- /dev/null +++ b/app/views/dedupes/_reassignment_summary.html.erb @@ -0,0 +1,15 @@ +<%# One column of the reassignment summary. Locals: border, title, counts (hash), empty (string). %> +
    +

    <%= title %>

    + <% if counts.any? %> +
    + <% counts.sort_by { |label, _count| label }.each do |label, count| %> + + <%= label %> <%= count %> + + <% end %> +
    + <% else %> +

    <%= empty %>

    + <% end %> +
    diff --git a/app/views/dedupes/index.html.erb b/app/views/dedupes/index.html.erb index 46b3dc8cee..6619ae985e 100644 --- a/app/views/dedupes/index.html.erb +++ b/app/views/dedupes/index.html.erb @@ -1,124 +1,109 @@ -<% color = DomainTheme.color_for(@dedupe[:domain]) %> <% content_for(:page_bg_class, "admin-only bg-blue-100") %> -
    +
    +
    - -
    -

    - Dedupe <%= @dedupe[:model_label_plural] %> -

    -
    - <%= link_to url_for(action: :dedupe_index), - class: "px-4 py-2 btn btn-secondary-outline" do %> - - Possible dupes - <% end %> - <%= link_to url_for(action: :index), - class: "px-4 py-2 btn btn-secondary-outline" do %> - - <%= @dedupe[:model_label_plural] %> - <% end %> -
    -
    + +
    + <%= link_to "← #{@dedupe[:model_label_plural]}", url_for(action: :index), + class: "text-sm #{eyebrow_link_class} px-2 py-1" %> +

    + Dedupe <%= @dedupe[:model_label_plural].downcase %> +

    +

    + Pick a <%= @dedupe[:model_label].downcase %> to delete and one to keep — its + records move to the one you keep before it's removed. +

    +
    - - <%= form_with url: url_for(action: :dedupe_preview), method: :get, data: { turbo: false }, - class: "bg-#{color}-100 text-white rounded-lg p-6 space-y-4" do |f| %> -
    - -
    - - <%= f.select @dedupe[:delete_id_param].to_sym, - options_for_select([["-- #{@dedupe[:model_label].downcase} to delete --", ""]] + @records_for_select), - {}, - class: "w-full px-4 py-2 rounded border border-gray-300 focus:ring-2 focus:ring-#{color}-500 focus:border-transparent text-gray-900", - required: true %> -
    + + <%= form_with url: url_for(action: :dedupe_preview), method: :get, data: { turbo: false }, + class: "rounded-xl bg-white border border-gray-200 p-6 space-y-4" do |f| %> +
    +
    + + <%= f.select @dedupe[:delete_id_param].to_sym, + options_for_select([ [ "— #{@dedupe[:model_label].downcase} to delete —", "" ] ] + @records_for_select), + {}, + class: "w-full px-4 py-2 rounded border border-gray-300 focus:border-blue-500 focus:ring-blue-200 text-gray-900", + required: true %> +
    +
    + + <%= f.select @dedupe[:keep_id_param].to_sym, + options_for_select([ [ "— #{@dedupe[:model_label].downcase} to keep —", "" ] ] + @records_for_select), + {}, + class: "w-full px-4 py-2 rounded border border-gray-300 focus:border-blue-500 focus:ring-blue-200 text-gray-900", + required: true %> +
    +
    +
    + <%= link_to "Clear", url_for(action: :dedupe_index), + class: "btn btn-secondary-outline" %> + <%= f.submit "Preview merge", class: "btn btn-primary" %> +
    + <% end %> - -
    - - <%= f.select @dedupe[:keep_id_param].to_sym, - options_for_select([["-- #{@dedupe[:model_label].downcase} to keep --", ""]] + @records_for_select), - {}, - class: "w-full px-4 py-2 rounded border border-gray-300 focus:ring-2 focus:ring-#{color}-500 focus:border-transparent text-gray-900", - required: true %> + + <% if @possible_duplicate_groups.any? %> +
    +

    + Possible duplicate groups (<%= @possible_duplicate_groups.count %>) +

    +
    + <% @possible_duplicate_groups.each do |group| %> + <% keep_record = group.records.first %> +
    +
    +

    + <%= group.label.presence || "(no name)" %> + · <%= group.records.count %> <%= @dedupe[:model_label_plural].downcase %> +

    -
    - - -
    - <%= f.submit "Preview this dedupe", - class: "px-6 py-2 btn btn-primary font-semibold" %> - <%= link_to "Clear filters", - url_for(action: :dedupe_index), - class: "px-6 py-2 bg-gray-200 text-gray-700 rounded hover:bg-gray-300 transition-colors font-semibold" %> -
    - <% end %> - - - <% if @possible_duplicate_groups.any? %> -
    -

    - Possible Duplicate Groups (<%= @possible_duplicate_groups.count %>) -

    -
    - <% @possible_duplicate_groups.each do |group| %> - <% keep_record = group.records.first %> -
    -
    -

    - "<%= group.label %>" (<%= group.records.count %> <%= @dedupe[:model_label_plural].downcase %>) -

    -
    - <% if group.reasons.any? %> -
    - <% group.reasons.each do |reason| %> - - <%= reason %> - - <% end %> -
    + <% if group.reasons.any? %> +
    + <% group.reasons.each do |reason| %> + rounded-full border px-2 py-0.5"> + <%= reason %> + + <% end %> +
    + <% end %> +
      + <% group.records.each do |record| %> +
    • + + <%= record.name.presence || "(no name)" %> + + · ID <%= record.id %><% if @dedupe[:record_extras] && (extra = @dedupe[:record_extras].call(record)).present? %> · <%= extra %><% end %> + + + <% if record == keep_record %> + Suggested keeper + <% else %> + <%= link_to "Preview merge →", + url_for(action: :dedupe_preview, + @dedupe[:delete_id_param] => record.id, + @dedupe[:keep_id_param] => keep_record.id), + data: { turbo: false }, + class: "text-sm #{DomainTheme.text_class_for(@dedupe[:domain], intensity: 700)} hover:underline whitespace-nowrap" %> <% end %> -
        - <% group.records.each do |record| %> -
      • - - <%= record.name %> - - (ID: <%= record.id %>, - <% if @dedupe[:record_extras] %><%= @dedupe[:record_extras].call(record) %>, <% end %> - <%= record.published? ? 'Published' : 'Unpublished' %>, - <%= record.public_send(@dedupe[:join_association]).count %> taggings) - - - <% if record != keep_record %> - <%= link_to "Preview merge →", - url_for(action: :dedupe_preview, - @dedupe[:delete_id_param] => record.id, - @dedupe[:keep_id_param] => keep_record.id), - data: { turbo: false }, - class: "text-sm #{DomainTheme.text_class_for(@dedupe[:domain], intensity: 700)} hover:underline whitespace-nowrap" %> - <% end %> -
      • - <% end %> -
      -
    +
  • <% end %> -
-
- <% else %> -
- -

- No duplicate <%= @dedupe[:model_label_plural].downcase %> detected! -

+
<% end %> -
+ + <% else %> +
+ +

No likely duplicate <%= @dedupe[:model_label_plural].downcase %> detected.

+
+ <% end %> + + diff --git a/app/views/dedupes/preview.html.erb b/app/views/dedupes/preview.html.erb index 3b958f115a..95bfc1ddbc 100644 --- a/app/views/dedupes/preview.html.erb +++ b/app/views/dedupes/preview.html.erb @@ -1,75 +1,64 @@ -<% color = DomainTheme.color_for(@dedupe[:domain]) %> <% content_for(:page_bg_class, "admin-only bg-blue-100") %> -
+
+
- -
-
-

- Preview <%= @dedupe[:model_label] %> Deduplication -

-

- Review the differences before merging. -

-

- The <%= @dedupe[:model_label].downcase %> on the left will be deleted and its associations reassigned to the right. -

-
-
- <%= link_to url_for(action: :dedupe_preview, @dedupe[:delete_id_param] => @record_to_keep.id, @dedupe[:keep_id_param] => @record_to_delete.id), - class: "px-4 py-2 bg-gray-200 text-gray-700 rounded hover:bg-gray-300 transition-colors inline-flex items-center gap-2 text-sm font-medium" do %> - Swap - <% end %> - <%= link_to url_for(action: :dedupe_index), - class: "px-4 py-2 bg-#{color}-600 text-white rounded hover:bg-#{color}-700 transition-colors inline-flex items-center gap-2 text-sm font-medium" do %> - Back to Dedupe - <% end %> -
-
- - - <%= form_with url: url_for(action: :dedupe_perform), method: :post, - data: { turbo: false, - turbo_confirm: "Are you sure you want to merge these #{@dedupe[:model_label_plural].downcase}? This action cannot be undone!" } do |f| %> - <%= f.hidden_field @dedupe[:delete_id_param].to_sym, value: @record_to_delete.id %> - <%= f.hidden_field @dedupe[:keep_id_param].to_sym, value: @record_to_keep.id %> - - <%= render "dedupes/preview", - domain: @dedupe[:domain], - record_to_delete: @record_to_delete, - record_to_keep: @record_to_keep, - delete_items: @delete_items, - keep_items: @keep_items, - item_type_col: @dedupe[:item_type_col], - item_id_col: @dedupe[:item_id_col], - model_label: @dedupe[:model_label], - keep_param_key: @dedupe[:keep_param_key], - form_builder: f, - belongs_to_options: @dedupe[:belongs_to_options] %> + +
+
+ <%= link_to "← Back to dedupe", url_for(action: :dedupe_index), + class: "text-sm #{eyebrow_link_class} px-2 py-1" %> +

+ Preview <%= @dedupe[:model_label].downcase %> merge +

+

+ The <%= @dedupe[:model_label].downcase %> on the left is deleted; its records + move to the one on the right. Edit the keeper's fields before merging. +

+
+ <%= link_to url_for(action: :dedupe_preview, @dedupe[:delete_id_param] => @record_to_keep.id, @dedupe[:keep_id_param] => @record_to_delete.id), + class: "btn btn-secondary-outline btn-sm shrink-0" do %> + Swap + <% end %> +
- -
-
- -

- Danger: This action cannot be undone. - "<%= @record_to_delete.name %>" (ID: <%= @record_to_delete.id %>) will be permanently deleted - and its <%= @delete_items.count %> associations moved to - "<%= @record_to_keep.name %>" (ID: <%= @record_to_keep.id %>). -

-
-
+ <%= form_with url: url_for(action: :dedupe_perform), method: :post, + data: { turbo: false, + turbo_confirm: "Merge these #{@dedupe[:model_label_plural].downcase}? This can't be undone." } do |f| %> + <%= f.hidden_field @dedupe[:delete_id_param].to_sym, value: @record_to_delete.id %> + <%= f.hidden_field @dedupe[:keep_id_param].to_sym, value: @record_to_keep.id %> - -
- <%= f.submit "Perform merge", - class: "px-6 py-3 bg-red-600 text-white rounded-lg hover:bg-red-700 transition-colors font-semibold" %> + <%= render "dedupes/preview", + domain: @dedupe[:domain], + record_to_delete: @record_to_delete, + record_to_keep: @record_to_keep, + reassignment_delete: @reassignment_delete, + reassignment_keep: @reassignment_keep, + model_label: @dedupe[:model_label], + keep_param_key: @dedupe[:keep_param_key], + editable_columns: @dedupe[:editable_columns], + form_builder: f, + belongs_to_options: @dedupe[:belongs_to_options] %> - <%= link_to "Cancel", - url_for(action: :dedupe_index), - class: "px-6 py-3 bg-gray-200 text-gray-700 rounded-lg hover:bg-gray-300 transition-colors font-semibold" %> -
- <% end %> + +
+
+ +

+ This can't be undone. + "<%= @record_to_delete.name %>" (ID <%= @record_to_delete.id %>) will be deleted and + <%= @reassignment_delete.values.sum %> associated + <%= "record".pluralize(@reassignment_delete.values.sum) %> moved to + "<%= @record_to_keep.name %>" (ID <%= @record_to_keep.id %>). +

+
+
+ +
+ <%= link_to "Cancel", url_for(action: :dedupe_index), class: "btn btn-secondary-outline" %> + <%= f.submit "Merge #{@dedupe[:model_label_plural].downcase}", class: "btn btn-danger" %>
+ <% end %> + +
diff --git a/app/views/organizations/index.html.erb b/app/views/organizations/index.html.erb index dc8a6bb8bd..3018d3ba43 100644 --- a/app/views/organizations/index.html.erb +++ b/app/views/organizations/index.html.erb @@ -1,15 +1,21 @@ <% content_for(:page_bg_class, "admin-or-auth") %> -
+
-
+

Organizations

- <% if allowed_to?(:new?, Organization) %> - <%= link_to "New Organization", - new_organization_path, - class: "admin-only bg-blue-100 btn btn-primary-outline" %> - <% end %> +
+ <% if allowed_to?(:manage?, Organization) %> + <%= link_to "Dedupe", dedupe_index_organizations_path, + class: "text-sm #{eyebrow_link_class} px-2 py-1" %> + <% end %> + <% if allowed_to?(:new?, Organization) %> + <%= link_to "New Organization", + new_organization_path, + class: "admin-only bg-blue-100 btn btn-primary-outline" %> + <% end %> +
diff --git a/config/features.yml b/config/features.yml index e31dfff8d1..65daf52635 100644 --- a/config/features.yml +++ b/config/features.yml @@ -2262,3 +2262,28 @@ pro_tips: - "Only mutations show: opening or printing a record never clutters its log." - "Deleting a child or attachment is recorded too — the entry keeps the name even after the file is gone." + +- name: "Merge duplicate organizations" + area: reporting + display_status: admin_facing + released_on: 2026-08-25 + pr_number: 2374 + action_path: "/organizations/dedupe_index" + summary: >- + Admins can now merge two duplicate organizations into one. The deduper + suggests likely duplicates (by name, name minus Inc/LLC, FileMaker code, and + shared address), then previews exactly what will move before merging. + description: >- + Reach it from the Dedupe link on the Organizations page. It clusters likely + duplicates and flags why each was grouped — same name, a matching FileMaker + code, or a shared address. Because FileMaker is the source of truth, two + records with different FileMaker codes are surfaced as a conflict to resolve by + hand rather than merged automatically. The preview shows a side-by-side of both + records with the kept record's key fields editable, plus a summary of every + associated record (affiliations, event registrations, reports, payments, + workshop logs, tags, and more) that moves to the kept organization. The merge + reassigns all of them and deletes the duplicate. It can't be undone. Admin-only. + pro_tips: + - "Edit the kept organization's fields right in the preview before you merge." + - "Use Swap if you picked the wrong record to keep." + - "A different FileMaker code on each record is a warning — decide which one is correct first." diff --git a/spec/requests/dedupable_spec.rb b/spec/requests/dedupable_spec.rb index 4d3ca4d586..e78324eba6 100644 --- a/spec/requests/dedupable_spec.rb +++ b/spec/requests/dedupable_spec.rb @@ -326,16 +326,19 @@ describe "GET dedupe_preview" do let!(:keep) { create(:organization, name: "Keep Org") } let!(:delete_rec) { create(:organization, name: "Delete Org") } + before { create(:affiliation, organization: delete_rec) } - it "renders the preview page" do + it "renders the preview with a reassignment summary and curated fields" do get dedupe_preview_organizations_path( organization_to_keep_id: keep.id, organization_to_delete_id: delete_rec.id ) expect(response).to have_http_status(:ok) - expect(response.body).to include("Keep Org") - expect(response.body).to include("Delete Org") + expect(response.body).to include("Keep Org", "Delete Org") + expect(response.body).to include("Records that will move", "Affiliations") + expect(response.body).to include("Website url") + expect(response.body).not_to include("Profile show age ranges") end end From 4b7d012966596b642dcfc00f0bf718af7ab81450 Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 10:07:32 -0400 Subject: [PATCH 3/9] Reword deduper index intro to avoid a/an agreement across models Co-Authored-By: Claude Opus 4.8 (1M context) --- app/views/dedupes/index.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/dedupes/index.html.erb b/app/views/dedupes/index.html.erb index 6619ae985e..43204495ee 100644 --- a/app/views/dedupes/index.html.erb +++ b/app/views/dedupes/index.html.erb @@ -10,7 +10,7 @@ Dedupe <%= @dedupe[:model_label_plural].downcase %>

- Pick a <%= @dedupe[:model_label].downcase %> to delete and one to keep — its + Pick which <%= @dedupe[:model_label].downcase %> to delete and which to keep — its records move to the one you keep before it's removed.

From 9e21e42f54898187a0789bbc5204a30c4e1e9c65 Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 10:56:23 -0400 Subject: [PATCH 4/9] Treat FileMaker code as a comma-separated set, unioned on merge FileMaker is the source of truth and a record can map to more than one FileMaker record, so the deduper now reads filemaker_code as a trimmed, comma-separated list. Candidate detection matches on set intersection (FM-27 no longer matches FM-273), and merging keeps the union of both records' codes so no FileMaker linkage is lost. The preview marks a union field being combined with a 'Kept on merge' badge rather than 'Will be dropped'. Adds a generic union_columns + merge_keeper hook to the Dedupable framework. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controllers/concerns/dedupable.rb | 8 ++++++++ app/controllers/organizations_controller.rb | 4 ++++ app/models/organization.rb | 17 ++++++++++++++++ .../organization_services/duplicate_finder.rb | 11 +++++----- app/views/dedupes/_preview.html.erb | 10 ++++++++-- .../dedupes/_reassignment_summary.html.erb | 4 ++-- app/views/dedupes/preview.html.erb | 1 + spec/models/organization_spec.rb | 20 +++++++++++++++++++ spec/requests/dedupable_spec.rb | 12 +++++++++++ .../duplicate_finder_spec.rb | 14 +++++++++++-- 10 files changed, 89 insertions(+), 12 deletions(-) diff --git a/app/controllers/concerns/dedupable.rb b/app/controllers/concerns/dedupable.rb index 8c61e006bf..fdfd962633 100644 --- a/app/controllers/concerns/dedupable.rb +++ b/app/controllers/concerns/dedupable.rb @@ -83,6 +83,13 @@ def dedupe_perform record_to_keep.update!(params.require(keep_param_key).permit(editable)) end + # Combine values that must survive the merge rather than be replaced wholesale + # (e.g. an org's FileMaker codes), so the kept record keeps both records' links. + if (hook = config[:merge_keeper]) + hook.call(record_to_keep, record_to_delete) + record_to_keep.save! + end + deduper = ModelDeduper.new(model_class: mc, logger: Rails.logger, dry_run: false, min_usage: 0) if respond_to?(:track_event, true) @@ -146,6 +153,7 @@ def build_dedupe_vars(config) keep_id_param: "#{mn}_to_keep_id", keep_param_key: "#{mn}_to_keep".to_sym, editable_columns: config[:editable_columns], + union_columns: Array(config[:union_columns]).map(&:to_s), belongs_to_options: opts.is_a?(Proc) ? opts.call : (opts || {}), record_extras: config[:record_extras] } diff --git a/app/controllers/organizations_controller.rb b/app/controllers/organizations_controller.rb index 5251bb6d5d..af88bcfdaf 100644 --- a/app/controllers/organizations_controller.rb +++ b/app/controllers/organizations_controller.rb @@ -292,6 +292,10 @@ def dedupe_config name filemaker_code email website_url agency_type description mission_vision_values notes organization_status_id organization_obligation_id ], + union_columns: %w[filemaker_code], + merge_keeper: ->(keep, delete) { + keep.filemaker_code = Organization.join_filemaker_codes(keep.filemaker_code, delete.filemaker_code) + }, belongs_to_options: -> { { "organization_status_id" => OrganizationStatus.order(:name), diff --git a/app/models/organization.rb b/app/models/organization.rb index 3c5ec6cd53..03f48833cc 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -285,6 +285,23 @@ def rollup_cache_version remote_searchable_by :name + # FileMaker is the source of truth, and a record can carry more than one + # FileMaker code (e.g. when two orgs that each mapped to a FileMaker record are + # merged). The `filemaker_code` column holds them as a trimmed, comma-separated + # list; these are the seam for reading and combining them. + def filemaker_codes + Organization.split_filemaker_codes(filemaker_code) + end + + def self.split_filemaker_codes(value) + value.to_s.split(",").map(&:strip).reject(&:blank?).uniq + end + + # A single normalized column value from any mix of code strings/lists. + def self.join_filemaker_codes(*values) + values.flatten.flat_map { |value| split_filemaker_codes(value) }.uniq.sort.join(", ").presence + end + # Returns the website as a clickable, scheme-qualified URL — prepending # https:// to a bare domain like "awbw.org" — or nil when the value is blank or # not a usable web address. Drives the external links on the org profile and diff --git a/app/services/organization_services/duplicate_finder.rb b/app/services/organization_services/duplicate_finder.rb index de596fde8a..2c70829282 100644 --- a/app/services/organization_services/duplicate_finder.rb +++ b/app/services/organization_services/duplicate_finder.rb @@ -8,7 +8,6 @@ module OrganizationServices # cluster whose members carry different FileMaker codes is surfaced as a # conflict for an admin to resolve, never silently treated as a clean merge. class DuplicateFinder - SOURCE_OF_TRUTH_FIELD = :filemaker_code LEGAL_SUFFIX = /\b(?:incorporated|inc|llc|l\.l\.c|corporation|corp|company|co|limited|ltd)\b\.?/i Group = Struct.new(:key, :label, :records, :reasons, keyword_init: true) @@ -21,7 +20,7 @@ def groups union = UnionFind.new(@organizations.map(&:id)) cluster(union) { |org| [ normalized_name(org).presence ].compact } cluster(union) { |org| [ suffixless_name(org).presence ].compact } - cluster(union) { |org| [ org.public_send(SOURCE_OF_TRUTH_FIELD).presence ].compact } + cluster(union) { |org| org.filemaker_codes } cluster(union) { |org| address_keys(org) } by_id = @organizations.index_by(&:id) @@ -63,11 +62,11 @@ def reasons_for(records) end def filemaker_reasons(records) - codes = records.map { |org| org.public_send(SOURCE_OF_TRUTH_FIELD).presence } - present = codes.compact.uniq - return [ "⚠ Different FileMaker codes (#{present.join(", ")}) — choose which to keep" ] if present.size > 1 + code_sets = records.map(&:filemaker_codes) + present = code_sets.flatten.uniq.sort return [] if present.empty? - return [ "FileMaker code on only one record" ] if codes.any?(&:blank?) + return [ "⚠ Multiple FileMaker codes (#{present.join(", ")}) — merging keeps all of them" ] if present.size > 1 + return [ "FileMaker code on only one record" ] if code_sets.any?(&:empty?) [ "Same FileMaker code (#{present.first})" ] end diff --git a/app/views/dedupes/_preview.html.erb b/app/views/dedupes/_preview.html.erb index 0145889324..02f737165f 100644 --- a/app/views/dedupes/_preview.html.erb +++ b/app/views/dedupes/_preview.html.erb @@ -8,6 +8,7 @@ model_label: Human label (e.g. "Organization") keep_param_key: Param key for keep fields (e.g. :organization_to_keep) editable_columns: Optional allowlist of column names to show/edit; nil shows all + union_columns: Column names whose values are combined into the keeper (not replaced) form_builder: The parent form builder (f) belongs_to_options: Hash of { column_name => collection } for select fields %> @@ -16,6 +17,7 @@ border = DomainTheme.border_class_for(domain, intensity: 200) header_bg = DomainTheme.bg_class_for(domain, intensity: 50) model_class = record_to_delete.class + union_columns ||= [] skip_columns = %w[id created_at updated_at legacy_id] columns = if editable_columns.present? @@ -69,14 +71,18 @@ else delete_val.presence || "—" end - is_different = delete_val != keep_val + union_column = union_columns.include?(col.name) + merged_in = union_column && delete_val.to_s.split(",").map(&:strip).reject(&:blank?).difference(keep_val.to_s.split(",").map(&:strip)).any? + is_different = !union_column && delete_val != keep_val %>
<%= label %> "><%= display_val %> - <% if is_different %> + <% if merged_in %> + Kept on merge + <% elsif is_different %> Will be dropped <% end %>
diff --git a/app/views/dedupes/_reassignment_summary.html.erb b/app/views/dedupes/_reassignment_summary.html.erb index 6b8ecb13c3..05b6001740 100644 --- a/app/views/dedupes/_reassignment_summary.html.erb +++ b/app/views/dedupes/_reassignment_summary.html.erb @@ -1,10 +1,10 @@ <%# One column of the reassignment summary. Locals: border, title, counts (hash), empty (string). %>
-

<%= title %>

+

<%= title %>

<% if counts.any? %>
<% counts.sort_by { |label, _count| label }.each do |label, count| %> - + <%= label %> <%= count %> <% end %> diff --git a/app/views/dedupes/preview.html.erb b/app/views/dedupes/preview.html.erb index 95bfc1ddbc..dac11c3711 100644 --- a/app/views/dedupes/preview.html.erb +++ b/app/views/dedupes/preview.html.erb @@ -36,6 +36,7 @@ model_label: @dedupe[:model_label], keep_param_key: @dedupe[:keep_param_key], editable_columns: @dedupe[:editable_columns], + union_columns: @dedupe[:union_columns], form_builder: f, belongs_to_options: @dedupe[:belongs_to_options] %> diff --git a/spec/models/organization_spec.rb b/spec/models/organization_spec.rb index 67ecf9948d..13fb7b8801 100644 --- a/spec/models/organization_spec.rb +++ b/spec/models/organization_spec.rb @@ -422,4 +422,24 @@ def version = Organization.find(organization.id).rollup_cache_version expect(Organization.awbw).to be_nil end end + + describe "FileMaker codes" do + it "parses a trimmed, de-duplicated list from the column" do + org = build(:organization, filemaker_code: " FM1 , FM2,FM1 ") + + expect(org.filemaker_codes).to eq(%w[FM1 FM2]) + end + + it "is empty when the column is blank" do + expect(build(:organization, filemaker_code: nil).filemaker_codes).to eq([]) + end + + it "combines mixed strings and lists into one sorted normalized value" do + expect(Organization.join_filemaker_codes("FM2, FM1", " FM3 ", nil, "FM1")).to eq("FM1, FM2, FM3") + end + + it "returns nil when there is nothing to combine" do + expect(Organization.join_filemaker_codes(nil, "")).to be_nil + end + end end diff --git a/spec/requests/dedupable_spec.rb b/spec/requests/dedupable_spec.rb index e78324eba6..be3181a1a0 100644 --- a/spec/requests/dedupable_spec.rb +++ b/spec/requests/dedupable_spec.rb @@ -369,6 +369,18 @@ expect(keep.reload.name).to eq("Canonical Org") end + + it "combines both organizations' FileMaker codes onto the keeper" do + keep.update!(filemaker_code: "FM-100") + delete_rec.update!(filemaker_code: "FM-273") + + post dedupe_perform_organizations_path, params: { + organization_to_delete_id: delete_rec.id, + organization_to_keep_id: keep.id + } + + expect(keep.reload.filemaker_code).to eq("FM-100, FM-273") + end end end end diff --git a/spec/services/organization_services/duplicate_finder_spec.rb b/spec/services/organization_services/duplicate_finder_spec.rb index aea6844556..cb7b4c7d8c 100644 --- a/spec/services/organization_services/duplicate_finder_spec.rb +++ b/spec/services/organization_services/duplicate_finder_spec.rb @@ -37,12 +37,22 @@ def org(attrs = {}) expect(described_class.new.groups.first.reasons).to include("FileMaker code on only one record") end - it "surfaces conflicting FileMaker codes as a choose-which conflict" do + it "surfaces multiple FileMaker codes as a keep-all conflict" do org(name: "Rivertown", filemaker_code: "FM1") org(name: "Rivertown", filemaker_code: "FM2") expect(described_class.new.groups.first.reasons) - .to include(a_string_matching(/Different FileMaker codes/)) + .to include(a_string_matching(/Multiple FileMaker codes.*merging keeps all/)) + end + + it "groups records that share one code in a comma-separated list" do + a = org(name: "Lakeside A", filemaker_code: "FM1, FM2") + b = org(name: "Lakeside B", filemaker_code: " FM2 ") + + group = described_class.new.groups.first + + expect(group.records).to contain_exactly(a, b) + expect(group.reasons).to include(a_string_matching(/Multiple FileMaker codes \(FM1, FM2\)/)) end it "groups organizations that share an address" do From 2c785c82c223b7226db78eaba5490302426a3407 Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 10:57:12 -0400 Subject: [PATCH 5/9] Update deduper feature note for FileMaker code union behavior Co-Authored-By: Claude Opus 4.8 (1M context) --- config/features.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/config/features.yml b/config/features.yml index 65daf52635..783e1a7bd0 100644 --- a/config/features.yml +++ b/config/features.yml @@ -2275,15 +2275,16 @@ shared address), then previews exactly what will move before merging. description: >- Reach it from the Dedupe link on the Organizations page. It clusters likely - duplicates and flags why each was grouped — same name, a matching FileMaker - code, or a shared address. Because FileMaker is the source of truth, two - records with different FileMaker codes are surfaced as a conflict to resolve by - hand rather than merged automatically. The preview shows a side-by-side of both - records with the kept record's key fields editable, plus a summary of every - associated record (affiliations, event registrations, reports, payments, - workshop logs, tags, and more) that moves to the kept organization. The merge - reassigns all of them and deletes the duplicate. It can't be undone. Admin-only. + duplicates and flags why each was grouped — same name, a shared FileMaker + code, or a shared address. FileMaker is the source of truth, and a record can + carry more than one code (a trimmed, comma-separated list); when two records + have different codes, merging keeps all of them so no FileMaker link is lost. + The preview shows a side-by-side of both records with the kept record's key + fields editable, plus a summary of every associated record (affiliations, + event registrations, reports, payments, workshop logs, tags, and more) that + moves to the kept organization. The merge reassigns all of them and deletes + the duplicate. It can't be undone. Admin-only. pro_tips: - "Edit the kept organization's fields right in the preview before you merge." - "Use Swap if you picked the wrong record to keep." - - "A different FileMaker code on each record is a warning — decide which one is correct first." + - "Both records' FileMaker codes are kept on the merged organization — a field flagged 'Kept on merge' is combined, not dropped." From b6e551ca0bdabc0353fcd32d610882f015ea0147 Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 11:01:03 -0400 Subject: [PATCH 6/9] Add organization deduper card under Deprecated data on admin home Co-Authored-By: Claude Opus 4.8 (1M context) --- app/helpers/admin_cards_helper.rb | 3 ++- spec/requests/admin/home_spec.rb | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/helpers/admin_cards_helper.rb b/app/helpers/admin_cards_helper.rb index 1ab9b302ae..d31961808a 100644 --- a/app/helpers/admin_cards_helper.rb +++ b/app/helpers/admin_cards_helper.rb @@ -66,7 +66,8 @@ def reference_cards # ----------------------------- def deprecated_data_cards [ - custom_card("Organization statuses", organization_statuses_path, icon: "🧮", color: :emerald, intensity: 100) + custom_card("Organization statuses", organization_statuses_path, icon: "🧮", color: :emerald, intensity: 100), + custom_card("Dedupe organizations", dedupe_index_organizations_path, icon: "🧹", color: :emerald, intensity: 100) ] end diff --git a/spec/requests/admin/home_spec.rb b/spec/requests/admin/home_spec.rb index 54e7def6da..0d1b48e7cb 100644 --- a/spec/requests/admin/home_spec.rb +++ b/spec/requests/admin/home_spec.rb @@ -18,4 +18,11 @@ expect(response.body).to include("Form submissions") expect(response.body).to include(%(href="#{form_submissions_path}")) end + + it "links to the organization deduper under deprecated data" do + get "/admin" + + expect(response.body).to include("Dedupe organizations") + expect(response.body).to include(%(href="#{dedupe_index_organizations_path}")) + end end From 67be90dc095a345e4d2f9756213d255067c4fe51 Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 11:03:02 -0400 Subject: [PATCH 7/9] Add category and sector deduper cards under Deprecated data Co-Authored-By: Claude Opus 4.8 (1M context) --- app/helpers/admin_cards_helper.rb | 4 +++- spec/requests/admin/home_spec.rb | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/helpers/admin_cards_helper.rb b/app/helpers/admin_cards_helper.rb index d31961808a..3eb00b3572 100644 --- a/app/helpers/admin_cards_helper.rb +++ b/app/helpers/admin_cards_helper.rb @@ -67,7 +67,9 @@ def reference_cards def deprecated_data_cards [ custom_card("Organization statuses", organization_statuses_path, icon: "🧮", color: :emerald, intensity: 100), - custom_card("Dedupe organizations", dedupe_index_organizations_path, icon: "🧹", color: :emerald, intensity: 100) + custom_card("Dedupe organizations", dedupe_index_organizations_path, icon: "🧹", color: DomainTheme.color_for(:organizations), intensity: 100), + custom_card("Dedupe categories", dedupe_index_categories_path, icon: "🧹", color: DomainTheme.color_for(:categories), intensity: 100), + custom_card("Dedupe sectors", dedupe_index_sectors_path, icon: "🧹", color: DomainTheme.color_for(:sectors), intensity: 100) ] end diff --git a/spec/requests/admin/home_spec.rb b/spec/requests/admin/home_spec.rb index 0d1b48e7cb..20cda01c67 100644 --- a/spec/requests/admin/home_spec.rb +++ b/spec/requests/admin/home_spec.rb @@ -19,10 +19,14 @@ expect(response.body).to include(%(href="#{form_submissions_path}")) end - it "links to the organization deduper under deprecated data" do + it "links to the record dedupers under deprecated data" do get "/admin" - expect(response.body).to include("Dedupe organizations") - expect(response.body).to include(%(href="#{dedupe_index_organizations_path}")) + expect(response.body).to include("Dedupe organizations", "Dedupe categories", "Dedupe sectors") + expect(response.body).to include( + %(href="#{dedupe_index_organizations_path}"), + %(href="#{dedupe_index_categories_path}"), + %(href="#{dedupe_index_sectors_path}") + ) end end From 6182f7a46d477eaaeabb58a38f9c8b30e1a19f0c Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 11:26:21 -0400 Subject: [PATCH 8/9] Show associated names, reassign analytics/audit, and guard unhandled refs - Reassignment summary now lists a few truncated record names per association, not just the rollup count. - Merge repoints ahoy_events + versions to the kept record so analytics and audit history survive instead of orphaning. - Preview surfaces attached files (logo) that are deleted with the duplicate, and adds space above the action buttons. - Coverage safeguard: a data-driven scan blocks the merge (banner + disabled button + server guard) if anything references the record that the deduper won't reassign, so a future association can't silently break referential integrity. Billing (Pay gem) links are deferred to rubyforgood/awbw#2378. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controllers/concerns/dedupable.rb | 13 +- app/services/model_deduper.rb | 143 ++++++++++++++++++ app/views/dedupes/_preview.html.erb | 4 +- .../dedupes/_reassignment_summary.html.erb | 24 ++- app/views/dedupes/preview.html.erb | 62 +++++++- spec/requests/dedupable_spec.rb | 19 +++ spec/services/model_deduper_spec.rb | 73 +++++++++ 7 files changed, 321 insertions(+), 17 deletions(-) diff --git a/app/controllers/concerns/dedupable.rb b/app/controllers/concerns/dedupable.rb index fdfd962633..c72e9a34cf 100644 --- a/app/controllers/concerns/dedupable.rb +++ b/app/controllers/concerns/dedupable.rb @@ -40,8 +40,10 @@ def dedupe_preview alert: "#{mc.model_name.human} not found (ID: #{missing.join(', ')})." end deduper = ModelDeduper.new(model_class: mc) - @reassignment_delete = deduper.reassignment_counts(@record_to_delete) - @reassignment_keep = deduper.reassignment_counts(@record_to_keep) + @reassignment_delete = deduper.reassignment_preview(@record_to_delete) + @reassignment_keep = deduper.reassignment_preview(@record_to_keep) + @lost_references = deduper.lost_references(@record_to_delete) + @unhandled_references = deduper.unhandled_references(@record_to_delete) @dedupe = build_dedupe_vars(config) render "dedupes/preview" @@ -77,6 +79,13 @@ def dedupe_perform record_to_delete = mc.find(params["#{mn}_to_delete_id"]) record_to_keep = mc.find(params["#{mn}_to_keep_id"]) + unhandled = ModelDeduper.new(model_class: mc).unhandled_references(record_to_delete) + if unhandled.any? + tables = unhandled.map { |ref| ref[:table] }.uniq.join(", ") + return redirect_to url_for(action: :dedupe_index), + alert: "Can't merge: #{tables} still reference this #{mc.model_name.human.downcase} and the deduper doesn't reassign them. A developer needs to teach ModelDeduper about them before merging." + end + keep_param_key = "#{mn}_to_keep" if params[keep_param_key].present? editable = mc.column_names - %w[id created_at updated_at legacy_id] diff --git a/app/services/model_deduper.rb b/app/services/model_deduper.rb index f7160095b7..6ee684cecb 100644 --- a/app/services/model_deduper.rb +++ b/app/services/model_deduper.rb @@ -48,6 +48,127 @@ def reassignment_counts(record) end end + # Like #reassignment_counts, but also a few sample record names per association + # so the preview can show *what* moves, not just how many. Each entry is + # { label:, count:, names: [ up to SAMPLE_SIZE display strings ] }. + SAMPLE_SIZE = 3 + + def reassignment_preview(record) + reassignable_joins.filter_map do |join| + scope = join[:join_class].where(join[:foreign_key] => record.id) + count = scope.count + next if count.zero? + + { + label: join_label(join[:join_class]), + count: count, + names: scope.limit(SAMPLE_SIZE).map { |related| record_display(related) } + } + end.sort_by { |entry| entry[:label] } + end + + # A best-effort human name for an associated record: its own name/title, else a + # named parent (e.g. an affiliation's person), else its id. + def record_display(record) + own = display_string(record) + return own if own + + record.class.reflect_on_all_associations(:belongs_to).each do |assoc| + next if assoc.polymorphic? + + related = record.try(assoc.name) + parent = display_string(related) + return parent if parent + end + "##{record.id}" + end + + def display_string(record) + return unless record + + %i[name title full_name].filter_map { |method| record.try(method).presence }.first + end + + # Polymorphic references the merge repoints to the kept record (type stays, id + # moves) so analytics and audit history follow the survivor instead of orphaning. + # table => the polymorphic association name (its columns are name_type/name_id). + REASSIGNED_POLYMORPHIC_REFERENCES = { "ahoy_events" => "resource", "versions" => "item" }.freeze + + # References that are lost with the deleted record (its own attached files purge) + # rather than moved — surfaced on the preview so an admin sees the loss. + LOST_POLYMORPHIC_REFERENCES = { "active_storage_attachments" => "record" }.freeze + + # Billing links between orgs need a deliberate decision (rubyforgood/awbw#2378); + # skipped here so they neither block nor silently move. + DEFERRED_REFERENCE_TABLES = %w[ + pay_customers pay_merchants pay_subscriptions pay_charges pay_payment_methods pay_webhooks + ].freeze + + # Framework internals that purge with the record and aren't worth surfacing. + IGNORED_REFERENCE_TABLES = %w[ + action_text_rich_texts action_text_mentions ckeditor_assets + active_storage_blobs active_storage_variant_records ahoy_visits + ].freeze + + HANDLED_ELSEWHERE_TABLES = ( + REASSIGNED_POLYMORPHIC_REFERENCES.keys + LOST_POLYMORPHIC_REFERENCES.keys + + DEFERRED_REFERENCE_TABLES + IGNORED_REFERENCE_TABLES + ).freeze + + # Attached files (e.g. the logo) that are deleted with the record, not moved. + # Each entry is { label:, count: } — drives the preview's "will be lost" note. + def lost_references(record) + LOST_POLYMORPHIC_REFERENCES.filter_map do |table, association| + next unless reference_table_present?(table) + + klass = model_for_table(table) + id_column = "#{association}_id" + next unless klass.column_names.include?(id_column) + + count = klass.where("#{association}_type" => model_class.polymorphic_name, id_column => record.id).count + next if count.zero? + + { label: "Attached files (e.g. logo)", count: count } + end + end + + # Defensive safeguard: tables that still reference `record` but that the merge + # would NOT reassign — e.g. a new association added to the model that this flow + # doesn't yet account for. Data-driven (it checks the actual record) and + # independent of any `dependent:` option, so a future association can't silently + # orphan rows or break referential integrity. Each entry is { table:, column: }. + def unhandled_references(record) + covered = reassignable_joins.map { |join| [ join[:join_class].table_name, join[:foreign_key].to_s ] }.to_set + connection = model_class.connection + convention_fk = "#{model_class.model_name.singular}_id" + polymorphic_name = model_class.polymorphic_name + + connection.tables.flat_map do |table| + next [] if table == model_class.table_name || HANDLED_ELSEWHERE_TABLES.include?(table) + + column_names = connection.columns(table).map(&:name) + gaps = [] + + if column_names.include?(convention_fk) && !covered.include?([ table, convention_fk ]) && + model_for_table(table).where(convention_fk => record.id).exists? + gaps << { table: table, column: convention_fk } + end + + column_names.each do |column| + next unless column.end_with?("_type") + + id_column = column.sub(/_type\z/, "_id") + next unless column_names.include?(id_column) + next if covered.include?([ table, id_column ]) + next unless model_for_table(table).where(column => polymorphic_name, id_column => record.id).exists? + + gaps << { table: table, column: id_column } + end + + gaps.uniq + end.uniq + end + private # Every association that references this model by a foreign key — whether a @@ -196,11 +317,33 @@ def merge_duplicate(primary, dupe, usage) merge_join(primary, dupe, join) end + reassign_polymorphic_references(primary, dupe) + dupe.reload.destroy! logger.info " deleted #{model_label} #{dupe.id}" end end + # Repoint analytics/audit rows (ahoy events, versions) from the dupe to the kept + # record so its history survives the merge. Type column stays; only the id moves. + def reassign_polymorphic_references(primary, dupe) + REASSIGNED_POLYMORPHIC_REFERENCES.each do |table, association| + next unless reference_table_present?(table) + + klass = model_for_table(table) + id_column = "#{association}_id" + next unless klass.column_names.include?(id_column) + + moved = klass.where("#{association}_type" => model_class.polymorphic_name, id_column => dupe.id) + .update_all(id_column => primary.id) + logger.info " moved #{moved} #{table} to primary" if moved > 0 + end + end + + def reference_table_present?(table) + model_class.connection.data_source_exists?(table) + end + def merge_join(primary, dupe, join) jc = join[:join_class] fk = join[:foreign_key] diff --git a/app/views/dedupes/_preview.html.erb b/app/views/dedupes/_preview.html.erb index 02f737165f..d5b1b38ca0 100644 --- a/app/views/dedupes/_preview.html.erb +++ b/app/views/dedupes/_preview.html.erb @@ -111,6 +111,6 @@ <% end %> - <%= render "dedupes/reassignment_summary", border: border, title: "Records that will move", counts: reassignment_delete, empty: "Nothing to move" %> - <%= render "dedupes/reassignment_summary", border: border, title: "Records already here", counts: reassignment_keep, empty: "None yet" %> + <%= render "dedupes/reassignment_summary", border: border, title: "Records that will move", entries: reassignment_delete, empty: "Nothing to move" %> + <%= render "dedupes/reassignment_summary", border: border, title: "Records already here", entries: reassignment_keep, empty: "None yet" %>
diff --git a/app/views/dedupes/_reassignment_summary.html.erb b/app/views/dedupes/_reassignment_summary.html.erb index 05b6001740..ca689b734c 100644 --- a/app/views/dedupes/_reassignment_summary.html.erb +++ b/app/views/dedupes/_reassignment_summary.html.erb @@ -1,14 +1,22 @@ -<%# One column of the reassignment summary. Locals: border, title, counts (hash), empty (string). %> +<%# One column of the reassignment summary. Locals: border, title, entries (array of + { label:, count:, names: [..] }), empty (string). %>

<%= title %>

- <% if counts.any? %> -
- <% counts.sort_by { |label, _count| label }.each do |label, count| %> - - <%= label %> <%= count %> - + <% if entries.any? %> +
    + <% entries.each do |entry| %> + <% extra = entry[:count] - entry[:names].size %> +
  • + <%= entry[:label] %> + (<%= entry[:count] %>) + <% if entry[:names].any? %> + · + <%= entry[:names].map { |name| truncate(name, length: 28) }.join(", ") %><%= " +#{extra} more" if extra > 0 %> + + <% end %> +
  • <% end %> -
+ <% else %>

<%= empty %>

<% end %> diff --git a/app/views/dedupes/preview.html.erb b/app/views/dedupes/preview.html.erb index dac11c3711..0856627cae 100644 --- a/app/views/dedupes/preview.html.erb +++ b/app/views/dedupes/preview.html.erb @@ -21,6 +21,28 @@ <% end %>
+ + <% if @unhandled_references.any? %> +
+
+ +
+

Merge blocked — unhandled associations

+

+ These reference this <%= @dedupe[:model_label].downcase %> but the deduper wouldn't move them, so merging + could orphan data or break referential integrity: +

+
    + <% @unhandled_references.each do |ref| %> +
  • <%= ref[:table] %>.<%= ref[:column] %>
  • + <% end %> +
+

A developer needs to teach ModelDeduper about them before this merge is safe.

+
+
+
+ <% end %> + <%= form_with url: url_for(action: :dedupe_perform), method: :post, data: { turbo: false, turbo_confirm: "Merge these #{@dedupe[:model_label_plural].downcase}? This can't be undone." } do |f| %> @@ -40,24 +62,54 @@ form_builder: f, belongs_to_options: @dedupe[:belongs_to_options] %> + + <% if @lost_references.any? %> +
+
+ +
+

Deleted, not moved

+

+ These belong to "<%= @record_to_delete.name %>" and are removed with it — they are + not transferred to "<%= @record_to_keep.name %>": +

+
    + <% @lost_references.each do |entry| %> +
  • <%= entry[:label] %> (<%= entry[:count] %>)
  • + <% end %> +
+
+
+
+ <% end %> + -
+ <% moved_total = @reassignment_delete.sum { |entry| entry[:count] } %> +

This can't be undone. "<%= @record_to_delete.name %>" (ID <%= @record_to_delete.id %>) will be deleted and - <%= @reassignment_delete.values.sum %> associated - <%= "record".pluralize(@reassignment_delete.values.sum) %> moved to + <%= moved_total %> associated + <%= "record".pluralize(moved_total) %> moved to "<%= @record_to_keep.name %>" (ID <%= @record_to_keep.id %>).

-
+ <% blocked = @unhandled_references.any? %> +
+ <% if blocked %> + + Merge disabled — unhandled associations (see above). + + <% end %> <%= link_to "Cancel", url_for(action: :dedupe_index), class: "btn btn-secondary-outline" %> - <%= f.submit "Merge #{@dedupe[:model_label_plural].downcase}", class: "btn btn-danger" %> + <%= f.submit "Merge #{@dedupe[:model_label_plural].downcase}", + disabled: blocked, + class: "btn btn-danger #{"cursor-not-allowed opacity-50" if blocked}" %>
<% end %> diff --git a/spec/requests/dedupable_spec.rb b/spec/requests/dedupable_spec.rb index be3181a1a0..656dbf9252 100644 --- a/spec/requests/dedupable_spec.rb +++ b/spec/requests/dedupable_spec.rb @@ -370,6 +370,25 @@ expect(keep.reload.name).to eq("Canonical Org") end + it "blocks the merge when an association would be orphaned" do + create(:other_response, promotable: delete_rec) + + get dedupe_preview_organizations_path( + organization_to_delete_id: delete_rec.id, + organization_to_keep_id: keep.id + ) + expect(response.body).to include("Merge blocked") + + expect { + post dedupe_perform_organizations_path, params: { + organization_to_delete_id: delete_rec.id, + organization_to_keep_id: keep.id + } + }.not_to change(Organization, :count) + expect(response).to redirect_to(dedupe_index_organizations_path) + expect(Organization.exists?(delete_rec.id)).to be true + end + it "combines both organizations' FileMaker codes onto the keeper" do keep.update!(filemaker_code: "FM-100") delete_rec.update!(filemaker_code: "FM-273") diff --git a/spec/services/model_deduper_spec.rb b/spec/services/model_deduper_spec.rb index 33be8418f6..1ce987bf40 100644 --- a/spec/services/model_deduper_spec.rb +++ b/spec/services/model_deduper_spec.rb @@ -179,4 +179,77 @@ expect(story.reload.organization_id).to eq(keep.id) end end + + describe "#unhandled_references (coverage safeguard)" do + subject(:service) { described_class.new(model_class: Organization) } + + let(:org) { create(:organization, name: "Solo Org") } + + it "is empty when every reference is reassignable" do + create(:affiliation, organization: org) + create(:report, organization: org) + + expect(service.unhandled_references(org)).to eq([]) + end + + it "flags a polymorphic reference the deduper does not reassign" do + # OtherResponse#owner is covered, but #promotable is not — a promotable + # pointing at the org would be orphaned on merge. + create(:other_response, promotable: org) + + expect(service.unhandled_references(org)) + .to include(a_hash_including(table: "other_responses", column: "promotable_id")) + end + + it "ignores gem-managed infrastructure that references the org" do + blob = ActiveStorage::Blob.create_before_direct_upload!( + filename: "logo.png", byte_size: 1, checksum: "x", content_type: "image/png" + ) + ActiveStorage::Attachment.create!(name: "logo", record: org, blob: blob) + + tables = service.unhandled_references(org).map { |ref| ref[:table] } + expect(tables).not_to include("active_storage_attachments") + end + end + + describe "analytics/audit reassignment" do + subject(:service) { described_class.new(model_class: Organization, logger: logger, dry_run: false) } + + let!(:keep) { create(:organization, name: "Keep Org") } + let!(:dupe) { create(:organization, name: "Dupe Org") } + let(:ahoy_events) { Class.new(ActiveRecord::Base) { self.table_name = "ahoy_events" } } + let(:versions) { Class.new(ActiveRecord::Base) { self.table_name = "versions" } } + + it "repoints ahoy events to the kept organization" do + event = ahoy_events.create!(name: "view", resource_type: "Organization", resource_id: dupe.id, time: Time.current) + + service.merge(keep, dupe) + expect(event.reload.resource_id).to eq(keep.id) + end + + it "repoints paper_trail versions to the kept organization" do + version = versions.create!(event: "update", item_type: "Organization", item_id: dupe.id) + + service.merge(keep, dupe) + expect(version.reload.item_id).to eq(keep.id) + end + end + + describe "#lost_references" do + subject(:service) { described_class.new(model_class: Organization) } + + it "reports attached files that are deleted with the record" do + org = create(:organization) + blob = ActiveStorage::Blob.create_before_direct_upload!( + filename: "logo.png", byte_size: 1, checksum: "x", content_type: "image/png" + ) + ActiveStorage::Attachment.create!(name: "logo", record: org, blob: blob) + + expect(service.lost_references(org)).to include(a_hash_including(count: 1)) + end + + it "is empty when the record has no attached files" do + expect(service.lost_references(create(:organization))).to eq([]) + end + end end From 58675c0491e0bd3a36b194cdafe9fd6677edba0a Mon Sep 17 00:00:00 2001 From: Mae Beale Date: Tue, 25 Aug 2026 11:28:41 -0400 Subject: [PATCH 9/9] Show associated names, reassign analytics/audit, and guard unhandled refs - Reassignment summary now lists a few truncated record names per association, not just the rollup count. - Merge repoints ahoy_events + versions to the kept record so analytics and audit history survive instead of orphaning. - Preview surfaces attached files (logo) that are deleted with the duplicate, and adds space above the action buttons. - Coverage safeguard: a data-driven scan blocks the merge (banner + disabled button + server guard) if anything references the record that the deduper won't reassign, so a future association can't silently break referential integrity. Billing (Pay gem) links are deferred to rubyforgood/awbw#2378. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/services/model_deduper.rb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/services/model_deduper.rb b/app/services/model_deduper.rb index 6ee684cecb..66582306e6 100644 --- a/app/services/model_deduper.rb +++ b/app/services/model_deduper.rb @@ -91,12 +91,15 @@ def display_string(record) # Polymorphic references the merge repoints to the kept record (type stays, id # moves) so analytics and audit history follow the survivor instead of orphaning. - # table => the polymorphic association name (its columns are name_type/name_id). - REASSIGNED_POLYMORPHIC_REFERENCES = { "ahoy_events" => "resource", "versions" => "item" }.freeze + # table => [ type_column, id_column ]. + REASSIGNED_POLYMORPHIC_REFERENCES = { + "ahoy_events" => %w[resource_type resource_id], + "versions" => %w[item_type item_id] + }.freeze # References that are lost with the deleted record (its own attached files purge) # rather than moved — surfaced on the preview so an admin sees the loss. - LOST_POLYMORPHIC_REFERENCES = { "active_storage_attachments" => "record" }.freeze + LOST_POLYMORPHIC_REFERENCES = { "active_storage_attachments" => %w[record_type record_id] }.freeze # Billing links between orgs need a deliberate decision (rubyforgood/awbw#2378); # skipped here so they neither block nor silently move. @@ -118,14 +121,13 @@ def display_string(record) # Attached files (e.g. the logo) that are deleted with the record, not moved. # Each entry is { label:, count: } — drives the preview's "will be lost" note. def lost_references(record) - LOST_POLYMORPHIC_REFERENCES.filter_map do |table, association| + LOST_POLYMORPHIC_REFERENCES.filter_map do |table, (type_column, id_column)| next unless reference_table_present?(table) klass = model_for_table(table) - id_column = "#{association}_id" next unless klass.column_names.include?(id_column) - count = klass.where("#{association}_type" => model_class.polymorphic_name, id_column => record.id).count + count = klass.where(type_column => model_class.polymorphic_name, id_column => record.id).count next if count.zero? { label: "Attached files (e.g. logo)", count: count } @@ -327,14 +329,13 @@ def merge_duplicate(primary, dupe, usage) # Repoint analytics/audit rows (ahoy events, versions) from the dupe to the kept # record so its history survives the merge. Type column stays; only the id moves. def reassign_polymorphic_references(primary, dupe) - REASSIGNED_POLYMORPHIC_REFERENCES.each do |table, association| + REASSIGNED_POLYMORPHIC_REFERENCES.each do |table, (type_column, id_column)| next unless reference_table_present?(table) klass = model_for_table(table) - id_column = "#{association}_id" next unless klass.column_names.include?(id_column) - moved = klass.where("#{association}_type" => model_class.polymorphic_name, id_column => dupe.id) + moved = klass.where(type_column => model_class.polymorphic_name, id_column => dupe.id) .update_all(id_column => primary.id) logger.info " moved #{moved} #{table} to primary" if moved > 0 end