From 5833c870182c036214c58c60181ca4cdeb9e2dc2 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Wed, 16 Sep 2026 18:15:07 +0200 Subject: [PATCH 1/4] Generate sitemap.xml Crawlers requesting /sitemap.xml hit the chapter catch-all route and got a 404. The previous attempt (PR #2349) shipped sitemap_generator with a static sitemap.xml.gz containing only the root URL, which went stale and did not fix the 404s for the bare /sitemap.xml path. Serve the sitemap from a dynamic endpoint instead: - Add SitemapsController and an xml.builder view rendering static pages, active chapters, workshops, events and meetings, each with a lastmod value from updated_at. - Fragment-cache each section, keyed by its records' latest updated_at and bounded by expires_in, so unchanged sections are not re-rendered and stale sections (for example after record deletions) self-heal. Timestamps use to_f because raw Time values in cache keys are stringified with second precision. - Add updated_at indexes on workshops, events and meetings so the cache key computation does not do a sequential scan per request, and set public HTTP caching (1 hour) on the response so the CDN absorbs repeat crawler fetches. - Enable gzip in the Heroku nginx config for proxied responses, so the uncompressed XML is compressed (the Heroku router does not compress). - Remove sitemap_generator and the committed public/sitemap.xml.gz; robots.txt now points at /sitemap.xml. --- Gemfile | 1 - Gemfile.lock | 4 - app/controllers/sitemaps_controller.rb | 8 ++ app/helpers/sitemaps_helper.rb | 17 ++++ app/views/sitemaps/show.xml.builder | 21 +++++ config/nginx.conf.erb | 9 ++ config/routes.rb | 2 + config/sitemap.rb | 32 -------- ...dated_at_indexes_for_sitemap_cache_keys.rb | 9 ++ db/schema.rb | 5 +- public/robots.txt | 2 +- public/sitemap.xml.gz | Bin 328 -> 0 bytes spec/requests/sitemap_spec.rb | 77 ++++++++++++++++++ 13 files changed, 148 insertions(+), 39 deletions(-) create mode 100644 app/controllers/sitemaps_controller.rb create mode 100644 app/helpers/sitemaps_helper.rb create mode 100644 app/views/sitemaps/show.xml.builder delete mode 100644 config/sitemap.rb create mode 100644 db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb delete mode 100644 public/sitemap.xml.gz create mode 100644 spec/requests/sitemap_spec.rb diff --git a/Gemfile b/Gemfile index 56e293d2a..325deb032 100644 --- a/Gemfile +++ b/Gemfile @@ -141,6 +141,5 @@ gem 'prawn' gem 'prawn-svg', '~> 0.35' gem 'carrierwave-aws', '~> 1.6' -gem 'sitemap_generator', '~> 7.1' gem 'solid_cache', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 49b807ec4..72fd672df 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -551,8 +551,6 @@ GEM activemodel (>= 7.0) simplecov (1.1.1) simplecov-lcov (0.9.0) - sitemap_generator (7.1.1) - builder (~> 3.0) slop (3.6.0) snaky_hash (2.0.6) hashie (>= 0.1.0, < 6) @@ -710,7 +708,6 @@ DEPENDENCIES simple_form simplecov simplecov-lcov - sitemap_generator (~> 7.1) solid_cache (~> 1.0) sprockets-rails stimulus-rails @@ -930,7 +927,6 @@ CHECKSUMS simple_form (5.4.1) sha256=58c3d229034c7e5545035c3271b6f030ef730c340b9d7d8eb730e0a385b20808 simplecov (1.1.1) sha256=25825ef13f0b2e74694d769817dad6ab8e90131dabdaa666e522fea105521e78 simplecov-lcov (0.9.0) sha256=7a77a31e200a595ed4b0249493056efd0c920601f53d2ef135ca34ee796346cd - sitemap_generator (7.1.1) sha256=ab2a133d512a7b33ed713a27a9977f61d8379c9943e3b79900243fc1c4f7f81d slop (3.6.0) sha256=76ccab03be66bfcab4838cdc07cab019cd3e192a3538266246749e79e4788803 snaky_hash (2.0.6) sha256=3663cae48cdef582b517025cf8a39d8789996eaf0b4ed89e2f0624836505654a solid_cache (1.0.10) sha256=bc05a2fb3ac78a6f43cbb5946679cf9db67dd30d22939ededc385cb93e120d41 diff --git a/app/controllers/sitemaps_controller.rb b/app/controllers/sitemaps_controller.rb new file mode 100644 index 000000000..6e4a80176 --- /dev/null +++ b/app/controllers/sitemaps_controller.rb @@ -0,0 +1,8 @@ +class SitemapsController < ApplicationController + def show + # Crawler-facing endpoint; let CDNs absorb repeat fetches. The fragment + # caches still bound the DB cost of a cache-miss render. + expires_in 1.hour, public: true + render xml: render_to_string(formats: [:xml]) + end +end diff --git a/app/helpers/sitemaps_helper.rb b/app/helpers/sitemaps_helper.rb new file mode 100644 index 000000000..4fb921f81 --- /dev/null +++ b/app/helpers/sitemaps_helper.rb @@ -0,0 +1,17 @@ +module SitemapsHelper + def sitemap_static_urls + [root_url, code_of_conduct_url, coaches_url, teaching_guide_url, faq_url, + attendance_policy_url, student_guide_url, privacy_policy_url, cookie_policy_url, + breach_code_of_conduct_url, volunteer_url, fundraise_url, donate_url, + codebar_stories_podcast_url] + end + + def sitemap_record_sections + [ + { name: 'chapters', records: Chapter.active, url: ->(chapter) { chapter_url(chapter.slug) } }, + { name: 'workshops', records: Workshop.all, url: ->(workshop) { workshop_url(workshop) } }, + { name: 'events', records: Event.all, url: ->(event) { event_url(event) } }, + { name: 'meetings', records: Meeting.all, url: ->(meeting) { meeting_url(meeting) } } + ] + end +end diff --git a/app/views/sitemaps/show.xml.builder b/app/views/sitemaps/show.xml.builder new file mode 100644 index 000000000..f2e0a060f --- /dev/null +++ b/app/views/sitemaps/show.xml.builder @@ -0,0 +1,21 @@ +xml.instruct! + +# `maximum(:updated_at).to_f` — a raw Time in a cache key is stringified with +# second precision, so changes made within the same second would be missed. +# expires_in makes sections stale after record deletions self-heal. +xml.urlset('xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9') do + cache 'sitemap/static', expires_in: 1.week do + sitemap_static_urls.each { |url| xml.url { xml.loc(url) } } + end + + sitemap_record_sections.each do |section| + cache ['sitemap', section[:name], section[:records].maximum(:updated_at).to_f], expires_in: 1.day do + section[:records].find_each do |record| + xml.url do + xml.loc(section[:url].call(record)) + xml.lastmod(record.updated_at.utc.iso8601) + end + end + end + end +end diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb index 80114eceb..f58067432 100644 --- a/config/nginx.conf.erb +++ b/config/nginx.conf.erb @@ -36,6 +36,15 @@ http { set $plausible_script_url https://plausible.io/js/pa-PFruVsE_br97UUCRXE_6f.js; set $plausible_event_url https://plausible.io/api/event; + # Gzip: the Heroku router does not compress, and Cloudflare only re-compresses + # when the origin has not already. Compress proxied responses here. + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 5; + gzip_min_length 1024; + gzip_types application/xml application/json text/plain text/css application/javascript text/javascript; + # Plausible: Proxy script.js (cached) location = /js/script.js { proxy_cache plausible_cache; diff --git a/config/routes.rb b/config/routes.rb index 1703ba314..4c02490c1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -192,6 +192,8 @@ post "check-in/w/:code" => "check_ins#create" get "check-in/w/:code/confirm" => "check_ins#confirm", as: :check_in_w_confirm + get 'sitemap.xml', to: 'sitemaps#show', as: :sitemap + get 'cookie-policy' => 'pages#show', id: 'cookie-policy' get 'privacy-policy' => 'pages#show', id: 'privacy-policy' get 'breach-code-of-conduct' => 'pages#show', id: 'breach-code-of-conduct' diff --git a/config/sitemap.rb b/config/sitemap.rb deleted file mode 100644 index 6b744bd2b..000000000 --- a/config/sitemap.rb +++ /dev/null @@ -1,32 +0,0 @@ -# This file is generated with `rake sitemap:install` -# To update `public/sitemap.xml.gz` run `rake sitemap:refresh`. -# -# See https://github.com/kjvarga/sitemap_generator?tab=readme-ov-file#rake-tasks - -# Set the host name for URL creation -SitemapGenerator::Sitemap.default_host = 'https://codebar.io' - -SitemapGenerator::Sitemap.create do - # Put links creation logic here. - # - # The root path '/' and sitemap index file are added automatically for you. - # Links are added to the Sitemap in the order they are specified. - # - # Usage: add(path, options={}) - # (default options are used if you don't specify) - # - # Defaults: :priority => 0.5, :changefreq => 'weekly', - # :lastmod => Time.now, :host => default_host - # - # Examples: - # - # Add '/articles' - # - # add articles_path, :priority => 0.7, :changefreq => 'daily' - # - # Add all articles: - # - # Article.find_each do |article| - # add article_path(article), :lastmod => article.updated_at - # end -end diff --git a/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb new file mode 100644 index 000000000..bd8be4f4c --- /dev/null +++ b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb @@ -0,0 +1,9 @@ +class AddUpdatedAtIndexesForSitemapCacheKeys < ActiveRecord::Migration[8.1] + disable_ddl_transaction! + + def change + add_index :workshops, :updated_at, algorithm: :concurrently + add_index :events, :updated_at, algorithm: :concurrently + add_index :meetings, :updated_at, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 2191103ab..235fce36a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_09_08_085057) do +ActiveRecord::Schema[8.1].define(version: 2026_09_16_120000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -235,6 +235,7 @@ t.index ["check_in_code"], name: "index_events_on_check_in_code", unique: true t.index ["date_and_time"], name: "index_events_on_date_and_time" t.index ["slug"], name: "index_events_on_slug", unique: true + t.index ["updated_at"], name: "index_events_on_updated_at" t.index ["venue_id"], name: "index_events_on_venue_id" end @@ -415,6 +416,7 @@ t.datetime "updated_at", precision: nil t.integer "venue_id" t.index ["slug"], name: "index_meetings_on_slug", unique: true + t.index ["updated_at"], name: "index_meetings_on_updated_at" t.index ["venue_id"], name: "index_meetings_on_venue_id" end @@ -652,6 +654,7 @@ t.index ["check_in_code"], name: "index_workshops_on_check_in_code", unique: true t.index ["created_by_id"], name: "index_workshops_on_created_by_id" t.index ["date_and_time"], name: "index_workshops_on_date_and_time" + t.index ["updated_at"], name: "index_workshops_on_updated_at" end add_foreign_key "invitation_log_entries", "invitation_logs" diff --git a/public/robots.txt b/public/robots.txt index 7da454548..fc588ec65 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,2 +1,2 @@ # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -Sitemap: https://codebar.io/sitemap.xml.gz +Sitemap: https://codebar.io/sitemap.xml diff --git a/public/sitemap.xml.gz b/public/sitemap.xml.gz deleted file mode 100644 index 0a516f7ebcfba48f2def5df7340a85a3ddada3be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 328 zcmV-O0k{4iiwFRrc*tk~1C^3bZ-XEhhrf#9x(H}zG7Y7-ox0PyWLJw+$@t@dRr~En z(JpS;!QS9~pXcRge93N9GC6bWgwb1&K@1YEnQ4S>x8UvYoIZgaC0|xL?vuc3SB*|= zLFav6Av6pF95&F{7UdXc=>28?$~vy=H&_Z{@hgR?6kco3> zE+P!eHKvsuGNb-_X@pS*vzvVu4L57;2rq5T2e($f zvdi4c9EDsRe4671wFA1Hs17>_VVeGt;|E)(>mL-Z^5ci1^X&Ak7E`VX<__4N7 z7=o^KB5UWBX=sk~BE@Bz6^A^pHbwPVJm9>-IB+vUSXb+opDh2R1Lq&|LeQc_dMk_- a{z5}cf_`oU#T01TZTby3UZS`P0{{RhnxIwy diff --git a/spec/requests/sitemap_spec.rb b/spec/requests/sitemap_spec.rb new file mode 100644 index 000000000..3d2d43119 --- /dev/null +++ b/spec/requests/sitemap_spec.rb @@ -0,0 +1,77 @@ +require 'rails_helper' + +RSpec.describe 'Sitemap' do + let!(:chapter) { Fabricate(:chapter) } + let!(:workshop) { Fabricate(:workshop_no_sponsor, chapter:) } + let!(:event) { Fabricate(:event) } + let!(:meeting) { Fabricate(:meeting) } + + it 'serves the sitemap as XML' do + get '/sitemap.xml' + + expect(response).to have_http_status(:ok) + expect(response.media_type).to eq('application/xml') + end + + it 'lists chapters, workshops, events, meetings and static pages' do + get '/sitemap.xml' + + body = response.body + expect(body).to include(root_url) + expect(body).to include(chapter_url(chapter.slug)) + expect(body).to include(workshop_url(workshop)) + expect(body).to include(event_url(event)) + expect(body).to include(meeting_url(meeting)) + expect(body).to include(code_of_conduct_url) + expect(body).to include(faq_url) + expect(body).to include(privacy_policy_url) + end + + it 'includes lastmod for records' do + get '/sitemap.xml' + + expect(response.body).to include("#{workshop.reload.updated_at.utc.iso8601}") + expect(response.headers['Cache-Control']).to include('public') + end + + it 'includes new records once their section cache key changes' do + with_fragment_caching do + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + + new_workshop = Fabricate(:workshop_no_sponsor, chapter:) + get '/sitemap.xml' + + expect(response.body).to include(workshop_url(new_workshop)) + end + end + + it 'stops listing a deleted record once the section cache expires' do + Fabricate(:workshop_no_sponsor, chapter:) + workshop.update_columns(updated_at: 1.hour.ago) + with_fragment_caching do + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + + workshop.destroy + travel 2.days do + get '/sitemap.xml' + end + + expect(response.body).not_to include(workshop_url(workshop)) + end + end + + def with_fragment_caching + old_cache = Rails.cache + old_perform_caching = ActionController::Base.perform_caching + Rails.cache = ActiveSupport::Cache::MemoryStore.new + ActionController::Base.cache_store = Rails.cache + ActionController::Base.perform_caching = true + yield + ensure + Rails.cache = old_cache + ActionController::Base.cache_store = old_cache + ActionController::Base.perform_caching = old_perform_caching + end +end From f66a4f4c3118bb3e76acc85a8f9ae19c779b2834 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Tue, 22 Sep 2026 14:44:14 +0200 Subject: [PATCH 2/4] Make the sitemap updated_at index migration re-runnable A partial CREATE INDEX CONCURRENTLY failure leaves the migration recorded as applied with no index present (observed locally), and a re-run then fails on the existing relation name. Add if_not_exists to each add_index so a retry after cleanup succeeds instead of failing the next deploy. --- ...20000_add_updated_at_indexes_for_sitemap_cache_keys.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb index bd8be4f4c..047d3f140 100644 --- a/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb +++ b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb @@ -2,8 +2,10 @@ class AddUpdatedAtIndexesForSitemapCacheKeys < ActiveRecord::Migration[8.1] disable_ddl_transaction! def change - add_index :workshops, :updated_at, algorithm: :concurrently - add_index :events, :updated_at, algorithm: :concurrently - add_index :meetings, :updated_at, algorithm: :concurrently + # if_not_exists keeps a re-run after a partial concurrent-index failure + # (an INVALID or half-built index) from failing the next deploy. + add_index :workshops, :updated_at, algorithm: :concurrently, if_not_exists: true + add_index :events, :updated_at, algorithm: :concurrently, if_not_exists: true + add_index :meetings, :updated_at, algorithm: :concurrently, if_not_exists: true end end From b2923b7a98cb26ec303f096fd7c660e454b023fa Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Tue, 22 Sep 2026 14:44:14 +0200 Subject: [PATCH 3/4] Redirect sitemap.xml.gz to the dynamic sitemap endpoint --- config/routes.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/routes.rb b/config/routes.rb index 85e85b84b..8fbc52ea5 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -192,6 +192,10 @@ post "check-in/w/:code" => "check_ins#create" get "check-in/w/:code/confirm" => "check_ins#confirm", as: :check_in_w_confirm + # Consumers pinned to the old static path keep working while crawlers + # migrate via robots.txt (e.g. an old Search Console submission). + get 'sitemap.xml.gz', to: redirect('/sitemap.xml', status: 301) + get 'sitemap.xml', to: 'sitemaps#show', as: :sitemap get 'cookie-policy' => 'pages#show', id: 'cookie-policy' From cfd86e987a56d87de9a46753bec87028aa00b6b8 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Tue, 22 Sep 2026 14:44:19 +0200 Subject: [PATCH 4/4] Harden the sitemap request specs Assert that inactive chapters are excluded, that the stale fragment is still served before the section cache expires (proving caching is engaged), and pin the max-age value of the Cache-Control header. --- spec/requests/sitemap_spec.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/spec/requests/sitemap_spec.rb b/spec/requests/sitemap_spec.rb index 3d2d43119..fff3e0a6c 100644 --- a/spec/requests/sitemap_spec.rb +++ b/spec/requests/sitemap_spec.rb @@ -2,6 +2,7 @@ RSpec.describe 'Sitemap' do let!(:chapter) { Fabricate(:chapter) } + let!(:inactive_chapter) { Fabricate(:chapter, active: false) } let!(:workshop) { Fabricate(:workshop_no_sponsor, chapter:) } let!(:event) { Fabricate(:event) } let!(:meeting) { Fabricate(:meeting) } @@ -27,11 +28,18 @@ expect(body).to include(privacy_policy_url) end + it 'excludes inactive chapters' do + get '/sitemap.xml' + + expect(response.body).not_to include(chapter_url(inactive_chapter.slug)) + end + it 'includes lastmod for records' do get '/sitemap.xml' expect(response.body).to include("#{workshop.reload.updated_at.utc.iso8601}") expect(response.headers['Cache-Control']).to include('public') + expect(response.headers['Cache-Control']).to include('max-age=3600') end it 'includes new records once their section cache key changes' do @@ -54,6 +62,12 @@ expect(response.body).to include(workshop_url(workshop)) workshop.destroy + + # The stale fragment is still served before expiry — proves caching is + # actually engaged, so the post-expiry assertion is meaningful. + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + travel 2.days do get '/sitemap.xml' end @@ -62,6 +76,13 @@ end end + it 'redirects the old sitemap.xml.gz path to the new endpoint' do + get '/sitemap.xml.gz' + + expect(response).to redirect_to('/sitemap.xml') + expect(response).to have_http_status(:moved_permanently) + end + def with_fragment_caching old_cache = Rails.cache old_perform_caching = ActionController::Base.perform_caching