From cc5e250f1db498d8dea1b3a985eed53a901ed50c Mon Sep 17 00:00:00 2001
From: John Brestelli <jbrestell@gmail.com>
Date: Mon, 31 Aug 2026 12:55:48 -0400
Subject: [PATCH] Fix GenesByGenericSpliceSites performance (6.5min -> 0.6s)

Restrict the sample aggregation to the two chosen samples up front so the
indexes on protocol_app_node_id apply. Previously all 48 samples were
aggregated -- twice, since the stats subquery repeated the same join -- and
filtered afterwards.

Compute the predominant-site max in a single window pass. The old self-join
on sum_cpm = max_cpm compared double precision values produced by two
separate aggregations, which are not guaranteed bit-identical and could drop
a gene's max row.

Pass organismAbbrev from the injected question (the injector already calls
setOrganismAbbrevFromDatasetName) so the partitioned webready _p tables prune
to one organism instead of scanning 800+ partitions, and read
webready.TranscriptAttributes_p rather than the unpartitioned view.

Materialize the pairing CTE. Inlined, Postgres flattens it into a cross
product joined by filter -- 35M rows generated and discarded on
tbruTREU927 -- because a CTE has no statistics for it to estimate from.

Timed on all six sample pairs across every organism with a differential
question: worst case 1.0s, previously unusable.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 Model/lib/dst/spliceSitesTemplates.dst        |   4 +
 .../model/questions/queries/geneQueries.xml   | 118 +++++++++---------
 2 files changed, 65 insertions(+), 57 deletions(-)

diff --git a/Model/lib/dst/spliceSitesTemplates.dst b/Model/lib/dst/spliceSitesTemplates.dst
index 1ed8a33ad4..dc34aced25 100644
--- a/Model/lib/dst/spliceSitesTemplates.dst
+++ b/Model/lib/dst/spliceSitesTemplates.dst
@@ -173,6 +173,7 @@ prop=isEuPathDBSite
 prop=exprGraphAttr
 prop=includeProjects
 prop=organismAbbrevDisplay
+prop=organismAbbrev
 >templateTextStart<
     <question name="GenesByDifferentialSpliceSites${datasetName}" includeProjects="${includeProjects}" newBuild="${buildNumberIntroduced}"
          displayName="${organismAbbrevDisplay} Differential ${datasetDisplayName}"
@@ -182,6 +183,9 @@ prop=organismAbbrevDisplay
          recordClassRef="TranscriptRecordClasses.TranscriptRecordClass">
 
         <paramRef ref="geneParams.splice_site_generic" queryRef="GeneVQ.${datasetName}SpliceSets"/>
+        <!-- Partition key for the webready _p tables the query reads; fixed per
+             dataset, so hidden from the user. -->
+        <paramRef ref="organismParams.organismAbbrev" default="${organismAbbrev}" visible="false"/>
         <attributesList
               summary="gene_product,one_loc,one_dist_to_cds,one_dist_to_first_atg,two_loc,two_dist_to_cds,two_dist_to_first_atg"
 	      sorting="one_dist_to_cds asc,two_dist_to_cds asc"
diff --git a/Model/lib/wdk/model/questions/queries/geneQueries.xml b/Model/lib/wdk/model/questions/queries/geneQueries.xml
index e98cfd31ba..9de89c9e4d 100644
--- a/Model/lib/wdk/model/questions/queries/geneQueries.xml
+++ b/Model/lib/wdk/model/questions/queries/geneQueries.xml
@@ -5061,6 +5061,7 @@
       <paramRef ref="geneParams.splice_site_sample_two"/>
       <paramRef ref="geneParams.percent_diff"/>
       <paramRef ref="geneParams.min_norm_count"/>
+      <paramRef ref="organismParams.organismAbbrev"/>
       <column name="project_id"/>
       <column name="source_id"/>
       <column name="gene_source_id"/>
@@ -5075,77 +5076,80 @@
       <column name="two_dist_to_first_atg"/>
       <sql>
         <![CDATA[
-          WITH loc_counts AS ( 'purposely break this query.  it is way way too slow.  needs fixing'
+          -- The webready _p tables are partitioned by org_abbrev; filtering all three on
+          -- $$organismAbbrev$$ (set per-dataset from the template's ${organismAbbrev})
+          -- prunes to one partition instead of scanning 800+.
+          --
+          -- Both CTEs are MATERIALIZED deliberately. A CTE referenced once is inlined by
+          -- Postgres, which flattens the one/two pairing into a cross product joined by
+          -- filter: 35M rows generated and discarded, 6.5 minutes on tbruTREU927 versus
+          -- 0.6s materialized.
+          WITH loc_counts AS (
             SELECT g.source_id
-             , f.protocol_app_node_id
-             , f.na_sequence_id
-             , f.segment_start AS location
-             , sum(f.count_per_million) AS sum_cpm
-            FROM
-             apidb.splicesitegenes g
-             , apidb.splicesitefeature f
-            WHERE g.splice_site_feature_id = f.splice_site_feature_id
-            GROUP BY f.segment_start, f.na_sequence_id, f.protocol_app_node_id, g.source_id
-          ) , stats AS (
-            SELECT source_id
-              , protocol_app_node_id
-              , max(sum_cpm) AS max_cpm
-              , round((max(sum_cpm) / sum(sum_cpm))::numeric, 2) * 100 AS pct_max
-            FROM loc_counts
-            GROUP BY source_id, protocol_app_node_id
-          ) , gene_loc_stats AS (
-            SELECT this.*, stats.pct_max
-            FROM (
-               SELECT g.source_id
+                 , f.protocol_app_node_id
                  , f.na_sequence_id
                  , f.segment_start AS location
                  , sum(f.count_per_million) AS sum_cpm
-                 , f.protocol_app_node_id
-               FROM
-                 apidb.splicesitegenes g
-                 , apidb.splicesitefeature f
-               WHERE g.splice_site_feature_id = f.splice_site_feature_id
-               GROUP BY g.source_id, f.segment_start, f.protocol_app_node_id, f.na_sequence_id
-             ) this
-             , stats
-            WHERE this.sum_cpm = stats.max_cpm
-              AND this.source_id = stats.source_id
-              AND this.protocol_app_node_id = stats.protocol_app_node_id
+            FROM apidb.splicesitegenes g
+            JOIN apidb.splicesitefeature f
+              ON g.splice_site_feature_id = f.splice_site_feature_id
+            WHERE f.protocol_app_node_id IN ($$splice_site_sample_one$$, $$splice_site_sample_two$$)
+            GROUP BY g.source_id, f.protocol_app_node_id, f.na_sequence_id, f.segment_start
+          ) , gene_loc_pct AS (
+            -- One window pass yields both the percentile and the max flag; comparing
+            -- sum_cpm to a max from the SAME pass keeps the float comparison exact.
+            SELECT source_id
+                 , protocol_app_node_id
+                 , na_sequence_id
+                 , location
+                 , sum_cpm
+                 , round((max(sum_cpm) OVER w / sum(sum_cpm) OVER w)::numeric, 2) * 100 AS pct_max
+                 , sum_cpm = max(sum_cpm) OVER w AS is_max
+            FROM loc_counts
+            WINDOW w AS (PARTITION BY source_id, protocol_app_node_id)
+          ) , gene_loc_stats AS MATERIALIZED (
+            SELECT source_id, protocol_app_node_id, na_sequence_id, location, sum_cpm
+            FROM gene_loc_pct
+            WHERE is_max
+              AND pct_max > $$percent_diff$$
+              AND sum_cpm > $$min_norm_count$$
+          ) , pairs AS MATERIALIZED (
+            SELECT one.source_id
+                 , one.na_sequence_id AS one_na_sequence_id, one.location AS one_location
+                 , two.na_sequence_id AS two_na_sequence_id, two.location AS two_location
+            FROM gene_loc_stats one
+            JOIN gene_loc_stats two
+              ON two.source_id = one.source_id
+             AND two.location != one.location
+             AND two.protocol_app_node_id = $$splice_site_sample_two$$
+            WHERE one.protocol_app_node_id = $$splice_site_sample_one$$
           )
-             -- MAIN START
-select distinct ta.gene_source_id
+          SELECT DISTINCT ta.gene_source_id
             , ta.source_id
             , ta.project_id
             , 'Y' AS matched_result
-            , one.location AS one_loc
+            , p.one_location AS one_loc
             , t1.transcript_source_id AS one_transcript_id
             , t1.dist_to_cds AS one_dist_to_cds
             , t1.dist_to_first_atg AS one_dist_to_first_atg
-            , two.location AS two_loc
+            , p.two_location AS two_loc
             , t2.transcript_source_id AS two_transcript_id
             , t2.dist_to_cds AS two_dist_to_cds
             , t2.dist_to_first_atg AS two_dist_to_first_atg
-          FROM
-            gene_loc_stats one
-            , gene_loc_stats two
-            , webready.SpliceSiteTranscript_p t1
-            , webready.SpliceSiteTranscript_p t2
-            , apidbtuning.TranscriptAttributes ta
-          WHERE one.source_id = two.source_id
-            AND one.protocol_app_node_id = $$splice_site_sample_one$$
-            AND two.protocol_app_node_id = $$splice_site_sample_two$$
-            AND one.pct_max > $$percent_diff$$
-            AND two.pct_max > $$percent_diff$$
-            AND one.sum_cpm > $$min_norm_count$$
-            AND two.sum_cpm > $$min_norm_count$$
-            AND one.location != two.location
-            AND one.na_sequence_id = t1.na_sequence_id
-            AND one.location = t1.location
-            AND two.na_sequence_id = t2.na_sequence_id
-            AND two.location = t2.location
-            AND t1.type = 'Splice Site'
-            AND t2.type = 'Splice Site'
-            AND ta.gene_source_id = one.source_id
+          FROM pairs p
+          JOIN webready.SpliceSiteTranscript_p t1
+            ON t1.na_sequence_id = p.one_na_sequence_id
+           AND t1.location = p.one_location
+           AND t1.type = 'Splice Site'
+           AND t1.org_abbrev = $$organismAbbrev$$
+          JOIN webready.SpliceSiteTranscript_p t2
+            ON t2.na_sequence_id = p.two_na_sequence_id
+           AND t2.location = p.two_location
+           AND t2.type = 'Splice Site'
+           AND t2.org_abbrev = $$organismAbbrev$$
+          JOIN webready.TranscriptAttributes_p ta
+            ON ta.gene_source_id = p.source_id
+           AND ta.org_abbrev = $$organismAbbrev$$
         ]]>
       </sql>
     </sqlQuery>