Skip to content

fix: Optimize TaskQueueDB matching with composite indices and EXISTS #8462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fstagni merged 2 commits into from
Feb 24, 2026
+20 −29
Merged

fix: Optimize TaskQueueDB matching with composite indices and EXISTS #8462

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
88ae595
fix: Add composite (TQId, Value) indices to TaskQueue multi-value tables
chrisburr Feb 19, 2026
fe366e0
fix: Refactor COUNT subqueries to EXISTS in TaskQueueDB
chrisburr Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 20 additions & 29 deletions src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ def __initializeDB(self):
self.__tablesDesc[tableName] = {
"Fields": {"TQId": "INTEGER(11) UNSIGNED NOT NULL", "Value": "VARCHAR(64) NOT NULL"},
"PrimaryKey": ["TQId", "Value"],
"Indexes": {"TaskIndex": ["TQId"], f"{multiField}Index": ["Value"]},
"Indexes": {"TaskIndex": ["TQId", "Value"], f"{multiField}Index": ["Value"]},
"ForeignKeys": {"TQId": "tq_TaskQueues.TQId"},
}

Expand Down Expand Up @@ -508,24 +508,21 @@ def __generateTQFindSQL(
for field in multiValueDefFields:
tableName = f"`tq_TQTo{field}`"
if field in tqDefDict and tqDefDict[field]:
firstQuery = (
"SELECT COUNT(%s.Value) \
FROM %s \
WHERE %s.TQId = `tq_TaskQueues`.TQId"
% (tableName, tableName, tableName)
)
grouping = f"GROUP BY {tableName}.TQId"
valuesList = List.uniqueElements([value.strip() for value in tqDefDict[field] if value.strip()])
numValues = len(valuesList)
secondQuery = "{} AND {}.Value in ({})".format(
firstQuery,
tableName,
",".join(["%s" % str(value) for value in valuesList]),
valuesStr = ",".join([f"{str(value)}" for value in valuesList])
# Exact count match: TQ must have exactly numValues entries
sqlCondList.append(
f"( SELECT COUNT({tableName}.Value) FROM {tableName} WHERE {tableName}.TQId = `tq_TaskQueues`.TQId ) = {numValues}"
)
# All values must be in the TQ
sqlCondList.append(
f"NOT EXISTS ( SELECT 1 FROM {tableName} WHERE {tableName}.TQId = `tq_TaskQueues`.TQId AND {tableName}.Value NOT IN ({valuesStr}) )"
)
sqlCondList.append(f"{numValues} = ({firstQuery} {grouping})")
sqlCondList.append(f"{numValues} = ({secondQuery} {grouping})")
else:
sqlCondList.append(f"`tq_TaskQueues`.TQId not in ( SELECT DISTINCT {tableName}.TQId from {tableName} )")
sqlCondList.append(
f"NOT EXISTS ( SELECT 1 FROM {tableName} WHERE {tableName}.TQId = `tq_TaskQueues`.TQId )"
)

# Handle RAM requirements matching
hasRAMRequirements = "MinRAM" in tqDefDict or "MaxRAM" in tqDefDict
Expand Down Expand Up @@ -859,14 +856,11 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None)
# that the GridCE matches explicitly so the COUNT can not be 0. In this case we skip this
# condition
sqlMultiCondList.append(
"( SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId = tq.TQId ) = 0"
% (fullTableN, fullTableN, fullTableN)
f"NOT EXISTS ( SELECT 1 FROM {fullTableN} WHERE {fullTableN}.TQId = tq.TQId )"
)
sqlMultiCondList.append(
self.__generateSQLSubCond(
"%%s IN ( SELECT %s.Value \
FROM %s \
WHERE %s.TQId = tq.TQId )"
"EXISTS ( SELECT 1 FROM %s WHERE %s.TQId = tq.TQId AND %s.Value = %%s )"
% (fullTableN, fullTableN, fullTableN),
tqMatchDict.get(field),
)
Expand All @@ -878,9 +872,7 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None)
if field in bannedJobMatchFields:
fullTableN = f"`tq_TQToBanned{field}s`"
csql = self.__generateSQLSubCond(
"%%s not in ( SELECT %s.Value \
FROM %s \
WHERE %s.TQId = tq.TQId )"
"NOT EXISTS ( SELECT 1 FROM %s WHERE %s.TQId = tq.TQId AND %s.Value = %%s )"
% (fullTableN, fullTableN, fullTableN),
tqMatchDict[field],
boolOp="OR",
Expand Down Expand Up @@ -920,7 +912,7 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None)

sqlCondList.append(
self.__generateSQLSubCond(
f"%%s not in ( SELECT {fullTableN}.Value FROM {fullTableN} WHERE {fullTableN}.TQId = tq.TQId )",
f"NOT EXISTS ( SELECT 1 FROM {fullTableN} WHERE {fullTableN}.TQId = tq.TQId AND {fullTableN}.Value = %%s )",
b_fv,
boolOp="OR",
)
Expand Down Expand Up @@ -952,15 +944,14 @@ def __generateTagSQLSubCond(tableName, tagMatchList):
"""Generate SQL condition where ALL the specified multiValue requirements must be
present in the matching resource list
"""
sql1 = f"SELECT COUNT({tableName}.Value) FROM {tableName} WHERE {tableName}.TQId=tq.TQId"
if not tagMatchList:
sql2 = sql1 + f" AND {tableName}.Value=''"
sql = f"NOT EXISTS ( SELECT 1 FROM {tableName} WHERE {tableName}.TQId=tq.TQId AND {tableName}.Value != '' )"
else:
if isinstance(tagMatchList, (list, tuple)):
sql2 = sql1 + f" AND {tableName}.Value in ( {','.join([('%s' % v) for v in tagMatchList])} )"
valuesStr = ",".join([f"{v}" for v in tagMatchList])
else:
sql2 = sql1 + f" AND {tableName}.Value={tagMatchList}"
sql = "( " + sql1 + " ) = (" + sql2 + " )"
valuesStr = tagMatchList
sql = f"NOT EXISTS ( SELECT 1 FROM {tableName} WHERE {tableName}.TQId=tq.TQId AND {tableName}.Value NOT IN ( {valuesStr} ) )"
return sql

@staticmethod
Expand Down
Loading