Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,36 @@ curl 'http://localhost:8080/impacted_targets?from=main&to=my-feature-branch'
}
```

* `GET /impacted_targets_with_distances?from=<rev>&to=<rev>` — like `/impacted_targets`, but each
impacted target is annotated with its build-graph distance metrics: `targetDistance` (the number of
dependency hops to the nearest directly-changed target) and `packageDistance` (how many of those
hops cross a package boundary). Directly-changed targets sit at distance `0`. Requires the server
to have been started with `--trackDeps` (see below); otherwise this endpoint returns `400`. The
same optional `targetType` filter applies.

```bash
curl 'http://localhost:8080/impacted_targets_with_distances?from=main&to=my-feature-branch'
```

```json
{
"from": "9a1c0e2…",
"to": "3f7b8d4…",
"impactedTargets": [
{"label": "//foo:bar", "targetDistance": 0, "packageDistance": 0},
{"label": "//foo:baz", "targetDistance": 1, "packageDistance": 1}
]
}
```

Notes and current limitations:

* Distance metrics (`/impacted_targets_with_distances`) require the dependency-edge graph, which is
only tracked when the server is started with `--trackDeps`. Tracking deps grows each cached hash
entry, so it is opt-in. The flag is folded into the cache key, so enabling or disabling it never
reuses a previously cached entry of the other kind. This mirrors the `generate-hashes --depEdgesFile`
/ `get-impacted-targets --depEdgesFile` flow used by the CLI.

* The service checks out revisions inside `--workspacePath`, so point it at a dedicated clone, not a
working tree you edit. All workspace-mutating work (git checkout + `bazel query`) is serialized,
so a single instance answers one cold query at a time; the per-SHA cache absorbs the rest.
Expand Down Expand Up @@ -394,8 +422,8 @@ Command-line utility to analyze the state of the bazel build graph

```terminal
Usage: bazel-diff serve [-hkvV] [--[no-]excludeExternalTargets]
[--no-initial-fetch] [--[no-]useCquery]
[-b=<bazelPath>] --cacheDir=<cacheDir>
[--no-initial-fetch] [--[no-]trackDeps] [--[no-]
useCquery] [-b=<bazelPath>] --cacheDir=<cacheDir>
[--cqueryExpression=<cqueryExpression>]
[--fineGrainedHashExternalReposFile=<fineGrainedHashExte
rnalReposFile>] [--gitEngine=<gitEngine>]
Expand Down Expand Up @@ -456,6 +484,10 @@ targets between two git revisions, caching generated hashes per commit SHA.
-so, --bazelStartupOptions=<bazelStartupOptions>
Additional space separated Bazel client startup
options used when invoking Bazel
--[no-]trackDeps Track dependency edges and persist them per commit
SHA so build-graph distance metrics can be served
via /impacted_targets_with_distances. Increases
cache size and memory. Defaults to false.
--[no-]useCquery If true, use cquery instead of query when
generating dependency graphs.
-v, --verbose Display query string, missing files and elapsed time
Expand Down
21 changes: 18 additions & 3 deletions cli/src/main/kotlin/com/bazel_diff/cli/ServeCommand.kt
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,16 @@ class ServeCommand : Callable<Int> {
description = ["If true, exclude external targets (do not query //external:all-targets)."])
var excludeExternalTargets = false

@CommandLine.Option(
names = ["--trackDeps"],
negatable = true,
description =
[
"Track dependency edges and persist them per commit SHA so build-graph distance " +
"metrics can be served via /impacted_targets_with_distances. Increases cache " +
"size and memory. Defaults to false."])
var trackDeps = false

override fun call(): Int {
org.koin.core.context.GlobalContext.stopKoin()
startKoin {
Expand All @@ -191,7 +201,7 @@ class ServeCommand : Callable<Int> {
useCquery,
cqueryExpression,
keepGoing,
false,
trackDeps,
fineGrainedHashExternalRepos,
fineGrainedHashExternalReposFile,
excludeExternalTargets,
Expand Down Expand Up @@ -238,8 +248,10 @@ class ServeCommand : Callable<Int> {
storage,
computeConfigFingerprint(),
loadSeedFilepaths(),
ignoredRuleHashingAttributes)
val impactedTargetsService = ImpactedTargetsService(gitClient, hashService)
ignoredRuleHashingAttributes,
trackDeps)
val impactedTargetsService =
ImpactedTargetsService(gitClient, hashService, depsTracked = trackDeps)

val ready = AtomicBoolean(false)
val server = BazelDiffServer(port, impactedTargetsService) { ready.get() }
Expand Down Expand Up @@ -316,6 +328,9 @@ class ServeCommand : Callable<Int> {
append("ignoredRuleHashingAttributes=")
.append(ignoredRuleHashingAttributes.sorted().joinToString(","))
.append('\n')
// Distinct cache keys for deps-tracked vs deps-less runs: a deps-tracking server must never
// serve a deps-less cache entry (its distance queries would have no edges to traverse).
append("trackDeps=").append(trackDeps).append('\n')
append("version=").append(VersionProvider().version.firstOrNull() ?: "unknown").append('\n')
}
return sha256 { putBytes(canonical.toByteArray()) }.toHexString().take(12)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@ import org.koin.core.component.inject

data class TargetDistanceMetrics(val targetDistance: Int, val packageDistance: Int) {}

/** An impacted target paired with its build-graph distance metrics. */
data class ImpactedTargetWithDistance(
val label: String,
val targetDistance: Int,
val packageDistance: Int
)

class CalculateImpactedTargetsInteractor : KoinComponent {
private val gson: Gson by inject()
private val logger: Logger by inject()
Expand Down Expand Up @@ -94,6 +101,33 @@ class CalculateImpactedTargetsInteractor : KoinComponent {
toModuleGraphJson: String? = null,
excludeExternalTargets: Boolean = false,
) {
val impactedTargets =
computeImpactedTargetsWithDistances(
from,
to,
depEdges,
targetTypes,
fromModuleGraphJson,
toModuleGraphJson,
excludeExternalTargets)
outputWriter.use { writer -> writer.write(gson.toJson(impactedTargets)) }
}

/**
* Computes the impacted targets between [from] and [to] together with their build-graph distance
* metrics, filtered by [targetTypes]/[excludeExternalTargets] and ordered the same way the
* non-distance path orders its output. Returned in-memory so both the CLI writer path
* ([executeWithDistances]) and the query service consume identical data without reparsing JSON.
*/
fun computeImpactedTargetsWithDistances(
from: Map<String, TargetHash>,
to: Map<String, TargetHash>,
depEdges: Map<String, List<String>>,
targetTypes: Set<String>?,
fromModuleGraphJson: String? = null,
toModuleGraphJson: String? = null,
excludeExternalTargets: Boolean = false,
): List<ImpactedTargetWithDistance> {
val typeFilter = TargetTypeFilter(targetTypes, to)

// Quick check: if module graph JSON is identical, skip module change detection entirely
Expand Down Expand Up @@ -121,21 +155,12 @@ class CalculateImpactedTargetsInteractor : KoinComponent {
}

val ordering = impactedTargetOrdering(to, from)
impactedTargets
return impactedTargets
.filterKeys { typeFilter.accepts(it) }
.filterKeys { !excludeExternalTargets || !it.startsWith("//external:") }
.toSortedMap(ordering)
.let { filtered ->
outputWriter.use { writer ->
writer.write(
gson.toJson(
filtered.map {
mapOf(
"label" to it.key,
"targetDistance" to it.value.targetDistance,
"packageDistance" to it.value.packageDistance)
}))
}
.map {
ImpactedTargetWithDistance(it.key, it.value.targetDistance, it.value.packageDistance)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ import java.io.FileReader
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject

data class HashFileData(val hashes: Map<String, TargetHash>, val moduleGraphJson: String?)
data class HashFileData(
val hashes: Map<String, TargetHash>,
val moduleGraphJson: String?,
val depEdges: Map<String, List<String>> = emptyMap()
)

class DeserialiseHashesInteractor : KoinComponent {
private val gson: Gson by inject()
Expand Down Expand Up @@ -40,9 +44,19 @@ class DeserialiseHashesInteractor : KoinComponent {
val hashesMap: Map<String, String> = gson.fromJson(jsonObject.get("hashes"), hashesShape)
val hashes = hashesMap.mapValues { TargetHash.fromJson(it.value) }

val moduleGraphJson = jsonObject.getAsJsonObject("metadata")?.get("moduleGraphJson")?.asString
val metadata = jsonObject.getAsJsonObject("metadata")
val moduleGraphJson = metadata?.get("moduleGraphJson")?.asString

return HashFileData(hashes, moduleGraphJson)
// The query service persists the dependency-edge adjacency list (label -> direct dep labels)
// under metadata.depEdges when started with --trackDeps, so build-graph distance metrics can
// be computed on a cache hit without re-tracking deps. Absent for CLI-produced hashes.
val depEdges: Map<String, List<String>> =
metadata?.get("depEdges")?.let {
val depShape = object : TypeToken<Map<String, List<String>>>() {}.type
gson.fromJson<Map<String, List<String>>>(it, depShape)
} ?: emptyMap()

return HashFileData(hashes, moduleGraphJson, depEdges)
} else {
// Legacy format - just a flat map of hashes
val shape = object : TypeToken<Map<String, String>>() {}.type
Expand Down
28 changes: 26 additions & 2 deletions cli/src/main/kotlin/com/bazel_diff/server/BazelDiffServer.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ import org.koin.core.component.inject
* lame-ducked by a fatal git error.
* - `GET /impacted_targets?from=<rev>&to=<rev>[&targetType=Rule,SourceFile]` -- returns `{"from":
* <sha>, "to": <sha>, "impactedTargets": [...]}`.
* - `GET /impacted_targets_with_distances?from=<rev>&to=<rev>[&targetType=...]` -- like above but
* each impacted target is `{"label", "targetDistance", "packageDistance"}`. Requires the server
* to have been started with `--trackDeps`; returns `400` otherwise.
*
* Built on the JDK's [HttpServer] so the service needs no new third-party dependency. The handler
* pool is unbounded (cached) so that health checks are always served even while a long hash
Expand All @@ -47,6 +50,8 @@ class BazelDiffServer(
}
httpServer.createContext("/health", ::handleHealth)
httpServer.createContext("/impacted_targets", ::handleImpactedTargets)
httpServer.createContext(
"/impacted_targets_with_distances", ::handleImpactedTargetsWithDistances)
httpServer.start()
server = httpServer
logger.i { "bazel-diff query service listening on port ${boundPort()} " }
Expand Down Expand Up @@ -80,6 +85,24 @@ class BazelDiffServer(
}

private fun handleImpactedTargets(exchange: HttpExchange) =
handleQuery(exchange) { from, to, targetTypes ->
impactedTargetsProvider.getImpactedTargets(from, to, targetTypes)
}

private fun handleImpactedTargetsWithDistances(exchange: HttpExchange) =
handleQuery(exchange) { from, to, targetTypes ->
impactedTargetsProvider.getImpactedTargetsWithDistances(from, to, targetTypes)
}

/**
* Shared handling for the impacted-targets endpoints: enforces GET + readiness, parses and
* validates `from`/`to`/`targetType`, then serializes the result of [compute] as JSON, mapping
* the known failure modes to the appropriate status codes.
*/
private fun handleQuery(
exchange: HttpExchange,
compute: (from: String, to: String, targetTypes: Set<String>?) -> Any
) =
withExchange(exchange) {
if (!exchange.requestMethod.equals("GET", ignoreCase = true)) {
respondJson(exchange, 405, mapOf("error" to "method not allowed, use GET"))
Expand All @@ -102,8 +125,9 @@ class BazelDiffServer(
params["targetType"]?.split(",")?.map { it.trim() }?.filter { it.isNotEmpty() }?.toSet()

try {
val result = impactedTargetsProvider.getImpactedTargets(from, to, targetTypes)
respondJson(exchange, 200, result)
respondJson(exchange, 200, compute(from, to, targetTypes))
} catch (e: DistancesUnavailableException) {
respondJson(exchange, 400, mapOf("error" to (e.message ?: "distances unavailable")))
} catch (e: GitClientException) {
logger.e(e) { "git error computing impacted targets" }
respondJson(exchange, 400, mapOf("error" to "git error: ${e.message}"))
Expand Down
40 changes: 31 additions & 9 deletions cli/src/main/kotlin/com/bazel_diff/server/HashService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,17 @@ interface HashProvider {
* server started with different flags never reads another configuration's cached hashes.
* @param seedFilepaths seed files passed through to [BuildGraphHasher].
* @param ignoredRuleHashingAttributes rule attributes ignored when hashing.
* @param trackDeps when true, persist each revision's dependency-edge adjacency list in the cache
* so build-graph distance metrics can be served. Requires the hasher to have been built with
* dep-tracking on (so [TargetHash.deps] is populated).
*/
class HashService(
private val gitClient: GitClient,
private val storage: HashCacheStorage,
private val configFingerprint: String,
private val seedFilepaths: Set<Path>,
private val ignoredRuleHashingAttributes: Set<String>,
private val trackDeps: Boolean = false,
) : HashProvider, KoinComponent {
private val buildGraphHasher: BuildGraphHasher by inject()
private val bazelModService: BazelModService by inject()
Expand Down Expand Up @@ -89,24 +93,42 @@ class HashService(
buildGraphHasher.hashAllBazelTargetsAndSourcefiles(
seedFilepaths, ignoredRuleHashingAttributes)
val moduleGraphJson = runBlocking { bazelModService.getModuleGraphJson() }
val depEdges = depEdgesOf(hashes)
storage.put(
cacheKey(sha), serialize(hashes, moduleGraphJson).toByteArray(StandardCharsets.UTF_8))
HashFileData(hashes, moduleGraphJson)
cacheKey(sha),
serialize(hashes, moduleGraphJson, depEdges).toByteArray(StandardCharsets.UTF_8))
HashFileData(hashes, moduleGraphJson, depEdges)
}

/**
* The dependency-edge adjacency list (label -> direct dep labels) when [trackDeps] is on, else
* empty. Derived from [TargetHash.deps], the same way `generate-hashes --depEdgesFile` derives
* it.
*/
private fun depEdgesOf(hashes: Map<String, TargetHash>): Map<String, List<String>> =
if (trackDeps) hashes.mapValues { it.value.deps ?: emptyList() } else emptyMap()

/**
* Serializes hashes into the same JSON shape `generate-hashes` writes (see
* [com.bazel_diff.interactor.GenerateHashesInteractor]), so cached entries are interchangeable
* with hashes produced by the CLI. Target type is always included for the richest data.
* with hashes produced by the CLI. Target type is always included for the richest data. When
* [depEdges] is non-empty (i.e. --trackDeps), it is persisted under `metadata.depEdges` so
* distance metrics can be served on a cache hit.
*/
private fun serialize(hashes: Map<String, TargetHash>, moduleGraphJson: String?): String {
private fun serialize(
hashes: Map<String, TargetHash>,
moduleGraphJson: String?,
depEdges: Map<String, List<String>>
): String {
val serializedHashes = hashes.mapValues { it.value.toJson(true) }
val output =
if (moduleGraphJson != null) {
mapOf(
"hashes" to hashes.mapValues { it.value.toJson(true) },
"metadata" to mapOf("moduleGraphJson" to moduleGraphJson))
if (moduleGraphJson != null || depEdges.isNotEmpty()) {
val metadata = mutableMapOf<String, Any>()
if (moduleGraphJson != null) metadata["moduleGraphJson"] = moduleGraphJson
if (depEdges.isNotEmpty()) metadata["depEdges"] = depEdges
mapOf("hashes" to serializedHashes, "metadata" to metadata)
} else {
hashes.mapValues { it.value.toJson(true) }
serializedHashes
}
return gson.toJson(output)
}
Expand Down
Loading
Loading