Skip to content

feat(evaluate): refuse a decision from a local policy #1302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ToreMerkely merged 2 commits into from
Oct 26, 2026
Merged

feat(evaluate): refuse a decision from a local policy #1302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d45befc
feat(evaluate): refuse a decision from a local policy
ToreMerkely Oct 26, 2026
116574a
docs(evaluate): say in the --control help that it needs --identifier
ToreMerkely Oct 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions cmd/kosli/evaluatePolicy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ An entrypoint annotated outside the policy package, or on a function, is refused
Pass ` + "`--control`" + ` to record the outcome as a decision against that control, in the
` + "`--flow`" + ` and ` + "`--trail`" + ` given. The decision is recorded where the policy runs, so
the verdict is never asserted from here. Without ` + "`--control`" + ` nothing is recorded.
Only a published policy can record a decision, so ` + "`--control`" + ` needs ` + "`--identifier`" + `.
Comment thread
ToreMerkely marked this conversation as resolved.

Use ` + "`--params`" + ` to pass values the policy reads as ` + "`data.params`" + `. For a
published policy, they must match the params schema of its current version.
Expand Down Expand Up @@ -107,10 +108,10 @@ kosli evaluate policy \
--api-token yourAPIToken \
--org yourOrgName

# evaluate a policy and record the outcome as a decision:
# evaluate a published policy and record the outcome as a decision:
kosli evaluate policy \
--context trail=yourFlowName/yourTrailName \
--path yourPolicyFile.rego \
--identifier yourPolicyIdentifier \
--control yourControlIdentifier \
--flow yourFlowName \
--trail yourTrailName \
Expand Down Expand Up @@ -207,6 +208,12 @@ func (o *evaluatePolicyOptions) run(out io.Writer) error {
return fmt.Errorf("--path takes a file or a directory on this machine, not a URL")
}
}
// Publishing is admin-only and validated, so files from here would let
// whoever records a decision also write the policy behind it.
if o.control != "" && len(policyRefs) > 0 {
return fmt.Errorf("a decision can only be recorded from a published policy: " +
"publish it with 'kosli publish rego-policy', then evaluate it with --identifier")
}

// Refused before the request: a format refused where it is printed would
// leave a decision recorded, and a rerun would record a second.
Expand Down
53 changes: 43 additions & 10 deletions cmd/kosli/evaluatePolicy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestTooManyContextsAreRefusedBefore
func (suite *EvaluatePolicyCommandTestSuite) TestControlRecordsADecisionWhereItIsTold() {
server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

_, _, _, _, err := executeCommandC(suite.cmd(server.URL,
_, _, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail "+
"--fingerprint b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"))

Expand All @@ -480,7 +480,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestControlRecordsADecisionWhereItI
func (suite *EvaluatePolicyCommandTestSuite) TestTheDecisionNameDefaultsToTheControl() {
server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

_, _, _, _, err := executeCommandC(suite.cmd(server.URL,
_, _, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail"))

require.NoError(suite.T(), err)
Expand All @@ -493,7 +493,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestTheDecisionNameDefaultsToTheCon
func (suite *EvaluatePolicyCommandTestSuite) TestAGivenNameIsSentAsGiven() {
server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

_, _, _, _, err := executeCommandC(suite.cmd(server.URL,
_, _, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail --name code-review-decision"))

require.NoError(suite.T(), err)
Expand Down Expand Up @@ -546,7 +546,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestWhatCannotBeAskedForIsRefusedBe
suite.Run(test.name, func() {
server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

_, combined, _, _, err := executeCommandC(suite.cmd(server.URL, test.extra))
_, combined, _, _, err := executeCommandC(suite.identifierCmd(server.URL, test.extra))

require.Error(suite.T(), err)
for _, says := range test.says {
Expand All @@ -562,13 +562,46 @@ func (suite *EvaluatePolicyCommandTestSuite) TestWhatCannotBeAskedForIsRefusedBe
}
}

func (suite *EvaluatePolicyCommandTestSuite) TestADecisionFromLocalFilesIsRefusedBeforeAnyRequest() {
for _, files := range []string{"--path", "--policy"} {
suite.Run(files, func() {
server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

_, combined, _, _, err := executeCommandC(fmt.Sprintf(
"evaluate policy --context trail=my-flow/my-trail %s testdata/policies/allow-all.rego "+
"--control SDLC-CTRL-0007 --flow release --trail my-trail "+
"--host %s --org test-org --api-token test-token --max-api-retries 0",
files, server.URL))

require.Error(suite.T(), err)
require.Contains(suite.T(), err.Error(), "kosli publish rego-policy")
require.Contains(suite.T(), err.Error(), "--identifier")
require.Empty(suite.T(), fake.created, "nothing is sent")
require.NotContains(suite.T(), combined, "RESULT")
})
}
}

// KOSLI_CONTROL binds to --control, so a pipeline that exports it is refused
// with --path as a typed --control is.
func (suite *EvaluatePolicyCommandTestSuite) TestAControlFromTheEnvironmentIsRefusedWithAPath() {
suite.T().Setenv("KOSLI_CONTROL", "SDLC-CTRL-0007")
server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

_, _, _, _, err := executeCommandC(suite.cmd(server.URL, "--flow release --trail my-trail"))

require.Error(suite.T(), err)
require.Contains(suite.T(), err.Error(), "kosli publish rego-policy")
require.Empty(suite.T(), fake.created, "nothing is sent")
}

// The environment satisfies the destination as the flags do.
func (suite *EvaluatePolicyCommandTestSuite) TestTheDestinationCanComeFromTheEnvironment() {
suite.T().Setenv("KOSLI_FLOW", "release")
suite.T().Setenv("KOSLI_TRAIL", "my-trail")
server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

_, _, _, _, err := executeCommandC(suite.cmd(server.URL, "--control SDLC-CTRL-0007"))
_, _, _, _, err := executeCommandC(suite.identifierCmd(server.URL, "--control SDLC-CTRL-0007"))

require.NoError(suite.T(), err)
decision := fake.created[0]["decision"].(map[string]any)
Expand All @@ -581,7 +614,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestTheDestinationCanComeFromTheEnv
func (suite *EvaluatePolicyCommandTestSuite) TestADenialAsksForItsDecisionToo() {
server, fake := newFakeEvaluations(suite.T(), verdictDenied)

_, combined, _, _, err := executeCommandC(suite.cmd(server.URL,
_, combined, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail --assert"))

require.Error(suite.T(), err, "--assert still fails the step")
Expand All @@ -593,7 +626,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestADenialAsksForItsDecisionToo()
func (suite *EvaluatePolicyCommandTestSuite) TestTheRecordedDecisionIsReported() {
server, _ := newFakeEvaluations(suite.T(), verdictAllowedWithDecision)

_, combined, _, _, err := executeCommandC(suite.cmd(server.URL,
_, combined, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail"))

require.NoError(suite.T(), err)
Expand All @@ -603,7 +636,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestTheRecordedDecisionIsReported()
func (suite *EvaluatePolicyCommandTestSuite) TestTheRecordedDecisionIsReportedInJson() {
server, _ := newFakeEvaluations(suite.T(), verdictAllowedWithDecision)

_, combined, _, _, err := executeCommandC(suite.cmd(server.URL,
_, combined, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail --output json"))

require.NoError(suite.T(), err)
Expand Down Expand Up @@ -692,7 +725,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestARefusalIsReportedAsTheServerPu
server := newRefusingServer(suite.T(), test.status,
fmt.Sprintf(`{"message":%q}`, test.message))

_, combined, _, _, err := executeCommandC(suite.cmd(server.URL,
_, combined, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail"))

require.Error(suite.T(), err)
Expand All @@ -710,7 +743,7 @@ func (suite *EvaluatePolicyCommandTestSuite) TestAClassifiedFailureIsNotADenial(
`{"id":"01EVAL","status":"failed","requested_at":1.0,"recorded_at":1.0,`+
`"error":{"kind":"compile","message":"policy.rego:4: unexpected token"}}`)

_, combined, _, _, err := executeCommandC(suite.cmd(server.URL,
_, combined, _, _, err := executeCommandC(suite.identifierCmd(server.URL,
"--control SDLC-CTRL-0007 --flow release --trail my-trail --assert"))

require.Error(suite.T(), err)
Expand Down
2 changes: 1 addition & 1 deletion cmd/kosli/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ Paths the list already matches stay excluded whatever is later added there, so k
policyOutputRuleFlag = "[optional] [experimental] Name of a policy rule to add to the JSON output, next to allow and violations. Can be repeated, or given as a comma-separated list. Subject to change."
policyAssertFlag = "[optional] Exit with a non-zero status when the policy denies. Without it the verdict is printed and the command exits 0."
policyContextFlag = "What to evaluate, as trail=<flow>/<trail>. Repeat it to evaluate several trails at one instant."
policyControlFlag = "[optional] Record the outcome as a decision against this control. Without it nothing is recorded."
policyControlFlag = "[optional] Record the outcome as a decision against this control. Only a published policy (--identifier) can record one. Without it nothing is recorded."
policyDecisionFlowFlag = "[conditional] The Kosli flow the decision is recorded in. Only required with --control."
policyDecisionTrailFlag = "[conditional] The Kosli trail the decision is recorded in. Only required with --control."
policyDecisionNameFlag = "[defaulted] The attestation name the decision is recorded under. Defaults to <control>-decision."
Expand Down
Loading