Skip to content

Add Detailed Logging to Cases Retention Evaluation Job #8740

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sanjacornelius wants to merge 10 commits into
base: epic/FOUR-29101
Choose a base branch
from
+173 −12
Open

Add Detailed Logging to Cases Retention Evaluation Job #8740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
a2e353d
add details logs of EvaluateProcessRetentionJob
sanjacornelius Feb 20, 2026
a1223b0
Update log messages for better clarity
sanjacornelius Feb 20, 2026
dc5ca0d
Prevent job from running on system & template processs
sanjacornelius Feb 20, 2026
1ff22c9
Resolve ambiguous id field
sanjacornelius Feb 20, 2026
e5525b1
Add tests to ensure job does not run on templates and system processes
sanjacornelius Feb 20, 2026
4148e83
CursorBot Fix: Use exists() instead of count() for better performance
sanjacornelius Feb 24, 2026
343b1f0
Cursor Bot Fix: Fix ineffective system categories/templates test
sanjacornelius Feb 24, 2026
c08dc75
Merge branch 'epic/FOUR-29101' into task/FOUR-29113
sanjacornelius Mar 5, 2026
03695a0
PHPCS FIXER: Remove Space
sanjacornelius Mar 5, 2026
9579d01
Remove duplicate comments
sanjacornelius Mar 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 27 additions & 5 deletions ProcessMaker/Console/Commands/EvaluateCaseRetention.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
use Illuminate\Console\Command;
use ProcessMaker\Jobs\EvaluateProcessRetentionJob;
use ProcessMaker\Models\Process;
use ProcessMaker\Models\ProcessCategory;

class EvaluateCaseRetention extends Command
{
Expand Down Expand Up @@ -37,16 +38,37 @@ public function handle()
}

$this->info('Case retention policy is enabled');
$this->info('Evaluating and deleting cases past their retention period');
$this->info('Dispatching retention evaluation jobs for all processes');

// Process all processes when retention policy is enabled
// Processes without retention_period will default to one_year
Process::chunkById(100, function ($processes) {
// Get system category IDs to exclude
$systemCategoryIds = ProcessCategory::where('is_system', true)->pluck('id');

// Exclude processes that are templates or in system categories
$jobCount = 0;
$query = Process::where('is_template', '!=', 1);

// Exclude processes in system categories
if ($systemCategoryIds->isNotEmpty()) {
$query->where(function ($q) use ($systemCategoryIds) {
$q->where(function ($subQuery) use ($systemCategoryIds) {
$subQuery->whereNotIn('process_category_id', $systemCategoryIds)
->orWhereNull('process_category_id');
});
})
->whereDoesntHave('categories', function ($q) use ($systemCategoryIds) {
// Exclude processes with any category assignment to system categories
$q->whereIn('process_categories.id', $systemCategoryIds);
});
}

$query->chunkById(100, function ($processes) use (&$jobCount) {
foreach ($processes as $process) {
dispatch(new EvaluateProcessRetentionJob($process->id));
$jobCount++;
}
});

$this->info('Cases retention evaluation complete');
$this->info("Dispatched {$jobCount} retention evaluation job(s) to the queue");
$this->info('Jobs will be processed asynchronously by queue workers');
}
}
79 changes: 72 additions & 7 deletions ProcessMaker/Jobs/EvaluateProcessRetentionJob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,45 @@ public function __construct(public int $processId)
*/
public function handle(): void
{
$startTime = microtime(true);

Log::info('EvaluateProcessRetentionJob: Starting evaluation', [
'process_id' => $this->processId,
]);

// Only run if case retention policy is enabled
$enabled = config('app.case_retention_policy_enabled', false);
if (!$enabled) {
Log::info('EvaluateProcessRetentionJob: Case retention policy is disabled, skipping', [
'process_id' => $this->processId,
]);

return;
}

$process = Process::find($this->processId);
if (!$process) {
Log::error('CaseRetentionJob: Process not found', ['process_id' => $this->processId]);
Log::error('EvaluateProcessRetentionJob: Process not found', [
'process_id' => $this->processId,
]);

return;
}

// Skip template processes
if ($process->is_template) {
Log::info('EvaluateProcessRetentionJob: Skipping template process', [
'process_id' => $this->processId,
]);

return;
}

// Skip processes in system categories
if ($process->categories()->where('is_system', true)->exists()) {
Log::info('EvaluateProcessRetentionJob: Skipping process in system category', [
'process_id' => $this->processId,
]);

return;
}
Expand All @@ -54,15 +84,25 @@ public function handle(): void
default => 12, // Default to one_year
};

Log::info('EvaluateProcessRetentionJob: Retention configuration loaded', [
'process_id' => $this->processId,
'process_name' => $process->name,
'retention_period' => $retentionPeriod,
'retention_months' => $retentionMonths,
]);

// Default retention_updated_at to now if not set
// This means the retention policy applies from now for processes without explicit retention settings
$retentionUpdatedAt = isset($process->properties['retention_updated_at'])
? Carbon::parse($process->properties['retention_updated_at'])
: Carbon::now();

// Check if there are any process requests for this process
// If not, nothing to delete
if (!ProcessRequest::where('process_id', $this->processId)->exists()) {
Log::info('EvaluateProcessRetentionJob: No process requests found, nothing to evaluate', [
'process_id' => $this->processId,
]);

return;
}

Expand All @@ -80,15 +120,25 @@ public function handle(): void
// For cases created after retention_updated_at: cutoff is now - retention_period
$newCasesCutoff = $now->copy()->subMonths($retentionMonths);

Log::info('EvaluateProcessRetentionJob: Retention cutoff dates calculated', [
'process_id' => $this->processId,
'retention_updated_at' => $retentionUpdatedAt->toIso8601String(),
'old_cases_cutoff' => $oldCasesCutoff->toIso8601String(),
'new_cases_cutoff' => $newCasesCutoff->toIso8601String(),
'current_time' => $now->toIso8601String(),
]);

// Use subquery to get process request IDs
$processRequestSubquery = ProcessRequest::where('process_id', $this->processId)->select('id');

// Collect all ProcessRequest IDs that will be deleted (to delete them after all chunks are processed)
$processRequestIdsToDelete = [];
$totalDeleted = 0;
$chunkCount = 0;

CaseNumber::whereIn('process_request_id', $processRequestSubquery)
->where($this->buildRetentionQuery($retentionUpdatedAt, $oldCasesCutoff, $newCasesCutoff))
->chunkById(100, function ($cases) use (&$processRequestIdsToDelete) {
->chunkById(100, function ($cases) use (&$processRequestIdsToDelete, &$totalDeleted, &$chunkCount) {
$caseIds = $cases->pluck('id')->all();
$processRequestIds = $cases->pluck('process_request_id')->unique()->all();

Expand Down Expand Up @@ -116,10 +166,15 @@ public function handle(): void
$this->deleteTaskDraftMedia($draftIds);
$this->deleteTaskDrafts($processRequestTokenIds);

// TODO: Add logs to track the number of cases deleted
// Get deleted timestamp
// $deletedAt = Carbon::now();
// RetentionPolicyLog::record($process->id, $caseIds, $deletedAt);
$chunkCount++;
$chunkSize = count($caseIds);
$totalDeleted += $chunkSize;

Log::info('EvaluateProcessRetentionJob: Deleted chunk of cases', [
'process_id' => $this->processId,
'chunk_number' => $chunkCount,
'cases_deleted' => $chunkSize,
]);
});

// Delete ProcessRequests after all chunks are processed
Expand All @@ -142,6 +197,16 @@ public function handle(): void
$this->dispatchSavedSearchRecount();
}
}

$endTime = microtime(true);
$executionTime = round(($endTime - $startTime) * 1000, 2);

Log::info('EvaluateProcessRetentionJob: Evaluation completed', [
'process_id' => $this->processId,
'total_cases_deleted' => $totalDeleted,
'total_chunks_processed' => $chunkCount,
'execution_time_ms' => $executionTime,
]);
}

/**
Expand Down
74 changes: 74 additions & 0 deletions tests/Jobs/EvaluateProcessRetentionJobTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
use ProcessMaker\Models\Media;
use ProcessMaker\Models\Notification;
use ProcessMaker\Models\Process;
use ProcessMaker\Models\ProcessCategory;
use ProcessMaker\Models\ProcessRequest;
use ProcessMaker\Models\ProcessRequestToken;
use Tests\TestCase;
Expand Down Expand Up @@ -522,4 +523,77 @@ public function testItDeletesAllRelatedRecordsWhenCasesExceedRetentionPeriod()
// Check that notification has been deleted
$this->assertNull(Notification::find($notification->id), 'The notification should be deleted');
}

public function testItDoesNotRunForTemplates()
{
// Create a template process
$process = Process::factory()->create([
'is_template' => 1,
'properties' => [
'retention_period' => self::RETENTION_PERIOD,
],
]);
$process->save();
$process->refresh();

// Create a process request
$processRequest = ProcessRequest::factory()->create();
$processRequest->process_id = $process->id;
$processRequest->save();
$processRequest->refresh();

// Create an old case that should be deleted if this wasn't a template
// 13 months ago is older than 1 year retention period
$oldCaseDate = Carbon::now()->subMonths(13);
$oldCase = CaseNumber::factory()->create([
'process_request_id' => $processRequest->id,
]);
$oldCase->created_at = $oldCaseDate;
$oldCase->save();

// Dispatch the job
EvaluateProcessRetentionJob::dispatchSync($process->id);

// The case should NOT be deleted because the process is a template
$this->assertNotNull(CaseNumber::find($oldCase->id), 'The case should NOT be deleted because the process is a template');
$this->assertDatabaseCount('case_numbers', 2);
}

public function testItDoesNotRunForProcessesInSystemCategories()
{
$category = ProcessCategory::factory()->create([
'is_system' => 1,
]);
// Create a process in a system category
$process = Process::factory()->create([
'process_category_id' => $category->id,
'properties' => [
'retention_period' => self::RETENTION_PERIOD,
],
]);
$process->save();
$process->refresh();

// Create a process request
$processRequest = ProcessRequest::factory()->create();
$processRequest->process_id = $process->id;
$processRequest->save();
$processRequest->refresh();

// Create an old case that should be deleted if this wasn't in a system category
// 13 months ago is older than 1 year retention period
$oldCaseDate = Carbon::now()->subMonths(13);
$oldCase = CaseNumber::factory()->create([
'process_request_id' => $processRequest->id,
]);
$oldCase->created_at = $oldCaseDate;
$oldCase->save();

// Dispatch the job
EvaluateProcessRetentionJob::dispatchSync($process->id);

// The case should NOT be deleted because the process is in a system category
$this->assertNotNull(CaseNumber::find($oldCase->id), 'The case should NOT be deleted because the process is in a system category');
$this->assertDatabaseCount('case_numbers', 2);
}
}
Loading