diff --git a/.github/workflows/daily_trading_bot.yml b/.github/workflows/daily_trading_bot.yml index ae6d1d7..7d92e8e 100644 --- a/.github/workflows/daily_trading_bot.yml +++ b/.github/workflows/daily_trading_bot.yml @@ -233,6 +233,36 @@ jobs: cat results/ai_runtime_plan.json + - name: Validate Cerebrium primary configuration + id: validate_cerebrium_config + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) }} + run: | + configured_inference_url="${CEREBRIUM_INFERENCE_URL}" + if [ -z "${configured_inference_url}" ]; then + configured_inference_url="${CEREBRIUM_TRAINED_MODEL_URL}" + fi + if [ -z "${configured_inference_url}" ]; then + configured_inference_url="${TRAINED_MODEL_INFERENCE_URL}" + fi + if [ -n "${configured_inference_url}" ]; then + echo "has_inference_url=true" >> "$GITHUB_OUTPUT" + echo "inference_url_source=configured" + else + echo "has_inference_url=false" >> "$GITHUB_OUTPUT" + echo "inference_url_source=missing" + echo "AI primary inference URL is missing across CEREBRIUM_INFERENCE_URL/CEREBRIUM_TRAINED_MODEL_URL/TRAINED_MODEL_INFERENCE_URL" >&2 + fi + { + echo "runtime_mode=${runtime_mode}" + echo "selected_backend=${selected_backend}" + echo "selected_compute_name=" + echo "selected_disk_gb=0" + echo "reason=${reason}" + } >> "$GITHUB_OUTPUT" + python -c 'import json, os, pathlib; payload={"runtime_mode": os.environ["runtime_mode"], "selected_backend": os.environ["selected_backend"], "selected_compute_name": "", "selected_disk_gb": 0, "reason": os.environ["reason"], "preflight": {"router_policy": "cerebrium_primary_then_distilled"}}; pathlib.Path("results/ai_runtime_plan.json").write_text(json.dumps(payload, indent=2)+"\n", encoding="utf-8"); print(json.dumps(payload, indent=2))' + cat results/ai_runtime_plan.json + + - name: Validate Cerebrium primary configuration id: validate_cerebrium_config if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) }} @@ -270,6 +300,14 @@ jobs: echo "AI reason: ${{ steps.plan_ai_runtime.outputs.reason }}" echo "AI inference URL configured: ${{ steps.validate_cerebrium_config.outputs.has_inference_url }}" + - name: Emit AI runtime decision + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) }} + run: | + echo "AI runtime mode: ${{ steps.plan_ai_runtime.outputs.runtime_mode }}" + echo "AI backend: ${{ steps.plan_ai_runtime.outputs.selected_backend }}" + echo "AI reason: ${{ steps.plan_ai_runtime.outputs.reason }}" + echo "AI inference URL configured: ${{ steps.validate_cerebrium_config.outputs.has_inference_url }}" + - name: Warm Cerebrium inference app id: warm_cerebrium_inference if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_config.outputs.has_inference_url == 'true' }} @@ -503,31 +541,7 @@ jobs: python main.py daily_job - name: Run AI Trading Bot (Emergency Distilled Retry) - id: run_ai_bot_distilled_retry - if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) && steps.run_ai_bot_distilled_local.outcome == 'failure' }} - continue-on-error: true - timeout-minutes: 60 - env: - SMTP_SERVER: ${{ secrets.SMTP_SERVER }} - SMTP_PORT: ${{ secrets.SMTP_PORT }} - SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} - SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} - RECIPIENT_EMAIL: ${{ secrets.RECIPIENT_EMAIL }} - TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} - TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} - ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} - LOG_LEVEL: ${{ secrets.LOG_LEVEL }} - DISABLE_CORE_TRADING: "1" - AI_RUNTIME_MODE: "distilled_local" - AI_ROUTER_REASON: "distilled_emergency_retry_after_failure" - TRAINED_MODEL_TIMEOUT_SECONDS: "450" - TRAINED_MODEL_MAX_RETRIES: "1" - run: | - python quant_platform/scripts/prepare_ai_only_retry.py --results-dir results - python main.py daily_job - - - name: Run AI Trading Bot (Emergency Distilled Retry) - id: run_ai_bot_distilled_retry + id: run_ai_bot_distilled_emergency_retry if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) && steps.run_ai_bot_distilled_local.outcome == 'failure' }} continue-on-error: true timeout-minutes: 60 @@ -551,7 +565,7 @@ jobs: python main.py daily_job - name: Send AI Failure Report - if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) && (((steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_config.outputs.has_inference_url == 'true') && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success')) && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full') && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_retry.outcome != 'success')) }} + if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) && (((steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_config.outputs.has_inference_url == 'true') && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success')) && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full') && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success')) }} continue-on-error: true env: SMTP_SERVER: ${{ secrets.SMTP_SERVER }} @@ -569,7 +583,7 @@ jobs: RUN_AI_CEREBRIUM_OUTCOME: ${{ steps.run_ai_bot_cerebrium.outcome }} RUN_AI_CEREBRIUM_RETRY_OUTCOME: ${{ steps.run_ai_bot_cerebrium_retry.outcome }} RUN_AI_DISTILLED_OUTCOME: ${{ steps.run_ai_bot_distilled_local.outcome }} - RUN_AI_DISTILLED_RETRY_OUTCOME: ${{ steps.run_ai_bot_distilled_retry.outcome }} + RUN_AI_DISTILLED_RETRY_OUTCOME: ${{ steps.run_ai_bot_distilled_emergency_retry.outcome }} run: | source_step="Run AI Trading Bot" detail="AI daily job failed before sending its report email. Run: ${RUN_URL}" @@ -673,7 +687,7 @@ jobs: RUN_AI_CEREBRIUM_RETRY_OUTCOME: ${{ steps.run_ai_bot_cerebrium_retry.outcome }} RUN_AI_STUDIO_OUTCOME: ${{ steps.run_ai_bot_in_lightning_studio.outcome }} RUN_AI_DISTILLED_OUTCOME: ${{ steps.run_ai_bot_distilled_local.outcome }} - RUN_AI_DISTILLED_RETRY_OUTCOME: ${{ steps.run_ai_bot_distilled_retry.outcome }} + RUN_AI_DISTILLED_RETRY_OUTCOME: ${{ steps.run_ai_bot_distilled_emergency_retry.outcome }} run: | core_ok="false" ai_ok="false" @@ -731,7 +745,7 @@ jobs: RUN_AI_CEREBRIUM_OUTCOME: ${{ steps.run_ai_bot_cerebrium.outcome }} RUN_AI_CEREBRIUM_RETRY_OUTCOME: ${{ steps.run_ai_bot_cerebrium_retry.outcome }} RUN_AI_DISTILLED_OUTCOME: ${{ steps.run_ai_bot_distilled_local.outcome }} - RUN_AI_DISTILLED_RETRY_OUTCOME: ${{ steps.run_ai_bot_distilled_retry.outcome }} + RUN_AI_DISTILLED_RETRY_OUTCOME: ${{ steps.run_ai_bot_distilled_emergency_retry.outcome }} run: | rc=0 if [ "${BASE_DEPS_OUTCOME}" != "success" ]; then