ref: Remove Kafka/Arroyo consumer code

.envrc

@@ -2,12 +2,7 @@
 
 # Local development environment variables
 export KAFKA_BOOTSTRAP_SERVERS="localhost:9092"
-export KAFKA_GROUP_ID="launchpad-devservices"
-export KAFKA_TOPICS="preprod-artifact-events"
-export LAUNCHPAD_CREATE_KAFKA_TOPIC="1"
 export LAUNCHPAD_ENV="development"
-export LAUNCHPAD_HOST="0.0.0.0"
-export LAUNCHPAD_PORT="2218"
 export LAUNCHPAD_RPC_SHARED_SECRET="launchpad-also-very-long-value-haha"
 export SENTRY_BASE_URL="http://localhost:8000"
 

.github/workflows/ci.yml

@@ -64,75 +64,20 @@ jobs:
       - name: Install dependencies
         run: make install-dev
 
-      - name: Start Kafka with devservices
-        run: |
-          .venv/bin/devservices up --mode default
-
-          echo "Waiting for Kafka to be ready..."
-          KAFKA_READY=false
-          for i in {1..30}; do
-            KAFKA_CONTAINER=$(docker ps -qf "name=kafka")
-            if [ -z "$KAFKA_CONTAINER" ]; then
-              echo "Waiting for Kafka container to start... attempt $i/30"
-              sleep 2
-              continue
-            fi
-
-            HEALTH_STATUS=$(docker inspect --format='{{.State.Health.Status}}' $KAFKA_CONTAINER 2>/dev/null || echo "none")
-            if [ "$HEALTH_STATUS" = "healthy" ]; then
-              echo "Kafka is ready!"
-              KAFKA_READY=true
-              break
-            fi
-            echo "Waiting for Kafka health check (status: $HEALTH_STATUS)... attempt $i/30"
-            sleep 2
-          done
-
-          if [ "$KAFKA_READY" = "false" ]; then
-            echo "ERROR: Kafka failed to become healthy after 60 seconds"
-            echo "=== Docker containers ==="
-            docker ps -a
-            echo "=== Kafka logs ==="
-            docker logs $(docker ps -aqf "name=kafka") --tail 100 || echo "Could not get Kafka logs"
-            exit 1
-          fi
-
-          docker ps
-
       - name: Build Docker image with test fixtures
         run: docker build --build-arg TEST_BUILD=true -t launchpad-test .
 
       - name: Run all tests in Docker
         run: |
-          # Get Kafka container info for network connectivity
-          KAFKA_CONTAINER=$(docker ps -qf "name=kafka")
-          KAFKA_NETWORK=$(docker inspect $KAFKA_CONTAINER --format='{{range $net,$v := .NetworkSettings.Networks}}{{$net}}{{end}}')
-
           docker run --rm \
-            --network $KAFKA_NETWORK \
             -e LAUNCHPAD_ENV=development \
-            -e LAUNCHPAD_HOST=localhost \
-            -e LAUNCHPAD_PORT=2218 \
             -e LAUNCHPAD_RPC_SHARED_SECRET="launchpad-test-secret" \
-            -e KAFKA_BOOTSTRAP_SERVERS="kafka:9093" \
-            -e KAFKA_GROUP_ID="launchpad-test-ci" \
-            -e KAFKA_TOPICS="preprod-artifact-events" \
             --entrypoint python launchpad-test -m pytest -n auto tests/ --ignore=tests/e2e -v
 
-      - name: Show Kafka logs on failure
-        if: failure()
-        run: |
-          if docker ps -qf "name=kafka" >/dev/null 2>&1; then
-            echo "=== Kafka logs ==="
-            docker logs $(docker ps -qf "name=kafka") --tail 100
-          fi
-
       - name: Test CLI installation and basic functionality in Docker
         run: |
           docker run --rm \
             -e LAUNCHPAD_ENV=development \
-            -e LAUNCHPAD_HOST=localhost \
-            -e LAUNCHPAD_PORT=2218 \
             launchpad-test --help
 
   e2e:
@@ -141,106 +86,76 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
 
-      - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
-        with:
-          enable-cache: true
-          cache-dependency-glob: pyproject.toml
-
-      - name: Install dependencies
-        run: make install-dev
+      - name: Build E2E Docker images
+        run: docker compose -f docker-compose.e2e.yml build
 
-      - name: Start Kafka with devservices
+      - name: Start E2E services
         run: |
-          .venv/bin/devservices up --mode default
-
-          echo "Waiting for Kafka to be ready..."
-          KAFKA_READY=false
+          docker compose -f docker-compose.e2e.yml up -d kafka
+          echo "Waiting for Kafka to be healthy..."
           for i in {1..30}; do
-            KAFKA_CONTAINER=$(docker ps -qf "name=kafka")
-            if [ -z "$KAFKA_CONTAINER" ]; then
-              echo "Waiting for Kafka container to start... attempt $i/30"
-              sleep 2
-              continue
-            fi
-
-            HEALTH_STATUS=$(docker inspect --format='{{.State.Health.Status}}' $KAFKA_CONTAINER 2>/dev/null || echo "none")
-            if [ "$HEALTH_STATUS" = "healthy" ]; then
+            if docker compose -f docker-compose.e2e.yml ps kafka | grep -q "healthy"; then
               echo "Kafka is ready!"
-              KAFKA_READY=true
               break
             fi
-            echo "Waiting for Kafka health check (status: $HEALTH_STATUS)... attempt $i/30"
-            sleep 2
+            echo "Waiting for Kafka... attempt $i/30"
+            sleep 3
           done
-
-          if [ "$KAFKA_READY" = "false" ]; then
-            echo "ERROR: Kafka failed to become healthy after 60 seconds"
-            echo "=== Docker containers ==="
-            docker ps -a
-            echo "=== Kafka logs ==="
-            docker logs $(docker ps -aqf "name=kafka") --tail 100 || echo "Could not get Kafka logs"
+          if ! docker compose -f docker-compose.e2e.yml ps kafka | grep -q "healthy"; then
+            echo "ERROR: Kafka failed to become healthy"
+            docker compose -f docker-compose.e2e.yml logs kafka
             exit 1
           fi
 
-          docker ps
-
-      - name: Create Kafka topic
-        run: |
-          KAFKA_CONTAINER=$(docker ps -qf "name=kafka")
-          echo "Creating preprod-artifact-events topic..."
-          docker exec $KAFKA_CONTAINER kafka-topics --bootstrap-server localhost:9092 --create --topic preprod-artifact-events --partitions 1 --replication-factor 1 --if-not-exists
-          echo "Topic created successfully"
-
-      - name: Build E2E Docker images
-        run: docker compose -f docker-compose.e2e.yml build
-
-      - name: Start E2E services
-        run: |
-          # Start services in detached mode (minio, mock-sentry-api, launchpad)
-          docker compose -f docker-compose.e2e.yml up -d minio mock-sentry-api launchpad
+          docker compose -f docker-compose.e2e.yml up -d taskbroker minio mock-sentry-api
+          echo "Waiting for mock-sentry-api to be healthy..."
+          for i in {1..20}; do
+            if docker compose -f docker-compose.e2e.yml ps mock-sentry-api | grep -q "healthy"; then
+              echo "Mock Sentry API is ready!"
+              break
+            fi
+            echo "Waiting for Mock API... attempt $i/20"
+            sleep 3
+          done
+          if ! docker compose -f docker-compose.e2e.yml ps mock-sentry-api | grep -q "healthy"; then
+            echo "ERROR: Mock Sentry API failed to become healthy"
+            docker compose -f docker-compose.e2e.yml logs mock-sentry-api
+            exit 1
+          fi
 
-          # Wait for launchpad to be healthy
-          echo "Waiting for Launchpad to be healthy..."
-          LAUNCHPAD_READY=false
+          docker compose -f docker-compose.e2e.yml up -d launchpad
+          echo "Waiting for Launchpad worker to be healthy..."
           for i in {1..30}; do
             if docker compose -f docker-compose.e2e.yml ps launchpad | grep -q "healthy"; then
               echo "Launchpad is ready!"
-              LAUNCHPAD_READY=true
               break
             fi
             echo "Waiting for Launchpad... attempt $i/30"
             sleep 5
           done
-
-          if [ "$LAUNCHPAD_READY" = "false" ]; then
-            echo "ERROR: Launchpad failed to become healthy"
+          if ! docker compose -f docker-compose.e2e.yml ps launchpad | grep -q "healthy"; then
+            echo "ERROR: Launchpad worker failed to become healthy"
             docker compose -f docker-compose.e2e.yml logs launchpad
             exit 1
           fi
 
-          # Show running services
           docker compose -f docker-compose.e2e.yml ps
 
       - name: Run E2E tests
-        run: |
-          docker compose -f docker-compose.e2e.yml run --rm e2e-tests
-        timeout-minutes: 10
+        run: docker compose -f docker-compose.e2e.yml run --rm e2e-tests pytest e2e_tests/test_e2e_flow.py -v --tb=short
+        timeout-minutes: 5
 
       - name: Show service logs on failure
         if: failure()
         run: |
           echo "=== Launchpad logs ==="
           docker compose -f docker-compose.e2e.yml logs launchpad
+          echo "=== TaskBroker logs ==="
+          docker compose -f docker-compose.e2e.yml logs taskbroker
           echo "=== Mock API logs ==="
           docker compose -f docker-compose.e2e.yml logs mock-sentry-api
           echo "=== Kafka logs ==="
-          docker logs $(docker ps -qf "name=kafka") --tail 100 || echo "Could not get Kafka logs"
+          docker compose -f docker-compose.e2e.yml logs kafka
 
       - name: Cleanup E2E environment
         if: always()

Dockerfile

@@ -1,5 +1,5 @@
 # Build libdispatch for the strip binary
 FROM --platform=linux/amd64 debian:12-slim AS libdispatch-build
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
@@ -106,12 +106,9 @@
 # Switch to app user
 USER app
 
-# Expose ports
-EXPOSE 2218
-
 ARG LAUNCHPAD_VERSION_SHA
 ENV LAUNCHPAD_VERSION_SHA=$LAUNCHPAD_VERSION_SHA
 
 # Default command
 ENTRYPOINT ["launchpad"]
-CMD ["serve"]
+CMD ["worker"]

Makefile

@@ -29,15 +29,7 @@ test-unit:
 test-integration:
 	$(PYTHON_VENV) -m pytest -n auto tests/integration/ -v
 
-test-e2e:  ## Run E2E tests with Docker Compose (requires devservices up)
-	@echo "Ensuring devservices Kafka is running..."
-	@if ! docker ps | grep -q kafka; then \
-		echo "Starting devservices..."; \
-		devservices up --mode default; \
-		sleep 10; \
-	else \
-		echo "Kafka already running"; \
-	fi
+test-e2e:  ## Run E2E tests with Docker Compose
 	@echo "Starting E2E test environment..."
 	docker compose -f docker-compose.e2e.yml up --build --abort-on-container-exit --exit-code-from e2e-tests
 	@echo "Cleaning up E2E environment..."
@@ -107,22 +99,10 @@ run-cli:  ## Run the CLI tool (use ARGS="..." to pass arguments, DEBUG=1 to run
 		$(PYTHON_VENV) -m launchpad.cli $(ARGS); \
 	fi
 
-serve:  ## Start the Launchpad server with proper Kafka configuration
-	@echo "Ensuring Kafka topics exist..."
-	$(PYTHON_VENV) scripts/ensure_kafka_topics.py
-	@echo "Starting Launchpad server..."
-	$(PYTHON_VENV) -m launchpad.cli serve --verbose
-
-worker:  ## Start the Launchpad TaskWorker (no HTTP server)
+worker:  ## Start the Launchpad TaskWorker
 	@echo "Starting Launchpad TaskWorker..."
 	$(PYTHON_VENV) -m launchpad.cli worker --verbose
 
-test-kafka-message:  ## Send a test message to Kafka (requires Kafka running)
-	$(PYTHON_VENV) scripts/test_kafka.py --count 1
-
-test-kafka-multiple:  ## Send multiple test messages to Kafka
-	$(PYTHON_VENV) scripts/test_kafka.py --count 5 --interval 0
-
 test-download-artifact:
 	$(PYTHON_VENV) scripts/test_download_artifact.py --verbose
 
@@ -132,31 +112,6 @@ test-artifact-update:
 test-artifact-size-analysis-upload:
 	$(PYTHON_VENV) scripts/test_artifact_size_analysis_upload.py --verbose
 
-test-service-integration:  ## Run full integration test with devservices
-	@echo "Starting Kafka services via devservices..."
-	@devservices up
-	@echo "Waiting for Kafka to be ready..."
-	@sleep 10
-	@echo "Starting Launchpad server in background..."
-	@set -e; \
-	$(PYTHON_VENV) -m launchpad.cli serve --verbose & \
-	LAUNCHPAD_PID=$$!; \
-	echo "Launchpad started with PID: $$LAUNCHPAD_PID"; \
-	sleep 5; \
-	echo "Sending test messages..."; \
-	$(PYTHON_VENV) scripts/test_kafka.py --count 3 --interval 1; \
-	sleep 5; \
-	echo "Stopping Launchpad gracefully..."; \
-	kill -TERM $$LAUNCHPAD_PID 2>/dev/null && echo "SIGTERM sent" || echo "Process not found"; \
-	sleep 8; \
-	if kill -0 $$LAUNCHPAD_PID 2>/dev/null; then \
-		echo "Process still running, sending SIGKILL..."; \
-		kill -KILL $$LAUNCHPAD_PID 2>/dev/null || true; \
-		sleep 2; \
-	fi; \
-	echo "Stopping devservices..."; \
-	devservices down
-
 # Show current status
 status:
 	@echo "Python version: $$($(PYTHON_VENV) --version)"

devservices/config.yml

@@ -3,47 +3,36 @@ x-sentry-service-config:
   version: 0.1
   service_name: launchpad
   dependencies:
-    kafka:
-      description: Shared instance of kafka used by sentry services
-      remote:
-        repo_name: sentry-shared-kafka
-        branch: main
-        repo_link: https://github.com/getsentry/sentry-shared-kafka.git
     launchpad:
       description: Service that powers preprod artifact analysis
   modes:
-    default: [kafka]
-    containerized: [kafka, launchpad]
+    default: []
+    containerized: [launchpad]
 
 x-programs:
   devserver:
-    command: make serve
+    command: make worker
 
-# Assuming we only have remote dependencies (currently the case), then below is only relevant when running launchpad as a dependency of the monolith.
+# Below is only relevant when running launchpad as a dependency of the monolith.
 services:
   launchpad:
     image: ghcr.io/getsentry/launchpad:nightly
-    ports:
-      - 127.0.0.1:2218:2218 # Bind to localhost only - no external access
     command:
-      - serve
+      - worker
       - --verbose
-    healthcheck:
-      test: curl -f http://127.0.0.1:2218/health || exit 1
-      interval: 10s
-      timeout: 5s
-      retries: 3
-      start_period: 30s
     environment:
       PYTHONUNBUFFERED: 1
-      KAFKA_BOOTSTRAP_SERVERS: kafka:9093
-      KAFKA_GROUP_ID: launchpad-devservices
-      KAFKA_TOPICS: preprod-artifact-events
-      LAUNCHPAD_HOST: "0.0.0.0" # Inside container, but port binding restricts access
-      LAUNCHPAD_PORT: "2218"
       LAUNCHPAD_ENV: "development"
       SENTRY_BASE_URL: "http://host.docker.internal:8000"
       LAUNCHPAD_RPC_SHARED_SECRET: "launchpad-also-very-long-value-haha"
+      LAUNCHPAD_WORKER_RPC_HOST: "host.docker.internal:50051"
+      LAUNCHPAD_WORKER_CONCURRENCY: "1"
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f /tmp/health ]"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
     platform: linux/amd64
     extra_hosts:
       host.docker.internal: host-gateway