Update to Python 3.13

.github/workflows/tmm-api-deploy-aws.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Typing - mypy
         run: uv run mypy .
       - name: Lint - ruff
-        run: uv run ruff .
+        run: uv run ruff check
       - name: Lint - black
         run: uv run black --check .
 

docker-compose.int-test.yaml

@@ -11,11 +11,11 @@ services:
     environment:
       INFLUX_CONFIG_FILE: /run/secrets/influx_config      
       TMM_BUCKET_FILE: /run/secrets/tmm_bucket
-      TMM_API_KEY_FILE: /run/secrets/tmm_api_key
+      TMM_AUTH_SECRET: /run/secrets/tmm_auth_secret
     secrets:
       - influx_config
       - tmm_bucket
-      - tmm_api_key
+      - tmm_auth_secret
     ports:
       - "5000:5000"
 
@@ -29,5 +29,5 @@ secrets:
     file: ./examples/influx_config.ini
   tmm_bucket:
     file: ./examples/tmm_bucket
-  tmm_api_key:
-    file: ./examples/tmm_api_key
+  tmm_auth_secret:
+    file: ./examples/tmm_auth_secret

docker-compose.yaml

@@ -13,13 +13,13 @@ services:
     environment:
       INFLUX_CONFIG_FILE: /run/secrets/influx_config
       TMM_BUCKET_FILE: /run/secrets/tmm_bucket
-      TMM_API_KEY_FILE: /run/secrets/tmm_api_key
+      TMM_AUTH_SECRET_FILE: /run/secrets/tmm_auth_secret
       DEVELOPMENT_MODE: true
       ENABLE_WRITE: true
     secrets:
       - influx_config
       - tmm_bucket
-      - tmm_api_key
+      - tmm_auth_secret
     ports:
       - "5000:5000"
 
@@ -92,5 +92,5 @@ secrets:
     file: ./examples/influx_config.ini
   tmm_bucket:
     file: ./examples/tmm_bucket
-  tmm_api_key:
-    file: ./examples/tmm_api_key
+  tmm_auth_secret:
+    file: ./examples/tmm_auth_secret

examples/tmm_auth_secret

@@ -0,0 +1 @@
+secret

services/tmm-api/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-alpine as base
+FROM python:3.13-alpine as base
 
 ENV PYTHONFAULTHANDLER=1 \
     PYTHONHASHSEED=random \

services/tmm-api/Makefile

@@ -13,7 +13,7 @@ dist/lambda_package.zip: pyproject.toml uv.lock $(shell find tmm_api)
 
 	uv build --wheel
 
-	uvx -p 3.11.13 pip install -t dist/lambda_package dist/*.whl 
+	uvx -p 3.13 pip install -t dist/lambda_package dist/*.whl 
 #	uv pip install -t dist/lambda_package dist/*.whl 
 
 	cd dist/lambda_package && \

services/tmm-api/noxfile.py

@@ -18,7 +18,7 @@ def black(session):
 @nox.session(python=False)
 def ruff(session):
     session.run("uv", "sync", "--quiet", external=True)
-    session.run("uv", "run", "ruff", ".", external=True)
+    session.run("uv", "run", "ruff", "check", external=True)
 
 
 @nox.session(python=False)

services/tmm-api/pyproject.toml

@@ -3,27 +3,27 @@ name = "tmm-api"
 version = "0.1.0"
 description = "Teuto Moisture Map Api"
 authors = [{ name = "code4bielefeld" }]
-requires-python = "~=3.11.0"
+requires-python = "~=3.13.0"
 license = "MIT"
 dependencies = [
-    "influxdb-client>=1.31.0,<2",
-    "fastapi>=0.87.0,<0.88",
-    "uvicorn>=0.19.0,<0.20",
-    "python-multipart>=0.0.5,<0.0.6",
-    "mangum>=0.17.0,<0.18",
-    "cachetools>=5.3.0,<6",
-    "boto3>=1.28.26,<2",
+    "influxdb-client>=1.31.0",
+    "fastapi>=0.87.0",
+    "uvicorn>=0.19.0",
+    "python-multipart>=0.0.5",
+    "mangum>=0.17.0",
+    "cachetools>=5.3.0",
+    "boto3>=1.28.26",
 ]
 
 [dependency-groups]
 dev = [
-    "black>=22.10.0,<23",
-    "boto3-stubs>=1.28.26,<2",
-    "mypy>=0.982,<0.983",
-    "nox>=2023.4.22,<2024",
-    "ruff>=0.0.128,<0.0.129",
-    "pytest>=7.1.2,<8",
-    "watchfiles>=0.18.1,<0.19",
+    "black>=22.10.0",
+    "boto3-stubs>=1.28.26",
+    "mypy>=0.982",
+    "nox>=2023.4.22",
+    "ruff>=0.0.128",
+    "pytest>=7.1.2",
+    "watchfiles>=0.18.1",
 ]
 
 [build-system]
@@ -32,6 +32,19 @@ build-backend = "hatchling.build"
 
 [tool.ruff]
 line-length = 120
+exclude = [
+    ".git",
+    "__pycache__",
+    "*.egg-info",
+    ".nox",
+    ".pytest_cache",
+    ".mypy_cache",
+    "dist",
+    ".venv",
+    "scratch",
+]
+
+[tool.ruff.lint]
 select = [
     "C",
     "E",
@@ -44,17 +57,6 @@ select = [
     #,"D" # todo
 ]
 ignore = ["E501"]
-exclude = [
-    ".git",
-    "__pycache__",
-    "*.egg-info",
-    ".nox",
-    ".pytest_cache",
-    ".mypy_cache",
-    "dist",
-    ".venv",
-    "scratch",
-]
 
 [tool.black]
 line-length = 120

services/tmm-api/terraform/Makefile

@@ -18,4 +18,4 @@ init:
 	terraform init
 
 clean:
-	rm -rf dev.tfplan $(ENV).tfplan
+	rm -rf dev.tfplan prod.tfplan $(ENV).tfplan

services/tmm-api/terraform/lambda.tf

@@ -24,7 +24,8 @@ resource "aws_s3_object" "lambda_source_code" {
 
 resource "aws_lambda_function" "api_lambda_function" {
   function_name = "${local.prefix}_lambda"
-  runtime       = "python3.11"
+  runtime       = "python3.13"
+  timeout       = 10 
   role          = aws_iam_role.api_lambda_function_role.arn
 
   handler          = local.lambda_function_handler

services/tmm-api/test/ttn/test_TTNMessage.py

@@ -10,7 +10,9 @@ class TestTTNMessageParsing(unittest.TestCase):
     def test_parse_payload(self):
         filepath = f"{dirname(__file__)}/../dragino_ttn_payload.json"
 
-        message = TTNMessage.parse_file(filepath)
+        with open(filepath) as f:
+            json_str = f.read()
+        message = TTNMessage.model_validate_json(json_str)
         measurement = message.to_measurement()
 
         self.assertEqual(measurement.battery, 3.304, "Battery value not ok")

services/tmm-api/tmm_api/app.py

@@ -1,5 +1,5 @@
 # app.py
-from logging import StreamHandler, getLogger
+from logging import getLogger
 import logging
 from threading import Lock
 from cachetools import TTLCache, cached
@@ -25,8 +25,8 @@
 app = FastAPI(title="BodenfeuchteAPI")
 
 logger = getLogger(__name__)
-logger.setLevel(logging.INFO)
-logger.addHandler(StreamHandler())
+getLogger().setLevel(logging.INFO)
+# getLogger().addHandler(StreamHandler())
 logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%m/%d/%Y %I:%M:%S %p")
 
 
@@ -46,6 +46,7 @@
     )
     def ttn_dragino(message: TTNMessage, TMM_APIKEY: str = Header()):  # noqa: B008,N803
         if not is_auth(message.end_device_ids.device_id, TMM_APIKEY):
+            logger.info(f"Rejected sensor data for device: {message.end_device_ids.device_id} due to invalid API key")
             return JSONResponse(status_code=401, content={"error": "Unauthorized"})
         measurement = message.to_measurement()
         measurement.write_to_influx()
@@ -60,10 +61,13 @@ def ttn_dragino(message: TTNMessage, TMM_APIKEY: str = Header()):  # noqa: B008,
 @app.get("/mapData", response_model=MapData, response_model_exclude_none=True)
 @cached(cache=TTLCache(maxsize=1000, ttl=600), lock=Lock())
 def map_data(days: int = 1):
+    logger.debug(f"Exporting map data for the last {days} days")
     """
     This method exports the moisture data for the current day.
     """
-    return export_moisture_map_data(days)
+    data = export_moisture_map_data(days)
+    logger.debug(f"Exported {len(data.records)} records")
+    return data
 
 
 @app.get("/sensorData/{sensor}", response_model=SensorReport)

services/tmm-api/tmm_api/common/secrets.py

@@ -7,7 +7,7 @@
 @cached(cache={})
 def get_secret(name: str) -> str | None:
     if secret_file := os.environ.get(f"{name}_FILE"):
-        return open(secret_file, "r").read().strip()
+        return open(secret_file).read().strip()
     elif ssm_param_name := os.environ.get(f"{name}_SSM_NAME"):
         return get_aws_ssm_parameter(ssm_param_name)
     else:

services/tmm-api/tmm_api/common/sensor_metadata.py

@@ -23,7 +23,7 @@ def get_sensors_metadata():
         }
     except Exception as e:
         print(f"Error retrieving sensor metadata: {e}")
-        raise HTTPException(status_code=500, detail="Error retrieving sensor metadata")
+        raise HTTPException(status_code=500, detail="Error retrieving sensor metadata") from e
 
 
 def convert_dyno_to_plain(dynamo_item):
@@ -47,4 +47,4 @@ def write_sensor_metadata(sensor):
         return response
     except Exception as e:
         print(f"Error saving sensor metadata: {e}")
-        raise HTTPException(status_code=500, detail="Error saving sensor metadata")
+        raise HTTPException(status_code=500, detail="Error saving sensor metadata") from e

services/tmm-api/tmm_api/export/map_overview.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 from datetime import datetime
+from logging import getLogger
 from zoneinfo import ZoneInfo
 
 from tmm_api.common.influx import get_influx_client
@@ -11,6 +12,8 @@
 fieldname = "soil_moisture"
 bucket = get_secret("TMM_BUCKET")
 
+logger = getLogger(__name__)
+
 
 @dataclass
 class Record:
@@ -67,20 +70,23 @@ def export_moisture_map_data(days: int = 1) -> MapData:
         |> group(columns: ["device"])
 
         //latest |> yield(name: "latest")
-        
+
         joined = join.inner(
             left: average,
             right: latest,
             on: (l, r) => l.device == r.device,
             as: (l, r) => ({{r with last_update: r._time, avg_soil_moisture: l.soil_moisture, avg_soil_temperature: l.soil_temperature, avg_soil_conductivity: l.soil_conductivity, avg_battery: l.battery}}),
         ) |> drop(columns: ["_time"])
-                
+
         joined |> yield(name: "joined")
     """
     with get_influx_client() as client:
+        logger.debug("Executing query to export map data")
         query_api = client.query_api()
         results = query_api.query(query=query)
+        logger.debug("Retrieved results from InfluxDB")
         metadata = get_sensors_metadata()
+        logger.debug(f"Retrieved metadata for {len(metadata)} sensors")
 
         def get_field(sensor_id, field) -> typing.Any:
             sensor = metadata.get(sensor_id)