From 31e45a5096c31bffc3a16836217ac89cf829ed75 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:10:33 +0000 Subject: [PATCH 01/16] localstack --- poetry.lock | 301 +----------------- pyproject.toml | 1 - .../config/config.py | 12 +- tests/docker-compose.yml | 37 ++- tests/integration/conftest.py | 167 ++++++---- 5 files changed, 134 insertions(+), 384 deletions(-) diff --git a/poetry.lock b/poetry.lock index 20e8d40c4..f019e41b4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -217,18 +217,6 @@ files = [ [package.extras] tests = ["mypy (>=1.14.0)", "pytest", "pytest-asyncio"] -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -groups = ["dev"] -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - [[package]] name = "attrs" version = "25.3.0" @@ -454,18 +442,6 @@ files = [ {file = "brunns_row-2.1.0-py3-none-any.whl", hash = "sha256:0953f231c812658a7cfd3240f1f6fac92e7b5bba247cfc22a4afa134f2a540f3"}, ] -[[package]] -name = "cachetools" -version = "6.1.0" -description = "Extensible memoizing collections and decorators" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "cachetools-6.1.0-py3-none-any.whl", hash = "sha256:1c7bb3cf9193deaf3508b7c5f2a79986c13ea38965c5adcff1f84519cf39163e"}, - {file = "cachetools-6.1.0.tar.gz", hash = "sha256:b4c4f404392848db3ce7aac34950d17be4d864da4b8b66911008e430bc544587"}, -] - [[package]] name = "certifi" version = "2025.6.15" @@ -684,7 +660,7 @@ version = "8.2.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, @@ -949,54 +925,6 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] -[[package]] -name = "dill" -version = "0.3.6" -description = "serialize all of python" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, - {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] - -[[package]] -name = "dnslib" -version = "0.9.26" -description = "Simple library to encode/decode DNS wire-format packets" -optional = false -python-versions = "*" -groups = ["dev"] -files = [ - {file = "dnslib-0.9.26-py3-none-any.whl", hash = "sha256:e68719e633d761747c7e91bd241019ef5a2b61a63f56025939e144c841a70e0d"}, - {file = "dnslib-0.9.26.tar.gz", hash = "sha256:be56857534390b2fbd02935270019bacc5e6b411d156cb3921ac55a7fb51f1a8"}, -] - -[[package]] -name = "dnspython" -version = "2.7.0" -description = "DNS toolkit" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86"}, - {file = "dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1"}, -] - -[package.extras] -dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "hypercorn (>=0.16.0)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "quart-trio (>=0.11.0)", "sphinx (>=7.2.0)", "sphinx-rtd-theme (>=2.0.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] -dnssec = ["cryptography (>=43)"] -doh = ["h2 (>=4.1.0)", "httpcore (>=1.0.0)", "httpx (>=0.26.0)"] -doq = ["aioquic (>=1.0.0)"] -idna = ["idna (>=3.7)"] -trio = ["trio (>=0.23)"] -wmi = ["wmi (>=1.5.1)"] - [[package]] name = "docopt" version = "0.6.2" @@ -1477,24 +1405,6 @@ files = [ {file = "lazy_object_proxy-1.11.0.tar.gz", hash = "sha256:18874411864c9fbbbaa47f9fc1dd7aea754c86cfde21278ef427639d1dd78e9c"}, ] -[[package]] -name = "localstack" -version = "4.12.0" -description = "LocalStack - A fully functional local Cloud stack" -optional = false -python-versions = "*" -groups = ["dev"] -files = [ - {file = "localstack-4.12.0.tar.gz", hash = "sha256:87e0824d3115fc72fe78efa6d8bda942e68e69e38b74173c4bff3091efaea1ea"}, -] - -[package.dependencies] -localstack-core = "*" -localstack-ext = "4.12.0" - -[package.extras] -runtime = ["localstack-core[runtime]", "localstack-ext[runtime] (==4.12.0)"] - [[package]] name = "localstack-client" version = "2.10" @@ -1512,73 +1422,6 @@ boto3 = "*" [package.extras] test = ["black", "coverage", "flake8", "isort", "localstack", "pytest"] -[[package]] -name = "localstack-core" -version = "4.12.0" -description = "The core library and runtime of LocalStack" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "localstack_core-4.12.0-py3-none-any.whl", hash = "sha256:730de1c4d71aff00512523b41ba3250f869f9cc774b713e04736851b7ac6cedb"}, - {file = "localstack_core-4.12.0.tar.gz", hash = "sha256:893359fb2392b95a1587e109120904dd4001e640238a2d4a87f8b46c6cb67ba6"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" -cachetools = ">=5.0" -click = ">=8.2.0" -cryptography = "*" -dill = "0.3.6" -dnslib = ">=0.9.10" -dnspython = ">=1.16.0" -plux = ">=1.10" -psutil = ">=5.4.8" -python-dotenv = ">=0.19.1" -pyyaml = ">=5.1" -requests = ">=2.20.0" -rich = ">=12.3.0" -semver = ">=2.10" - -[package.extras] -base-runtime = ["Werkzeug (>=3.1.3)", "awscrt (>=0.13.14,!=0.27.1)", "boto3 (==1.42.4)", "botocore (==1.42.4)", "cbor2 (>=5.5.0)", "dnspython (>=1.16.0)", "docker (>=6.1.1)", "hypercorn (>=0.14.4)", "jsonpatch (>=1.24)", "jsonpointer (>=3.0.0)", "jsonschema (>=4.25.1)", "localstack-twisted (>=23.0)", "openapi-core (>=0.19.2)", "pyopenssl (>=23.0.0)", "python-dateutil (>=2.9.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.7)", "typing-extensions (>=4.15.0)", "urllib3 (>=2.0.7)", "xmltodict (>=0.13.0)"] -dev = ["Cython", "coveralls (>=3.3.1)", "deptry (>=0.13.0)", "localstack-core[test]", "mypy", "networkx (>=2.8.4)", "openapi-spec-validator (>=0.7.1)", "pandoc", "pre-commit (>=3.5.0)", "pypandoc", "rstr (>=3.2.0)", "ruff (>=0.3.3)"] -runtime = ["airspeed-ext (>=0.6.3)", "antlr4-python3-runtime (==4.13.2)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.105.0)", "awscli (==1.43.10)", "crontab (>=0.22.6)", "cryptography (>=41.0.5)", "jinja2 (>=3.1.6)", "jpype1 (>=1.6.0)", "jsonpath-ng (>=1.6.1)", "jsonpath-rw (>=1.4.0)", "kclpy-ext (>=3.0.0)", "localstack-core[base-runtime]", "moto-ext[all] (>=5.1.12.post22)", "opensearch-py (>=2.4.1)", "pydantic (>=2.11.9)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "responses (>=0.25.8)"] -test = ["aws-cdk-lib (>=2.88.0)", "coverage[toml] (>=5.5)", "httpx[http2] (>=0.25)", "json5 (>=0.12.1)", "localstack-core[runtime]", "localstack-snapshot (>=0.1.1)", "pluggy (>=1.3.0)", "pytest (>=7.4.2)", "pytest-httpserver (>=1.1.2)", "pytest-rerunfailures (>=12.0)", "pytest-split (>=0.8.0)", "pytest-tinybird (>=0.5.0)", "websocket-client (>=1.7.0)"] -typehint = ["boto3-stubs[acm,acm-pca,amplify,apigateway,apigatewayv2,appconfig,appconfigdata,application-autoscaling,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codebuild,codecommit,codeconnections,codedeploy,codepipeline,codestar-connections,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,identitystore,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalyticsv2,kms,lakeformation,lambda,logs,managedblockchain,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,pinpoint,pipes,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,verifiedpermissions,wafv2,xray]", "localstack-core[dev]"] - -[[package]] -name = "localstack-ext" -version = "4.12.0" -description = "Extensions for LocalStack" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "localstack_ext-4.12.0.tar.gz", hash = "sha256:010ac6ea245305aae29eb1a025e2ebe4e620bae37b296cb145460d307cf89a51"}, -] - -[package.dependencies] -click = ">=7.1" -cryptography = "*" -localstack-core = "4.12.0" -packaging = "*" -plux = ">=1.10.0" -PyJWT = {version = ">=1.7.0", extras = ["crypto"]} -pyotp = ">=2.9.0" -python-dateutil = ">=2.8" -pyyaml = ">=5.1" -requests = ">=2.20.0" -rich = ">=12.3.0" -tabulate = "*" -windows-curses = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -package = ["localstack-obfuscator (>=0.3.0)"] -runtime = ["Whoosh (>=2.7.4)", "airspeed-ext (>=0.6.9)", "alembic (>=1.16.0)", "avro (>=1.11.0)", "aws-encryption-sdk (>=3.1.0)", "aws-json-term-matcher (>=0.1.5)", "aws-sam-translator (>=1.105.0)", "boto3", "botocore", "cachetools (>=5.5.2)", "cedarpy (>=4.1.0)", "confluent-kafka", "distro", "docker (>=7.1.0)", "dulwich (>=0.19.16)", "fastavro (>=1.12.0)", "graphql-core (>=3.0.3)", "hypercorn (>=0.14.4)", "janus (>=0.5.0)", "javascript", "jinja2 (>=3.1.6)", "jsonpatch (>=1.32)", "jsonpath-ng (>=1.7.0)", "jsonschema (>=4.25.1)", "kubernetes (>=21.7.0)", "libvirt-python", "localstack-core[runtime] (==4.12.0)", "localstack-py-avro-schema (==3.9.9)", "mysql-replication", "opentelemetry-api", "opentelemetry-propagator-aws-xray", "opentelemetry-sdk", "orjson", "paho-mqtt (>=1.5)", "parquet[snappy] (>=1.3.1)", "parse (>=1.19.0)", "pg8000 (>=1.10)", "postgres (>=2.2.2)", "postgresql-proxy (>=0.2.0)", "psutil (>=7.1.0)", "psycopg2-binary (>=2.9.10)", "pycdlib (>=1.14.0)", "pycognito (>=2024.5.1)", "pydantic (<2.12)", "pydantic (>=2.11.9)", "pyftpdlib (>=1.5.6)", "pyhive[hive-pure-sasl] (>=0.7.0)", "pyiceberg (>=0.9.0)", "pymysql", "pyopenssl (>=25.3.0)", "pyparsing (>=3.2.5)", "python-dxf (>=12.1.1)", "readerwriterlock (>=1.0.7)", "redis (>=5.0,<6.0)", "rolo", "rsa (>=4.0)", "semver (>=3.0.4)", "setuptools", "sql-metadata (>=2.6.0)", "sqlalchemy (>=2.0.0)", "sqlglot[rs]", "srp-ext (>=1.0.7.1)", "testing.common.database (>=1.1.0)", "trino (>=0.328.0)", "typing-extensions (>=4.15.0)", "urllib3 (>=2.5.0)", "websocket-client (>=1.8.0)", "websockets (>=8.1,<14)", "werkzeug (>=3.1.3)", "xmltodict (>=1.0.2)"] -test = ["PyAthena[pandas]", "aiohttp", "async-timeout", "aws-cdk-lib (>=2.88.0)", "aws-cdk.aws-cognito-identitypool-alpha", "aws_cdk.aws_neptune_alpha", "aws_cdk.aws_redshift_alpha", "aws_xray_sdk (>=2.4.2)", "awsiotsdk", "awsiotsdk", "awswrangler (>=3.5.2)", "coverage[toml] (>=5.0.0)", "deepdiff (>=5.5.0)", "deptry (>=0.13.0)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "gremlinpython (<3.8.0)", "jws (>=0.1.3)", "kafka-python", "localstack-core[test] (==4.12.0)", "localstack-ext[runtime]", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "paramiko", "playwright", "portalocker", "pre-commit (>=3.5.0)", "pyarrow", "pymongo", "pymssql (>=2.2.8)", "pypandoc", "pytest-httpserver (>=1.0.1)", "pytest-instafail (>=0.4.2)", "pytest-mock (>=3.14.0)", "pytest-playwright", "python-terraform", "redshift_connector", "ruff (>=0.1.0)", "stomp.py (>=8.0.1)", "uefivars (>=1.2)"] -typehint = ["boto3-stubs[acm,amplify,apigateway,apigatewayv2,appconfig,appsync,athena,autoscaling,backup,batch,bedrock,bedrock-runtime,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,s3tables,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,xray]", "localstack-ext[test]"] - [[package]] name = "lxml" version = "5.4.0" @@ -1743,31 +1586,6 @@ files = [ [package.dependencies] typing-extensions = "*" -[[package]] -name = "markdown-it-py" -version = "3.0.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, - {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -code-style = ["pre-commit (>=3.0,<4.0)"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - [[package]] name = "markupsafe" version = "3.0.2" @@ -1839,18 +1657,6 @@ files = [ {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - [[package]] name = "moto" version = "5.1.19" @@ -2170,21 +1976,6 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] -[[package]] -name = "plux" -version = "1.12.1" -description = "A dynamic code loading framework for building pluggable Python distributions" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "plux-1.12.1-py3-none-any.whl", hash = "sha256:b4aa4e67329f2fcd73fb28096a8a9304f5912ee6cce39994eac567e8eec65488"}, - {file = "plux-1.12.1.tar.gz", hash = "sha256:1ed44a6edbb7343f4711ff75ddaf87bed066c53625d41b63a0b4edd3f77792ba"}, -] - -[package.extras] -dev = ["black (==22.3.0)", "isort (==5.9.1)", "pytest (==6.2.4)", "setuptools"] - [[package]] name = "ply" version = "3.11" @@ -2348,30 +2139,6 @@ files = [ {file = "propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168"}, ] -[[package]] -name = "psutil" -version = "7.0.0" -description = "Cross-platform lib for process and system monitoring in Python. NOTE: the syntax of this script MUST be kept compatible with Python 2.7." -optional = false -python-versions = ">=3.6" -groups = ["dev"] -files = [ - {file = "psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"}, - {file = "psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"}, - {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91"}, - {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34"}, - {file = "psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993"}, - {file = "psutil-7.0.0-cp36-cp36m-win32.whl", hash = "sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17"}, - {file = "psutil-7.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e"}, - {file = "psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99"}, - {file = "psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553"}, - {file = "psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456"}, -] - -[package.extras] -dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest", "pytest-cov", "pytest-xdist", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] -test = ["pytest", "pytest-xdist", "setuptools"] - [[package]] name = "pyasn1" version = "0.6.2" @@ -2673,9 +2440,6 @@ files = [ {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, ] -[package.dependencies] -cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} - [package.extras] crypto = ["cryptography (>=3.4.0)"] dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] @@ -3024,25 +2788,6 @@ files = [ [package.dependencies] six = "*" -[[package]] -name = "rich" -version = "14.0.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.8.0" -groups = ["dev"] -files = [ - {file = "rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0"}, - {file = "rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725"}, -] - -[package.dependencies] -markdown-it-py = ">=2.2.0" -pygments = ">=2.13.0,<3.0.0" - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - [[package]] name = "rpds-py" version = "0.25.1" @@ -3328,21 +3073,6 @@ files = [ [package.dependencies] tenacity = "*" -[[package]] -name = "tabulate" -version = "0.9.0" -description = "Pretty-print tabular data" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, - {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, -] - -[package.extras] -widechars = ["wcwidth"] - [[package]] name = "tenacity" version = "9.1.2" @@ -3473,33 +3203,6 @@ files = [ [package.extras] test = ["pytest (>=3.0.0)"] -[[package]] -name = "windows-curses" -version = "2.4.1" -description = "Support for the standard curses module on Windows" -optional = false -python-versions = "*" -groups = ["dev"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "windows_curses-2.4.1-cp310-cp310-win32.whl", hash = "sha256:53d711e07194d0d3ff7ceff29e0955b35479bc01465d46c3041de67b8141db2f"}, - {file = "windows_curses-2.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:325439cd4f37897a1de8a9c068a5b4c432f9244bf9c855ee2fbeb3fa721a770c"}, - {file = "windows_curses-2.4.1-cp311-cp311-win32.whl", hash = "sha256:4fa1a176bfcf098d0c9bb7bc03dce6e83a4257fc0c66ad721f5745ebf0c00746"}, - {file = "windows_curses-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fd7d7a9cf6c1758f46ed76b8c67f608bc5fcd5f0ca91f1580fd2d84cf41c7f4f"}, - {file = "windows_curses-2.4.1-cp312-cp312-win32.whl", hash = "sha256:bdbe7d58747408aef8a9128b2654acf6fbd11c821b91224b9a046faba8c6b6ca"}, - {file = "windows_curses-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c9c2635faf171a229caca80e1dd760ab00db078e2a285ba2f667bbfcc31777c"}, - {file = "windows_curses-2.4.1-cp313-cp313-win32.whl", hash = "sha256:05d1ca01e5199a435ccb6c8c2978df4a169cdff1ec99ab15f11ded9de8e5be26"}, - {file = "windows_curses-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8cf653f8928af19c103ae11cfed38124f418dcdd92643c4cd17239c0cec2f9da"}, - {file = "windows_curses-2.4.1-cp36-cp36m-win32.whl", hash = "sha256:6a5a831cabaadde41a6856fea5a0c68c74b7d11d332a816e5a5e6c84577aef3a"}, - {file = "windows_curses-2.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e61be805edc390ccfdeaf0e0c39736d931d3c4a007d6bf0f98d1e792ce437796"}, - {file = "windows_curses-2.4.1-cp37-cp37m-win32.whl", hash = "sha256:a36b8fd4e410ddfb1a8eb65af2116c588e9f99b2ff3404412317440106755485"}, - {file = "windows_curses-2.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:db776df70c10bd523c4a1ab0a7624a1d58c7d47f83ec49c6988f05bc1189e7b8"}, - {file = "windows_curses-2.4.1-cp38-cp38-win32.whl", hash = "sha256:e9ce84559f80de7ec770d28c3b2991e0da51748def04e25a3c08ada727cfac2d"}, - {file = "windows_curses-2.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:618e31458fedba2cf8105485ff00533ece780026c544142fc1647a20dc6c7641"}, - {file = "windows_curses-2.4.1-cp39-cp39-win32.whl", hash = "sha256:775a2e0fefeddfdb0e00b3fa6c4f21caf9982db34df30e4e62c49caaef7b5e56"}, - {file = "windows_curses-2.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:4588213f7ef3b0c24c5cb9e309653d7a84c1792c707561e8b471d466ca79f2b8"}, -] - [[package]] name = "wireup" version = "2.2.2" @@ -3741,4 +3444,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "383eb0e3581570630fc52690d61866a180122fe61c891bea564a82f354a99992" +content-hash = "a3ad1996923b878a8227dbbeb7d3a1455650c67163d8c79fbb5700a3ab1ff157" diff --git a/pyproject.toml b/pyproject.toml index cf9a9c3a4..a2bcf6978 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ awscli-local = "^0.22.2" polyfactory = "^3.2.0" pyright = "^1.1.407" brunns-matchers = "^2.9.0" -localstack = "^4.12.0" pytest-docker = "^3.2.3" stamina = "^25.2.0" pytest-freezer = "^0.4.9" diff --git a/src/eligibility_signposting_api/config/config.py b/src/eligibility_signposting_api/config/config.py index 52f3111cc..bb23991a5 100644 --- a/src/eligibility_signposting_api/config/config.py +++ b/src/eligibility_signposting_api/config/config.py @@ -26,7 +26,7 @@ def config() -> dict[str, Any]: audit_bucket_name = BucketName(os.getenv("AUDIT_BUCKET_NAME", "test-audit-bucket")) hashing_secret_name = HashSecretName(os.getenv("HASHING_SECRET_NAME", "test_secret")) aws_default_region = AwsRegion(os.getenv("AWS_DEFAULT_REGION", "eu-west-1")) - enable_xray_patching = bool(os.getenv("ENABLE_XRAY_PATCHING", "false")) + enable_xray_patching = os.getenv("ENABLE_XRAY_PATCHING", "false").lower() == "true" kinesis_audit_stream_to_s3 = AwsKinesisFirehoseStreamName( os.getenv("KINESIS_AUDIT_STREAM_TO_S3", "test_kinesis_audit_stream_to_s3") ) @@ -51,21 +51,21 @@ def config() -> dict[str, Any]: "log_level": log_level, } - local_stack_endpoint = "http://localhost:4566" + moto_server_endpoint = "http://localhost:4566" return { "aws_access_key_id": AwsAccessKey(os.getenv("AWS_ACCESS_KEY_ID", "dummy_key")), "aws_default_region": aws_default_region, "aws_secret_access_key": AwsSecretAccessKey(os.getenv("AWS_SECRET_ACCESS_KEY", "dummy_secret")), - "dynamodb_endpoint": URL(os.getenv("DYNAMODB_ENDPOINT", local_stack_endpoint)), + "dynamodb_endpoint": URL(os.getenv("DYNAMODB_ENDPOINT", moto_server_endpoint)), "person_table_name": person_table_name, - "s3_endpoint": URL(os.getenv("S3_ENDPOINT", local_stack_endpoint)), + "s3_endpoint": URL(os.getenv("S3_ENDPOINT", moto_server_endpoint)), "rules_bucket_name": rules_bucket_name, "audit_bucket_name": audit_bucket_name, "consumer_mapping_bucket_name": consumer_mapping_bucket_name, - "firehose_endpoint": URL(os.getenv("FIREHOSE_ENDPOINT", local_stack_endpoint)), + "firehose_endpoint": URL(os.getenv("FIREHOSE_ENDPOINT", moto_server_endpoint)), "kinesis_audit_stream_to_s3": kinesis_audit_stream_to_s3, "enable_xray_patching": enable_xray_patching, - "secretsmanager_endpoint": URL(os.getenv("SECRET_MANAGER_ENDPOINT", local_stack_endpoint)), + "secretsmanager_endpoint": URL(os.getenv("SECRET_MANAGER_ENDPOINT", moto_server_endpoint)), "hashing_secret_name": hashing_secret_name, "log_level": log_level, } diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index e6b90dd5a..06d19ad7e 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,20 +1,31 @@ services: - localstack: -# container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" - image: localstack/localstack:4.4.0 # See https://hub.docker.com/r/localstack/localstack/tags + moto-server: + image: motoserver/moto:latest ports: - - "4566:4566" # LocalStack Gateway - - "4510-4559:4510-4559" # external services port range + - "4566:5000" environment: - # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ - - DEBUG=${LOCALSTACK_DEBUG:-0} - - DEFAULT_REGION=${AWS_DEFAULT_REGION:-eu-west-1} - - LAMBDA_EXECUTOR=docker + - MOTO_PORT=5000 + - MOTO_LAMBDA_EXECUTOR=docker + - MOTO_LAMBDA_IMAGE_PYTHON313=public.ecr.aws/lambda/python:3.13 volumes: - - "${LOCALSTACK_VOLUME_DIR:-../volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" + - /var/run/docker.sock:/var/run/docker.sock + networks: + - test-network healthcheck: - test: "curl -f http://localhost:4566/health || exit 1" + test: "curl -f http://localhost:5000/health || exit 1" interval: 5s timeout: 5s - retries: 10 + retries: 5 + +# mock-api-gateway: +# image: openresty/openresty:alpine +# ports: +# - "9123:9123" +# volumes: +# - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro +# networks: +# - test-network + +networks: + test-network: + driver: bridge diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0cdf7f235..9c4c68d00 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -52,21 +52,29 @@ UNIQUE_CONSUMER_HEADER = "nhse-product-id" +MOTO_PORT = 5000 -@pytest.fixture(scope="session") -def localstack(request: pytest.FixtureRequest) -> URL: - if url := os.getenv("RUNNING_LOCALSTACK_URL", None): - logger.info("localstack already running on %s", url) - return URL(url) +@pytest.fixture(scope="session", autouse=True) +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "dummy_key" + os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret" + os.environ["AWS_SECURITY_TOKEN"] = "dummy_token" + os.environ["AWS_SESSION_TOKEN"] = "dummy_session_token" + os.environ["AWS_DEFAULT_REGION"] = AWS_REGION - docker_ip: str = request.getfixturevalue("docker_ip") - docker_services: Services = request.getfixturevalue("docker_services") +@pytest.fixture(scope="session") +def moto_server(request: pytest.FixtureRequest) -> URL: + docker_services = request.getfixturevalue("docker_services") + docker_ip = request.getfixturevalue("docker_ip") - logger.info("Starting localstack") - port = docker_services.port_for("localstack", 4566) + # Change "moto" to "moto-server" to match your docker-compose.yml + port = docker_services.port_for("moto-server", 5000) url = URL(f"http://{docker_ip}:{port}") - docker_services.wait_until_responsive(timeout=30.0, pause=0.1, check=lambda: is_responsive(url)) - logger.info("localstack running on %s", url) + + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_responsive(url) + ) return url @@ -82,57 +90,56 @@ def is_responsive(url: URL) -> bool: @pytest.fixture(scope="session") def boto3_session() -> Session: - return Session(aws_access_key_id="fake", aws_secret_access_key="fake", region_name=AWS_REGION) - + return Session(aws_access_key_id="fake", aws_secret_access_key="fake", aws_session_token="fake", region_name=AWS_REGION) @pytest.fixture(scope="session") -def api_gateway_client(boto3_session: Session, localstack: URL) -> BaseClient: - return boto3_session.client("apigateway", endpoint_url=str(localstack)) +def api_gateway_client(boto3_session: Session, moto_server: URL) -> BaseClient: + return boto3_session.client("apigateway", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def lambda_client(boto3_session: Session, localstack: URL) -> BaseClient: - return boto3_session.client("lambda", endpoint_url=str(localstack)) +def lambda_client(boto3_session: Session, moto_server: URL) -> BaseClient: + return boto3_session.client("lambda", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def dynamodb_client(boto3_session: Session, localstack: URL) -> BaseClient: - return boto3_session.client("dynamodb", endpoint_url=str(localstack)) +def dynamodb_client(boto3_session: Session, moto_server: URL) -> BaseClient: + return boto3_session.client("dynamodb", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def dynamodb_resource(boto3_session: Session, localstack: URL) -> ServiceResource: - return boto3_session.resource("dynamodb", endpoint_url=str(localstack)) +def dynamodb_resource(boto3_session: Session, moto_server: URL) -> ServiceResource: + return boto3_session.resource("dynamodb", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def logs_client(boto3_session: Session, localstack: URL) -> BaseClient: - return boto3_session.client("logs", endpoint_url=str(localstack)) +def logs_client(boto3_session: Session, moto_server: URL) -> BaseClient: + return boto3_session.client("logs", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def iam_client(boto3_session: Session, localstack: URL) -> BaseClient: - return boto3_session.client("iam", endpoint_url=str(localstack)) +def iam_client(boto3_session: Session, moto_server: URL) -> BaseClient: + return boto3_session.client("iam", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def s3_client(boto3_session: Session, localstack: URL) -> BaseClient: - return boto3_session.client("s3", endpoint_url=str(localstack)) +def s3_client(boto3_session: Session, moto_server: URL) -> BaseClient: + return boto3_session.client("s3", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def firehose_client(boto3_session: Session, localstack: URL) -> BaseClient: - return boto3_session.client("firehose", endpoint_url=str(localstack)) +def firehose_client(boto3_session: Session, moto_server: URL) -> BaseClient: + return boto3_session.client("firehose", endpoint_url=str(moto_server)) @pytest.fixture(scope="session") -def secretsmanager_client(boto3_session: Session, localstack: URL) -> BaseClient: +def secretsmanager_client(boto3_session: Session, moto_server: URL) -> BaseClient: """ - Provides a boto3 Secrets Manager client bound to LocalStack. + Provides a boto3 Secrets Manager client bound to Moto. Seeds a test secret for use in integration tests. """ client: BaseClient = boto3_session.client( - service_name="secretsmanager", endpoint_url=str(localstack), region_name="eu-west-1" + service_name="secretsmanager", endpoint_url=str(moto_server), region_name="eu-west-1" ) secret_name = AWS_SECRET_NAME @@ -231,14 +238,28 @@ def iam_role(iam_client: BaseClient) -> Generator[str]: @pytest.fixture(scope="session") def lambda_zip() -> Path: - build_result = subprocess.run(["make", "build"], capture_output=True, text=True, check=False) # noqa: S607 - assert build_result.returncode == 0, f"'make build' failed: {build_result.stderr}" - return Path("dist/lambda.zip") + # Run the build command + build_result = subprocess.run(["make", "build"], capture_output=True, text=True, check=False) + + # If it fails, print the actual stderr so you can see WHY in the pytest logs + if build_result.returncode != 0: + pytest.fail( + f"'make build' failed with code {build_result.returncode}.\nSTDOUT: {build_result.stdout}\nSTDERR: {build_result.stderr}") + + zip_path = Path("dist/lambda.zip") + if not zip_path.exists(): + pytest.fail(f"Build succeeded but {zip_path} was not created.") + + return zip_path @pytest.fixture(scope="session") def flask_function(lambda_client: BaseClient, iam_role: str, lambda_zip: Path) -> Generator[str]: function_name = "eligibility_signposting_api" + + # Use the container name 'moto-server' on the internal Docker network + moto_internal_endpoint = "http://moto-server:5000" + with lambda_zip.open("rb") as zipfile: lambda_client.create_function( FunctionName=function_name, @@ -246,22 +267,19 @@ def flask_function(lambda_client: BaseClient, iam_role: str, lambda_zip: Path) - Role=iam_role, Handler="eligibility_signposting_api.app.lambda_handler", Code={"ZipFile": zipfile.read()}, - Architectures=["x86_64"], - Timeout=180, Environment={ "Variables": { - "DYNAMODB_ENDPOINT": os.getenv("LOCALSTACK_INTERNAL_ENDPOINT", "http://localstack:4566/"), - "S3_ENDPOINT": os.getenv("LOCALSTACK_INTERNAL_ENDPOINT", "http://localstack:4566/"), - "FIREHOSE_ENDPOINT": os.getenv("LOCALSTACK_INTERNAL_ENDPOINT", "http://localstack:4566/"), - "SECRET_MANAGER_ENDPOINT": os.getenv("LOCALSTACK_INTERNAL_ENDPOINT", "http://localstack:4566/"), + # IMPORTANT: These must use the internal Docker DNS name + "DYNAMODB_ENDPOINT": moto_internal_endpoint, + "S3_ENDPOINT": moto_internal_endpoint, + "FIREHOSE_ENDPOINT": moto_internal_endpoint, + "SECRET_MANAGER_ENDPOINT": moto_internal_endpoint, "AWS_REGION": AWS_REGION, "LOG_LEVEL": "DEBUG", } }, ) - logger.info("loaded zip") wait_for_function_active(function_name, lambda_client) - logger.info("function active") yield function_name lambda_client.delete_function(FunctionName=function_name) @@ -304,23 +322,32 @@ def wait_for_function_active(function_name, lambda_client): @pytest.fixture(scope="session") -def configured_api_gateway(api_gateway_client, lambda_client, flask_function: str): +def configured_api_gateway(api_gateway_client, lambda_client, flask_function: str, moto_server: URL): region = lambda_client.meta.region_name - api = api_gateway_client.create_rest_api(name="API Gateway Lambda integration") + # 1. Create the REST API + api = api_gateway_client.create_rest_api( + name="API Gateway Lambda integration", + endpointConfiguration={'types': ['REGIONAL']} + ) rest_api_id = api["id"] + # 2. Get Root Resource ID resources = api_gateway_client.get_resources(restApiId=rest_api_id) root_id = next(item["id"] for item in resources["items"] if item["path"] == "/") + # 3. Create Resource Path: /patient-check/{id} patient_check_res = api_gateway_client.create_resource( restApiId=rest_api_id, parentId=root_id, pathPart="patient-check" ) patient_check_id = patient_check_res["id"] - id_res = api_gateway_client.create_resource(restApiId=rest_api_id, parentId=patient_check_id, pathPart="{id}") + id_res = api_gateway_client.create_resource( + restApiId=rest_api_id, parentId=patient_check_id, pathPart="{id}" + ) resource_id = id_res["id"] + # 4. Create GET Method api_gateway_client.put_method( restApiId=rest_api_id, resourceId=resource_id, @@ -329,11 +356,13 @@ def configured_api_gateway(api_gateway_client, lambda_client, flask_function: st requestParameters={"method.request.path.id": True}, ) - # Integration with actual region + # 5. Setup Lambda Integration + # Moto uses account 123456789012 by default lambda_uri = ( f"arn:aws:apigateway:{region}:lambda:path/2015-03-31/functions/" - f"arn:aws:lambda:{region}:000000000000:function:{flask_function}/invocations" + f"arn:aws:lambda:{region}:123456789012:function:{flask_function}/invocations" ) + api_gateway_client.put_integration( restApiId=rest_api_id, resourceId=resource_id, @@ -344,28 +373,31 @@ def configured_api_gateway(api_gateway_client, lambda_client, flask_function: st passthroughBehavior="WHEN_NO_MATCH", ) - # Permission with matching region + # 6. Add Permission for API Gateway to invoke Lambda lambda_client.add_permission( FunctionName=flask_function, StatementId="apigateway-access", Action="lambda:InvokeFunction", Principal="apigateway.amazonaws.com", - SourceArn=f"arn:aws:execute-api:{region}:000000000000:{rest_api_id}/*/GET/patient-check/*", + SourceArn=f"arn:aws:execute-api:{region}:123456789012:{rest_api_id}/*/GET/patient-check/*", ) - # Deploy the API + # 7. Create Deployment and Stage api_gateway_client.create_deployment(restApiId=rest_api_id, stageName="dev") + # 8. Construct the Moto-compatible Invoke URL + moto_base_url = str(localstack).rstrip("/") + invoke_url = f"{moto_base_url}/restapis/{rest_api_id}/dev/_user_request_" + return { "rest_api_id": rest_api_id, "resource_id": resource_id, - "invoke_url": f"http://{rest_api_id}.execute-api.localhost.localstack.cloud:4566/dev/patient-check/{{id}}", + "invoke_url": invoke_url, } - @pytest.fixture def api_gateway_endpoint(configured_api_gateway: dict) -> URL: - return URL(f"http://{configured_api_gateway['rest_api_id']}.execute-api.localhost.localstack.cloud:4566/dev") + return URL(configured_api_gateway["invoke_url"]) @pytest.fixture(scope="session") @@ -727,17 +759,22 @@ def audit_bucket(s3_client: BaseClient) -> Generator[BucketName]: @pytest.fixture(autouse=True) def firehose_delivery_stream(firehose_client: BaseClient, audit_bucket: BucketName) -> dict[str, Any]: - return firehose_client.create_delivery_stream( - DeliveryStreamName="test_kinesis_audit_stream_to_s3", - DeliveryStreamType="DirectPut", - ExtendedS3DestinationConfiguration={ - "BucketARN": f"arn:aws:s3:::{audit_bucket}", - "RoleARN": "arn:aws:iam::000000000000:role/firehose_delivery_role", - "Prefix": "audit-logs/", - "BufferingHints": {"SizeInMBs": 1, "IntervalInSeconds": 60}, - "CompressionFormat": "UNCOMPRESSED", - }, - ) + stream_name = "test_kinesis_audit_stream_to_s3" + + try: + return firehose_client.create_delivery_stream( + DeliveryStreamName=stream_name, + DeliveryStreamType="DirectPut", + ExtendedS3DestinationConfiguration={ + "BucketARN": f"arn:aws:s3:::{audit_bucket}", + "RoleARN": "arn:aws:iam::123456789012:role/firehose_delivery_role", + "Prefix": "audit-logs/", + "BufferingHints": {"SizeInMBs": 1, "IntervalInSeconds": 60}, + "CompressionFormat": "UNCOMPRESSED", + }, + ) + except firehose_client.exceptions.ResourceInUseException: + return firehose_client.describe_delivery_stream(DeliveryStreamName=stream_name) @pytest.fixture(scope="class") From ac10a45e9ae4539f9bf41660b676c407376d4182 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:57:06 +0000 Subject: [PATCH 02/16] localstack --- tests/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 06d19ad7e..4031a619f 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,12 +1,16 @@ services: moto-server: image: motoserver/moto:latest + container_name: moto-server ports: - "4566:5000" environment: - MOTO_PORT=5000 - MOTO_LAMBDA_EXECUTOR=docker + # 1. Map the runtime to official AWS image - MOTO_LAMBDA_IMAGE_PYTHON313=public.ecr.aws/lambda/python:3.13 + # 2. Tell Moto to place the Lambda container on the same network + - MOTO_DOCKER_NETWORK=test-network volumes: - /var/run/docker.sock:/var/run/docker.sock networks: From c9776f32bd62b64c5072686e3b038e4591e1d417 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:57:10 +0000 Subject: [PATCH 03/16] localstack --- tests/integration/conftest.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9c4c68d00..ed219feea 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -54,6 +54,10 @@ MOTO_PORT = 5000 +@pytest.fixture(scope="session") +def docker_compose_project_name(): + return "tests" + @pytest.fixture(scope="session", autouse=True) def aws_credentials(): """Mocked AWS Credentials for moto.""" @@ -63,18 +67,17 @@ def aws_credentials(): os.environ["AWS_SESSION_TOKEN"] = "dummy_session_token" os.environ["AWS_DEFAULT_REGION"] = AWS_REGION + @pytest.fixture(scope="session") def moto_server(request: pytest.FixtureRequest) -> URL: docker_services = request.getfixturevalue("docker_services") + # This must match the container_name in docker-compose docker_ip = request.getfixturevalue("docker_ip") - - # Change "moto" to "moto-server" to match your docker-compose.yml + # port_for maps the INTERNAL 5000 to whatever random EXTERNAL port was assigned port = docker_services.port_for("moto-server", 5000) url = URL(f"http://{docker_ip}:{port}") - docker_services.wait_until_responsive( - timeout=30.0, pause=0.1, check=lambda: is_responsive(url) - ) + docker_services.wait_until_responsive(timeout=30.0, pause=0.2, check=lambda: is_responsive(url)) return url @@ -257,8 +260,8 @@ def lambda_zip() -> Path: def flask_function(lambda_client: BaseClient, iam_role: str, lambda_zip: Path) -> Generator[str]: function_name = "eligibility_signposting_api" - # Use the container name 'moto-server' on the internal Docker network - moto_internal_endpoint = "http://moto-server:5000" + # Use the INTERNAL docker name. Inside the Docker network, Moto is always 5000. + moto_internal = "http://moto-server:5000" with lambda_zip.open("rb") as zipfile: lambda_client.create_function( @@ -269,13 +272,11 @@ def flask_function(lambda_client: BaseClient, iam_role: str, lambda_zip: Path) - Code={"ZipFile": zipfile.read()}, Environment={ "Variables": { - # IMPORTANT: These must use the internal Docker DNS name - "DYNAMODB_ENDPOINT": moto_internal_endpoint, - "S3_ENDPOINT": moto_internal_endpoint, - "FIREHOSE_ENDPOINT": moto_internal_endpoint, - "SECRET_MANAGER_ENDPOINT": moto_internal_endpoint, + "DYNAMODB_ENDPOINT": moto_internal, + "S3_ENDPOINT": moto_internal, + "SECRET_MANAGER_ENDPOINT": moto_internal, + "FIREHOSE_ENDPOINT": moto_internal, "AWS_REGION": AWS_REGION, - "LOG_LEVEL": "DEBUG", } }, ) @@ -283,7 +284,6 @@ def flask_function(lambda_client: BaseClient, iam_role: str, lambda_zip: Path) - yield function_name lambda_client.delete_function(FunctionName=function_name) - @pytest.fixture(autouse=True) def clean_audit_bucket(s3_client: BaseClient, audit_bucket: str): objects_to_delete = [] @@ -386,7 +386,9 @@ def configured_api_gateway(api_gateway_client, lambda_client, flask_function: st api_gateway_client.create_deployment(restApiId=rest_api_id, stageName="dev") # 8. Construct the Moto-compatible Invoke URL - moto_base_url = str(localstack).rstrip("/") + # Inside configured_api_gateway fixture + moto_base_url = str(moto_server).rstrip("/") + # Moto routes via: /restapis///_user_request_/ invoke_url = f"{moto_base_url}/restapis/{rest_api_id}/dev/_user_request_" return { From 9afef83e27f6b6172371cb12c86545697d5ee16d Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Tue, 17 Feb 2026 11:27:34 +0000 Subject: [PATCH 04/16] ELI-662: working sam --- tests/docker-compose.yml | 58 +++++++++++++++++++++++++--------------- tests/locals.json | 6 +++++ tests/template.yaml | 34 +++++++++++++++++++++++ 3 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 tests/locals.json create mode 100644 tests/template.yaml diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 4031a619f..4167a12fa 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -4,31 +4,47 @@ services: container_name: moto-server ports: - "4566:5000" - environment: - - MOTO_PORT=5000 - - MOTO_LAMBDA_EXECUTOR=docker - # 1. Map the runtime to official AWS image - - MOTO_LAMBDA_IMAGE_PYTHON313=public.ecr.aws/lambda/python:3.13 - # 2. Tell Moto to place the Lambda container on the same network - - MOTO_DOCKER_NETWORK=test-network + networks: + - test-network + + sam-api: + image: public.ecr.aws/sam/build-python3.13:latest + container_name: sam-api + command: + - sam + - local + - start-api + - --host + - 0.0.0.0 + - --template + - tests/template.yaml + - --env-vars + - tests/locals.json + - --docker-network + - tests_test-network + # Force SAM to use the host gateway to find the Lambda container + - --container-host + - host.docker.internal + - --container-host-interface + - 0.0.0.0 volumes: + - ..:/var/opt:ro - /var/run/docker.sock:/var/run/docker.sock + working_dir: /var/opt + ports: + - "3000:3000" + environment: + - AWS_ACCESS_KEY_ID=testing + - AWS_SECRET_ACCESS_KEY=testing + - AWS_DEFAULT_REGION=eu-west-1 + # Give it more breathing room for the handshake + - SAM_CLI_CONTAINER_CONNECTION_TIMEOUT=60 + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + - moto-server networks: - test-network - healthcheck: - test: "curl -f http://localhost:5000/health || exit 1" - interval: 5s - timeout: 5s - retries: 5 - -# mock-api-gateway: -# image: openresty/openresty:alpine -# ports: -# - "9123:9123" -# volumes: -# - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro -# networks: -# - test-network networks: test-network: diff --git a/tests/locals.json b/tests/locals.json new file mode 100644 index 000000000..61ec182f8 --- /dev/null +++ b/tests/locals.json @@ -0,0 +1,6 @@ +{ + "EligibilityFunction": { + "PYTHONPATH": "/var/task", + "AWS_REGION": "eu-west-1" + } +} diff --git a/tests/template.yaml b/tests/template.yaml new file mode 100644 index 000000000..d4e95b69c --- /dev/null +++ b/tests/template.yaml @@ -0,0 +1,34 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: SAM Template using pre-built Zip for Eligibility Signposting API + +Globals: + Function: + Timeout: 30 + MemorySize: 256 + Runtime: python3.13 + Handler: eligibility_signposting_api.app.lambda_handler + Environment: + Variables: + DYNAMODB_ENDPOINT: http://moto-server:5000 + S3_ENDPOINT: http://moto-server:5000 + SECRET_MANAGER_ENDPOINT: http://moto-server:5000 + FIREHOSE_ENDPOINT: http://moto-server:5000 + AWS_REGION: eu-west-1 + PERSON_TABLE_NAME: test_eligibility_datastore + RULES_BUCKET_NAME: test-rules-bucket + CONSUMER_MAPPING_BUCKET_NAME: test-consumer-mapping-bucket + AUDIT_BUCKET_NAME: test-audit-bucket + +Resources: + EligibilityFunction: + Type: AWS::Serverless::Function + Properties: + # Point this to the location of your zip file relative to the template + CodeUri: ../dist/lambda.zip + Events: + GetEligibility: + Type: Api + Properties: + Path: /patient-check/{id} + Method: get From 4f66423406d9352c8682abb4b654c08cb1c391b0 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Tue, 17 Feb 2026 11:45:46 +0000 Subject: [PATCH 05/16] ELI-662: working sam --- tests/docker-compose.yml | 10 ++++++---- tests/locals.json | 4 ++++ tests/template.yaml | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 4167a12fa..79003529d 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -22,14 +22,16 @@ services: - tests/locals.json - --docker-network - tests_test-network - # Force SAM to use the host gateway to find the Lambda container - --container-host - host.docker.internal - --container-host-interface - 0.0.0.0 volumes: - - ..:/var/opt:ro - /var/run/docker.sock:/var/run/docker.sock + # Mount project root to /var/opt inside the container + - ${PWD}/..:/var/opt:ro + # CRITICAL: Create a 'tmp' folder in your 'tests' directory and mount it to /tmp + - ${PWD}/tmp:/tmp:rw working_dir: /var/opt ports: - "3000:3000" @@ -37,8 +39,8 @@ services: - AWS_ACCESS_KEY_ID=testing - AWS_SECRET_ACCESS_KEY=testing - AWS_DEFAULT_REGION=eu-west-1 - # Give it more breathing room for the handshake - - SAM_CLI_CONTAINER_CONNECTION_TIMEOUT=60 + # Tell SAM the host's absolute path for the Zip and the shared /tmp + - SAM_DOCKER_VOLUME_BASEDIR=/Users/karthikeyanthangavel/PycharmProjects/eligibility-signposting-api extra_hosts: - "host.docker.internal:host-gateway" depends_on: diff --git a/tests/locals.json b/tests/locals.json index 61ec182f8..ca6f89cde 100644 --- a/tests/locals.json +++ b/tests/locals.json @@ -1,6 +1,10 @@ { "EligibilityFunction": { "PYTHONPATH": "/var/task", + "DYNAMODB_ENDPOINT": "http://moto-server:5000", + "S3_ENDPOINT": "http://moto-server:5000", + "AWS_ACCESS_KEY_ID": "testing", + "AWS_SECRET_ACCESS_KEY": "testing", "AWS_REGION": "eu-west-1" } } diff --git a/tests/template.yaml b/tests/template.yaml index d4e95b69c..bb9b6495d 100644 --- a/tests/template.yaml +++ b/tests/template.yaml @@ -25,7 +25,7 @@ Resources: Type: AWS::Serverless::Function Properties: # Point this to the location of your zip file relative to the template - CodeUri: ../dist/lambda.zip + CodeUri: ./dist/lambda.zip Events: GetEligibility: Type: Api From 97697bd870aa8de1de33cedfb1fe235972fe7074 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Wed, 18 Feb 2026 18:50:36 +0000 Subject: [PATCH 06/16] AWS RIE for lambda and moto for other services, Cloud watch cannot be configured for testing --- tests/docker-compose.yml | 65 +++-- tests/gateway.js | 25 ++ tests/integration/conftest.py | 254 ++---------------- tests/integration/lambda/conftest.py | 111 ++++++++ .../lambda/test_app_running_as_lambda.py | 53 ++-- tests/locals.json | 10 - tests/nginx.conf | 113 ++++++++ tests/template.yaml | 34 --- 8 files changed, 324 insertions(+), 341 deletions(-) create mode 100644 tests/gateway.js create mode 100644 tests/integration/lambda/conftest.py delete mode 100644 tests/locals.json create mode 100644 tests/nginx.conf delete mode 100644 tests/template.yaml diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 79003529d..fe186265d 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -7,47 +7,46 @@ services: networks: - test-network - sam-api: - image: public.ecr.aws/sam/build-python3.13:latest - container_name: sam-api - command: - - sam - - local - - start-api - - --host - - 0.0.0.0 - - --template - - tests/template.yaml - - --env-vars - - tests/locals.json - - --docker-network - - tests_test-network - - --container-host - - host.docker.internal - - --container-host-interface - - 0.0.0.0 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - # Mount project root to /var/opt inside the container - - ${PWD}/..:/var/opt:ro - # CRITICAL: Create a 'tmp' folder in your 'tests' directory and mount it to /tmp - - ${PWD}/tmp:/tmp:rw - working_dir: /var/opt + lambda-api: + image: public.ecr.aws/lambda/python:3.13 + container_name: lambda-api ports: - - "3000:3000" + - "4567:8080" + platform: linux/amd64 + volumes: + - ../dist:/tmp:ro environment: - - AWS_ACCESS_KEY_ID=testing - - AWS_SECRET_ACCESS_KEY=testing + - AWS_ACCESS_KEY_ID=dummy_key + - AWS_SECRET_ACCESS_KEY=dummy_secret - AWS_DEFAULT_REGION=eu-west-1 - # Tell SAM the host's absolute path for the Zip and the shared /tmp - - SAM_DOCKER_VOLUME_BASEDIR=/Users/karthikeyanthangavel/PycharmProjects/eligibility-signposting-api - extra_hosts: - - "host.docker.internal:host-gateway" + - PYTHONPATH=/var/task + - AWS_ENDPOINT_URL=http://moto-server:5000 + - DYNAMODB_ENDPOINT=http://moto-server:5000 + - S3_ENDPOINT=http://moto-server:5000 + - SECRET_MANAGER_ENDPOINT=http://moto-server:5000 + - FIREHOSE_ENDPOINT=http://moto-server:5000 + - LOG_LEVEL=INFO + entrypoint: /bin/sh + command: + - "-c" + - | + mkdir -p /var/task && + python3 -m zipfile -e /tmp/lambda.zip /var/task && + exec /usr/local/bin/aws-lambda-rie python3 -m awslambdaric eligibility_signposting_api.app.lambda_handler depends_on: - moto-server networks: - test-network + api-gateway-mock: + image: openresty/openresty:alpine + ports: + - "9123:9123" + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + networks: + - test-network + networks: test-network: driver: bridge diff --git a/tests/gateway.js b/tests/gateway.js new file mode 100644 index 000000000..30dd84513 --- /dev/null +++ b/tests/gateway.js @@ -0,0 +1,25 @@ +function unwrap(r) { + r.subrequest('/proxy' + r.uri, { method: r.method }, function(reply) { + + if (reply.status !== 200) { + r.return(502, "Lambda Bridge Error: " + reply.status); + return; + } + + // Check for empty/undefined body + if (!reply.responseBody) { + ngx.log(ngx.ERR, "CRITICAL: Lambda returned 200 but body is EMPTY/UNDEFINED"); + r.return(502, "Lambda returned empty response. Check Python logs for logic errors."); + return; + } + + try { + var response = JSON.parse(reply.responseBody); + r.return(response.statusCode || 200, response.body || ""); + } catch (e) { + ngx.log(ngx.ERR, "JSON Parse Error: " + e.message + " | Body: " + reply.responseBody); + r.return(502, "Invalid JSON from Lambda"); + } + }); +} +export default { unwrap }; diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ed219feea..fffbb273b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -5,6 +5,7 @@ import subprocess from collections.abc import Callable, Generator from pathlib import Path +from typing import List from typing import TYPE_CHECKING, Any import httpx @@ -13,8 +14,8 @@ from boto3 import Session from boto3.resources.base import ServiceResource from botocore.client import BaseClient +from botocore.exceptions import ClientError from faker import Faker -from httpx import RequestError from yarl import URL from eligibility_signposting_api.model import eligibility_status @@ -40,7 +41,7 @@ from tests.fixtures.builders.repos.person import person_rows_builder if TYPE_CHECKING: - from pytest_docker.plugin import Services + pass logger = logging.getLogger(__name__) @@ -54,10 +55,12 @@ MOTO_PORT = 5000 + @pytest.fixture(scope="session") def docker_compose_project_name(): return "tests" + @pytest.fixture(scope="session", autouse=True) def aws_credentials(): """Mocked AWS Credentials for moto.""" @@ -83,12 +86,10 @@ def moto_server(request: pytest.FixtureRequest) -> URL: def is_responsive(url: URL) -> bool: try: - response = httpx.get(str(url)) - response.raise_for_status() - except RequestError: + response = httpx.get(str(url), timeout=2.0) + return response.status_code < 500 + except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.TimeoutException): return False - else: - return True @pytest.fixture(scope="session") @@ -96,14 +97,8 @@ def boto3_session() -> Session: return Session(aws_access_key_id="fake", aws_secret_access_key="fake", aws_session_token="fake", region_name=AWS_REGION) @pytest.fixture(scope="session") -def api_gateway_client(boto3_session: Session, moto_server: URL) -> BaseClient: - return boto3_session.client("apigateway", endpoint_url=str(moto_server)) - - -@pytest.fixture(scope="session") -def lambda_client(boto3_session: Session, moto_server: URL) -> BaseClient: - return boto3_session.client("lambda", endpoint_url=str(moto_server)) - +def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient: + return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) @pytest.fixture(scope="session") def dynamodb_client(boto3_session: Session, moto_server: URL) -> BaseClient: @@ -114,6 +109,18 @@ def dynamodb_client(boto3_session: Session, moto_server: URL) -> BaseClient: def dynamodb_resource(boto3_session: Session, moto_server: URL) -> ServiceResource: return boto3_session.resource("dynamodb", endpoint_url=str(moto_server)) +def get_log_messages(flask_function: str, logs_client: BaseClient) -> list[str]: + for attempt in stamina.retry_context(on=ClientError, attempts=20, timeout=120): + with attempt: + log_streams = logs_client.describe_log_streams( + logGroupName=f"/aws/lambda/{flask_function}", orderBy="LastEventTime", descending=True + ) + assert log_streams["logStreams"] != [] + log_stream_name = log_streams["logStreams"][0]["logStreamName"] + log_events = logs_client.get_log_events( + logGroupName=f"/aws/lambda/{flask_function}", logStreamName=log_stream_name, limit=100 + ) + return [e["message"] for e in log_events["events"]] @pytest.fixture(scope="session") def logs_client(boto3_session: Session, moto_server: URL) -> BaseClient: @@ -169,121 +176,6 @@ def secretsmanager_client(boto3_session: Session, moto_server: URL) -> BaseClien return client -@pytest.fixture(scope="session") -def iam_role(iam_client: BaseClient) -> Generator[str]: - role_name = "LambdaExecutionRole" - policy_name = "LambdaCloudWatchPolicy" - - # Define IAM Trust Policy for Lambda Execution Role - trust_policy = { - "Version": "2012-10-17", - "Statement": [ - {"Effect": "Allow", "Principal": {"Service": "lambda.amazonaws.com"}, "Action": "sts:AssumeRole"} - ], - } - - # Create IAM Role - role = iam_client.create_role( - RoleName=role_name, - AssumeRolePolicyDocument=json.dumps(trust_policy), - Description="Role for Lambda execution with CloudWatch logging permissions", - ) - - # Define IAM Policy for CloudWatch Logs - log_policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], - "Resource": "arn:aws:logs:*:*:*", - } - ], - } - dynamodb_policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "dynamodb:GetItem", - "dynamodb:PutItem", - "dynamodb:UpdateItem", - "dynamodb:DeleteItem", - "dynamodb:Scan", - "dynamodb:Query", - ], - "Resource": "arn:aws:dynamodb:*:*:table/*", - } - ], - } - - # Create CloudWatch Logs policy (as before) - log_policy_resp = iam_client.create_policy(PolicyName=policy_name, PolicyDocument=json.dumps(log_policy)) - log_policy_arn = log_policy_resp["Policy"]["Arn"] - iam_client.attach_role_policy(RoleName=role_name, PolicyArn=log_policy_arn) - - # Create DynamoDB policy - ddb_policy_resp = iam_client.create_policy( - PolicyName="LambdaDynamoDBPolicy", PolicyDocument=json.dumps(dynamodb_policy) - ) - ddb_policy_arn = ddb_policy_resp["Policy"]["Arn"] - iam_client.attach_role_policy(RoleName=role_name, PolicyArn=ddb_policy_arn) - - yield role["Role"]["Arn"] - - iam_client.detach_role_policy(RoleName=role_name, PolicyArn=log_policy_arn) - iam_client.delete_policy(PolicyArn=log_policy_arn) - iam_client.detach_role_policy(RoleName=role_name, PolicyArn=ddb_policy_arn) - iam_client.delete_policy(PolicyArn=ddb_policy_arn) - iam_client.delete_role(RoleName=role_name) - - -@pytest.fixture(scope="session") -def lambda_zip() -> Path: - # Run the build command - build_result = subprocess.run(["make", "build"], capture_output=True, text=True, check=False) - - # If it fails, print the actual stderr so you can see WHY in the pytest logs - if build_result.returncode != 0: - pytest.fail( - f"'make build' failed with code {build_result.returncode}.\nSTDOUT: {build_result.stdout}\nSTDERR: {build_result.stderr}") - - zip_path = Path("dist/lambda.zip") - if not zip_path.exists(): - pytest.fail(f"Build succeeded but {zip_path} was not created.") - - return zip_path - - -@pytest.fixture(scope="session") -def flask_function(lambda_client: BaseClient, iam_role: str, lambda_zip: Path) -> Generator[str]: - function_name = "eligibility_signposting_api" - - # Use the INTERNAL docker name. Inside the Docker network, Moto is always 5000. - moto_internal = "http://moto-server:5000" - - with lambda_zip.open("rb") as zipfile: - lambda_client.create_function( - FunctionName=function_name, - Runtime="python3.13", - Role=iam_role, - Handler="eligibility_signposting_api.app.lambda_handler", - Code={"ZipFile": zipfile.read()}, - Environment={ - "Variables": { - "DYNAMODB_ENDPOINT": moto_internal, - "S3_ENDPOINT": moto_internal, - "SECRET_MANAGER_ENDPOINT": moto_internal, - "FIREHOSE_ENDPOINT": moto_internal, - "AWS_REGION": AWS_REGION, - } - }, - ) - wait_for_function_active(function_name, lambda_client) - yield function_name - lambda_client.delete_function(FunctionName=function_name) - @pytest.fixture(autouse=True) def clean_audit_bucket(s3_client: BaseClient, audit_bucket: str): objects_to_delete = [] @@ -300,108 +192,6 @@ def clean_audit_bucket(s3_client: BaseClient, audit_bucket: str): ) -@pytest.fixture(scope="session") -def flask_function_url(lambda_client: BaseClient, flask_function: str) -> URL: - response = lambda_client.create_function_url_config(FunctionName=flask_function, AuthType="NONE") - return URL(response["FunctionUrl"]) - - -class FunctionNotActiveError(Exception): - """Lambda Function not yet active""" - - -def wait_for_function_active(function_name, lambda_client): - for attempt in stamina.retry_context(on=FunctionNotActiveError, attempts=20, timeout=120): - with attempt: - logger.info("waiting") - response = lambda_client.get_function(FunctionName=function_name) - function_state = response["Configuration"]["State"] - logger.info("function_state %s", function_state) - if function_state != "Active": - raise FunctionNotActiveError - - -@pytest.fixture(scope="session") -def configured_api_gateway(api_gateway_client, lambda_client, flask_function: str, moto_server: URL): - region = lambda_client.meta.region_name - - # 1. Create the REST API - api = api_gateway_client.create_rest_api( - name="API Gateway Lambda integration", - endpointConfiguration={'types': ['REGIONAL']} - ) - rest_api_id = api["id"] - - # 2. Get Root Resource ID - resources = api_gateway_client.get_resources(restApiId=rest_api_id) - root_id = next(item["id"] for item in resources["items"] if item["path"] == "/") - - # 3. Create Resource Path: /patient-check/{id} - patient_check_res = api_gateway_client.create_resource( - restApiId=rest_api_id, parentId=root_id, pathPart="patient-check" - ) - patient_check_id = patient_check_res["id"] - - id_res = api_gateway_client.create_resource( - restApiId=rest_api_id, parentId=patient_check_id, pathPart="{id}" - ) - resource_id = id_res["id"] - - # 4. Create GET Method - api_gateway_client.put_method( - restApiId=rest_api_id, - resourceId=resource_id, - httpMethod="GET", - authorizationType="NONE", - requestParameters={"method.request.path.id": True}, - ) - - # 5. Setup Lambda Integration - # Moto uses account 123456789012 by default - lambda_uri = ( - f"arn:aws:apigateway:{region}:lambda:path/2015-03-31/functions/" - f"arn:aws:lambda:{region}:123456789012:function:{flask_function}/invocations" - ) - - api_gateway_client.put_integration( - restApiId=rest_api_id, - resourceId=resource_id, - httpMethod="GET", - type="AWS_PROXY", - integrationHttpMethod="POST", - uri=lambda_uri, - passthroughBehavior="WHEN_NO_MATCH", - ) - - # 6. Add Permission for API Gateway to invoke Lambda - lambda_client.add_permission( - FunctionName=flask_function, - StatementId="apigateway-access", - Action="lambda:InvokeFunction", - Principal="apigateway.amazonaws.com", - SourceArn=f"arn:aws:execute-api:{region}:123456789012:{rest_api_id}/*/GET/patient-check/*", - ) - - # 7. Create Deployment and Stage - api_gateway_client.create_deployment(restApiId=rest_api_id, stageName="dev") - - # 8. Construct the Moto-compatible Invoke URL - # Inside configured_api_gateway fixture - moto_base_url = str(moto_server).rstrip("/") - # Moto routes via: /restapis///_user_request_/ - invoke_url = f"{moto_base_url}/restapis/{rest_api_id}/dev/_user_request_" - - return { - "rest_api_id": rest_api_id, - "resource_id": resource_id, - "invoke_url": invoke_url, - } - -@pytest.fixture -def api_gateway_endpoint(configured_api_gateway: dict) -> URL: - return URL(configured_api_gateway["invoke_url"]) - - @pytest.fixture(scope="session") def person_table(dynamodb_resource: ServiceResource) -> Generator[Any]: table = dynamodb_resource.create_table( diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py new file mode 100644 index 000000000..d73b92e10 --- /dev/null +++ b/tests/integration/lambda/conftest.py @@ -0,0 +1,111 @@ +import subprocess +from pathlib import Path +from typing import Callable + +import pytest +from yarl import URL + +from tests.integration.conftest import is_responsive + + +@pytest.fixture(scope="session") +def lambda_zip() -> Path: + # Determine project root (directory containing this conftest.py) + project_root = Path(__file__).resolve().parents[3] + + build_result = subprocess.run( + ["make", "build"], + cwd=project_root, + capture_output=True, + text=True, + check=False, + ) + + if build_result.returncode != 0: + pytest.fail( + f"'make build' failed with code {build_result.returncode}.\n" + f"STDOUT: {build_result.stdout}\n" + f"STDERR: {build_result.stderr}" + ) + + zip_path = project_root / "dist/lambda.zip" + if not zip_path.exists(): + pytest.fail(f"Build succeeded but {zip_path} was not created.") + + return zip_path + +@pytest.fixture(scope="session") +def lambda_runtime_url(request: pytest.FixtureRequest, lambda_zip: Path) -> URL: + docker_services = request.getfixturevalue("docker_services") + docker_ip = request.getfixturevalue("docker_ip") + + wait_for_zip_in_container(docker_services, "lambda-api", "/tmp/lambda.zip") + force_lambda_reload(docker_services) + + port = docker_services.port_for("lambda-api", 8080) + base_url = URL(f"http://{docker_ip}:{port}") + + docker_services.wait_until_responsive( + timeout=30.0, + pause=1.0, + check=lambda: is_responsive(base_url) + ) + return base_url + + +@pytest.fixture(scope="session") +def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url) -> URL: + docker_services = request.getfixturevalue("docker_services") + docker_ip = request.getfixturevalue("docker_ip") + + port = docker_services.port_for("api-gateway-mock", 9123) + + base_url = URL(f"http://{docker_ip}:{port}") + health_url = URL(f"http://{docker_ip}:{port}/health") + + docker_services.wait_until_responsive( + timeout=30.0, + pause=1.0, + check=lambda: is_responsive(health_url) + ) + return base_url + +def wait_for_zip_in_container(docker_services, service: str, path: str) -> None: + def _check() -> bool: + try: + docker_services._docker_compose.execute( + f"exec {service} test -f {path}" + ) + return True + except Exception: + return False + + docker_services.wait_until_responsive( + timeout=30, + pause=1, + check=_check, + ) + + +def force_lambda_reload(docker_services): + # Stop and remove lambda-api + docker_services._docker_compose.execute(f"stop lambda-api") + docker_services._docker_compose.execute(f"rm -f lambda-api") + + # Start lambda-api fresh so it re-reads the mounted ZIP + docker_services._docker_compose.execute(f"up --build -d lambda-api") + + +@pytest.fixture +def lambda_logs(docker_services) -> Callable[[], list[str]]: + def _get_messages() -> list[str]: + return get_lambda_logs(docker_services) + + return _get_messages + + +def get_lambda_logs(docker_services) -> list[str]: + result: bytes = docker_services._docker_compose.execute("logs --no-color lambda-api") + raw_lines = result.decode("utf-8").splitlines() + return [line.partition("|")[-1].strip() for line in raw_lines] + diff --git a/tests/integration/lambda/test_app_running_as_lambda.py b/tests/integration/lambda/test_app_running_as_lambda.py index 08618038b..cda79a16a 100644 --- a/tests/integration/lambda/test_app_running_as_lambda.py +++ b/tests/integration/lambda/test_app_running_as_lambda.py @@ -2,6 +2,7 @@ import json import logging from http import HTTPStatus +from typing import Callable, List import httpx import stamina @@ -34,10 +35,10 @@ def test_install_and_call_lambda_flask( lambda_client: BaseClient, - flask_function: str, persisted_person: NHSNumber, consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa: ARG001 consumer_id: ConsumerId, + secretsmanager_client: BaseClient, ): """Given lambda installed into localstack, run it via boto3 lambda client""" # Given @@ -68,12 +69,12 @@ def test_install_and_call_lambda_flask( "isBase64Encoded": False, } response = lambda_client.invoke( - FunctionName=flask_function, + FunctionName="function", InvocationType="RequestResponse", Payload=json.dumps(request_payload), - LogType="Tail", + # LogType="Tail", #TODO ) - log_output = base64.b64decode(response["LogResult"]).decode("utf-8") + # log_output = base64.b64decode(response["LogResult"]).decode("utf-8") # Then assert_that(response, has_entries(StatusCode=HTTPStatus.OK)) @@ -84,13 +85,14 @@ def test_install_and_call_lambda_flask( has_entries(statusCode=HTTPStatus.OK, body=is_json_that(has_key("processedSuggestions"))), ) - assert_that(log_output, contains_string("checking nhs_number")) + # assert_that(log_output, contains_string("checking nhs_number")) def test_install_and_call_flask_lambda_over_http( persisted_person: NHSNumber, consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa: ARG001 consumer_id: ConsumerId, + secretsmanager_client: BaseClient, api_gateway_endpoint: URL, ): """Given api-gateway and lambda installed into localstack, run it via http""" @@ -111,13 +113,13 @@ def test_install_and_call_flask_lambda_over_http( def test_install_and_call_flask_lambda_with_unknown_nhs_number( # noqa: PLR0913 - flask_function: str, persisted_person: NHSNumber, consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa: ARG001 consumer_id: ConsumerId, logs_client: BaseClient, api_gateway_endpoint: URL, secretsmanager_client: BaseClient, # noqa: ARG001 + lambda_logs: Callable[[], List[str]] ): """Given lambda installed into localstack, run it via http, with a nonexistent NHS number specified""" # Given @@ -145,15 +147,15 @@ def test_install_and_call_flask_lambda_with_unknown_nhs_number( # noqa: PLR0913 severity="error", code="processing", diagnostics=f"NHS Number '{nhs_number!s}' was not " - f"recognised by the Eligibility Signposting API", + f"recognised by the Eligibility Signposting API", details={ "coding": [ { "system": "https://fhir.nhs.uk/STU3/ValueSet/Spine-ErrorOrWarningCode-1", "code": "REFERENCE_NOT_FOUND", "display": "The given NHS number was not found in our datasets. " - "This could be because the number is incorrect or " - "some other reason we cannot process that number.", + "This could be because the number is incorrect or " + "some other reason we cannot process that number.", } ] }, @@ -164,27 +166,13 @@ def test_install_and_call_flask_lambda_with_unknown_nhs_number( # noqa: PLR0913 ), ) - messages = get_log_messages(flask_function, logs_client) + messages = lambda_logs() assert_that( messages, has_item(contains_string(f"NHS Number '{nhs_number}' was not recognised by the Eligibility Signposting API")), ) -def get_log_messages(flask_function: str, logs_client: BaseClient) -> list[str]: - for attempt in stamina.retry_context(on=ClientError, attempts=20, timeout=120): - with attempt: - log_streams = logs_client.describe_log_streams( - logGroupName=f"/aws/lambda/{flask_function}", orderBy="LastEventTime", descending=True - ) - assert log_streams["logStreams"] != [] - log_stream_name = log_streams["logStreams"][0]["logStreamName"] - log_events = logs_client.get_log_events( - logGroupName=f"/aws/lambda/{flask_function}", logStreamName=log_stream_name, limit=100 - ) - return [e["message"] for e in log_events["events"]] - - def test_given_nhs_number_in_path_matches_with_nhs_number_in_headers_and_check_if_audited( # noqa: PLR0913 lambda_client: BaseClient, # noqa:ARG001 persisted_person: NHSNumber, @@ -194,8 +182,8 @@ def test_given_nhs_number_in_path_matches_with_nhs_number_in_headers_and_check_i s3_client: BaseClient, audit_bucket: BucketName, api_gateway_endpoint: URL, - flask_function: str, - logs_client: BaseClient, + lambda_logs: Callable[[], List[str]], + secretsmanager_client: BaseClient, ): # Given # When @@ -274,7 +262,7 @@ def test_given_nhs_number_in_path_matches_with_nhs_number_in_headers_and_check_i assert_that(audit_data["response"]["lastUpdated"], is_not(equal_to(""))) assert_that(audit_data["response"]["condition"], equal_to(expected_conditions)) - messages = get_log_messages(flask_function, logs_client) + messages = lambda_logs() assert_that( messages, has_item(contains_string("Defaulting category query param to 'ALL' as no value was provided")), @@ -332,8 +320,10 @@ def test_given_nhs_number_in_path_does_not_match_with_nhs_number_in_headers_resu def test_given_nhs_number_not_present_in_headers_results_in_valid_for_application_restricted_users( - lambda_client: BaseClient, # noqa:ARG001 + lambda_client: BaseClient, + secretsmanager_client: BaseClient, # noqa:ARG001 persisted_person: NHSNumber, + consumer_to_active_rsv_campaign_mapping: ConsumerMapping, api_gateway_endpoint: URL, ): # Given @@ -632,7 +622,7 @@ def test_token_formatting_in_eligibility_response_and_audit( # noqa: PLR0913 s3_client: BaseClient, audit_bucket: BucketName, api_gateway_endpoint: URL, - flask_function: str, # noqa:ARG001 + # noqa:ARG001 logs_client: BaseClient, # noqa:ARG001 ): invoke_url = f"{api_gateway_endpoint}/patient-check/{person_with_all_data}" @@ -683,8 +673,7 @@ def test_incorrect_token_causes_internal_server_error( # noqa: PLR0913 s3_client: BaseClient, audit_bucket: BucketName, api_gateway_endpoint: URL, - flask_function: str, - logs_client: BaseClient, +lambda_logs: Callable[[], List[str]] ): invoke_url = f"{api_gateway_endpoint}/patient-check/{person_with_all_data}" response = httpx.get( @@ -727,7 +716,7 @@ def test_incorrect_token_causes_internal_server_error( # noqa: PLR0913 assert len(objects) == 0 # Check there are no audit logs assert_that( - get_log_messages(flask_function, logs_client), + lambda_logs(), has_item(contains_string("Invalid attribute name 'ICECREAM' in token '[[PERSON.ICECREAM]]'.")), ) diff --git a/tests/locals.json b/tests/locals.json deleted file mode 100644 index ca6f89cde..000000000 --- a/tests/locals.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "EligibilityFunction": { - "PYTHONPATH": "/var/task", - "DYNAMODB_ENDPOINT": "http://moto-server:5000", - "S3_ENDPOINT": "http://moto-server:5000", - "AWS_ACCESS_KEY_ID": "testing", - "AWS_SECRET_ACCESS_KEY": "testing", - "AWS_REGION": "eu-west-1" - } -} diff --git a/tests/nginx.conf b/tests/nginx.conf new file mode 100644 index 000000000..6534d6c98 --- /dev/null +++ b/tests/nginx.conf @@ -0,0 +1,113 @@ +server { + listen 9123; + server_name _; + + underscores_in_headers on; + + location /health { + access_log off; + return 200 'healthy'; + } + + location / { + lua_need_request_body on; + + content_by_lua_block { + local cjson = require "cjson.safe" + + -- Extract request details + local headers = ngx.req.get_headers() + local method = ngx.req.get_method() + local uri = ngx.var.uri + local query = ngx.var.args or "" + local body = ngx.req.get_body_data() + + -- Parse query params into API Gateway v2 format + local args = ngx.req.get_uri_args() + local query_params = {} + + for k, v in pairs(args) do + -- API Gateway v2 always uses string values + if type(v) == "table" then + query_params[k] = v[#v] -- take last value if multi + else + query_params[k] = tostring(v) + end + end + + -- Extract path parameter + local persisted_person = nil + local m = ngx.re.match(uri, [[^/patient-check/([^/]+)$]]) + if m then + persisted_person = m[1] + end + + -- Build API Gateway v2 event + local event = { + version = "2.0", + routeKey = method .. " /patient-check/{persisted_person}", + rawPath = uri, + rawQueryString = query, + headers = headers, + queryStringParameters = next(query_params) and query_params or nil, + pathParameters = persisted_person and { persisted_person = persisted_person } or {}, + requestContext = { + http = { + method = method, + path = uri, + protocol = "HTTP/1.1", + sourceIp = ngx.var.remote_addr + } + }, + body = body, + isBase64Encoded = false + } + + local json = cjson.encode(event) + + -- Invoke Lambda RIE + local res = ngx.location.capture( + "/invoke_lambda", + { + method = ngx.HTTP_POST, + body = json, + ctx = {} + } + ) + + if not res or not res.body then + ngx.status = 502 + ngx.say("Lambda invocation failed") + return ngx.exit(ngx.status) + end + + -- Decode Lambda proxy response + local lambda_resp = cjson.decode(res.body) + if not lambda_resp then + ngx.status = 502 + ngx.say("Invalid Lambda JSON: ", res.body) + return ngx.exit(ngx.status) + end + + -- Set HTTP status + ngx.status = lambda_resp.statusCode or 200 + + -- Set headers + if lambda_resp.headers then + for k, v in pairs(lambda_resp.headers) do + ngx.header[k] = v + end + end + + -- Write body + ngx.say(lambda_resp.body) + + return ngx.exit(ngx.status) + } + } + + location /invoke_lambda { + proxy_pass http://lambda-api:8080/2015-03-31/functions/function/invocations; + proxy_read_timeout 30s; + } +} diff --git a/tests/template.yaml b/tests/template.yaml deleted file mode 100644 index bb9b6495d..000000000 --- a/tests/template.yaml +++ /dev/null @@ -1,34 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 -Description: SAM Template using pre-built Zip for Eligibility Signposting API - -Globals: - Function: - Timeout: 30 - MemorySize: 256 - Runtime: python3.13 - Handler: eligibility_signposting_api.app.lambda_handler - Environment: - Variables: - DYNAMODB_ENDPOINT: http://moto-server:5000 - S3_ENDPOINT: http://moto-server:5000 - SECRET_MANAGER_ENDPOINT: http://moto-server:5000 - FIREHOSE_ENDPOINT: http://moto-server:5000 - AWS_REGION: eu-west-1 - PERSON_TABLE_NAME: test_eligibility_datastore - RULES_BUCKET_NAME: test-rules-bucket - CONSUMER_MAPPING_BUCKET_NAME: test-consumer-mapping-bucket - AUDIT_BUCKET_NAME: test-audit-bucket - -Resources: - EligibilityFunction: - Type: AWS::Serverless::Function - Properties: - # Point this to the location of your zip file relative to the template - CodeUri: ./dist/lambda.zip - Events: - GetEligibility: - Type: Api - Properties: - Path: /patient-check/{id} - Method: get From 6018f09867a86c0d69a644e7ca610db330ffc81f Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 12:34:36 +0000 Subject: [PATCH 07/16] moved lambda client to lambda conftest --- tests/integration/conftest.py | 4 ---- tests/integration/lambda/conftest.py | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fffbb273b..75a04bd95 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -96,15 +96,11 @@ def is_responsive(url: URL) -> bool: def boto3_session() -> Session: return Session(aws_access_key_id="fake", aws_secret_access_key="fake", aws_session_token="fake", region_name=AWS_REGION) -@pytest.fixture(scope="session") -def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient: - return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) @pytest.fixture(scope="session") def dynamodb_client(boto3_session: Session, moto_server: URL) -> BaseClient: return boto3_session.client("dynamodb", endpoint_url=str(moto_server)) - @pytest.fixture(scope="session") def dynamodb_resource(boto3_session: Session, moto_server: URL) -> ServiceResource: return boto3_session.resource("dynamodb", endpoint_url=str(moto_server)) diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index d73b92e10..4c78f6fce 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -3,6 +3,8 @@ from typing import Callable import pytest +from boto3 import Session +from botocore.client import BaseClient from yarl import URL from tests.integration.conftest import is_responsive @@ -52,6 +54,9 @@ def lambda_runtime_url(request: pytest.FixtureRequest, lambda_zip: Path) -> URL: ) return base_url +@pytest.fixture(scope="session") +def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient: + return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) @pytest.fixture(scope="session") def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url) -> URL: From 1f94c19bb66af7abef86567e6a097e90386f508d Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:46:07 +0000 Subject: [PATCH 08/16] use docker compose profiling --- poetry.lock | 14 ++++- pyproject.toml | 2 +- tests/docker-compose.yml | 5 ++ tests/integration/conftest.py | 3 -- tests/integration/lambda/conftest.py | 78 ++++++++++++++++------------ 5 files changed, 63 insertions(+), 39 deletions(-) diff --git a/poetry.lock b/poetry.lock index f019e41b4..1f2455bca 100644 --- a/poetry.lock +++ b/poetry.lock @@ -442,6 +442,18 @@ files = [ {file = "brunns_row-2.1.0-py3-none-any.whl", hash = "sha256:0953f231c812658a7cfd3240f1f6fac92e7b5bba247cfc22a4afa134f2a540f3"}, ] +[[package]] +name = "cachetools" +version = "7.0.1" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "cachetools-7.0.1-py3-none-any.whl", hash = "sha256:8f086515c254d5664ae2146d14fc7f65c9a4bce75152eb247e5a9c5e6d7b2ecf"}, + {file = "cachetools-7.0.1.tar.gz", hash = "sha256:e31e579d2c5b6e2944177a0397150d312888ddf4e16e12f1016068f0c03b8341"}, +] + [[package]] name = "certifi" version = "2025.6.15" @@ -3444,4 +3456,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "a3ad1996923b878a8227dbbeb7d3a1455650c67163d8c79fbb5700a3ab1ff157" +content-hash = "c5064b43e402173391286c84cff772c1776fdf816a8fbd229cfdafa26da4b456" diff --git a/pyproject.toml b/pyproject.toml index a2bcf6978..4fbaa3f8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ behave = "^1.3.3" python-dotenv = "^1.2.1" openapi-spec-validator = "^0.7.2" pip-licenses = "^5.5.0" - +cachetools = "^7.0.1" [tool.poetry-plugin-lambda-build] docker-image = "public.ecr.aws/sam/build-python3.13:1.139-x86_64" # See https://gallery.ecr.aws/search?searchTerm=%22python%22&architecture=x86-64&popularRegistries=amazon&verified=verified&operatingSystems=Linux diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index fe186265d..346ac6a67 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,5 +1,6 @@ services: moto-server: + #used for s3, dynamodb, kinesis, secret manager image: motoserver/moto:latest container_name: moto-server ports: @@ -8,6 +9,8 @@ services: - test-network lambda-api: + #used for lambda simulation + profiles: ["lambda-test"] image: public.ecr.aws/lambda/python:3.13 container_name: lambda-api ports: @@ -39,6 +42,8 @@ services: - test-network api-gateway-mock: + #used for api gateway simulation + profiles: ["lambda-test"] image: openresty/openresty:alpine ports: - "9123:9123" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 75a04bd95..ce070f144 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,10 +2,7 @@ import json import logging import os -import subprocess from collections.abc import Callable, Generator -from pathlib import Path -from typing import List from typing import TYPE_CHECKING, Any import httpx diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index 4c78f6fce..459b052a3 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -1,3 +1,4 @@ +import os import subprocess from pathlib import Path from typing import Callable @@ -10,10 +11,17 @@ from tests.integration.conftest import is_responsive +def get_project_root() -> Path: + """Finds the project root by looking for the 'dist' directory or a git folder.""" + current = Path(__file__).resolve() + for parent in current.parents: + if (parent / "dist").exists() or (parent / ".git").exists(): + return parent + return Path(__file__).resolve().parents[3] + @pytest.fixture(scope="session") def lambda_zip() -> Path: - # Determine project root (directory containing this conftest.py) - project_root = Path(__file__).resolve().parents[3] + project_root = get_project_root() build_result = subprocess.run( ["make", "build"], @@ -37,29 +45,51 @@ def lambda_zip() -> Path: return zip_path @pytest.fixture(scope="session") -def lambda_runtime_url(request: pytest.FixtureRequest, lambda_zip: Path) -> URL: +def lambda_runtime_url(request, lambda_zip): + """ + kick-starts the lambda simulation + """ docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") - - wait_for_zip_in_container(docker_services, "lambda-api", "/tmp/lambda.zip") - force_lambda_reload(docker_services) + project_root = get_project_root() + compose_file = project_root / "tests/docker-compose.yml" + + env = os.environ.copy() + env["COMPOSE_PROFILES"] = "lambda-test" + + subprocess.run( + [ + "docker", "compose", + "-f", str(compose_file), + "up", "-d", "--build", "--force-recreate", + "lambda-api", "api-gateway-mock", + ], + env=env, + check=True, + capture_output=True, + text=True, + ) port = docker_services.port_for("lambda-api", 8080) base_url = URL(f"http://{docker_ip}:{port}") docker_services.wait_until_responsive( - timeout=30.0, - pause=1.0, - check=lambda: is_responsive(base_url) + timeout=30.0, pause=0.5, check=lambda: is_responsive(base_url) ) + return base_url + + @pytest.fixture(scope="session") def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient: return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) @pytest.fixture(scope="session") def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url) -> URL: + """ + kick-starts the api-gateway lambda simulation + """ docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") @@ -75,34 +105,12 @@ def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url) -> ) return base_url -def wait_for_zip_in_container(docker_services, service: str, path: str) -> None: - def _check() -> bool: - try: - docker_services._docker_compose.execute( - f"exec {service} test -f {path}" - ) - return True - except Exception: - return False - - docker_services.wait_until_responsive( - timeout=30, - pause=1, - check=_check, - ) - - -def force_lambda_reload(docker_services): - # Stop and remove lambda-api - docker_services._docker_compose.execute(f"stop lambda-api") - docker_services._docker_compose.execute(f"rm -f lambda-api") - - # Start lambda-api fresh so it re-reads the mounted ZIP - docker_services._docker_compose.execute(f"up --build -d lambda-api") - @pytest.fixture def lambda_logs(docker_services) -> Callable[[], list[str]]: + """Returns a callable that fetches the latest lambda-api logs, + allowing tests to inspect runtime output on demand.""" + def _get_messages() -> list[str]: return get_lambda_logs(docker_services) @@ -110,6 +118,8 @@ def _get_messages() -> list[str]: def get_lambda_logs(docker_services) -> list[str]: + """returns logs from lambda-api container""" + result: bytes = docker_services._docker_compose.execute("logs --no-color lambda-api") raw_lines = result.decode("utf-8").splitlines() return [line.partition("|")[-1].strip() for line in raw_lines] From e7a15c5b649c394739dcab84ac1859607c3c011b Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 15:22:12 +0000 Subject: [PATCH 09/16] lint fixes --- tests/docker-compose.yml | 1 + tests/integration/conftest.py | 27 ++--- tests/integration/lambda/conftest.py | 100 +++++++++++++----- .../lambda/test_app_running_as_lambda.py | 32 +++--- 4 files changed, 101 insertions(+), 59 deletions(-) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 346ac6a67..640d97bb9 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,6 +1,7 @@ services: moto-server: #used for s3, dynamodb, kinesis, secret manager + # lambda cannot be used, because its 3.11 (older) image: motoserver/moto:latest container_name: moto-server ports: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ce070f144..7d2b0c732 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,7 +3,7 @@ import logging import os from collections.abc import Callable, Generator -from typing import TYPE_CHECKING, Any +from typing import Any import httpx import pytest @@ -37,9 +37,6 @@ from tests.fixtures.builders.model.rule import RulesMapperFactory from tests.fixtures.builders.repos.person import person_rows_builder -if TYPE_CHECKING: - pass - logger = logging.getLogger(__name__) AWS_REGION = "eu-west-1" @@ -52,6 +49,8 @@ MOTO_PORT = 5000 +HTTP_SERVER_ERROR = 500 + @pytest.fixture(scope="session") def docker_compose_project_name(): @@ -62,9 +61,9 @@ def docker_compose_project_name(): def aws_credentials(): """Mocked AWS Credentials for moto.""" os.environ["AWS_ACCESS_KEY_ID"] = "dummy_key" - os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret" - os.environ["AWS_SECURITY_TOKEN"] = "dummy_token" - os.environ["AWS_SESSION_TOKEN"] = "dummy_session_token" + os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret" # noqa: S105 + os.environ["AWS_SECURITY_TOKEN"] = "dummy_token" # noqa: S105 + os.environ["AWS_SESSION_TOKEN"] = "dummy_session_token" # noqa: S105 os.environ["AWS_DEFAULT_REGION"] = AWS_REGION @@ -84,24 +83,30 @@ def moto_server(request: pytest.FixtureRequest) -> URL: def is_responsive(url: URL) -> bool: try: response = httpx.get(str(url), timeout=2.0) - return response.status_code < 500 except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.TimeoutException): return False + else: + # Use the constant instead of the raw number + return response.status_code < HTTP_SERVER_ERROR @pytest.fixture(scope="session") def boto3_session() -> Session: - return Session(aws_access_key_id="fake", aws_secret_access_key="fake", aws_session_token="fake", region_name=AWS_REGION) + return Session( + aws_access_key_id="fake", aws_secret_access_key="fake", aws_session_token="fake", region_name=AWS_REGION + ) @pytest.fixture(scope="session") def dynamodb_client(boto3_session: Session, moto_server: URL) -> BaseClient: return boto3_session.client("dynamodb", endpoint_url=str(moto_server)) + @pytest.fixture(scope="session") def dynamodb_resource(boto3_session: Session, moto_server: URL) -> ServiceResource: return boto3_session.resource("dynamodb", endpoint_url=str(moto_server)) + def get_log_messages(flask_function: str, logs_client: BaseClient) -> list[str]: for attempt in stamina.retry_context(on=ClientError, attempts=20, timeout=120): with attempt: @@ -115,10 +120,6 @@ def get_log_messages(flask_function: str, logs_client: BaseClient) -> list[str]: ) return [e["message"] for e in log_events["events"]] -@pytest.fixture(scope="session") -def logs_client(boto3_session: Session, moto_server: URL) -> BaseClient: - return boto3_session.client("logs", endpoint_url=str(moto_server)) - @pytest.fixture(scope="session") def iam_client(boto3_session: Session, moto_server: URL) -> BaseClient: diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index 459b052a3..87674d62d 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -1,7 +1,8 @@ import os +import shutil import subprocess +from collections.abc import Callable from pathlib import Path -from typing import Callable import pytest from boto3 import Session @@ -12,19 +13,25 @@ def get_project_root() -> Path: - """Finds the project root by looking for the 'dist' directory or a git folder.""" + """Find the project root by locating 'dist' or '.git'.""" current = Path(__file__).resolve() for parent in current.parents: if (parent / "dist").exists() or (parent / ".git").exists(): return parent return Path(__file__).resolve().parents[3] + @pytest.fixture(scope="session") def lambda_zip() -> Path: + """Build the lambda.zip artifact using `make build`.""" project_root = get_project_root() - build_result = subprocess.run( - ["make", "build"], + make_path = shutil.which("make") + if not make_path: + pytest.fail("The 'make' executable was not found in the system PATH.") + + build_result = subprocess.run( # noqa: S603 + [make_path, "build"], cwd=project_root, capture_output=True, text=True, @@ -34,61 +41,81 @@ def lambda_zip() -> Path: if build_result.returncode != 0: pytest.fail( f"'make build' failed with code {build_result.returncode}.\n" - f"STDOUT: {build_result.stdout}\n" - f"STDERR: {build_result.stderr}" + f"STDOUT:\n{build_result.stdout}\n" + f"STDERR:\n{build_result.stderr}" ) - zip_path = project_root / "dist/lambda.zip" + zip_path = project_root / "dist" / "lambda.zip" if not zip_path.exists(): pytest.fail(f"Build succeeded but {zip_path} was not created.") return zip_path + @pytest.fixture(scope="session") -def lambda_runtime_url(request, lambda_zip): +def lambda_runtime_url(request, lambda_zip): # noqa: ARG001 """ - kick-starts the lambda simulation + Start the lambda simulation using docker compose. """ docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") project_root = get_project_root() compose_file = project_root / "tests/docker-compose.yml" + # Activate the profile without using the --profile flag env = os.environ.copy() env["COMPOSE_PROFILES"] = "lambda-test" - subprocess.run( + docker_path = shutil.which("docker") + if not docker_path: + pytest.fail("Docker executable not found in PATH") + + result = subprocess.run( # noqa: S603 [ - "docker", "compose", - "-f", str(compose_file), - "up", "-d", "--build", "--force-recreate", - "lambda-api", "api-gateway-mock", + docker_path, + "compose", + "-f", + str(compose_file), + "up", + "-d", + "--build", + "--force-recreate", + "lambda-api", + "api-gateway-mock", ], env=env, - check=True, capture_output=True, text=True, + check=False, ) + if result.returncode != 0: + pytest.fail( + f"Docker compose failed with code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + port = docker_services.port_for("lambda-api", 8080) base_url = URL(f"http://{docker_ip}:{port}") docker_services.wait_until_responsive( - timeout=30.0, pause=0.5, check=lambda: is_responsive(base_url) + timeout=30.0, + pause=0.5, + check=lambda: is_responsive(base_url), ) return base_url - @pytest.fixture(scope="session") def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient: + """Return a boto3 Lambda client pointing at the simulated lambda runtime.""" return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) + @pytest.fixture(scope="session") -def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url) -> URL: +def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url): # noqa: ARG001 """ - kick-starts the api-gateway lambda simulation + Start and validate the API Gateway mock. """ docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") @@ -101,15 +128,15 @@ def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url) -> docker_services.wait_until_responsive( timeout=30.0, pause=1.0, - check=lambda: is_responsive(health_url) + check=lambda: is_responsive(health_url), ) + return base_url @pytest.fixture def lambda_logs(docker_services) -> Callable[[], list[str]]: - """Returns a callable that fetches the latest lambda-api logs, - allowing tests to inspect runtime output on demand.""" + """Return a callable that fetches the latest lambda-api logs.""" def _get_messages() -> list[str]: return get_lambda_logs(docker_services) @@ -117,10 +144,29 @@ def _get_messages() -> list[str]: return _get_messages -def get_lambda_logs(docker_services) -> list[str]: - """returns logs from lambda-api container""" +def get_lambda_logs(docker_services) -> list[str]: # noqa :ARG001 + """Fetch logs from the lambda-api container.""" + raw_docker = shutil.which("docker") + if not raw_docker: + return ["Error: Docker not found"] - result: bytes = docker_services._docker_compose.execute("logs --no-color lambda-api") - raw_lines = result.decode("utf-8").splitlines() - return [line.partition("|")[-1].strip() for line in raw_lines] + docker_path = Path(raw_docker).resolve() + project_root = get_project_root() + compose_file = (project_root / "tests" / "docker-compose.yml").resolve() + + result = subprocess.run( # noqa: S603 + [ + str(docker_path), + "compose", + "-f", + str(compose_file), + "logs", + "--no-color", + "lambda-api", + ], + capture_output=True, + text=True, + check=False, + ) + return [line.partition("|")[-1].strip() for line in result.stdout.splitlines()] diff --git a/tests/integration/lambda/test_app_running_as_lambda.py b/tests/integration/lambda/test_app_running_as_lambda.py index cda79a16a..355edf168 100644 --- a/tests/integration/lambda/test_app_running_as_lambda.py +++ b/tests/integration/lambda/test_app_running_as_lambda.py @@ -1,13 +1,10 @@ -import base64 import json import logging +from collections.abc import Callable from http import HTTPStatus -from typing import Callable, List import httpx -import stamina from botocore.client import BaseClient -from botocore.exceptions import ClientError from brunns.matchers.data import json_matching as is_json_that from brunns.matchers.response import is_response from freezegun import freeze_time @@ -38,7 +35,7 @@ def test_install_and_call_lambda_flask( persisted_person: NHSNumber, consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa: ARG001 consumer_id: ConsumerId, - secretsmanager_client: BaseClient, + secretsmanager_client: BaseClient, # noqa :ARG001 ): """Given lambda installed into localstack, run it via boto3 lambda client""" # Given @@ -92,7 +89,7 @@ def test_install_and_call_flask_lambda_over_http( persisted_person: NHSNumber, consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa: ARG001 consumer_id: ConsumerId, - secretsmanager_client: BaseClient, + secretsmanager_client: BaseClient, # noqa:ARG001 api_gateway_endpoint: URL, ): """Given api-gateway and lambda installed into localstack, run it via http""" @@ -116,10 +113,9 @@ def test_install_and_call_flask_lambda_with_unknown_nhs_number( # noqa: PLR0913 persisted_person: NHSNumber, consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa: ARG001 consumer_id: ConsumerId, - logs_client: BaseClient, api_gateway_endpoint: URL, secretsmanager_client: BaseClient, # noqa: ARG001 - lambda_logs: Callable[[], List[str]] + lambda_logs: Callable[[], list[str]], ): """Given lambda installed into localstack, run it via http, with a nonexistent NHS number specified""" # Given @@ -147,15 +143,15 @@ def test_install_and_call_flask_lambda_with_unknown_nhs_number( # noqa: PLR0913 severity="error", code="processing", diagnostics=f"NHS Number '{nhs_number!s}' was not " - f"recognised by the Eligibility Signposting API", + f"recognised by the Eligibility Signposting API", details={ "coding": [ { "system": "https://fhir.nhs.uk/STU3/ValueSet/Spine-ErrorOrWarningCode-1", "code": "REFERENCE_NOT_FOUND", "display": "The given NHS number was not found in our datasets. " - "This could be because the number is incorrect or " - "some other reason we cannot process that number.", + "This could be because the number is incorrect or " + "some other reason we cannot process that number.", } ] }, @@ -182,8 +178,8 @@ def test_given_nhs_number_in_path_matches_with_nhs_number_in_headers_and_check_i s3_client: BaseClient, audit_bucket: BucketName, api_gateway_endpoint: URL, - lambda_logs: Callable[[], List[str]], - secretsmanager_client: BaseClient, + lambda_logs: Callable[[], list[str]], + secretsmanager_client: BaseClient, # noqa:ARG001 ): # Given # When @@ -320,10 +316,10 @@ def test_given_nhs_number_in_path_does_not_match_with_nhs_number_in_headers_resu def test_given_nhs_number_not_present_in_headers_results_in_valid_for_application_restricted_users( - lambda_client: BaseClient, - secretsmanager_client: BaseClient, # noqa:ARG001 + lambda_client: BaseClient, # noqa:ARG001 + secretsmanager_client: BaseClient, # noqa:ARG001 persisted_person: NHSNumber, - consumer_to_active_rsv_campaign_mapping: ConsumerMapping, + consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa:ARG001 api_gateway_endpoint: URL, ): # Given @@ -622,8 +618,6 @@ def test_token_formatting_in_eligibility_response_and_audit( # noqa: PLR0913 s3_client: BaseClient, audit_bucket: BucketName, api_gateway_endpoint: URL, - # noqa:ARG001 - logs_client: BaseClient, # noqa:ARG001 ): invoke_url = f"{api_gateway_endpoint}/patient-check/{person_with_all_data}" response = httpx.get( @@ -673,7 +667,7 @@ def test_incorrect_token_causes_internal_server_error( # noqa: PLR0913 s3_client: BaseClient, audit_bucket: BucketName, api_gateway_endpoint: URL, -lambda_logs: Callable[[], List[str]] + lambda_logs: Callable[[], list[str]], ): invoke_url = f"{api_gateway_endpoint}/patient-check/{person_with_all_data}" response = httpx.get( From af8055cb93f0982c2f337683936a1eb5ebb19ab4 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 15:55:28 +0000 Subject: [PATCH 10/16] docker clean up --- tests/integration/lambda/conftest.py | 23 ++++++++++++------- .../lambda/test_app_running_as_lambda.py | 9 ++++---- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index 87674d62d..b3a30ceb9 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -103,13 +103,15 @@ def lambda_runtime_url(request, lambda_zip): # noqa: ARG001 check=lambda: is_responsive(base_url), ) - return base_url - + yield base_url -@pytest.fixture(scope="session") -def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient: - """Return a boto3 Lambda client pointing at the simulated lambda runtime.""" - return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) + subprocess.run( # noqa: S603 + [docker_path, "compose", "-f", str(compose_file), "rm", "-f", "-s", "lambda-api", "api-gateway-mock"], + check=False, + env=env, + capture_output=True, + text=True, + ) @pytest.fixture(scope="session") @@ -134,6 +136,12 @@ def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url): # return base_url +@pytest.fixture(scope="session") +def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient: + """Return a boto3 Lambda client pointing at the simulated lambda runtime.""" + return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) + + @pytest.fixture def lambda_logs(docker_services) -> Callable[[], list[str]]: """Return a callable that fetches the latest lambda-api logs.""" @@ -168,5 +176,4 @@ def get_lambda_logs(docker_services) -> list[str]: # noqa :ARG001 text=True, check=False, ) - - return [line.partition("|")[-1].strip() for line in result.stdout.splitlines()] + return [line.split("|", 1)[-1].strip() for line in result.stdout.splitlines()] diff --git a/tests/integration/lambda/test_app_running_as_lambda.py b/tests/integration/lambda/test_app_running_as_lambda.py index 355edf168..c717936eb 100644 --- a/tests/integration/lambda/test_app_running_as_lambda.py +++ b/tests/integration/lambda/test_app_running_as_lambda.py @@ -30,16 +30,16 @@ logger = logging.getLogger(__name__) -def test_install_and_call_lambda_flask( +def test_install_and_call_lambda_flask( # noqa: PLR0913 lambda_client: BaseClient, persisted_person: NHSNumber, consumer_to_active_rsv_campaign_mapping: ConsumerMapping, # noqa: ARG001 consumer_id: ConsumerId, secretsmanager_client: BaseClient, # noqa :ARG001 + lambda_logs: Callable[[], list[str]], ): """Given lambda installed into localstack, run it via boto3 lambda client""" # Given - # When request_payload = { "version": "2.0", @@ -69,9 +69,7 @@ def test_install_and_call_lambda_flask( FunctionName="function", InvocationType="RequestResponse", Payload=json.dumps(request_payload), - # LogType="Tail", #TODO ) - # log_output = base64.b64decode(response["LogResult"]).decode("utf-8") # Then assert_that(response, has_entries(StatusCode=HTTPStatus.OK)) @@ -82,7 +80,8 @@ def test_install_and_call_lambda_flask( has_entries(statusCode=HTTPStatus.OK, body=is_json_that(has_key("processedSuggestions"))), ) - # assert_that(log_output, contains_string("checking nhs_number")) + messages = lambda_logs() + assert_that(messages, has_item(contains_string("checking nhs_number"))) def test_install_and_call_flask_lambda_over_http( From 354cd5ba643f5cb0886977cc26cfcf35d5185df6 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:04:37 +0000 Subject: [PATCH 11/16] docker clean up --- tests/integration/lambda/test_app_running_as_lambda.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/lambda/test_app_running_as_lambda.py b/tests/integration/lambda/test_app_running_as_lambda.py index c717936eb..eab073dd5 100644 --- a/tests/integration/lambda/test_app_running_as_lambda.py +++ b/tests/integration/lambda/test_app_running_as_lambda.py @@ -80,6 +80,7 @@ def test_install_and_call_lambda_flask( # noqa: PLR0913 has_entries(statusCode=HTTPStatus.OK, body=is_json_that(has_key("processedSuggestions"))), ) + # assert logs from lambda container messages = lambda_logs() assert_that(messages, has_item(contains_string("checking nhs_number"))) From bff73dfd58db9af4ddd3a715c1495f01a47473ca Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:18:49 +0000 Subject: [PATCH 12/16] test - commit --- tests/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 640d97bb9..2037051c8 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -18,7 +18,7 @@ services: - "4567:8080" platform: linux/amd64 volumes: - - ../dist:/tmp:ro + - ../dist/lambda.zip:/tmp/lambda.zip:ro environment: - AWS_ACCESS_KEY_ID=dummy_key - AWS_SECRET_ACCESS_KEY=dummy_secret From c09c2df34b19517c9ffca9c1d2e9deed58809e79 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:38:57 +0000 Subject: [PATCH 13/16] test - commit --- tests/integration/lambda/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index b3a30ceb9..d2416e8e0 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -98,8 +98,8 @@ def lambda_runtime_url(request, lambda_zip): # noqa: ARG001 base_url = URL(f"http://{docker_ip}:{port}") docker_services.wait_until_responsive( - timeout=30.0, - pause=0.5, + timeout=60.0, + pause=1, check=lambda: is_responsive(base_url), ) From a9850a2cdcf4da8490bf4579460698322293053b Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 20 Feb 2026 20:00:28 +0000 Subject: [PATCH 14/16] reduced subprocess usage --- tests/integration/conftest.py | 9 +++--- tests/integration/lambda/conftest.py | 45 +++++++--------------------- 2 files changed, 16 insertions(+), 38 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7d2b0c732..6bdd20c90 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -13,6 +13,7 @@ from botocore.client import BaseClient from botocore.exceptions import ClientError from faker import Faker +from httpx import RequestError from yarl import URL from eligibility_signposting_api.model import eligibility_status @@ -82,12 +83,12 @@ def moto_server(request: pytest.FixtureRequest) -> URL: def is_responsive(url: URL) -> bool: try: - response = httpx.get(str(url), timeout=2.0) - except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.TimeoutException): + response = httpx.get(str(url)) + response.raise_for_status() + except RequestError: return False else: - # Use the constant instead of the raw number - return response.status_code < HTTP_SERVER_ERROR + return True @pytest.fixture(scope="session") diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index d2416e8e0..190abb154 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -1,7 +1,6 @@ import os import shutil import subprocess -from collections.abc import Callable from pathlib import Path import pytest @@ -142,38 +141,16 @@ def lambda_client(boto3_session: Session, lambda_runtime_url: URL) -> BaseClient return boto3_session.client("lambda", endpoint_url=str(lambda_runtime_url)) -@pytest.fixture -def lambda_logs(docker_services) -> Callable[[], list[str]]: - """Return a callable that fetches the latest lambda-api logs.""" - - def _get_messages() -> list[str]: - return get_lambda_logs(docker_services) - - return _get_messages - - -def get_lambda_logs(docker_services) -> list[str]: # noqa :ARG001 - """Fetch logs from the lambda-api container.""" - raw_docker = shutil.which("docker") - if not raw_docker: - return ["Error: Docker not found"] +def get_lambda_logs(docker_services) -> list[str]: + """ + Fetch logs from the lambda-api container using the internal pytest-docker executor. + This replaces manual subprocess calls and path resolution. + """ + try: + result = docker_services._docker_compose.execute("logs --no-color lambda-api") # noqa: SLF001 - docker_path = Path(raw_docker).resolve() - project_root = get_project_root() - compose_file = (project_root / "tests" / "docker-compose.yml").resolve() + output = result.decode("utf-8") if isinstance(result, bytes) else str(result) - result = subprocess.run( # noqa: S603 - [ - str(docker_path), - "compose", - "-f", - str(compose_file), - "logs", - "--no-color", - "lambda-api", - ], - capture_output=True, - text=True, - check=False, - ) - return [line.split("|", 1)[-1].strip() for line in result.stdout.splitlines()] + return [line.split("|", 1)[-1].strip() for line in output.splitlines()] + except Exception as e: # noqa: BLE001 + return [f"Error fetching logs: {e!s}"] From d348500275055e93a57e249737b51952f3097d12 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Sat, 21 Feb 2026 00:03:30 +0000 Subject: [PATCH 15/16] lint fix --- ...-compose.yml => docker-compose.lambda.yml} | 24 ++--- tests/docker-compose.moto.yml | 19 ++++ tests/integration/conftest.py | 38 ++++++-- tests/integration/lambda/conftest.py | 88 ++++++------------- 4 files changed, 85 insertions(+), 84 deletions(-) rename tests/{docker-compose.yml => docker-compose.lambda.yml} (77%) create mode 100644 tests/docker-compose.moto.yml diff --git a/tests/docker-compose.yml b/tests/docker-compose.lambda.yml similarity index 77% rename from tests/docker-compose.yml rename to tests/docker-compose.lambda.yml index 2037051c8..f3f191b7b 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.lambda.yml @@ -1,17 +1,8 @@ -services: - moto-server: - #used for s3, dynamodb, kinesis, secret manager - # lambda cannot be used, because its 3.11 (older) - image: motoserver/moto:latest - container_name: moto-server - ports: - - "4566:5000" - networks: - - test-network +include: + - docker-compose.moto.yml +services: lambda-api: - #used for lambda simulation - profiles: ["lambda-test"] image: public.ecr.aws/lambda/python:3.13 container_name: lambda-api ports: @@ -37,19 +28,22 @@ services: mkdir -p /var/task && python3 -m zipfile -e /tmp/lambda.zip /var/task && exec /usr/local/bin/aws-lambda-rie python3 -m awslambdaric eligibility_signposting_api.app.lambda_handler - depends_on: - - moto-server networks: - test-network + depends_on: + moto-server: + condition: service_healthy + api-gateway-mock: #used for api gateway simulation - profiles: ["lambda-test"] image: openresty/openresty:alpine ports: - "9123:9123" volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - lambda-api networks: - test-network diff --git a/tests/docker-compose.moto.yml b/tests/docker-compose.moto.yml new file mode 100644 index 000000000..e81121da7 --- /dev/null +++ b/tests/docker-compose.moto.yml @@ -0,0 +1,19 @@ +services: + moto-server: + #used for s3, dynamodb, kinesis, secret manager + # lambda cannot be used, because its 3.11 (older) + image: motoserver/moto:latest + container_name: moto-server + ports: + - "4566:5000" + networks: + - test-network + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/"] + interval: 1s + timeout: 1s + retries: 30 + +networks: + test-network: + driver: bridge diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6bdd20c90..ff93d9d2a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,6 +3,7 @@ import logging import os from collections.abc import Callable, Generator +from pathlib import Path from typing import Any import httpx @@ -53,11 +54,6 @@ HTTP_SERVER_ERROR = 500 -@pytest.fixture(scope="session") -def docker_compose_project_name(): - return "tests" - - @pytest.fixture(scope="session", autouse=True) def aws_credentials(): """Mocked AWS Credentials for moto.""" @@ -68,16 +64,42 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = AWS_REGION +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + root = Path(pytestconfig.rootpath) / "tests" + return [str(root / "docker-compose.moto.yml"), str(root / "docker-compose.lambda.yml")] + + +@pytest.fixture(scope="session") +def docker_compose_project_name(): + return "eligibility-integration" + + +@pytest.fixture(scope="session") +def docker_setup(): + return [] + + +@pytest.fixture(scope="session") +def docker_compose_services(): + return [] + + @pytest.fixture(scope="session") def moto_server(request: pytest.FixtureRequest) -> URL: docker_services = request.getfixturevalue("docker_services") - # This must match the container_name in docker-compose docker_ip = request.getfixturevalue("docker_ip") - # port_for maps the INTERNAL 5000 to whatever random EXTERNAL port was assigned + + docker_services._docker_compose.execute("up -d moto-server") # noqa : SLF001 + port = docker_services.port_for("moto-server", 5000) url = URL(f"http://{docker_ip}:{port}") - docker_services.wait_until_responsive(timeout=30.0, pause=0.2, check=lambda: is_responsive(url)) + docker_services.wait_until_responsive( + timeout=30.0, + pause=0.2, + check=lambda: is_responsive(url), + ) return url diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index 190abb154..dc34367dd 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -1,11 +1,12 @@ -import os import shutil import subprocess +from collections.abc import Callable from pathlib import Path import pytest from boto3 import Session from botocore.client import BaseClient +from pytest_docker import Services from yarl import URL from tests.integration.conftest import is_responsive @@ -52,87 +53,42 @@ def lambda_zip() -> Path: @pytest.fixture(scope="session") -def lambda_runtime_url(request, lambda_zip): # noqa: ARG001 - """ - Start the lambda simulation using docker compose. - """ +def lambda_runtime_url(request, lambda_zip: Path) -> URL: # noqa : ARG001 docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") - project_root = get_project_root() - compose_file = project_root / "tests/docker-compose.yml" - - # Activate the profile without using the --profile flag - env = os.environ.copy() - env["COMPOSE_PROFILES"] = "lambda-test" - - docker_path = shutil.which("docker") - if not docker_path: - pytest.fail("Docker executable not found in PATH") - - result = subprocess.run( # noqa: S603 - [ - docker_path, - "compose", - "-f", - str(compose_file), - "up", - "-d", - "--build", - "--force-recreate", - "lambda-api", - "api-gateway-mock", - ], - env=env, - capture_output=True, - text=True, - check=False, - ) - if result.returncode != 0: - pytest.fail( - f"Docker compose failed with code {result.returncode}.\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" - ) + docker_services._docker_compose.execute("up lambda-api") # noqa : SLF001 port = docker_services.port_for("lambda-api", 8080) base_url = URL(f"http://{docker_ip}:{port}") + # The RIE expects this path for invocations + health_url = base_url / "2015-03-31/functions/function/invocations" + docker_services.wait_until_responsive( timeout=60.0, - pause=1, - check=lambda: is_responsive(base_url), - ) - - yield base_url - - subprocess.run( # noqa: S603 - [docker_path, "compose", "-f", str(compose_file), "rm", "-f", "-s", "lambda-api", "api-gateway-mock"], - check=False, - env=env, - capture_output=True, - text=True, + pause=2, + check=lambda: is_responsive(health_url), ) + return base_url @pytest.fixture(scope="session") -def api_gateway_endpoint(request: pytest.FixtureRequest, lambda_runtime_url): # noqa: ARG001 - """ - Start and validate the API Gateway mock. - """ +def api_gateway_endpoint(request, lambda_runtime_url: URL) -> URL: # noqa: ARG001 docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") - port = docker_services.port_for("api-gateway-mock", 9123) - - base_url = URL(f"http://{docker_ip}:{port}") - health_url = URL(f"http://{docker_ip}:{port}/health") + docker_services._docker_compose.execute("up api-gateway-mock") # noqa: SLF001 + port = docker_services.port_for("api-gateway-mock", 9123) + url = URL(f"http://{docker_ip}:{port}") + health_url = url / "health" docker_services.wait_until_responsive( timeout=30.0, - pause=1.0, + pause=0.2, check=lambda: is_responsive(health_url), ) - - return base_url + return url @pytest.fixture(scope="session") @@ -154,3 +110,13 @@ def get_lambda_logs(docker_services) -> list[str]: return [line.split("|", 1)[-1].strip() for line in output.splitlines()] except Exception as e: # noqa: BLE001 return [f"Error fetching logs: {e!s}"] + + +@pytest.fixture +def lambda_logs(docker_services: Services) -> Callable[[], list[str]]: + """Fixture to provide access to container logs.""" + + def _get_messages() -> list[str]: + return get_lambda_logs(docker_services) + + return _get_messages From 96de4533c6e77017b69c53ec3824a4be6834ee33 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Sat, 21 Feb 2026 00:11:31 +0000 Subject: [PATCH 16/16] fix services trigger --- tests/integration/lambda/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/lambda/conftest.py b/tests/integration/lambda/conftest.py index dc34367dd..f77e1c5d1 100644 --- a/tests/integration/lambda/conftest.py +++ b/tests/integration/lambda/conftest.py @@ -57,7 +57,7 @@ def lambda_runtime_url(request, lambda_zip: Path) -> URL: # noqa : ARG001 docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") - docker_services._docker_compose.execute("up lambda-api") # noqa : SLF001 + docker_services._docker_compose.execute("up -d lambda-api") # noqa : SLF001 port = docker_services.port_for("lambda-api", 8080) base_url = URL(f"http://{docker_ip}:{port}") @@ -78,7 +78,7 @@ def api_gateway_endpoint(request, lambda_runtime_url: URL) -> URL: # noqa: ARG0 docker_services = request.getfixturevalue("docker_services") docker_ip = request.getfixturevalue("docker_ip") - docker_services._docker_compose.execute("up api-gateway-mock") # noqa: SLF001 + docker_services._docker_compose.execute("up -d api-gateway-mock") # noqa: SLF001 port = docker_services.port_for("api-gateway-mock", 9123) url = URL(f"http://{docker_ip}:{port}")