From 09206c560291a37c3a11a49387cf9899ea77b449 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 29 Jul 2026 16:51:15 +0200 Subject: [PATCH 1/5] fix(aws-lambda): Increase shutdown timeout for lambda test infrastructure & added temporary workflow that runs aws_lambda tests in parallel Refs: #6911 --- .github/workflows/aws-lambda-flake-check.yml | 41 +++++++++++++++++++ .../aws_lambda/test_aws_lambda.py | 10 ++--- 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/aws-lambda-flake-check.yml diff --git a/.github/workflows/aws-lambda-flake-check.yml b/.github/workflows/aws-lambda-flake-check.yml new file mode 100644 index 0000000000..f1fff7a75a --- /dev/null +++ b/.github/workflows/aws-lambda-flake-check.yml @@ -0,0 +1,41 @@ +name: AWS Lambda Flake Check + +on: + pull_request: + push: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + aws-lambda-flake: + name: aws_lambda py${{ matrix.python-version }} #${{ matrix.attempt }} + timeout-minutes: 30 + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.11", "3.13"] + attempt: [1, 2, 3, 4, 5] + services: + docker: + image: docker:dind + options: --privileged + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + - name: Setup Test Env + run: uv sync + - name: Test aws_lambda (attempt ${{ matrix.attempt }}) + run: | + set -x + ./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda" diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 01ca19bd54..c9ecfc06e4 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -14,6 +14,7 @@ DOCKER_NETWORK_NAME = "lambda-test-network" SAM_TEMPLATE_FILE = "sam.template.yaml" +SAM_SHUTDOWN_TIMEOUT = 60 @pytest.fixture(scope="session", autouse=True) @@ -78,13 +79,12 @@ def before_test(): finally: print("[test_environment fixture] Tearing down AWS Lambda test infrastructure") - process.terminate() - process.wait(timeout=10) # Give it time to shut down gracefully - - # Force kill if still running - if process.poll() is None: + try: + process.wait(timeout=SAM_SHUTDOWN_TIMEOUT) + except subprocess.TimeoutExpired: process.kill() + process.wait() @pytest.fixture(autouse=True) From 3b203d23583ed187b805939700fb630e5a2c087b Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Thu, 30 Jul 2026 11:02:55 +0200 Subject: [PATCH 2/5] fix(aws-lambda): change container startup to lazy - each container now will be initiated on first invocation Refs: #6937 --- tests/integrations/aws_lambda/test_aws_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index c9ecfc06e4..3c7b793877 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -55,7 +55,7 @@ def test_environment(): "--template", SAM_TEMPLATE_FILE, "--warm-containers", - "EAGER", + "LAZY", # Start each Docker container on its function's first invocation "--docker-network", DOCKER_NETWORK_NAME, ], From 67fb31eb5f9d2158c49579bfdb1112b81aa3f4ef Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Thu, 30 Jul 2026 13:30:06 +0200 Subject: [PATCH 3/5] fix(aws-lambda): finalize SAM teardown flake fix Drop the temporary flake-check workflow, restore a 10s shutdown budget (local teardown averages ~7s), and escalate to kill on timeout so an uncaught TimeoutExpired cannot fail the suite. --- .github/workflows/aws-lambda-flake-check.yml | 41 ------------------- .../aws_lambda/test_aws_lambda.py | 4 +- 2 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 .github/workflows/aws-lambda-flake-check.yml diff --git a/.github/workflows/aws-lambda-flake-check.yml b/.github/workflows/aws-lambda-flake-check.yml deleted file mode 100644 index f1fff7a75a..0000000000 --- a/.github/workflows/aws-lambda-flake-check.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: AWS Lambda Flake Check - -on: - pull_request: - push: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - aws-lambda-flake: - name: aws_lambda py${{ matrix.python-version }} #${{ matrix.attempt }} - timeout-minutes: 30 - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.11", "3.13"] - attempt: [1, 2, 3, 4, 5] - services: - docker: - image: docker:dind - options: --privileged - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: false - - name: Setup Test Env - run: uv sync - - name: Test aws_lambda (attempt ${{ matrix.attempt }}) - run: | - set -x - ./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda" diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 3c7b793877..d365742c86 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -14,7 +14,8 @@ DOCKER_NETWORK_NAME = "lambda-test-network" SAM_TEMPLATE_FILE = "sam.template.yaml" -SAM_SHUTDOWN_TIMEOUT = 60 +# Teardown is typically ~7s; escalate with kill if SAM exceeds this. +SAM_SHUTDOWN_TIMEOUT = 10 @pytest.fixture(scope="session", autouse=True) @@ -83,6 +84,7 @@ def before_test(): try: process.wait(timeout=SAM_SHUTDOWN_TIMEOUT) except subprocess.TimeoutExpired: + # Don't fail the suite if SAM is slow to stop warm containers. process.kill() process.wait() From 0771f043a6aebc106a6efc7e5cf357f8e04be050 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Thu, 30 Jul 2026 13:35:05 +0200 Subject: [PATCH 4/5] fix(aws-lambda): one last check with temporary workflow --- .github/workflows/aws-lambda-flake-check.yml | 41 +++++++++++++++++++ .../aws_lambda/test_aws_lambda.py | 3 +- 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/aws-lambda-flake-check.yml diff --git a/.github/workflows/aws-lambda-flake-check.yml b/.github/workflows/aws-lambda-flake-check.yml new file mode 100644 index 0000000000..f1fff7a75a --- /dev/null +++ b/.github/workflows/aws-lambda-flake-check.yml @@ -0,0 +1,41 @@ +name: AWS Lambda Flake Check + +on: + pull_request: + push: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + aws-lambda-flake: + name: aws_lambda py${{ matrix.python-version }} #${{ matrix.attempt }} + timeout-minutes: 30 + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.11", "3.13"] + attempt: [1, 2, 3, 4, 5] + services: + docker: + image: docker:dind + options: --privileged + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + - name: Setup Test Env + run: uv sync + - name: Test aws_lambda (attempt ${{ matrix.attempt }}) + run: | + set -x + ./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda" diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index d365742c86..0b203af106 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -14,7 +14,6 @@ DOCKER_NETWORK_NAME = "lambda-test-network" SAM_TEMPLATE_FILE = "sam.template.yaml" -# Teardown is typically ~7s; escalate with kill if SAM exceeds this. SAM_SHUTDOWN_TIMEOUT = 10 @@ -82,9 +81,9 @@ def before_test(): print("[test_environment fixture] Tearing down AWS Lambda test infrastructure") process.terminate() try: + # Teardown is typically ~7s; escalate with kill if SAM exceeds this. process.wait(timeout=SAM_SHUTDOWN_TIMEOUT) except subprocess.TimeoutExpired: - # Don't fail the suite if SAM is slow to stop warm containers. process.kill() process.wait() From 87862c3bc6181fd5ecf2dcff13dbb025b3f3a20a Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Thu, 30 Jul 2026 13:52:16 +0200 Subject: [PATCH 5/5] chore(aws-lambda): remove temporary aws-flake workflow --- .github/workflows/aws-lambda-flake-check.yml | 41 -------------------- 1 file changed, 41 deletions(-) delete mode 100644 .github/workflows/aws-lambda-flake-check.yml diff --git a/.github/workflows/aws-lambda-flake-check.yml b/.github/workflows/aws-lambda-flake-check.yml deleted file mode 100644 index f1fff7a75a..0000000000 --- a/.github/workflows/aws-lambda-flake-check.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: AWS Lambda Flake Check - -on: - pull_request: - push: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - aws-lambda-flake: - name: aws_lambda py${{ matrix.python-version }} #${{ matrix.attempt }} - timeout-minutes: 30 - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.11", "3.13"] - attempt: [1, 2, 3, 4, 5] - services: - docker: - image: docker:dind - options: --privileged - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: false - - name: Setup Test Env - run: uv sync - - name: Test aws_lambda (attempt ${{ matrix.attempt }}) - run: | - set -x - ./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda"