diff --git a/docker/dts/README.md b/docker/dts/README.md index 0ecd021..5e4bcfe 100644 --- a/docker/dts/README.md +++ b/docker/dts/README.md @@ -1,20 +1,26 @@ -The `docker-compose.yaml` file in this directory is primarily intended for testing the DTS services of the TESP API. +The `docker-compose.yaml` file in this directory is primarily intended for testing the DTS services of the TESP API. Smoke tests exercise **only the HTTP service**. To ensure proper execution of the tests, run `docker-compose.yaml` both in this directory and in the `/tesp-api` directory with: ``` docker compose up -d --build ``` - # DTS Example of data transfer server using HTTP, S3 and FTP. -Project uses Docker and deploy 4 containers: +### Current status + +- HTTP is the only DTS service implemented in this repository and used by smoke tests. +- S3 runs MinIO as an external service for manual testing. +- FTP runs the upstream `ftpserver` container and uses S3 (MinIO) as its storage backend. + +`docker-compose.yaml` deploys 4 containers: - s3 - ftp - http - clients -The `clients` container contains clients for the used protocols. So you doesn't need to install the clients on you local computer to test it. +The `clients` container contains clients for the used protocols, so you do not need to install the clients on your local computer to test it. + ## Deploy @@ -24,11 +30,20 @@ The `clients` container contains clients for the used protocols. So you doesn't ### HTTP -Upload -`curl -i -X POST -F "file=@up-file.txt;filename=file.txt" service-http:5000/upload` +The HTTP service provides multiple routes: -Download -`curl -X GET -o down-file.txt service-http:5000/download/file.txt` +**Upload** (saves to `/data/uploaded_data/`) +`curl -i -X POST -F "file=@up-file.txt" localhost:5000/upload` + +**Download (legacy route)** +`curl -X GET -o down-file.txt localhost:5000/download/file.txt` + +**Browse test_data (smoke test compatible)** +`curl localhost:5000/test_data/test.txt` +`curl localhost:5000/test_data/input_dir/` + +**List all files** +`curl localhost:5000/list` ### S3 Create bucket @@ -89,19 +104,30 @@ lftp -p 2121 -e "put /tmp/sample.jpg; get $(basename $SAMPLE_DATE) -o /tmp/sampl ##### Upload -POST + `http://localhost:5000/upload` while payload is sent as 'file' HTTP body parameter. +POST to `http://localhost:5000/upload` with file as multipart form data: +```bash +curl -F "file=@/tmp/qwerty" http://localhost:5000/upload ``` + +With subdirectory: +```bash curl -F "file=@/tmp/qwerty" http://localhost:5000/upload/foo/bar ``` - ##### Download -GET + `http://localhost:5000/download` while payload is sent as 'file' HTTP body parameter. -``` -wget http://localhost:5000/download/foo/bar/qwerty -``` +Legacy route: +```bash +wget http://localhost:5000/download/foo/bar/qwerty +``` +Smoke test compatible route (serves from mounted test_data): +```bash +curl http://localhost:5000/test_data/test.txt +``` ##### List of all data -GET + `http://localhost:5000/list` + +```bash +curl http://localhost:5000/list +``` diff --git a/docker/dts/docker-compose.yaml b/docker/dts/docker-compose.yaml index 5434d6d..f72438b 100644 --- a/docker/dts/docker-compose.yaml +++ b/docker/dts/docker-compose.yaml @@ -31,6 +31,7 @@ services: - "5000:5000" volumes: - ../../tests/test_data:/data + - ../../tests/uploaded_data:/data/uploaded_data tests-clients: build: diff --git a/docker/dts/http/main_flask.py b/docker/dts/http/main_flask.py index df94809..e598e64 100644 --- a/docker/dts/http/main_flask.py +++ b/docker/dts/http/main_flask.py @@ -1,11 +1,12 @@ import os import subprocess -from flask import Flask, request, send_file, jsonify +from flask import Flask, request, send_file, send_from_directory, jsonify, abort from werkzeug.utils import secure_filename from logger_config import logger app = Flask(__name__) DATA_DIR = '/data' +UPLOAD_DIR = '/data/uploaded_data' @app.route('/upload', defaults={'target_path': ''}, methods=['POST'], strict_slashes=False) @@ -18,16 +19,29 @@ def upload_file(target_path): if file.filename == '': return 'No selected file', 400 - target_dir = os.path.join(DATA_DIR, target_path) + # Support nested paths in filename (e.g., "subdir/file.txt") + raw_filename = file.filename + secure_parts = [secure_filename(part) for part in raw_filename.split('/') if part] + + if target_path: + target_dir = os.path.join(UPLOAD_DIR, target_path) + else: + target_dir = UPLOAD_DIR + + if len(secure_parts) > 1: + target_dir = os.path.join(target_dir, *secure_parts[:-1]) + os.makedirs(target_dir, exist_ok=True) - - filename = secure_filename(file.filename) - file.save(os.path.join(target_dir, filename)) - return 'File uploaded successfully.', 200 + filename = secure_parts[-1] if secure_parts else secure_filename(raw_filename) + save_path = os.path.join(target_dir, filename) + file.save(save_path) + + return jsonify({"status": "ok", "saved_as": save_path}), 200 @app.route('/download/', methods=['GET']) def download_file(file_path): + """Legacy download route for backwards compatibility.""" logger.info(f"path { file_path }") target_file = os.path.join(DATA_DIR, file_path) @@ -37,6 +51,31 @@ def download_file(file_path): return 'File not found.', 404 +@app.route('/test_data/', methods=['GET']) +@app.route('/test_data/', methods=['GET']) +def browse_test_data(subpath=''): + """Serve files and directory listings from /data (mounted as test_data).""" + full_path = os.path.join(DATA_DIR, subpath) + + if os.path.isdir(full_path): + # Directory listing + items = os.listdir(full_path) + links = [] + for item in items: + item_path = os.path.join('test_data', subpath, item) if subpath else os.path.join('test_data', item) + if os.path.isdir(os.path.join(full_path, item)): + item_path += '/' + links.append(f"{item}
") + return "" + "\n".join(links) + "" + + elif os.path.isfile(full_path): + # Serve file + return send_from_directory(os.path.dirname(full_path), os.path.basename(full_path)) + + else: + abort(404, description=f"{subpath} not found") + + @app.route('/list', methods=['GET']) def list_data(): try: @@ -55,4 +94,6 @@ def list_data(): if __name__ == '__main__': + os.makedirs(UPLOAD_DIR, exist_ok=True) app.run(host='0.0.0.0', debug=True, port=5000) + diff --git a/tests/README.md b/tests/README.md index b2c706f..40d8a54 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,10 +1,78 @@ # Testing -To test functionality of TESP API, running `docker-compose.yaml` with: + +Integration tests for TESP-API. All tests run against a live TESP-API instance at `http://localhost:8080`. + +## Prerequisites + +Start TESP-API + MongoDB + Pulsar from the project root: + +```bash +docker compose --profile pulsar up -d --build ``` -docker compose up -d --build + +## Running Tests + +### 1. Start the upload server + +For I/O tests (`test_inputs`, `test_dir_input`, `test_volumes`, `test_envs`), the test fixtures download/upload files via HTTP. Start the local file server: + +```bash +python3 tests/upload_server.py ``` -is necessary both in `/tesp-api` and `/tesp-api/docker/dts` -Tests themselves can be run with: + +This serves `tests/test_data/` on port 5000 and accepts uploads to `tests/uploaded_data/`. + +### 2. Run smoke tests + +```bash +python3 -m pytest tests/smoke_tests.py -v ``` -python3 -m pytest smoke_tests.py + +### Load/stress test (100 concurrent tasks) + +```bash +python3 -m pytest tests/load_stress_test.py -v ``` + +## Platform Notes + +The test fixtures use `http://172.17.0.1:5000` to reach the host from inside containers. This works on **Linux** (Docker bridge gateway IP). + + +## Test Fixtures + +All task JSON fixtures are in `test_jsons/`. Key files: + +| Fixture | Description | Requires upload_server | +|---|---|---| +| `state_true.json` | Simple task that should succeed | No | +| `state_false.json` | Task designed to fail | No | +| `cancel.json` | Long-running task (`sleep infinity`) for cancellation | No | +| `multi_true.json` | Three sequential executors | No | +| `inputs.json` | HTTP download + inline content | **Yes** | +| `dir-io.json` | Directory input/output via HTTP | **Yes** | +| `volumes.json` | Shared volume between executors | **Yes** | +| `envs.json` | Environment variables with output upload | **Yes** | +| `workdir.json` | Working directory test | No | + +## DTS (Data Transfer Services) + +The `docker/dts/` directory contains an **alternative** test infrastructure with MinIO (S3), FTP, and HTTP services running in containers. At the moment, only HTTP service is compatible with the smoke tests. + +### Using DTS instead of upload_server.py + +```bash +# Start DTS (from project root) +cd docker/dts && docker compose up -d --build && cd ../.. + +# Run tests (DTS serves on the same port 5000) +python3 -m pytest tests/smoke_tests.py -v +``` + +DTS mounts `tests/test_data/` and `tests/uploaded_data/` so the same fixture URLs work. + +## Cleanup + +The test suite automatically cleans up: +- `uploaded_data/` directory created by upload tests +- Leftover Ubuntu containers from failed tests