Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .github/workflows/aws-replicator.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
name: LocalStack AWS Replicator Extension Tests
name: LocalStack AWS Proxy Extension Tests

on:
push:
paths:
- aws-replicator/**
- aws-proxy/**
branches:
- main
pull_request:
paths:
- .github/workflows/aws-replicator.yml
- aws-replicator/**
- .github/workflows/aws-proxy.yml
- aws-proxy/**
workflow_dispatch:

jobs:
Expand Down Expand Up @@ -49,7 +49,7 @@ jobs:
# build and install extension
localstack extensions init
(
cd aws-replicator
cd aws-proxy
make install
. .venv/bin/activate
pip install --upgrade --pre localstack localstack-ext
Expand All @@ -68,7 +68,7 @@ jobs:

- name: Run linter
run: |
cd aws-replicator
cd aws-proxy
(. .venv/bin/activate; pip install --upgrade --pre localstack localstack-ext)
make lint

Expand All @@ -78,7 +78,7 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.TEST_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_AWS_SECRET_ACCESS_KEY }}
run: |
cd aws-replicator
cd aws-proxy
make test

- name: Deploy and test sample app
Expand All @@ -88,7 +88,7 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_AWS_SECRET_ACCESS_KEY }}
LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
run: |
cd aws-replicator/example
cd aws-proxy/example
make test

- name: Print LocalStack logs
Expand Down
2 changes: 1 addition & 1 deletion CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
### Extensions ###
######################

/aws-replicator/ @whummer
/aws-proxy/ @whummer @nik-localstack
/diagnosis-viewer/ @thrau @silv-io
/hello-world/ @lukqw @thrau
/http-bin/ @thrau @dominikschubert
Expand Down
20 changes: 10 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,16 @@ $ localstack extensions install "git+https://github.com/localstack/localstack-ex
Here is the current list of extensions developed by the LocalStack team and their support status.
You can install the respective extension by calling `localstack install <Install name>`.

| Extension | Install name | Version | Support status |
| --------- | ------------ | ------- | -------------- |
| [AWS replicator](https://github.com/localstack/localstack-extensions/tree/main/aws-replicator) | localstack-extension-aws-replicator | 0.1.7 | Experimental |
| [Diagnosis Viewer](https://github.com/localstack/localstack-extensions/tree/main/diagnosis-viewer) | localstack-extension-diagnosis-viewer | 0.1.0 | Stable |
| [Hello World](https://github.com/localstack/localstack-extensions/tree/main/hello-world) | localstack-extension-hello-world | 0.1.0 | Stable |
| [httpbin](https://github.com/localstack/localstack-extensions/tree/main/httpbin) | localstack-extension-httpbin | 0.1.0 | Stable |
| [MailHog](https://github.com/localstack/localstack-extensions/tree/main/mailhog) | localstack-extension-mailhog | 0.1.0 | Stable |
| [Miniflare](https://github.com/localstack/localstack-extensions/tree/main/miniflare) | localstack-extension-miniflare | 0.1.0 | Experimental |
| [Stripe](https://github.com/localstack/localstack-extensions/tree/main/stripe) | localstack-extension-stripe | 0.2.0 | Stable |
| [Terraform Init](https://github.com/localstack/localstack-extensions/tree/main/terraform-init) | localstack-extension-terraform-init | 0.2.0 | Experimental |
| Extension | Install name | Version | Support status |
|----------------------------------------------------------------------------------------------------| ------------ |---------| -------------- |
| [AWS Proxy](https://github.com/localstack/localstack-extensions/tree/main/aws-proxy) | localstack-extension-aws-proxy | 0.2.0 | Experimental |
| [Diagnosis Viewer](https://github.com/localstack/localstack-extensions/tree/main/diagnosis-viewer) | localstack-extension-diagnosis-viewer | 0.1.0 | Stable |
| [Hello World](https://github.com/localstack/localstack-extensions/tree/main/hello-world) | localstack-extension-hello-world | 0.1.0 | Stable |
| [httpbin](https://github.com/localstack/localstack-extensions/tree/main/httpbin) | localstack-extension-httpbin | 0.1.0 | Stable |
| [MailHog](https://github.com/localstack/localstack-extensions/tree/main/mailhog) | localstack-extension-mailhog | 0.1.0 | Stable |
| [Miniflare](https://github.com/localstack/localstack-extensions/tree/main/miniflare) | localstack-extension-miniflare | 0.1.0 | Experimental |
| [Stripe](https://github.com/localstack/localstack-extensions/tree/main/stripe) | localstack-extension-stripe | 0.2.0 | Stable |
| [Terraform Init](https://github.com/localstack/localstack-extensions/tree/main/terraform-init) | localstack-extension-terraform-init | 0.2.0 | Experimental |


## Developing Extensions
Expand Down
File renamed without changes.
3 changes: 3 additions & 0 deletions aws-proxy/MANIFEST.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
recursive-include aws_proxy *.html
recursive-include aws_proxy *.js
recursive-include aws_proxy *.png
6 changes: 3 additions & 3 deletions aws-replicator/Makefile → aws-proxy/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,12 @@ dist: venv

build: ## Build the extension
mkdir -p build
cp -r setup.py setup.cfg README.md aws_replicator build/
cp -r setup.py setup.cfg README.md aws_proxy build/
(cd build && python setup.py sdist)

enable: $(wildcard ./build/dist/localstack_extension_aws_replicator-*.tar.gz) ## Enable the extension in LocalStack
enable: $(wildcard ./build/dist/localstack_extension_aws_proxy-*.tar.gz) ## Enable the extension in LocalStack
$(VENV_RUN); \
pip uninstall --yes localstack-extension-aws-replicator; \
pip uninstall --yes localstack-extension-aws-proxy; \
localstack extensions -v install file://$?

publish: clean-dist venv dist
Expand Down
27 changes: 14 additions & 13 deletions aws-replicator/README.md → aws-proxy/README.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
AWS Cloud Proxy Extension (experimental)
========================================
[![Install LocalStack Extension](https://localstack.cloud/gh/extension-badge.svg)](https://app.localstack.cloud/extensions/remote?url=git+https://github.com/localstack/localstack-extensions/#egg=localstack-extension-aws-replicator&subdirectory=aws-replicator)
[![Install LocalStack Extension](https://localstack.cloud/gh/extension-badge.svg)](https://app.localstack.cloud/extensions/remote?url=git+https://github.com/localstack/localstack-extensions/#egg=localstack-extension-aws-proxy&subdirectory=aws-proxy)

A LocalStack extension to proxy and integrate AWS resources into your local machine.
This enables one flavor of "hybrid" or "remocal" setups where you can easily bridge the gap between LocalStack (local resources) and remote AWS (resources in the real cloud).

⚠️ Please note that this extension is experimental and still under active development.

⚠️ Note: Given that the scope of this extension has recently changed (see [below](#resource-replicator-cli-deprecated)), it may get renamed from `aws-replicator` to `cloud-proxy` in an upcoming release.
⚠️ Note: Given that the scope of this extension has recently changed (see [below](#resource-replicator-cli-deprecated)) - it has been renamed from `aws-replicator` to `aws-proxy`.

## Prerequisites

Expand All @@ -33,11 +33,11 @@ For example, in order to forward all API calls for DynamoDB/S3/Cognito to real A
```
$ localstack start -d
```
2. Enable LocalStack AWS replicator from the Web Application Extension Library
2. Enable LocalStack AWS Proxy from the Web Application Extension Library
3. After installation restart Localstack
4. Install the AWS replicator CLI package
4. Install the AWS Proxy CLI package
```
$ pip install localstack-extension-aws-replicator
$ pip install localstack-extension-aws-proxy
```
5. Configure real cloud account credentials in a new terminal session to allow access
```
Expand All @@ -53,12 +53,12 @@ $ localstack aws proxy -s dynamodb,s3,cognito-idp

1. Start Localstack with extra CORS
```
EXTRA_CORS_ALLOWED_ORIGINS=https://aws-replicator.localhost.localstack.cloud:4566 localstack start -d
EXTRA_CORS_ALLOWED_ORIGINS=https://aws-proxy.localhost.localstack.cloud:4566 localstack start -d
```

2. Enable Localstack AWS replicator from the Web Application Extension Library
2. Enable Localstack AWS Proxy from the Web Application Extension Library

3. Once the extension is installed, it will expose a small configuration endpoint in your LocalStack container under the following endpoint: http://localhost:4566/_localstack/aws-replicator/index.html .
3. Once the extension is installed, it will expose a small configuration endpoint in your LocalStack container under the following endpoint: http://localhost:4566/_localstack/aws-proxy/index.html .

4. Use this Web UI to define the proxy configuration (in YAML syntax), as well as the AWS credentials (AWS access key ID, secret access key, and optionally session token) and save configuration. The proxy should report enabled state and on the host a proxy container should spawn.

Expand Down Expand Up @@ -111,22 +111,23 @@ A more comprehensive sample, involving local Lambda functions combined with remo
### Configuration

In addition to the proxy services configuration shown above, the following configs can be used to customize the behavior of the extension itself (simply pass them as environment variables to the main LocalStack container):
* `REPLICATOR_CLEANUP_PROXY_CONTAINERS`: whether to clean up (remove) the proxy Docker containers once they shut down (default `1`). Can be set to `0` to help debug issues, e.g., if a proxy container starts up and exits immediately.
* `REPLICATOR_LOCALSTACK_HOST`: the target host to use when the proxy container connects to the LocalStack main container (automatically determined by default)
* `REPLICATOR_PROXY_DOCKER_FLAGS`: additional flags that should be passed when creating the proxy Docker containers
* `PROXY_CLEANUP_CONTAINERS`: whether to clean up (remove) the proxy Docker containers once they shut down (default `1`). Can be set to `0` to help debug issues, e.g., if a proxy container starts up and exits immediately.
* `PROXY_LOCALSTACK_HOST`: the target host to use when the proxy container connects to the LocalStack main container (automatically determined by default)
* `PROXY_DOCKER_FLAGS`: additional flags that should be passed when creating the proxy Docker containers

**Note:** Due to some recent changes in the core framework, make sure to start up your LocalStack container with the `GATEWAY_SERVER=hypercorn` configuration enabled, for backwards compatibility. This will be fixed in an upcoming release.

## Resource Replicator CLI (deprecated)

Note: Previous versions of this extension also offered a "replicate" mode to copy/clone (rather than proxy) resources from an AWS account into the local instance.
This functionality has been removed from this extension, and is now being migrated to a new extension (more details following soon).
This functionality has been removed from this extension, and is now available directly in the LocalStack Pro image (see [here](https://docs.localstack.cloud/aws/tooling/aws-replicator)).

If you wish to access the deprecated instructions, they can be found [here](https://github.com/localstack/localstack-extensions/blob/fe0c97e8a9d94f72c80358493e51ce6c1da535dc/aws-replicator/README.md#resource-replicator-cli).

## Change Log

* `0.1.25`: Fix dynamodb proxying for read-only mode.
* `0.2.0`: Rename extension from `localstack-extension-aws-replicator` to `localstack-extension-aws-proxy`
* `0.1.25`: Fix dynamodb proxying for read-only mode
* `0.1.24`: Fix healthcheck probe for proxy container
* `0.1.23`: Fix unpinned React.js dependencies preventing webui from loading
* `0.1.22`: Fix auth-related imports that prevent the AWS proxy from starting
Expand Down
1 change: 1 addition & 0 deletions aws-proxy/aws_proxy/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
name = "aws-proxy"
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
from localstack.config import external_service_url
from localstack.constants import AWS_REGION_US_EAST_1, DOCKER_IMAGE_NAME_PRO, LOCALHOST_HOSTNAME
from localstack.http import Request
from localstack.pro.core.bootstrap.licensingv2 import (
ENV_LOCALSTACK_API_KEY,
ENV_LOCALSTACK_AUTH_TOKEN,
)
from localstack.utils.aws.aws_responses import requests_response
from localstack.utils.bootstrap import setup_logging
from localstack.utils.collections import select_attributes
Expand All @@ -29,35 +33,23 @@
from localstack.utils.strings import short_uid, to_bytes, to_str, truncate
from requests import Response

from aws_replicator import config as repl_config
from aws_replicator.client.utils import truncate_content
from aws_replicator.config import HANDLER_PATH_PROXIES
from aws_replicator.shared.constants import HEADER_HOST_ORIGINAL
from aws_replicator.shared.models import AddProxyRequest, ProxyConfig
from aws_proxy import config as repl_config
from aws_proxy.client.utils import truncate_content
from aws_proxy.config import HANDLER_PATH_PROXIES
from aws_proxy.shared.constants import HEADER_HOST_ORIGINAL
from aws_proxy.shared.models import AddProxyRequest, ProxyConfig

from .http2_server import run_server

try:
from localstack.pro.core.bootstrap.licensingv2 import (
ENV_LOCALSTACK_API_KEY,
ENV_LOCALSTACK_AUTH_TOKEN,
)
except ImportError:
# TODO remove once we don't need compatibility with <3.6 anymore
from localstack_ext.bootstrap.licensingv2 import (
ENV_LOCALSTACK_API_KEY,
ENV_LOCALSTACK_AUTH_TOKEN,
)

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.INFO)
if localstack_config.DEBUG:
LOG.setLevel(logging.DEBUG)

# TODO make configurable
CLI_PIP_PACKAGE = "localstack-extension-aws-replicator"
CLI_PIP_PACKAGE = "localstack-extension-aws-proxy"
# note: enable the line below temporarily for testing:
# CLI_PIP_PACKAGE = "git+https://github.com/localstack/localstack-extensions/@branch#egg=localstack-extension-aws-replicator&subdirectory=aws-replicator"
# CLI_PIP_PACKAGE = "git+https://github.com/localstack/localstack-extensions/@branch#egg=localstack-extension-aws-proxy&subdirectory=aws-proxy"

CONTAINER_NAME_PREFIX = "ls-aws-proxy-"
CONTAINER_CONFIG_FILE = "/tmp/ls.aws.proxy.yml"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,50 +4,31 @@
import click
import yaml
from localstack.cli import LocalstackCli, LocalstackCliPlugin, console
from localstack.logging.setup import setup_logging
from localstack.pro.core.cli.aws import aws
from localstack.pro.core.config import is_auth_token_configured
from localstack.utils.files import load_file

from aws_replicator.shared.models import ProxyConfig, ProxyServiceConfig
from aws_proxy.shared.models import ProxyConfig, ProxyServiceConfig

try:
from localstack.pro.core.bootstrap.auth import get_platform_auth_headers
from localstack.pro.core.cli.aws import aws
from localstack.pro.core.config import is_auth_token_configured
except ImportError:
# Only support anything over version 3.6
from localstack.pro.core.bootstrap.auth import get_auth_headers as get_platform_auth_headers
from localstack.pro.core.cli.aws import aws
from localstack.pro.core.config import is_api_key_configured as is_auth_token_configured


class AwsReplicatorPlugin(LocalstackCliPlugin):
name = "aws-replicator"
class AwsProxyPlugin(LocalstackCliPlugin):
name = "aws-proxy"

def should_load(self) -> bool:
return _is_logged_in() or is_auth_token_configured()
return is_auth_token_configured()

def attach(self, cli: LocalstackCli) -> None:
group: click.Group = cli.group
if not group.get_command(ctx=None, cmd_name="aws"):
group.add_command(aws)
aws.add_command(cmd_aws_proxy)
aws.add_command(cmd_aws_replicate)


# TODO: remove over time as we're phasing out the `login` command
def _is_logged_in() -> bool:
try:
get_platform_auth_headers()
return True
except Exception:
return False


@click.command(name="proxy", help="Start up an authentication proxy against real AWS")
@click.option(
"-s",
"--services",
help="Comma-delimited list of services to replicate (e.g., sqs,s3)",
help="Comma-delimited list of services to proxy (e.g., sqs,s3)",
required=False,
)
@click.option(
Expand All @@ -74,7 +55,7 @@ def _is_logged_in() -> bool:
required=False,
)
def cmd_aws_proxy(services: str, config: str, container: bool, port: int, host: str):
from aws_replicator.client.auth_proxy import start_aws_auth_proxy_in_container
from aws_proxy.client.auth_proxy import start_aws_auth_proxy_in_container

config_json: ProxyConfig = {"services": {}}
if config:
Expand All @@ -90,7 +71,7 @@ def cmd_aws_proxy(services: str, config: str, container: bool, port: int, host:
return start_aws_auth_proxy_in_container(config_json)

# note: deferring the import here, to avoid import errors in CLI context
from aws_replicator.client.auth_proxy import start_aws_auth_proxy
from aws_proxy.client.auth_proxy import start_aws_auth_proxy

proxy = start_aws_auth_proxy(config_json, port=port)
proxy.join()
Expand All @@ -99,20 +80,5 @@ def cmd_aws_proxy(services: str, config: str, container: bool, port: int, host:
sys.exit(1)


@click.command(name="replicate", help="Replicate the state of an AWS account into LocalStack")
@click.option(
"-s",
"--services",
help="Comma-delimited list of services to replicate (e.g., sqs,s3)",
required=True,
)
def cmd_aws_replicate(services: str):
from aws_replicator.client.replicate import replicate_state_into_local

setup_logging()
services = _split_string(services)
replicate_state_into_local(services)


def _split_string(string):
return [s.strip().lower() for s in re.split(r"[\s,]+", string) if s.strip()]
12 changes: 12 additions & 0 deletions aws-proxy/aws_proxy/client/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from typing import Union

from localstack.utils.functions import run_safe
from localstack.utils.strings import to_str, truncate


# TODO: add to common utils
def truncate_content(content: Union[str, bytes], max_length: int = None):
max_length = max_length or 100
if isinstance(content, bytes):
content = run_safe(lambda: to_str(content)) or content
return truncate(content, max_length=max_length)
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@
from localstack.constants import INTERNAL_RESOURCE_PATH

# handler path within the internal /_localstack endpoint
HANDLER_PATH_REPLICATE = f"{INTERNAL_RESOURCE_PATH}/aws/replicate"
HANDLER_PATH_PROXY = f"{INTERNAL_RESOURCE_PATH}/aws/proxy"
HANDLER_PATH_PROXIES = f"{INTERNAL_RESOURCE_PATH}/aws/proxies"

# whether to clean up proxy containers (set to "0" to investigate startup issues)
CLEANUP_PROXY_CONTAINERS = is_env_not_false("REPLICATOR_CLEANUP_PROXY_CONTAINERS")
CLEANUP_PROXY_CONTAINERS = is_env_not_false("PROXY_CLEANUP_CONTAINERS")

# additional Docker flags to pass to the proxy containers
PROXY_DOCKER_FLAGS = (os.getenv("REPLICATOR_PROXY_DOCKER_FLAGS") or "").strip()
PROXY_DOCKER_FLAGS = (os.getenv("PROXY_DOCKER_FLAGS") or "").strip()

# LS hostname to use for proxy Docker container to register itself at the main container
PROXY_LOCALSTACK_HOST = (os.getenv("REPLICATOR_LOCALSTACK_HOST") or "").strip()
PROXY_LOCALSTACK_HOST = (os.getenv("PROXY_LOCALSTACK_HOST") or "").strip()
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
except ImportError:
from localstack.constants import TEST_AWS_ACCESS_KEY_ID

from aws_replicator.shared.constants import HEADER_HOST_ORIGINAL
from aws_replicator.shared.models import ProxyInstance, ProxyServiceConfig
from aws_proxy.shared.constants import HEADER_HOST_ORIGINAL
from aws_proxy.shared.models import ProxyInstance, ProxyServiceConfig

LOG = logging.getLogger(__name__)

Expand Down
Loading