diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index a2eab4eae..2e39c1800 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -55,12 +55,16 @@ + os.environ.get("CONTAINER_ENGINE_EXECUTABLE", "docker").upper() + "with GPU capabilites : " + os.environ.get("GPU_DEVICE", "nvidia.com/gpu=all") + + " network_disabled for the competition container is set to " + + os.environ.get("COMPETITION_CONTAINER_NETWORK_DISABLED", "False") ) else: logger.info( "Using " + os.environ.get("CONTAINER_ENGINE_EXECUTABLE", "docker").upper() - + " without GPU capabilities" + + " without GPU capabilities. " + + "network_disabled for the competition container is set to " + + os.environ.get("COMPETITION_CONTAINER_NETWORK_DISABLED", "False") ) if os.environ.get("CONTAINER_ENGINE_EXECUTABLE", "docker").lower() == "docker": @@ -1007,6 +1011,26 @@ async def _run_program_directory(self, program_dir, kind): if kind == "ingestion" else self.program_container_name ) + # Disable or not the competition container access to Internet (False by default) + container_network_disabled = os.environ.get( + "COMPETITION_CONTAINER_NETWORK_DISABLED", "" + ) + + # HTTP and HTTPS proxy for the competition container if needed + competition_container_proxy_http = os.environ.get( + "COMPETITION_CONTAINER_HTTP_PROXY", "" + ) + competition_container_proxy_http = ( + "http_proxy=" + competition_container_proxy_http + ) + + competition_container_proxy_https = os.environ.get( + "COMPETITION_CONTAINER_HTTPS_PROXY", "" + ) + competition_container_proxy_https = ( + "https_proxy=" + competition_container_proxy_https + ) + container = client.create_container( self.container_image, name=container_name, @@ -1015,7 +1039,12 @@ async def _run_program_directory(self, program_dir, kind): volumes=volumes_host, command=command, working_dir="/app/program", - environment=["PYTHONUNBUFFERED=1"], + environment=[ + "PYTHONUNBUFFERED=1", + competition_container_proxy_http, + competition_container_proxy_https, + ], + network_disabled=container_network_disabled.lower() == "true", ) logger.debug("Created container : " + str(container)) logger.info("Volume configuration of the container: ") diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md index 9976bcdd9..fdf9e9e70 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md @@ -40,13 +40,37 @@ You will get your Broker URL from the instance. More information about Queues [h Make a file `.env` and put this in it: ```ini title=".env" +####################################################################### +# Connection URL # +####################################################################### BROKER_URL=pyamqp://:@www.codabench.org:5672/ -HOST_DIRECTORY=/codabench # If SSL isn't enabled, then comment or remove the following line BROKER_USE_SSL=True + +####################################################################### +# Temporary Storage # +####################################################################### +HOST_DIRECTORY=/codabench + + +####################################################################### +# Runtime # +####################################################################### +CONTAINER_ENGINE_EXECUTABLE=docker #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all + +####################################################################### +# Network # +####################################################################### +#COMPETITION_CONTAINER_NETWORK_DISABLED=False + +#COMPETITION_CONTAINER_HTTP_PROXY=https://example_proxy:123 +#COMPETITION_CONTAINER_HTTPS_PROXY=http://example_proxy:1233 ``` +By default, the competition container created by the compute worker has access to internet. If you want to remove this access, you can uncomment `COMPETITION_CONTAINER_NETWORK_DISABLED` and set it to `True` + +If the VM hosting the compute worker is behind a proxy, and you want to allow the competition container to access internet, you will also need to set the proxy for the competition container to use, in which case you can use `COMPETITION_CONTAINER_HTTP_PROXY` !!! note - The broker URL is a unique identifier of the job queue that the worker should listen to. To create a queue or obtain the broker URL of an existing queue, you can refer to [Queue Management](Queue-Management.md) docs page. diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md index bd308e5bb..26dab2591 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md @@ -18,15 +18,38 @@ unqualified-search-registries = ["docker.io"] Create the `.env` file in order to add the compute worker into a queue (here, the default queue is used. If you use a particular queue, then, fill in your BROKER_URL generated when creating this particular queue) : ```ini title=".env" +####################################################################### +# Connection URL # +####################################################################### BROKER_URL=pyamqp://:@www.codabench.org:5672/ -HOST_DIRECTORY=/codabench # If SSL isn't enabled, then comment or remove the following line BROKER_USE_SSL=True + +####################################################################### +# Temporary Storage # +####################################################################### +HOST_DIRECTORY=/codabench + + +####################################################################### +# Runtime # +####################################################################### CONTAINER_ENGINE_EXECUTABLE=podman #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all + +####################################################################### +# Network # +####################################################################### +#COMPETITION_CONTAINER_NETWORK_DISABLED=False + +#COMPETITION_CONTAINER_HTTP_PROXY=https://example_proxy:123 +#COMPETITION_CONTAINER_HTTPS_PROXY=http://example_proxy:1233 ``` +By default, the competition container created by the compute worker has access to internet. If you want to remove this access, you can uncomment `COMPETITION_CONTAINER_NETWORK_DISABLED` and set it to `True` + +If the VM hosting the compute worker is behind a proxy, and you want to allow the competition container to access internet, you will also need to set the proxy for the competition container to use, in which case you can use `COMPETITION_CONTAINER_HTTP_PROXY` You will also need to create the `codabench` folder defined in the `.env` file, as well as change its permissions to the user that is running the compute worker. diff --git a/src/factories.py b/src/factories.py index c76384f5c..a149db4d0 100644 --- a/src/factories.py +++ b/src/factories.py @@ -195,9 +195,9 @@ class ColumnFactory(DjangoModelFactory): class Meta: model = Column - title = factory.Faker('word') - key = factory.Faker('word') index = factory.Sequence(lambda n: n) + title = factory.LazyAttribute(lambda n: f"Col_{n.index}") + key = factory.LazyAttribute(lambda n: f"col_{n.index}") leaderboard = factory.SubFactory(LeaderboardFactory) sorting = factory.LazyAttribute(lambda n: random.choice(['asc', 'desc']))