Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions test/data/my-environment.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
name: xcengine
channels:
- conda-forge
dependencies:
- python >=3.11
25 changes: 24 additions & 1 deletion test/test_cli.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import pathlib
import re
import urllib
from unittest.mock import patch, ANY, MagicMock
Expand Down Expand Up @@ -86,7 +87,7 @@ def test_image_build(builder_mock, tmp_path, specify_dir, specify_env):
tag=tag,
build_dir=(build_dir if specify_dir else ANY),
)
instance_mock.build.assert_called_once_with()
instance_mock.build.assert_called_once_with(skip_build=False)


@patch("xcengine.cli.ContainerRunner")
Expand Down Expand Up @@ -180,3 +181,25 @@ def urlopen(url):
assert count == 2
assert passed_url == f"http://localhost:{port}"
open_mock.assert_called_once_with(f"http://localhost:{port}/viewer")

@patch("docker.from_env")
def test_image_skip_build_save_dockerfile_and_env(from_env_mock, tmp_path):
build_dir = tmp_path / "build"
nb_path = pathlib.Path(__file__).parent / "data" / "noparamtest.ipynb"
cli_runner = CliRunner()
result = cli_runner.invoke(
cli,
[
"image",
"build",
"--skip-build",
"--build-dir",
str(build_dir),
str(nb_path),
],
)
assert result.exit_code == 0
assert (build_dir / "Dockerfile").is_file()
assert (build_dir / "environment.yml").is_file()
from_env_mock.assert_not_called()

28 changes: 28 additions & 0 deletions test/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,3 +330,31 @@ def test_image_builder_notebook_config(tmp_path):
config = image_builder.script_creator.nb_params.config
assert config["environment_file"] == "my-environment.yml"
assert config["container_image_tag"] == "my-tag"


def test_image_builder_write_dockerfile(tmp_path):
ImageBuilder.write_dockerfile(
dockerfilepath := tmp_path / "as-yet-nonexistent-dir" / "Dockerfile"
)
with open(dockerfilepath) as fh:
content = fh.read()
assert content.startswith("FROM ")
assert "\nENTRYPOINT " in content


@patch("docker.from_env")
def test_image_builder_build_skip_build(from_env_mock, tmp_path):
build_dir = tmp_path / "build"
image_builder = ImageBuilder(
pathlib.Path(__file__).parent / "data" / "noparamtest.ipynb",
None,
build_dir,
None,
)
image_builder.build(skip_build=True)
from_env_mock.assert_not_called()
env_path = build_dir / "environment.yml"
assert env_path.is_file()
with open(env_path) as fh:
env_dict = yaml.safe_load(fh)
assert {"name", "channels", "dependencies"} <= set(env_dict)
20 changes: 16 additions & 4 deletions xcengine/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,13 +146,21 @@ def image_cli():
help="Write a CWL file defining an Earth Observation Application Package "
"to the specified path.",
)
@click.option(
"-s",
"--skip-build",
is_flag=True,
help="Prepare the Dockerfile and build context, but don't actually build "
"the image"
)
@notebook_argument
def build(
build_dir: pathlib.Path,
notebook: pathlib.Path,
environment: pathlib.Path,
tag: str,
eoap: pathlib.Path,
skip_build: bool
) -> None:
if environment is None:
LOGGER.info("No environment file specified on command line.")
Expand All @@ -166,13 +174,13 @@ class InitArgs(TypedDict):
if build_dir:
image_builder = ImageBuilder(build_dir=build_dir, **init_args)
os.makedirs(build_dir, exist_ok=True)
image = image_builder.build()
image = image_builder.build(skip_build=skip_build)
else:
with tempfile.TemporaryDirectory() as temp_dir:
image_builder = ImageBuilder(
build_dir=pathlib.Path(temp_dir), **init_args
)
image = image_builder.build()
image = image_builder.build(skip_build=skip_build)
if eoap:

class IndentDumper(yaml.Dumper):
Expand All @@ -186,7 +194,11 @@ def increase_indent(self, flow=False, indentless=False):
Dumper=IndentDumper,
)
)
print(f"Built image with tags {image.tags}")
print(
f"Built image with tags {image.tags}"
if image is not None else
"No image built"
)


@image_cli.command(
Expand Down Expand Up @@ -233,7 +245,7 @@ def increase_indent(self, flow=False, indentless=False):
help="Keep container after it has finished running.",
)
@click.option(
"-b",
"-w",
"--open-browser",
is_flag=True,
help="After the server has started, open a web browser window "
Expand Down
48 changes: 30 additions & 18 deletions xcengine/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,10 @@ def __init__(
LOGGER.info(f"No environment found at {notebook_sibling}")
self.environment = None

def build(self) -> Image:
def build(
self,
skip_build: bool = False,
) -> Image | None:
self.script_creator.convert_notebook_to_script(self.build_dir)
if self.environment:
with open(self.environment, "r") as fh:
Expand All @@ -247,7 +250,8 @@ def build(self) -> Image:
self.add_packages_to_environment(env_def, ["xcube", "pystac"])
with open(self.build_dir / "environment.yml", "w") as fh:
fh.write(yaml.safe_dump(env_def))
return self._build_image()
self.write_dockerfile(self.build_dir / "Dockerfile")
return None if skip_build else self._build_image()

@staticmethod
def export_conda_env() -> dict:
Expand Down Expand Up @@ -308,22 +312,6 @@ def ensure_present(pkg: str):

def _build_image(self) -> docker.models.images.Image:
client = docker.from_env()
dockerfile = textwrap.dedent("""
FROM mambaorg/micromamba:1.5.10-noble-cuda-12.6.0
COPY Dockerfile Dockerfile
COPY environment.yml environment.yml
RUN micromamba install -y -n base -f environment.yml && \
micromamba clean --all --yes
WORKDIR /home/mambauser
COPY user_code.py user_code.py
COPY execute.py execute.py
COPY parameters.yaml parameters.yaml
COPY parameters.py parameters.py
COPY util.py util.py
ENTRYPOINT ["/usr/local/bin/_entrypoint.sh", "python", "/home/mambauser/execute.py"]
""")
with open(self.build_dir / "Dockerfile", "w") as fh:
fh.write(dockerfile)
LOGGER.info(f"Building image with tag {self.tag}...")
try:
image, logs = client.images.build(
Expand All @@ -338,6 +326,30 @@ def _build_image(self) -> docker.models.images.Image:
LOGGER.info("Docker image built.")
return image

@staticmethod
def write_dockerfile(destination: pathlib.Path) -> None:
LOGGER.info(f"Writing Dockerfile to {destination}...")
destination.parent.mkdir(parents=True, exist_ok=True)
with open(destination, "w") as fh:
fh.write(textwrap.dedent("""\
FROM mambaorg/micromamba:1.5.10-noble-cuda-12.6.0
COPY Dockerfile Dockerfile
COPY environment.yml environment.yml
RUN micromamba install -y -n base -f environment.yml && \\
micromamba clean --all --yes
WORKDIR /home/mambauser
COPY user_code.py user_code.py
COPY execute.py execute.py
COPY parameters.yaml parameters.yaml
COPY parameters.py parameters.py
COPY util.py util.py
ENTRYPOINT [ \\
"/usr/local/bin/_entrypoint.sh", \\
"python", \\
"/home/mambauser/execute.py" \\
]
"""))

def create_cwl(self):
return self.script_creator.create_cwl(self.tag)

Expand Down