Skip to content

Commit 7de6499

Browse files
mdellwegmdellweg
committed
Move some stances around in the ci scripts
1 parent 7b32c6e commit 7de6499

4 files changed

Lines changed: 191 additions & 244 deletions

File tree

‎templates/github/.github/workflows/scripts/before_install.sh.j2‎

Lines changed: 134 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,70 +2,157 @@
22

33
{% include 'header.j2' %}
44

5+
# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file.
6+
#
7+
# It requires the following environment:
8+
# TEST - The name of the scenario to prepare.
9+
#
10+
# It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario.
11+
12+
set -eu -o pipefail
13+
514
# make sure this script runs at the repo root
615
cd "$(dirname "$(realpath -e "$0")")"/../../..
716

8-
set -mveuo pipefail
17+
if [ -f .github/workflows/scripts/pre_before_install.sh ]; then
18+
source .github/workflows/scripts/pre_before_install.sh
19+
fi
20+
21+
COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
22+
COMPONENT_SOURCE="./{{ plugin_name }}/dist/{{ plugin_name | snake }}-${COMPONENT_VERSION}-py3-none-any.whl"
923

10-
if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ]
11-
then
12-
BRANCH_BUILD=0
13-
else
14-
BRANCH_BUILD=1
15-
BRANCH="${GITHUB_REF##refs/heads/}"
24+
{%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %}
25+
{%- if test_s3 %}
26+
if [ "$TEST" = "s3" ]; then
27+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[s3]"
28+
fi
29+
{%- endif %}
30+
{%- if test_azure %}
31+
if [ "$TEST" = "azure" ]; then
32+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[azure]"
1633
fi
17-
if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ]
18-
then
19-
TAG_BUILD=0
20-
else
21-
TAG_BUILD=1
22-
BRANCH="${GITHUB_REF##refs/tags/}"
34+
{%- endif %}
35+
{%- if test_gcp %}
36+
if [ "$TEST" = "gcp" ]; then
37+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[google]"
2338
fi
39+
{%- endif %}
2440

25-
COMMIT_MSG=$(git log --format=%B --no-merges -1)
26-
export COMMIT_MSG
41+
if [[ "$TEST" = "pulp" ]]; then
42+
python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt
43+
fi
44+
if [[ "$TEST" = "lowerbounds" ]]; then
45+
python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt
46+
fi
2747

28-
{% if setup_py -%}
29-
COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py)
30-
{%- else -%}
31-
COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
48+
{%- if test_reroute %}
49+
export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "{{ api_root }}")
50+
{%- else %}
51+
export PULP_API_ROOT="{{ api_root }}"
3252
{%- endif %}
3353

34-
mkdir .ci/ansible/vars || true
35-
echo "---" > .ci/ansible/vars/main.yaml
36-
echo "legacy_component_name: {{ plugin_name | snake }}" >> .ci/ansible/vars/main.yaml
37-
echo "component_name: {{ plugin_app_label }}" >> .ci/ansible/vars/main.yaml
38-
echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml
54+
echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
3955

40-
export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh
41-
export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh
56+
# Compose the scenario definition.
57+
mkdir -p .ci/ansible/vars
4258

43-
if [ -f $PRE_BEFORE_INSTALL ]; then
44-
source $PRE_BEFORE_INSTALL
45-
fi
59+
cat > .ci/ansible/vars/main.yaml << VARSYAML
60+
---
61+
scenario: "${TEST}"
62+
legacy_component_name: "{{ plugin_name | snake }}"
63+
component_name: "{{ plugin_app_label }}"
64+
component_version: "${COMPONENT_VERSION}"
65+
pulp_env: {{ pulp_env | tojson }}
66+
pulp_settings: {{ pulp_settings | tojson }}
67+
pulp_scheme: "{{ pulp_scheme }}"
68+
{%- if ci_base_image.count(":") %}
69+
pulp_default_container: "{{ ci_base_image }}"
70+
{% else %}
71+
pulp_default_container: "{{ ci_base_image + ":latest" }}"
72+
{% endif -%}
73+
api_root: "${PULP_API_ROOT}"
74+
image:
75+
name: "pulp"
76+
tag: "ci_build"
77+
plugins:
78+
- name: "{{ plugin_name }}"
79+
source: "${COMPONENT_SOURCE}"
80+
ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
81+
upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
82+
lowerounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
83+
{%- if extra_files %}
84+
extra_files:
85+
{%- for item in extra_files %}
86+
- origin: "{{ item.origin }}"
87+
destination: "{{ item.destination }}"
88+
{%- endfor %}
89+
{%- endif %}
90+
services:
91+
- name: "pulp"
92+
image: "pulp:ci_build"
93+
volumes:
94+
- "./settings:/etc/pulp"
95+
- "./ssh:/keys/"
96+
- "~/.config:/var/lib/pulp/.config"
97+
- "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
98+
env:
99+
PULP_WORKERS: "4"
100+
PULP_HTTPS: "{{ 'true' if pulp_scheme == 'https' else 'false' }}"
101+
{%- if docker_fixtures %}
102+
- name: "pulp-fixtures"
103+
image: "docker.io/pulp/pulp-fixtures:latest"
104+
env:
105+
BASE_URL: "http://pulp-fixtures:8080"
106+
{%- endif %}
107+
VARSYAML
46108

47-
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "{{ plugin_default_branch }}" ]
48-
then
49-
echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml
109+
if [ "$TEST" = "s3" ]; then
110+
MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
111+
MINIO_SECRET_KEY=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
112+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
113+
- name: "minio"
114+
image: "minio/minio"
115+
env:
116+
MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}"
117+
MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}"
118+
command: "server /data"
119+
s3_test: true
120+
minio_access_key: "${MINIO_ACCESS_KEY}"
121+
minio_secret_key: "${MINIO_SECRET_KEY}"
122+
pulp_scenario_settings: {{ pulp_settings_s3 | tojson }}
123+
pulp_scenario_env: {{ pulp_env_s3 | tojson }}
124+
VARSYAML
50125
fi
51126

52-
for i in {1..3}
53-
do
54-
ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
55-
done
56-
if [[ $s -gt 0 ]]
57-
then
58-
echo "Failed to install amazon.aws"
59-
exit $s
127+
if [ "$TEST" = "azure" ]; then
128+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
129+
- name: "ci-azurite"
130+
image: "mcr.microsoft.com/azure-storage/azurite"
131+
command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"
132+
azure_test: true
133+
pulp_scenario_settings: {{ pulp_settings_azure | tojson }}
134+
pulp_scenario_env: {{ pulp_env_azure | tojson }}
135+
VARSYAML
60136
fi
61137

62-
if [[ "$TEST" = "pulp" ]]; then
63-
python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt
64-
fi
65-
if [[ "$TEST" = "lowerbounds" ]]; then
66-
python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt
138+
if [ "$TEST" = "gcp" ]; then
139+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
140+
- name: "ci-gcp"
141+
image: "fsouza/fake-gcs-server"
142+
volumes:
143+
- "storage_data:/etc/pulp"
144+
command: " -scheme http"
145+
gcp_test: true
146+
pulp_scenario_settings: {{ pulp_settings_gcp | tojson }}
147+
pulp_scenario_env: {{ pulp_env_gcp | tojson }}
148+
VARSYAML
67149
fi
68150

69-
if [ -f $POST_BEFORE_INSTALL ]; then
70-
source $POST_BEFORE_INSTALL
151+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
152+
...
153+
VARSYAML
154+
cat .ci/ansible/vars/main.yaml
155+
156+
if [ -f .github/workflows/scripts/post_before_install.sh ]; then
157+
source .github/workflows/scripts/post_before_install.sh
71158
fi

‎templates/github/.github/workflows/scripts/before_script.sh.j2‎

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,48 +2,46 @@
22

33
{% include 'header.j2' %}
44

5+
# This script dumps some files to help understand the setup of the test scenario.
6+
7+
set -eu -o pipefail
8+
59
# make sure this script runs at the repo root
610
cd "$(dirname "$(realpath -e "$0")")"/../../..
711

8-
set -euv
9-
1012
source .github/workflows/scripts/utils.sh
1113

12-
export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh
13-
export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh
14-
15-
if [[ -f $PRE_BEFORE_SCRIPT ]]; then
16-
source $PRE_BEFORE_SCRIPT
14+
if [[ -f .github/workflows/scripts/pre_before_script.sh ]]; then
15+
source .github/workflows/scripts/pre_before_script.sh
1716
fi
1817

19-
# Developers should be able to reproduce the containers with this config
20-
echo "CI vars:"
21-
tail -v -n +1 .ci/ansible/vars/main.yaml
22-
2318
# Developers often want to know the final pulp config
24-
echo "PULP CONFIG:"
19+
echo
20+
echo "# Pulp config:"
2521
tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
2622

27-
echo "Containerfile:"
23+
echo
24+
echo "# Containerfile:"
2825
tail -v -n +1 .ci/ansible/Containerfile
2926

30-
echo "Constraints Files:"
31-
# The need not even exist.
27+
echo
28+
echo "# Constraints Files:"
29+
# They need not even exist.
3230
tail -v -n +1 ../*/*constraints.txt || true
3331

34-
# Needed for some functional tests
35-
cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
36-
cmd_prefix bash -c "usermod -a -G wheel pulp"
32+
echo
33+
echo "# pip list outside the container"
34+
pip list
3735

38-
if [[ "${REDIS_DISABLED:-false}" == true ]]; then
39-
cmd_prefix bash -c "s6-rc -d change redis"
40-
echo "The Redis service was disabled for $TEST"
41-
fi
36+
echo
37+
echo "# pip list inside the container"
38+
cmd_prefix bash -c "pip3 list"
39+
40+
echo
41+
echo "# State of the containers"
42+
docker ps -a
4243

43-
if [[ -f $POST_BEFORE_SCRIPT ]]; then
44-
source $POST_BEFORE_SCRIPT
44+
if [[ -f .github/workflows/scripts/post_before_script.sh ]]; then
45+
source .github/workflows/scripts/post_before_script.sh
4546
fi
4647

47-
# Lots of plugins try to use this path, and throw warnings if they cannot access it.
48-
cmd_prefix mkdir /.pytest_cache
49-
cmd_prefix chown pulp:pulp /.pytest_cache

0 commit comments

Comments
 (0)