Skip to content

Commit 1a1ae38

Browse files
mdellwegmdellweg
committed
Move some stances around in the ci scripts
1 parent 86437ce commit 1a1ae38

4 files changed

Lines changed: 181 additions & 242 deletions

File tree

‎templates/github/.github/workflows/scripts/before_install.sh.j2‎

Lines changed: 127 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,70 +2,150 @@
22

33
{% include 'header.j2' %}
44

5+
set -euo pipefail
6+
57
# make sure this script runs at the repo root
68
cd "$(dirname "$(realpath -e "$0")")"/../../..
79

8-
set -mveuo pipefail
10+
if [ -f .github/workflows/scripts/pre_before_install.sh ]; then
11+
source .github/workflows/scripts/pre_before_install.sh
12+
fi
13+
14+
COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
15+
COMPONENT_SOURCE="./{{ plugin_name }}/dist/{{ plugin_name | snake }}-${COMPONENT_VERSION}-py3-none-any.whl"
916

10-
if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ]
11-
then
12-
BRANCH_BUILD=0
13-
else
14-
BRANCH_BUILD=1
15-
BRANCH="${GITHUB_REF##refs/heads/}"
17+
{%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %}
18+
{%- if test_s3 %}
19+
if [ "$TEST" = "s3" ]; then
20+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[s3]"
1621
fi
17-
if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ]
18-
then
19-
TAG_BUILD=0
20-
else
21-
TAG_BUILD=1
22-
BRANCH="${GITHUB_REF##refs/tags/}"
22+
{%- endif %}
23+
{%- if test_azure %}
24+
if [ "$TEST" = "azure" ]; then
25+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[azure]"
26+
fi
27+
{%- endif %}
28+
{%- if test_gcp %}
29+
if [ "$TEST" = "gcp" ]; then
30+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[google]"
2331
fi
32+
{%- endif %}
2433

25-
COMMIT_MSG=$(git log --format=%B --no-merges -1)
26-
export COMMIT_MSG
34+
if [[ "$TEST" = "pulp" ]]; then
35+
python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt
36+
fi
37+
if [[ "$TEST" = "lowerbounds" ]]; then
38+
python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt
39+
fi
2740

28-
{% if setup_py -%}
29-
COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py)
30-
{%- else -%}
31-
COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
41+
{%- if test_reroute %}
42+
export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "{{ api_root }}")
43+
{%- else %}
44+
export PULP_API_ROOT="{{ api_root }}"
3245
{%- endif %}
3346

34-
mkdir .ci/ansible/vars || true
35-
echo "---" > .ci/ansible/vars/main.yaml
36-
echo "legacy_component_name: {{ plugin_name | snake }}" >> .ci/ansible/vars/main.yaml
37-
echo "component_name: {{ plugin_app_label }}" >> .ci/ansible/vars/main.yaml
38-
echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml
47+
echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
3948

40-
export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh
41-
export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh
49+
# Compose the scenario definition.
50+
mkdir -p .ci/ansible/vars
4251

43-
if [ -f $PRE_BEFORE_INSTALL ]; then
44-
source $PRE_BEFORE_INSTALL
45-
fi
52+
cat > .ci/ansible/vars/main.yaml << VARSYAML
53+
---
54+
scenario: "${TEST}"
55+
legacy_component_name: "{{ plugin_name | snake }}"
56+
component_name: "{{ plugin_app_label }}"
57+
component_version: "${COMPONENT_VERSION}"
58+
pulp_env: {{ pulp_env | tojson }}
59+
pulp_settings: {{ pulp_settings | tojson }}
60+
pulp_scheme: "{{ pulp_scheme }}"
61+
{%- if ci_base_image.count(":") %}
62+
pulp_default_container: "{{ ci_base_image }}"
63+
{% else %}
64+
pulp_default_container: "{{ ci_base_image + ":latest" }}"
65+
{% endif -%}
66+
api_root: "${PULP_API_ROOT}"
67+
image:
68+
name: "pulp"
69+
tag: "ci_build"
70+
plugins:
71+
- name: "{{ plugin_name }}"
72+
source: "${COMPONENT_SOURCE}"
73+
ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
74+
upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
75+
lowerounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
76+
{%- if extra_files %}
77+
extra_files:
78+
{%- for item in extra_files %}
79+
- origin: "{{ item.origin }}"
80+
destination: "{{ item.destination }}"
81+
{%- endfor %}
82+
{%- endif %}
83+
services:
84+
- name: "pulp"
85+
image: "pulp:ci_build"
86+
volumes:
87+
- "./settings:/etc/pulp"
88+
- "./ssh:/keys/"
89+
- "~/.config:/var/lib/pulp/.config"
90+
- "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
91+
env:
92+
PULP_WORKERS: "4"
93+
PULP_HTTPS: "{{ 'true' if pulp_scheme == 'https' else 'false' }}"
94+
{%- if docker_fixtures %}
95+
- name: "pulp-fixtures"
96+
image: "docker.io/pulp/pulp-fixtures:latest"
97+
env:
98+
BASE_URL: "http://pulp-fixtures:8080"
99+
{%- endif %}
100+
VARSYAML
46101

47-
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "{{ plugin_default_branch }}" ]
48-
then
49-
echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml
102+
if [ "$TEST" = "s3" ]; then
103+
MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
104+
MINIO_SECRET_KEY=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
105+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
106+
- name: "minio"
107+
image: "minio/minio"
108+
env:
109+
MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}"
110+
MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}"
111+
command: "server /data"
112+
s3_test: true
113+
minio_access_key: "${MINIO_ACCESS_KEY}"
114+
minio_secret_key: "${MINIO_SECRET_KEY}"
115+
pulp_scenario_settings: {{ pulp_settings_s3 | tojson }}
116+
pulp_scenario_env: {{ pulp_env_s3 | tojson }}
117+
VARSYAML
50118
fi
51119

52-
for i in {1..3}
53-
do
54-
ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
55-
done
56-
if [[ $s -gt 0 ]]
57-
then
58-
echo "Failed to install amazon.aws"
59-
exit $s
120+
if [ "$TEST" = "azure" ]; then
121+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
122+
- name: "ci-azurite"
123+
image: "mcr.microsoft.com/azure-storage/azurite"
124+
command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"
125+
azure_test: true
126+
pulp_scenario_settings: {{ pulp_settings_azure | tojson }}
127+
pulp_scenario_env: {{ pulp_env_azure | tojson }}
128+
VARSYAML
60129
fi
61130

62-
if [[ "$TEST" = "pulp" ]]; then
63-
python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt
64-
fi
65-
if [[ "$TEST" = "lowerbounds" ]]; then
66-
python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt
131+
if [ "$TEST" = "gcp" ]; then
132+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
133+
- name: "ci-gcp"
134+
image: "fsouza/fake-gcs-server"
135+
volumes:
136+
- "storage_data:/etc/pulp"
137+
command: " -scheme http"
138+
gcp_test: true
139+
pulp_scenario_settings: {{ pulp_settings_gcp | tojson }}
140+
pulp_scenario_env: {{ pulp_env_gcp | tojson }}
141+
VARSYAML
67142
fi
68143

69-
if [ -f $POST_BEFORE_INSTALL ]; then
70-
source $POST_BEFORE_INSTALL
144+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
145+
...
146+
VARSYAML
147+
cat .ci/ansible/vars/main.yaml
148+
149+
if [ -f .github/workflows/scripts/post_before_install.sh ]; then
150+
source .github/workflows/scripts/post_before_install.sh
71151
fi

‎templates/github/.github/workflows/scripts/before_script.sh.j2‎

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,48 +2,45 @@
22

33
{% include 'header.j2' %}
44

5+
# This script dumps some files to help understand the setup of the test scenario.
6+
57
# make sure this script runs at the repo root
68
cd "$(dirname "$(realpath -e "$0")")"/../../..
79

810
set -euv
911

1012
source .github/workflows/scripts/utils.sh
1113

12-
export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh
13-
export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh
14-
15-
if [[ -f $PRE_BEFORE_SCRIPT ]]; then
16-
source $PRE_BEFORE_SCRIPT
14+
if [[ -f .github/workflows/scripts/pre_before_script.sh ]]; then
15+
source .github/workflows/scripts/pre_before_script.sh
1716
fi
1817

19-
# Developers should be able to reproduce the containers with this config
20-
echo "CI vars:"
21-
tail -v -n +1 .ci/ansible/vars/main.yaml
22-
2318
# Developers often want to know the final pulp config
24-
echo "PULP CONFIG:"
19+
echo
20+
echo "# Pulp config:"
2521
tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
2622

27-
echo "Containerfile:"
23+
echo
24+
echo "# Containerfile:"
2825
tail -v -n +1 .ci/ansible/Containerfile
2926

30-
echo "Constraints Files:"
31-
# The need not even exist.
27+
echo
28+
echo "# Constraints Files:"
29+
# They need not even exist.
3230
tail -v -n +1 ../*/*constraints.txt || true
3331

34-
# Needed for some functional tests
35-
cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
36-
cmd_prefix bash -c "usermod -a -G wheel pulp"
32+
echo
33+
echo "# pip list outside the container"
3734

38-
if [[ "${REDIS_DISABLED:-false}" == true ]]; then
39-
cmd_prefix bash -c "s6-rc -d change redis"
40-
echo "The Redis service was disabled for $TEST"
41-
fi
35+
echo
36+
echo "# pip list inside the container"
37+
cmd_prefix bash -c "pip3 list"
38+
39+
echo
40+
echo "# State of the containers"
41+
docker ps -a
4242

43-
if [[ -f $POST_BEFORE_SCRIPT ]]; then
44-
source $POST_BEFORE_SCRIPT
43+
if [[ -f .github/workflows/scripts/post_before_script.sh ]]; then
44+
source .github/workflows/scripts/post_before_script.sh
4545
fi
4646

47-
# Lots of plugins try to use this path, and throw warnings if they cannot access it.
48-
cmd_prefix mkdir /.pytest_cache
49-
cmd_prefix chown pulp:pulp /.pytest_cache

0 commit comments

Comments
 (0)