From 6a809e20d9b8bf6db2c7f504aa1f5687b5dbfdfe Mon Sep 17 00:00:00 2001 From: ternera Date: Mon, 17 Nov 2025 20:12:16 -0600 Subject: [PATCH 1/5] started on docker implimentation --- .dockerignore | 34 +++++++++++++++++ DOCKER_README.md | 0 Dockerfile | 32 ++++++++++++++++ docker-compose.yml | 70 +++++++++++++++++++++++++++++++++++ docker-entrypoint.sh | 28 ++++++++++++++ editgroups/settings/docker.py | 29 +++++++++++++++ 6 files changed, 193 insertions(+) create mode 100644 .dockerignore create mode 100644 DOCKER_README.md create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh create mode 100644 editgroups/settings/docker.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2649130 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,34 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +*.so +*.egg +*.egg-info +dist +build +.git +.gitignore +.vscode +.idea +*.swp +*.swo +*~ +.DS_Store +db.sqlite3 +*.sqlite3 +.env +venv +env +ENV +.venv +staticfiles +media +*.log +.coverage +htmlcov +.pytest_cache +.mypy_cache +node_modules +postgres_data diff --git a/DOCKER_README.md b/DOCKER_README.md new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3e74fdc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + postgresql-client \ + libpq-dev \ + gcc \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + git \ + netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir psycopg2-binary + +COPY . . + +RUN mkdir -p /app/static + +EXPOSE 8000 + +COPY docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0fa9d18 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,70 @@ +version: '3.8' + +services: + db: + image: postgres:15 + environment: + POSTGRES_DB: editgroups + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + web: + build: . + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/app + ports: + - "8000:8000" + environment: + - DJANGO_SETTINGS_MODULE=editgroups.settings.docker + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + celery: + build: . + command: celery -A editgroups worker -l info + volumes: + - .:/app + environment: + - DJANGO_SETTINGS_MODULE=editgroups.settings.docker + depends_on: + - db + - redis + - web + + celery-beat: + build: . + command: celery -A editgroups beat -l info + volumes: + - .:/app + environment: + - DJANGO_SETTINGS_MODULE=editgroups.settings.docker + depends_on: + - db + - redis + - web + +volumes: + postgres_data: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..397ece3 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +echo "Waiting for PostgreSQL..." +while ! nc -z db 5432; do + sleep 0.1 +done +echo "PostgreSQL started" + +echo "Running migrations..." +python manage.py migrate --noinput + +echo "Collecting static files..." +python manage.py collectstatic --noinput + +echo "Creating superuser if it doesn't exist..." +python manage.py shell -c " +from django.contrib.auth import get_user_model +User = get_user_model() +if not User.objects.filter(username='admin').exists(): + User.objects.create_superuser('admin', 'admin@example.com', 'admin') + print('Superuser created: admin/admin') +else: + print('Superuser already exists') +" || true + +exec "$@" diff --git a/editgroups/settings/docker.py b/editgroups/settings/docker.py new file mode 100644 index 0000000..6274c34 --- /dev/null +++ b/editgroups/settings/docker.py @@ -0,0 +1,29 @@ +from .common import * + +DEBUG = True +ALLOWED_HOSTS = ['*'] + +SECRET_KEY = '20oj&tj8uaruseitlrise,tries,uirsetur36746209etus7e' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'editgroups', + 'USER': 'postgres', + 'PASSWORD': 'postgres', + 'HOST': 'db', + 'PORT': '5432', + 'DISABLE_SERVER_SIDE_CURSORS': False, + } +} + +SOCIAL_AUTH_MEDIAWIKI_KEY = 'your_mediawiki_key' +SOCIAL_AUTH_MEDIAWIKI_SECRET = 'your_mediawiki_secret' +SOCIAL_AUTH_MEDIAWIKI_URL = 'https://www.wikidata.org/w/index.php' +SOCIAL_AUTH_MEDIAWIKI_CALLBACK = 'http://localhost:8000/oauth/complete/mediawiki/' + +REDIS_HOST = 'redis' +REDIS_PORT = 6379 +REDIS_DB = 0 +REDIS_PASSWORD = '' +REDIS_KEY_PREFIX = 'editgroups_' From ea6cefc1a31ad34f5e40410d4dff886bd08cd73a Mon Sep 17 00:00:00 2001 From: ternera Date: Tue, 18 Nov 2025 11:36:50 -0600 Subject: [PATCH 2/5] fixed a few bugs --- DOCKER_README.md | 0 celerybeat-schedule | Bin 0 -> 16384 bytes docker-compose.yml | 2 ++ docker-entrypoint.sh | 16 ++++++++------- editgroups/settings/docker.py | 37 +++++++++++++++++++--------------- editgroups/wsgi.py | 17 +++++++++++++++- 6 files changed, 48 insertions(+), 24 deletions(-) delete mode 100644 DOCKER_README.md create mode 100644 celerybeat-schedule diff --git a/DOCKER_README.md b/DOCKER_README.md deleted file mode 100644 index e69de29..0000000 diff --git a/celerybeat-schedule b/celerybeat-schedule new file mode 100644 index 0000000000000000000000000000000000000000..9e8fd745a64bcaa7483210b990bbf1f20b60fba3 GIT binary patch literal 16384 zcmeI(&ubGw6bJB0x-p5V^rxU!)I-HcVNpuY3dKWNET|Pl!DX`f##{ubUZTKWxAp0uX=z1Rwwb2tWV=5P$##AOL}*C~(1b8qq?)0S-n0;{f9T z;{f*o>Hz8h>Hz8h>Hz8h>HzA%e^v)Z>|M=9U1V2m@~~jzvRnS*Hkh5?+3??^N4xx_ z*RVaY-m1U>4*B~7c1Fx@r^ZeWHB&{qA2j159zW}Jms8 zQ6uHef%nqiZEb7G{eIBbbv2e`omC#ouo1~v@Qzlsn5VqG?l*WOwK{Q!);7B-zd@Z@ z*PFU73ZkGY%}8>YJmWf-ZtBJ3 Date: Wed, 25 Feb 2026 23:57:28 -0300 Subject: [PATCH 3/5] fix: wsgi.py --- editgroups/wsgi.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) mode change 120000 => 100644 editgroups/wsgi.py diff --git a/editgroups/wsgi.py b/editgroups/wsgi.py deleted file mode 120000 index 61d7e1a..0000000 --- a/editgroups/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for editgroups project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "editgroups.settings") - -application = app = get_wsgi_application() diff --git a/editgroups/wsgi.py b/editgroups/wsgi.py new file mode 100644 index 0000000..73b792c --- /dev/null +++ b/editgroups/wsgi.py @@ -0,0 +1,8 @@ +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "editgroups.settings") + +application = get_wsgi_application() + From 58d94e355076c00b46bd295d27c4f84be681b7db Mon Sep 17 00:00:00 2001 From: arcstur Date: Wed, 25 Feb 2026 23:57:51 -0300 Subject: [PATCH 4/5] refactor: make docker closer to Toolforge * use MariaDB instead of Postgres * use Toolforge's python image * use docker.py settings as secret, and use dev * remove docker-entrypoint and use that in docker-compose --- Dockerfile | 38 ++++++------ docker-compose.yml | 109 ++++++++++++++++++---------------- docker-entrypoint.sh | 30 ---------- editgroups/settings/docker.py | 46 ++++++-------- 4 files changed, 97 insertions(+), 126 deletions(-) delete mode 100755 docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 3e74fdc..0a9c737 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,36 @@ -FROM python:3.11-slim +FROM docker-registry.tools.wmflabs.org/toolforge-python311-sssd-web:latest -ENV PYTHONUNBUFFERED=1 - -WORKDIR /app +WORKDIR /root/www/python/ RUN apt-get update && apt-get install -y \ - postgresql-client \ - libpq-dev \ gcc \ libxml2-dev \ libxslt1-dev \ zlib1g-dev \ git \ netcat-openbsd \ + curl \ && rm -rf /var/lib/apt/lists/* -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -RUN pip install --no-cache-dir psycopg2-binary - -COPY . . +# Necessary flags for mysqlclient driver +ENV MYSQLCLIENT_CFLAGS="-I/usr/include/mariadb/" +ENV MYSQLCLIENT_LDFLAGS="-L/usr/lib/x86_64-linux-gnu/ -lmariadb" +ENV VIRTUAL_ENV /root/www/python/venv +ENV PATH="/root/www/python/venv/bin:${PATH}" +ENV DJANGO_SETTINGS_MODULE=editgroups.settings +ENV PYTHONPATH="/root/www/python/src:${PYTHONPATH}" +COPY requirements.txt ./src/requirements.txt +RUN echo "mysqlclient==2.2.7" >> /root/www/python/src/requirements.txt +RUN webservice-python-bootstrap -RUN mkdir -p /app/static +COPY . ./src +WORKDIR /root/www/python/src/ -EXPOSE 8000 +RUN echo "from .dev import *" > /root/www/python/src/editgroups/settings/__init__.py +COPY editgroups/settings/docker.py /root/www/python/src/editgroups/settings/secret.py +RUN echo "source /root/www/python/venv/bin/activate" >> /root/.bashrc # useful when entering the shell -COPY docker-entrypoint.sh /docker-entrypoint.sh -RUN chmod +x /docker-entrypoint.sh +# Just to make sure virtual environment is working properly +RUN ["/bin/bash", "-c", "source ../venv/bin/activate && python3 manage.py collectstatic --no-input"] -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] +EXPOSE 8000 diff --git a/docker-compose.yml b/docker-compose.yml index becacdd..dfadb04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,72 +1,77 @@ -version: '3.8' - services: - db: - image: postgres:15 + mariadb: + image: mariadb:11.7.2 environment: - POSTGRES_DB: editgroups - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + MARIADB_DATABASE: ${DB_NAME:-editgroups} + MARIADB_ROOT_PASSWORD: ${DB_PASSWORD:-editgroups} volumes: - - postgres_data:/var/lib/postgresql/data - ports: - - "5432:5432" + - mariadb:/var/lib/mysql healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s + test: healthcheck.sh --connect --innodb_initialized + start_period: 10s + interval: 10s timeout: 5s - retries: 5 + retries: 3 + expose: + - 3306 redis: - image: redis:7-alpine - ports: - - "6379:6379" + image: redis:7.4 + expose: + - 6379 healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s timeout: 3s retries: 5 - web: - build: . - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - .:/app + app: ports: - - "8000:8000" - environment: - - DJANGO_SETTINGS_MODULE=editgroups.settings.docker - - RUN_MIGRATIONS=true - depends_on: - db: - condition: service_healthy - redis: - condition: service_healthy + - 8000:8000 + healthcheck: + test: curl --fail http://localhost:8000/ || exit 1 + start_period: 10s + interval: 10s + command: > + /bin/bash -c " + django-admin collectstatic --no-input && + django-admin migrate && + django-admin runserver 0.0.0.0:8000 + " + <<: &app + build: + context: . + dockerfile: Dockerfile + depends_on: + mariadb: + condition: service_healthy + redis: + condition: service_healthy + environment: + - SECRET_KEY=${DJANGO_SECRET_KEY:-} + - DB_NAME=${DB_NAME:-editgroups} + - DB_PASSWORD=${DB_NAME:-editgroups} celery: - build: . - command: celery -A editgroups worker -l info - volumes: - - .:/app - environment: - - DJANGO_SETTINGS_MODULE=editgroups.settings.docker - - C_FORCE_ROOT=true + <<: *app + command: > + /bin/bash -c " + source ../venv/bin/activate + C_FORCE_ROOT=1 python3 ../venv/bin/celery --app=editgroups.celery:app worker -l INFO -B --concurrency=3 --max-memory-per-child=50000 + " depends_on: - - db - - redis - - web - - celery-beat: - build: . - command: celery -A editgroups beat -l info - volumes: - - .:/app - environment: - - DJANGO_SETTINGS_MODULE=editgroups.settings.docker + app: + condition: service_healthy + listener: + <<: *app + command: > + /bin/bash -c " + source ../venv/bin/activate + django-admin listener + " depends_on: - - db - - redis - - web + app: + condition: service_healthy volumes: - postgres_data: + mariadb: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100755 index ac512d0..0000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -set -e - -echo "Waiting for PostgreSQL" -while ! nc -z db 5432; do - sleep 0.1 -done -echo "PostgreSQL started" - -if [ "$RUN_MIGRATIONS" = "true" ]; then - echo "Running migrations" - python manage.py migrate --noinput - - echo "Collecting static files" - python manage.py collectstatic --noinput - - echo "Creating superuser if it doesn't exist" - python manage.py shell -c " -from django.contrib.auth import get_user_model -User = get_user_model() -if not User.objects.filter(username='admin').exists(): - User.objects.create_superuser('admin', 'admin@example.com', 'admin') - print('Superuser created: admin/admin') -else: - print('Superuser already exists') -" || true -fi - -exec "$@" diff --git a/editgroups/settings/docker.py b/editgroups/settings/docker.py index 913c31a..16f17fd 100644 --- a/editgroups/settings/docker.py +++ b/editgroups/settings/docker.py @@ -1,34 +1,26 @@ import os -import sys -from types import ModuleType -secret = ModuleType('editgroups.settings.secret') -secret.SECRET_KEY = '20oj&tj8uaruseitlrise,tries,uirsetur36746209etus7e' -secret.DATABASES = { +SECRET_KEY = 'django-insecure-0ldt)=t=w(%l)7b28n=x!#9ciei^yuox3204(#(r!-fh^59+a-' +DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'editgroups', - 'USER': 'postgres', - 'PASSWORD': 'postgres', - 'HOST': 'db', - 'PORT': '5432', - 'DISABLE_SERVER_SIDE_CURSORS': False, + 'ENGINE': 'django.db.backends.mysql', + 'HOST': 'mariadb', + 'NAME': os.getenv("DB_NAME"), + 'PASSWORD': os.getenv("DB_PASSWORD"), + 'OPTIONS': { + 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", + 'charset': 'utf8mb4', + }, } } -secret.SOCIAL_AUTH_MEDIAWIKI_KEY = 'your_mediawiki_key' -secret.SOCIAL_AUTH_MEDIAWIKI_SECRET = 'your_mediawiki_secret' -secret.SOCIAL_AUTH_MEDIAWIKI_URL = 'https://www.wikidata.org/w/index.php' -secret.SOCIAL_AUTH_MEDIAWIKI_CALLBACK = 'http://localhost:8000/oauth/complete/mediawiki/' -secret.REDIS_HOST = 'redis' -secret.REDIS_PORT = 6379 -secret.REDIS_DB = 0 -secret.REDIS_PASSWORD = '' -secret.REDIS_KEY_PREFIX = 'editgroups_' -sys.modules['editgroups.settings.secret'] = secret +SOCIAL_AUTH_MEDIAWIKI_KEY = 'your_mediawiki_key' +SOCIAL_AUTH_MEDIAWIKI_SECRET = 'your_mediawiki_secret' +SOCIAL_AUTH_MEDIAWIKI_URL = 'https://www.wikidata.org/w/index.php' +SOCIAL_AUTH_MEDIAWIKI_CALLBACK = 'http://localhost:8000/oauth/complete/mediawiki/' -from .common import * - -DEBUG = True -ALLOWED_HOSTS = ['*'] -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) \ No newline at end of file +REDIS_HOST = 'redis' +REDIS_PORT = 6379 +REDIS_DB = 0 +REDIS_PASSWORD = '' +REDIS_KEY_PREFIX = 'editgroups_' From 9aa054167e0a47324294627db90f21db772fd800 Mon Sep 17 00:00:00 2001 From: arcstur Date: Thu, 26 Feb 2026 00:01:51 -0300 Subject: [PATCH 5/5] fix: remove unnecessary file --- .gitignore | 1 + celerybeat-schedule | Bin 16384 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 celerybeat-schedule diff --git a/.gitignore b/.gitignore index 4eff705..2e62f58 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ editgroups/settings/secret.py editgroups/settings/__init__.py static/ celerybeat-schedule.db +celerybeat-schedule docs/_build/ .project diff --git a/celerybeat-schedule b/celerybeat-schedule deleted file mode 100644 index 9e8fd745a64bcaa7483210b990bbf1f20b60fba3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI(&ubGw6bJB0x-p5V^rxU!)I-HcVNpuY3dKWNET|Pl!DX`f##{ubUZTKWxAp0uX=z1Rwwb2tWV=5P$##AOL}*C~(1b8qq?)0S-n0;{f9T z;{f*o>Hz8h>Hz8h>Hz8h>HzA%e^v)Z>|M=9U1V2m@~~jzvRnS*Hkh5?+3??^N4xx_ z*RVaY-m1U>4*B~7c1Fx@r^ZeWHB&{qA2j159zW}Jms8 zQ6uHef%nqiZEb7G{eIBbbv2e`omC#ouo1~v@Qzlsn5VqG?l*WOwK{Q!);7B-zd@Z@ z*PFU73ZkGY%}8>YJmWf-ZtBJ3