From 797f8a68248f8fb78e2187fbae2f5e546f47b58a Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Mon, 24 Feb 2025 19:36:16 +0100 Subject: [PATCH 1/3] Support '/' in branches --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b8c125b91..bd5511253 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -80,10 +80,10 @@ jobs: id: get_version run: | pip -q install setuptools_scm - # '+' is not supported in Docker Image tags + # '+' and '/' is not supported in Docker Image tags scm_version=$(echo 'from setuptools_scm import get_version; print(get_version(version_scheme="release-branch-semver"))' | python) branch_version_suffix=${GITHUB_HEAD_REF:+-$GITHUB_HEAD_REF} - echo "internetnl_version=$scm_version$branch_version_suffix" | tr '+' '-'| tee -a "$GITHUB_OUTPUT" + echo "internetnl_version=$scm_version$branch_version_suffix" | tr '+' '-'| tr '/' '-' | tee -a "$GITHUB_OUTPUT" # login to pull images from Github registry - name: Login to GitHub Container Registry From d60d670667d3a1432e5a75d8f698fc2cc29c4497 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Mon, 24 Feb 2025 17:27:28 +0100 Subject: [PATCH 2/3] Document frontend development --- Makefile | 28 +++++++++++++++++-- .../Docker-development-environment.md | 22 +++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9b72ed779..f66a88c6d 100644 --- a/Makefile +++ b/Makefile @@ -399,6 +399,7 @@ qa: fix check test env ?= environment ?= ${env} test_args ?= ${testargs} +services ?= ${service} ifeq (${environment},dev) environment = develop endif @@ -440,13 +441,16 @@ docker-compose: ${DOCKER_COMPOSE_CMD} ${args} up docker-compose-up: - ${DOCKER_COMPOSE_UP_PULL_CMD} up --wait --no-build --remove-orphans ${services} + ${DOCKER_COMPOSE_UP_PULL_CMD} up --wait --no-build --remove-orphans --timeout=0 ${services} @if [ "${environment}" = "test" ]; then echo -e "\nšŸš€ Running on http://localhost:8081"; fi @if [ "${environment}" = "develop" ]; then echo -e "\nšŸš€ Running on http://localhost:8080"; fi @if [ "${environment}" = "batch-test" ]; then echo -e "\nšŸš€ Running on http://localhost:8081"; fi +up-no-wait: + ${DOCKER_COMPOSE_UP_PULL_CMD} up --detach --no-build --remove-orphans --timeout=0 ${services} + run docker-compose-run: - ${DOCKER_COMPOSE_UP_PULL_CMD} up --no-build ${services} + ${DOCKER_COMPOSE_UP_PULL_CMD} up --watch --remove-orphans --timeout=0 ${services} restart docker-compose-restart: ${DOCKER_COMPOSE_CMD} restart --no-deps ${services} @@ -512,7 +516,7 @@ stop docker-compose-stop: # stop and remove all containers, but keep volumes (eg: routinator cache, databases) down docker-compose-down: - ${DOCKER_COMPOSE_CMD} down + ${DOCKER_COMPOSE_CMD} down --timeout=0 down-remove-volumes docker-compose-down-remove-volumes: ${DOCKER_COMPOSE_CMD} down --volumes @@ -718,3 +722,21 @@ ${TMPDIR}/tranco_list_%k.txt: ${TMPDIR}/tranco_list.txt ${TMPDIR}/tranco_list.txt: # download tranco list, unzip, convert from csv to plain list of domains curl -Ls https://tranco-list.eu/download_daily/4Q39X | bsdtar -xOf - | cut -d, -f2 > $@ + +# convenience target, will build and run all services for development and output logs for the relevant ones +develop: + # (re)build all services + ${MAKE} build env=develop + # bring entire environment up + ${MAKE} up-no-wait env=develop + # wait and log on relevant services + ${MAKE} run env=develop services='app webserver worker worker-slow worker-nassl' + # shut everything down + ${MAKE} down env=develop + +# same as develop, but focus on frontend only +develop_frontend: + # only bring up what is needed for frontend development + ${MAKE} build run env=develop services='app webserver port-expose' + # shut everything down + ${MAKE} down env=develop diff --git a/documentation/Docker-development-environment.md b/documentation/Docker-development-environment.md index bcf08ce20..793fdc75c 100644 --- a/documentation/Docker-development-environment.md +++ b/documentation/Docker-development-environment.md @@ -64,6 +64,28 @@ Multiple services can be specified like so: Using a Docker runtime with more memory and CPU cores improves rebuild/restart speed. +### Auto reload + +Once the development environment is up and running it will autotically rebuild/restart/reload when certain parts of the source code change. Currently changes to the following files will be detected and acted upon: + +- Python sources in `checks/`, `internetnl/`, `interface/`: will reload `app` and `worker*` using Django autoreload, when needed open browser sessions will reload. +- CSS files in `frontend/css/`: will rebuild CSS files and open browser sessions will reload. +- Django template files (`.html`) in `interface/templates/`: open browser sessions will reload. + +### Foreground development + +Instead of running in the background the application stack can also be started in foreground mode using the command: + + make develop + +This convenience command will build all relevant services, start the entire stack and show log output for the most relevant services. The Docker Compose application will keep running in the foreground. Terminating it (with `ctrl-c`) will shutdown all running services and bring down the entire application stack. + +#### Frontend development + +When only developing on frontend parts of the application (eg: `html`, `css`, `javascript`, Django templates/views), a smaller subset of the application can be started using: + + make develop_frontend + ### Running/debugging individual probes It is possible to run individual probes with debug logging using a manage command. This provides a quick way to iterate over changes regarding test/probe code without having to go through the web interface. For this run the following command: From 939675426c9b117d574cf0e40ce3ac330bd1e315 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Fri, 21 Feb 2025 19:32:14 +0100 Subject: [PATCH 3/3] Auto reload when CSS/JS changes in development --- .github/workflows/docker.yml | 10 +-- docker/Dockerfile | 9 ++ docker/compose.development.yaml | 24 +++++- docker/defaults.env | 6 +- docker/develop.env | 3 + .../nginx_templates/port-expose.conf.template | 8 ++ .../nginx_templates/app.conf.template | 16 ++++ documentation/Docker-getting-started.md | 3 +- internetnl/settings.py | 8 ++ internetnl/urls.py | 5 ++ requirements-dev.in | 5 +- requirements-dev.txt | 83 +++++++++++-------- 12 files changed, 134 insertions(+), 46 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bd5511253..be2fcc284 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -267,7 +267,7 @@ jobs: sudo apt-get update # upgrade Docker - sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\* + sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\* - name: Debug info run: | @@ -440,7 +440,7 @@ jobs: sudo apt-get update # upgrade Docker - sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\* + sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\* - name: Debug info run: | @@ -523,7 +523,7 @@ jobs: sudo apt-get update # upgrade Docker - sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\* + sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\* - name: Debug info run: | @@ -575,7 +575,7 @@ jobs: - name: Start development environment # disable autoreload in CI as it makes workers unstable - run: DEVSERVER_ARGS=--noreload make up env=develop + run: INTERNETNL_AUTORELOAD=False make up env=develop - name: Run development environment tests run: make develop-tests @@ -637,7 +637,7 @@ jobs: sudo apt-get update # upgrade Docker - sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\* + sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\* - name: Debug info run: | diff --git a/docker/Dockerfile b/docker/Dockerfile index e67859661..e81a2d728 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -219,6 +219,15 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=$RELEASE ENTRYPOINT [ "python3", "./manage.py"] CMD ["runserver", "0.0.0.0:8080"] +# app image with development tools for better live reloading +FROM app AS app-dev + +# required for compose `develop` mode file sync/copy +USER root + +# browser reload +COPY --from=linttest-deps /usr/local/lib/python${PYTHON_VERSION}/dist-packages/django_browser_reload /usr/local/lib/python${PYTHON_VERSION}/dist-packages/django_browser_reload + # supplement application with unittest, lint dependencies FROM build-app AS linttest ARG PYTHON_VERSION diff --git a/docker/compose.development.yaml b/docker/compose.development.yaml index b967519ab..5042f27ae 100644 --- a/docker/compose.development.yaml +++ b/docker/compose.development.yaml @@ -20,9 +20,29 @@ services: - IPV4_IP_RABBITMQ_INTERNAL app: + build: + target: app-dev + develop: + watch: + # auto rebuild/reload when CSS/JS changes + - path: ../frontend/ + action: sync+exec + target: /app/frontend + exec: + command: /bin/sh -c "python3 bin/frontend.py js;python3 bin/frontend.py css;" # use Django runserver for better debug abilities during development - entrypoint: ["/bin/sh"] - command: ["-c", "./manage.py runserver 0.0.0.0:8080 $DEVSERVER_ARGS"] + entrypoint: ["/bin/bash"] + # run CSS auto rebuild in the background and start devserver + command: + - "-c" + - > + if [ "$INTERNETNL_AUTORELOAD" = "True" ]; then + ./manage.py runserver 0.0.0.0:8080 + else + ./manage.py runserver 0.0.0.0:8080 --noreload + fi; + environment: + - INTERNETNL_AUTORELOAD volumes: - batch_results:/app/batch_results # mount sources using volumes for quicker dev cycles diff --git a/docker/defaults.env b/docker/defaults.env index 01075f829..d562212f6 100644 --- a/docker/defaults.env +++ b/docker/defaults.env @@ -278,8 +278,8 @@ LANGUAGES=en,nl # enable caching, set to off to disable NGINX_PROXY_CACHE=default_cache -# used to disable autoreload in CI -DEVSERVER_ARGS= - # how long Django rendered pages are cached in Redis DB PAGE_CACHE_TIME_SECONDS=300 + +# allows to disable autoreload in CI +INTERNETNL_AUTORELOAD=False diff --git a/docker/develop.env b/docker/develop.env index 52cc4508c..87e58bf07 100644 --- a/docker/develop.env +++ b/docker/develop.env @@ -77,3 +77,6 @@ PAGE_CACHE_TIME_SECONDS=0 # Routinator is slow to start initially and requires a lot of resources which is not ideal for # development environments. COMPOSE_PROFILES=monitoring + +# automatically reload browser when html/css/js changes +INTERNETNL_AUTORELOAD=True diff --git a/docker/port-expose/nginx_templates/port-expose.conf.template b/docker/port-expose/nginx_templates/port-expose.conf.template index cdc7e22c4..314bc6e6a 100644 --- a/docker/port-expose/nginx_templates/port-expose.conf.template +++ b/docker/port-expose/nginx_templates/port-expose.conf.template @@ -17,6 +17,14 @@ server { proxy_pass http://$IPV4_IP_WEBSERVER_INTERNAL/connection/; } + location /__reload__/events/ { + proxy_set_header Host ${INTERNETNL_DOMAINNAME}; + proxy_ssl_name ${INTERNETNL_DOMAINNAME}; + proxy_ssl_server_name on; + proxy_ssl_verify off; + proxy_pass https://$IPV4_IP_WEBSERVER_INTERNAL:443; + proxy_buffering off; + } } server { diff --git a/docker/webserver/nginx_templates/app.conf.template b/docker/webserver/nginx_templates/app.conf.template index bc49f227c..c1ea8f177 100644 --- a/docker/webserver/nginx_templates/app.conf.template +++ b/docker/webserver/nginx_templates/app.conf.template @@ -164,6 +164,22 @@ server { proxy_pass http://${IPV4_IP_APP_INTERNAL}:8080; } + # disable buffering on event stream for CSS development + location /__reload__/events/ { + # disable this location when not in development + set $debug "${DEBUG}"; + if ($debug != "True"){ + return 404; + } + + # same as / above, but only with things required in development + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://${IPV4_IP_APP_INTERNAL}:8080; + # disable buffering for streaming response + proxy_buffering off; + } + # security.txt file location /.well-known/ { # basic auth should not apply to this path diff --git a/documentation/Docker-getting-started.md b/documentation/Docker-getting-started.md index 88103a75c..b71129360 100644 --- a/documentation/Docker-getting-started.md +++ b/documentation/Docker-getting-started.md @@ -58,7 +58,8 @@ There are 2 environment for development, the "development" and "integration test | Server | Django runserver | uWSGI | uWSGI | | Python source files | Mounted | from build image | from build image | | Website | http://localhost:8080 | http://localhost:8081 | http://localhost:8081 | -| Autoreload | `.py` files | No | No | +| Autoreload | `.py`, `css`, `js` and | No | No | +| | `html` files | | | | Batch API enabled | Yes | No | Yes | | Tests | Yes | Yes | Yes | | Test command | `make develop-tests` | `make integration-tests` | `make batch-tests` | diff --git a/internetnl/settings.py b/internetnl/settings.py index 3f9923f10..d154a2842 100644 --- a/internetnl/settings.py +++ b/internetnl/settings.py @@ -117,6 +117,8 @@ ENABLE_HOF = get_boolean_env("ENABLE_HOF", True) +AUTORELOAD = get_boolean_env("INTERNETNL_AUTORELOAD", False) + # -- End of manual configuration """ @@ -141,6 +143,8 @@ "django_hosts", "django_statsd", ] +if AUTORELOAD: + INSTALLED_APPS += ["django_browser_reload"] TEMPLATES = [ { @@ -173,6 +177,10 @@ "django_hosts.middleware.HostsResponseMiddleware", "internetnl.custom_middlewares.ActivateTranslationMiddleware", ] +if AUTORELOAD: + MIDDLEWARE += [ + "django_browser_reload.middleware.BrowserReloadMiddleware", + ] ROOT_URLCONF = "internetnl.urls" ROOT_HOSTCONF = "internetnl.hosts" diff --git a/internetnl/urls.py b/internetnl/urls.py index 8cf81c7d1..6e8e2b805 100644 --- a/internetnl/urls.py +++ b/internetnl/urls.py @@ -1,8 +1,13 @@ # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors # SPDX-License-Identifier: Apache-2.0 from django.urls import include, path +from django.conf import settings urlpatterns = [ path("", include("interface.urls")), ] + +if settings.AUTORELOAD: + urlpatterns += [path("__reload__/", include("django_browser_reload.urls"))] + handler404 = "interface.views.page404" diff --git a/requirements-dev.in b/requirements-dev.in index 5c634c35b..9126cb5cb 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -22,4 +22,7 @@ black pylama # requirements management -pip-tools \ No newline at end of file +pip-tools + +# for automatic css reload +django-browser-reload diff --git a/requirements-dev.txt b/requirements-dev.txt index 9ef703467..4a25c7aee 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,14 @@ # # pip-compile requirements-dev.in # -attrs==23.1.0 +asgiref==3.8.1 # via - # -c requirements.txt + # -c /source/requirements.txt + # django + # django-browser-reload +attrs==24.3.0 + # via + # -c /source/requirements.txt # pytest autoflake==2.0.1 # via -r requirements-dev.in @@ -14,42 +19,48 @@ black==23.1.0 # via -r requirements-dev.in build==0.10.0 # via pip-tools -certifi==2023.7.22 +certifi==2024.12.14 # via - # -c requirements.txt + # -c /source/requirements.txt # requests -charset-normalizer==3.2.0 +charset-normalizer==3.4.1 # via - # -c requirements.txt + # -c /source/requirements.txt # requests -click==8.1.7 +click==8.1.8 # via - # -c requirements.txt + # -c /source/requirements.txt # black # pip-tools coverage[toml]==7.1.0 # via # -r requirements-dev.in # pytest-cov +django==4.2.18 + # via + # -c /source/requirements.txt + # django-browser-reload +django-browser-reload==1.18.0 + # via -r requirements-dev.in django-coverage==1.2.4 # via -r requirements-dev.in freezegun==1.2.2 # via -r requirements-dev.in -idna==3.7 +idna==3.10 # via - # -c requirements.txt + # -c /source/requirements.txt # requests iniconfig==2.0.0 # via - # -c requirements.txt + # -c /source/requirements.txt # pytest mccabe==0.7.0 # via pylama mypy-extensions==1.0.0 # via black -packaging==23.1 +packaging==24.2 # via - # -c requirements.txt + # -c /source/requirements.txt # black # build # pytest @@ -59,13 +70,13 @@ pip-tools==6.14.0 # via -r requirements-dev.in platformdirs==3.0.0 # via black -pluggy==1.2.0 +pluggy==1.5.0 # via - # -c requirements.txt + # -c /source/requirements.txt # pytest py==1.11.0 # via - # -c requirements.txt + # -c /source/requirements.txt # pytest # pytest-html pycodestyle==2.10.0 @@ -82,7 +93,7 @@ pyproject-hooks==1.0.0 # via build pytest==6.2.5 # via - # -c requirements.txt + # -c /source/requirements.txt # -r requirements-dev.in # pytest-base-url # pytest-cov @@ -116,13 +127,13 @@ pytest-selenium==3.0.0 # via -r requirements-dev.in pytest-variables==1.9.0 # via pytest-selenium -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via - # -c requirements.txt + # -c /source/requirements.txt # freezegun -requests==2.32.0 +requests==2.31.0 # via - # -c requirements.txt + # -c /source/requirements.txt # pytest-base-url # pytest-selenium # requests-mock @@ -133,47 +144,51 @@ responses==0.22.0 # via pytest-responses selenium==3.141.0 # via - # -c requirements.txt + # -c /source/requirements.txt # pytest-selenium -six==1.16.0 +six==1.17.0 # via - # -c requirements.txt + # -c /source/requirements.txt # python-dateutil # requests-mock # tenacity snowballstemmer==2.2.0 # via pydocstyle +sqlparse==0.5.3 + # via + # -c /source/requirements.txt + # django tenacity==6.3.1 # via pytest-selenium toml==0.10.2 # via - # -c requirements.txt + # -c /source/requirements.txt # pytest # responses -tomli==2.0.1 +tomli==2.2.1 # via - # -c requirements.txt + # -c /source/requirements.txt # autoflake # black # build # coverage # pip-tools - # pyproject-hooks types-toml==0.10.8.3 # via responses -typing-extensions==4.7.1 +typing-extensions==4.12.2 # via - # -c requirements.txt + # -c /source/requirements.txt + # asgiref # black -urllib3==2.0.7 +urllib3==2.3.0 # via - # -c requirements.txt + # -c /source/requirements.txt # requests # responses # selenium -wheel==0.41.2 +wheel==0.45.1 # via - # -c requirements.txt + # -c /source/requirements.txt # pip-tools # The following packages are considered to be unsafe in a requirements file: