internetstandards · mxsasha · Feb 25, 2025 · Feb 24, 2025 · Feb 24, 2025 · Feb 21, 2025
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -80,10 +80,10 @@ jobs:
         id: get_version
         run: |
           pip -q install setuptools_scm
-          # '+' is not supported in Docker Image tags
+          # '+' and '/' is not supported in Docker Image tags
           scm_version=$(echo 'from setuptools_scm import get_version; print(get_version(version_scheme="release-branch-semver"))' | python)
           branch_version_suffix=${GITHUB_HEAD_REF:+-$GITHUB_HEAD_REF}
-          echo "internetnl_version=$scm_version$branch_version_suffix" | tr '+' '-'| tee -a "$GITHUB_OUTPUT"
+          echo "internetnl_version=$scm_version$branch_version_suffix" | tr '+' '-'| tr '/' '-' | tee -a "$GITHUB_OUTPUT"
 
       # login to pull images from Github registry
       - name: Login to GitHub Container Registry
@@ -267,7 +267,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
 
       - name: Debug info
         run: |
@@ -440,7 +440,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
 
       - name: Debug info
         run: |
@@ -523,7 +523,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
 
       - name: Debug info
         run: |
@@ -575,7 +575,7 @@ jobs:
 
       - name: Start development environment
         # disable autoreload in CI as it makes workers unstable
-        run: DEVSERVER_ARGS=--noreload make up env=develop
+        run: INTERNETNL_AUTORELOAD=False make up env=develop
 
       - name: Run development environment tests
         run: make develop-tests
@@ -637,7 +637,7 @@ jobs:
           sudo apt-get update
 
           # upgrade Docker
-          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*
+          sudo apt install --upgrade docker-ce docker-compose-plugin=2.33.0\*
 
       - name: Debug info
         run: |

diff --git a/Makefile b/Makefile
@@ -399,6 +399,7 @@ qa: fix check test
 env ?=
 environment ?= ${env}
 test_args ?= ${testargs}
+services ?= ${service}
 ifeq (${environment},dev)
 	environment = develop
 endif
@@ -440,13 +441,16 @@ docker-compose:
 	${DOCKER_COMPOSE_CMD} ${args}
 
 up docker-compose-up:
-	${DOCKER_COMPOSE_UP_PULL_CMD} up --wait --no-build --remove-orphans ${services}
+	${DOCKER_COMPOSE_UP_PULL_CMD} up --wait --no-build --remove-orphans --timeout=0 ${services}
 	@if [ "${environment}" = "test" ]; then echo -e "\n🚀 Running on http://localhost:8081"; fi
 	@if [ "${environment}" = "develop" ]; then echo -e "\n🚀 Running on http://localhost:8080"; fi
 	@if [ "${environment}" = "batch-test" ]; then echo -e "\n🚀 Running on http://localhost:8081"; fi
 
+up-no-wait:
+	${DOCKER_COMPOSE_UP_PULL_CMD} up --detach --no-build --remove-orphans --timeout=0 ${services}
+
 run docker-compose-run:
-	${DOCKER_COMPOSE_UP_PULL_CMD} up --no-build ${services}
+	${DOCKER_COMPOSE_UP_PULL_CMD} up --watch --remove-orphans --timeout=0 ${services}
 
 restart docker-compose-restart:
 	${DOCKER_COMPOSE_CMD} restart --no-deps ${services}
@@ -512,7 +516,7 @@ stop docker-compose-stop:
 
 # stop and remove all containers, but keep volumes (eg: routinator cache, databases)
 down docker-compose-down:
-	${DOCKER_COMPOSE_CMD} down
+	${DOCKER_COMPOSE_CMD} down --timeout=0
 
 down-remove-volumes docker-compose-down-remove-volumes:
 	${DOCKER_COMPOSE_CMD} down --volumes
@@ -718,3 +722,21 @@ ${TMPDIR}/tranco_list_%k.txt: ${TMPDIR}/tranco_list.txt
 ${TMPDIR}/tranco_list.txt:
 	# download tranco list, unzip, convert from csv to plain list of domains
 	curl -Ls https://tranco-list.eu/download_daily/4Q39X | bsdtar -xOf - | cut -d, -f2 > $@
+
+# convenience target, will build and run all services for development and output logs for the relevant ones
+develop:
+	# (re)build all services
+	${MAKE} build env=develop
+	# bring entire environment up
+	${MAKE} up-no-wait env=develop
+	# wait and log on relevant services
+	${MAKE} run env=develop services='app webserver worker worker-slow worker-nassl'
+	# shut everything down
+	${MAKE} down env=develop
+
+# same as develop, but focus on frontend only
+develop_frontend:
+	# only bring up what is needed for frontend development
+	${MAKE} build run env=develop services='app webserver port-expose'
+	# shut everything down
+	${MAKE} down env=develop
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -219,6 +219,15 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=$RELEASE
 ENTRYPOINT [ "python3", "./manage.py"]
 CMD ["runserver", "0.0.0.0:8080"]
 
+# app image with development tools for better live reloading
+FROM app AS app-dev
+
+# required for compose `develop` mode file sync/copy
+USER root
+
+# browser reload
+COPY --from=linttest-deps /usr/local/lib/python${PYTHON_VERSION}/dist-packages/django_browser_reload /usr/local/lib/python${PYTHON_VERSION}/dist-packages/django_browser_reload
+
 # supplement application with unittest, lint dependencies
 FROM build-app AS linttest
 ARG PYTHON_VERSION

diff --git a/docker/compose.development.yaml b/docker/compose.development.yaml
@@ -20,9 +20,29 @@ services:
       - IPV4_IP_RABBITMQ_INTERNAL
 
   app:
+    build:
+      target: app-dev
+    develop:
+      watch:
+        # auto rebuild/reload when CSS/JS changes
+        - path: ../frontend/
+          action: sync+exec
+          target: /app/frontend
+          exec:
+            command: /bin/sh -c "python3 bin/frontend.py js;python3 bin/frontend.py css;"
     # use Django runserver for better debug abilities during development
-    entrypoint: ["/bin/sh"]
-    command: ["-c", "./manage.py runserver 0.0.0.0:8080 $DEVSERVER_ARGS"]
+    entrypoint: ["/bin/bash"]
+    # run CSS auto rebuild in the background and start devserver
+    command:
+      - "-c"
+      - >
+        if [ "$INTERNETNL_AUTORELOAD" = "True" ]; then
+          ./manage.py runserver 0.0.0.0:8080
+        else
+          ./manage.py runserver 0.0.0.0:8080 --noreload
+        fi;
+    environment:
+      - INTERNETNL_AUTORELOAD
     volumes:
       - batch_results:/app/batch_results
       # mount sources using volumes for quicker dev cycles

diff --git a/docker/defaults.env b/docker/defaults.env
@@ -278,8 +278,8 @@ LANGUAGES=en,nl
 # enable caching, set to off to disable
 NGINX_PROXY_CACHE=default_cache
 
-# used to disable autoreload in CI
-DEVSERVER_ARGS=
-
 # how long Django rendered pages are cached in Redis DB
 PAGE_CACHE_TIME_SECONDS=300
+
+# allows to disable autoreload in CI
+INTERNETNL_AUTORELOAD=False
diff --git a/docker/develop.env b/docker/develop.env
@@ -77,3 +77,6 @@ PAGE_CACHE_TIME_SECONDS=0
 # Routinator is slow to start initially and requires a lot of resources which is not ideal for
 # development environments.
 COMPOSE_PROFILES=monitoring
+
+# automatically reload browser when html/css/js changes
+INTERNETNL_AUTORELOAD=True
diff --git a/docker/port-expose/nginx_templates/port-expose.conf.template b/docker/port-expose/nginx_templates/port-expose.conf.template
@@ -17,6 +17,14 @@ server {
         proxy_pass http://$IPV4_IP_WEBSERVER_INTERNAL/connection/;
     }
 
+    location /__reload__/events/ {
+        proxy_set_header Host ${INTERNETNL_DOMAINNAME};
+        proxy_ssl_name ${INTERNETNL_DOMAINNAME};
+        proxy_ssl_server_name on;
+        proxy_ssl_verify off;
+        proxy_pass https://$IPV4_IP_WEBSERVER_INTERNAL:443;
+        proxy_buffering off;
+    }
 }
 
 server {

diff --git a/docker/webserver/nginx_templates/app.conf.template b/docker/webserver/nginx_templates/app.conf.template
@@ -164,6 +164,22 @@ server {
         proxy_pass http://${IPV4_IP_APP_INTERNAL}:8080;
     }
 
+    # disable buffering on event stream for CSS development
+    location /__reload__/events/ {
+        # disable this location when not in development
+        set $debug "${DEBUG}";
+        if ($debug != "True"){
+            return 404;
+        }
+
+        # same as / above, but only with things required in development
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_pass http://${IPV4_IP_APP_INTERNAL}:8080;
+        # disable buffering for streaming response
+        proxy_buffering off;
+    }
+
     # security.txt file
     location /.well-known/ {
         # basic auth should not apply to this path

diff --git a/documentation/Docker-development-environment.md b/documentation/Docker-development-environment.md
@@ -64,6 +64,28 @@ Multiple services can be specified like so:
 
 Using a Docker runtime with more memory and CPU cores improves rebuild/restart speed.
 
+### Auto reload
+
+Once the development environment is up and running it will autotically rebuild/restart/reload when certain parts of the source code change. Currently changes to the following files will be detected and acted upon:
+
+- Python sources in  `checks/`, `internetnl/`, `interface/`: will reload `app` and `worker*` using Django autoreload, when needed open browser sessions will reload.
+- CSS files in `frontend/css/`: will rebuild CSS files and open browser sessions will reload.
+- Django template files (`.html`) in `interface/templates/`: open browser sessions will reload.
+
+### Foreground development
+
+Instead of running in the background the application stack can also be started in foreground mode using the command:
+
+    make develop
+
+This convenience command will build all relevant services, start the entire stack and show log output for the most relevant services. The Docker Compose application will keep running in the foreground. Terminating it (with `ctrl-c`) will shutdown all running services and bring down the entire application stack.
+
+#### Frontend development
+
+When only developing on frontend parts of the application (eg: `html`, `css`, `javascript`, Django templates/views), a smaller subset of the application can be started using:
+
+    make develop_frontend
+
 ### Running/debugging individual probes
 
 It is possible to run individual probes with debug logging using a manage command. This provides a quick way to iterate over changes regarding test/probe code without having to go through the web interface. For this run the following command:

diff --git a/documentation/Docker-getting-started.md b/documentation/Docker-getting-started.md
@@ -58,7 +58,8 @@ There are 2 environment for development, the "development" and "integration test
 | Server              | Django runserver          | uWSGI                        | uWSGI                     |
 | Python source files | Mounted                   | from build image             | from build image          |
 | Website             | http://localhost:8080     | http://localhost:8081        | http://localhost:8081     |
-| Autoreload          | `.py` files               | No                           | No                        |
+| Autoreload          | `.py`, `css`, `js` and    | No                           | No                        |
+|                     | `html` files              |                              |                           |
 | Batch API enabled   | Yes                       | No                           | Yes                       |
 | Tests               | Yes                       | Yes                          | Yes                       |
 | Test command        | `make develop-tests`      | `make integration-tests`     | `make batch-tests`        |

diff --git a/internetnl/settings.py b/internetnl/settings.py
@@ -117,6 +117,8 @@
 
 ENABLE_HOF = get_boolean_env("ENABLE_HOF", True)
 
+AUTORELOAD = get_boolean_env("INTERNETNL_AUTORELOAD", False)
+
 # -- End of manual configuration
 
 """
@@ -141,6 +143,8 @@
     "django_hosts",
     "django_statsd",
 ]
+if AUTORELOAD:
+    INSTALLED_APPS += ["django_browser_reload"]
 
 TEMPLATES = [
     {
@@ -173,6 +177,10 @@
     "django_hosts.middleware.HostsResponseMiddleware",
     "internetnl.custom_middlewares.ActivateTranslationMiddleware",
 ]
+if AUTORELOAD:
+    MIDDLEWARE += [
+        "django_browser_reload.middleware.BrowserReloadMiddleware",
+    ]
 
 ROOT_URLCONF = "internetnl.urls"
 ROOT_HOSTCONF = "internetnl.hosts"

diff --git a/internetnl/urls.py b/internetnl/urls.py
@@ -1,8 +1,13 @@
 # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors
 # SPDX-License-Identifier: Apache-2.0
 from django.urls import include, path
+from django.conf import settings
 
 urlpatterns = [
     path("", include("interface.urls")),
 ]
+
+if settings.AUTORELOAD:
+    urlpatterns += [path("__reload__/", include("django_browser_reload.urls"))]
+
 handler404 = "interface.views.page404"
diff --git a/requirements-dev.in b/requirements-dev.in
@@ -22,4 +22,7 @@ black
 pylama
 
 # requirements management
-pip-tools
+pip-tools
+
+# for automatic css reload
+django-browser-reload