Add external alerting via Webhook, Slack, and PagerDuty notification adapters

.credo.exs

@@ -130,7 +130,7 @@
           {Credo.Check.Refactor.MatchInCondition, []},
           {Credo.Check.Refactor.NegatedConditionsInUnless, []},
           {Credo.Check.Refactor.NegatedConditionsWithElse, []},
-          {Credo.Check.Refactor.Nesting, []},
+          {Credo.Check.Refactor.Nesting, max_nesting: 3},
           {Credo.Check.Refactor.RedundantWithClauseResult, []},
           {Credo.Check.Refactor.RejectReject, []},
           {Credo.Check.Refactor.UnlessWithElse, []},

README.md

@@ -38,6 +38,10 @@ Upon deployment, the following dashboard becomes available, providing easy acces
   - Supports wildcard domain certificates (e.g. `*.example.com`) through configurable DNS providers (rout353 and cloudflare).
   - Supports automatic certificate import/export workflows for cloud providers such as AWS ACM/Route53.
   - Automatically renews certificates before expiration with configurable renewal thresholds and validation intervals.
+- Supports external alerting for deployment lifecycle events via configurable notification channels:
+  - Webhook — generic HTTP POST to any endpoint (n8n, Zapier, custom scripts, etc.)
+  - Slack — formatted Incoming Webhook messages with per-event emoji and mrkdwn.
+  - PagerDuty — Events API v2 incidents with automatic severity mapping.
 - Supports local environments for non-cloud deployments
 - Provides rollback functionality if a monitored app version remains unstable for 10 minutes (time configurable).
 - Rolled-back monitored app versions are ghosted, preventing their redeployment.
@@ -78,6 +82,7 @@ Since OTP distribution is heavily used between the DeployEx and Monitored Applic
 
 | DeployEx version                                                          | <img src="https://img.shields.io/badge/OTP-27-green.svg"/> | <img src="https://img.shields.io/badge/OTP-28-green.svg"/> |
 | ------------------------------------------------------------------------- | ---------------------------------------------------------- | ---------------------------------------------------------- |
+| :soon: [**0.9.4**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.4) | **27.3.4.13**                                              | **28.5.0.2**                                               |
 | [**0.9.3**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.3) | **27.3.4.13**                                              | **28.5.0.2**                                               |
 | [**0.9.2**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.2) | **27.3.4.13**                                              | **28.5.0.2**                                               |
 | [**0.9.1**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.1) | **27.3.4.9**                                               | **28.4.1**                                                 |

apps/deployer/lib/deployer/application.ex

@@ -29,6 +29,7 @@ defmodule Deployer.Application do
       {:ok, _pid} = response ->
         Deployer.Monitor.init_all_monitor_supervisors()
         init_all_workers()
+        notify_startup()
         response
 
       {:error, reason} = response ->
@@ -37,16 +38,44 @@ defmodule Deployer.Application do
     end
   end
 
+  @impl true
+  def stop(_state) do
+    notify_shutdown()
+  end
+
   if_not_test do
     alias Deployer.Engine
 
     defp application_servers, do: [Deployer.Status.Application]
 
     defp init_all_workers, do: Engine.init_all_workers()
+
+    defp notify_startup do
+      version = Application.spec(:deployer, :vsn) |> to_string()
+
+      Foundation.Notifications.notify(:deployment_started, %{
+        node: node(),
+        sname: "deployex",
+        version: version
+      })
+    end
+
+    defp notify_shutdown do
+      Foundation.Notifications.notify(:deployment_complete, %{
+        node: node(),
+        sname: "deployex",
+        status: :ok,
+        message: "shutdown"
+      })
+    end
   else
     defp application_servers, do: []
 
     defp init_all_workers, do: :ok
+
+    defp notify_startup, do: :ok
+
+    defp notify_shutdown, do: :ok
   end
 
   defp gcp_app_credentials do

apps/deployer/lib/deployer/deployex.ex

@@ -19,6 +19,13 @@ defmodule Deployer.Deployex do
   def force_terminate(sleep_time) do
     Logger.warning("Deployex was requested to terminate, see you soon!!!")
 
+    Foundation.Notifications.notify(:deployment_complete, %{
+      node: node(),
+      sname: "deployex",
+      status: :ok,
+      message: "forced terminate"
+    })
+
     :timer.sleep(sleep_time)
 
     deployex_path = Application.fetch_env!(:foundation, :install_path)

apps/deployer/lib/deployer/hot_upgrade/application.ex

@@ -594,15 +594,31 @@ defmodule Deployer.HotUpgrade.Application do
       @events_topic,
       {:hot_upgrade_complete, Node.self(), sname, :ok, "Hot upgrade applied successfully!"}
     )
+
+    Foundation.Notifications.notify(:deployment_complete, %{
+      node: Node.self(),
+      sname: sname,
+      status: :ok,
+      message: "Hot upgrade applied successfully!"
+    })
   end
 
   @spec notify_error(sname :: String.t(), result :: any()) :: :ok
   defp notify_error(sname, result) do
+    message = "Upgrade failed: #{inspect(result)}"
+
     Phoenix.PubSub.broadcast(
       Deployer.PubSub,
       @events_topic,
-      {:hot_upgrade_complete, Node.self(), sname, :error, "Upgrade failed: #{inspect(result)}"}
+      {:hot_upgrade_complete, Node.self(), sname, :error, message}
     )
+
+    Foundation.Notifications.notify(:deployment_complete, %{
+      node: Node.self(),
+      sname: sname,
+      status: :error,
+      message: message
+    })
   end
 
   defp check_jellyfish_files(files, from_version, to_version) do

apps/deployer/lib/deployer/monitor/application.ex

@@ -157,6 +157,14 @@ defmodule Deployer.Monitor.Application do
     # Update the number of crash restarts
     crash_restart_count = state.crash_restart_count + 1
 
+    Foundation.Notifications.notify(:crash_restart, %{
+      node: Node.self(),
+      sname: state.sname,
+      name: state.name,
+      language: state.language,
+      crash_restart_count: crash_restart_count
+    })
+
     # Retry with backoff pattern
     trigger_run_service(state.sname, 2 * crash_restart_count * 1000)
 
@@ -243,6 +251,12 @@ defmodule Deployer.Monitor.Application do
     version = version_map.version
 
     notify_new_deploy = fn ->
+      Foundation.Notifications.notify(:deployment_started, %{
+        node: Node.self(),
+        sname: sname,
+        version: version
+      })
+
       Phoenix.PubSub.broadcast(
         Deployer.PubSub,
         @new_deploy_topic,

apps/deployex_web/test/deployex_web/ui_settings_test.exs

@@ -1,5 +1,5 @@
 defmodule DeployexWeb.UiSettingsTest do
-  use DeployexWeb.ConnCase, async: true
+  use DeployexWeb.ConnCase, async: false
 
   import DeployexWeb.AccountsFixtures
 

apps/foundation/lib/config_provider/env/config.ex

@@ -37,6 +37,7 @@ defmodule Foundation.ConfigProvider.Env.Config do
             {:applications, yaml_config.applications},
             {:config_checksum, yaml_config.config_checksum},
             {:monitoring, yaml_config.monitoring},
+            {:notifications, yaml_config.notifications},
             {:logs_retention_time_ms, yaml_config.logs_retention_time_ms},
             {:install_path, yaml_config.install_path},
             {:var_path, yaml_config.var_path},

apps/foundation/lib/foundation/application.ex

@@ -10,9 +10,12 @@ defmodule Foundation.Application do
     children =
       [
         {Phoenix.PubSub, name: Foundation.PubSub},
+        {Finch, name: Foundation.Finch},
         Foundation.Catalog.Local,
         Foundation.Certificates.Manager.Supervisor,
-        Foundation.Certificate
+        Foundation.Certificate,
+        Foundation.Notifications.Supervisor,
+        Foundation.Notifications
       ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html

apps/foundation/lib/foundation/certificates/manager/manager.ex

@@ -80,11 +80,22 @@ defmodule Foundation.Certificates.Manager do
           "Certificate check completed successfully for app: #{state.app_name}, domains: #{inspect(certificate.domains)}"
         )
 
+        Foundation.Notifications.notify(:certificate_renewed, %{
+          app_name: state.app_name,
+          domains: state.domains
+        })
+
       {:retry_after, seconds} when seconds >= 0 ->
         Logger.warning("Acme for #{state.app_name} requested retry after #{seconds} s")
 
       {:error, reason} ->
         Logger.error("Certificate renewal check failed: #{inspect(reason)}")
+
+        Foundation.Notifications.notify(:certificate_failed, %{
+          app_name: state.app_name,
+          domains: state.domains,
+          reason: inspect(reason)
+        })
     end
 
     _ = Process.send_after(self(), :check_and_renew, state.certificate_check_interval_ms)

apps/foundation/lib/foundation/notifications.ex

@@ -0,0 +1,156 @@
+defmodule Foundation.Notifications do
+  @moduledoc """
+  Dispatches deployment lifecycle events to configured external notification channels.
+
+  `Foundation.Notifications.notify/2` broadcasts an event to a per-event
+  `Foundation.PubSub` topic (e.g. `"deployex::notifications::crash_restart"`).
+  Each `Foundation.Notifications.Worker` — one per entry in the `notifications:`
+  YAML list — subscribes only to the topics matching its `events` list at startup,
+  so PubSub routing handles delivery without any filtering in the worker.  This means:
+
+  - **Callers are never blocked** — `Phoenix.PubSub.broadcast/3` returns immediately.
+  - **Channels are isolated** — a slow or crashing adapter only affects its own worker.
+  - **Workers are supervised** — `Foundation.Notifications.Supervisor` restarts a
+    crashed worker automatically.
+
+  ## Supported events
+
+  | Event atom                       | Trigger                                                    | Payload keys                                                                |
+  |----------------------------------|------------------------------------------------------------|-----------------------------------------------------------------------------|
+  | `:crash_restart`                 | Monitored app crashed and is being restarted               | `node`, `sname`, `name`, `language`, `crash_restart_count`                 |
+  | `:deployment_started`            | New deployment was initiated for an sname                  | `node`, `sname`, `version`                                                  |
+  | `:deployment_complete`           | Hot-upgrade finished (success or failure)                  | `node`, `sname`, `status` (`:ok`/`:error`), `message`                      |
+  | `:watchdog_threshold_exceeded`   | Watchdog exceeded a resource threshold and restarted an app| `node`, `sname`, `type`, `current_percentage`, `restart_threshold_percent` |
+  | `:certificate_renewed`           | TLS certificate was successfully renewed                   | `app_name`, `domains`                                                       |
+  | `:certificate_failed`            | TLS certificate renewal failed                             | `app_name`, `domains`, `reason`                                             |
+
+  ## Available adapters
+
+  | YAML value    | Module                                | Description                           |
+  |---------------|---------------------------------------|---------------------------------------|
+  | `"webhook"`   | `Foundation.Notifications.Webhook`   | Generic HTTP POST with a JSON body    |
+  | `"slack"`     | `Foundation.Notifications.Slack`     | Slack Incoming Webhook message        |
+  | `"pagerduty"` | `Foundation.Notifications.PagerDuty` | PagerDuty Events API v2 incident      |
+
+  ## Configuration (deployex.yaml)
+
+  Multiple adapters can run in parallel.  Each entry is independent and subscribes
+  to its own subset of events.
+
+      notifications:
+        - adapter: "slack"
+          url: "https://hooks.slack.com/services/T.../B.../..."
+          enabled: true
+          events:
+            - "crash_restart"
+            - "deployment_complete"
+            - "watchdog_threshold_exceeded"
+          options:
+            username: "DeployEx"
+            icon_emoji: ":rocket:"
+
+        - adapter: "pagerduty"
+          enabled: true
+          events:
+            - "crash_restart"
+            - "watchdog_threshold_exceeded"
+            - "certificate_failed"
+          options:
+            routing_key: "abc123def456..."
+
+        - adapter: "webhook"
+          url: "https://internal.example.com/hooks/deployex"
+          enabled: true
+          events:
+            - "crash_restart"
+            - "deployment_started"
+            - "deployment_complete"
+            - "watchdog_threshold_exceeded"
+            - "certificate_renewed"
+            - "certificate_failed"
+
+  ## Adding a new adapter
+
+  Implement `Foundation.Notifications.Adapter` and add a new clause to
+  `Foundation.Yaml.notification_adapter/1`.  See `Foundation.Notifications.Adapter`
+  for a step-by-step guide and a skeleton implementation.
+  """
+
+  @topic_prefix "deployex::notifications"
+
+  use GenServer
+
+  import Foundation.Macros
+
+  require Logger
+
+  ### ==========================================================================
+  ### Callback functions
+  ### ==========================================================================
+  def start_link(args) do
+    GenServer.start_link(__MODULE__, args, name: __MODULE__)
+  end
+
+  @impl true
+  def init(_args) do
+    Logger.info("Initializing Notification Server")
+    {:ok, %{}, {:continue, :start_notification_manager}}
+  end
+
+  @impl true
+  def handle_continue(:start_notification_manager, state) do
+    initialize_notification_manager()
+    {:noreply, state}
+  end
+
+  @doc """
+  Returns the PubSub topic for a specific event.
+
+  Each `Foundation.Notifications.Worker` subscribes to `topic(event)` for
+  every event in its list.  `notify/2` broadcasts to the same per-event topic,
+  so PubSub routing delivers the message only to workers that subscribed to it.
+
+  ## Example
+
+      iex> Foundation.Notifications.topic(:crash_restart)
+      "deployex::notifications::crash_restart"
+  """
+  @spec topic(event :: atom()) :: String.t()
+  def topic(event), do: "#{@topic_prefix}::#{event}"
+
+  @doc """
+  Broadcasts `event` with `payload` to all workers subscribed to that event.
+
+  Returns `:ok` immediately; delivery to each adapter happens asynchronously
+  inside the respective `Foundation.Notifications.Worker` process.
+  """
+  @spec notify(event :: atom(), payload :: map()) :: :ok
+  def notify(event, payload) do
+    Phoenix.PubSub.broadcast(Foundation.PubSub, topic(event), {event, payload})
+  end
+
+  ### ==========================================================================
+  ### Private Functions
+  ### ==========================================================================
+
+  if_not_test do
+    alias Foundation.Notifications.Worker
+
+    def initialize_notification_manager do
+      :foundation
+      |> Application.fetch_env!(:notifications)
+      |> Enum.map(&to_notification_struct/1)
+      |> Enum.map(&Foundation.Notifications.Supervisor.start_notification_worker/1)
+    end
+
+    defp to_notification_struct(%Foundation.Yaml.Notification{} = config) do
+      struct!(Worker, Map.from_struct(config))
+    end
+
+    defp to_notification_struct(config) do
+      struct!(Worker, config)
+    end
+  else
+    defp initialize_notification_manager, do: :ok
+  end
+end