diff --git a/.credo.exs b/.credo.exs index cd5c809e..2ac6e8e3 100644 --- a/.credo.exs +++ b/.credo.exs @@ -130,7 +130,7 @@ {Credo.Check.Refactor.MatchInCondition, []}, {Credo.Check.Refactor.NegatedConditionsInUnless, []}, {Credo.Check.Refactor.NegatedConditionsWithElse, []}, - {Credo.Check.Refactor.Nesting, []}, + {Credo.Check.Refactor.Nesting, max_nesting: 3}, {Credo.Check.Refactor.RedundantWithClauseResult, []}, {Credo.Check.Refactor.RejectReject, []}, {Credo.Check.Refactor.UnlessWithElse, []}, diff --git a/README.md b/README.md index 3424b35f..3ee8cff7 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,10 @@ Upon deployment, the following dashboard becomes available, providing easy acces - Supports wildcard domain certificates (e.g. `*.example.com`) through configurable DNS providers (rout353 and cloudflare). - Supports automatic certificate import/export workflows for cloud providers such as AWS ACM/Route53. - Automatically renews certificates before expiration with configurable renewal thresholds and validation intervals. +- Supports external alerting for deployment lifecycle events via configurable notification channels: + - Webhook — generic HTTP POST to any endpoint (n8n, Zapier, custom scripts, etc.) + - Slack — formatted Incoming Webhook messages with per-event emoji and mrkdwn. + - PagerDuty — Events API v2 incidents with automatic severity mapping. - Supports local environments for non-cloud deployments - Provides rollback functionality if a monitored app version remains unstable for 10 minutes (time configurable). - Rolled-back monitored app versions are ghosted, preventing their redeployment. @@ -78,6 +82,7 @@ Since OTP distribution is heavily used between the DeployEx and Monitored Applic | DeployEx version | | | | ------------------------------------------------------------------------- | ---------------------------------------------------------- | ---------------------------------------------------------- | +| :soon: [**0.9.4**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.4) | **27.3.4.13** | **28.5.0.2** | | [**0.9.3**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.3) | **27.3.4.13** | **28.5.0.2** | | [**0.9.2**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.2) | **27.3.4.13** | **28.5.0.2** | | [**0.9.1**](https://github.com/thiagoesteves/deployex/releases/tag/0.9.1) | **27.3.4.9** | **28.4.1** | diff --git a/apps/deployer/lib/deployer/application.ex b/apps/deployer/lib/deployer/application.ex index 33b61217..f154b689 100644 --- a/apps/deployer/lib/deployer/application.ex +++ b/apps/deployer/lib/deployer/application.ex @@ -29,6 +29,7 @@ defmodule Deployer.Application do {:ok, _pid} = response -> Deployer.Monitor.init_all_monitor_supervisors() init_all_workers() + notify_startup() response {:error, reason} = response -> @@ -37,16 +38,44 @@ defmodule Deployer.Application do end end + @impl true + def stop(_state) do + notify_shutdown() + end + if_not_test do alias Deployer.Engine defp application_servers, do: [Deployer.Status.Application] defp init_all_workers, do: Engine.init_all_workers() + + defp notify_startup do + version = Application.spec(:deployer, :vsn) |> to_string() + + Foundation.Notifications.notify(:deployment_started, %{ + node: node(), + sname: "deployex", + version: version + }) + end + + defp notify_shutdown do + Foundation.Notifications.notify(:deployment_complete, %{ + node: node(), + sname: "deployex", + status: :ok, + message: "shutdown" + }) + end else defp application_servers, do: [] defp init_all_workers, do: :ok + + defp notify_startup, do: :ok + + defp notify_shutdown, do: :ok end defp gcp_app_credentials do diff --git a/apps/deployer/lib/deployer/deployex.ex b/apps/deployer/lib/deployer/deployex.ex index 59a3c380..57e29af5 100644 --- a/apps/deployer/lib/deployer/deployex.ex +++ b/apps/deployer/lib/deployer/deployex.ex @@ -19,6 +19,13 @@ defmodule Deployer.Deployex do def force_terminate(sleep_time) do Logger.warning("Deployex was requested to terminate, see you soon!!!") + Foundation.Notifications.notify(:deployment_complete, %{ + node: node(), + sname: "deployex", + status: :ok, + message: "forced terminate" + }) + :timer.sleep(sleep_time) deployex_path = Application.fetch_env!(:foundation, :install_path) diff --git a/apps/deployer/lib/deployer/hot_upgrade/application.ex b/apps/deployer/lib/deployer/hot_upgrade/application.ex index 44cf6738..eef85ad2 100644 --- a/apps/deployer/lib/deployer/hot_upgrade/application.ex +++ b/apps/deployer/lib/deployer/hot_upgrade/application.ex @@ -594,15 +594,31 @@ defmodule Deployer.HotUpgrade.Application do @events_topic, {:hot_upgrade_complete, Node.self(), sname, :ok, "Hot upgrade applied successfully!"} ) + + Foundation.Notifications.notify(:deployment_complete, %{ + node: Node.self(), + sname: sname, + status: :ok, + message: "Hot upgrade applied successfully!" + }) end @spec notify_error(sname :: String.t(), result :: any()) :: :ok defp notify_error(sname, result) do + message = "Upgrade failed: #{inspect(result)}" + Phoenix.PubSub.broadcast( Deployer.PubSub, @events_topic, - {:hot_upgrade_complete, Node.self(), sname, :error, "Upgrade failed: #{inspect(result)}"} + {:hot_upgrade_complete, Node.self(), sname, :error, message} ) + + Foundation.Notifications.notify(:deployment_complete, %{ + node: Node.self(), + sname: sname, + status: :error, + message: message + }) end defp check_jellyfish_files(files, from_version, to_version) do diff --git a/apps/deployer/lib/deployer/monitor/application.ex b/apps/deployer/lib/deployer/monitor/application.ex index 32c7c06c..6aebe7d2 100644 --- a/apps/deployer/lib/deployer/monitor/application.ex +++ b/apps/deployer/lib/deployer/monitor/application.ex @@ -157,6 +157,14 @@ defmodule Deployer.Monitor.Application do # Update the number of crash restarts crash_restart_count = state.crash_restart_count + 1 + Foundation.Notifications.notify(:crash_restart, %{ + node: Node.self(), + sname: state.sname, + name: state.name, + language: state.language, + crash_restart_count: crash_restart_count + }) + # Retry with backoff pattern trigger_run_service(state.sname, 2 * crash_restart_count * 1000) @@ -243,6 +251,12 @@ defmodule Deployer.Monitor.Application do version = version_map.version notify_new_deploy = fn -> + Foundation.Notifications.notify(:deployment_started, %{ + node: Node.self(), + sname: sname, + version: version + }) + Phoenix.PubSub.broadcast( Deployer.PubSub, @new_deploy_topic, diff --git a/apps/deployex_web/test/deployex_web/ui_settings_test.exs b/apps/deployex_web/test/deployex_web/ui_settings_test.exs index a355b888..4ffbeb14 100644 --- a/apps/deployex_web/test/deployex_web/ui_settings_test.exs +++ b/apps/deployex_web/test/deployex_web/ui_settings_test.exs @@ -1,5 +1,5 @@ defmodule DeployexWeb.UiSettingsTest do - use DeployexWeb.ConnCase, async: true + use DeployexWeb.ConnCase, async: false import DeployexWeb.AccountsFixtures diff --git a/apps/foundation/lib/config_provider/env/config.ex b/apps/foundation/lib/config_provider/env/config.ex index de4d0ab3..c943715b 100644 --- a/apps/foundation/lib/config_provider/env/config.ex +++ b/apps/foundation/lib/config_provider/env/config.ex @@ -37,6 +37,7 @@ defmodule Foundation.ConfigProvider.Env.Config do {:applications, yaml_config.applications}, {:config_checksum, yaml_config.config_checksum}, {:monitoring, yaml_config.monitoring}, + {:notifications, yaml_config.notifications}, {:logs_retention_time_ms, yaml_config.logs_retention_time_ms}, {:install_path, yaml_config.install_path}, {:var_path, yaml_config.var_path}, diff --git a/apps/foundation/lib/foundation/application.ex b/apps/foundation/lib/foundation/application.ex index 34345704..236e3b81 100644 --- a/apps/foundation/lib/foundation/application.ex +++ b/apps/foundation/lib/foundation/application.ex @@ -10,9 +10,12 @@ defmodule Foundation.Application do children = [ {Phoenix.PubSub, name: Foundation.PubSub}, + {Finch, name: Foundation.Finch}, Foundation.Catalog.Local, Foundation.Certificates.Manager.Supervisor, - Foundation.Certificate + Foundation.Certificate, + Foundation.Notifications.Supervisor, + Foundation.Notifications ] # See https://hexdocs.pm/elixir/Supervisor.html diff --git a/apps/foundation/lib/foundation/certificates/manager/manager.ex b/apps/foundation/lib/foundation/certificates/manager/manager.ex index aa7282d0..286a4647 100644 --- a/apps/foundation/lib/foundation/certificates/manager/manager.ex +++ b/apps/foundation/lib/foundation/certificates/manager/manager.ex @@ -80,11 +80,22 @@ defmodule Foundation.Certificates.Manager do "Certificate check completed successfully for app: #{state.app_name}, domains: #{inspect(certificate.domains)}" ) + Foundation.Notifications.notify(:certificate_renewed, %{ + app_name: state.app_name, + domains: state.domains + }) + {:retry_after, seconds} when seconds >= 0 -> Logger.warning("Acme for #{state.app_name} requested retry after #{seconds} s") {:error, reason} -> Logger.error("Certificate renewal check failed: #{inspect(reason)}") + + Foundation.Notifications.notify(:certificate_failed, %{ + app_name: state.app_name, + domains: state.domains, + reason: inspect(reason) + }) end _ = Process.send_after(self(), :check_and_renew, state.certificate_check_interval_ms) diff --git a/apps/foundation/lib/foundation/notifications.ex b/apps/foundation/lib/foundation/notifications.ex new file mode 100644 index 00000000..12e519cb --- /dev/null +++ b/apps/foundation/lib/foundation/notifications.ex @@ -0,0 +1,156 @@ +defmodule Foundation.Notifications do + @moduledoc """ + Dispatches deployment lifecycle events to configured external notification channels. + + `Foundation.Notifications.notify/2` broadcasts an event to a per-event + `Foundation.PubSub` topic (e.g. `"deployex::notifications::crash_restart"`). + Each `Foundation.Notifications.Worker` — one per entry in the `notifications:` + YAML list — subscribes only to the topics matching its `events` list at startup, + so PubSub routing handles delivery without any filtering in the worker. This means: + + - **Callers are never blocked** — `Phoenix.PubSub.broadcast/3` returns immediately. + - **Channels are isolated** — a slow or crashing adapter only affects its own worker. + - **Workers are supervised** — `Foundation.Notifications.Supervisor` restarts a + crashed worker automatically. + + ## Supported events + + | Event atom | Trigger | Payload keys | + |----------------------------------|------------------------------------------------------------|-----------------------------------------------------------------------------| + | `:crash_restart` | Monitored app crashed and is being restarted | `node`, `sname`, `name`, `language`, `crash_restart_count` | + | `:deployment_started` | New deployment was initiated for an sname | `node`, `sname`, `version` | + | `:deployment_complete` | Hot-upgrade finished (success or failure) | `node`, `sname`, `status` (`:ok`/`:error`), `message` | + | `:watchdog_threshold_exceeded` | Watchdog exceeded a resource threshold and restarted an app| `node`, `sname`, `type`, `current_percentage`, `restart_threshold_percent` | + | `:certificate_renewed` | TLS certificate was successfully renewed | `app_name`, `domains` | + | `:certificate_failed` | TLS certificate renewal failed | `app_name`, `domains`, `reason` | + + ## Available adapters + + | YAML value | Module | Description | + |---------------|---------------------------------------|---------------------------------------| + | `"webhook"` | `Foundation.Notifications.Webhook` | Generic HTTP POST with a JSON body | + | `"slack"` | `Foundation.Notifications.Slack` | Slack Incoming Webhook message | + | `"pagerduty"` | `Foundation.Notifications.PagerDuty` | PagerDuty Events API v2 incident | + + ## Configuration (deployex.yaml) + + Multiple adapters can run in parallel. Each entry is independent and subscribes + to its own subset of events. + + notifications: + - adapter: "slack" + url: "https://hooks.slack.com/services/T.../B.../..." + enabled: true + events: + - "crash_restart" + - "deployment_complete" + - "watchdog_threshold_exceeded" + options: + username: "DeployEx" + icon_emoji: ":rocket:" + + - adapter: "pagerduty" + enabled: true + events: + - "crash_restart" + - "watchdog_threshold_exceeded" + - "certificate_failed" + options: + routing_key: "abc123def456..." + + - adapter: "webhook" + url: "https://internal.example.com/hooks/deployex" + enabled: true + events: + - "crash_restart" + - "deployment_started" + - "deployment_complete" + - "watchdog_threshold_exceeded" + - "certificate_renewed" + - "certificate_failed" + + ## Adding a new adapter + + Implement `Foundation.Notifications.Adapter` and add a new clause to + `Foundation.Yaml.notification_adapter/1`. See `Foundation.Notifications.Adapter` + for a step-by-step guide and a skeleton implementation. + """ + + @topic_prefix "deployex::notifications" + + use GenServer + + import Foundation.Macros + + require Logger + + ### ========================================================================== + ### Callback functions + ### ========================================================================== + def start_link(args) do + GenServer.start_link(__MODULE__, args, name: __MODULE__) + end + + @impl true + def init(_args) do + Logger.info("Initializing Notification Server") + {:ok, %{}, {:continue, :start_notification_manager}} + end + + @impl true + def handle_continue(:start_notification_manager, state) do + initialize_notification_manager() + {:noreply, state} + end + + @doc """ + Returns the PubSub topic for a specific event. + + Each `Foundation.Notifications.Worker` subscribes to `topic(event)` for + every event in its list. `notify/2` broadcasts to the same per-event topic, + so PubSub routing delivers the message only to workers that subscribed to it. + + ## Example + + iex> Foundation.Notifications.topic(:crash_restart) + "deployex::notifications::crash_restart" + """ + @spec topic(event :: atom()) :: String.t() + def topic(event), do: "#{@topic_prefix}::#{event}" + + @doc """ + Broadcasts `event` with `payload` to all workers subscribed to that event. + + Returns `:ok` immediately; delivery to each adapter happens asynchronously + inside the respective `Foundation.Notifications.Worker` process. + """ + @spec notify(event :: atom(), payload :: map()) :: :ok + def notify(event, payload) do + Phoenix.PubSub.broadcast(Foundation.PubSub, topic(event), {event, payload}) + end + + ### ========================================================================== + ### Private Functions + ### ========================================================================== + + if_not_test do + alias Foundation.Notifications.Worker + + def initialize_notification_manager do + :foundation + |> Application.fetch_env!(:notifications) + |> Enum.map(&to_notification_struct/1) + |> Enum.map(&Foundation.Notifications.Supervisor.start_notification_worker/1) + end + + defp to_notification_struct(%Foundation.Yaml.Notification{} = config) do + struct!(Worker, Map.from_struct(config)) + end + + defp to_notification_struct(config) do + struct!(Worker, config) + end + else + defp initialize_notification_manager, do: :ok + end +end diff --git a/apps/foundation/lib/foundation/notifications/adapter.ex b/apps/foundation/lib/foundation/notifications/adapter.ex new file mode 100644 index 00000000..3655e10c --- /dev/null +++ b/apps/foundation/lib/foundation/notifications/adapter.ex @@ -0,0 +1,56 @@ +defmodule Foundation.Notifications.Adapter do + @moduledoc """ + Behaviour that every notification channel must implement. + + ## Implementing a new adapter + + 1. Create a module that `@behaviour Foundation.Notifications.Adapter`. + 2. Implement `notify/3`. Return `:ok` on success or `{:error, reason}` on + failure — `Foundation.Notifications` will log the error automatically. + 3. Add a new clause to `Foundation.Yaml.Notifications` so the adapter + can be selected from `deployex.yaml`. + + ### Example skeleton + + defmodule Foundation.Notifications.MyChannel do + @behaviour Foundation.Notifications.Adapter + + alias Foundation.Notifications.Worker + + @impl true + def notify(event, payload, %Notifications.Worker{url: url, options: options}) do + # build and send the notification + :ok + end + end + + ### Adapter-specific configuration + + Anything in the `options:` map of a notification entry is passed through + in `config.options` with keys converted to atoms at parse time, e.g.: + + options: + api_token: "secret" + channel: "#alerts" + + accessed as `Map.get(options, :api_token)` inside `notify/3`. + """ + + alias Foundation.Notifications.Worker + + @doc """ + Deliver a single event notification. + + Called asynchronously by `Foundation.Notifications.notify/2` for every enabled + adapter whose `events` list includes the given `event`. + + ## Parameters + + - `event` — one of the atoms defined in `Foundation.Notifications` (e.g. `:crash_restart`) + - `payload` — map with event-specific fields; keys are atoms + - `config` — the `Foundation.Notifications.Worker` struct for this adapter instance, + containing `:url`, `:options`, and so on + """ + @callback notify(event :: atom(), payload :: map(), config :: Worker.t()) :: + :ok | {:error, term()} +end diff --git a/apps/foundation/lib/foundation/notifications/pagerduty.ex b/apps/foundation/lib/foundation/notifications/pagerduty.ex new file mode 100644 index 00000000..5bf86acc --- /dev/null +++ b/apps/foundation/lib/foundation/notifications/pagerduty.ex @@ -0,0 +1,154 @@ +defmodule Foundation.Notifications.PagerDuty do + @moduledoc """ + Notification adapter that creates PagerDuty incidents via the Events API v2. + + ## How it works + + Each event is translated into a PagerDuty *trigger* action and posted to the + [PagerDuty Events API v2](https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgw-send-an-alert-event). + The routing key determines which PagerDuty service and escalation policy receives + the incident. + + ## Worker configuration + + notifications: + - adapter: "pagerduty" + enabled: true + url: "https://events.pagerduty.com/v2/enqueue" # optional — override the API endpoint + events: + - "crash_restart" + - "watchdog_threshold_exceeded" + - "certificate_failed" + options: + routing_key: "abc123def456..." # required — PagerDuty integration key + + The `url:` field is optional. When omitted, the standard PagerDuty Events API + endpoint (`https://events.pagerduty.com/v2/enqueue`) is used. Override it only + if you are running a PagerDuty on-premises installation. + + ## Finding your routing key + + 1. In PagerDuty, open *Services* → your service → *Integrations*. + 2. Add an *Events API v2* integration (or use an existing one). + 3. Copy the *Integration Key* — that is your `routing_key`. + + ## Alert severity mapping + + | DeployEx event | PagerDuty severity | + |-------------------------------|--------------------| + | `crash_restart` | `error` | + | `deployment_started` | `info` | + | `deployment_complete` (ok) | `info` | + | `deployment_complete` (error) | `error` | + | `watchdog_threshold_exceeded` | `critical` | + | `certificate_renewed` | `info` | + | `certificate_failed` | `error` | + + ## Payload sent to PagerDuty + + { + "routing_key": "...", + "event_action": "trigger", + "payload": { + "summary": "crash_restart — myapp-1 on myapp@prod-1", + "severity": "error", + "source": "myapp@prod-1", + "custom_details": { ... event payload ... } + } + } + """ + + @behaviour Foundation.Notifications.Adapter + + require Logger + + alias Foundation.Notifications.Worker + + @default_api_url "https://events.pagerduty.com/v2/enqueue" + + @impl true + @spec notify(event :: atom(), payload :: map(), config :: Worker.t()) :: + :ok | {:error, term()} + def notify(event, payload, %Worker{url: url, options: options}) do + routing_key = options[:routing_key] + api_url = url || @default_api_url + + body = + Jason.encode!(%{ + routing_key: routing_key, + event_action: "trigger", + payload: %{ + summary: format_summary(event, payload), + severity: event_severity(event, payload), + source: event_source(payload), + custom_details: payload + } + }) + + headers = [{"content-type", "application/json"}] + + :post + |> Finch.build(api_url, headers, body) + |> Finch.request(Foundation.Finch) + |> handle_response(event, api_url) + end + + ### ========================================================================== + ### Private functions + ### ========================================================================== + + defp handle_response({:ok, %Finch.Response{status: status}}, _event, _url) + when status in 200..299 do + :ok + end + + defp handle_response({:ok, %Finch.Response{status: status, body: body}}, event, url) do + Logger.warning( + "PagerDuty notification failed for event=#{event} url=#{url} status=#{status} body=#{body}" + ) + + {:error, {:http_error, status}} + end + + defp handle_response({:error, reason}, event, url) do + Logger.error( + "PagerDuty notification error for event=#{event} url=#{url} reason=#{inspect(reason)}" + ) + + {:error, reason} + end + + defp format_summary(:crash_restart, payload), + do: "crash_restart — #{payload.sname} on #{payload.node}" + + defp format_summary(:deployment_started, payload), + do: "deployment_started — #{payload.sname} on #{payload.node} (version #{payload.version})" + + defp format_summary(:deployment_complete, payload), + do: "deployment_complete (#{payload.status}) — #{payload.sname} on #{payload.node}" + + defp format_summary(:watchdog_threshold_exceeded, payload), + do: + "watchdog_threshold_exceeded — #{payload.type} at #{payload.current_percentage}% on #{payload.node}" + + defp format_summary(:certificate_renewed, payload), + do: "certificate_renewed — #{payload.app_name} (#{Enum.join(payload.domains, ", ")})" + + defp format_summary(:certificate_failed, payload), + do: "certificate_failed — #{payload.app_name}: #{payload.reason}" + + defp format_summary(event, payload), + do: "#{event} — #{inspect(payload)}" + + defp event_severity(:crash_restart, _payload), do: "error" + defp event_severity(:deployment_started, _payload), do: "info" + defp event_severity(:deployment_complete, %{status: :ok}), do: "info" + defp event_severity(:deployment_complete, %{status: :error}), do: "error" + defp event_severity(:watchdog_threshold_exceeded, _payload), do: "critical" + defp event_severity(:certificate_renewed, _payload), do: "info" + defp event_severity(:certificate_failed, _payload), do: "error" + defp event_severity(_event, _payload), do: "info" + + defp event_source(%{node: node}), do: to_string(node) + defp event_source(_payload), do: "deployex" +end diff --git a/apps/foundation/lib/foundation/notifications/slack.ex b/apps/foundation/lib/foundation/notifications/slack.ex new file mode 100644 index 00000000..6fbb1585 --- /dev/null +++ b/apps/foundation/lib/foundation/notifications/slack.ex @@ -0,0 +1,152 @@ +defmodule Foundation.Notifications.Slack do + @moduledoc """ + Notification adapter that delivers events to a Slack channel via Incoming Webhooks. + + ## How it works + + Each event is formatted into a human-readable Slack message and posted as an HTTP + POST to the configured Incoming Webhook URL. The URL already encodes the target + workspace and channel, so no additional auth header is needed. + + ## Worker configuration + + notifications: + - adapter: "slack" + url: "https://hooks.slack.com/services" + enabled: true + events: + - "crash_restart" + - "deployment_started" + - "deployment_complete" + - "watchdog_threshold_exceeded" + - "certificate_renewed" + - "certificate_failed" + options: + username: "DeployEx" # optional, default: "DeployEx" + icon_emoji: ":rocket:" # optional, default: ":robot_face:" + + ## Setting up a Slack Incoming Webhook + + 1. Go to https://api.slack.com/apps → *Create New App* → *From scratch*. + 2. Under *Features*, choose *Incoming Webhooks* and activate them. + 3. Click *Add New Webhook to Workspace*, pick a channel, and copy the URL. + 4. Paste that URL as the `url:` value in `deployex.yaml`. + + ## Message format + + Messages use Slack's `mrkdwn` formatting. Examples: + + 🚨 *crash_restart* — `myapp-1` on `myapp@prod-1` + Crash count: *3* + + ✅ *deployment_complete* — `myapp-1` on `myapp@prod-1` + Status: *ok* — Hot upgrade applied successfully! + """ + + @behaviour Foundation.Notifications.Adapter + + require Logger + + alias Foundation.Notifications.Worker + + @default_username "DeployEx" + @default_icon_emoji ":robot_face:" + + @impl true + @spec notify(event :: atom(), payload :: map(), config :: Worker.t()) :: + :ok | {:error, term()} + def notify(event, payload, %Worker{url: url, options: options}) do + body = + Jason.encode!(%{ + text: format_message(event, payload), + username: options[:username] || @default_username, + icon_emoji: options[:icon_emoji] || @default_icon_emoji + }) + + headers = [{"content-type", "application/json"}] + + :post + |> Finch.build(url, headers, body) + |> Finch.request(Foundation.Finch) + |> handle_response(event, url) + end + + ### ========================================================================== + ### Private functions + ### ========================================================================== + + defp handle_response({:ok, %Finch.Response{status: status}}, _event, _url) + when status in 200..299 do + :ok + end + + defp handle_response({:ok, %Finch.Response{status: status, body: body}}, event, url) do + Logger.warning( + "Slack notification failed for event=#{event} url=#{url} status=#{status} body=#{body}" + ) + + {:error, {:http_error, status}} + end + + defp handle_response({:error, reason}, event, url) do + Logger.error( + "Slack notification error for event=#{event} url=#{url} reason=#{inspect(reason)}" + ) + + {:error, reason} + end + + defp format_message(:crash_restart, payload) do + """ + 🚨 *crash_restart* — `#{payload.sname}` on `#{payload.node}` + Crash count: *#{payload.crash_restart_count}*\ + """ + end + + defp format_message(:deployment_started, payload) do + """ + 🚀 *deployment_started* — `#{payload.sname}` on `#{payload.node}` + Version: *#{payload.version}*\ + """ + end + + defp format_message(:deployment_complete, %{status: :ok} = payload) do + """ + ✅ *deployment_complete* — `#{payload.sname}` on `#{payload.node}` + Status: *ok* — #{payload.message}\ + """ + end + + defp format_message(:deployment_complete, %{status: :error} = payload) do + """ + ❌ *deployment_complete* — `#{payload.sname}` on `#{payload.node}` + Status: *error* — #{payload.message}\ + """ + end + + defp format_message(:watchdog_threshold_exceeded, payload) do + """ + ⚠️ *watchdog_threshold_exceeded* — `#{payload.sname}` on `#{payload.node}` + Resource: *#{payload.type}* at *#{payload.current_percentage}%* (threshold: #{payload.restart_threshold_percent}%)\ + """ + end + + defp format_message(:certificate_renewed, payload) do + """ + 🔒 *certificate_renewed* — `#{payload.app_name}` + Domains: #{Enum.join(payload.domains, ", ")}\ + """ + end + + defp format_message(:certificate_failed, payload) do + """ + 🔓 *certificate_failed* — `#{payload.app_name}` + Domains: #{Enum.join(payload.domains, ", ")} + Reason: #{payload.reason}\ + """ + end + + defp format_message(event, payload) do + "ℹ️ *#{event}*\n#{inspect(payload)}" + end +end diff --git a/apps/foundation/lib/foundation/notifications/supervisor.ex b/apps/foundation/lib/foundation/notifications/supervisor.ex new file mode 100644 index 00000000..7ba6d73c --- /dev/null +++ b/apps/foundation/lib/foundation/notifications/supervisor.ex @@ -0,0 +1,45 @@ +defmodule Foundation.Notifications.Supervisor do + @moduledoc """ + Supervisor that owns one `Foundation.Notifications.Worker` per notification + channel defined in the `notifications:` YAML list. + + Children are built from `Application.get_env(:foundation, :notifications, [])` + at startup, so the number and type of workers matches the runtime configuration + loaded by `Foundation.ConfigProvider.Env.Config`. + + Each worker is assigned a unique child ID based on its position in the list, + so multiple entries with the same adapter type are all started correctly. + """ + + use DynamicSupervisor + + alias Foundation.Notifications.Worker + + ### ========================================================================== + ### GenServer Callbacks + ### ========================================================================== + def start_link(init_arg) do + DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + @impl true + def init(_init_arg) do + DynamicSupervisor.init(strategy: :one_for_one, max_restarts: 20) + end + + ### ========================================================================== + ### Public APIs + ### ========================================================================== + + @spec start_notification_worker(config :: Worker.t()) :: + {:ok, pid} | {:error, pid(), :already_started} + def start_notification_worker(config) do + spec = %{ + id: Worker, + start: {Worker, :start_link, [config]}, + restart: :transient + } + + DynamicSupervisor.start_child(__MODULE__, spec) + end +end diff --git a/apps/foundation/lib/foundation/notifications/webhook.ex b/apps/foundation/lib/foundation/notifications/webhook.ex new file mode 100644 index 00000000..6aaba779 --- /dev/null +++ b/apps/foundation/lib/foundation/notifications/webhook.ex @@ -0,0 +1,106 @@ +defmodule Foundation.Notifications.Webhook do + @moduledoc """ + Notification adapter that delivers events as JSON HTTP POST requests. + + This is the generic, receiver-agnostic adapter. Point it at any endpoint + that accepts a POST with a JSON body (e.g. custom scripts, n8n, Zapier, + Make, AWS API Gateway, etc.). + + ## Worker configuration + + notifications: + - adapter: "webhook" + url: "https://example.com/hooks/deployex" + enabled: true + events: + - "crash_restart" + - "deployment_started" + - "deployment_complete" + - "watchdog_threshold_exceeded" + - "certificate_renewed" + - "certificate_failed" + + The `options:` key is accepted but currently unused; it is reserved for + future extensions such as custom headers or HMAC signing. + + ## Request format + + `Content-Type: application/json`, body: + + { + "event": "crash_restart", + "timestamp": "2025-06-18T14:30:00.000000Z", + "payload": { + "node": "myapp@prod-1", + "sname": "myapp-1", + "name": "myapp", + "language": "elixir", + "crash_restart_count": 3 + } + } + + A `2xx` HTTP response is treated as success. Any other status or transport + error is returned as `{:error, reason}` and logged at the `warning` level. + + ## Payload fields per event + + | Event | Fields in `payload` | + |--------------------------------|------------------------------------------------------------------------| + | `crash_restart` | `node`, `sname`, `name`, `language`, `crash_restart_count` | + | `deployment_started` | `node`, `sname`, `version` | + | `deployment_complete` | `node`, `sname`, `status` (`:ok`/`:error`), `message` | + | `watchdog_threshold_exceeded` | `node`, `sname`, `type`, `current_percentage`, `restart_threshold_percent` | + | `certificate_renewed` | `app_name`, `domains` | + | `certificate_failed` | `app_name`, `domains`, `reason` | + """ + + @behaviour Foundation.Notifications.Adapter + + require Logger + + alias Foundation.Notifications.Worker + + @impl true + @spec notify(event :: atom(), payload :: map(), config :: Worker.t()) :: + :ok | {:error, term()} + def notify(event, payload, %Worker{url: url}) do + body = + Jason.encode!(%{ + event: event, + timestamp: DateTime.utc_now() |> DateTime.to_iso8601(), + payload: payload + }) + + headers = [{"content-type", "application/json"}] + + :post + |> Finch.build(url, headers, body) + |> Finch.request(Foundation.Finch) + |> handle_response(event, url) + end + + ### ========================================================================== + ### Private functions + ### ========================================================================== + + defp handle_response({:ok, %Finch.Response{status: status}}, _event, _url) + when status in 200..299 do + :ok + end + + defp handle_response({:ok, %Finch.Response{status: status, body: body}}, event, url) do + Logger.warning( + "Webhook notification failed for event=#{event} url=#{url} status=#{status} body=#{body}" + ) + + {:error, {:http_error, status}} + end + + defp handle_response({:error, reason}, event, url) do + Logger.error( + "Webhook notification error for event=#{event} url=#{url} reason=#{inspect(reason)}" + ) + + {:error, reason} + end +end diff --git a/apps/foundation/lib/foundation/notifications/worker.ex b/apps/foundation/lib/foundation/notifications/worker.ex new file mode 100644 index 00000000..6689aff5 --- /dev/null +++ b/apps/foundation/lib/foundation/notifications/worker.ex @@ -0,0 +1,67 @@ +defmodule Foundation.Notifications.Worker do + @moduledoc """ + GenServer that backs a single notification channel. + + On startup the worker subscribes to one `Foundation.PubSub` topic per event + in its `events` list (e.g. `"deployex::notifications::crash_restart"`). + PubSub routing ensures only relevant messages arrive, so `handle_info/2` + calls the adapter unconditionally — no filtering needed. + + Disabled workers (`enabled: false`) skip all subscriptions and therefore + never receive any messages. + + Workers are started by `Foundation.Notifications.Supervisor`, one per entry + in the `notifications:` YAML list. + """ + + use GenServer + + require Logger + + alias Foundation.Notifications + + defstruct [:adapter, :url, :enabled, :events, options: %{}] + + @type t :: %__MODULE__{ + adapter: module(), + url: String.t() | nil, + enabled: boolean(), + events: [atom()], + options: map() + } + + ### ========================================================================== + ### GenServer callbacks + ### ========================================================================== + def start_link(%__MODULE__{} = config) do + GenServer.start_link(__MODULE__, config) + end + + @impl true + def init(%__MODULE__{adapter: adapter, enabled: enabled, events: events} = config) do + Logger.info("Initializing Notifications Worker for adapter: #{inspect(adapter)}") + + if enabled do + Enum.each(events, fn event -> + Phoenix.PubSub.subscribe(Foundation.PubSub, Notifications.topic(event)) + end) + end + + {:ok, config} + end + + @impl true + def handle_info({event, payload}, config) do + case config.adapter.notify(event, payload, config) do + :ok -> + :ok + + {:error, reason} -> + Logger.error( + "Notifications.Worker adapter=#{inspect(config.adapter)} event=#{event} reason=#{inspect(reason)}" + ) + end + + {:noreply, config} + end +end diff --git a/apps/foundation/lib/foundation/yaml.ex b/apps/foundation/lib/foundation/yaml.ex index 7ec49e5e..9cb4d1c1 100644 --- a/apps/foundation/lib/foundation/yaml.ex +++ b/apps/foundation/lib/foundation/yaml.ex @@ -129,6 +129,43 @@ defmodule Foundation.Yaml do } end + defmodule Notification do + @moduledoc """ + Configuration for a single notification channel. + + Each entry in the top-level `notifications:` YAML list is parsed into one of + these structs and stored in the `:foundation` application environment under + the `:notifications` key. + + ## Fields + + - `:adapter` — the module that handles delivery (e.g. `Foundation.Notifications.Webhook`) + - `:url` — destination URL; required for `webhook` and `slack`, optional for `pagerduty` + (which defaults to the standard Events API endpoint) + - `:enabled` — set to `false` to silence a channel without removing it; defaults to `true` + - `:events` — list of event atoms this channel subscribes to (empty list = no deliveries) + - `:options` — adapter-specific key/value pairs parsed directly from the YAML `options:` map + + ## Adapter-specific options + + | Adapter | Key | Required | Description | + |-------------|-----------------|----------|------------------------------------------| + | `pagerduty` | `routing_key` | yes | PagerDuty integration / routing key | + | `slack` | `username` | no | Bot display name (default: `"DeployEx"`) | + | `slack` | `icon_emoji` | no | Bot emoji icon (default: `":robot_face:"`) | + """ + + defstruct [:adapter, :url, :enabled, :events, options: %{}] + + @type t :: %__MODULE__{ + adapter: module(), + url: String.t() | nil, + enabled: boolean(), + events: [atom()], + options: map() + } + end + defmodule Application do @moduledoc """ Provides structure to define Application feature @@ -180,6 +217,7 @@ defmodule Foundation.Yaml do logs_retention_time_ms: nil, monitoring: [], applications: [], + notifications: [], config_checksum: nil @type t :: %__MODULE__{ @@ -204,6 +242,7 @@ defmodule Foundation.Yaml do logs_retention_time_ms: non_neg_integer() | nil, monitoring: [{atom(), Foundation.Yaml.Monitoring.t()}] | [], applications: [Foundation.Yaml.Application.t()] | [], + notifications: [Foundation.Yaml.Notification.t()] | [], # Checksum of the YAML configuration file content. # Used internally to detect configuration changes and trigger dynamic reloads. # This value is computed from the file contents, not the file metadata. @@ -328,6 +367,7 @@ defmodule Foundation.Yaml do logs_retention_time_ms: data["logs_retention_time_ms"] || @default_logs_retention_time_ms, monitoring: parse_monitoring_list(data["monitoring"]), applications: parse_applications(data["applications"]), + notifications: parse_notifications(data["notifications"]), config_checksum: checksum } end @@ -464,4 +504,40 @@ defmodule Foundation.Yaml do certificate_arn: opts["certificate_arn"] } end + + defp parse_notifications(nil), do: [] + + defp parse_notifications(notifications) do + Enum.map(notifications, &parse_notification/1) + end + + defp parse_notification(data) do + %Foundation.Yaml.Notification{ + adapter: notification_adapter(data["adapter"]), + url: data["url"], + enabled: data["enabled"] != false, + events: parse_notification_events(data["events"] || []), + options: atomize_keys(data["options"]) + } + end + + defp notification_adapter("webhook"), do: Foundation.Notifications.Webhook + defp notification_adapter("slack"), do: Foundation.Notifications.Slack + defp notification_adapter("pagerduty"), do: Foundation.Notifications.PagerDuty + defp notification_adapter(adapter), do: raise("Notification adapter #{adapter} not supported") + + defp parse_notification_events(events) do + Enum.map(events, &parse_notification_event/1) + end + + defp parse_notification_event("crash_restart"), do: :crash_restart + defp parse_notification_event("deployment_started"), do: :deployment_started + defp parse_notification_event("deployment_complete"), do: :deployment_complete + defp parse_notification_event("watchdog_threshold_exceeded"), do: :watchdog_threshold_exceeded + defp parse_notification_event("certificate_renewed"), do: :certificate_renewed + defp parse_notification_event("certificate_failed"), do: :certificate_failed + defp parse_notification_event(event), do: raise("Unknown notification event: #{event}") + + defp atomize_keys(nil), do: %{} + defp atomize_keys(map), do: Map.new(map, fn {k, v} -> {String.to_atom(k), v} end) end diff --git a/apps/foundation/test/certificates/public_key_test.exs b/apps/foundation/test/certificates/public_key_test.exs index 81074bdc..40fdbe20 100644 --- a/apps/foundation/test/certificates/public_key_test.exs +++ b/apps/foundation/test/certificates/public_key_test.exs @@ -1,5 +1,5 @@ defmodule Foundation.Certificates.PublicKeyTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias Foundation.Certificates.PublicKey diff --git a/apps/foundation/test/config_provider/env/config_test.exs b/apps/foundation/test/config_provider/env/config_test.exs index 6488c5b5..09e71ef0 100644 --- a/apps/foundation/test/config_provider/env/config_test.exs +++ b/apps/foundation/test/config_provider/env/config_test.exs @@ -16,6 +16,7 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do @yaml_gcp_path "#{@file_paths}/deployex-gcp.yaml" @yaml_gcp_release_error "#{@file_paths}/deployex-gcp-release-error.yaml" @yaml_gcp_secrets_error "#{@file_paths}/deployex-gcp-secrets-error.yaml" + @yaml_notifications "#{@file_paths}/deployex-notifications.yaml" @tag :capture_log test "init/1 with success" do @@ -79,6 +80,7 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do {:config_checksum, "1df842a56db380a76735783d8055d45a5d9558a9446a9076e596796532b3b59c"}, {:monitoring, []}, + {:notifications, []}, {:logs_retention_time_ms, 3_600_000}, {:install_path, "/opt/deployex"}, {:var_path, "/var/lib/deployex"}, @@ -183,6 +185,7 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do {:config_checksum, "80b62c3694c0311d2d9567fbf777b419bced562b5bd5196432d811faf128f845"}, {:monitoring, []}, + {:notifications, []}, {:logs_retention_time_ms, 3_600_000}, {:install_path, "/opt/deployex"}, {:var_path, "/var/lib/deployex"}, @@ -303,6 +306,7 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do restart_threshold_percent: 85 } ]}, + {:notifications, []}, {:logs_retention_time_ms, 3_600_000}, {:install_path, "/opt/install/deployex"}, {:var_path, "/var/lib/install/deployex"}, @@ -408,6 +412,7 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do restart_threshold_percent: 85 } ]}, + {:notifications, []}, {:logs_retention_time_ms, 3_600_000}, {:install_path, "/opt/deployex"}, {:var_path, "/var/lib/deployex"}, @@ -474,6 +479,7 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do {:config_checksum, "9b6cb25872864716974f64a1477510d4127b0d7b3063e75d8c39e9dd00fb578d"}, {:monitoring, []}, + {:notifications, []}, {:logs_retention_time_ms, 3_600_000}, {:install_path, "/opt/install/deployex"}, {:var_path, "/var/lib/install/deployex"}, @@ -580,6 +586,7 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do {:config_checksum, "5c1d8cb90a8661ac4fefad8d1ad2aa760d4d7257f61bd310e9aadf31c3b97968"}, {:monitoring, []}, + {:notifications, []}, {:logs_retention_time_ms, 3_600_000}, {:install_path, "/opt/deployex"}, {:var_path, "/var/lib/deployex"}, @@ -666,6 +673,100 @@ defmodule Foundation.ConfigProvider.Env.ConfigTest do end end + @tag :capture_log + test "load/3 with success for AWS with notifications" do + with_mocks([ + {System, [:passthrough], + [get_env: fn "DEPLOYEX_CONFIG_YAML_PATH" -> @yaml_notifications end]} + ]) do + assert [ + {:ex_aws, [region: "sa-east-1"]}, + {:deployex_web, + [ + {DeployexWeb.Endpoint, + [ + url: [host: "deployex.example.com"], + http: [port: 5001] + ]} + ]}, + {:observer_web, [data_retention_period: 3_600_000]}, + {:deployer, + [ + {Deployer.Release, + [adapter: Deployer.Release.S3, bucket: "myapp-prod-distribution"]} + ]}, + {:foundation, + [ + {:env, "prod"}, + {:applications, + [ + %{ + env: [], + name: "myphoenixapp", + monitoring: [], + replicas: 3, + language: "elixir", + deploy_rollback_timeout_ms: 600_000, + deploy_schedule_interval_ms: 5_000, + replica_ports: [] + } + ]}, + {:config_checksum, + "3da24d11865bc40bab27e2545a5547f83005a3137e7d3b8a2bc4a65d7f00ea16"}, + {:monitoring, []}, + {:notifications, + [ + %Foundation.Yaml.Notification{ + adapter: Foundation.Notifications.Webhook, + url: "https://hooks.example.com/deployex", + enabled: true, + events: [ + :crash_restart, + :deployment_started, + :deployment_complete, + :watchdog_threshold_exceeded, + :certificate_renewed, + :certificate_failed + ], + options: %{} + }, + %Foundation.Yaml.Notification{ + adapter: Foundation.Notifications.Slack, + url: "https://hooks.slack.com/services/T000/B000/XXX", + enabled: true, + events: [:crash_restart, :deployment_complete], + options: %{username: "DeployEx-Bot", icon_emoji: ":rocket:"} + }, + %Foundation.Yaml.Notification{ + adapter: Foundation.Notifications.PagerDuty, + url: nil, + enabled: true, + events: [:crash_restart, :watchdog_threshold_exceeded], + options: %{routing_key: "abc123def456"} + }, + %Foundation.Yaml.Notification{ + adapter: Foundation.Notifications.Webhook, + url: "https://hooks2.example.com/deployex", + enabled: false, + events: [:crash_restart], + options: %{} + } + ]}, + {:logs_retention_time_ms, 3_600_000}, + {:install_path, "/opt/deployex"}, + {:var_path, "/var/lib/deployex"}, + {:log_path, "/var/log/deployex"}, + {:monitored_app_log_path, "/var/log/monitored-apps"}, + {Foundation.ConfigProvider.Secrets.Manager, + [ + adapter: Foundation.ConfigProvider.Secrets.Aws, + path: "deployex-myapp-prod-secrets" + ]} + ]} + ] = Config.load([], []) + end + end + test "load/3 Yaml file not found, keep the configuration" do with_mocks([ {System, [:passthrough], [get_env: fn "DEPLOYEX_CONFIG_YAML_PATH" -> nil end]} diff --git a/apps/foundation/test/notifications/notifications_test.exs b/apps/foundation/test/notifications/notifications_test.exs new file mode 100644 index 00000000..65c63488 --- /dev/null +++ b/apps/foundation/test/notifications/notifications_test.exs @@ -0,0 +1,162 @@ +defmodule Foundation.NotificationsTest do + use ExUnit.Case, async: false + + import Mock + + alias Foundation.Notifications + alias Foundation.Notifications.Webhook + alias Foundation.Notifications.Worker + + @payload %{node: :deployex@host, sname: "myapp-1", crash_restart_count: 1} + + describe "topic/1" do + test "returns a binary topic string for an event" do + assert is_binary(Notifications.topic(:crash_restart)) + end + + test "different events produce different topics" do + assert Notifications.topic(:crash_restart) != Notifications.topic(:deployment_complete) + end + + test "topic contains the event name" do + assert Notifications.topic(:crash_restart) =~ "crash_restart" + end + end + + describe "notify/2" do + test "broadcasts to the per-event topic on Foundation.PubSub" do + Phoenix.PubSub.subscribe(Foundation.PubSub, Notifications.topic(:crash_restart)) + + Notifications.notify(:crash_restart, @payload) + + assert_receive {:crash_restart, received_payload} + assert received_payload == @payload + end + + test "does not deliver to a subscriber on a different event topic" do + Phoenix.PubSub.subscribe(Foundation.PubSub, Notifications.topic(:deployment_complete)) + + Notifications.notify(:crash_restart, @payload) + + refute_receive {:crash_restart, _}, 50 + end + + test "returns :ok" do + assert :ok = Notifications.notify(:crash_restart, @payload) + end + end + + describe "Worker integration" do + @tag :capture_log + test "worker calls its adapter when a matching event is received" do + config = %Worker{ + adapter: Webhook, + url: "https://hooks.example.com/deployex", + enabled: true, + events: [:crash_restart], + options: %{} + } + + {:ok, worker} = Worker.start_link(config) + + with_mocks([ + {Webhook, [], [notify: fn _event, _payload, _config -> :ok end]} + ]) do + Notifications.notify(:crash_restart, @payload) + + # allow the worker's handle_info to run + Process.sleep(50) + + assert called(Webhook.notify(:crash_restart, @payload, config)) + end + + GenServer.stop(worker) + end + + @tag :capture_log + test "worker ignores events not in its list" do + config = %Worker{ + adapter: Webhook, + url: "https://hooks.example.com/deployex", + enabled: true, + events: [:watchdog_threshold_exceeded], + options: %{} + } + + {:ok, worker} = Worker.start_link(config) + + with_mocks([ + {Webhook, [], [notify: fn _event, _payload, _config -> :ok end]} + ]) do + Notifications.notify(:crash_restart, @payload) + + Process.sleep(50) + + refute called(Webhook.notify(:_, :_, :_)) + end + + GenServer.stop(worker) + end + + @tag :capture_log + test "disabled worker never calls its adapter" do + config = %Worker{ + adapter: Webhook, + url: "https://hooks.example.com/deployex", + enabled: false, + events: [:crash_restart], + options: %{} + } + + {:ok, worker} = Worker.start_link(config) + + with_mocks([ + {Webhook, [], [notify: fn _event, _payload, _config -> :ok end]} + ]) do + Notifications.notify(:crash_restart, @payload) + + Process.sleep(50) + + refute called(Webhook.notify(:_, :_, :_)) + end + + GenServer.stop(worker) + end + + @tag :capture_log + test "two workers for different channels each receive the same event" do + config_a = %Worker{ + adapter: Webhook, + url: "https://a.example.com", + enabled: true, + events: [:crash_restart], + options: %{} + } + + config_b = %Worker{ + adapter: Webhook, + url: "https://b.example.com", + enabled: true, + events: [:crash_restart], + options: %{} + } + + {:ok, worker_a} = Worker.start_link(config_a) + {:ok, worker_b} = Worker.start_link(config_b) + + with_mocks([ + {Webhook, [], [notify: fn _event, _payload, _config -> :ok end]} + ]) do + Notifications.notify(:crash_restart, @payload) + + Process.sleep(50) + + assert called(Webhook.notify(:crash_restart, @payload, config_a)) + assert called(Webhook.notify(:crash_restart, @payload, config_b)) + end + + GenServer.stop(worker_a) + GenServer.stop(worker_b) + end + end +end diff --git a/apps/foundation/test/notifications/pagerduty_test.exs b/apps/foundation/test/notifications/pagerduty_test.exs new file mode 100644 index 00000000..a79d0af1 --- /dev/null +++ b/apps/foundation/test/notifications/pagerduty_test.exs @@ -0,0 +1,188 @@ +defmodule Foundation.Notifications.PagerDutyTest do + use ExUnit.Case, async: false + + import Mock + + alias Foundation.Notifications.PagerDuty + alias Foundation.Notifications.Worker + + @config %Worker{ + adapter: PagerDuty, + url: nil, + enabled: true, + events: [:crash_restart], + options: %{routing_key: "abc123def456"} + } + + describe "notify/3" do + @tag :capture_log + test "returns :ok on a 2xx response" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 202, headers: [], body: ~s({"status":"success"})}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 1} + assert :ok = PagerDuty.notify(:crash_restart, payload, @config) + end + end + + @tag :capture_log + test "returns {:error, {:http_error, status}} on non-2xx" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 400, headers: [], body: "Bad Request"}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 1} + assert {:error, {:http_error, 400}} = PagerDuty.notify(:crash_restart, payload, @config) + end + end + + @tag :capture_log + test "returns {:error, reason} on transport error" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> {:error, :timeout} end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 1} + assert {:error, :timeout} = PagerDuty.notify(:crash_restart, payload, @config) + end + end + + test "posts to the default PagerDuty API URL when config url is nil" do + test_pid = self() + + with_mocks([ + {Finch, [], + [ + build: fn :post, url, _headers, _body -> + send(test_pid, {:url, url}) + %{} + end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 202, headers: [], body: ""}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 1} + PagerDuty.notify(:crash_restart, payload, @config) + + assert_receive {:url, url} + assert url == "https://events.pagerduty.com/v2/enqueue" + end + end + + test "uses a custom URL when provided in config" do + test_pid = self() + custom_config = %{@config | url: "https://acme.pagerduty.com/v2/enqueue"} + + with_mocks([ + {Finch, [], + [ + build: fn :post, url, _headers, _body -> + send(test_pid, {:url, url}) + %{} + end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 202, headers: [], body: ""}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 1} + PagerDuty.notify(:crash_restart, payload, custom_config) + + assert_receive {:url, url} + assert url == "https://acme.pagerduty.com/v2/enqueue" + end + end + + test "posts JSON with routing_key, event_action, and structured payload" do + test_pid = self() + + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, headers, body -> + send(test_pid, {:request, headers, body}) + %{} + end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 202, headers: [], body: ""}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 1} + PagerDuty.notify(:crash_restart, payload, @config) + + assert_receive {:request, headers, body} + + assert {"content-type", "application/json"} in headers + + decoded = Jason.decode!(body) + assert decoded["routing_key"] == "abc123def456" + assert decoded["event_action"] == "trigger" + assert is_map(decoded["payload"]) + assert is_binary(decoded["payload"]["summary"]) + assert decoded["payload"]["severity"] == "error" + assert decoded["payload"]["source"] == "app@host" + assert is_map(decoded["payload"]["custom_details"]) + end + end + + test "assigns correct severity for each event" do + severities = [ + {:crash_restart, %{node: :n@h, sname: "s", crash_restart_count: 1}, "error"}, + {:deployment_started, %{node: :n@h, sname: "s", version: "1.0"}, "info"}, + {:deployment_complete, %{node: :n@h, sname: "s", status: :ok, message: "ok"}, "info"}, + {:deployment_complete, %{node: :n@h, sname: "s", status: :error, message: "fail"}, + "error"}, + {:watchdog_threshold_exceeded, + %{ + node: :n@h, + sname: "s", + type: :memory, + current_percentage: 96, + restart_threshold_percent: 95 + }, "critical"}, + {:certificate_renewed, %{app_name: "app", domains: ["ex.com"]}, "info"}, + {:certificate_failed, %{app_name: "app", domains: ["ex.com"], reason: "timeout"}, "error"} + ] + + test_pid = self() + + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, body -> + send(test_pid, {:body, body}) + %{} + end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 202, headers: [], body: ""}} + end + ]} + ]) do + Enum.each(severities, fn {event, payload, expected_severity} -> + PagerDuty.notify(event, payload, @config) + assert_receive {:body, body} + decoded = Jason.decode!(body) + + assert decoded["payload"]["severity"] == expected_severity, + "expected #{expected_severity} for #{event}, got #{decoded["payload"]["severity"]}" + end) + end + end + end +end diff --git a/apps/foundation/test/notifications/slack_test.exs b/apps/foundation/test/notifications/slack_test.exs new file mode 100644 index 00000000..c55efe01 --- /dev/null +++ b/apps/foundation/test/notifications/slack_test.exs @@ -0,0 +1,153 @@ +defmodule Foundation.Notifications.SlackTest do + use ExUnit.Case, async: false + + import Mock + + alias Foundation.Notifications.Slack + alias Foundation.Notifications.Worker + + @config %Worker{ + adapter: Slack, + url: "https://hooks.slack.com/services/T000/B000/XXX", + enabled: true, + events: [:crash_restart], + options: %{username: "DeployEx", icon_emoji: ":robot_face:"} + } + + describe "notify/3" do + @tag :capture_log + test "returns :ok on a 2xx response" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 200, headers: [], body: "ok"}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 2} + assert :ok = Slack.notify(:crash_restart, payload, @config) + end + end + + @tag :capture_log + test "returns {:error, {:http_error, status}} on non-2xx" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 400, headers: [], body: "no_text"}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 2} + assert {:error, {:http_error, 400}} = Slack.notify(:crash_restart, payload, @config) + end + end + + @tag :capture_log + test "returns {:error, reason} on transport error" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> {:error, :econnrefused} end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 2} + assert {:error, :econnrefused} = Slack.notify(:crash_restart, payload, @config) + end + end + + test "posts JSON with text, username and icon_emoji fields" do + test_pid = self() + + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, headers, body -> + send(test_pid, {:request, headers, body}) + %{} + end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 200, headers: [], body: "ok"}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 2} + Slack.notify(:crash_restart, payload, @config) + + assert_receive {:request, headers, body} + + assert {"content-type", "application/json"} in headers + + decoded = Jason.decode!(body) + assert is_binary(decoded["text"]) + assert decoded["username"] == "DeployEx" + assert decoded["icon_emoji"] == ":robot_face:" + end + end + + test "uses default username and icon when options are absent" do + test_pid = self() + config_no_opts = %{@config | options: %{}} + + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, body -> + send(test_pid, {:body, body}) + %{} + end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 200, headers: [], body: "ok"}} + end + ]} + ]) do + payload = %{node: :app@host, sname: "myapp-1", crash_restart_count: 1} + Slack.notify(:crash_restart, payload, config_no_opts) + + assert_receive {:body, body} + + decoded = Jason.decode!(body) + assert decoded["username"] == "DeployEx" + assert decoded["icon_emoji"] == ":robot_face:" + end + end + + test "formats all supported events without raising" do + events_and_payloads = [ + {:crash_restart, %{node: :n@h, sname: "s-1", crash_restart_count: 1}}, + {:deployment_started, %{node: :n@h, sname: "s-1", version: "1.0.0"}}, + {:deployment_complete, %{node: :n@h, sname: "s-1", status: :ok, message: "done"}}, + {:deployment_complete, %{node: :n@h, sname: "s-1", status: :error, message: "fail"}}, + {:watchdog_threshold_exceeded, + %{ + node: :n@h, + sname: "s-1", + type: :memory, + current_percentage: 96, + restart_threshold_percent: 95 + }}, + {:certificate_renewed, %{app_name: "myapp", domains: ["example.com"]}}, + {:certificate_failed, %{app_name: "myapp", domains: ["example.com"], reason: "timeout"}} + ] + + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 200, headers: [], body: "ok"}} + end + ]} + ]) do + Enum.each(events_and_payloads, fn {event, payload} -> + assert :ok = Slack.notify(event, payload, @config) + end) + end + end + end +end diff --git a/apps/foundation/test/notifications/supervisor_test.exs b/apps/foundation/test/notifications/supervisor_test.exs new file mode 100644 index 00000000..4a5b0b67 --- /dev/null +++ b/apps/foundation/test/notifications/supervisor_test.exs @@ -0,0 +1,50 @@ +defmodule Foundation.Notifications.SupervisorTest do + use ExUnit.Case, async: false + + alias Foundation.Notifications.Supervisor, as: NotifSupervisor + alias Foundation.Notifications.Worker + + @worker_config %Worker{ + adapter: Foundation.Notifications.Webhook, + url: "https://hooks.example.com", + enabled: true, + events: [:crash_restart], + options: %{} + } + + describe "start_notification_worker/1" do + @tag :capture_log + test "returns {:ok, pid} and starts a live worker process" do + {:ok, pid} = NotifSupervisor.start_notification_worker(@worker_config) + + assert is_pid(pid) + assert Process.alive?(pid) + + GenServer.stop(pid) + end + + @tag :capture_log + test "each call adds a worker to the supervisor" do + before = DynamicSupervisor.count_children(NotifSupervisor).workers + + {:ok, pid1} = NotifSupervisor.start_notification_worker(@worker_config) + {:ok, pid2} = NotifSupervisor.start_notification_worker(@worker_config) + + assert DynamicSupervisor.count_children(NotifSupervisor).workers == before + 2 + + GenServer.stop(pid1) + GenServer.stop(pid2) + end + + @tag :capture_log + test "disabled worker is still started (it simply skips subscriptions)" do + config = %Worker{@worker_config | enabled: false} + + {:ok, pid} = NotifSupervisor.start_notification_worker(config) + + assert Process.alive?(pid) + + GenServer.stop(pid) + end + end +end diff --git a/apps/foundation/test/notifications/webhook_test.exs b/apps/foundation/test/notifications/webhook_test.exs new file mode 100644 index 00000000..ee205cf3 --- /dev/null +++ b/apps/foundation/test/notifications/webhook_test.exs @@ -0,0 +1,107 @@ +defmodule Foundation.Notifications.WebhookTest do + use ExUnit.Case, async: false + + import Mock + + alias Foundation.Notifications.Webhook + alias Foundation.Notifications.Worker + + @config %Worker{ + adapter: Webhook, + url: "https://hooks.example.com/deployex", + enabled: true, + events: [:crash_restart], + options: %{} + } + + @payload %{node: :deployex@host, sname: "myapp-1", crash_restart_count: 1} + + describe "notify/3" do + @tag :capture_log + test "returns :ok on a 2xx response" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 200, headers: [], body: "ok"}} + end + ]} + ]) do + assert :ok = Webhook.notify(:crash_restart, @payload, @config) + end + end + + @tag :capture_log + test "returns :ok for any 2xx status code" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 204, headers: [], body: ""}} + end + ]} + ]) do + assert :ok = Webhook.notify(:crash_restart, @payload, @config) + end + end + + @tag :capture_log + test "returns {:error, {:http_error, status}} on a non-2xx response" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 500, headers: [], body: "Internal Server Error"}} + end + ]} + ]) do + assert {:error, {:http_error, 500}} = + Webhook.notify(:crash_restart, @payload, @config) + end + end + + @tag :capture_log + test "returns {:error, reason} on a transport error" do + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, _headers, _body -> %{} end, + request: fn _req, Foundation.Finch -> {:error, :timeout} end + ]} + ]) do + assert {:error, :timeout} = Webhook.notify(:crash_restart, @payload, @config) + end + end + + test "posts JSON with event, timestamp and payload fields" do + test_pid = self() + + with_mocks([ + {Finch, [], + [ + build: fn :post, _url, headers, body -> + send(test_pid, {:request_body, headers, body}) + %{} + end, + request: fn _req, Foundation.Finch -> + {:ok, %Finch.Response{status: 200, headers: [], body: "ok"}} + end + ]} + ]) do + Webhook.notify(:crash_restart, @payload, @config) + + assert_receive {:request_body, headers, body} + + assert {"content-type", "application/json"} in headers + + decoded = Jason.decode!(body) + assert decoded["event"] == "crash_restart" + assert is_binary(decoded["timestamp"]) + assert is_map(decoded["payload"]) + end + end + end +end diff --git a/apps/foundation/test/support/files/deployex-notifications.yaml b/apps/foundation/test/support/files/deployex-notifications.yaml new file mode 100644 index 00000000..8e14960f --- /dev/null +++ b/apps/foundation/test/support/files/deployex-notifications.yaml @@ -0,0 +1,47 @@ +account_name: "prod" +hostname: "deployex.example.com" +port: 5001 +release_adapter: "s3" +release_bucket: "myapp-prod-distribution" +secrets_adapter: "aws" +secrets_path: "deployex-myapp-prod-secrets" +aws_region: "sa-east-1" +version: "0.4.0" +otp_version: 26 +os_target: "ubuntu-20.04" +notifications: + - adapter: "webhook" + url: "https://hooks.example.com/deployex" + enabled: true + events: + - "crash_restart" + - "deployment_started" + - "deployment_complete" + - "watchdog_threshold_exceeded" + - "certificate_renewed" + - "certificate_failed" + - adapter: "slack" + url: "https://hooks.slack.com/services/T000/B000/XXX" + enabled: true + events: + - "crash_restart" + - "deployment_complete" + options: + username: "DeployEx-Bot" + icon_emoji: ":rocket:" + - adapter: "pagerduty" + enabled: true + events: + - "crash_restart" + - "watchdog_threshold_exceeded" + options: + routing_key: "abc123def456" + - adapter: "webhook" + url: "https://hooks2.example.com/deployex" + enabled: false + events: + - "crash_restart" +applications: + - name: "myphoenixapp" + language: "elixir" + replicas: 3 diff --git a/apps/foundation/test/yaml_test.exs b/apps/foundation/test/yaml_test.exs index 0455a02a..2926f64c 100644 --- a/apps/foundation/test/yaml_test.exs +++ b/apps/foundation/test/yaml_test.exs @@ -19,6 +19,7 @@ defmodule Foundation.YamlTest do @yaml_aws_optional "#{@file_paths}/deployex-aws-optional.yaml" @yaml_deployex_aws_no_replica_ports "#{@file_paths}/deployex-aws-no-replica-ports.yaml" @yaml_dns_cloudflare "#{@file_paths}/deployex-dns-cloudflare.yaml" + @yaml_notifications "#{@file_paths}/deployex-notifications.yaml" describe "load/0" do test "successfully loads and parses YAML configuration" do @@ -578,4 +579,92 @@ defmodule Foundation.YamlTest do assert cert.dns_options.ttl == 1 end end + + describe "notifications parsing" do + test "parses a webhook notification entry" do + with_mocks([ + {System, [:passthrough], + [get_env: fn "DEPLOYEX_CONFIG_YAML_PATH" -> @yaml_notifications end]} + ]) do + {:ok, config} = Yaml.load() + + [webhook | _] = config.notifications + + assert %Yaml.Notification{} = webhook + assert webhook.adapter == Foundation.Notifications.Webhook + assert webhook.url == "https://hooks.example.com/deployex" + assert webhook.enabled == true + assert webhook.options == %{} + + assert :crash_restart in webhook.events + assert :deployment_started in webhook.events + assert :deployment_complete in webhook.events + assert :watchdog_threshold_exceeded in webhook.events + assert :certificate_renewed in webhook.events + assert :certificate_failed in webhook.events + end + end + + test "parses a slack notification entry with options" do + with_mocks([ + {System, [:passthrough], + [get_env: fn "DEPLOYEX_CONFIG_YAML_PATH" -> @yaml_notifications end]} + ]) do + {:ok, config} = Yaml.load() + + [_, slack | _] = config.notifications + + assert slack.adapter == Foundation.Notifications.Slack + assert slack.url == "https://hooks.slack.com/services/T000/B000/XXX" + assert slack.enabled == true + assert slack.options[:username] == "DeployEx-Bot" + assert slack.options[:icon_emoji] == ":rocket:" + assert :crash_restart in slack.events + assert :deployment_complete in slack.events + end + end + + test "parses a pagerduty notification entry with options" do + with_mocks([ + {System, [:passthrough], + [get_env: fn "DEPLOYEX_CONFIG_YAML_PATH" -> @yaml_notifications end]} + ]) do + {:ok, config} = Yaml.load() + + [_, _, pagerduty | _] = config.notifications + + assert pagerduty.adapter == Foundation.Notifications.PagerDuty + assert pagerduty.url == nil + assert pagerduty.enabled == true + assert pagerduty.options[:routing_key] == "abc123def456" + assert :crash_restart in pagerduty.events + assert :watchdog_threshold_exceeded in pagerduty.events + end + end + + test "parses a disabled notification entry" do + with_mocks([ + {System, [:passthrough], + [get_env: fn "DEPLOYEX_CONFIG_YAML_PATH" -> @yaml_notifications end]} + ]) do + {:ok, config} = Yaml.load() + + [_, _, _, disabled] = config.notifications + + assert disabled.adapter == Foundation.Notifications.Webhook + assert disabled.enabled == false + assert disabled.url == "https://hooks2.example.com/deployex" + end + end + + test "defaults to empty notifications list when key is absent" do + with_mocks([ + {System, [:passthrough], + [get_env: fn "DEPLOYEX_CONFIG_YAML_PATH" -> @yaml_aws_default end]} + ]) do + {:ok, config} = Yaml.load() + assert config.notifications == [] + end + end + end end diff --git a/apps/sentinel/lib/sentinel/watchdog/watchdog.ex b/apps/sentinel/lib/sentinel/watchdog/watchdog.ex index eeb5d20a..e4a7e1b7 100644 --- a/apps/sentinel/lib/sentinel/watchdog/watchdog.ex +++ b/apps/sentinel/lib/sentinel/watchdog/watchdog.ex @@ -383,6 +383,14 @@ defmodule Sentinel.Watchdog do %{sname: sname} = Catalog.node_info(node) Monitor.restart(sname) + Foundation.Notifications.notify(:watchdog_threshold_exceeded, %{ + node: node, + sname: sname, + type: type, + current_percentage: current_percentage, + restart_threshold_percent: restart_threshold_percent + }) + :ok end @@ -446,6 +454,14 @@ defmodule Sentinel.Watchdog do %{sname: sname} = Catalog.node_info(node) Monitor.restart(sname) + Foundation.Notifications.notify(:watchdog_threshold_exceeded, %{ + node: node, + sname: sname, + type: :memory, + current_percentage: current_percentage, + restart_threshold_percent: restart_threshold_percent + }) + :ok end diff --git a/config/config.exs b/config/config.exs index d1791603..deaa3c97 100644 --- a/config/config.exs +++ b/config/config.exs @@ -34,6 +34,7 @@ config :foundation, monitored_app_log_path: "/var/log/monitored-apps", monitoring: [], applications: [], + notifications: [], healthcheck_logging: false, logs_retention_time_ms: :timer.hours(1), config_checksum: nil diff --git a/config/dev.exs b/config/dev.exs index b2b725d8..8c63191e 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -85,6 +85,24 @@ config :foundation, restart_threshold_percent: 95 } ], + notifications: [], + # NOTE: Enable for notifications testing + # notifications: [ + # %{ + # adapter: Foundation.Notifications.Webhook, + # url: "https://example.com/hooks/deployex", + # enabled: true, + # events: [ + # :crash_restart, + # :deployment_started, + # :deployment_complete, + # :watchdog_threshold_exceeded, + # :certificate_renewed, + # :certificate_failed + # ], + # options: %{} + # } + # ], applications: [ %{ name: "myphoenixapp", diff --git a/devops/installer/deployex-aws.yaml b/devops/installer/deployex-aws.yaml index 461c85f6..deb3d7a2 100644 --- a/devops/installer/deployex-aws.yaml +++ b/devops/installer/deployex-aws.yaml @@ -12,6 +12,35 @@ otp_tls_certificates: "/usr/local/share/ca-certificates" os_target: "ubuntu-24.04" metrics_retention_time_ms: 3600000 logs_retention_time_ms: 3600000 +notifications: + - adapter: "webhook" # options: webhook, slack, pagerduty + url: "https://example.com/hooks/deployex" + enabled: true + events: + - "crash_restart" + - "deployment_started" + - "deployment_complete" + - "watchdog_threshold_exceeded" + - "certificate_renewed" + - "certificate_failed" + - adapter: "slack" + url: "https://hooks.slack.com/services/T.../B.../..." + enabled: true + events: + - "crash_restart" + - "deployment_complete" + - "watchdog_threshold_exceeded" + options: + username: "DeployEx" + icon_emoji: ":rocket:" + - adapter: "pagerduty" + enabled: true + events: + - "crash_restart" + - "watchdog_threshold_exceeded" + - "certificate_failed" + options: + routing_key: "YOUR_PAGERDUTY_INTEGRATION_KEY" monitoring: - type: "memory" enable_restart: true diff --git a/devops/installer/deployex-gcp.yaml b/devops/installer/deployex-gcp.yaml index 41025003..f3d0a4cf 100644 --- a/devops/installer/deployex-gcp.yaml +++ b/devops/installer/deployex-gcp.yaml @@ -12,6 +12,35 @@ otp_tls_certificates: "/usr/local/share/ca-certificates" os_target: "ubuntu-24.04" metrics_retention_time_ms: 3600000 logs_retention_time_ms: 3600000 +notifications: + - adapter: "webhook" # options: webhook, slack, pagerduty + url: "https://example.com/hooks/deployex" + enabled: true + events: + - "crash_restart" + - "deployment_started" + - "deployment_complete" + - "watchdog_threshold_exceeded" + - "certificate_renewed" + - "certificate_failed" + - adapter: "slack" + url: "https://hooks.slack.com/services/T.../B.../..." + enabled: true + events: + - "crash_restart" + - "deployment_complete" + - "watchdog_threshold_exceeded" + options: + username: "DeployEx" + icon_emoji: ":rocket:" + - adapter: "pagerduty" + enabled: true + events: + - "crash_restart" + - "watchdog_threshold_exceeded" + - "certificate_failed" + options: + routing_key: "YOUR_PAGERDUTY_INTEGRATION_KEY" monitoring: - type: "memory" enable_restart: true diff --git a/guides/docs/yaml/README.md b/guides/docs/yaml/README.md index db45f15d..f3e160a1 100644 --- a/guides/docs/yaml/README.md +++ b/guides/docs/yaml/README.md @@ -67,6 +67,45 @@ monitored_app_log_path: "/var/log/monitored-apps" # Monitored apps log pa metrics_retention_time_ms: 3600000 # Metrics retention: 1 hour (default: 3600000) logs_retention_time_ms: 3600000 # Logs retention: 1 hour (default: 3600000) +# ============================================================================ +# EXTERNAL NOTIFICATIONS (optional) +# ============================================================================ + +notifications: + # Webhook adapter — generic HTTP POST to any endpoint + - adapter: "webhook" # Adapter: webhook, slack or pagerduty + url: "https://example.com/hooks/deployex" # HTTP POST destination + enabled: true + events: # One or more of the six supported events + - "crash_restart" # Monitored app crashed and was restarted + - "deployment_started" # New deployment was initiated + - "deployment_complete" # Deployment finished (success or failure) + - "watchdog_threshold_exceeded" # Resource threshold crossed; app restarted + - "certificate_renewed" # TLS certificate successfully renewed + - "certificate_failed" # TLS certificate renewal failed + + # Slack Incoming Webhook adapter + - adapter: "slack" + url: "https://hooks.slack.com/services/T.../B.../..." # Slack Incoming Webhook URL + enabled: true + events: + - "crash_restart" + - "deployment_complete" + - "watchdog_threshold_exceeded" + options: + username: "DeployEx" # Bot display name (default: DeployEx) + icon_emoji: ":rocket:" # Bot icon emoji (default: :robot_face:) + + # PagerDuty Events API v2 adapter + - adapter: "pagerduty" + enabled: true # url: omit to use the default PagerDuty endpoint + events: + - "crash_restart" + - "watchdog_threshold_exceeded" + - "certificate_failed" + options: + routing_key: "abc123def456..." # PagerDuty integration key (required) + # ============================================================================ # HOST-LEVEL MONITORING (optional) # ============================================================================ @@ -225,6 +264,7 @@ These fields require a **full DeployEx restart**: - `secrets_path` - Secrets path - `aws_region` - AWS region - `google_credentials` - GCP credentials path +- `notifications` - External notification channels (requires restart to add, remove, or reconfigure) #### System Requirements - `version` - DeployEx version diff --git a/mix/shared.exs b/mix/shared.exs index ae940f8c..94d73d21 100644 --- a/mix/shared.exs +++ b/mix/shared.exs @@ -1,5 +1,5 @@ defmodule Mix.Shared do - def version, do: "0.9.3" + def version, do: "0.9.4-rc1" def elixir, do: "~> 1.16"