diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f5f60267..28cb98a19 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,35 @@ Thank you for your interest in contributing to this project! 5. Push to the branch (`git push origin feature/amazing-feature`) 6. Open a Pull Request +### Test without TEE + +Maintainers can run a development image without TDX to test VMM and guest changes. + +> [!WARNING] +> A no-TEE VM has no hardware isolation or attestation. Its disk is unencrypted, and its temporary app keys are not stable across boots. Do not use this mode for production workloads or secrets. + +Create an app compose file without KMS, Gateway, or TEE requirements: + +```bash +python3 vmm/src/vmm-cli.py compose \ + --name no-tee-dev \ + --docker-compose ./docker-compose.yml \ + --key-provider none \ + --output ./app-compose.json +``` + +Deploy it with an image whose metadata sets `is_dev` to `true`: + +```bash +python3 vmm/src/vmm-cli.py deploy \ + --name no-tee-dev \ + --image "YOUR_DEVELOPMENT_IMAGE" \ + --compose ./app-compose.json \ + --no-tee +``` + +Attestation and runtime measurement APIs return errors in this mode. TEE and storage modes cannot change after VM creation; recreate the VM to switch modes. + ## Commit Convention This project uses [Conventional Commits](https://www.conventionalcommits.org/). Please format your commit messages as: diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index 81e23363a..d24ae78d4 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -99,7 +99,22 @@ log "Syncing system time..." # Let the chronyd correct the system time immediately; keep booting if chronyd is not ready yet. chronyc makestep || log "Warning: chronyc makestep failed; continuing" -if [[ -e /dev/sev-guest ]] || grep -qw sev_guest /sys/kernel/config/tsm/report/*/provider 2>/dev/null; then +case "$(get_cmdline_value dstack.no_tee || true)" in +1 | true | yes | on) + NO_TEE=true + ;; +0 | false | no | off | "") + NO_TEE=false + ;; +*) + log "Error: invalid dstack.no_tee value" + exit 1 + ;; +esac + +if $NO_TEE; then + log "Development-only no-TEE mode enabled" +elif [[ -e /dev/sev-guest ]] || grep -qw sev_guest /sys/kernel/config/tsm/report/*/provider 2>/dev/null; then log "SEV-SNP guest device/TSM provider detected" elif [[ -e /dev/tdx_guest ]]; then log "TDX guest device detected" @@ -127,7 +142,9 @@ setup_tsm() { mkdir -p /sys/kernel/config/tsm/report/com.intel.dcap fi } -setup_tsm || true +if ! $NO_TEE; then + setup_tsm || true +fi # Setup dstack system log "Preparing dstack system..." diff --git a/cert-client/src/lib.rs b/cert-client/src/lib.rs index 8b5b329a1..78f5d6abf 100644 --- a/cert-client/src/lib.rs +++ b/cert-client/src/lib.rs @@ -2,11 +2,16 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::{Context, Result}; +use std::time::{Duration, UNIX_EPOCH}; + +use anyhow::{bail, Context, Result}; use dstack_kms_rpc::{kms_client::KmsClient, SignCertRequest}; use dstack_types::{AppKeys, KeyProvider}; use ra_rpc::client::{RaClient, RaClientConfig}; -use ra_tls::cert::{generate_ra_cert, CaCert, CertSigningRequestV2}; +use ra_tls::{ + cert::{generate_ra_cert, CaCert, CertConfigV2, CertRequest, CertSigningRequestV2}, + rcgen::SubjectPublicKeyInfo, +}; pub enum CertRequestClient { Local { @@ -19,6 +24,37 @@ pub enum CertRequestClient { } impl CertRequestClient { + pub fn sign_unattested(&self, pubkey: &[u8], config: &CertConfigV2) -> Result> { + if config.ext_quote || config.ext_app_info { + bail!("Attestation is unavailable in no-TEE mode"); + } + let CertRequestClient::Local { ca } = self else { + bail!("Unattested certificates require a local key provider"); + }; + let pki = SubjectPublicKeyInfo::from_der(pubkey).context("Failed to parse public key")?; + let req = CertRequest::builder() + .key(&pki) + .subject(&config.subject) + .maybe_org_name(config.org_name.as_deref()) + .alt_names(&config.subject_alt_names) + .usage_server_auth(config.usage_server_auth) + .usage_client_auth(config.usage_client_auth) + .special_usage("app:custom") + .maybe_not_before( + config + .not_before + .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds)), + ) + .maybe_not_after( + config + .not_after + .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds)), + ) + .build(); + let cert = ca.sign(req).context("Failed to sign certificate")?; + Ok(vec![cert.pem(), ca.pem_cert.clone()]) + } + pub async fn sign_csr( &self, csr: &CertSigningRequestV2, diff --git a/dstack-util/src/main.rs b/dstack-util/src/main.rs index c9ec7ef70..8ab8be053 100644 --- a/dstack-util/src/main.rs +++ b/dstack-util/src/main.rs @@ -840,6 +840,21 @@ fn gen_app_keys_from_seed( provider: KeyProviderKind, mr: Option>, ) -> Result { + let (key, disk_key, k256_key, key_provider) = derive_app_keys_from_seed(seed, provider, mr)?; + make_app_keys(&key, &disk_key, &k256_key, 1, key_provider) +} + +fn gen_dev_app_keys_from_seed(seed: &[u8]) -> Result { + let (key, disk_key, k256_key, key_provider) = + derive_app_keys_from_seed(seed, KeyProviderKind::None, None)?; + make_dev_app_keys(&key, &disk_key, &k256_key, 1, key_provider) +} + +fn derive_app_keys_from_seed( + seed: &[u8], + provider: KeyProviderKind, + mr: Option>, +) -> Result<(KeyPair, KeyPair, SigningKey, KeyProvider)> { let key = derive_p256_key_pair_from_bytes(seed, &["app-key".as_bytes()])?; let disk_key = derive_p256_key_pair_from_bytes(seed, &["app-disk-key".as_bytes()])?; let k256_key = derive_key(seed, &["app-k256-key".as_bytes()], 32)?; @@ -860,7 +875,7 @@ fn gen_app_keys_from_seed( anyhow::bail!("KMS keys must be fetched from the KMS server") } }; - make_app_keys(&key, &disk_key, &k256_key, 1, key_provider) + Ok((key, disk_key, k256_key, key_provider)) } fn make_app_keys( @@ -886,15 +901,54 @@ fn make_app_keys( .self_signed() .context("Failed to self-sign certificate")?; - Ok(AppKeys { + Ok(app_keys_with_cert( + disk_key, + k256_key, + cert.pem(), + key_provider, + )) +} + +fn make_dev_app_keys( + app_key: &KeyPair, + disk_key: &KeyPair, + k256_key: &SigningKey, + ca_level: u8, + key_provider: KeyProvider, +) -> Result { + use ra_tls::cert::CertRequest; + let req = CertRequest::builder() + .subject("App Root Cert") + .key(app_key) + .ca_level(ca_level) + .build(); + let cert = req + .self_signed() + .context("Failed to self-sign certificate")?; + + Ok(app_keys_with_cert( + disk_key, + k256_key, + cert.pem(), + key_provider, + )) +} + +fn app_keys_with_cert( + disk_key: &KeyPair, + k256_key: &SigningKey, + ca_cert: String, + key_provider: KeyProvider, +) -> AppKeys { + AppKeys { disk_crypt_key: sha256(&disk_key.serialize_der()).to_vec(), env_crypt_key: vec![], k256_key: k256_key.to_bytes().to_vec(), k256_signature: vec![], - gateway_app_id: "".to_string(), - ca_cert: cert.pem(), + gateway_app_id: String::new(), + ca_cert, key_provider, - }) + } } async fn cmd_notify_host(args: HostNotifyArgs) -> Result<()> { diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index bd4f778ad..0a825ebd2 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -45,7 +45,7 @@ use tracing::{info, warn}; use crate::{ cmd_show_mrs, crypto::dh_decrypt, - gen_app_keys_from_seed, + gen_app_keys_from_seed, gen_dev_app_keys_from_seed, host_api::HostApi, utils::{ deserialize_json_file, sha256, sha256_file, AppCompose, AppKeys, KeyProviderKind, SysConfig, @@ -159,6 +159,15 @@ impl FromStr for FsType { struct DstackOptions { storage_encrypted: bool, storage_fs: FsType, + no_tee: bool, +} + +fn parse_cmdline_bool(key: &str, value: &str) -> Result { + match value { + "0" | "false" | "no" | "off" => Ok(false), + "1" | "true" | "yes" | "on" => Ok(true), + _ => bail!("Invalid value for {key}: {value}"), + } } fn parse_dstack_options(shared: &HostShared) -> Result { @@ -167,19 +176,16 @@ fn parse_dstack_options(shared: &HostShared) -> Result { let mut options = DstackOptions { storage_encrypted: true, // Default to encryption enabled storage_fs: FsType::Zfs, // Default to ZFS + no_tee: false, }; for param in cmdline.split_whitespace() { if let Some(value) = param.strip_prefix("dstack.storage_encrypted=") { - match value { - "0" | "false" | "no" | "off" => options.storage_encrypted = false, - "1" | "true" | "yes" | "on" => options.storage_encrypted = true, - _ => { - bail!("Invalid value for dstack.storage_encrypted: {value}"); - } - } + options.storage_encrypted = parse_cmdline_bool("dstack.storage_encrypted", value)?; } else if let Some(value) = param.strip_prefix("dstack.storage_fs=") { options.storage_fs = value.parse().context("Failed to parse dstack.storage_fs")?; + } else if let Some(value) = param.strip_prefix("dstack.no_tee=") { + options.no_tee = parse_cmdline_bool("dstack.no_tee", value)?; } } @@ -189,6 +195,13 @@ fn parse_dstack_options(shared: &HostShared) -> Result { Ok(options) } +fn emit_runtime_event_for_setup(opts: &DstackOptions, event: &str, payload: &[u8]) -> Result<()> { + if opts.no_tee { + return Ok(()); + } + emit_runtime_event(event, payload) +} + #[derive(Clone)] pub struct HostShareDir { base_dir: PathBuf, @@ -759,10 +772,10 @@ fn platform_instance_binding() -> Result>> { } } -fn emit_key_provider_info(provider_info: &KeyProviderInfo) -> Result<()> { +fn emit_key_provider_info(provider_info: &KeyProviderInfo, opts: &DstackOptions) -> Result<()> { info!("Key provider info: {provider_info:?}"); let provider_info_json = serde_json::to_vec(&provider_info)?; - emit_runtime_event("key-provider", &provider_info_json)?; + emit_runtime_event_for_setup(opts, "key-provider", &provider_info_json)?; Ok(()) } @@ -1109,6 +1122,7 @@ struct Stage1<'a> { vmm: HostApi, shared: HostShared, keys: AppKeys, + no_tee: bool, } impl<'a> Stage0<'a> { @@ -1316,16 +1330,23 @@ impl<'a> Stage0<'a> { .context("failed to generate TPM app keys") } - async fn request_app_keys(&self) -> Result { + async fn request_app_keys(&self, opts: &DstackOptions) -> Result { let key_provider = self.shared.app_compose.key_provider(); + if opts.no_tee && !key_provider.is_none() { + bail!("no-TEE mode requires key_provider=none"); + } match key_provider { KeyProviderKind::Kms => self.request_app_keys_from_kms().await, KeyProviderKind::Local => self.get_keys_from_local_key_provider().await, KeyProviderKind::None => { info!("No key provider is enabled, generating temporary app keys"); let seed: [u8; 32] = rand::thread_rng().gen(); - gen_app_keys_from_seed(&seed, KeyProviderKind::None, None) - .context("Failed to generate app keys") + if opts.no_tee { + gen_dev_app_keys_from_seed(&seed).context("Failed to generate app keys") + } else { + gen_app_keys_from_seed(&seed, KeyProviderKind::None, None) + .context("Failed to generate app keys") + } } KeyProviderKind::Tpm => { info!("Generating app keys from TPM"); @@ -1657,14 +1678,18 @@ impl<'a> Stage0<'a> { Ok(()) } - fn measure_app_info(&self) -> Result { + fn measure_app_info(&self, opts: &DstackOptions) -> Result { let compose_hash = sha256_file(self.shared.dir.app_compose_file())?; let truncated_compose_hash = truncate(&compose_hash, 20); let key_provider = self.shared.app_compose.key_provider(); let mut instance_info = self.shared.instance_info.clone(); - let is_snp = AttestationMode::detect() - .map(|mode| mode == AttestationMode::DstackAmdSevSnp) - .unwrap_or(false); + let is_snp = if opts.no_tee { + false + } else { + AttestationMode::detect() + .map(|mode| mode == AttestationMode::DstackAmdSevSnp) + .unwrap_or(false) + }; if instance_info.app_id.is_empty() { instance_info.app_id = truncated_compose_hash.to_vec(); @@ -1689,7 +1714,7 @@ impl<'a> Stage0<'a> { } else { let mut id_path = instance_info.instance_id_seed.clone(); id_path.extend_from_slice(&instance_info.app_id); - if !is_snp { + if !is_snp && !opts.no_tee { if let Some(binding) = platform_instance_binding()? { info!("mixing platform per-instance binding into instance_id"); id_path.extend_from_slice(&binding); @@ -1706,31 +1731,33 @@ impl<'a> Stage0<'a> { // no KMS to bind it, the relying party MUST gate the compose_hash // (which launcher build) separately from the app_id (which app). - emit_runtime_event("system-preparing", &[])?; - emit_runtime_event("app-id", &instance_info.app_id)?; - emit_runtime_event("compose-hash", &compose_hash)?; - emit_runtime_event("instance-id", &instance_id)?; - emit_runtime_event("boot-mr-done", &[])?; + emit_runtime_event_for_setup(opts, "system-preparing", &[])?; + emit_runtime_event_for_setup(opts, "app-id", &instance_info.app_id)?; + emit_runtime_event_for_setup(opts, "compose-hash", &compose_hash)?; + emit_runtime_event_for_setup(opts, "instance-id", &instance_id)?; + emit_runtime_event_for_setup(opts, "boot-mr-done", &[])?; Ok(AppInfo { instance_info, compose_hash, }) } - fn verify_app(&self, app_info: &AppInfo, keys: &AppKeys) -> Result<()> { - config_id_verifier::verify_mr_config_id( - &app_info.compose_hash, - &app_info - .instance_info - .app_id - .as_slice() - .try_into() - .ok() - .context("Invalid app id")?, - &app_info.instance_info.instance_id, - keys.key_provider.kind(), - keys.key_provider.id(), - )?; + fn verify_app(&self, app_info: &AppInfo, keys: &AppKeys, opts: &DstackOptions) -> Result<()> { + if !opts.no_tee { + config_id_verifier::verify_mr_config_id( + &app_info.compose_hash, + &app_info + .instance_info + .app_id + .as_slice() + .try_into() + .ok() + .context("Invalid app id")?, + &app_info.instance_info.instance_id, + keys.key_provider.kind(), + keys.key_provider.id(), + )?; + } self.verify_key_provider_id(keys.key_provider.id())?; let kp_info = match &keys.key_provider { KeyProvider::None { .. } => KeyProviderInfo::new("none".into(), "".into()), @@ -1744,38 +1771,46 @@ impl<'a> Stage0<'a> { KeyProviderInfo::new("kms".into(), hex::encode(pubkey)) } }; - emit_key_provider_info(&kp_info)?; + emit_key_provider_info(&kp_info, opts)?; Ok(()) } async fn setup_fs(self) -> Result> { + let opts = parse_dstack_options(&self.shared).context("Failed to parse kernel cmdline")?; + if opts.no_tee { + if opts.storage_encrypted { + bail!("no-TEE mode requires storage encryption disabled"); + } + warn!("Development-only no-TEE mode enabled; storage is unencrypted"); + } let app_info = self - .measure_app_info() + .measure_app_info(&opts) .context("Failed to measure app info")?; - if self.shared.app_compose.key_provider().is_kms() { + if self.shared.app_compose.key_provider().is_kms() && !opts.no_tee { cmd_show_mrs()?; } - self.vmm - .notify_q("boot.progress", "requesting app keys") - .await; + let key_progress = if opts.no_tee { + "generating temporary app keys" + } else { + "requesting app keys" + }; + self.vmm.notify_q("boot.progress", key_progress).await; let app_keys = self - .request_app_keys() + .request_app_keys(&opts) .await .context("Failed to request app keys")?; if app_keys.disk_crypt_key.is_empty() { bail!("Failed to get valid key phrase from KMS"); } - self.verify_app(&app_info, &app_keys) + self.verify_app(&app_info, &app_keys, &opts) .context("Failed to verify app")?; // Save app keys let keys_json = serde_json::to_string(&app_keys).context("Failed to serialize app keys")?; fs::write(self.app_keys_file(), keys_json).context("Failed to write app keys")?; - // Parse kernel command line options - let opts = parse_dstack_options(&self.shared).context("Failed to parse kernel cmdline")?; - emit_runtime_event("storage-fs", opts.storage_fs.to_string().as_bytes())?; + emit_runtime_event_for_setup(&opts, "storage-fs", opts.storage_fs.to_string().as_bytes())?; info!( "Filesystem options: encryption={}, filesystem={:?}", opts.storage_encrypted, opts.storage_fs @@ -1791,10 +1826,10 @@ impl<'a> Stage0<'a> { &serde_json::to_string(&app_info.instance_info)?, ) .await; - emit_runtime_event("system-ready", &[])?; + emit_runtime_event_for_setup(&opts, "system-ready", &[])?; self.vmm.notify_q("boot.progress", "data disk ready").await; - if !self.shared.app_compose.key_provider().is_kms() { + if !self.shared.app_compose.key_provider().is_kms() && !opts.no_tee { cmd_show_mrs()?; } Ok(Stage1 { @@ -1802,6 +1837,7 @@ impl<'a> Stage0<'a> { shared: self.shared, vmm: self.vmm, keys: app_keys, + no_tee: opts.no_tee, }) } } @@ -1903,6 +1939,7 @@ impl Stage1<'_> { "core": { "pccs_url": self.shared.sys_config.pccs_url, "data_disks": data_disks, + "no_tee": self.no_tee, } } }); @@ -2171,6 +2208,22 @@ fn test_validate_luks2_header() { .contains("Invalid LUKS keyslot encryption")); } +#[test] +fn test_parse_cmdline_bool() { + for value in ["1", "true", "yes", "on"] { + assert!(parse_cmdline_bool("test", value).unwrap()); + } + for value in ["0", "false", "no", "off"] { + assert!(!parse_cmdline_bool("test", value).unwrap()); + } + assert_eq!( + parse_cmdline_bool("test", "invalid") + .unwrap_err() + .to_string(), + "Invalid value for test: invalid" + ); +} + #[cfg(test)] fn test_app_compose( manifest_version: serde_json::Value, diff --git a/guest-agent/src/config.rs b/guest-agent/src/config.rs index 7f94185d7..d771a46ab 100644 --- a/guest-agent/src/config.rs +++ b/guest-agent/src/config.rs @@ -51,6 +51,8 @@ pub struct Config { #[serde(default)] pub pccs_url: Option, pub data_disks: HashSet, + #[serde(default)] + pub no_tee: bool, } fn deserialize_app_compose<'de, D>(deserializer: D) -> Result diff --git a/guest-agent/src/rpc_service.rs b/guest-agent/src/rpc_service.rs index 89ad6ac87..ce4ca7835 100644 --- a/guest-agent/src/rpc_service.rs +++ b/guest-agent/src/rpc_service.rs @@ -65,11 +65,20 @@ struct AppStateInner { impl AppStateInner { fn info_attestation(&self) -> Result { + if self.config.no_tee { + anyhow::bail!("Attestation is unavailable in no-TEE mode"); + } self.platform.attestation_for_info() } async fn issue_cert(&self, key: &KeyPair, config: CertConfigV2) -> Result> { let pubkey = key.public_key_der(); + if self.config.no_tee { + return self + .cert_client + .sign_unattested(&pubkey, &config) + .context("Failed to sign development certificate"); + } let attestation = self .platform .certificate_attestation(&pubkey) @@ -113,6 +122,9 @@ impl AppStateInner { impl AppState { fn maybe_request_demo_cert(&self) { + if self.config().no_tee { + return; + } let state = self.inner.clone(); if !state .demo_cert @@ -171,16 +183,25 @@ impl AppState { } fn quote_response(&self, report_data: [u8; 64]) -> Result { + if self.config().no_tee { + anyhow::bail!("Attestation is unavailable in no-TEE mode"); + } self.inner .platform .quote_response(report_data, &self.inner.vm_config) } fn attest_response(&self, report_data: [u8; 64]) -> Result { + if self.config().no_tee { + anyhow::bail!("Attestation is unavailable in no-TEE mode"); + } self.inner.platform.attest_response(report_data) } fn emit_event(&self, event: &str, payload: &[u8]) -> Result<()> { + if self.config().no_tee { + anyhow::bail!("Runtime measurements are unavailable in no-TEE mode"); + } self.inner.platform.emit_event(event, payload) } } @@ -693,6 +714,10 @@ mod tests { } async fn setup_test_state() -> (AppState, tempfile::NamedTempFile) { + setup_test_state_with_mode(false).await + } + + async fn setup_test_state_with_mode(no_tee: bool) -> (AppState, tempfile::NamedTempFile) { let mut temp_attestation_file = tempfile::NamedTempFile::new().unwrap(); let attestation = include_bytes!("../fixtures/attestation.bin"); @@ -733,6 +758,7 @@ mod tests { sys_config_file: String::new().into(), pccs_url: None, data_disks: HashSet::new(), + no_tee, }; const DUMMY_PEM_KEY: &str = r#"-----BEGIN PRIVATE KEY----- @@ -810,6 +836,7 @@ pNs85uhOZE8z2jr8Pg== struct TestSimulatorPlatform { attestation: VersionedAttestation, + deny_attestation: bool, } fn patch_report_data( @@ -821,10 +848,12 @@ pNs85uhOZE8z2jr8Pg== impl PlatformBackend for TestSimulatorPlatform { fn attestation_for_info(&self) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); Ok(self.attestation.clone()) } fn certificate_attestation(&self, pubkey: &[u8]) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); let report_data = ra_tls::attestation::QuoteContentType::RaTlsCert.to_report_data(pubkey); let attestation = patch_report_data(&self.attestation, report_data); @@ -836,6 +865,7 @@ pNs85uhOZE8z2jr8Pg== report_data: [u8; 64], vm_config: &str, ) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); let attestation = patch_report_data(&self.attestation, report_data); let Some(quote) = attestation.platform.tdx_quote().map(ToOwned::to_owned) else { return Err(anyhow::anyhow!("Quote not found")); @@ -853,6 +883,7 @@ pNs85uhOZE8z2jr8Pg== } fn attest_response(&self, report_data: [u8; 64]) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); let attestation = patch_report_data(&self.attestation, report_data); Ok(AttestResponse { attestation: VersionedAttestation::V1 { attestation }.to_bytes()?, @@ -860,6 +891,7 @@ pNs85uhOZE8z2jr8Pg== } fn emit_event(&self, _event: &str, _payload: &[u8]) -> Result<()> { + assert!(!self.deny_attestation, "unexpected measurement request"); Ok(()) } } @@ -875,6 +907,7 @@ pNs85uhOZE8z2jr8Pg== &std::fs::read(temp_attestation_file.path()).unwrap(), ) .unwrap(), + deny_attestation: no_tee, }), }; @@ -886,6 +919,55 @@ pNs85uhOZE8z2jr8Pg== ) } + #[tokio::test] + async fn no_tee_supports_unattested_keys_and_rejects_attestation() { + let (state, _guard) = setup_test_state_with_mode(true).await; + let handler = InternalRpcHandler { + state: state.clone(), + }; + let response = handler + .get_tls_key(GetTlsKeyArgs { + subject: "development".to_string(), + usage_server_auth: true, + ..Default::default() + }) + .await + .unwrap(); + assert!(!response.key.is_empty()); + assert_eq!(response.certificate_chain.len(), 2); + + for err in [ + get_info(&state, false).await.unwrap_err(), + state.quote_response([0; 64]).unwrap_err(), + state.attest_response([0; 64]).unwrap_err(), + ] { + assert_eq!(err.to_string(), "Attestation is unavailable in no-TEE mode"); + } + assert_eq!( + state.emit_event("test", &[]).unwrap_err().to_string(), + "Runtime measurements are unavailable in no-TEE mode" + ); + + for request in [ + GetTlsKeyArgs { + usage_ra_tls: true, + ..Default::default() + }, + GetTlsKeyArgs { + with_app_info: true, + ..Default::default() + }, + ] { + let err = InternalRpcHandler { + state: state.clone(), + } + .get_tls_key(request) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("Attestation is unavailable in no-TEE mode")); + } + } + #[tokio::test] async fn test_verify_ed25519_success() { let (state, _guard) = setup_test_state().await; diff --git a/vmm/rpc/proto/vmm_rpc.proto b/vmm/rpc/proto/vmm_rpc.proto index 8d79fea66..ceae5937e 100644 --- a/vmm/rpc/proto/vmm_rpc.proto +++ b/vmm/rpc/proto/vmm_rpc.proto @@ -97,7 +97,7 @@ message VmConfiguration { repeated string gateway_urls = 15; // The VM is stopped bool stopped = 16; - // Disable confidential computing (fallback to non-TEE VM). + // Development only. Disables TEE and disk encryption. bool no_tee = 17; // Per-VM networking mode override (if unset, uses global cvm.networking). optional NetworkingConfig networking = 18; @@ -161,7 +161,7 @@ message UpdateVmRequest { optional uint32 memory = 15; optional uint32 disk_size = 16; optional string image = 17; - // Disable or re-enable TEE for an existing VM. + // Deprecated. TEE mode cannot change after VM creation. optional bool no_tee = 18; } diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 1d7c358cf..e3a3b06ee 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -8,10 +8,10 @@ use dstack_port_forward::{ForwardRule, ForwardService, Protocol as FwdProtocol}; use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_kms_rpc::kms_client::KmsClient; -use dstack_types::mr_config::MrConfigV3; use dstack_types::shared_filenames::{ APP_COMPOSE, ENCRYPTED_ENV, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, }; +use dstack_types::{mr_config::MrConfigV3, AppCompose}; use dstack_vmm_rpc::{ self as pb, GpuInfo, ReloadVmsResponse, StatusRequest, StatusResponse, VmConfiguration, }; @@ -130,6 +130,40 @@ pub(crate) fn round_up(value: u32, multiple: u32) -> u32 { value + (multiple - remainder) } +pub(crate) fn validate_no_tee_compose(no_tee: bool, app_compose: &AppCompose) -> Result<()> { + if !no_tee { + return Ok(()); + } + if !app_compose.key_provider().is_none() { + bail!("no-TEE mode requires key_provider=none"); + } + if app_compose.gateway_enabled() { + bail!("no-TEE mode does not support the gateway"); + } + if app_compose + .requirements + .as_ref() + .is_some_and(|requirements| { + requirements.platforms.is_some() || requirements.tdx_measure_acpi_tables.is_some() + }) + { + bail!("no-TEE mode does not support TEE requirements"); + } + Ok(()) +} + +pub(crate) fn validate_image_name(name: &str) -> Result<()> { + if name.len() > 64 + || name.contains("..") + || !name + .chars() + .all(|c| c.is_alphanumeric() || matches!(c, '_' | '-' | '.')) + { + bail!("Invalid image name"); + } + Ok(()) +} + /// Get the NUMA node associated with a PCI device. pub(crate) fn pci_numa_node(device: &str) -> Result { // Ensure the device string only contains valid hexadecimal characters and colons. @@ -264,21 +298,15 @@ impl App { ) -> Result<()> { let vm_work_dir = VmWorkDir::new(work_dir.as_ref()); let manifest = vm_work_dir.manifest().context("Failed to read manifest")?; - if manifest.image.len() > 64 - || manifest.image.contains("..") - || !manifest - .image - .chars() - .all(|c| c.is_alphanumeric() || c == '_' || c == '-' || c == '.') - { - bail!("Invalid image name"); - } + validate_image_name(&manifest.image)?; let image_path = self.config.image.path.join(&manifest.image); let image = Image::load(&image_path).context("Failed to load image")?; + image.validate_no_tee(manifest.no_tee)?; let vm_id = manifest.id.clone(); let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; { let mut states = self.lock(); let cid = states @@ -843,22 +871,16 @@ impl App { ) -> Result { let vm_work_dir = VmWorkDir::new(work_dir.as_ref()); let manifest = vm_work_dir.manifest().context("Failed to read manifest")?; - if manifest.image.len() > 64 - || manifest.image.contains("..") - || !manifest - .image - .chars() - .all(|c| c.is_alphanumeric() || c == '_' || c == '-' || c == '.') - { - bail!("Invalid image name"); - } + validate_image_name(&manifest.image)?; let image_path = self.config.image.path.join(&manifest.image); let image = Image::load(&image_path).context("Failed to load image")?; + image.validate_no_tee(manifest.no_tee)?; let vm_id = manifest.id.clone(); let already_running = cids_assigned.contains_key(&vm_id); let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; let mut is_new = false; { @@ -1259,6 +1281,7 @@ pub(crate) fn make_sys_config( mr_config: Option, requirements: Option<&dstack_types::Requirements>, ) -> Result { + validate_image_name(&manifest.image)?; let image_path = cfg.image.path.join(&manifest.image); let image = Image::load(image_path).context("Failed to load image info")?; let img_ver = image.info.version_tuple().unwrap_or((0, 0, 0)); @@ -1619,6 +1642,32 @@ mod tests { assert_eq!(effective_vcpu_count(3, None), 3); } + #[test] + fn no_tee_compose_rejects_attestation_dependencies() { + fn app_compose(extra: serde_json::Value) -> AppCompose { + let mut value = serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "docker-compose", + "key_provider": "none" + }); + value + .as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + serde_json::from_value(value).unwrap() + } + + validate_no_tee_compose(true, &app_compose(serde_json::json!({}))).unwrap(); + for extra in [ + serde_json::json!({"key_provider": "kms"}), + serde_json::json!({"gateway_enabled": true}), + serde_json::json!({"requirements": {"platforms": ["tdx"]}}), + ] { + assert!(validate_no_tee_compose(true, &app_compose(extra)).is_err()); + } + } + #[test] fn tdx_auto_variant_uses_legacy_for_low_non_2g_memory() -> Result<()> { let config = test_tdx_config()?; diff --git a/vmm/src/app/image.rs b/vmm/src/app/image.rs index 40f7df4fd..107a48f4c 100644 --- a/vmm/src/app/image.rs +++ b/vmm/src/app/image.rs @@ -82,6 +82,10 @@ pub struct Image { } impl Image { + pub(crate) fn validate_no_tee(&self, no_tee: bool) -> Result<()> { + validate_no_tee(no_tee, self.info.is_dev, self.info.cmdline.as_deref()) + } + /// Firmware blob to launch with, given whether this is an AMD SEV-SNP guest. /// SEV-SNP prefers the dedicated SEV firmware (`bios_sev`) and falls back to /// the generic `bios`; TDX always uses `bios`. @@ -94,6 +98,23 @@ impl Image { } } +fn validate_no_tee(no_tee: bool, is_dev: bool, cmdline: Option<&str>) -> Result<()> { + for param in cmdline.unwrap_or_default().split_whitespace() { + if param == "dstack.no_tee" { + bail!("image command line cannot enable dstack.no_tee"); + } + if let Some(value) = param.strip_prefix("dstack.no_tee=") { + if !matches!(value, "0" | "false" | "no" | "off") { + bail!("image command line cannot enable dstack.no_tee"); + } + } + } + if no_tee && !is_dev { + bail!("no-TEE mode requires a development image"); + } + Ok(()) +} + impl Image { pub fn load(base_path: impl AsRef) -> Result { let base_path = base_path.as_ref().absolutize()?; @@ -210,3 +231,32 @@ fn guess_version(base_path: &Path) -> Option { }; Some(version.to_string()) } + +#[cfg(test)] +mod tests { + use super::validate_no_tee; + + #[test] + fn no_tee_requires_development_image() { + assert!(validate_no_tee(false, false, None).is_ok()); + assert!(validate_no_tee(false, true, None).is_ok()); + assert!(validate_no_tee(true, true, None).is_ok()); + + let err = validate_no_tee(true, false, None).unwrap_err(); + assert_eq!(err.to_string(), "no-TEE mode requires a development image"); + } + + #[test] + fn image_cmdline_cannot_enable_no_tee() { + for cmdline in ["dstack.no_tee", "dstack.no_tee=1", "dstack.no_tee=invalid"] { + let err = validate_no_tee(false, false, Some(cmdline)).unwrap_err(); + assert_eq!( + err.to_string(), + "image command line cannot enable dstack.no_tee" + ); + } + for value in ["0", "false", "no", "off"] { + assert!(validate_no_tee(false, false, Some(&format!("dstack.no_tee={value}"))).is_ok()); + } + } +} diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 0749fc95b..9b79d7b78 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -478,8 +478,8 @@ impl VmState { #[cfg(test)] mod tests { use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, sanitize_optional, - virtio_pci_device, + amd_sev_snp_memory_backend_arg, guest_kernel_cmdline, parse_amd_sev_snp_qmp_capabilities, + sanitize_optional, virtio_pci_device, }; #[test] @@ -533,6 +533,27 @@ mod tests { "virtio-blk-pci,drive=hd0" ); } + + #[test] + fn no_tee_is_added_to_the_guest_cmdline() { + assert_eq!(guest_kernel_cmdline(None, false), None); + assert_eq!( + guest_kernel_cmdline(Some("console=ttyS0"), false).as_deref(), + Some("console=ttyS0") + ); + assert_eq!( + guest_kernel_cmdline(None, true).as_deref(), + Some("dstack.no_tee=1 dstack.storage_encrypted=0") + ); + assert_eq!( + guest_kernel_cmdline(Some("console=ttyS0"), true).as_deref(), + Some("console=ttyS0 dstack.no_tee=1 dstack.storage_encrypted=0") + ); + assert_eq!( + guest_kernel_cmdline(Some("dstack.no_tee=false console=ttyS0"), true).as_deref(), + Some("console=ttyS0 dstack.no_tee=1 dstack.storage_encrypted=0") + ); + } } fn virtio_pci_device(device: &str, snp: bool) -> String { @@ -543,6 +564,19 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { } } +fn guest_kernel_cmdline(image_cmdline: Option<&str>, no_tee: bool) -> Option { + let image_cmdline = image_cmdline.unwrap_or_default().trim(); + if !no_tee { + return (!image_cmdline.is_empty()).then(|| image_cmdline.to_string()); + } + let mut params = image_cmdline + .split_whitespace() + .filter(|param| !param.starts_with("dstack.no_tee=")) + .collect::>(); + params.extend(["dstack.no_tee=1", "dstack.storage_encrypted=0"]); + Some(params.join(" ")) +} + impl VmConfig { pub fn config_qemu( &self, @@ -550,6 +584,7 @@ impl VmConfig { cfg: &CvmConfig, gpus: &GpuConfig, ) -> Result> { + self.image.validate_no_tee(self.manifest.no_tee)?; let workdir = VmWorkDir::new(workdir); let serial_file = workdir.serial_file(); let serial_pty = workdir.serial_pty(); @@ -900,10 +935,10 @@ impl VmConfig { } } - // SNP app identity is bound through HOST_DATA, so the measured cmdline - // remains the image-provided cmdline. - let cmdline = self.image.info.cmdline.clone(); - if let Some(cmdline) = cmdline { + // no-TEE VMs need an explicit guest-side development flag. + if let Some(cmdline) = + guest_kernel_cmdline(self.image.info.cmdline.as_deref(), self.manifest.no_tee) + { command.arg("-append").arg(cmdline); } diff --git a/vmm/src/main_service.rs b/vmm/src/main_service.rs index 88aa884c1..9426dd00e 100644 --- a/vmm/src/main_service.rs +++ b/vmm/src/main_service.rs @@ -22,7 +22,10 @@ use or_panic::ResultOrPanic; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; -use crate::app::{App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir}; +use crate::app::{ + validate_image_name, validate_no_tee_compose, App, AttachMode, GpuConfig, GpuSpec, Image, + Manifest, PortMapping, VmWorkDir, +}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -287,9 +290,19 @@ impl RpcHandler { } } +fn validate_no_tee_update(current: bool, requested: Option) -> Result<()> { + if requested.is_some_and(|requested| requested != current) { + bail!("TEE mode cannot be changed after VM creation"); + } + Ok(()) +} + impl VmmRpc for RpcHandler { async fn create_vm(self, request: VmConfiguration) -> Result { let manifest = create_manifest_from_vm_config(request.clone(), &self.app.config.cvm)?; + let app_compose: AppCompose = + serde_json::from_str(&request.compose_file).context("Invalid compose file")?; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; let id = manifest.id.clone(); let app_id = manifest.app_id.clone(); let vm_work_dir = self.app.work_dir(&id); @@ -375,10 +388,27 @@ impl VmmRpc for RpcHandler { } async fn update_vm(self, request: UpdateVmRequest) -> Result { + let vm_work_dir = self.app.work_dir(&request.id); + let mut manifest = vm_work_dir.manifest().context("Failed to read manifest")?; + validate_no_tee_update(manifest.no_tee, request.no_tee)?; + + let app_compose = if request.compose_file.is_empty() { + vm_work_dir + .app_compose() + .context("Failed to read compose file")? + } else { + serde_json::from_str(&request.compose_file).context("Invalid compose file")? + }; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; + + if let Some(image_name) = request.image.as_deref() { + validate_image_name(image_name)?; + let image = Image::load(self.app.config.image.path.join(image_name)) + .context("Failed to load image")?; + image.validate_no_tee(manifest.no_tee)?; + } + let new_id = if !request.compose_file.is_empty() { - // check the compose file is valid - let _app_compose: AppCompose = - serde_json::from_str(&request.compose_file).context("Invalid compose file")?; let compose_file_path = self.compose_file_path(&request.id); if !compose_file_path.exists() { bail!("The instance {} not found", request.id); @@ -400,8 +430,6 @@ impl VmmRpc for RpcHandler { fs::write(user_config_path, &request.user_config) .context("Failed to write user config")?; } - let vm_work_dir = self.app.work_dir(&request.id); - let mut manifest = vm_work_dir.manifest().context("Failed to read manifest")?; self.apply_resource_updates( &request.id, &mut manifest, @@ -415,9 +443,6 @@ impl VmmRpc for RpcHandler { if let Some(gpus) = request.gpus { manifest.gpus = Some(self.resolve_gpus(&gpus)?); } - if let Some(no_tee) = request.no_tee { - manifest.no_tee = no_tee; - } if request.update_ports { manifest.port_map = request .ports @@ -749,3 +774,17 @@ impl RpcCall for RpcHandler { }) } } + +#[cfg(test)] +mod tests { + use super::validate_no_tee_update; + + #[test] + fn tee_mode_is_immutable() { + validate_no_tee_update(false, None).unwrap(); + validate_no_tee_update(false, Some(false)).unwrap(); + validate_no_tee_update(true, Some(true)).unwrap(); + assert!(validate_no_tee_update(false, Some(true)).is_err()); + assert!(validate_no_tee_update(true, Some(false)).is_err()); + } +} diff --git a/vmm/src/one_shot.rs b/vmm/src/one_shot.rs index 3773fbcb2..4cb75367c 100644 --- a/vmm/src/one_shot.rs +++ b/vmm/src/one_shot.rs @@ -2,7 +2,9 @@ // // SPDX-License-Identifier: Apache-2.0 -use crate::app::{make_sys_config, Image, VmConfig, VmWorkDir}; +use crate::app::{ + make_sys_config, validate_image_name, validate_no_tee_compose, Image, VmConfig, VmWorkDir, +}; use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; @@ -79,9 +81,11 @@ pub async fn run_one_shot( let manifest = create_manifest_from_vm_config(vm_config.clone(), &config.cvm)?; // Load image + validate_image_name(&manifest.image)?; let image_path = config.image.path.join(&manifest.image); let image = Image::load(&image_path) .with_context(|| format!("Failed to load image: {}", image_path.display()))?; + image.validate_no_tee(manifest.no_tee)?; // Create or use specified workdir and setup files let workdir_path = match workdir_option { @@ -117,7 +121,7 @@ pub async fn run_one_shot( fs_err::create_dir_all(&shared_dir).context("Failed to create shared directory")?; // Create app compose file content and parse AppCompose instance - let (app_compose_content, _app_compose) = if vm_config.compose_file.is_empty() { + let (app_compose_content, app_compose) = if vm_config.compose_file.is_empty() { // Create default compose JSON directly as string let gateway_enabled = !vm_config.gateway_urls.is_empty(); let kms_enabled = !vm_config.kms_urls.is_empty(); @@ -204,6 +208,8 @@ Compose file content (first 200 chars): } }; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; + // Write the JSON string directly (no serialization needed) fs_err::write(vm_work_dir.app_compose_path(), app_compose_content) .context("Failed to write app compose file")?; diff --git a/vmm/src/vmm-cli.py b/vmm/src/vmm-cli.py index 290ee2440..260b70faf 100755 --- a/vmm/src/vmm-cli.py +++ b/vmm/src/vmm-cli.py @@ -999,7 +999,6 @@ def update_vm( attach_all: bool = False, no_gpus: bool = False, kms_urls: Optional[List[str]] = None, - no_tee: Optional[bool] = None, ) -> None: """Update multiple aspects of a VM in one command.""" # Validate: --env-file requires --kms-url @@ -1163,10 +1162,6 @@ def update_vm( updates.append("GPUs (none)") upgrade_params["gpus"] = gpu_config - if no_tee is not None: - upgrade_params["no_tee"] = no_tee - updates.append("TEE disabled" if no_tee else "TEE enabled") - if len(upgrade_params) > 1: # more than just the id self.rpc_call("UpgradeApp", upgrade_params) @@ -1754,7 +1749,7 @@ def _patched_format_help(): "--no-tee", dest="no_tee", action="store_true", - help="Disable Intel TDX / run without TEE", + help="Disable TEE and disk encryption for development", ) deploy_parser.add_argument( "--tee", @@ -1906,22 +1901,6 @@ def _patched_format_help(): help="Detach all GPUs from the VM", ) - # TDX toggle - tee_group = update_parser.add_mutually_exclusive_group() - tee_group.add_argument( - "--no-tee", - dest="no_tee", - action="store_true", - help="Disable Intel TDX / run without TEE", - ) - tee_group.add_argument( - "--tee", - dest="no_tee", - action="store_false", - help="Enable Intel TDX for the VM", - ) - update_parser.set_defaults(no_tee=None) - # KMS URL for environment encryption update_parser.add_argument("--kms-url", action="append", type=str, help="KMS URL") @@ -2006,7 +1985,6 @@ def _patched_format_help(): attach_all=args.ppcie, no_gpus=args.no_gpus if hasattr(args, "no_gpus") else False, kms_urls=args.kms_url, - no_tee=args.no_tee, ) elif args.command == "kms": if not args.kms_action: diff --git a/vmm/ui/src/components/CreateVmDialog.ts b/vmm/ui/src/components/CreateVmDialog.ts index 22f473435..0e378bea5 100644 --- a/vmm/ui/src/components/CreateVmDialog.ts +++ b/vmm/ui/src/components/CreateVmDialog.ts @@ -164,7 +164,6 @@ const CreateVmDialogComponent = { - diff --git a/vmm/ui/src/composables/useVmManager.ts b/vmm/ui/src/composables/useVmManager.ts index a994ee1d8..6eed60763 100644 --- a/vmm/ui/src/composables/useVmManager.ts +++ b/vmm/ui/src/composables/useVmManager.ts @@ -115,7 +115,6 @@ type VmFormState = { public_logs: boolean; public_sysinfo: boolean; public_tcbinfo: boolean; - no_tee: boolean; pin_numa: boolean; hugepages: boolean; net_mode: string; @@ -165,7 +164,6 @@ type CloneConfigDialogState = { gateway_urls?: string[]; hugepages: boolean; pin_numa: boolean; - no_tee: boolean; encrypted_env?: Uint8Array; app_id?: string; stopped: boolean; @@ -198,7 +196,6 @@ function createVmFormState(preLaunchScript: string): VmFormState { public_logs: true, public_sysinfo: true, public_tcbinfo: true, - no_tee: false, pin_numa: false, hugepages: false, net_mode: '', @@ -252,7 +249,6 @@ function createCloneConfigDialogState(): CloneConfigDialogState { gateway_urls: undefined, hugepages: false, pin_numa: false, - no_tee: false, encrypted_env: undefined, app_id: undefined, stopped: false, @@ -443,7 +439,6 @@ type CreateVmPayloadSource = { user_config?: string; hugepages?: boolean; pin_numa?: boolean; - no_tee?: boolean; net_mode?: string; gpus?: VmmTypes.IGpuConfig; kms_urls?: string[]; @@ -466,7 +461,6 @@ type CreateVmPayloadSource = { user_config: source.user_config || '', hugepages: !!source.hugepages, pin_numa: !!source.pin_numa, - no_tee: source.no_tee ?? false, networking: source.net_mode ? { mode: source.net_mode } : undefined, gpus: source.gpus, kms_urls: source.kms_urls?.filter((url) => url && url.trim().length) ?? [], @@ -1003,7 +997,6 @@ type CreateVmPayloadSource = { user_config: vmForm.value.user_config, hugepages: vmForm.value.hugepages, pin_numa: vmForm.value.pin_numa, - no_tee: vmForm.value.no_tee, net_mode: vmForm.value.net_mode, gpus: configGpu(vmForm.value) || undefined, kms_urls: vmForm.value.kms_urls, @@ -1154,7 +1147,6 @@ type CreateVmPayloadSource = { public_tcbinfo: !!theVm.appCompose?.public_tcbinfo, pin_numa: !!config.pin_numa, hugepages: !!config.hugepages, - no_tee: !!config.no_tee, net_mode: config.networking?.mode || '', user_config: config.user_config || '', stopped: !!config.stopped, @@ -1184,7 +1176,6 @@ type CreateVmPayloadSource = { user_config: source.user_config, hugepages: source.hugepages, pin_numa: source.pin_numa, - no_tee: source.no_tee, gpus: source.gpus, kms_urls: source.kms_urls, gateway_urls: source.gateway_urls, diff --git a/vmm/ui/src/templates/app.html b/vmm/ui/src/templates/app.html index 7b6a63bdc..b0cee85de 100644 --- a/vmm/ui/src/templates/app.html +++ b/vmm/ui/src/templates/app.html @@ -307,7 +307,7 @@

dstack-vmm

TEE - {{ vm.configuration?.no_tee ? 'Disabled' : 'Enabled' }} + {{ vm.configuration?.no_tee ? 'Disabled (development)' : 'Enabled' }}
GPUs