chore: add shared helpers for the integration:ng test suites

Shared device onboarding (auth requests signed with RSA, ECDSA
P-224/256/384/521 and Ed25519 keys), artifact building, context-aware
polling and raw-request helpers in tests/common, with the existing
suites refactored onto them. Groundwork for porting the python
integration tests suite by suite; the python tests keep running in
parallel until the port is complete.

Signed-off-by: Patryk Targowicz <patryk.targowicz@northern.tech>

Author: p-targowicz

backend/tests/runner/go.mod

@@ -195,7 +195,7 @@ require (
 	golang.org/x/crypto v0.54.0 // indirect
 	golang.org/x/net v0.56.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sync v0.22.0
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/term v0.45.0 // indirect
 	golang.org/x/text v0.40.0 // indirect

backend/tests/runner/tests/common/artifact.go

@@ -0,0 +1,154 @@
+//nolint:all // This is all test code
+package common
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/mendersoftware/mender-artifact/artifact"
+	"github.com/mendersoftware/mender-artifact/awriter"
+	"github.com/mendersoftware/mender-artifact/handlers"
+)
+
+// ArtifactOption modifies the WriteArtifactArgs an artifact is built from.
+type ArtifactOption func(args *awriter.WriteArtifactArgs) error
+
+func WithCompatibleDevices(compatibleDevices []string) ArtifactOption {
+	return func(args *awriter.WriteArtifactArgs) error {
+		args.Depends.CompatibleDevices = compatibleDevices
+		return nil
+	}
+}
+
+func WithUpdates(updates []handlers.Composer) ArtifactOption {
+	return func(args *awriter.WriteArtifactArgs) error {
+		args.Updates.Updates = updates
+		return nil
+	}
+}
+
+func WithModuleImage(module *handlers.ModuleImage) ArtifactOption {
+	return func(args *awriter.WriteArtifactArgs) error {
+		args.TypeInfoV3.Type = module.GetUpdateType()
+		args.Updates.Updates = []handlers.Composer{module}
+		return nil
+	}
+}
+
+// WithDependsProvides sets the type-info depends/provides maps, used by
+// the artifact selection semantics tests.
+func WithDependsProvides(depends, provides map[string]string) ArtifactOption {
+	return func(args *awriter.WriteArtifactArgs) error {
+		if depends != nil {
+			deps := make(artifact.TypeInfoDepends, len(depends))
+			for k, v := range depends {
+				deps[k] = v
+			}
+			args.TypeInfoV3.ArtifactDepends = deps
+		}
+		if provides != nil {
+			provs := make(artifact.TypeInfoProvides, len(provides))
+			for k, v := range provides {
+				provs[k] = v
+			}
+			args.TypeInfoV3.ArtifactProvides = provs
+		}
+		return nil
+	}
+}
+
+// WithPayloadFile attaches a randomly-generated update payload of the given
+// size (in bytes) to the artifact's module image, so the resulting artifact
+// file has a controllable, predictable size. This is needed by tests that
+// exercise deployments' "select the smallest matching artifact" tie-break
+// logic. Must be applied after the option that sets the module image (or
+// rely on the default one CreateArtifact installs).
+func WithPayloadFile(t interface{ TempDir() string }, size int) ArtifactOption {
+	return func(args *awriter.WriteArtifactArgs) error {
+		f, err := os.CreateTemp(t.TempDir(), "mender-artifact-payload-*")
+		if err != nil {
+			return fmt.Errorf("WithPayloadFile: failed to create payload file: %w", err)
+		}
+		defer f.Close()
+
+		data := make([]byte, size)
+		if _, err := rand.Read(data); err != nil {
+			return fmt.Errorf("WithPayloadFile: failed to generate random payload: %w", err)
+		}
+		if _, err := f.Write(data); err != nil {
+			return fmt.Errorf("WithPayloadFile: failed to write payload file: %w", err)
+		}
+
+		if len(args.Updates.Updates) == 0 {
+			return fmt.Errorf("WithPayloadFile: no updates to attach a payload to")
+		}
+		mi, ok := args.Updates.Updates[len(args.Updates.Updates)-1].(*handlers.ModuleImage)
+		if !ok {
+			return fmt.Errorf("WithPayloadFile: last update is not a *handlers.ModuleImage")
+		}
+		if err := mi.SetUpdateFiles([]*handlers.DataFile{{Name: f.Name()}}); err != nil {
+			return fmt.Errorf("WithPayloadFile: failed to set update files: %w", err)
+		}
+		return nil
+	}
+}
+
+// CreateArtifact writes a minimal module-image mender artifact to a temp
+// file and returns it seeked to the start, ready for upload.
+func CreateArtifact(
+	name string,
+	t interface{ TempDir() string },
+	artifactArgsOpts ...ArtifactOption,
+) (*os.File, error) {
+
+	artifactDst := path.Join(t.TempDir(), fmt.Sprintf("%s.mender", name))
+	file, err := os.Create(artifactDst)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create %s: %w", artifactDst, err)
+	}
+
+	w := awriter.NewWriter(file, artifact.NewCompressorGzip())
+	i := handlers.NewModuleImage("foo")
+
+	args := &awriter.WriteArtifactArgs{
+		Format:  "mender",
+		Version: 3,
+		Name:    name,
+		Provides: &artifact.ArtifactProvides{
+			ArtifactName: name,
+		},
+		Depends: &artifact.ArtifactDepends{
+			CompatibleDevices: []string{"foo"},
+		},
+		TypeInfoV3: &artifact.TypeInfoV3{
+			Type:             i.GetUpdateType(),
+			ArtifactProvides: artifact.TypeInfoProvides{"foo": "bar"},
+			ArtifactDepends:  artifact.TypeInfoDepends{"foo": "bar"},
+		},
+		Updates: &awriter.Updates{
+			Updates: []handlers.Composer{i},
+		},
+	}
+
+	for _, opt := range artifactArgsOpts {
+		if err := opt(args); err != nil {
+			return nil, fmt.Errorf("failed to apply artifact option: %w", err)
+		}
+	}
+
+	err = w.WriteArtifact(args)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write module-image artifact: %w", err)
+	}
+
+	// Seek to the start of the file so the caller can read it
+	_, err = file.Seek(0, io.SeekStart)
+	if err != nil {
+		return nil, fmt.Errorf("failed to prepare %s for reading: %w", artifactDst, err)
+	}
+
+	return file, nil
+}

backend/tests/runner/tests/common/crypto.go

@@ -0,0 +1,206 @@
+//nolint:all // This is all test code
+package common
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/ed25519"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/pem"
+	"fmt"
+	"net"
+)
+
+// generateDeviceKeys generates the RSA key-pair a test device
+// authenticates with.
+func generateDeviceKeys() (*rsa.PrivateKey, *rsa.PublicKey, error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		return nil, nil, err
+	}
+	return privateKey, &privateKey.PublicKey, nil
+}
+
+// signData signs data with SHA256+PKCS1v15 and returns the base64-encoded
+// signature, the format deviceauth expects in the X-MEN-Signature header.
+func signData(privateKey *rsa.PrivateKey, data []byte) (string, error) {
+	hash := sha256.Sum256(data)
+	signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, hash[:])
+	if err != nil {
+		return "", err
+	}
+	return base64.StdEncoding.EncodeToString(signature), nil
+}
+
+// ExportPublicKeyPEM renders a public key (RSA, ECDSA or Ed25519) as the
+// PKIX PEM string used in device auth requests. Panics if the key cannot be
+// marshaled.
+func ExportPublicKeyPEM(pubkey any) string {
+	pubASN1, err := x509.MarshalPKIXPublicKey(pubkey)
+	if err != nil {
+		panic(fmt.Errorf("failed to marshal public key: %w", err))
+	}
+	pubBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "PUBLIC KEY",
+		Bytes: pubASN1,
+	})
+	return string(pubBytes)
+}
+
+// EC curve identifiers mirroring the Python test suite's EC_CURVE_* constants
+// (testutils/util/crypto.py).
+var (
+	ECCurveP224 = elliptic.P224()
+	ECCurveP256 = elliptic.P256()
+	ECCurveP384 = elliptic.P384()
+	ECCurveP521 = elliptic.P521()
+)
+
+// generateECDeviceKeys generates an ECDSA key-pair on the given curve.
+func generateECDeviceKeys(curve elliptic.Curve) (*ecdsa.PrivateKey, *ecdsa.PublicKey, error) {
+	privateKey, err := ecdsa.GenerateKey(curve, rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+	return privateKey, &privateKey.PublicKey, nil
+}
+
+// generateEd25519DeviceKeys generates an Ed25519 key-pair.
+func generateEd25519DeviceKeys() (ed25519.PrivateKey, ed25519.PublicKey, error) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+	return priv, pub, nil
+}
+
+// signDataEC signs data with SHA256+ECDSA and returns the base64-encoded
+// ASN.1/DER signature, the format deviceauth expects in the X-MEN-Signature
+// header.
+func signDataEC(privateKey *ecdsa.PrivateKey, data []byte) (string, error) {
+	hash := sha256.Sum256(data)
+	signature, err := ecdsa.SignASN1(rand.Reader, privateKey, hash[:])
+	if err != nil {
+		return "", err
+	}
+	return base64.StdEncoding.EncodeToString(signature), nil
+}
+
+// signDataEd25519 signs the raw (unhashed) data with Ed25519, per the Mender
+// device auth signing convention, and returns the base64-encoded signature.
+func signDataEd25519(privateKey ed25519.PrivateKey, data []byte) (string, error) {
+	return base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, data)), nil
+}
+
+// Key kinds exercised by the device auth test suite, mirroring the key types
+// generated by testutils/util/crypto.py.
+const (
+	KeyKindRSA     = "rsa"
+	KeyKindECP224  = "ec-p224"
+	KeyKindECP256  = "ec-p256"
+	KeyKindECP384  = "ec-p384"
+	KeyKindECP521  = "ec-p521"
+	KeyKindEd25519 = "ed25519"
+)
+
+// KeyKinds lists all key kinds exercised by the device auth test suite.
+var KeyKinds = []string{
+	KeyKindRSA,
+	KeyKindECP224,
+	KeyKindECP256,
+	KeyKindECP384,
+	KeyKindECP521,
+	KeyKindEd25519,
+}
+
+// KeyPair is a signing key-pair for a test device, abstracting over the key
+// types deviceauth accepts (RSA, ECDSA P-224/256/384/521, Ed25519).
+type KeyPair struct {
+	Kind    string
+	Private crypto.Signer
+	Public  any
+}
+
+// NewKeyPair generates a key-pair of the given kind (one of the KeyKind*
+// constants).
+func NewKeyPair(kind string) (*KeyPair, error) {
+	switch kind {
+	case KeyKindRSA:
+		priv, pub, err := generateDeviceKeys()
+		if err != nil {
+			return nil, err
+		}
+		return &KeyPair{Kind: kind, Private: priv, Public: pub}, nil
+	case KeyKindECP224:
+		return newECKeyPair(kind, ECCurveP224)
+	case KeyKindECP256:
+		return newECKeyPair(kind, ECCurveP256)
+	case KeyKindECP384:
+		return newECKeyPair(kind, ECCurveP384)
+	case KeyKindECP521:
+		return newECKeyPair(kind, ECCurveP521)
+	case KeyKindEd25519:
+		priv, pub, err := generateEd25519DeviceKeys()
+		if err != nil {
+			return nil, err
+		}
+		return &KeyPair{Kind: kind, Private: priv, Public: pub}, nil
+	default:
+		return nil, fmt.Errorf("unsupported key kind %q", kind)
+	}
+}
+
+func newECKeyPair(kind string, curve elliptic.Curve) (*KeyPair, error) {
+	priv, pub, err := generateECDeviceKeys(curve)
+	if err != nil {
+		return nil, err
+	}
+	return &KeyPair{Kind: kind, Private: priv, Public: pub}, nil
+}
+
+// PublicKeyPEM renders the public key as PKIX PEM.
+func (k *KeyPair) PublicKeyPEM() string {
+	return ExportPublicKeyPEM(k.Public)
+}
+
+// Sign signs data with the scheme appropriate for the key kind and returns
+// the base64-encoded signature for the X-MEN-Signature header.
+func (k *KeyPair) Sign(data []byte) (string, error) {
+	return SignAuthRequest(k.Private, data)
+}
+
+// SignAuthRequest signs data with the scheme appropriate for the concrete
+// type of privateKey (RSA, ECDSA or Ed25519) and returns the base64-encoded
+// signature for the X-MEN-Signature header.
+func SignAuthRequest(privateKey crypto.Signer, data []byte) (string, error) {
+	switch priv := privateKey.(type) {
+	case *rsa.PrivateKey:
+		return signData(priv, data)
+	case *ecdsa.PrivateKey:
+		return signDataEC(priv, data)
+	case ed25519.PrivateKey:
+		return signDataEd25519(priv, data)
+	default:
+		return "", fmt.Errorf("unsupported private key type %T", priv)
+	}
+}
+
+// RandomMAC returns a random unicast, locally administered MAC address for
+// use as device identity data.
+func RandomMAC() (net.HardwareAddr, error) {
+	mac := make([]byte, 6)
+	_, err := rand.Read(mac)
+	if err != nil {
+		return nil, err
+	}
+
+	mac[0] &= 0xfe // Set to Unicast
+	mac[0] |= 0x02 // Set to Locally Administered
+
+	return mac, nil
+}