Skip to content

Commit 54734be

Browse files
3pointer3pointer
committed
tici: mount tidb root auth secret for meta config
1 parent ad95f42 commit 54734be

2 files changed

Lines changed: 134 additions & 105 deletions

File tree

‎pkg/manager/member/tici_member_manager.go‎

Lines changed: 97 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ package member
1515

1616
import (
1717
"fmt"
18-
neturl "net/url"
1918
"strings"
2019

2120
"github.com/pingcap/tidb-operator/pkg/apis/label"
@@ -39,7 +38,12 @@ import (
3938
"k8s.io/utils/pointer"
4039
)
4140

42-
const ticiMetaConfigHashAnnotation = "tidb.pingcap.com/tici-meta-config-hash"
41+
const (
42+
ticiMetaRootAuthHashAnnotation = "tidb.pingcap.com/tici-meta-root-auth-hash"
43+
ticiMetaConfigMountPath = "/etc/tici"
44+
ticiMetaConfigTemplateMountPath = "/etc/tici-template"
45+
ticiMetaRootAuthMountPath = "/etc/tici-root-auth"
46+
)
4347

4448
// ticiMemberManager implements manager.Manager.
4549
type ticiMemberManager struct {
@@ -208,12 +212,17 @@ func (m *ticiMemberManager) syncTiCIMetaStatefulSet(tc *v1alpha1.TidbCluster) er
208212
return nil
209213
}
210214

211-
secret, err := m.syncTiCIMetaConfigSecret(tc)
215+
cm, err := m.syncTiCIMetaConfigMap(tc, oldSts)
212216
if err != nil {
213217
return err
214218
}
215219

216-
newSts, err := getNewTiCIMetaStatefulSet(tc, secret)
220+
rootAuthHash, err := getTiDBRootAuthHash(tc, m.deps.SecretLister)
221+
if err != nil {
222+
return err
223+
}
224+
225+
newSts, err := getNewTiCIMetaStatefulSet(tc, cm, rootAuthHash)
217226
if err != nil {
218227
return err
219228
}
@@ -343,12 +352,24 @@ func (m *ticiMemberManager) syncTiCIWorkerStatus(tc *v1alpha1.TidbCluster, sts *
343352
return volumes.SyncVolumeStatus(m.podVolumeModifier, m.deps.PodLister, tc, v1alpha1.TiCIWorkerMemberType)
344353
}
345354

346-
func (m *ticiMemberManager) syncTiCIMetaConfigSecret(tc *v1alpha1.TidbCluster) (*corev1.Secret, error) {
347-
newSecret, err := getTiCIMetaConfigSecret(tc, m.deps.SecretLister)
355+
func (m *ticiMemberManager) syncTiCIMetaConfigMap(tc *v1alpha1.TidbCluster, set *appsv1.StatefulSet) (*corev1.ConfigMap, error) {
356+
newCm, err := getTiCIMetaConfigMap(tc)
357+
if err != nil {
358+
return nil, err
359+
}
360+
361+
var inUseName string
362+
if set != nil {
363+
inUseName = mngerutils.FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool {
364+
return strings.HasPrefix(name, controller.TiCIMetaMemberName(tc.Name))
365+
})
366+
}
367+
368+
err = mngerutils.UpdateConfigMapIfNeed(m.deps.ConfigMapLister, tc.BaseTiCIMetaSpec().ConfigUpdateStrategy(), inUseName, newCm)
348369
if err != nil {
349370
return nil, err
350371
}
351-
return m.deps.TypedControl.CreateOrUpdateSecret(tc, newSecret)
372+
return m.deps.TypedControl.CreateOrUpdateConfigMap(tc, newCm)
352373
}
353374

354375
func (m *ticiMemberManager) syncTiCIWorkerConfigMap(tc *v1alpha1.TidbCluster, set *appsv1.StatefulSet) (*corev1.ConfigMap, error) {
@@ -371,32 +392,43 @@ func (m *ticiMemberManager) syncTiCIWorkerConfigMap(tc *v1alpha1.TidbCluster, se
371392
return m.deps.TypedControl.CreateOrUpdateConfigMap(tc, newCm)
372393
}
373394

374-
func getTiCIMetaConfigSecret(tc *v1alpha1.TidbCluster, secretLister corelisterv1.SecretLister) (*corev1.Secret, error) {
375-
password := ""
376-
if secretLister != nil {
377-
if passwordSecret, err := secretLister.Secrets(tc.Namespace).Get(controller.TiDBInitSecret(tc.Name)); err == nil {
378-
password = string(passwordSecret.Data[constants.TidbRootKey])
379-
}
380-
}
381-
382-
configText, err := buildTiCIMetaConfigWithPassword(tc, password)
395+
func getTiCIMetaConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) {
396+
configText, err := buildTiCIMetaConfig(tc)
383397
if err != nil {
384398
return nil, err
385399
}
386400
name := controller.TiCIMetaMemberName(tc.Name)
387401
instanceName := tc.GetInstanceName()
388402
labels := label.New().Instance(instanceName).TiCIMeta().Labels()
389403

390-
secret := &corev1.Secret{
404+
cm := &corev1.ConfigMap{
391405
ObjectMeta: metav1.ObjectMeta{
392406
Name: name,
393407
Namespace: tc.Namespace,
394408
Labels: labels,
395409
OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)},
396410
},
397-
Data: map[string][]byte{"config-file": []byte(configText)},
411+
Data: map[string]string{"config-file": configText},
412+
}
413+
return cm, nil
414+
}
415+
416+
func getTiDBRootAuthHash(tc *v1alpha1.TidbCluster, secretLister corelisterv1.SecretLister) (string, error) {
417+
if secretLister == nil {
418+
return "", nil
419+
}
420+
rootSecret, err := secretLister.Secrets(tc.Namespace).Get(controller.TiDBInitSecret(tc.Name))
421+
if err != nil {
422+
if errors.IsNotFound(err) {
423+
return "", nil
424+
}
425+
return "", err
426+
}
427+
rootAuth, ok := rootSecret.Data[constants.TidbRootKey]
428+
if !ok {
429+
return "", nil
398430
}
399-
return secret, nil
431+
return mngerutils.Sha256Sum(map[string]string{constants.TidbRootKey: string(rootAuth)})
400432
}
401433

402434
func getTiCIWorkerConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) {
@@ -420,7 +452,7 @@ func getTiCIWorkerConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error)
420452
return cm, nil
421453
}
422454

423-
func getNewTiCIMetaStatefulSet(tc *v1alpha1.TidbCluster, secret *corev1.Secret) (*appsv1.StatefulSet, error) {
455+
func getNewTiCIMetaStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap, rootAuthHash string) (*appsv1.StatefulSet, error) {
424456
if tc.Spec.TiCI == nil || tc.Spec.TiCI.Meta == nil {
425457
return nil, nil
426458
}
@@ -433,12 +465,8 @@ func getNewTiCIMetaStatefulSet(tc *v1alpha1.TidbCluster, secret *corev1.Secret)
433465
stsName := controller.TiCIMetaMemberName(tcName)
434466
podLabels := util.CombineStringMap(stsLabels, baseSpec.Labels())
435467
podAnnotations := util.CombineStringMap(baseSpec.Annotations(), controller.AnnProm(v1alpha1.DefaultTiCIMetaStatusPort, "/metrics"))
436-
if secret != nil {
437-
sum, err := mngerutils.Sha256Sum(map[string]string{"config-file": getSecretStringValue(secret, "config-file")})
438-
if err != nil {
439-
return nil, fmt.Errorf("failed to hash TiCI meta config secret for [%s/%s], error: %v", ns, tcName, err)
440-
}
441-
podAnnotations[ticiMetaConfigHashAnnotation] = sum
468+
if rootAuthHash != "" {
469+
podAnnotations[ticiMetaRootAuthHashAnnotation] = rootAuthHash
442470
}
443471
stsAnnotations := getStsAnnotations(tc.Annotations, label.TiCIMetaLabelVal)
444472
headlessSvcName := controller.TiCIMetaPeerMemberName(tcName)
@@ -449,16 +477,27 @@ func getNewTiCIMetaStatefulSet(tc *v1alpha1.TidbCluster, secret *corev1.Secret)
449477
volMounts = append(volMounts, storageVolMounts...)
450478
volMounts = append(volMounts, spec.AdditionalVolumeMounts...)
451479

452-
configMountPath := "/etc/tici"
453-
if secret != nil {
454-
volMounts = append(volMounts, corev1.VolumeMount{Name: "config", MountPath: configMountPath})
455-
vols = append(vols, corev1.Volume{Name: "config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
456-
SecretName: secret.Name,
457-
Items: []corev1.KeyToPath{{Key: "config-file", Path: "tici.toml"}},
458-
}}})
459-
}
460-
461-
args := renderTiCIStartArgs(tc, v1alpha1.TiCIMetaMemberType, headlessSvcName)
480+
if cm != nil {
481+
volMounts = append(volMounts,
482+
corev1.VolumeMount{Name: "config", MountPath: ticiMetaConfigTemplateMountPath, ReadOnly: true},
483+
corev1.VolumeMount{Name: "runtime-config", MountPath: ticiMetaConfigMountPath},
484+
corev1.VolumeMount{Name: "tidb-root-auth", MountPath: ticiMetaRootAuthMountPath, ReadOnly: true},
485+
)
486+
vols = append(vols,
487+
corev1.Volume{Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{
488+
LocalObjectReference: corev1.LocalObjectReference{Name: cm.Name},
489+
Items: []corev1.KeyToPath{{Key: "config-file", Path: "tici.toml"}},
490+
}}},
491+
corev1.Volume{Name: "runtime-config", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
492+
corev1.Volume{Name: "tidb-root-auth", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
493+
SecretName: controller.TiDBInitSecret(tcName),
494+
Items: []corev1.KeyToPath{{Key: constants.TidbRootKey, Path: constants.TidbRootKey}},
495+
Optional: pointer.BoolPtr(true),
496+
}}},
497+
)
498+
}
499+
500+
args := renderTiCIMetaStartScript(tc, headlessSvcName)
462501

463502
envs := []corev1.EnvVar{
464503
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
@@ -697,11 +736,28 @@ func renderTiCIStartArgs(tc *v1alpha1.TidbCluster, memberType v1alpha1.MemberTyp
697736
return strings.Join(args, " ")
698737
}
699738

700-
func buildTiCIMetaConfig(tc *v1alpha1.TidbCluster) (string, error) {
701-
return buildTiCIMetaConfigWithPassword(tc, "")
739+
func renderTiCIMetaStartScript(tc *v1alpha1.TidbCluster, headlessSvcName string) string {
740+
configTemplatePath := fmt.Sprintf("%s/tici.toml", ticiMetaConfigTemplateMountPath)
741+
configPath := fmt.Sprintf("%s/tici.toml", ticiMetaConfigMountPath)
742+
rootAuthPath := fmt.Sprintf("%s/%s", ticiMetaRootAuthMountPath, constants.TidbRootKey)
743+
744+
lines := []string{
745+
"set -e",
746+
fmt.Sprintf("config_template=%q", configTemplatePath),
747+
fmt.Sprintf("config_file=%q", configPath),
748+
fmt.Sprintf("root_auth_file=%q", rootAuthPath),
749+
`if [ -s "${root_auth_file}" ]; then`,
750+
` encoded_root_auth=$(od -An -tx1 -v "${root_auth_file}" | tr -d ' \n' | sed 's/../%&/g')`,
751+
` sed "s#mysql://root@#mysql://root:${encoded_root_auth}@#" "${config_template}" > "${config_file}"`,
752+
"else",
753+
` cp "${config_template}" "${config_file}"`,
754+
"fi",
755+
renderTiCIStartArgs(tc, v1alpha1.TiCIMetaMemberType, headlessSvcName),
756+
}
757+
return strings.Join(lines, "\n")
702758
}
703759

704-
func buildTiCIMetaConfigWithPassword(tc *v1alpha1.TidbCluster, password string) (string, error) {
760+
func buildTiCIMetaConfig(tc *v1alpha1.TidbCluster) (string, error) {
705761
s3, err := buildTiCIS3Config(tc)
706762
if err != nil {
707763
return "", err
@@ -712,10 +768,9 @@ func buildTiCIMetaConfigWithPassword(tc *v1alpha1.TidbCluster, password string)
712768
tidbHost := controller.TiDBMemberName(tc.Name)
713769
tidbPort := tc.Spec.TiDB.GetServicePort()
714770
pdAddr := fmt.Sprintf("%s:%d", controller.PDMemberName(tc.Name), v1alpha1.DefaultPDClientPort)
715-
tidbDSN := buildTiDBServerDSN(tidbHost, tidbPort, password)
716771

717772
baseConfig := fmt.Sprintf(`[tidb-server]
718-
dsns = ["%s"]
773+
dsns = ["mysql://root@%s:%d"]
719774
720775
[server]
721776
pd-addr = "%s"
@@ -728,42 +783,13 @@ secret-key = "%s"
728783
use-path-style = %t
729784
bucket = "%s"
730785
prefix = "%s"
731-
`, tidbDSN, pdAddr, s3.Endpoint, s3.Region, s3.AccessKey, s3.SecretKey, s3.UsePathStyle, s3.Bucket, s3.Prefix)
786+
`, tidbHost, tidbPort, pdAddr, s3.Endpoint, s3.Region, s3.AccessKey, s3.SecretKey, s3.UsePathStyle, s3.Bucket, s3.Prefix)
732787
if tc.Spec.TiCI != nil && tc.Spec.TiCI.Meta != nil {
733788
return appendTiCICustomConfig(baseConfig, tc.Spec.TiCI.Meta.Config)
734789
}
735790
return baseConfig, nil
736791
}
737792

738-
func buildTiDBServerDSN(host string, port int32, password string) string {
739-
user := neturl.User("root")
740-
if password != "" {
741-
user = neturl.UserPassword("root", password)
742-
}
743-
return (&neturl.URL{
744-
Scheme: "mysql",
745-
User: user,
746-
Host: fmt.Sprintf("%s:%d", host, port),
747-
}).String()
748-
}
749-
750-
func getSecretStringValue(secret *corev1.Secret, key string) string {
751-
if secret == nil {
752-
return ""
753-
}
754-
if secret.StringData != nil {
755-
if value, ok := secret.StringData[key]; ok {
756-
return value
757-
}
758-
}
759-
if secret.Data != nil {
760-
if value, ok := secret.Data[key]; ok {
761-
return string(value)
762-
}
763-
}
764-
return ""
765-
}
766-
767793
func buildTiCIWorkerConfig(tc *v1alpha1.TidbCluster) (string, error) {
768794
s3, err := buildTiCIS3Config(tc)
769795
if err != nil {

‎pkg/manager/member/tici_member_manager_test.go‎

Lines changed: 37 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ package member
1515

1616
import (
1717
"encoding/json"
18-
neturl "net/url"
1918
"strings"
2019
"testing"
2120

@@ -79,29 +78,19 @@ data-dir = "/data/tici-meta"`
7978
}
8079
}
8180

82-
func TestBuildTiCIMetaConfigWithPassword(t *testing.T) {
81+
func TestRenderTiCIMetaStartScriptInjectsRootAuthFromSecretVolume(t *testing.T) {
8382
tc := newTidbClusterForTiCIConfig()
8483

85-
cfg, err := buildTiCIMetaConfigWithPassword(tc, `p@ss:/?#[]!$&'()*+,;=`)
86-
if err != nil {
87-
t.Fatalf("build meta config with password failed: %v", err)
88-
}
89-
90-
wrapper := tcconfig.New(map[string]interface{}{})
91-
if err := wrapper.UnmarshalTOML([]byte(cfg)); err != nil {
92-
t.Fatalf("meta config should be valid TOML, got err: %v, config: %s", err, cfg)
93-
}
94-
dsns := wrapper.Get("tidb-server.dsns")
95-
if dsns == nil || len(dsns.MustStringSlice()) != 1 {
96-
t.Fatalf("meta config should include one generated dsn, got: %s", cfg)
97-
}
98-
parsed, err := neturl.Parse(dsns.MustStringSlice()[0])
99-
if err != nil {
100-
t.Fatalf("dsn should be a valid url, got err: %v, dsn: %s", err, dsns.MustStringSlice()[0])
101-
}
102-
password, ok := parsed.User.Password()
103-
if !ok || password != `p@ss:/?#[]!$&'()*+,;=` {
104-
t.Fatalf("dsn should preserve the original password, got: %q", password)
84+
script := renderTiCIMetaStartScript(tc, "tici-test-tici-meta-peer")
85+
for _, expected := range []string{
86+
`root_auth_file="/etc/tici-root-auth/root"`,
87+
`od -An -tx1 -v "${root_auth_file}"`,
88+
`sed "s#mysql://root@#mysql://root:${encoded_root_auth}@#"`,
89+
`exec /tici-server meta --config=/etc/tici/tici.toml`,
90+
} {
91+
if !strings.Contains(script, expected) {
92+
t.Fatalf("expected meta start script to contain %q, got: %s", expected, script)
93+
}
10594
}
10695
}
10796

@@ -203,36 +192,50 @@ func TestPrepareTiCIRollingUpgrade(t *testing.T) {
203192
}
204193
}
205194

206-
func TestGetNewTiCIMetaStatefulSetUsesSecretConfig(t *testing.T) {
195+
func TestGetNewTiCIMetaStatefulSetUsesConfigMapAndRootAuthSecret(t *testing.T) {
207196
tc := newTidbClusterForTiCIConfig()
208-
secret := &corev1.Secret{
197+
cm := &corev1.ConfigMap{
209198
ObjectMeta: metav1.ObjectMeta{
210199
Name: "tici-test-tici-meta",
211200
Namespace: "test-ns",
212201
},
213-
StringData: map[string]string{"config-file": "[server]\npd-addr = \"x\"\n"},
202+
Data: map[string]string{"config-file": "[server]\npd-addr = \"x\"\n"},
214203
}
215204

216-
sts, err := getNewTiCIMetaStatefulSet(tc, secret)
205+
sts, err := getNewTiCIMetaStatefulSet(tc, cm, "root-auth-hash")
217206
if err != nil {
218207
t.Fatalf("build TiCI meta statefulset failed: %v", err)
219208
}
220209
if sts == nil {
221210
t.Fatal("expected non-nil TiCI meta statefulset")
211+
return
222212
}
223213

224-
foundSecretVolume := false
214+
foundConfigVolume := false
215+
foundRuntimeConfigVolume := false
216+
foundRootAuthVolume := false
225217
for _, volume := range sts.Spec.Template.Spec.Volumes {
226-
if volume.Name == "config" && volume.Secret != nil && volume.Secret.SecretName == secret.Name {
227-
foundSecretVolume = true
228-
break
218+
if volume.Name == "config" && volume.ConfigMap != nil && volume.ConfigMap.Name == cm.Name {
219+
foundConfigVolume = true
229220
}
221+
if volume.Name == "runtime-config" && volume.EmptyDir != nil {
222+
foundRuntimeConfigVolume = true
223+
}
224+
if volume.Name == "tidb-root-auth" && volume.Secret != nil && volume.Secret.SecretName == "tici-test-init" {
225+
foundRootAuthVolume = true
226+
}
227+
}
228+
if !foundConfigVolume {
229+
t.Fatalf("expected TiCI meta statefulset to mount config from configmap, got volumes: %+v", sts.Spec.Template.Spec.Volumes)
230+
}
231+
if !foundRuntimeConfigVolume {
232+
t.Fatalf("expected TiCI meta statefulset to include writable runtime config volume, got volumes: %+v", sts.Spec.Template.Spec.Volumes)
230233
}
231-
if !foundSecretVolume {
232-
t.Fatalf("expected TiCI meta statefulset to mount config from secret, got volumes: %+v", sts.Spec.Template.Spec.Volumes)
234+
if !foundRootAuthVolume {
235+
t.Fatalf("expected TiCI meta statefulset to mount TiDB root auth secret, got volumes: %+v", sts.Spec.Template.Spec.Volumes)
233236
}
234-
if sts.Spec.Template.Annotations[ticiMetaConfigHashAnnotation] == "" {
235-
t.Fatalf("expected TiCI meta statefulset to include %s annotation", ticiMetaConfigHashAnnotation)
237+
if sts.Spec.Template.Annotations[ticiMetaRootAuthHashAnnotation] != "root-auth-hash" {
238+
t.Fatalf("expected TiCI meta statefulset to include root auth hash annotation, got: %s", sts.Spec.Template.Annotations[ticiMetaRootAuthHashAnnotation])
236239
}
237240
}
238241

0 commit comments

Comments
 (0)