Skip to content

crl-release-26.2: valsep: account for blob-file compression in split-decision size estimate #6140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RaduBerinde merged 1 commit into from
Jul 31, 2026
+70 −47
Merged

crl-release-26.2: valsep: account for blob-file compression in split-decision size estimate #6140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 23 additions & 9 deletions internal/manifest/blob_metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"fmt"
"iter"
"maps"
"math/bits"
"slices"
"strings"
"sync/atomic"
Expand Down Expand Up @@ -65,15 +66,10 @@ func MakeBlobReference(
}
}
return BlobReference{
FileID: fileID,
ValueSize: valueSize,
BackingValueSize: backingValueSize,
// valueSize
// Reference size = ----------------- × phys.Size
// phys.ValueSize
//
// We perform the multiplication first to avoid floating point arithmetic.
EstimatedPhysicalSize: (valueSize * phys.Size) / phys.ValueSize,
FileID: fileID,
ValueSize: valueSize,
BackingValueSize: backingValueSize,
EstimatedPhysicalSize: phys.EstimatedReferencePhysicalSize(valueSize),
}
}

Expand Down Expand Up @@ -163,6 +159,24 @@ func (m *PhysicalBlobFile) FileInfo() (base.FileType, base.DiskFileNum) {
return base.FileTypeBlob, m.FileNum
}

// EstimatedReferencePhysicalSize returns an estimate of the physical (on-disk)
// size occupied by a reference to valueSize uncompressed bytes of values stored
// in this blob file. It scales the file's physical size by the fraction of the
// file's values that are referenced:
//
// valueSize
// Reference size = ----------- × m.Size
// m.ValueSize
//
// The multiplication is performed in 128-bit precision to avoid overflow (and
// to avoid floating point arithmetic). The quotient is guaranteed to fit in a
// uint64 because valueSize ≤ m.ValueSize, so the result is at most m.Size.
func (m *PhysicalBlobFile) EstimatedReferencePhysicalSize(valueSize uint64) uint64 {
hi, lo := bits.Mul64(valueSize, m.Size)
q, _ := bits.Div64(hi, lo, m.ValueSize)
return q
}

// UserKeyBounds returns the user key bounds of the blob file, if known.
func (m *PhysicalBlobFile) UserKeyBounds() base.UserKeyBounds {
// TODO(jackson): Add bounds for blob files.
Expand Down
27 changes: 14 additions & 13 deletions testdata/compaction/value_separation
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,17 +689,18 @@ Blob files:
compact a-zzzz
----
L6:
000052:[a@1#0,SET-ckw@1#0,SET] seqnums:[#0-#0] points:[a@1#0,SET-ckw@1#0,SET] size:23102 blobrefs:[(B000007: 51136), (B000009: 51136), (B000011: 6592); depth:1]
000053:[ckx@1#0,SET-ewd@1#0,SET] seqnums:[#0-#0] points:[ckx@1#0,SET-ewd@1#0,SET] size:22438 blobrefs:[(B000011: 44544), (B000013: 51136), (B000015: 13824); depth:1]
000054:[ewe@1#0,SET-hgu@1#0,SET] seqnums:[#0-#0] points:[ewe@1#0,SET-hgu@1#0,SET] size:23225 blobrefs:[(B000015: 37312), (B000017: 51136), (B000019: 19968); depth:1]
000055:[hgv@1#0,SET-js@1#0,SET] seqnums:[#0-#0] points:[hgv@1#0,SET-js@1#0,SET] size:22564 blobrefs:[(B000019: 31168), (B000021: 51136), (B000023: 27072); depth:1]
000056:[jsa@1#0,SET-mcx@1#0,SET] seqnums:[#0-#0] points:[jsa@1#0,SET-mcx@1#0,SET] size:23096 blobrefs:[(B000023: 24128), (B000025: 51136), (B000027: 33600); depth:1]
000057:[mcy@1#0,SET-onw@1#0,SET] seqnums:[#0-#0] points:[mcy@1#0,SET-onw@1#0,SET] size:23043 blobrefs:[(B000027: 17600), (B000029: 51200), (B000031: 40128); depth:1]
000058:[onx@1#0,SET-qyv@1#0,SET] seqnums:[#0-#0] points:[onx@1#0,SET-qyv@1#0,SET] size:23002 blobrefs:[(B000031: 11008), (B000033: 51200), (B000035: 46720); depth:1]
000059:[qyw@1#0,SET-tjs@1#0,SET] seqnums:[#0-#0] points:[qyw@1#0,SET-tjs@1#0,SET] size:23114 blobrefs:[(B000035: 4416), (B000037: 51200), (B000039: 51136), (B000041: 2112); depth:1]
000060:[tjt@1#0,SET-vuj@1#0,SET] seqnums:[#0-#0] points:[tjt@1#0,SET-vuj@1#0,SET] size:23522 blobrefs:[(B000041: 49088), (B000043: 51200), (B000045: 8128); depth:1]
000061:[vuk@1#0,SET-yez@1#0,SET] seqnums:[#0-#0] points:[vuk@1#0,SET-yez@1#0,SET] size:23539 blobrefs:[(B000045: 43008), (B000047: 51200), (B000049: 14144); depth:1]
000062:[yf@1#0,SET-zzz@1#0,SET] seqnums:[#0-#0] points:[yf@1#0,SET-zzz@1#0,SET] size:17782 blobrefs:[(B000049: 36992), (B000051: 44288); depth:1]
000052:[a@1#0,SET-ciw@1#0,SET] seqnums:[#0-#0] points:[a@1#0,SET-ciw@1#0,SET] size:22391 blobrefs:[(B000007: 51136), (B000009: 51136), (B000011: 3136); depth:1]
000053:[cix@1#0,SET-esc@1#0,SET] seqnums:[#0-#0] points:[cix@1#0,SET-esc@1#0,SET] size:21772 blobrefs:[(B000011: 48000), (B000013: 51136), (B000015: 6848); depth:1]
000054:[esd@1#0,SET-hba@1#0,SET] seqnums:[#0-#0] points:[esd@1#0,SET-hba@1#0,SET] size:22360 blobrefs:[(B000015: 44288), (B000017: 51136), (B000019: 10048); depth:1]
000055:[hbb@1#0,SET-jjt@1#0,SET] seqnums:[#0-#0] points:[hbb@1#0,SET-jjt@1#0,SET] size:22651 blobrefs:[(B000019: 41088), (B000021: 51136), (B000023: 12800); depth:1]
000056:[jju@1#0,SET-lss@1#0,SET] seqnums:[#0-#0] points:[jju@1#0,SET-lss@1#0,SET] size:22351 blobrefs:[(B000023: 38400), (B000025: 51136), (B000027: 15936); depth:1]
000057:[lst@1#0,SET-obq@1#0,SET] seqnums:[#0-#0] points:[lst@1#0,SET-obq@1#0,SET] size:22367 blobrefs:[(B000027: 35264), (B000029: 51200), (B000031: 19008); depth:1]
000058:[obr@1#0,SET-qkv@1#0,SET] seqnums:[#0-#0] points:[obr@1#0,SET-qkv@1#0,SET] size:21931 blobrefs:[(B000031: 32128), (B000033: 51200), (B000035: 22528); depth:1]
000059:[qkw@1#0,SET-stv@1#0,SET] seqnums:[#0-#0] points:[qkw@1#0,SET-stv@1#0,SET] size:22111 blobrefs:[(B000035: 28608), (B000037: 51200), (B000039: 25728); depth:1]
000060:[stw@1#0,SET-vct@1#0,SET] seqnums:[#0-#0] points:[stw@1#0,SET-vct@1#0,SET] size:22382 blobrefs:[(B000039: 25408), (B000041: 51200), (B000043: 28864); depth:1]
000061:[vcu@1#0,SET-xls@1#0,SET] seqnums:[#0-#0] points:[vcu@1#0,SET-xls@1#0,SET] size:22302 blobrefs:[(B000043: 22336), (B000045: 51136), (B000047: 32000); depth:1]
000062:[xlt@1#0,SET-zur@1#0,SET] seqnums:[#0-#0] points:[xlt@1#0,SET-zur@1#0,SET] size:22340 blobrefs:[(B000047: 19200), (B000049: 51136), (B000051: 35136); depth:1]
000063:[zus@1#0,SET-zzz@1#0,SET] seqnums:[#0-#0] points:[zus@1#0,SET-zzz@1#0,SET] size:2667 blobrefs:[(B000051: 9152); depth:1]
Blob files:
B000007 physical:{000007 size:[53138 (52KB)] vals:[51136 (50KB)]}
B000009 physical:{000009 size:[53138 (52KB)] vals:[51136 (50KB)]}
Expand Down Expand Up @@ -730,8 +731,8 @@ excise-dryrun b c
----
would excise 1 files.
del-table: L6 000052
add-table: L6 000063(000052):[a@1#0,SET-azz@1#0,SET] seqnums:[#0-#0] points:[a@1#0,SET-azz@1#0,SET] size:11748(23102) blobrefs:[(B000007: 26004), (B000009: 26004), (B000011: 3352); depth:1]
add-table: L6 000064(000052):[c@1#0,SET-ckw@1#0,SET] seqnums:[#0-#0] points:[c@1#0,SET-ckw@1#0,SET] size:6302(23102) blobrefs:[(B000007: 13949), (B000009: 13949), (B000011: 1798); depth:1]
add-table: L6 000064(000052):[a@1#0,SET-azz@1#0,SET] seqnums:[#0-#0] points:[a@1#0,SET-azz@1#0,SET] size:11748(22391) blobrefs:[(B000007: 26829), (B000009: 26829), (B000011: 1645); depth:1]
add-table: L6 000065(000052):[c@1#0,SET-ciw@1#0,SET] seqnums:[#0-#0] points:[c@1#0,SET-ciw@1#0,SET] size:5598(22391) blobrefs:[(B000007: 12784), (B000009: 12784), (B000011: 784); depth:1]
add-backing: 000052


Expand Down
4 changes: 2 additions & 2 deletions valsep/testdata/value_separation_policy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ RawWriter.Add("c#9,SET", "paddy", false)

estimated-sizes
----
file: 0, references: 9
file: 0, references: 123

close-output
----
Expand All @@ -145,7 +145,7 @@ RawWriter.AddWithBlobHandle("e#9,SET", "(f0,blk0,id1,len5)", 5, false)

estimated-sizes
----
file: 0, references: 5
file: 0, references: 11

close-output
----
Expand Down
54 changes: 31 additions & 23 deletions valsep/value_separator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,16 @@ type ValueSeparator struct {
type pendingReference struct {
blobFileID base.BlobFileID
valueSize uint64
// preserved indicates whether this reference is being carried over
// from an existing blob reference, or is a new reference being
// written to a new blob file.
preserved bool
// phys is the physical blob file backing a preserved reference. It is nil
// for new (non-preserved) references being written to a new blob file.
phys *manifest.PhysicalBlobFile
}

// preserved reports whether this reference is being carried over from an
// existing blob reference (as opposed to a new reference being written to a new
// blob file). A preserved reference has its source physical blob file resolved.
func (r pendingReference) preserved() bool {
return r.phys != nil
}

type blobWriterAndMeta struct {
Expand Down Expand Up @@ -202,17 +208,17 @@ func (vs *ValueSeparator) EstimatedReferenceSize() uint64 {
// When we're writing to new blob files, the size of the blob file itself is
// a better estimate of the disk space consumed than the uncompressed value
// sizes, so we use vs.EstimatedFileSize.
//
// TODO(jackson): The totalValueSize is the uncompressed value sizes. With
// compressible data, it overestimates the disk space consumed by the blob
// references. It also does not include the blob file's index block or
// footer, so it can underestimate if values are completely incompressible.
//
// Should we compute a compression ratio per blob file and scale the
// references appropriately?
size := vs.EstimatedFileSize()
for i := range vs.blobTiers {
size += vs.blobTiers[i].totalPreservedValueSize
// For preserved references, scale the uncompressed value size by the source
// blob file's compression ratio. This uses the same computation as the
// EstimatedPhysicalSize set by manifest.MakeBlobReference, so this in-flight
// estimate converges to the size the output table will ultimately report.
for i := range vs.currPendingReferences {
ref := &vs.currPendingReferences[i]
if !ref.preserved() {
continue
}
size += ref.phys.EstimatedReferencePhysicalSize(ref.valueSize)
}
return size
}
Expand Down Expand Up @@ -343,11 +349,18 @@ func (vs *ValueSeparator) preserveBlobReference(
}); refIdx != -1 {
refID = base.BlobReferenceID(refIdx)
} else {
// First reference to this blob file in the current output. Resolve its
// physical file now (once per unique reference); it's needed to scale
// the reference's size by the file's compression ratio.
phys, ok := vs.inputBlobPhysicalFiles[fileID]
if !ok {
return errors.AssertionFailedf("pebble: blob file %s not found among input sstables", errors.Safe(fileID))
}
refID = base.BlobReferenceID(len(vs.currPendingReferences))
vs.currPendingReferences = append(vs.currPendingReferences, pendingReference{
blobFileID: fileID,
valueSize: 0,
preserved: true,
phys: phys,
})
}
if invariants.Enabled && vs.currPendingReferences[refID].blobFileID != fileID {
Expand Down Expand Up @@ -436,7 +449,7 @@ func (vs *ValueSeparator) maybeCheckInvariants() {
// sum of all the reference value sizes.
totalValueSize := uint64(0)
for _, ref := range vs.currPendingReferences {
if ref.preserved {
if ref.preserved() {
totalValueSize += ref.valueSize
} else {
panic("no new references should exist in preserveAllHotBlobReferences mode")
Expand All @@ -461,15 +474,10 @@ func (vs *ValueSeparator) FinishOutput() (ValueSeparationMetadata, error) {
// these references must exist in the input files.
for i := range vs.currPendingReferences {
ref := vs.currPendingReferences[i]
if !ref.preserved {
if !ref.preserved() {
continue
}
phys, ok := vs.inputBlobPhysicalFiles[ref.blobFileID]
if !ok {
return ValueSeparationMetadata{},
errors.AssertionFailedf("pebble: blob file %s not found among input sstables", errors.Safe(ref.blobFileID))
}
references[i] = manifest.MakeBlobReference(ref.blobFileID, ref.valueSize, ref.valueSize, phys)
references[i] = manifest.MakeBlobReference(ref.blobFileID, ref.valueSize, ref.valueSize, ref.phys)
}

newBlobFiles, newReferencedValueSize, err := vs.closeWriters(references)
Expand Down
Loading