sharedcache: implement minimal version of metadata persistence

This commit implements an approach to persisting metadata thought of by Radu.
The metadata being persisted is the mapping from <filenum, logical block
offset> to the index of the cache block at which the data is located on disk.
The basic approach is to write changes to the metadata, plus metadata pointed
at by a ptr based on a sequence number, to a circular log that can be replayed
at server start time.

As of this commit, metadata persistence is neither performant nor safe in the
presence of crashes or certain errors happening at file write time. We need to
also implement the working set idea thought of by Radu, but this commit is
large enough without that.

Author: joshimhoff

objstorage/objstorageprovider/provider.go

@@ -106,6 +106,11 @@ type Settings struct {
 		// 2*runtime.GOMAXPROCS is used as the shard count.
 		CacheShardCount int
 
+		// If true, metadata on the mapping from logical block indexes to cache block indexes
+		// is persisted, implying that the contents in the cache are usable after a server
+		// restart.
+		PersistMetadata bool
+
 		// TODO(radu): allow the cache to live on another FS/location (e.g. to use
 		// instance-local SSD).
 	}

objstorage/objstorageprovider/shared.go

@@ -80,7 +80,8 @@ func (p *provider) sharedInit() error {
 			numShards = 2 * runtime.GOMAXPROCS(0)
 		}
 
-		p.shared.cache, err = sharedcache.Open(p.st.FS, p.st.FSDirName, blockSize, p.st.Shared.CacheSizeBytes, numShards)
+		p.shared.cache, err = sharedcache.Open(
+			p.st.FS, p.st.FSDirName, blockSize, p.st.Shared.CacheSizeBytes, numShards, p.st.Shared.PersistMetadata)
 		if err != nil {
 			return errors.Wrapf(err, "pebble: could not open shared object cache")
 		}