ceremonyclient/pebble/internal/metamorphic/metaflags/meta_flags.go

// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

// Package metaflags defines command-line flags for the metamorphic tests and
// provides functionality to construct the respective
// metamorphic.RunOptions/RunOnceOptions.
package metaflags

import (
	"flag"
	"fmt"
	"math"
	"regexp"

	"github.com/cockroachdb/pebble/internal/randvar"
	"github.com/cockroachdb/pebble/metamorphic"
)

// CommonFlags contains flags that apply to both metamorphic.Run and
// metamorphic.RunOnce/Compare.
type CommonFlags struct {
	// Dir is the directory storing test state. See "dir" flag below.
	Dir string
	// Seed for generation of random operations. See "seed" flag below.
	Seed uint64
	// ErrorRate is the rate of injected filesystem errors. See "error-rate" flag
	// below.
	ErrorRate float64
	// FailRE causes the test to fail if the output matches this regex. See "fail"
	// flag below.
	FailRE string
	// Keep determines if the DB directory is kept on successful runs. See "keep"
	// flag below.
	Keep bool
	// MaxThreads used by a single run. See "max-threads" flag below.
	MaxThreads int
	// NumInstances is the number of Pebble instances to create in one run. See
	// "num-instances" flag below.
	NumInstances int
}

func initCommonFlags() *CommonFlags {
	c := &CommonFlags{}
	flag.StringVar(&c.Dir, "dir", "_meta",
		"the directory storing test state")

	flag.Uint64Var(&c.Seed, "seed", 0,
		"a pseudorandom number generator seed")

	// TODO: default error rate to a non-zero value. Currently, retrying is
	// non-deterministic because of the Ierator.*WithLimit() methods since
	// they may say that the Iterator is not valid, but be positioned at a
	// certain key that can be returned in the future if the limit is changed.
	// Since that key is hidden from clients of Iterator, the retryableIter
	// using SeekGE will not necessarily position the Iterator that saw an
	// injected error at the same place as an Iterator that did not see that
	// error.
	flag.Float64Var(&c.ErrorRate, "error-rate", 0.0,
		"rate of errors injected into filesystem operations (0 ≤ r < 1)")

	flag.StringVar(&c.FailRE, "fail", "",
		"fail the test if the supplied regular expression matches the output")

	flag.BoolVar(&c.Keep, "keep", false,
		"keep the DB directory even on successful runs")

	flag.IntVar(&c.MaxThreads, "max-threads", math.MaxInt,
		"limit execution of a single run to the provided number of threads; must be ≥ 1")

	flag.IntVar(&c.NumInstances, "num-instances", 1, "number of pebble instances to create (default: 1)")

	return c
}

// RunOnceFlags contains flags that apply only to metamorphic.RunOnce/Compare.
type RunOnceFlags struct {
	*CommonFlags
	// RunDir applies to metamorphic.RunOnce and contains the specific
	// configuration of the run. See "run-dir" flag below.
	RunDir string
	// Compare applies to metamorphic.Compare. See "compare" flag below.
	Compare string
}

func initRunOnceFlags(c *CommonFlags) *RunOnceFlags {
	ro := &RunOnceFlags{CommonFlags: c}
	flag.StringVar(&ro.RunDir, "run-dir", "",
		"the specific configuration to (re-)run (used for post-mortem debugging)")

	flag.StringVar(&ro.Compare, "compare", "",
		`comma separated list of options files to compare. The result of each run is compared with
the result of the run from the first options file in the list. Example, -compare
random-003,standard-000. The dir flag should have the directory containing these directories.
Example, -dir _meta/200610-203012.077`)
	return ro
}

// RunFlags contains flags that apply only to metamorphic.Run.
type RunFlags struct {
	*CommonFlags
	// FS controls the type of filesystems to use. See "fs" flag below.
	FS string
	// TraceFile for execution tracing. See "trace-file" flag below.
	TraceFile string
	// Ops describes how the total number of operations is generated. See "ops" flags below.
	Ops randvar.Flag
	// InnerBinary is the binary to invoke for a single run. See "inner-binary"
	// flag below.
	InnerBinary string
	// PreviousOps is the path to the ops file of a previous run. See the
	// "previous-ops" flag below.
	PreviousOps string
	// InitialStatePath is the path to a database data directory from a previous
	// run. See the "initial-state" flag below.
	InitialStatePath string
	// InitialStateDesc is a human-readable description of the initial database
	// state. See "initial-state-desc" flag below.
	InitialStateDesc string
}

func initRunFlags(c *CommonFlags) *RunFlags {
	r := &RunFlags{CommonFlags: c}
	flag.StringVar(&r.FS, "fs", "rand",
		`force the tests to use either memory or disk-backed filesystems (valid: "mem", "disk", "rand")`)

	flag.StringVar(&r.TraceFile, "trace-file", "",
		"write an execution trace to `<run-dir>/file`")

	if err := r.Ops.Set("uniform:5000-10000"); err != nil {
		panic(err)
	}
	flag.Var(&r.Ops, "ops", "uniform:5000-10000")

	flag.StringVar(&r.InnerBinary, "inner-binary", "",
		`binary to run for each instance of the test (this same binary by default); cannot be used
with --run-dir or --compare`)

	// The following options may be used for split-version metamorphic testing.
	// To perform split-version testing, the client runs the metamorphic tests
	// on an earlier Pebble SHA passing the `--keep` flag. The client then
	// switches to the later Pebble SHA, setting the below options to point to
	// the `ops` file and one of the previous run's data directories.

	flag.StringVar(&r.PreviousOps, "previous-ops", "",
		"path to an ops file, used to prepopulate the set of keys operations draw from")

	flag.StringVar(&r.InitialStatePath, "initial-state", "",
		"path to a database's data directory, used to prepopulate the test run's databases")

	flag.StringVar(&r.InitialStateDesc, "initial-state-desc", "",
		`a human-readable description of the initial database state.
		If set this parameter is written to the OPTIONS to aid in
		debugging. It's intended to describe the lineage of a
		database's state, including sufficient information for
		reproduction (eg, SHA, prng seed, etc).`)
	return r
}

// InitRunOnceFlags initializes the flags that are used for a single run of the
// metamorphic test.
func InitRunOnceFlags() *RunOnceFlags {
	return initRunOnceFlags(initCommonFlags())
}

// InitAllFlags initializes all metamorphic test flags: those used for a
// single run, and those used for a "top level" run.
func InitAllFlags() (*RunOnceFlags, *RunFlags) {
	c := initCommonFlags()
	return initRunOnceFlags(c), initRunFlags(c)
}

// MakeRunOnceOptions constructs RunOnceOptions based on the flags.
func (ro *RunOnceFlags) MakeRunOnceOptions() []metamorphic.RunOnceOption {
	onceOpts := []metamorphic.RunOnceOption{
		metamorphic.MaxThreads(ro.MaxThreads),
	}
	if ro.Keep {
		onceOpts = append(onceOpts, metamorphic.KeepData{})
	}
	if ro.FailRE != "" {
		onceOpts = append(onceOpts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(ro.FailRE)})
	}
	if ro.ErrorRate > 0 {
		onceOpts = append(onceOpts, metamorphic.InjectErrorsRate(ro.ErrorRate))
	}
	if ro.NumInstances > 1 {
		onceOpts = append(onceOpts, metamorphic.MultiInstance(ro.NumInstances))
	}
	return onceOpts
}

// MakeRunOptions constructs RunOptions based on the flags.
func (r *RunFlags) MakeRunOptions() []metamorphic.RunOption {
	opts := []metamorphic.RunOption{
		metamorphic.Seed(r.Seed),
		metamorphic.OpCount(r.Ops.Static),
		metamorphic.MaxThreads(r.MaxThreads),
	}
	if r.Keep {
		opts = append(opts, metamorphic.KeepData{})
	}
	if r.FailRE != "" {
		opts = append(opts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(r.FailRE)})
	}
	if r.ErrorRate > 0 {
		opts = append(opts, metamorphic.InjectErrorsRate(r.ErrorRate))
	}
	if r.TraceFile != "" {
		opts = append(opts, metamorphic.RuntimeTrace(r.TraceFile))
	}
	if r.PreviousOps != "" {
		opts = append(opts, metamorphic.ExtendPreviousRun(r.PreviousOps, r.InitialStatePath, r.InitialStateDesc))
	}
	if r.NumInstances > 1 {
		opts = append(opts, metamorphic.MultiInstance(r.NumInstances))
	}

	// If the filesystem type was forced, all tests will use that value.
	switch r.FS {
	case "", "rand", "default":
		// No-op. Use the generated value for the filesystem.
	case "disk":
		opts = append(opts, metamorphic.UseDisk)
	case "mem":
		opts = append(opts, metamorphic.UseInMemory)
	default:
		panic(fmt.Sprintf("unknown forced filesystem type: %q", r.FS))
	}
	if r.InnerBinary != "" {
		opts = append(opts, metamorphic.InnerBinary(r.InnerBinary))
	}
	return opts
}