ceremonyclient/pebble/testdata/compaction_delete_only_hints

# The first few cases are adapted from this ASCII example. The y-axis is
# sequence numbers and the x-axis is the user key space. LSM levels are
# omitted from the visualization.
#
# 250
#       +--------00004 (fragmented)------+
#       V                                |
#       |-b...230:h-|                    |
# _______________________________________V_____________ snapshot #210
# 200               |--h.RANGEDEL.200:r--|
#
# _____________________________________________________ snapshot #180
#
# 150                     +--------+
#           +---------+   | 000006 |
#           | 000005  |   |        |
#           +_________+   |        |
# 100_____________________|________|___________________ snapshot #100
#                         +--------+
# _____________________________________________________ snapshot #70
#                             +---------------+
#  50                         | 000007        |
#                             |               |
#                             +---------------+
# ______________________________________________________________
#     a b c d e f g h i j k l m n o p q r s t u v w x y z

define snapshots=(70, 100, 180, 210)
L0
b.RANGEDEL.230:h h.RANGEDEL.200:r
L2
d.SET.110:d i.SET.140:i
L3
k.SET.90:k o.SET.150:o
L4
m.SET.30:m u.SET.60:u
----
0.0:
  000004:[b#230,RANGEDEL-r#inf,RANGEDEL]
2:
  000005:[d#110,SET-i#140,SET]
3:
  000006:[k#90,SET-o#150,SET]
4:
  000007:[m#30,SET-u#60,SET]

# Test a hint that is blocked by open snapshots. No compaction should occur
# and the hint should not be removed.

get-hints
----
L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only)

maybe-compact
----
Deletion hints:
  L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only)
Compactions:
  (none)

# Adopt the same LSM but without snapshots 100, 180 and 210.

define snapshots=(70)
L0
b.RANGEDEL.230:h h.RANGEDEL.200:r
L2
d.SET.110:d i.SET.140:i
L3
k.SET.90:k o.SET.150:o
L4
m.SET.30:m u.SET.60:u
----
0.0:
  000004:[b#230,RANGEDEL-r#inf,RANGEDEL]
2:
  000005:[d#110,SET-i#140,SET]
3:
  000006:[k#90,SET-o#150,SET]
4:
  000007:[m#30,SET-u#60,SET]

get-hints
----
L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only)

maybe-compact
----
Deletion hints:
  (none)
Compactions:
  [JOB 100] compacted(delete-only) L2 [000005] (677B) Score=0.00 + L3 [000006] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s

# Verify that compaction correctly handles the presence of multiple
# overlapping hints which might delete a file multiple times. All of the
# resolvable hints should be removed.

define snapshots=(70)
L0
a.RANGEDEL.300:k
L1
b.RANGEDEL.230:h h.RANGEDEL.200:r
L2
d.SET.110:d i.SET.140:i
L3
k.SET.90:k o.SET.150:o
L4
m.SET.30:m u.SET.60:u
----
0.0:
  000004:[a#300,RANGEDEL-k#inf,RANGEDEL]
1:
  000005:[b#230,RANGEDEL-r#inf,RANGEDEL]
2:
  000006:[d#110,SET-i#140,SET]
3:
  000007:[k#90,SET-o#150,SET]
4:
  000008:[m#30,SET-u#60,SET]

get-hints
----
L0.000004 a-k seqnums(tombstone=300-300, file-smallest=110, type=point-key-only)
L1.000005 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only)

maybe-compact
----
Deletion hints:
  (none)
Compactions:
  [JOB 100] compacted(delete-only) L2 [000006] (677B) Score=0.00 + L3 [000007] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s

# Test a range tombstone that is already compacted into L6.

define snapshots=(70)
L0
m.SET.300:m b.RANGEDEL.230:h h.RANGEDEL.200:r
L2
d.SET.110:d i.SET.140:i
L3
k.SET.90:k o.SET.150:o
L4
m.SET.30:m u.SET.60:u
----
0.0:
  000004:[b#230,RANGEDEL-r#inf,RANGEDEL]
2:
  000005:[d#110,SET-i#140,SET]
3:
  000006:[k#90,SET-o#150,SET]
4:
  000007:[m#30,SET-u#60,SET]

get-hints
----
L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only)

compact a-z
----
5:
  000008:[b#230,RANGEDEL-u#0,SET]

maybe-compact
----
Deletion hints:
  (none)
Compactions:
  (none)

# The same test case, without snapshots, with a table (000008) that exists
# within the range del user key bounds, but above it in the LSM.

define
L1
b.RANGEDEL.230:h h.RANGEDEL.200:r
L2
d.SET.110:d i.SET.140:i
L3
k.SET.90:k o.SET.150:o
L4
m.SET.30:m u.SET.60:u
L0
e.SET.240:e m.SET.260:m
----
0.0:
  000008:[e#240,SET-m#260,SET]
1:
  000004:[b#230,RANGEDEL-r#inf,RANGEDEL]
2:
  000005:[d#110,SET-i#140,SET]
3:
  000006:[k#90,SET-o#150,SET]
4:
  000007:[m#30,SET-u#60,SET]

get-hints
----
L1.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only)

# Tables 000005 and 000006 can be deleted as their largest sequence numbers fall
# below the smallest sequence number of the range del. Table 000007 falls
# outside the user key bounds, and table 000008 exists at a sequence number
# above the range del, so neither are deleted.

maybe-compact
----
Deletion hints:
  (none)
Compactions:
  [JOB 100] compacted(delete-only) L2 [000005] (677B) Score=0.00 + L3 [000006] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s

# A deletion hint present on an sstable in a higher level should NOT result in a
# deletion-only compaction incorrectly removing an sstable in L6 following an
# elision-only compaction that zeroes the sequence numbers in an L6 table.
#
# This is a regression test for pebble#1285.

# Create an sstable at L6. We expect that the SET survives the following
# sequence of compactions.
define snapshots=(10, 25)
L6
a.SET.20:b a.RANGEDEL.15:z
----
6:
  000004:[a#20,SETWITHDEL-z#inf,RANGEDEL]

# Note that this test depends on stats being present on the sstables, so we
# collect hints here. We expect none, as the table is in L6.
get-hints
----
(none)

# Place a compaction hint on a non-existent table in a higher level in the LSM.
#
# The selection of the sequence numbers for the hints is nuanced, and warrants
# some explanation. The largest tombstone sequence number (27) and file smallest
# sequence number (0) were chosen such that they fall into different snapshot
# stripes, which ensures the hint is not resolved and dropped. The deletion
# range 5-27 is also chosen such that it covers the sequence number range from
# the table, i.e. 15-20, which *appears* to make the keys eligible for deletion.
force-set-hints
L0.000001 a-z 0 5-27 point_key_only
----
L0.000001 a-z seqnums(tombstone=5-27, file-smallest=0, type=point-key-only)

# Hints on the table are unchanged, as the new sstable is at L6, and hints are
# not generated on tables at this level.
get-hints
----
L0.000001 a-z seqnums(tombstone=5-27, file-smallest=0, type=point-key-only)

# Closing snapshot 10 triggers an elision-only compaction in L6 rather than a
# deletion-only compaction, as the earliest snapshot that remains open is 25,
# preventing the delete compaction hint from being resolved as it does not exist
# in the same snapshot stripe as the table in L6.
close-snapshot
10
----
[JOB 100] compacted(elision-only) L6 [000004] (741B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (662B), in 1.0s (2.0s total), output rate 662B/s

# The deletion hint was removed by the elision-only compaction.
get-hints
----
(none)

# The LSM contains the key, as expected.
iter
first
next
----
a: (b, .)
.

# Closing the next snapshot should NOT trigger another compaction, as the
# deletion hint was removed in the elision-only compaction.
close-snapshot
25
----
(none)

# The key remains in the LSM.
iter
first
next
----
a: (b, .)
.

# Construct a scenario with tables containing a mixture of range dels and range
# key dels that sit within different types of hints.
#
#   +------- 000013 (internally fragmented spans) ----|
#   |                                                 V
#   |                       |-------------------------| m.RANGEKEYDEL:z
#   |               |-------|                           i.RANGEKEYDEL:m
#   V         |-----------------------|                 f.RANGEDEL:r
#   |---------|                                         a.RANGEDEL:f
#               +-+             +---+             +---+
#               | | 000006      |   | 000009      |   | 000012 <- Point keys only.
#               +-+             +---+             +---+
#         +---+           +---+             +---+
#         |   | 000005    |   | 000008      |   | 000011       <- Range keys only.
#         +---+           +---+             +---+
#   +---+           +---+             +---+
#   |   | 000004    |   | 000007      |   | 000010             <- Point and range keys.
#   +---+           +---+             +---+
# __________________________________________________________
#   a b c d e f g h i j k l m n o p q r s t u v w x y z
#
# Note that table 000013 contains both range dels and range key dels that have
# been internally fragmented. After defragmentation there are three hints
# created:
# - [a, i) - a point-key-only hint
# - [i, r) - a point-and-range-key hint
# - [r, z) - a range-key-only hint
#
# Based on the defragmented hints, the following tables can be deleted:
# - 000006: covered by range del hint [a, i), table contains only point keys.
# - 000007: covered by mixed hint [i, r), table contains point and range keys.
# - 000008: covered by mixed hint [i, r), table contains only range keys.
# - 000009: covered by mixed hint [i, r), table contains only point keys.
# - 000011: covered by range key hint [r, z), table contains only range keys.
#

# NOTE: the LSM shown in the example above is created bottom-up via ingestions.

reset
----

ingest ext
set a a
range-key-set a c @1 foo
set c c
----
OK

ingest ext
range-key-set d f @2 bar
----
OK

ingest ext
set g g
set h h
----
OK

ingest ext
set i i
range-key-set i k @1 v1
set k k
----
OK

ingest ext
range-key-set l n @2 bar
----
OK

ingest ext
set o o
set q q
----
OK

ingest ext
set r r
range-key-set r t @1 v1
set t t
----
OK

ingest ext
range-key-set u w @2 bar
----
OK

ingest ext
set x x
set z z
----
OK

ingest ext
del-range a f
del-range f r
range-key-del i m
range-key-del m z
----
OK

describe-lsm
----
0.0:
  000013:[a#19,RANGEDEL-z#inf,RANGEKEYDEL]
6:
  000004:[a#10,RANGEKEYSET-c#10,SET]
  000005:[d#11,RANGEKEYSET-f#inf,RANGEKEYSET]
  000006:[g#12,SET-h#12,SET]
  000007:[i#13,RANGEKEYSET-k#13,SET]
  000008:[l#14,RANGEKEYSET-n#inf,RANGEKEYSET]
  000009:[o#15,SET-q#15,SET]
  000010:[r#16,RANGEKEYSET-t#16,SET]
  000011:[u#17,RANGEKEYSET-w#inf,RANGEKEYSET]
  000012:[x#18,SET-z#18,SET]

get-hints
----
L0.000013 a-i seqnums(tombstone=19-19, file-smallest=12, type=point-key-only)
L0.000013 i-r seqnums(tombstone=19-19, file-smallest=13, type=point-and-range-key)
L0.000013 r-z seqnums(tombstone=19-19, file-smallest=17, type=range-key-only)

maybe-compact
----
Deletion hints:
  (none)
Compactions:
  [JOB 100] compacted(delete-only) L6 [000006 000007 000008 000009 000011] (3.9KB) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s