mirror of
https://github.com/clockworklabs/SpacetimeDB.git
synced 2026-05-08 00:30:03 -04:00
Kim Altintop
cfd0d4b712
When a new commitlog segment is created, allocate disk space for it up to the maximum segment size. Also do this when resuming writes to an existing segment, such that segments created without preallocation will allocate as well when the database is opened. Preallocation is gated behind the feature "fallocate", because it is not always desirable to preallocate, e.g. for local `standalone` users. The feature can only be enabled on Linux targets, because allocation is done using the Linux-specific `fallocate(2)` system call. Unlike `ftruncate(2)` or the portable `posix_fallocate(3)`, `fallocate(2)` supports allocating disk space without zeroing. This is currently required, because the commitlog format does not handle padding bytes. If not enough space can be allocated, the commitlog refuses writes. For commitlogs that were created without preallocation, this means that the commitlog cannot even be opened in this situation. The local durability impl will crash if it detects that the commitlog is unable to allocate enough space. This means that a database will eventually crash and be unable to start in an out-of-space situation. Allocated space is not included in the reported size of the commitlog. Instead, allocated blocks are reported separately. # Expected complexity level and risk 3 - Disk size monitoring may need to be adjusted. # Testing - [x] Adds a test that demonstrates the crash behavior of [`spacetimedb_durability::Local`] when there is insufficient space. The test performs I/O against a loop device. - [x] Modified the `repo::Memory` impl so that it can run out of space. No test currently utilizes this, but existing tests assuming infinite space still pass.
60 lines
1.8 KiB
TOML
60 lines
1.8 KiB
TOML
[package]
|
|
name = "spacetimedb-commitlog"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
rust-version.workspace = true
|
|
license-file = "LICENSE"
|
|
|
|
description = "Implementation of the SpacetimeDB commitlog."
|
|
|
|
[features]
|
|
default = ["serde"]
|
|
# Enable streaming reads + writes
|
|
streaming = ["dep:async-stream", "dep:bytes", "dep:futures", "dep:tokio", "dep:tokio-util"]
|
|
# Enable types + impls useful for testing
|
|
test = ["dep:env_logger"]
|
|
# Enable `fallocate` of segments
|
|
fallocate = ["dep:nix"]
|
|
|
|
[dependencies]
|
|
async-stream = { workspace = true, optional = true }
|
|
bitflags.workspace = true
|
|
bytes= { workspace = true, optional = true }
|
|
crc32c.workspace = true
|
|
futures = { workspace = true, optional = true }
|
|
itertools.workspace = true
|
|
log.workspace = true
|
|
memmap2 = "0.9.4"
|
|
nix = { workspace = true, optional = true, features = ["fs"] }
|
|
serde = { workspace = true, optional = true }
|
|
spacetimedb-fs-utils.workspace = true
|
|
spacetimedb-paths.workspace = true
|
|
spacetimedb-primitives.workspace = true
|
|
spacetimedb-sats.workspace = true
|
|
tempfile.workspace = true
|
|
thiserror.workspace = true
|
|
tokio = { workspace = true, optional = true }
|
|
tokio-util = { workspace = true, optional = true, features = ["io-util"] }
|
|
zstd-framed.workspace = true
|
|
|
|
# For the 'test' feature
|
|
env_logger = { workspace = true, optional = true }
|
|
pretty_assertions.workspace = true
|
|
|
|
[dev-dependencies]
|
|
# Enable streaming in tests
|
|
# Also enable 'test' feature, so integration tests can use the helpers.
|
|
spacetimedb-commitlog = { path = ".", features = ["test", "streaming"] }
|
|
|
|
env_logger.workspace = true
|
|
once_cell.workspace = true
|
|
pretty_assertions = { workspace = true, features = ["unstable"] }
|
|
proptest-derive.workspace = true
|
|
proptest.workspace = true
|
|
rand.workspace = true
|
|
tempfile.workspace = true
|
|
tokio-stream = { version = "0.1.17", features = ["fs"] }
|
|
|
|
[lints]
|
|
workspace = true
|