mirror of
https://github.com/clockworklabs/SpacetimeDB.git
synced 2026-05-14 11:48:28 -04:00
Phoebe Goldman
e77b62f475
# Description of Changes We've run into a problem on Maincloud caused by a database that was writing a relatively small number of very large transactions. This was accruing many commitlog segments consuming hundreds of gigabytes of disk, but had not ever taken a snapshot, or compressed or archived any data, as the database had not progressed past one million transactions. With this PR, we take a snapshot every time the commitlog segment rotates. We still also snapshot every million transactions. One BitCraft database we looked at had 2.5 million transactions per commitlog segment, meaning that this change will not meaningfully affect the frequency of snapshots. The offending Maincloud database, however, had only 50 transactions per segment! # API and ABI breaking changes N/a # Expected complexity level and risk 3: Hastily made changes to finnicky code across several crates. # Testing I am unsure how to test these changes. - [ ] <!-- maybe a test you want to do --> - [ ] <!-- maybe a test you want a reviewer to do, so they can check it off when they're satisfied. -->
56 lines
1.7 KiB
TOML
56 lines
1.7 KiB
TOML
[package]
|
|
name = "spacetimedb-commitlog"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
rust-version.workspace = true
|
|
license-file = "LICENSE"
|
|
|
|
description = "Implementation of the SpacetimeDB commitlog."
|
|
|
|
[features]
|
|
default = ["serde"]
|
|
streaming = ["dep:async-stream", "dep:bytes", "dep:futures", "dep:tokio", "dep:tokio-util"]
|
|
# Enable types + impls useful for testing
|
|
test = ["dep:env_logger"]
|
|
|
|
[dependencies]
|
|
async-stream = { workspace = true, optional = true }
|
|
bitflags.workspace = true
|
|
bytes= { workspace = true, optional = true }
|
|
crc32c.workspace = true
|
|
futures = { workspace = true, optional = true }
|
|
itertools.workspace = true
|
|
log.workspace = true
|
|
memmap2 = "0.9.4"
|
|
serde = { workspace = true, optional = true }
|
|
spacetimedb-fs-utils.workspace = true
|
|
spacetimedb-paths.workspace = true
|
|
spacetimedb-primitives.workspace = true
|
|
spacetimedb-sats.workspace = true
|
|
tempfile.workspace = true
|
|
thiserror.workspace = true
|
|
tokio = { workspace = true, optional = true }
|
|
tokio-util = { workspace = true, optional = true, features = ["io-util"] }
|
|
zstd-framed.workspace = true
|
|
|
|
# For the 'test' feature
|
|
env_logger = { workspace = true, optional = true }
|
|
pretty_assertions.workspace = true
|
|
|
|
[dev-dependencies]
|
|
# Enable streaming in tests
|
|
# Also enable 'test' feature, so integration tests can use the helpers.
|
|
spacetimedb-commitlog = { path = ".", features = ["test", "streaming"] }
|
|
|
|
env_logger.workspace = true
|
|
once_cell.workspace = true
|
|
pretty_assertions = { workspace = true, features = ["unstable"] }
|
|
proptest-derive.workspace = true
|
|
proptest.workspace = true
|
|
rand.workspace = true
|
|
tempfile.workspace = true
|
|
tokio-stream = { version = "0.1.17", features = ["fs"] }
|
|
|
|
[lints]
|
|
workspace = true
|