mirror of
https://github.com/clockworklabs/SpacetimeDB.git
synced 2026-05-22 07:32:16 -04:00
Kim Altintop
2c3fc66f21
* commitlog: Panic on fsync failure Errors returned by `fsync(2)` are particularly nefarious, as it is mostly undefined what the state of the page cache is in this case. Since the log is synced asynchronously and not after every write, it is impossible to know up to which commit data can be considered durable -- except by reading the most recent segment from disk. Therefore, the reasonable thing to do is to prevent any further use of the log, and force users to re-load it from disk. Note that this is only half of the solution: an application restart may still read data from the page cache, which could be gone after a system restart. To fix this, we would need to employ direct I/O (i.e. `O_DIRECT`), which however is beyond the scope of this patch as it invalidates the use of most of `std::io`. * commitlog: Handle duplicate commits when iterating We cannot exclude the possibility of a false failure in I/O operations. In particular, `EIO` errors are difficult to attribute to a particular write, as they happen asynchronously during flush of the page cache. Because we do not bypass the page cache, the possibility exists that a particular commit is lost when it isn't, or that it is considered durable when it isn't. The former could lead to duplicate commits appearing in the log, while the latter could lead to a matching offset number, but with different commit payload. This patch thus ignores duplicates, and introduces a new error variant in the event the offset matches but the checksum doesn't. * durability: Manage the flush-and-sync task in this crate Since syncing the commitlog may now panic, it is more obvious to handle all async tasks here, so as to be able to handle the panic cases. Namely, if the `FlushAndSyncTask` panics, the `PersisterTask` is aborted. This will lead to the channel receiver being dropped, which in turn will cause the next `append_tx` call to panic. * commitlog: Remove async flush-and-sync Due to panic behaviour, it is now preferable to manage periodic sync at the use site of the commitlog crate. Hence remove `flush_and_sync_every` method, and with it the dependency on tokio.
30 lines
683 B
TOML
30 lines
683 B
TOML
[package]
|
|
name = "spacetimedb-commitlog"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
rust-version.workspace = true
|
|
license-file = "LICENSE"
|
|
|
|
description = "Implementation of the SpacetimeDB commitlog."
|
|
|
|
[features]
|
|
default = ["serde"]
|
|
|
|
[dependencies]
|
|
bitflags.workspace = true
|
|
crc32c.workspace = true
|
|
itertools.workspace = true
|
|
log.workspace = true
|
|
serde = { workspace = true, optional = true }
|
|
spacetimedb-primitives.workspace = true
|
|
spacetimedb-sats.workspace = true
|
|
thiserror.workspace = true
|
|
|
|
[dev-dependencies]
|
|
env_logger.workspace = true
|
|
once_cell.workspace = true
|
|
proptest-derive.workspace = true
|
|
proptest.workspace = true
|
|
rand.workspace = true
|
|
tempfile.workspace = true
|