Add timeouts, and test restore with network partition

.gitignore

@@ -12,4 +12,4 @@ libsql-sqlite3/**.o.tmp
 
 /bindings/c/generated
 /bindings/c/**.xcframework
-/bindings/**/.DS_Store
+/bindings/**/.DS_Store

bottomless/Cargo.toml

@@ -13,6 +13,7 @@ anyhow = "1.0.66"
 async-compression = { version = "0.4.4", features = ["tokio", "gzip", "zstd"] }
 aws-config = { version = "1" }
 aws-sdk-s3 = { version = "1" }
+aws-smithy-types = { version = "1" }
 bytes = "1"
 libsql-sys = { path = "../libsql-sys" }
 libsql_replication = { path = "../libsql-replication" }

bottomless/src/replicator.rs

@@ -15,6 +15,7 @@ use aws_sdk_s3::operation::list_objects::builders::ListObjectsFluentBuilder;
 use aws_sdk_s3::operation::list_objects::ListObjectsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::{Client, Config};
+use aws_smithy_types::timeout::TimeoutConfig;
 use bytes::{Buf, Bytes};
 use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
 use libsql_replication::injector::Injector as _;
@@ -121,6 +122,12 @@ pub struct Options {
     pub s3_max_parallelism: usize,
     /// Max number of retries for S3 operations
     pub s3_max_retries: u32,
+    /// Timeout for reading the first byte of an S3 response (seconds)
+    pub s3_read_timeout_secs: u64,
+    /// Timeout for establishing a TCP connection to S3 (seconds)
+    pub s3_connect_timeout_secs: u64,
+    /// Timeout for a single S3 operation attempt, including retries (seconds)
+    pub s3_operation_attempt_timeout_secs: u64,
     /// Skip snapshot upload per checkpoint.
     pub skip_snapshot: bool,
     /// Skip uploading snapshots on shutdown
@@ -145,6 +152,11 @@ impl Options {
             "LIBSQL_BOTTOMLESS_AWS_SECRET_ACCESS_KEY was not set"
         ))?;
         let session_token: Option<String> = self.session_token.clone();
+        let timeout_config = TimeoutConfig::builder()
+            .read_timeout(Duration::from_secs(self.s3_read_timeout_secs))
+            .connect_timeout(Duration::from_secs(self.s3_connect_timeout_secs))
+            .operation_attempt_timeout(Duration::from_secs(self.s3_operation_attempt_timeout_secs))
+            .build();
         let conf = loader
             .behavior_version(BehaviorVersion::latest())
             .region(Region::new(region))
@@ -159,6 +171,7 @@ impl Options {
                 aws_sdk_s3::config::retry::RetryConfig::standard()
                     .with_max_attempts(self.s3_max_retries),
             )
+            .timeout_config(timeout_config)
             .build();
 
         let s3_config = aws_sdk_s3::config::Builder::from(&conf)
@@ -233,6 +246,12 @@ impl Options {
             ),
         };
         let s3_max_retries = env_var_or("LIBSQL_BOTTOMLESS_S3_MAX_RETRIES", 10).parse::<u32>()?;
+        let s3_read_timeout_secs =
+            env_var_or("LIBSQL_BOTTOMLESS_S3_READ_TIMEOUT_SECS", 5).parse::<u64>()?;
+        let s3_connect_timeout_secs =
+            env_var_or("LIBSQL_BOTTOMLESS_S3_CONNECT_TIMEOUT_SECS", 5).parse::<u64>()?;
+        let s3_operation_attempt_timeout_secs =
+            env_var_or("LIBSQL_BOTTOMLESS_S3_OPERATION_ATTEMPT_TIMEOUT_SECS", 10).parse::<u64>()?;
         let cipher = match encryption_cipher {
             Some(cipher) => Cipher::from_str(&cipher)?,
             None => Cipher::default(),
@@ -261,6 +280,9 @@ impl Options {
             region,
             bucket_name,
             s3_max_retries,
+            s3_read_timeout_secs,
+            s3_connect_timeout_secs,
+            s3_operation_attempt_timeout_secs,
             skip_snapshot,
             skip_shutdown_upload,
         })

libsql-server/Cargo.toml

@@ -99,6 +99,7 @@ indicatif = "0.17.8"
 [dev-dependencies]
 arbitrary = { version = "1.3.0", features = ["derive_arbitrary"] }
 env_logger = "0.10"
+hex = "0.4"
 hyper = { workspace = true, features = ["client"] }
 insta = { version = "1.26.0", features = ["json"] }
 libsql = { path = "../libsql/"}
@@ -114,6 +115,7 @@ s3s-fs = "0.8.1"
 ring = { version = "0.17.8", features = ["std"] }
 tonic-build = "0.11"
 prost-build = "0.12"
+reqwest = { version = "0.11", features = ["json"] }
 
 [build-dependencies]
 vergen = { version = "8", features = ["build", "git", "gitcl"] }

libsql-server/src/namespace/meta_store.rs

@@ -129,6 +129,9 @@ pub async fn metastore_connection_maker(
                 max_batch_interval: config.backup_interval,
                 s3_max_parallelism: 32,
                 s3_max_retries: 10,
+                s3_read_timeout_secs: 5,
+                s3_connect_timeout_secs: 5,
+                s3_operation_attempt_timeout_secs: 10,
                 skip_snapshot: false,
                 skip_shutdown_upload: false,
             };