fix(store): retry GCS 408, 502 and client timeouts

crates/walgit-store/src/gcs.rs

@@ -1201,9 +1201,13 @@ fn is_retryable(e: &google_cloud_storage::Error) -> bool {
         );
     }
     if let Some(code) = e.http_status_code() {
-        return matches!(code, 503 | 504 | 429 | 500);
+        // The statuses GCS documents as retryable. The client's own policy
+        // retries all of 5xx; 501 is left out because an unimplemented
+        // request will not start working on a second try. Same set as
+        // `s3::is_transient_status`.
+        return matches!(code, 408 | 429 | 500 | 502 | 503 | 504);
     }
-    e.is_connect() || e.is_io() || e.is_transient_and_before_rpc()
+    e.is_connect() || e.is_io() || e.is_timeout() || e.is_transient_and_before_rpc()
 }
 
 fn map_error(key: &str, e: google_cloud_storage::Error) -> StoreError {
@@ -1384,6 +1388,32 @@ mod tests {
         assert!(is_retryable(&e));
     }
 
+    fn http_error(status: u16) -> google_cloud_storage::Error {
+        google_cloud_gax::error::Error::http(status, http::HeaderMap::new(), bytes::Bytes::new())
+    }
+
+    #[test]
+    fn transient_http_statuses_are_retryable() {
+        for status in [408, 429, 500, 502, 503, 504] {
+            assert!(
+                is_retryable(&http_error(status)),
+                "{status} should be retryable"
+            );
+        }
+        for status in [400, 403, 404, 409, 412, 501] {
+            assert!(
+                !is_retryable(&http_error(status)),
+                "{status} should be permanent"
+            );
+        }
+    }
+
+    #[test]
+    fn is_retryable_timeout() {
+        let e = google_cloud_gax::error::Error::timeout("attempt timed out");
+        assert!(is_retryable(&e));
+    }
+
     #[test]
     fn is_not_retryable_not_found() {
         let e = google_cloud_gax::error::Error::service(

crates/walgit-store/src/s3.rs

@@ -261,7 +261,7 @@ fn is_transient_code(code: &str) -> bool {
 /// HTTP statuses that mean the same, for services whose error codes we do not
 /// recognise (rustfs and other S3-compatible stores do not all use AWS codes).
 fn is_transient_status(status: u16) -> bool {
-    matches!(status, 429 | 500 | 502 | 503 | 504)
+    matches!(status, 408 | 429 | 500 | 502 | 503 | 504)
 }
 
 /// Whether an SDK failure is worth another attempt at walgit's layer.
@@ -1303,10 +1303,10 @@ mod tests {
 
     #[test]
     fn transient_statuses_are_recognised() {
-        for status in [429, 500, 502, 503, 504] {
+        for status in [408, 429, 500, 502, 503, 504] {
             assert!(is_transient_status(status), "{status} should be transient");
         }
-        for status in [400, 403, 404, 409, 412] {
+        for status in [400, 403, 404, 409, 412, 501] {
             assert!(!is_transient_status(status), "{status} should be permanent");
         }
     }