Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions packages/syft-enclave/src/syft_enclaves/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from syft_rds import SyftRDSClient, SyftRDSClientConfig
from syft_rds.waiting import JobCheck, JobStatusArg, wait_for_job

from syft.sync.transport.protocols import Transport
from syft.sync.connections.protocols import Connections
from syft.sync.encryption_policy import EncryptionPolicy
from syft.sync.peers.peer import Peer
from syft.sync.peers.peer_list import PeerList
Expand Down Expand Up @@ -783,18 +783,18 @@ def from_config(
cls,
config: SyftRDSClientConfig,
data_owners: list[str] | None = None,
transport: Transport | None = None,
connections: Connections | None = None,
) -> "SyftEnclaveClient":
"""Build a SyftEnclaveClient from an RDS config with a wrapped job_client.

Encryption rides along on ``config.sync`` via
``peer_manager_config.encryption_policy``; ``SyftboxManager.from_config``
(invoked inside ``SyftRDSClient.from_config``) loads/generates this
datasite's keys from its per-datasite key file, so no extra step is needed.
``transport`` replaces the one ``config.sync`` describes; tests pass
one over a mock Drive.
``connections`` replace the ones ``config.sync`` describes; tests pass
them over a mock Drive.
"""
rds = SyftRDSClient.from_config(config, transport)
rds = SyftRDSClient.from_config(config, connections)
rds.job_client = EnclaveJobClient(rds.job_client)

sync_engine = rds.sync_engine
Expand Down
8 changes: 4 additions & 4 deletions packages/syft-enclave/src/syft_enclaves/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
)
from syft.sync.connections.drive.drive_cached_client import DriveCachedClient
from syft.sync.syftbox_manager import SyftboxManager
from syft.sync.transport.protocols import Transport
from syft.sync.connections.protocols import Connections


def create_configs(
Expand Down Expand Up @@ -57,15 +57,15 @@ def create_clients(configs) -> tuple:
backing_store = MockDriveBackingStore()
return tuple(
SyftEnclaveClient.from_config(
c, transport=_mock_transport(backing_store, c.sync.email)
c, connections=_mock_connections(backing_store, c.sync.email)
)
for c in configs
)


def _mock_transport(backing_store: MockDriveBackingStore, email: str) -> Transport:
def _mock_connections(backing_store: MockDriveBackingStore, email: str) -> Connections:
service = MockDriveService(backing_store, email)
return SyftboxManager._drive_transport(
return SyftboxManager._drive_connections(
DriveCachedClient.from_service(email, service)
)

Expand Down
10 changes: 5 additions & 5 deletions packages/syft-rds/src/syft_rds/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pydantic import BaseModel, ConfigDict
from syft_migration import ProtocolSchema

from syft.sync.transport.protocols import Transport
from syft.sync.connections.protocols import Connections
from syft.sync.syftbox_manager import SyftboxManager
from syft.sync.peers.peer import Peer
from syft.sync.utils.waiting import DEFAULT_POLL_INTERVAL, DEFAULT_WAIT_TIMEOUT
Expand Down Expand Up @@ -152,11 +152,11 @@ def model_post_init(self, __context: Any) -> None:
def from_config(
cls,
config: "SyftRDSClientConfig",
transport: Transport | None = None,
connections: Connections | None = None,
) -> "SyftRDSClient":
"""``transport`` replaces the one ``config.sync`` describes; tests pass
one over a mock Drive (see ``SyftboxManager.from_config``)."""
sync_engine = SyftboxManager.from_config(config.sync, transport)
"""``connections`` replace the ones ``config.sync`` describes; tests
pass them over a mock Drive (see ``SyftboxManager.from_config``)."""
sync_engine = SyftboxManager.from_config(config.sync, connections)
# The job client selects a job protocol version per peer from the
# schema the peer advertises, as the dataset manager does for layouts.
job_client = JobClient.from_config(
Expand Down
4 changes: 2 additions & 2 deletions packages/syft-rds/tests/test_dataset_upload_private.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self):

# Get the DS's raw GDrive connection and search for private folders
ds_cached_client = drive_cached_client_of(
ds_manager.sync_engine._connection_router.transport
ds_manager.sync_engine._connection_router.connections
)
results = (
ds_cached_client.client.drive_service.files()
Expand All @@ -158,7 +158,7 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self):

# DO should find it with the same query
do_cached_client = drive_cached_client_of(
do_manager.sync_engine._connection_router.transport
do_manager.sync_engine._connection_router.connections
)
do_results = (
do_cached_client.client.drive_service.files()
Expand Down
22 changes: 11 additions & 11 deletions packages/syft-rds/tests/test_sync_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ def test_dataset_only_mock_data_uploaded():
ds_manager.sync()

files = drive_cached_client_of(
ds_manager.sync_engine._connection_router.transport
ds_manager.sync_engine._connection_router.connections
).client.drive_service._backing_store.files
list(files.keys())
file_objs = list(files.values())
Expand Down Expand Up @@ -285,7 +285,7 @@ def test_jobs():
# a short cut, but we could do this based on timing eventually (if there are items in the
# queue for longer than a certain time we start pushing)
do_cached_client = drive_cached_client_of(
do_manager.sync_engine._connection_router.transport
do_manager.sync_engine._connection_router.connections
)
inbox_folder_id = do_cached_client.get_own_datasite_inbox_id(ds_manager.email)
inbox_file_metadatas = do_cached_client.get_file_metadatas_from_folder(
Expand Down Expand Up @@ -1043,7 +1043,7 @@ def test_ds_dataset_cache_aware_sync():

# Get the mock backing store and directories for creating second pair
mock_backing_store = drive_cached_client_of(
ds_manager.sync_engine._connection_router.transport
ds_manager.sync_engine._connection_router.connections
).client.drive_service._backing_store
ds_folder = ds_manager.syftbox_folder
do_folder = do_manager.syftbox_folder
Expand All @@ -1062,10 +1062,10 @@ def test_ds_dataset_cache_aware_sync():

# Replace mock backing store to share dataset collections
drive_cached_client_of(
ds_manager2.sync_engine._connection_router.transport
ds_manager2.sync_engine._connection_router.connections
).client.drive_service._backing_store = mock_backing_store
drive_cached_client_of(
do_manager2.sync_engine._connection_router.transport
do_manager2.sync_engine._connection_router.connections
).client.drive_service._backing_store = mock_backing_store

# Load peers (already approved in shared backing store)
Expand Down Expand Up @@ -1143,7 +1143,7 @@ def test_do_dataset_cache_aware_sync():

# Get the mock backing store and directories for creating second pair
mock_backing_store = drive_cached_client_of(
ds_manager.sync_engine._connection_router.transport
ds_manager.sync_engine._connection_router.connections
).client.drive_service._backing_store
ds_folder = ds_manager.syftbox_folder
do_folder = do_manager.syftbox_folder
Expand All @@ -1162,10 +1162,10 @@ def test_do_dataset_cache_aware_sync():

# Replace mock backing store to share dataset collections
drive_cached_client_of(
ds_manager2.sync_engine._connection_router.transport
ds_manager2.sync_engine._connection_router.connections
).client.drive_service._backing_store = mock_backing_store
drive_cached_client_of(
do_manager2.sync_engine._connection_router.transport
do_manager2.sync_engine._connection_router.connections
).client.drive_service._backing_store = mock_backing_store

# Load peers (already approved in shared backing store)
Expand Down Expand Up @@ -1210,7 +1210,7 @@ def test_in_memory_connection_load_state():

# Get the backing store that will persist across "restarts"
backing_store = drive_cached_client_of(
do_manager1.sync_engine._connection_router.transport
do_manager1.sync_engine._connection_router.connections
).client.drive_service._backing_store
ds_folder = ds_manager1.syftbox_folder
do_folder = do_manager1.syftbox_folder
Expand Down Expand Up @@ -1277,10 +1277,10 @@ def test_in_memory_connection_load_state():
ds_cached_client2 = DriveCachedClient.from_service(ds_email, service_ds)

do_manager2 = SyftRDSClient.from_config(
do_config2, SyftboxManager._drive_transport(do_cached_client2)
do_config2, SyftboxManager._drive_connections(do_cached_client2)
)
ds_manager2 = SyftRDSClient.from_config(
ds_config2, SyftboxManager._drive_transport(ds_cached_client2)
ds_config2, SyftboxManager._drive_connections(ds_cached_client2)
)

# Load peers
Expand Down
20 changes: 10 additions & 10 deletions packages/syft-rds/tests/test_version_mismatch_flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from unittest.mock import patch

from syft.sync.connections.drive.drive_peering import DrivePeering
from syft.sync.connections.drive.drive_peering import DrivePeeringConnection
from syft.sync.connections.drive.gdrive_client import GOOGLE_FOLDER_MIME_TYPE
from syft.sync.connections.drive.drive_cached_client import (
GDRIVE_P2P_FOLDER_DATASITE_PREFIX,
Expand Down Expand Up @@ -52,7 +52,7 @@ def _find_versioned_p2p_folders(cached_client, peer_email, version):
def _get_backing_store(manager):
"""Extract the mock backing store from a manager's connection."""
cached_client = drive_cached_client_of(
manager.peer_manager.connection_router.transport
manager.peer_manager.connection_router.connections
)
return cached_client.client.drive_service._backing_store

Expand All @@ -76,7 +76,7 @@ def _reinitialize_manager(
mock_service = MockDriveService(backing_store, email)
cached_client = DriveCachedClient.from_service(email, mock_service)
manager = SyftRDSClient.from_config(
config, SyftboxManager._drive_transport(cached_client)
config, SyftboxManager._drive_connections(cached_client)
)

if has_ds_role:
Expand Down Expand Up @@ -104,7 +104,7 @@ def _simulate_continue_on_mismatch(manager, backing_store):
cached_client = DriveCachedClient.from_service(email, mock_service)

def read_remote(e, t):
return DrivePeering(cached_client).read_own_peer_info().version_info
return DrivePeeringConnection(cached_client).read_own_peer_info().version_info

with (
patch(
Expand Down Expand Up @@ -180,7 +180,7 @@ def test_version_mismatch_continues_and_repairs():

# -- Step 4: Record P2P folders and personal folder id before upgrade --
do_cached_client = drive_cached_client_of(
do_manager.peer_manager.connection_router.transport
do_manager.peer_manager.connection_router.connections
)
do_p2p_current = _find_versioned_p2p_folders(
do_cached_client, ds_manager.email, SYFT_VERSION
Expand Down Expand Up @@ -225,7 +225,7 @@ def test_version_mismatch_continues_and_repairs():

# Personal folder is adopted (same Drive id, new name), not recreated.
do_cached_client_new = drive_cached_client_of(
do_manager.peer_manager.connection_router.transport
do_manager.peer_manager.connection_router.connections
)
new_personal_name = f"{NEW_VERSION}#{do_email}"
new_personal_id = do_cached_client_new.client.find_folder_by_name(
Expand Down Expand Up @@ -263,7 +263,7 @@ def test_version_mismatch_continues_and_repairs():
)

ds_cached_client_new = drive_cached_client_of(
ds_manager.peer_manager.connection_router.transport
ds_manager.peer_manager.connection_router.connections
)
ds_personal_name = f"{NEW_VERSION}#{ds_email}"
ds_personal_id = ds_cached_client_new.client.find_folder_by_name(
Expand Down Expand Up @@ -348,15 +348,15 @@ def test_login_writes_the_remote_version_file_too():
use_in_memory_cache=False,
check_versions=True,
)
peering = do_manager.peer_manager.connection_router.transport.peering
peering = do_manager.peer_manager.connection_router.connections.peering_con
assert peering.read_own_peer_info().version_info.syft_client_version == SYFT_VERSION

with (
patch("syft.version.SYFT_VERSION", NEW_VERSION),
patch("syft.sync.version.version_info.SYFT_VERSION", NEW_VERSION),
):
upgraded = _upgraded_manager(do_manager)
new_peering = upgraded.peer_manager.connection_router.transport.peering
new_peering = upgraded.peer_manager.connection_router.connections.peering_con
# Still the pre-upgrade version: nothing has refreshed it yet.
assert (
new_peering.read_own_peer_info().version_info.syft_client_version
Expand Down Expand Up @@ -407,7 +407,7 @@ def test_the_mismatch_prompt_does_not_return_after_a_login():
handle_potential_version_mismatches_on_login,
)

peering = do_manager.peer_manager.connection_router.transport.peering
peering = do_manager.peer_manager.connection_router.connections.peering_con
with patch(
"syft.sync.login_utils._read_remote_version",
side_effect=lambda e, t: peering.read_own_peer_info().version_info,
Expand Down
4 changes: 2 additions & 2 deletions syft/gdrive_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,14 +314,14 @@ def delete_remote_syftbox(
verbose: If True (default), print deletion progress.
"""
from syft.sync.connections.drive.drive_cached_client import DriveCachedClient
from syft.sync.connections.drive.drive_owner_store import DriveOwnerStore
from syft.sync.connections.drive.drive_owner_store import DriveOwnerStoreConnection

email = _resolve_email(email)
token_path = _resolve_token_path(token_path)
cached_client = DriveCachedClient.from_token_path(
email=email, token_path=token_path
)
store = DriveOwnerStore(cached_client)
store = DriveOwnerStoreConnection(cached_client)

# Gather file IDs via folder hierarchy
folder_file_ids = set(store.gather_all_file_and_folder_ids())
Expand Down
31 changes: 31 additions & 0 deletions syft/sync/connections/config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
"""The login each connection of the router uses, as plain config."""

from typing import Self

from pydantic import BaseModel

from syft.sync.connections.base_connection import ConnectionConfig
from syft.sync.connections.protocols import CONNECTION_FIELDS


class ConnectionRouterConfig(BaseModel):
"""The login each connection of the router uses.
``SyftboxManager.from_config`` builds the ``Connections`` from it; only one
Google Drive login for every connection is built so far."""

p2p_con: ConnectionConfig
owner_store_con: ConnectionConfig
watcher_store_con: ConnectionConfig
collections_con: ConnectionConfig
peering_con: ConnectionConfig

@classmethod
def one_connection(cls, connection_config: ConnectionConfig) -> Self:
"""Every component over the same connection."""
return cls(**{name: connection_config for name in CONNECTION_FIELDS})

def connection_configs(self) -> list[ConnectionConfig]:
return [getattr(self, name) for name in CONNECTION_FIELDS]

def uses_one_connection(self) -> bool:
return all(config == self.p2p_con for config in self.connection_configs())
Loading
Loading