From 924c9d85745593009cc36843b80443f6368f3c5f Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 7 Aug 2026 18:21:25 +0200 Subject: [PATCH 01/18] refactor: first batch of cycle breaks --- .knip.json | 3 + discojs/package.json | 60 +- discojs/src/aggregator.spec.ts | 6 +- discojs/src/aggregator/aggregator.ts | 5 +- discojs/src/aggregator/byzantine.ts | 6 +- discojs/src/aggregator/get.ts | 5 +- discojs/src/aggregator/mean.ts | 5 +- discojs/src/aggregator/secure.ts | 5 +- discojs/src/client/client.ts | 33 +- .../decentralized/decentralized_client.ts | 32 +- discojs/src/client/decentralized/messages.ts | 40 +- discojs/src/client/decentralized/peer.ts | 2 +- .../client/decentralized/peer_pool.spec.ts | 6 +- discojs/src/client/decentralized/peer_pool.ts | 4 +- discojs/src/client/event_connection.ts | 27 +- .../src/client/federated/federated_client.ts | 22 +- discojs/src/client/federated/messages.ts | 27 +- discojs/src/client/get_client.ts | 41 + discojs/src/client/index.ts | 1 + discojs/src/client/local_client.ts | 2 +- discojs/src/client/messages.ts | 67 -- discojs/src/client/mtype.ts | 66 ++ discojs/src/client/utils.ts | 23 +- discojs/src/models/cards/CIFAR10Classifier.ts | 3 +- discojs/src/models/cards/MNISTClassifier.ts | 5 +- discojs/src/models/cards/dogClassifier.ts | 5 +- discojs/src/models/cards/titanicClassifier.ts | 5 +- discojs/src/models/cards/wikitext.ts | 5 +- .../implementations/CIFAR10ClassifierModel.ts | 2 +- .../implementations/MNISTClassifierModel.ts | 2 +- .../implementations/dogClassifierModel.ts | 2 +- .../src/models/implementations/gpt/index.ts | 3 +- .../src/models/implementations/hellaswag.ts | 5 +- discojs/src/models/implementations/index.ts | 4 + .../implementations/titanicClassifierModel.ts | 2 +- discojs/src/models/index.ts | 12 +- discojs/src/models/model_card.ts | 2 +- discojs/src/privacy.ts | 4 +- discojs/src/processing/image.spec.ts | 2 +- discojs/src/processing/image.ts | 2 +- discojs/src/processing/index.ts | 134 +--- .../{index.spec.ts => processing.spec.ts} | 6 +- discojs/src/processing/processing.ts | 123 +++ discojs/src/serialization/index.ts | 9 +- discojs/src/serialization/json_like.ts | 8 + discojs/src/serialization/model.ts | 19 +- discojs/src/serialization/task.ts | 11 +- discojs/src/serialization/weights.ts | 2 +- discojs/src/task/display_information.ts | 2 +- discojs/src/task/task.ts | 2 +- discojs/src/task/task_handler.ts | 8 +- discojs/src/task/task_provider.ts | 4 +- discojs/src/task/training_information.ts | 4 +- discojs/src/training/disco.ts | 38 +- discojs/src/training/trainer.ts | 32 +- discojs/src/training/types.ts | 3 + discojs/src/types/data_format.ts | 11 +- discojs/src/validator.ts | 29 +- discojs/tsconfig.lib.json | 16 + package.json | 2 + pnpm-lock.yaml | 759 ++++++++++++++++++ .../controllers/decentralized_controller.ts | 2 +- .../src/controllers/federated_controller.ts | 2 +- server/src/controllers/training_controller.ts | 8 +- 64 files changed, 1327 insertions(+), 460 deletions(-) create mode 100644 discojs/src/client/get_client.ts create mode 100644 discojs/src/client/mtype.ts create mode 100644 discojs/src/models/implementations/index.ts rename discojs/src/processing/{index.spec.ts => processing.spec.ts} (87%) create mode 100644 discojs/src/processing/processing.ts create mode 100644 discojs/src/serialization/json_like.ts create mode 100644 discojs/src/training/types.ts diff --git a/.knip.json b/.knip.json index 8336807d9..151e9a13b 100644 --- a/.knip.json +++ b/.knip.json @@ -4,6 +4,9 @@ "ignore": ["docs/**"], "ignoreDependencies": ["typescript-eslint", "supports-color"] }, + "discojs": { + "ignoreExportsUsedInFile": true + }, "webapp": { "ignoreDependencies": ["buffer"] }, diff --git a/discojs/package.json b/discojs/package.json index 93c15c651..9967694ad 100644 --- a/discojs/package.json +++ b/discojs/package.json @@ -23,30 +23,70 @@ "url": "https://github.com/epfml/disco/issues" }, "imports": { + "#root/*": { + "@disco/source": "./src/*.ts", + "types": "./dist/*.d.ts", + "default": "./dist/*.js" + }, + "#aggregator/*": { + "@disco/source": "./src/aggregator/*.ts", + "types": "./dist/aggregator/*.d.ts", + "default": "./dist/aggregator/*.js" + }, + "#client/*": { + "@disco/source": "./src/client/*.ts", + "types": "./dist/client/*.d.ts", + "default": "./dist/client/*.js" + }, + "#dataset/*": { + "@disco/source": "./src/dataset/*.ts", + "types": "./dist/dataset/*.d.ts", + "default": "./dist/dataset/*.js" + }, + "#types/*": { + "@disco/source": "./src/types/*.ts", + "types": "./dist/types/*.d.ts", + "default": "./dist/types/*.js" + }, + "#logging/*": { + "@disco/source": "./src/logging/*.ts", + "types": "./dist/logging/*.d.ts", + "default": "./dist/logging/*.js" + }, "#models/*": { "@disco/source": "./src/models/*.ts", "types": "./dist/models/*.d.ts", "default": "./dist/models/*.js" }, + "#processing/*": { + "@disco/source": "./src/processing/*.ts", + "types": "./dist/processing/*.d.ts", + "default": "./dist/processing/*.js" + }, + "#serialization/*": { + "@disco/source": "./src/serialization/*.ts", + "types": "./dist/serialization/*.d.ts", + "default": "./dist/serialization/*.js" + }, "#task/*": { "@disco/source": "./src/task/*.ts", "types": "./dist/task/*.d.ts", "default": "./dist/task/*.js" }, + "#training/*": { + "@disco/source": "./src/training/*.ts", + "types": "./dist/training/*.d.ts", + "default": "./dist/training/*.js" + }, + "#utils/*": { + "@disco/source": "./src/utils/*.ts", + "types": "./dist/utils/*.d.ts", + "default": "./dist/utils/*.js" + }, "#weights/*": { "@disco/source": "./src/weights/*.ts", "types": "./dist/weights/*.d.ts", "default": "./dist/weights/*.js" - }, - "#types/*": { - "@disco/source": "./src/types/*.ts", - "types": "./dist/types/*.d.ts", - "default": "./dist/types/*.js" - }, - "#dataset/*": { - "@disco/source": "./src/dataset/*.ts", - "types": "./dist/dataset/*.d.ts", - "default": "./dist/dataset/*.js" } }, "homepage": "https://github.com/epfml/disco#readme", diff --git a/discojs/src/aggregator.spec.ts b/discojs/src/aggregator.spec.ts index 40f5806e0..fb2d960e9 100644 --- a/discojs/src/aggregator.spec.ts +++ b/discojs/src/aggregator.spec.ts @@ -4,9 +4,9 @@ import { type Aggregator, MeanAggregator, SecureAggregator, -} from "./aggregator/index.js"; -import type { NodeID } from "./client/types.js"; -import { WeightsContainer } from "./index.js"; +} from "#aggregator/index"; +import type { NodeID } from "#client/index"; +import { WeightsContainer } from "#weights/index"; const AGGREGATORS: Set<[name: string, new () => Aggregator]> = Set.of< new () => Aggregator diff --git a/discojs/src/aggregator/aggregator.ts b/discojs/src/aggregator/aggregator.ts index 7f437a27d..404d26886 100644 --- a/discojs/src/aggregator/aggregator.ts +++ b/discojs/src/aggregator/aggregator.ts @@ -1,9 +1,10 @@ import createDebug from "debug"; import { Map, Set } from "immutable"; -import type { client, WeightsContainer } from "../index.js"; +import type { WeightsContainer } from "#weights/index"; +import type * as client from "#client/index"; -import { EventEmitter } from "../utils/event_emitter.js"; +import { EventEmitter } from "#utils/event_emitter"; const debug = createDebug("discojs:aggregator"); diff --git a/discojs/src/aggregator/byzantine.ts b/discojs/src/aggregator/byzantine.ts index 476ef90cc..0db000adc 100644 --- a/discojs/src/aggregator/byzantine.ts +++ b/discojs/src/aggregator/byzantine.ts @@ -2,8 +2,10 @@ import { Map } from "immutable"; import * as tf from "@tensorflow/tfjs"; import { AggregationStep } from "./aggregator.js"; import { MultiRoundAggregator, ThresholdType } from "./multiround.js"; -import { WeightsContainer, client } from "../index.js"; -import { aggregation } from "../index.js"; +import { aggregation } from "#weights/index"; + +import type { WeightsContainer } from "#weights/index"; +import type * as client from "#client/index"; /** * Byzantine-robust aggregator using Centered Clipping (CC), based on the diff --git a/discojs/src/aggregator/get.ts b/discojs/src/aggregator/get.ts index ee53e1721..e9c42ff91 100644 --- a/discojs/src/aggregator/get.ts +++ b/discojs/src/aggregator/get.ts @@ -1,5 +1,6 @@ -import type { DataType, Network, Task } from "../index.js"; -import { aggregator } from "../index.js"; +import type { DataType, Network } from "#types/index"; +import type { Task } from "#task/index"; +import * as aggregator from "#aggregator/index"; import { ByzantineRobustAggregator } from "./byzantine.js"; type AggregatorOptions = Partial<{ diff --git a/discojs/src/aggregator/mean.ts b/discojs/src/aggregator/mean.ts index a4abba6c4..58cedc82d 100644 --- a/discojs/src/aggregator/mean.ts +++ b/discojs/src/aggregator/mean.ts @@ -1,8 +1,9 @@ import type { Map } from "immutable"; import { AggregationStep } from "./aggregator.js"; import { MultiRoundAggregator, ThresholdType } from "./multiround.js"; -import type { WeightsContainer, client } from "../index.js"; -import { aggregation } from "../index.js"; +import type { WeightsContainer } from "#weights/index"; +import type * as client from "#client/index"; +import { aggregation } from "#weights/index"; /** * Mean aggregator whose aggregation step consists in computing the mean of the received weights. diff --git a/discojs/src/aggregator/secure.ts b/discojs/src/aggregator/secure.ts index c20221311..9f7abc1e0 100644 --- a/discojs/src/aggregator/secure.ts +++ b/discojs/src/aggregator/secure.ts @@ -2,8 +2,9 @@ import { Map, List, Range } from "immutable"; import * as tf from "@tensorflow/tfjs"; import { AggregationStep, Aggregator } from "./aggregator.js"; -import type { WeightsContainer, client } from "../index.js"; -import { aggregation } from "../index.js"; +import type { WeightsContainer } from "#weights/index"; +import { aggregation } from "#weights/index"; +import type * as client from "#client/index"; /** * Aggregator implementing secure multi-party computation for decentralized learning. diff --git a/discojs/src/client/client.ts b/discojs/src/client/client.ts index 038d8ffad..667c58d19 100644 --- a/discojs/src/client/client.ts +++ b/discojs/src/client/client.ts @@ -1,19 +1,18 @@ import createDebug from "debug"; -import type { - DataType, - Model, - Network, - RoundStatus, - Task, - WeightsContainer, -} from "../index.js"; -import { serialization } from "../index.js"; -import type { NodeID } from "./types.js"; -import type { EventConnection } from "./event_connection.js"; -import type { Aggregator } from "../aggregator/index.js"; -import { EventEmitter } from "../utils/event_emitter.js"; -import { type } from "./messages.js"; +import type { Model } from "#models/index"; +import type { RoundStatus } from "#training/index"; +import type { DataType, Network } from "#types/index"; +import type { Task } from "#task/index"; +import type { WeightsContainer } from "#weights/index"; +import type { Aggregator } from "#aggregator/index"; +import { EventEmitter } from "#utils/event_emitter"; + +import * as serialization from "#serialization/index"; + +import type { EventConnection } from "#client/event_connection"; +import type { NodeID } from "#client/types"; +import { MType } from "#client/mtype"; const debug = createDebug("discojs:client"); @@ -120,7 +119,7 @@ export abstract class Client extends EventEmitter<{ protected setupServerCallbacks(setMessageInversionFlag: () => void) { // Setup an event callback if the server signals that we should // wait for more participants - this.server.on(type.WaitingForMoreParticipants, (event) => { + this.server.on(MType.WaitingForMoreParticipants, (event) => { if (this.promiseForMoreParticipants !== undefined) throw new Error( "Server sent multiple WaitingForMoreParticipants messages", @@ -143,7 +142,7 @@ export abstract class Client extends EventEmitter<{ // and directly follows with an EnoughParticipants message when the 2nd participant joins // However, the EnoughParticipants can arrive before the NewNodeInfo (which can be much bigger) // so we check whether we received the EnoughParticipants before being assigned a node ID - this.server.once(type.EnoughParticipants, (event) => { + this.server.once(MType.EnoughParticipants, (event) => { if (this._ownId === undefined) { setMessageInversionFlag(); this.nbOfParticipants = event.nbOfParticipants; @@ -160,7 +159,7 @@ export abstract class Client extends EventEmitter<{ protected async createPromiseForMoreParticipants(): Promise { return new Promise((resolve) => { // "once" is important because we can't resolve the same promise multiple times - this.server.once(type.EnoughParticipants, (event) => { + this.server.once(MType.EnoughParticipants, (event) => { debug( `[${shortenId(this.ownId)}] received EnoughParticipants message from server`, ); diff --git a/discojs/src/client/decentralized/decentralized_client.ts b/discojs/src/client/decentralized/decentralized_client.ts index 474968102..bd71aece2 100644 --- a/discojs/src/client/decentralized/decentralized_client.ts +++ b/discojs/src/client/decentralized/decentralized_client.ts @@ -1,18 +1,20 @@ import createDebug from "debug"; import { Map, Set } from "immutable"; -import type { DataType, Model, WeightsContainer } from "../../index.js"; -import { serialization } from "../../index.js"; -import { Client, shortenId } from "../client.js"; -import { type NodeID } from "../index.js"; -import { type, type ClientConnected } from "../messages.js"; -import { timeout } from "../utils.js"; +import type { WeightsContainer } from "#weights/index"; +import type { Model } from "#models/index"; +import type { DataType } from "#types/index"; +import * as serialization from "#serialization/index"; +import { Client, shortenId } from "#client/client"; +import { type NodeID } from "#client/index"; +import { MType, type ClientConnected } from "#client/mtype"; +import { timeout } from "#client/utils"; import { WebSocketServer, waitMessage, type PeerConnection, waitMessageWithTimeout, -} from "../event_connection.js"; +} from "#client/event_connection"; import { PeerPool } from "./peer_pool.js"; import * as messages from "./messages.js"; @@ -73,7 +75,7 @@ export class DecentralizedClient extends Client<"decentralized"> { messages.isMessageFromServer, messages.isMessageToServer, ); - this.server.on(type.SignalForPeer, (event) => { + this.server.on(MType.SignalForPeer, (event) => { if (this.#pool === undefined) throw new Error("received signal but peer pool is undefined"); // Create a WebRTC connection with the peer @@ -85,13 +87,13 @@ export class DecentralizedClient extends Client<"decentralized"> { this.setupServerCallbacks(() => (receivedEnoughParticipants = true)); const msg: ClientConnected = { - type: type.ClientConnected, + type: MType.ClientConnected, }; this.server.send(msg); const { id, waitForMoreParticipants, nbOfParticipants } = await waitMessage( this.server, - type.NewDecentralizedNodeInfo, + MType.NewDecentralizedNodeInfo, ); this.nbOfParticipants = nbOfParticipants; @@ -145,7 +147,7 @@ export class DecentralizedClient extends Client<"decentralized"> { override async onRoundBeginCommunication(): Promise { // Notify the server we want to join the next round so that the server // waits for us to be ready before sending the list of peers for the round - this.server.send({ type: type.JoinRound }); + this.server.send({ type: MType.JoinRound }); // Store the promise for the current round's aggregation result. // We will await for it to resolve at the end of the round when exchanging weight updates. this.aggregationResult = this.aggregator.getPromiseForAggregation(); @@ -190,7 +192,7 @@ export class DecentralizedClient extends Client<"decentralized"> { // Reset peers list at each round of training to make sure client works with an updated peers // list, maintained by the server. Adds any received weights to the aggregator. // Tell the server we are ready for the next round - const readyMessage: messages.PeerIsReady = { type: type.PeerIsReady }; + const readyMessage: messages.PeerIsReady = { type: MType.PeerIsReady }; this.server.send(readyMessage); // Wait for the server to answer with the list of peers for the round @@ -200,7 +202,7 @@ export class DecentralizedClient extends Client<"decentralized"> { ); const receivedMessage = await waitMessage( this.server, - type.PeersForRound, + MType.PeersForRound, ); const peers = Set(receivedMessage.peers); @@ -250,7 +252,7 @@ export class DecentralizedClient extends Client<"decentralized"> { try { const message = await waitMessageWithTimeout( connection, - type.Payload, + MType.Payload, 60_000, "Timeout waiting for a contribution from peer " + peerId, ); @@ -337,7 +339,7 @@ export class DecentralizedClient extends Client<"decentralized"> { if (peer !== undefined) { const encoded = await serialization.weights.encode(payload); const msg: messages.PeerMessage = { - type: type.Payload, + type: MType.Payload, peer: id, aggregationRound: this.aggregator.round, communicationRound, diff --git a/discojs/src/client/decentralized/messages.ts b/discojs/src/client/decentralized/messages.ts index 30991c3eb..4aa618895 100644 --- a/discojs/src/client/decentralized/messages.ts +++ b/discojs/src/client/decentralized/messages.ts @@ -1,17 +1,17 @@ -import { serialization } from "../../index.js"; +import * as serialization from "#serialization/index"; import { type SignalData } from "./peer.js"; -import { isNodeID, type NodeID } from "../types.js"; -import { type, hasMessageType } from "../messages.js"; +import { isNodeID, type NodeID } from "#client/types"; +import { MType, hasMessageType } from "#client/mtype"; import type { ClientConnected, WaitingForMoreParticipants, EnoughParticipants, -} from "../messages.js"; +} from "#client/mtype"; /// Phase 0 communication (between server and peers) export interface NewDecentralizedNodeInfo { - type: type.NewDecentralizedNodeInfo; + type: MType.NewDecentralizedNodeInfo; id: NodeID; waitForMoreParticipants: boolean; nbOfParticipants: number; @@ -19,24 +19,24 @@ export interface NewDecentralizedNodeInfo { // WebRTC signal to forward to other node export interface SignalForPeer { - type: type.SignalForPeer; + type: MType.SignalForPeer; peer: NodeID; signal: SignalData; } // peer wants to join the next round export interface JoinRound { - type: type.JoinRound; + type: MType.JoinRound; } // peer who sent is ready export interface PeerIsReady { - type: type.PeerIsReady; + type: MType.PeerIsReady; } // server sends to each peer the list of peers to connect to export interface PeersForRound { - type: type.PeersForRound; + type: MType.PeersForRound; peers: NodeID[]; aggregationRound: number; } @@ -44,7 +44,7 @@ export interface PeersForRound { /// Phase 1 communication (between peers) export interface Payload { - type: type.Payload; + type: MType.Payload; peer: NodeID; aggregationRound: number; communicationRound: number; @@ -72,19 +72,19 @@ export function isMessageFromServer(o: unknown): o is MessageFromServer { if (!hasMessageType(o)) return false; switch (o.type) { - case type.NewDecentralizedNodeInfo: + case MType.NewDecentralizedNodeInfo: return ( "id" in o && isNodeID(o.id) && "waitForMoreParticipants" in o && typeof o.waitForMoreParticipants === "boolean" ); - case type.SignalForPeer: + case MType.SignalForPeer: return "peer" in o && isNodeID(o.peer) && "signal" in o; // TODO check signal content? - case type.PeersForRound: + case MType.PeersForRound: return "peers" in o && Array.isArray(o.peers) && o.peers.every(isNodeID); - case type.WaitingForMoreParticipants: - case type.EnoughParticipants: + case MType.WaitingForMoreParticipants: + case MType.EnoughParticipants: return true; } @@ -95,12 +95,12 @@ export function isMessageToServer(o: unknown): o is MessageToServer { if (!hasMessageType(o)) return false; switch (o.type) { - case type.ClientConnected: + case MType.ClientConnected: return true; - case type.SignalForPeer: + case MType.SignalForPeer: return "peer" in o && isNodeID(o.peer) && "signal" in o; // TODO check signal content? - case type.JoinRound: - case type.PeerIsReady: + case MType.JoinRound: + case MType.PeerIsReady: return true; } @@ -111,7 +111,7 @@ export function isPeerMessage(o: unknown): o is PeerMessage { if (!hasMessageType(o)) return false; switch (o.type) { - case type.Payload: + case MType.Payload: return ( "peer" in o && isNodeID(o.peer) && diff --git a/discojs/src/client/decentralized/peer.ts b/discojs/src/client/decentralized/peer.ts index df8d41461..0ec23d77e 100644 --- a/discojs/src/client/decentralized/peer.ts +++ b/discojs/src/client/decentralized/peer.ts @@ -2,7 +2,7 @@ import { List, Map, Range, Seq } from "immutable"; import wrtc from "@epfml/isomorphic-wrtc"; import SimplePeer from "simple-peer"; -import type { NodeID } from "../types.js"; +import type { NodeID } from "#client/types"; type MessageID = number; type ChunkID = number; diff --git a/discojs/src/client/decentralized/peer_pool.spec.ts b/discojs/src/client/decentralized/peer_pool.spec.ts index 435f7ef0e..9d8606f49 100644 --- a/discojs/src/client/decentralized/peer_pool.spec.ts +++ b/discojs/src/client/decentralized/peer_pool.spec.ts @@ -2,7 +2,7 @@ import { Map, Range } from "immutable"; import { assert, afterEach, beforeEach, describe, it } from "vitest"; import type { EventConnection, PeerConnection } from "../event_connection.js"; -import { type } from "../messages.js"; +import { MType } from "../messages.js"; import type { NodeID } from "../types.js"; import type { messages } from "./index.js"; @@ -47,7 +47,7 @@ describe("peer pool", { timeout: 10_000 }, () => { function mockWeights(id: NodeID): messages.Payload { return { - type: type.Payload, + type: MType.Payload, peer: id, payload: Uint8Array.of(1, 2, 3), aggregationRound: 0, @@ -112,7 +112,7 @@ describe("peer pool", { timeout: 10_000 }, () => { .map( async (peer) => await new Promise((resolve) => { - peer.on(type.Payload, (data) => { + peer.on(MType.Payload, (data) => { resolve(data); }); }), diff --git a/discojs/src/client/decentralized/peer_pool.ts b/discojs/src/client/decentralized/peer_pool.ts index dd004b3a5..c4c9ccac7 100644 --- a/discojs/src/client/decentralized/peer_pool.ts +++ b/discojs/src/client/decentralized/peer_pool.ts @@ -2,8 +2,8 @@ import createDebug from "debug"; import { Map, type Set } from "immutable"; import { Peer, type SignalData } from "./peer.js"; -import type { NodeID } from "../types.js"; -import { PeerConnection, type EventConnection } from "../event_connection.js"; +import type { NodeID } from "#client/types"; +import { PeerConnection, type EventConnection } from "#client/event_connection"; const debug = createDebug("discojs:client:decentralized:pool"); diff --git a/discojs/src/client/event_connection.ts b/discojs/src/client/event_connection.ts index ca253ba64..700b6994a 100644 --- a/discojs/src/client/event_connection.ts +++ b/discojs/src/client/event_connection.ts @@ -1,22 +1,23 @@ import createDebug from "debug"; import WebSocket from "isomorphic-ws"; import * as msgpack from "@msgpack/msgpack"; -import type { Peer, SignalData } from "./decentralized/peer.js"; -import type { NodeID } from "./types.js"; -import * as decentralizedMessages from "./decentralized/messages.js"; -import { type, type NarrowMessage, type Message } from "./messages.js"; -import { timeout } from "./utils.js"; +import type { Peer, SignalData } from "#client/decentralized/peer"; +import type { NodeID } from "#client/types"; +import * as decentralizedMessages from "#client/decentralized/messages"; +import { MType } from "#client/mtype"; +import { type NarrowMessage, type Message } from "#client/messages"; +import { timeout } from "#client/utils"; -import { EventEmitter } from "../utils/event_emitter.js"; +import { EventEmitter } from "#utils/event_emitter"; const debug = createDebug("discojs:client:connections"); export interface EventConnection { - on: ( + on: ( type: K, handler: (event: NarrowMessage) => void, ) => void; - once: ( + once: ( type: K, handler: (event: NarrowMessage) => void, ) => void; @@ -24,7 +25,7 @@ export interface EventConnection { disconnect: () => Promise; } -export async function waitMessage( +export async function waitMessage( connection: EventConnection, type: T, ): Promise> { @@ -36,7 +37,7 @@ export async function waitMessage( }); } -export async function waitMessageWithTimeout( +export async function waitMessageWithTimeout( connection: EventConnection, type: T, timeoutMs?: number, @@ -49,7 +50,7 @@ export async function waitMessageWithTimeout( } export class PeerConnection - extends EventEmitter<{ [K in type]: NarrowMessage }> + extends EventEmitter<{ [K in MType]: NarrowMessage }> implements EventConnection { constructor( @@ -63,7 +64,7 @@ export class PeerConnection async connect(): Promise { this.peer.on("signal", (signal) => { const msg: decentralizedMessages.SignalForPeer = { - type: type.SignalForPeer, + type: MType.SignalForPeer, peer: this.peer.id, signal, }; @@ -108,7 +109,7 @@ export class PeerConnection } export class WebSocketServer - extends EventEmitter<{ [K in type]: NarrowMessage }> + extends EventEmitter<{ [K in MType]: NarrowMessage }> implements EventConnection { private constructor( diff --git a/discojs/src/client/federated/federated_client.ts b/discojs/src/client/federated/federated_client.ts index 9e21bd46a..be2d08155 100644 --- a/discojs/src/client/federated/federated_client.ts +++ b/discojs/src/client/federated/federated_client.ts @@ -1,11 +1,13 @@ import createDebug from "debug"; -import { serialization } from "../../index.js"; -import type { DataType, Model, WeightsContainer } from "../../index.js"; -import { Client, shortenId } from "../client.js"; -import { type, type ClientConnected } from "../messages.js"; -import { waitMessage, WebSocketServer } from "../event_connection.js"; -import * as messages from "./messages.js"; +import type { Model } from "#models/index"; +import type { DataType } from "#types/index"; +import type { WeightsContainer } from "#weights/index"; +import * as serialization from "#serialization/index"; +import { Client, shortenId } from "#client/client"; +import { MType, type ClientConnected } from "#client/mtype"; +import { waitMessage, WebSocketServer } from "#client/event_connection"; +import * as messages from "#client/federated/messages"; const debug = createDebug("discojs:client:federated"); @@ -55,12 +57,12 @@ export class FederatedClient extends Client<"federated"> { this.aggregator.registerNode(SERVER_NODE_ID); const msg: ClientConnected = { - type: type.ClientConnected, + type: MType.ClientConnected, }; this.server.send(msg); const { id, waitForMoreParticipants, payload, round, nbOfParticipants } = - await waitMessage(this.server, type.NewFederatedNodeInfo); + await waitMessage(this.server, MType.NewFederatedNodeInfo); // This should come right after receiving the message to make sure // we don't miss a subsequent message from the server @@ -139,7 +141,7 @@ export class FederatedClient extends Client<"federated"> { if (payloadToServer === undefined) throw new Error("aggregator didn't make a payload for the server"); const msg: messages.SendPayload = { - type: type.SendPayload, + type: MType.SendPayload, payload: await serialization.weights.encode(payloadToServer), round: this.aggregator.round, }; @@ -157,7 +159,7 @@ export class FederatedClient extends Client<"federated"> { payload: payloadFromServer, round: serverRound, nbOfParticipants, - } = await waitMessage(this.server, type.ReceiveServerPayload); // Wait indefinitely for the server update + } = await waitMessage(this.server, MType.ReceiveServerPayload); // Wait indefinitely for the server update this.nbOfParticipants = nbOfParticipants; // Save the current participants const serverResult = serialization.weights.decode(payloadFromServer); this.aggregator.setRound(serverRound); diff --git a/discojs/src/client/federated/messages.ts b/discojs/src/client/federated/messages.ts index 4d4ee0e2a..99db9e13e 100644 --- a/discojs/src/client/federated/messages.ts +++ b/discojs/src/client/federated/messages.ts @@ -1,13 +1,12 @@ -import type { serialization } from "../../index.js"; +import * as serialization from "#serialization/index"; +import type { NodeID } from "#client/types"; -import { type NodeID } from "..//types.js"; - -import { type, hasMessageType } from "../messages.js"; +import { MType, hasMessageType } from "#client/mtype"; import type { ClientConnected, WaitingForMoreParticipants, EnoughParticipants, -} from "../messages.js"; +} from "#client/mtype"; // See ../messages.ts for doc export type MessageFederated = @@ -19,7 +18,7 @@ export type MessageFederated = | EnoughParticipants; export interface NewFederatedNodeInfo { - type: type.NewFederatedNodeInfo; + type: MType.NewFederatedNodeInfo; id: NodeID; waitForMoreParticipants: boolean; payload: serialization.Encoded; @@ -28,12 +27,12 @@ export interface NewFederatedNodeInfo { } export interface SendPayload { - type: type.SendPayload; + type: MType.SendPayload; payload: serialization.Encoded; round: number; } export interface ReceiveServerPayload { - type: type.ReceiveServerPayload; + type: MType.ReceiveServerPayload; payload: serialization.Encoded; round: number; nbOfParticipants: number; // number of peers contributing to a federated training @@ -45,12 +44,12 @@ export function isMessageFederated(raw: unknown): raw is MessageFederated { } switch (raw.type) { - case type.ClientConnected: - case type.NewFederatedNodeInfo: - case type.SendPayload: - case type.ReceiveServerPayload: - case type.WaitingForMoreParticipants: - case type.EnoughParticipants: + case MType.ClientConnected: + case MType.NewFederatedNodeInfo: + case MType.SendPayload: + case MType.ReceiveServerPayload: + case MType.WaitingForMoreParticipants: + case MType.EnoughParticipants: return true; } diff --git a/discojs/src/client/get_client.ts b/discojs/src/client/get_client.ts new file mode 100644 index 000000000..9678c8dbe --- /dev/null +++ b/discojs/src/client/get_client.ts @@ -0,0 +1,41 @@ +import type { DataType, Network } from "#types/index"; +import type { Task } from "#task/index"; +import type * as aggregator from "#aggregator/index"; + +// import * as clients from "#client/index"; +import { LocalClient } from "#client/local_client"; +import type { Client } from "#client/client"; +import { DecentralizedClient } from "#client/decentralized/decentralized_client"; +import { FederatedClient } from "#client/federated/federated_client"; + +export function getClient( + scheme: N | "local", + serverURL: URL, + task: Task, + aggregator: aggregator.Aggregator, +): Client { + switch (scheme) { + case "decentralized": { + const t = task as Task; + t.trainingInformation.scheme = scheme; + + return new DecentralizedClient(serverURL, t, aggregator); + } + case "federated": { + const t = task as Task; + t.trainingInformation.scheme = scheme; + + return new FederatedClient(serverURL, t, aggregator); + } + case "local": { + const t = task as Task; + t.trainingInformation.scheme = scheme; + + return new LocalClient(serverURL, t, aggregator); + } + default: { + const _: never = scheme; + throw new Error("should never happen"); + } + } +} diff --git a/discojs/src/client/index.ts b/discojs/src/client/index.ts index f084d5a2f..b6b8812e9 100644 --- a/discojs/src/client/index.ts +++ b/discojs/src/client/index.ts @@ -6,6 +6,7 @@ export * as aggregator from "../aggregator/index.js"; export * as decentralized from "./decentralized/index.js"; export * as federated from "./federated/index.js"; export * as messages from "./messages.js"; +export * as mtype from "./mtype.js"; export { getClient, timeout } from "./utils.js"; export { LocalClient } from "./local_client.js"; diff --git a/discojs/src/client/local_client.ts b/discojs/src/client/local_client.ts index 1b40cc6bb..4e3559fa6 100644 --- a/discojs/src/client/local_client.ts +++ b/discojs/src/client/local_client.ts @@ -1,4 +1,4 @@ -import { WeightsContainer } from "../index.js"; +import { WeightsContainer } from "#weights/index"; import { Client } from "./client.js"; /** diff --git a/discojs/src/client/messages.ts b/discojs/src/client/messages.ts index afc2f6556..660dbb692 100644 --- a/discojs/src/client/messages.ts +++ b/discojs/src/client/messages.ts @@ -1,60 +1,6 @@ import type * as decentralized from "./decentralized/messages.js"; import type * as federated from "./federated/messages.js"; -export enum type { - // Sent from client to server as first point of contact to join a task. - // The server answers with an node id in a NewFederatedNodeInfo - // or NewDecentralizedNodeInfo message - ClientConnected, - - /* Decentralized */ - // When a user joins a task with a ClientConnected message, the server - // answers with its peer id and also tells the client whether we are waiting - // for more participants before starting training - NewDecentralizedNodeInfo, - // Message sent by peers to the server to signal they want to - // join the next round - JoinRound, - // Message sent by nodes to server signaling they are ready to - // start the next round - PeerIsReady, - // Sent by the server to participating peers containing the list - // of peers for the round - PeersForRound, - // Message forwarded by the server from a client to another client - // to establish a peer-to-peer (WebRTC) connection - SignalForPeer, - // The weight update - Payload, - - /* Federated */ - // The server answers the ClientConnected message with the necessary information - // to start training: node id, latest model global weights, current round etc - NewFederatedNodeInfo, - // Message sent by server to notify clients that there are not enough - // participants to continue training - WaitingForMoreParticipants, - // Message sent by server to notify clients that there are now enough - // participants to start training collaboratively - EnoughParticipants, - SendPayload, - ReceiveServerPayload, -} - -export interface ClientConnected { - type: type.ClientConnected; -} - -export interface EnoughParticipants { - type: type.EnoughParticipants; - nbOfParticipants: number; -} - -export interface WaitingForMoreParticipants { - type: type.WaitingForMoreParticipants; - nbOfParticipants: number; -} - export type Message = | decentralized.MessageFromServer | decentralized.MessageToServer @@ -63,16 +9,3 @@ export type Message = // Retrieve a specific message interface from the type D. i.e. NarrowMessage => messages.PeerId type export type NarrowMessage = Extract; - -export function hasMessageType( - raw: unknown, -): raw is { type: type } & Record { - if (typeof raw !== "object" || raw === null) return false; - - const o = raw as Record; - if (!("type" in o && typeof o.type === "number" && o.type in type)) { - return false; - } - - return true; -} diff --git a/discojs/src/client/mtype.ts b/discojs/src/client/mtype.ts new file mode 100644 index 000000000..f271cbe08 --- /dev/null +++ b/discojs/src/client/mtype.ts @@ -0,0 +1,66 @@ +export enum MType { + // Sent from client to server as first point of contact to join a task. + // The server answers with an node id in a NewFederatedNodeInfo + // or NewDecentralizedNodeInfo message + ClientConnected, + + /* Decentralized */ + // When a user joins a task with a ClientConnected message, the server + // answers with its peer id and also tells the client whether we are waiting + // for more participants before starting training + NewDecentralizedNodeInfo, + // Message sent by peers to the server to signal they want to + // join the next round + JoinRound, + // Message sent by nodes to server signaling they are ready to + // start the next round + PeerIsReady, + // Sent by the server to participating peers containing the list + // of peers for the round + PeersForRound, + // Message forwarded by the server from a client to another client + // to establish a peer-to-peer (WebRTC) connection + SignalForPeer, + // The weight update + Payload, + + /* Federated */ + // The server answers the ClientConnected message with the necessary information + // to start training: node id, latest model global weights, current round etc + NewFederatedNodeInfo, + // Message sent by server to notify clients that there are not enough + // participants to continue training + WaitingForMoreParticipants, + // Message sent by server to notify clients that there are now enough + // participants to start training collaboratively + EnoughParticipants, + SendPayload, + ReceiveServerPayload, +} + +export function hasMessageType( + raw: unknown, +): raw is { type: MType } & Record { + if (typeof raw !== "object" || raw === null) return false; + + const o = raw as Record; + if (!("type" in o && typeof o.type === "number" && o.type in MType)) { + return false; + } + + return true; +} + +export interface ClientConnected { + type: MType.ClientConnected; +} + +export interface EnoughParticipants { + type: MType.EnoughParticipants; + nbOfParticipants: number; +} + +export interface WaitingForMoreParticipants { + type: MType.WaitingForMoreParticipants; + nbOfParticipants: number; +} diff --git a/discojs/src/client/utils.ts b/discojs/src/client/utils.ts index cfc3e6922..a2b37f11b 100644 --- a/discojs/src/client/utils.ts +++ b/discojs/src/client/utils.ts @@ -1,5 +1,12 @@ -import type { DataType, Network, Task } from "../index.js"; -import { client as clients, type aggregator } from "../index.js"; +import type { DataType, Network } from "#types/index"; +import type { Task } from "#task/index"; +import type * as aggregator from "#aggregator/index"; + +// import * as clients from "#client/index"; +import { LocalClient } from "#client/local_client"; +import { Client } from "#client/client"; +import { DecentralizedClient } from "#client/decentralized/decentralized_client"; +import { FederatedClient } from "#client/federated/federated_client"; // Time to wait for the others in milliseconds. const MAX_WAIT_PER_ROUND = 15_000; @@ -20,29 +27,25 @@ export function getClient( serverURL: URL, task: Task, aggregator: aggregator.Aggregator, -): clients.Client { +): Client { switch (scheme) { case "decentralized": { const t = task as Task; t.trainingInformation.scheme = scheme; - return new clients.decentralized.DecentralizedClient( - serverURL, - t, - aggregator, - ); + return new DecentralizedClient(serverURL, t, aggregator); } case "federated": { const t = task as Task; t.trainingInformation.scheme = scheme; - return new clients.federated.FederatedClient(serverURL, t, aggregator); + return new FederatedClient(serverURL, t, aggregator); } case "local": { const t = task as Task; t.trainingInformation.scheme = scheme; - return new clients.LocalClient(serverURL, t, aggregator); + return new LocalClient(serverURL, t, aggregator); } default: { const _: never = scheme; diff --git a/discojs/src/models/cards/CIFAR10Classifier.ts b/discojs/src/models/cards/CIFAR10Classifier.ts index c59b7c2eb..ddf9dd70f 100644 --- a/discojs/src/models/cards/CIFAR10Classifier.ts +++ b/discojs/src/models/cards/CIFAR10Classifier.ts @@ -1,4 +1,5 @@ -import { Model, ModelCard } from "#models/index"; +import { Model } from "#models/model"; +import { ModelCard } from "#models/model_card"; import { getModel } from "#models/implementations/CIFAR10ClassifierModel"; export const CIFAR10Classifier: ModelCard<"image"> = { diff --git a/discojs/src/models/cards/MNISTClassifier.ts b/discojs/src/models/cards/MNISTClassifier.ts index 0bb6c4ca8..8e7c264a2 100644 --- a/discojs/src/models/cards/MNISTClassifier.ts +++ b/discojs/src/models/cards/MNISTClassifier.ts @@ -1,5 +1,6 @@ -import { Model, ModelCard } from "../index.js"; -import { model } from "../implementations/MNISTClassifierModel.js"; +import { Model } from "#models/model"; +import { ModelCard } from "#models/model_card"; +import { model } from "#models/implementations/MNISTClassifierModel"; export const MNISTClassifier: ModelCard<"image"> = { card: { diff --git a/discojs/src/models/cards/dogClassifier.ts b/discojs/src/models/cards/dogClassifier.ts index 6e7c762c8..d37305bb2 100644 --- a/discojs/src/models/cards/dogClassifier.ts +++ b/discojs/src/models/cards/dogClassifier.ts @@ -1,5 +1,6 @@ -import { Model, ModelCard } from "../index.js"; -import { model } from "../implementations/dogClassifierModel.js"; +import { Model } from "#models/model"; +import { ModelCard } from "#models/model_card"; +import { model } from "#models/implementations/dogClassifierModel"; export const DogClassifier: ModelCard<"image"> = { card: { diff --git a/discojs/src/models/cards/titanicClassifier.ts b/discojs/src/models/cards/titanicClassifier.ts index a4b33e209..15fae9af5 100644 --- a/discojs/src/models/cards/titanicClassifier.ts +++ b/discojs/src/models/cards/titanicClassifier.ts @@ -1,5 +1,6 @@ -import { Model, ModelCard } from "../index.js"; -import { model } from "../implementations/titanicClassifierModel.js"; +import { Model } from "#models/model"; +import { ModelCard } from "#models/model_card"; +import { model } from "#models/implementations/titanicClassifierModel"; export const TitanicClassifier: ModelCard<"tabular"> = { card: { diff --git a/discojs/src/models/cards/wikitext.ts b/discojs/src/models/cards/wikitext.ts index c452693dd..bb2de098a 100644 --- a/discojs/src/models/cards/wikitext.ts +++ b/discojs/src/models/cards/wikitext.ts @@ -1,5 +1,6 @@ -import { Model, ModelCard } from "../index.js"; -import { GPT } from "../index.js"; +import { Model } from "#models/model"; +import { ModelCard } from "#models/model_card"; +import { GPT } from "#models/implementations/index"; export const Wikitext: ModelCard<"text"> = { card: { diff --git a/discojs/src/models/implementations/CIFAR10ClassifierModel.ts b/discojs/src/models/implementations/CIFAR10ClassifierModel.ts index dfb5dd1a8..c2ce700ca 100644 --- a/discojs/src/models/implementations/CIFAR10ClassifierModel.ts +++ b/discojs/src/models/implementations/CIFAR10ClassifierModel.ts @@ -1,6 +1,6 @@ import * as tf from "@tensorflow/tfjs"; -import { TFJS } from "../index.js"; +import { TFJS } from "#models/tfjs"; import baseModel from "./mobileNet_v1_025_224.js"; diff --git a/discojs/src/models/implementations/MNISTClassifierModel.ts b/discojs/src/models/implementations/MNISTClassifierModel.ts index 582874f8f..27104f60e 100644 --- a/discojs/src/models/implementations/MNISTClassifierModel.ts +++ b/discojs/src/models/implementations/MNISTClassifierModel.ts @@ -1,6 +1,6 @@ import * as tf from "@tensorflow/tfjs"; -import { TFJS } from "../index.js"; +import { TFJS } from "#models/tfjs"; export function model() { // Architecture from the PyTorch MNIST example (I made it slightly smaller, 650kB instead of 5MB) diff --git a/discojs/src/models/implementations/dogClassifierModel.ts b/discojs/src/models/implementations/dogClassifierModel.ts index ff66091d9..ce790b4de 100644 --- a/discojs/src/models/implementations/dogClassifierModel.ts +++ b/discojs/src/models/implementations/dogClassifierModel.ts @@ -1,6 +1,6 @@ import * as tf from "@tensorflow/tfjs"; -import { TFJS } from "../index.js"; +import { TFJS } from "#models/tfjs"; export function model() { const seed = 42; // set a seed to ensure reproducibility during GDHF demo diff --git a/discojs/src/models/implementations/gpt/index.ts b/discojs/src/models/implementations/gpt/index.ts index 25de47258..8b9791ae2 100644 --- a/discojs/src/models/implementations/gpt/index.ts +++ b/discojs/src/models/implementations/gpt/index.ts @@ -9,9 +9,10 @@ import * as tf from "@tensorflow/tfjs"; import { WeightsContainer } from "#weights/index"; import { Dataset, Batched } from "#dataset/index"; -import { Model, BatchLogs, EpochLogs } from "#models/index"; import type { DataFormat } from "#types/index"; +import { BatchLogs, EpochLogs } from "#models/logs"; +import { Model } from "#models/model"; import { GPTModel } from "./model.js"; import evaluate from "./evaluate.js"; import { DefaultGPTConfig, DefaultGenerationConfig } from "./config.js"; diff --git a/discojs/src/models/implementations/hellaswag.ts b/discojs/src/models/implementations/hellaswag.ts index b4c643fa3..4e5aa92c3 100644 --- a/discojs/src/models/implementations/hellaswag.ts +++ b/discojs/src/models/implementations/hellaswag.ts @@ -1,7 +1,8 @@ import * as tf from "@tensorflow/tfjs"; import { List } from "immutable"; -import type { Tokenizer, ONNXModel } from "#models/index"; -import { GPT } from "#models/index"; +import type { Tokenizer } from "#models/tokenizer"; +import type { ONNXModel } from "#models/onnx"; +import { GPT } from "#models/implementations/gpt/index"; export const HELLASWAG_URL = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl"; diff --git a/discojs/src/models/implementations/index.ts b/discojs/src/models/implementations/index.ts new file mode 100644 index 000000000..982195083 --- /dev/null +++ b/discojs/src/models/implementations/index.ts @@ -0,0 +1,4 @@ +export { GPT } from "./gpt/index.js"; +export type { GPTConfig } from "./gpt/config.js"; +export type { HellaSwagDataset, HellaSwagExample } from "./hellaswag.js"; +export { evaluate as evaluate_hellaswag, HELLASWAG_URL } from "./hellaswag.js"; diff --git a/discojs/src/models/implementations/titanicClassifierModel.ts b/discojs/src/models/implementations/titanicClassifierModel.ts index 9013f1565..4c637a8ef 100644 --- a/discojs/src/models/implementations/titanicClassifierModel.ts +++ b/discojs/src/models/implementations/titanicClassifierModel.ts @@ -1,6 +1,6 @@ import * as tf from "@tensorflow/tfjs"; -import { TFJS } from "../index.js"; +import { TFJS } from "#models/tfjs"; export function model() { const model = tf.sequential(); diff --git a/discojs/src/models/index.ts b/discojs/src/models/index.ts index b74b9dc6e..04b15fbcd 100644 --- a/discojs/src/models/index.ts +++ b/discojs/src/models/index.ts @@ -3,17 +3,17 @@ export type { BatchLogs, ValidationMetrics } from "./logs.js"; export { EpochLogs } from "./logs.js"; export { Tokenizer } from "./tokenizer.js"; -export { GPT } from "./implementations/gpt/index.js"; -export { ONNXModel } from "./onnx.js"; -export type { GPTConfig } from "./implementations/gpt/config.js"; export type { + GPTConfig, HellaSwagDataset, HellaSwagExample, -} from "./implementations/hellaswag.js"; +} from "./implementations/index.js"; export { - evaluate as evaluate_hellaswag, + GPT, + evaluate_hellaswag, HELLASWAG_URL, -} from "./implementations/hellaswag.js"; +} from "./implementations/index.js"; +export { ONNXModel } from "./onnx.js"; export { TFJS } from "./tfjs.js"; export type { ModelCard } from "./model_card.js"; export { ModelCardInfo } from "./model_card.js"; diff --git a/discojs/src/models/model_card.ts b/discojs/src/models/model_card.ts index 7eeffe3b0..6bd3f975a 100644 --- a/discojs/src/models/model_card.ts +++ b/discojs/src/models/model_card.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import type { Model } from "#models/index"; +import type { Model } from "#models/model"; import { DataType, dataTypeValues } from "#types/index"; export namespace ModelCardInfo { diff --git a/discojs/src/privacy.ts b/discojs/src/privacy.ts index c91cb0553..9829b694c 100644 --- a/discojs/src/privacy.ts +++ b/discojs/src/privacy.ts @@ -1,8 +1,8 @@ import * as tf from "@tensorflow/tfjs"; -import { WeightsContainer } from "./index.js"; +import { WeightsContainer } from "#weights/index"; -import type { WeightNormHistory } from "./training/trainer.js"; +import type { WeightNormHistory } from "#training/types"; /** Computes the Frobenius norm of the given weights. */ export async function frobeniusNorm(weights: tf.Tensor): Promise { diff --git a/discojs/src/processing/image.spec.ts b/discojs/src/processing/image.spec.ts index ebe9af8ce..c8563567c 100644 --- a/discojs/src/processing/image.spec.ts +++ b/discojs/src/processing/image.spec.ts @@ -1,6 +1,6 @@ import { Repeat, Seq } from "immutable"; import { describe, expect, it } from "vitest"; -import { Image } from "../index.js"; +import { Image } from "#dataset/index"; import { removeAlpha, resize } from "./image.js"; describe("resize", () => { diff --git a/discojs/src/processing/image.ts b/discojs/src/processing/image.ts index 4a78079d1..d1dcb07bd 100644 --- a/discojs/src/processing/image.ts +++ b/discojs/src/processing/image.ts @@ -2,7 +2,7 @@ import { Repeat, Seq } from "immutable"; import { createJimp } from "@jimp/core"; import * as jimpResize from "@jimp/plugin-resize"; -import { Image } from "../index.js"; +import { Image } from "#dataset/index"; /** Image where intensity is represented in the range 0..1 */ export class NormalizedImage< diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts index 751017ee4..3fed7348c 100644 --- a/discojs/src/processing/index.ts +++ b/discojs/src/processing/index.ts @@ -1,136 +1,4 @@ /** Dataset shapers, convenient to map with */ - -import { List } from "immutable"; - -import type { - Dataset, - DataFormat, - DataType, - Tabular, - Task, - Network, -} from "../index.js"; - -import * as processing from "./index.js"; - export * from "./image.js"; export * from "./tabular.js"; - -export function preprocess( - task: Task, - dataset: Dataset, -): Dataset { - switch (task.dataType) { - case "image": { - // cast as typescript doesn't reduce generic type - const d = dataset as Dataset; - const { IMAGE_H, IMAGE_W, LABEL_LIST } = task.trainingInformation; - - return d.map(([image, label]) => [ - processing.normalize( - processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), - ), - processing.indexInList(label, LABEL_LIST), - ]) as Dataset; - } - case "tabular": { - // cast as typescript doesn't reduce generic type - const d = dataset as Dataset; - const { inputColumns, outputColumn } = task.trainingInformation; - - return d.map((row) => { - const output = processing.extractColumn(row, outputColumn); - - return [ - extractToNumbers(inputColumns, row), - // TODO sanitization doesn't care about column distribution - output !== "" ? processing.convertToNumber(output) : 0, - ]; - }) as Dataset; - } - case "text": { - // cast as typescript doesn't reduce generic type - const d = dataset as Dataset; - - const { contextLength, tokenizer } = task.trainingInformation; - - return d - .map((text) => tokenizer.tokenize(text)) - .flatten() - .batch(contextLength + 1, 1) - .map((tokens) => [tokens.pop(), tokens.last()]) as Dataset< - DataFormat.ModelEncoded[D] - >; - } - } -} - -export function preprocessWithoutLabel( - task: Task, - dataset: Dataset, -): Dataset { - switch (task.dataType) { - case "image": { - // cast as typescript doesn't reduce generic type - const d = dataset as Dataset; - const { IMAGE_H, IMAGE_W } = task.trainingInformation; - - return d.map((image) => - processing.normalize( - processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), - ), - ); - } - case "tabular": { - // cast as typescript doesn't reduce generic type - const d = dataset as Dataset; - const { inputColumns } = task.trainingInformation; - - return d.map((row) => extractToNumbers(inputColumns, row)); - } - case "text": { - // cast as typescript doesn't reduce generic type - const d = dataset as Dataset; - - const { contextLength, tokenizer } = task.trainingInformation; - - return d - .map((text) => tokenizer.tokenize(text)) - .flatten() - .batch(contextLength); - } - } -} - -export function postprocess( - task: Task, - encoded: DataFormat.ModelEncoded[D][1], -): DataFormat.Inferred[D] { - switch (task.dataType) { - case "image": { - const labels = List(task.trainingInformation.LABEL_LIST); - - const v = labels.get(encoded); - if (v === undefined) throw new Error("index not found in labels"); - return v as DataFormat.Inferred[D]; - } - case "tabular": { - return encoded as DataFormat.Inferred[D]; - } - case "text": { - return task.trainingInformation.tokenizer.decode([ - encoded, - ]) as DataFormat.Inferred[D]; - } - } -} - -function extractToNumbers(columns: Iterable, row: Tabular) { - return ( - List(columns) - .map((column) => processing.extractColumn(row, column)) - // TODO sanitization doesn't care about column distribution - .map((v) => (v !== "" ? v : "0")) - .map(processing.convertToNumber) - ); -} +export * from "./processing.js"; diff --git a/discojs/src/processing/index.spec.ts b/discojs/src/processing/processing.spec.ts similarity index 87% rename from discojs/src/processing/index.spec.ts rename to discojs/src/processing/processing.spec.ts index 19aa2b47a..bf995b99b 100644 --- a/discojs/src/processing/index.spec.ts +++ b/discojs/src/processing/processing.spec.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from "vitest"; -import type { Task } from "../index.js"; -import { Dataset } from "../index.js"; +import { preprocess } from "#processing/processing"; -import { preprocess } from "./index.js"; +import type { Task } from "#task/index"; +import { Dataset } from "#dataset/index"; describe("preprocess", () => { it("throws on missing column in tabular", async () => { diff --git a/discojs/src/processing/processing.ts b/discojs/src/processing/processing.ts new file mode 100644 index 000000000..68e24a2f5 --- /dev/null +++ b/discojs/src/processing/processing.ts @@ -0,0 +1,123 @@ +import { List } from "immutable"; + +import type { Task } from "#task/task"; +import type { Dataset, Tabular } from "#dataset/index"; +import type { DataType, DataFormat, Network } from "#types/index"; + +import { normalize, removeAlpha, resize } from "./image.js"; +import { indexInList, extractColumn, convertToNumber } from "./tabular.js"; + +export function preprocess( + task: Task, + dataset: Dataset, +): Dataset { + switch (task.dataType) { + case "image": { + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { IMAGE_H, IMAGE_W, LABEL_LIST } = task.trainingInformation; + + return d.map(([image, label]) => [ + normalize(removeAlpha(resize(IMAGE_W, IMAGE_H, image))), + indexInList(label, LABEL_LIST), + ]) as Dataset; + } + case "tabular": { + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { inputColumns, outputColumn } = task.trainingInformation; + + return d.map((row) => { + const output = extractColumn(row, outputColumn); + + return [ + extractToNumbers(inputColumns, row), + // TODO sanitization doesn't care about column distribution + output !== "" ? convertToNumber(output) : 0, + ]; + }) as Dataset; + } + case "text": { + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + + const { contextLength, tokenizer } = task.trainingInformation; + + return d + .map((text) => tokenizer.tokenize(text)) + .flatten() + .batch(contextLength + 1, 1) + .map((tokens) => [tokens.pop(), tokens.last()]) as Dataset< + DataFormat.ModelEncoded[D] + >; + } + } +} + +export function preprocessWithoutLabel( + task: Task, + dataset: Dataset, +): Dataset { + switch (task.dataType) { + case "image": { + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { IMAGE_H, IMAGE_W } = task.trainingInformation; + + return d.map((image) => + normalize(removeAlpha(resize(IMAGE_W, IMAGE_H, image))), + ); + } + case "tabular": { + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { inputColumns } = task.trainingInformation; + + return d.map((row) => extractToNumbers(inputColumns, row)); + } + case "text": { + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + + const { contextLength, tokenizer } = task.trainingInformation; + + return d + .map((text) => tokenizer.tokenize(text)) + .flatten() + .batch(contextLength); + } + } +} + +export function postprocess( + task: Task, + encoded: DataFormat.ModelEncoded[D][1], +): DataFormat.Inferred[D] { + switch (task.dataType) { + case "image": { + const labels = List(task.trainingInformation.LABEL_LIST); + + const v = labels.get(encoded); + if (v === undefined) throw new Error("index not found in labels"); + return v as DataFormat.Inferred[D]; + } + case "tabular": { + return encoded as DataFormat.Inferred[D]; + } + case "text": { + return task.trainingInformation.tokenizer.decode([ + encoded, + ]) as DataFormat.Inferred[D]; + } + } +} + +function extractToNumbers(columns: Iterable, row: Tabular) { + return ( + List(columns) + .map((column) => extractColumn(row, column)) + // TODO sanitization doesn't care about column distribution + .map((v) => (v !== "" ? v : "0")) + .map(convertToNumber) + ); +} diff --git a/discojs/src/serialization/index.ts b/discojs/src/serialization/index.ts index 1dccea6e6..c59aaf689 100644 --- a/discojs/src/serialization/index.ts +++ b/discojs/src/serialization/index.ts @@ -5,11 +5,4 @@ export * as weights from "./weights.js"; export type { Encoded } from "./coder.js"; export { isEncoded } from "./coder.js"; -export type JSON = - | null - | undefined - | boolean - | number - | string - | JSON[] - | { [_: string]: JSON }; +export { JSONLike } from "./json_like.js"; diff --git a/discojs/src/serialization/json_like.ts b/discojs/src/serialization/json_like.ts new file mode 100644 index 000000000..30505b626 --- /dev/null +++ b/discojs/src/serialization/json_like.ts @@ -0,0 +1,8 @@ +export type JSONLike = + | null + | undefined + | boolean + | number + | string + | JSONLike[] + | { [_: string]: JSONLike }; diff --git a/discojs/src/serialization/model.ts b/discojs/src/serialization/model.ts index 5552bfca1..7a004d724 100644 --- a/discojs/src/serialization/model.ts +++ b/discojs/src/serialization/model.ts @@ -1,8 +1,9 @@ import type tf from "@tensorflow/tfjs"; -import type { DataType, Model } from "../index.js"; -import { models, serialization } from "../index.js"; -import { GPTConfig } from "../models/index.js"; +import { encode as w_encode, decode as w_decode } from "#serialization/weights"; +import { GPTConfig, GPT, TFJS } from "#models/index"; +import type { Model } from "#models/index"; +import type { DataType } from "#types/index"; import * as coder from "./coder.js"; import { Encoded, isEncoded } from "./coder.js"; @@ -14,13 +15,13 @@ const Type = { export async function encode(model: Model): Promise { switch (true) { - case model instanceof models.TFJS: { + case model instanceof TFJS: { const serialized = await model.serialize(); return coder.encode([Type.TFJS, ...serialized]); } - case model instanceof models.GPT: { + case model instanceof GPT: { const { weights, config } = model.serialize(); - const serializedWeights = await serialization.weights.encode(weights); + const serializedWeights = await w_encode(weights); return coder.encode([Type.GPT, serializedWeights, config]); } default: @@ -61,7 +62,7 @@ export async function decode(encoded: Encoded): Promise> { throw new Error("invalid TFJS model encoding: invalid DataType"); } - return await models.TFJS.deserialize([ + return await TFJS.deserialize([ datatype, // TODO totally unsafe casting rawModel as tf.io.ModelArtifacts, @@ -83,8 +84,8 @@ export async function decode(encoded: Encoded): Promise> { throw new Error( "invalid encoding, gpt-tfjs model weights should be an encoding of its weights", ); - const weights = serialization.weights.decode(rawModel); - return models.GPT.deserialize({ weights, config }); + const weights = w_decode(rawModel); + return GPT.deserialize({ weights, config }); } default: throw new Error("invalid encoding, model type unrecognized"); diff --git a/discojs/src/serialization/task.ts b/discojs/src/serialization/task.ts index ea1a36f47..935c6d79c 100644 --- a/discojs/src/serialization/task.ts +++ b/discojs/src/serialization/task.ts @@ -1,10 +1,11 @@ import { z } from "zod"; -import type { DataType, Network } from "../index.js"; -import { Task, Tokenizer } from "../index.js"; +import type { DataType, Network } from "#types/index"; +import { Task } from "#task/index"; +import { Tokenizer } from "#models/index"; -import type { JSON } from "./index.js"; +import type { JSONLike } from "./json_like.js"; -export function serializeToJSON(task: Task): JSON { +export function serializeToJSON(task: Task): JSONLike { switch (task.dataType) { case "image": case "tabular": @@ -23,7 +24,7 @@ export function serializeToJSON(task: Task): JSON { // Throws if an error serialized object is malformed export async function deserializeFromJSON( - serialized: JSON, + serialized: JSONLike, ): Promise> { return await z .looseObject({ diff --git a/discojs/src/serialization/weights.ts b/discojs/src/serialization/weights.ts index 52af2c8a4..5a40257ef 100644 --- a/discojs/src/serialization/weights.ts +++ b/discojs/src/serialization/weights.ts @@ -1,6 +1,6 @@ import * as tf from "@tensorflow/tfjs"; -import { WeightsContainer } from "../index.js"; +import { WeightsContainer } from "#weights/index"; import { Encoded } from "./coder.js"; import * as coder from "./coder.js"; diff --git a/discojs/src/task/display_information.ts b/discojs/src/task/display_information.ts index b03b91da2..c87c70a26 100644 --- a/discojs/src/task/display_information.ts +++ b/discojs/src/task/display_information.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import type { DataType } from "../types/index.js"; +import type { DataType } from "#types/index"; export namespace DisplayInformation { export const baseSchema = z.object({ diff --git a/discojs/src/task/task.ts b/discojs/src/task/task.ts index 2903239cc..4d63b327c 100644 --- a/discojs/src/task/task.ts +++ b/discojs/src/task/task.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import type { DataType, Network } from "../index.js"; +import type { DataType, Network } from "#types/index"; import { DisplayInformation } from "./display_information.js"; import { TrainingInformation } from "./training_information.js"; diff --git a/discojs/src/task/task_handler.ts b/discojs/src/task/task_handler.ts index 1fb6273ce..2c8cc8b04 100644 --- a/discojs/src/task/task_handler.ts +++ b/discojs/src/task/task_handler.ts @@ -1,8 +1,8 @@ import { Map, Seq } from "immutable"; -import type { DataType, Model, Network } from "../index.js"; -import { serialization } from "../index.js"; -import type { ModelCardInfo } from "#models/model_card"; +import type { DataType, Network } from "#types/index"; +import type { Model } from "#models/index"; +import * as serialization from "#serialization/index"; import type { Task } from "./task.js"; @@ -42,7 +42,7 @@ export async function fetchTasks( ): Promise>> { const response = await fetch(urlToTasks(base)); if (!response.ok) throw new Error(`fetch: HTTP status ${response.status}`); - const json = (await response.json()) as serialization.JSON; + const json = (await response.json()) as serialization.JSONLike; if (!Array.isArray(json)) throw new Error("invalid tasks response: expected a JSON array"); diff --git a/discojs/src/task/task_provider.ts b/discojs/src/task/task_provider.ts index 666856f85..4609a9746 100644 --- a/discojs/src/task/task_provider.ts +++ b/discojs/src/task/task_provider.ts @@ -1,4 +1,6 @@ -import type { DataType, Network, Task, ModelCard } from "../index.js"; +import type { DataType, Network } from "#types/index"; +import type { ModelCard } from "#models/index"; +import type { Task } from "#task/task"; export interface TaskProvider { getTask(): Promise>; diff --git a/discojs/src/task/training_information.ts b/discojs/src/task/training_information.ts index a8f041ebb..be7e4bd01 100644 --- a/discojs/src/task/training_information.ts +++ b/discojs/src/task/training_information.ts @@ -1,7 +1,7 @@ import { z } from "zod"; -import type { DataType, Network } from "../index.js"; -import { Tokenizer } from "../index.js"; +import type { DataType, Network } from "#types/index"; +import { Tokenizer } from "#models/index"; const privacySchema = z.object({ // reduce training accuracy and improve privacy. diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index 4a124f9bf..f0686394e 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -1,25 +1,19 @@ -import { - async_iterator, - client as clients, - BatchLogs, - ConsoleLogger, - EpochLogs, - Logger, - processing, - Dataset, -} from "../index.js"; -import type { - Batched, - DataFormat, - DataType, - Model, - Network, - Task, -} from "../index.js"; -import type { Aggregator } from "../aggregator/index.js"; -import { getAggregator } from "../aggregator/index.js"; -import { enumerate, split } from "../utils/async_iterator.js"; -import { EventEmitter } from "../utils/event_emitter.js"; +import type { Model } from "#models/index"; +import type { DataType, DataFormat, Network } from "#types/index"; +import type { Task } from "#task/index"; +import type { Batched } from "#dataset/index"; +import type { Aggregator } from "#aggregator/index"; + +import { Dataset } from "#dataset/index"; +import { ConsoleLogger, Logger } from "#logging/index"; +import { BatchLogs, EpochLogs } from "#models/index"; +import { getAggregator } from "#aggregator/index"; +import { enumerate, split } from "#utils/async_iterator"; +import { EventEmitter } from "#utils/event_emitter"; + +import * as clients from "#client/index"; +import * as processing from "#processing/index"; +import * as async_iterator from "#utils/async_iterator"; import { RoundLogs, Trainer } from "./trainer.js"; diff --git a/discojs/src/training/trainer.ts b/discojs/src/training/trainer.ts index 44c8f8cd3..56bc3167d 100644 --- a/discojs/src/training/trainer.ts +++ b/discojs/src/training/trainer.ts @@ -1,22 +1,20 @@ import * as tf from "@tensorflow/tfjs"; import { List, Repeat } from "immutable"; -import { - Batched, - BatchLogs, - Dataset, - DataFormat, - DataType, - EpochLogs, - Model, - Task, - WeightsContainer, - Network, - ValidationMetrics, -} from "../index.js"; -import { privacy } from "../index.js"; -import { Client } from "../client/index.js"; -import * as async_iterator from "../utils/async_iterator.js"; +import type { Model } from "#models/index"; +import type { DataFormat, DataType, Network } from "#types/index"; +import type { Batched } from "#dataset/index"; +import type { Task } from "#task/index"; + +import { Dataset } from "#dataset/index"; +import { BatchLogs, EpochLogs, ValidationMetrics } from "#models/index"; +import { WeightsContainer } from "#weights/index"; +import { Client } from "#client/index"; + +import * as async_iterator from "#utils/async_iterator"; +import * as privacy from "#root/privacy"; + +import type { WeightNormHistory } from "#training/types"; export interface RoundLogs { epochs: List; @@ -25,8 +23,6 @@ export interface RoundLogs { } /** List of weight update norms */ -export type WeightNormHistory = List>; - function appendWeightHistory( weightNormHistory: WeightNormHistory, wc: number[], diff --git a/discojs/src/training/types.ts b/discojs/src/training/types.ts new file mode 100644 index 000000000..e4ab2568a --- /dev/null +++ b/discojs/src/training/types.ts @@ -0,0 +1,3 @@ +import { List } from "immutable"; + +export type WeightNormHistory = List>; diff --git a/discojs/src/types/data_format.ts b/discojs/src/types/data_format.ts index 7047d4be8..4927b4e73 100644 --- a/discojs/src/types/data_format.ts +++ b/discojs/src/types/data_format.ts @@ -1,12 +1,7 @@ import { List } from "immutable"; -import type { - Image, - processing, - Tabular, - Text, - TokenizedText, -} from "../index.js"; +import type { Image, Tabular, Text, TokenizedText } from "#dataset/index"; +import { NormalizedImage } from "#processing/image"; /** * The data & label format goes through various stages. @@ -37,7 +32,7 @@ type Token = number; * prediction needs data input and outputs label **/ export interface ModelEncoded { - image: [image: processing.NormalizedImage<3>, label: number]; + image: [image: NormalizedImage<3>, label: number]; tabular: [row: List, number]; text: [line: TokenizedText, next: Token]; } diff --git a/discojs/src/validator.ts b/discojs/src/validator.ts index a1bd01c39..ac493e960 100644 --- a/discojs/src/validator.ts +++ b/discojs/src/validator.ts @@ -1,12 +1,12 @@ -import type { - Dataset, - DataFormat, - DataType, - Model, - Task, - Network, -} from "./index.js"; -import { processing } from "./index.js"; +import type { DataFormat, DataType, Network } from "#types/index"; +import type { Task } from "#task/index"; +import type { Dataset } from "#dataset/index"; +import type { Model } from "#models/index"; +import { + preprocess, + preprocessWithoutLabel, + postprocess, +} from "#processing/index"; export class Validator { readonly #model: Model; @@ -22,7 +22,7 @@ export class Validator { test( dataset: Dataset, ): Dataset> { - const preprocessed = processing.preprocess(this.task, dataset); + const preprocessed = preprocess(this.task, dataset); const batched = preprocessed.batch(this.task.trainingInformation.batchSize); const predictionWithTruth = batched @@ -34,8 +34,8 @@ export class Validator { .flatten(); return predictionWithTruth.map(([predicted, truth]) => ({ - predicted: processing.postprocess(this.task, predicted), - truth: processing.postprocess(this.task, truth), + predicted: postprocess(this.task, predicted), + truth: postprocess(this.task, truth), })); } @@ -43,14 +43,13 @@ export class Validator { async *infer( dataset: Dataset, ): AsyncGenerator { - const modelPredictions = processing - .preprocessWithoutLabel(this.task, dataset) + const modelPredictions = preprocessWithoutLabel(this.task, dataset) .batch(this.task.trainingInformation.batchSize) .map((batch) => this.#model.predict(batch)) .flatten(); const predictions = modelPredictions.map((prediction) => - processing.postprocess(this.task, prediction), + postprocess(this.task, prediction), ); for await (const e of predictions) yield e; diff --git a/discojs/tsconfig.lib.json b/discojs/tsconfig.lib.json index 90ec2cefa..5fe8c7a23 100644 --- a/discojs/tsconfig.lib.json +++ b/discojs/tsconfig.lib.json @@ -1,6 +1,22 @@ { "extends": "../tsconfig.base.lib.json", "compilerOptions": { + // Copied from `package.json` since some tools don't work without it + "paths": { + "#root/*": ["./src/*"], + "#aggregator/*": ["./src/aggregator/*"], + "#client/*": ["./src/client/*"], + "#dataset/*": ["./src/dataset/*"], + "#types/*": ["./src/types/*"], + "#logging/*": ["./src/logging/*"], + "#models/*": ["./src/models/*"], + "#processing/*": ["./src/processing/*"], + "#serialization/*": ["./src/serialization/*"], + "#task/*": ["./src/task/*"], + "#training/*": ["./src/training/*"], + "#utils/*": ["./src/utils/*"], + "#weights/*": ["./src/weights/*"] + }, "rootDir": "./src", "outDir": "dist", "composite": true diff --git a/package.json b/package.json index 2c0fdc52c..407fd72e2 100644 --- a/package.json +++ b/package.json @@ -15,10 +15,12 @@ "@vitest/eslint-plugin": "1.6.12", "@vue/eslint-config-prettier": "10.2.0", "@vue/eslint-config-typescript": "14.9.0", + "dpdm": "^4.3.0", "eslint": "10.0.3", "eslint-plugin-cypress": "6.2.0", "eslint-plugin-vue": "10.9.2", "knip": "6.31.0", + "madge": "8.0.0", "prettier": "3.6.2", "supports-color": "8.1.1", "typescript": "catalog:", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 363d1a0b5..ce47c4667 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -250,6 +250,9 @@ importers: '@vue/eslint-config-typescript': specifier: 14.9.0 version: 14.9.0(eslint-plugin-vue@10.9.2(@typescript-eslint/parser@8.62.1(eslint@10.0.3(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3))(eslint@10.0.3(jiti@2.7.0)(supports-color@8.1.1))(vue-eslint-parser@10.4.1(eslint@10.0.3(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)))(eslint@10.0.3(jiti@2.7.0)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@6.0.3) + dpdm: + specifier: ^4.3.0 + version: 4.3.0 eslint: specifier: 10.0.3 version: 10.0.3(jiti@2.7.0)(supports-color@8.1.1) @@ -262,6 +265,9 @@ importers: knip: specifier: 6.31.0 version: 6.31.0 + madge: + specifier: 8.0.0 + version: 8.0.0(supports-color@8.1.1)(typescript@6.0.3) prettier: specifier: 3.6.2 version: 3.6.2 @@ -750,6 +756,14 @@ packages: '@cypress/xvfb@1.2.4': resolution: {integrity: sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==} + '@dependents/detective-less@5.0.3': + resolution: {integrity: sha512-v6oD9Ukp+N7V4n6p5I/+mM5fIohSfkrDSGlFm5w/pYmchvbk+sMIHsLxrFJ5Lnujewj1BzWL0K84d88lwZAMQA==} + engines: {node: '>=18'} + + '@discoveryjs/json-ext@1.1.0': + resolution: {integrity: sha512-Xc3VhU02wqZ1HvHRJUwL09HkZSTvidqY5Ya0NXBSYOxAp+Ln9dcJr9fySI+CkONzP3PekQo9WdzCv0PGER/mOA==} + engines: {node: '>=14.17.0'} + '@emnapi/core@1.10.0': resolution: {integrity: sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==} @@ -1643,6 +1657,22 @@ packages: '@tokenizer/token@0.3.0': resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} + '@ts-graphviz/adapter@2.0.6': + resolution: {integrity: sha512-kJ10lIMSWMJkLkkCG5gt927SnGZcBuG0s0HHswGzcHTgvtUe7yk5/3zTEr0bafzsodsOq5Gi6FhQeV775nC35Q==} + engines: {node: '>=18'} + + '@ts-graphviz/ast@2.0.7': + resolution: {integrity: sha512-e6+2qtNV99UT6DJSoLbHfkzfyqY84aIuoV8Xlb9+hZAjgpum8iVHprGeAMQ4rF6sKUAxrmY8rfF/vgAwoPc3gw==} + engines: {node: '>=18'} + + '@ts-graphviz/common@2.1.5': + resolution: {integrity: sha512-S6/9+T6x8j6cr/gNhp+U2olwo1n0jKj/682QVqsh7yXWV6ednHYqxFw0ZsY3LyzT0N8jaZ6jQY9YD99le3cmvg==} + engines: {node: '>=18'} + + '@ts-graphviz/core@2.0.7': + resolution: {integrity: sha512-w071DSzP94YfN6XiWhOxnLpYT3uqtxJBDYdh6Jdjzt+Ce6DNspJsPQgpC7rbts/B8tEkq0LHoYuIF/O5Jh5rPg==} + engines: {node: '>=18'} + '@tsconfig/node20@20.1.9': resolution: {integrity: sha512-IjlTv1RsvnPtUcjTqtVsZExKVq+KQx4g5pCP5tI7rAs6Xesl2qFwSz/tPDBC4JajkL/MlezBu3gPUwqRHl+RIg==} @@ -2136,10 +2166,16 @@ packages: resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} engines: {node: '>=12'} + any-promise@1.3.0: + resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + anymatch@3.1.3: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + app-module-path@2.2.0: + resolution: {integrity: sha512-gkco+qxENJV+8vFcDiiFhuoSvRXb2a/QPqpSoWhVz829VNJfOTnELbBmPmNKFxf3xdNnw4DWCkzkDaavcX/1YQ==} + aproba@2.1.0: resolution: {integrity: sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==} @@ -2180,6 +2216,10 @@ packages: resolution: {integrity: sha512-m1Q/RaVOnTp9JxPX+F+Zn7IcLYMzM8kZofDImfsKZd8MbR+ikdOzTeztStWqfrqIxZnYWryyI9ePm3NGjnZgGw==} engines: {node: '>=20.19.0'} + ast-module-types@6.0.2: + resolution: {integrity: sha512-6KuK/7nZ/2Qh7sGuVEiwxjCxzTY2Pdb5mTo5z1e6/J8BA0tvjR7G8vQJKrQMTqwmnA3UPEyKIFX4YUS1DO1Hvw==} + engines: {node: '>=18'} + ast-walker-scope@0.9.0: resolution: {integrity: sha512-IJdzo2vLiElBxKzwS36VsCue/62d6IdWjnPB2v3nuPKeWGynp6FF/CYoLa5i/3jXH/z97ZDdsXz6abpgM6w07A==} engines: {node: '>=20.19.0'} @@ -2347,6 +2387,10 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + chart.js@4.5.1: resolution: {integrity: sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==} engines: {pnpm: '>=8'} @@ -2374,10 +2418,22 @@ packages: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} + cli-cursor@3.1.0: + resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} + engines: {node: '>=8'} + cli-cursor@5.0.0: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} + cli-spinners@2.9.2: + resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} + engines: {node: '>=6'} + + cli-spinners@3.4.0: + resolution: {integrity: sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==} + engines: {node: '>=18.20'} + cli-table3@0.6.1: resolution: {integrity: sha512-w0q/enDHhPLq44ovMGdQeeDLvwxwavsJX7oQGYt/LrBlYsyaxyDnp6z3QzFut/6kLLKnlcUVJLrpB7KBfgG/RA==} engines: {node: 10.* || >= 12.*} @@ -2389,6 +2445,14 @@ packages: cliui@7.0.4: resolution: {integrity: sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + + clone@1.0.4: + resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} + engines: {node: '>=0.8'} + color-convert@1.9.3: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} @@ -2439,6 +2503,10 @@ packages: resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} engines: {node: '>=14'} + commander@12.1.0: + resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} + engines: {node: '>=18'} + commander@6.2.1: resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==} engines: {node: '>= 6'} @@ -2451,6 +2519,9 @@ packages: resolution: {integrity: sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==} engines: {node: '>=4.0.0'} + commondir@1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -2696,6 +2767,9 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + defaults@1.0.4: + resolution: {integrity: sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==} + delaunator@5.1.0: resolution: {integrity: sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==} @@ -2710,15 +2784,67 @@ packages: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} + dependency-tree@11.5.0: + resolution: {integrity: sha512-K9zBwKDZrot3RkxizugpVSdImxULAg4Ycp3+ydy2r561k96oiiw6nfsOR15fwNDQ5BF2UXe+2JFM/H5Xz4MGQg==} + engines: {node: '>=18'} + hasBin: true + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + detective-amd@6.1.0: + resolution: {integrity: sha512-fmI6LGMvotqd49QaA3ZYw+q0aGp2yXmMjzIuY6fH9j9YFIXY/73yDhMwhX9cPbhWd+AH06NH1Di/LKOuCH0Ubg==} + engines: {node: '>=18'} + hasBin: true + + detective-cjs@6.1.1: + resolution: {integrity: sha512-pSh7mkCKEtLlmANqLu3KDFS3NV8Hx41jy/JF1/gAWOgU+Uo5QTkeI1tWNP4dWGo4L0E9j18Ez9EPsTleautKqA==} + engines: {node: '>=18'} + + detective-es6@5.0.2: + resolution: {integrity: sha512-+qHHGYhjupiVs4rnIpI9nZ5B130A4AmE35ZX1w33hb46vcZ7T3jfDbvmPw0FhWtMHn5BS5HHu7ZtnZ53bMcXZA==} + engines: {node: '>=18'} + + detective-postcss@8.0.4: + resolution: {integrity: sha512-DZ7M/hWPZyr17ZUdoQ+TVXaPj70mYr4XXrAE+GeJbca44haCvZgb191L/jLJmFYewhxRJuBd4lUtNSu986TXag==} + engines: {node: '>=18'} + peerDependencies: + postcss: ^8.4.47 + + detective-sass@6.0.2: + resolution: {integrity: sha512-i3xpXHDKS0qI2aFW4asQ7fqlPK00ndOVZELvQapFJCaF0VxYmsNWtd0AmvXbTLMk7bfO5VdIeorhY9KfmHVoVA==} + engines: {node: '>=18'} + + detective-scss@5.0.2: + resolution: {integrity: sha512-9JOEMZ8pDh3ShXmftq7hoQqqJsClaGgxo1hghfCeFlmKf5TC/Twtwb0PAaK8dXwpg9Z0uCmEYSrCxO+kel2eEg==} + engines: {node: '>=18'} + + detective-stylus@5.0.1: + resolution: {integrity: sha512-Dgn0bUqdGbE3oZJ+WCKf8Dmu7VWLcmRJGc6RCzBgG31DLIyai9WAoEhYRgIHpt/BCRMrnXLbGWGPQuBUrnF0TA==} + engines: {node: '>=18'} + + detective-typescript@14.1.2: + resolution: {integrity: sha512-bIeEn0eVi/JRsE1YizBR2ilnMlWRAIBJJ6kXCKNFxEEWhUcEY3R6I3KYIAy48ieURbD1hcb3Ebvl8AqeoPMSzg==} + engines: {node: '>=18'} + peerDependencies: + typescript: ^5.4.4 || ^6.0.2 + + detective-vue2@2.3.0: + resolution: {integrity: sha512-3gwbZPqVTm9sL9XdZsgEJ7x4x99O853VVZHapQAiEkGuMJMpFPjHDrecSgfqnS5JW3FJfYXesLZGvUOibjn49g==} + engines: {node: '>=18'} + peerDependencies: + typescript: ^5.4.4 || ^6.0.2 + domexception@4.0.0: resolution: {integrity: sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==} engines: {node: '>=12'} deprecated: Use your platform's native DOMException instead + dpdm@4.3.0: + resolution: {integrity: sha512-2ZrP5B3MHHo7mXWgNxntU5DhJIvXNP4hcMBdCJsuxEWenKMdf4h6XouuKMf3Sj2SAUNBy/ajcWAVIyiOZly6rw==} + hasBin: true + driver.js@1.4.0: resolution: {integrity: sha512-Gm64jm6PmcU+si21sQhBrTAM1JvUrR0QhNmjkprNLxohOBzul9+pNHXgQaT9lW84gwg9GMLB3NZGuGolsz5uew==} @@ -2819,6 +2945,11 @@ packages: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} + escodegen@2.1.0: + resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} + engines: {node: '>=6.0'} + hasBin: true + eslint-config-prettier@10.1.8: resolution: {integrity: sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==} hasBin: true @@ -2884,6 +3015,11 @@ packages: resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + esquery@1.7.0: resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} engines: {node: '>=0.10'} @@ -3008,6 +3144,11 @@ packages: resolution: {integrity: sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==} engines: {node: '>=20'} + filing-cabinet@5.5.1: + resolution: {integrity: sha512-PzLBTChlVPn6LnNxF0KWs+XqPziVh3Sfmz/3TXOymHxu6a9yhrDcQn7YwgpcRM6mqhR2WHVGPR8RU4fmcF1IVA==} + engines: {node: '>=18'} + hasBin: true + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -3070,6 +3211,10 @@ packages: fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fs-extra@11.4.0: + resolution: {integrity: sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==} + engines: {node: '>=14.14'} + fs-extra@9.1.0: resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} engines: {node: '>=10'} @@ -3094,6 +3239,10 @@ packages: engines: {node: '>=10'} deprecated: This package is no longer supported. + get-amd-module-type@6.0.2: + resolution: {integrity: sha512-7zShVYAYtMnj9S65CfN+hvpBCByfuB1OY8xID01nZEzXTZbx4YyysAfi+nMl95JSR6odt4q8TCj2W63KAoyVLQ==} + engines: {node: '>=18'} + get-browser-rtc@1.1.0: resolution: {integrity: sha512-MghbMJ61EJrRsDe7w1Bvqt3ZsBuqhce5nrn/XAwgwOXhcsz53/ltdxOse1h/8eKXj5slzxdsz56g5rzOFSGwfQ==} @@ -3109,6 +3258,9 @@ packages: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} + get-own-enumerable-property-symbols@3.0.2: + resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} + get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -3143,6 +3295,10 @@ packages: deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true + glob@13.0.6: + resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} + engines: {node: 18 || 20 || >=22} + glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me @@ -3155,6 +3311,11 @@ packages: resolution: {integrity: sha512-hjrNztw/VajQwOLsMNT1cbJiH2muO3OROCHnbehc8eY5JyD2gqz4AcMHPqgaOR59DjgUjYAYLeH699g/eWi2jw==} engines: {node: '>=18'} + gonzales-pe@4.3.0: + resolution: {integrity: sha512-otgSPpUmdWJ43VXyiNgEYE4luzHCL2pz4wQ0OnDluC6Eg4Ko3Vexy/SrSynglw/eR+OhkzmqFCZa/OFa/RgAOQ==} + engines: {node: '>=0.6.0'} + hasBin: true + google-protobuf@3.21.4: resolution: {integrity: sha512-MnG7N936zcKTco4Jd2PX2U96Kf9PxygAPKBug+74LHzmHXmceN16MmRcdgZv+DGef/S9YvQAfRsNCn4cjf9yyQ==} @@ -3284,6 +3445,10 @@ packages: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} + is-core-module@2.16.2: + resolution: {integrity: sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==} + engines: {node: '>= 0.4'} + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -3304,10 +3469,22 @@ packages: resolution: {integrity: sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==} engines: {node: '>=10'} + is-interactive@1.0.0: + resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} + engines: {node: '>=8'} + + is-interactive@2.0.0: + resolution: {integrity: sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==} + engines: {node: '>=12'} + is-number@7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} + is-obj@1.0.1: + resolution: {integrity: sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==} + engines: {node: '>=0.10.0'} + is-path-inside@3.0.3: resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} engines: {node: '>=8'} @@ -3318,6 +3495,10 @@ packages: is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + is-regexp@1.0.0: + resolution: {integrity: sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==} + engines: {node: '>=0.10.0'} + is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} @@ -3329,6 +3510,14 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + + is-url-superb@4.0.0: + resolution: {integrity: sha512-GI+WjezhPPcbM+tqE9LnmsY5qqjwHzTvjJ36wxYX5ujNXefSUJ/T17r5bqDV8yLhcgB59KTPNOc9O9cmHTPWsA==} + engines: {node: '>=10'} + is-what@5.5.0: resolution: {integrity: sha512-oG7cgbmg5kLYae2N5IVd3jm2s+vldjxJzK1pcu9LfpGuQ93MQSzo0okvRna+7y5ifrD+20FE8FvjusyGaz14fw==} engines: {node: '>=18'} @@ -3528,6 +3717,10 @@ packages: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} + log-symbols@7.0.1: + resolution: {integrity: sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg==} + engines: {node: '>=18'} + log-update@6.1.0: resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} engines: {node: '>=18'} @@ -3545,6 +3738,16 @@ packages: resolution: {integrity: sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==} engines: {node: 20 || >=22} + madge@8.0.0: + resolution: {integrity: sha512-9sSsi3TBPhmkTCIpVQF0SPiChj1L7Rq9kU2KDG1o6v2XH9cCw086MopjVCD+vuoL5v8S77DTbVopTO8OUiQpIw==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + typescript: ^5.4.4 + peerDependenciesMeta: + typescript: + optional: true + magic-string-ast@1.0.3: resolution: {integrity: sha512-CvkkH1i81zl7mmb94DsRiFeG9V2fR2JeuK8yDgS8oiZSFa++wWLEgZ5ufEOyLHbvSbD1gTRKv9NdX69Rnvr9JA==} engines: {node: '>=20.19.0'} @@ -3659,6 +3862,16 @@ packages: mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + module-definition@6.0.2: + resolution: {integrity: sha512-SvAU3lB0+Yjbq55yHY3wkRZBOh+fhU1SnIF3IFbTewv6mtAh7yUT8ACHAJ2mGIJ7tCes2QuCL/cl6m0JSZ/ArA==} + engines: {node: '>=18'} + hasBin: true + + module-lookup-amd@9.1.3: + resolution: {integrity: sha512-Jc3XmOaR9FdfMJSK8+vyLgsCkzm8z2L0NS6vrlRWi12DjS7MY7TMNE7E1yj8yXx837xtMDbKSSgcdXnFlJ2YLg==} + engines: {node: '>=18'} + hasBin: true + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -3708,6 +3921,10 @@ packages: encoding: optional: true + node-source-walk@7.0.2: + resolution: {integrity: sha512-71kFFjYaSshDTA8/a2HiTYPLdASWjLJxUyJxGE+ffxU+KhxSBtM9kiLUX+R2yooFdSFKMFpi4n3PFtDy6qXv8A==} + engines: {node: '>=18'} + nodemon@3.1.14: resolution: {integrity: sha512-jakjZi93UtB3jHMWsXL68FXSAosbLfY0In5gtKq3niLSkrWznrVBzXFNOEMJUfc9+Ke7SHWoAZsiMkNP3vq6Jw==} engines: {node: '>=10'} @@ -3781,6 +3998,14 @@ packages: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} + ora@5.4.1: + resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==} + engines: {node: '>=10'} + + ora@9.4.1: + resolution: {integrity: sha512-6VlU9MLXbjVQD04AZCMX28hVtA5bUoadvUqO76MUCVA0ilwJbMiHsITRPfyVm6p/BC0Av/BXMujx39WCe1LEqw==} + engines: {node: '>=20'} + ospath@1.2.2: resolution: {integrity: sha512-o6E5qJV5zkAbIDNhGSIlyOhScKXgQrSRMilfph0clDfM0nEnBOlKlH4sWDmG95BW/CvwNz0vmm7dJVtU2KlMiA==} @@ -3808,6 +4033,10 @@ packages: papaparse@5.5.3: resolution: {integrity: sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==} + parse-ms@2.1.0: + resolution: {integrity: sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==} + engines: {node: '>=6'} + parse-unit@1.0.1: resolution: {integrity: sha512-hrqldJHokR3Qj88EIlV/kAyAi/G5R2+R56TBANxNMy0uPlYcttx0jnMW6Yx5KsKPSbC3KddM/7qQm3+0wEXKxg==} @@ -3836,10 +4065,17 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + path-parse@1.0.7: + resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-scurry@1.11.1: resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} engines: {node: '>=16 || 14 >=14.18'} + path-scurry@2.0.2: + resolution: {integrity: sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==} + engines: {node: 18 || 20 || >=22} + path-to-regexp@8.4.2: resolution: {integrity: sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==} @@ -3900,10 +4136,20 @@ packages: platform@1.3.6: resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + postcss-selector-parser@7.1.1: resolution: {integrity: sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==} engines: {node: '>=4'} + postcss-values-parser@6.0.2: + resolution: {integrity: sha512-YLJpK0N1brcNJrs9WatuJFtHaV9q5aAOj+S4DI5S7jgHlRfm0PIbDCAFRYMQD5SHq7Fy6xsDhyutgS0QOAs0qw==} + engines: {node: '>=10'} + peerDependencies: + postcss: ^8.2.9 + postcss@8.5.16: resolution: {integrity: sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==} engines: {node: ^10 || ^12 || >=14} @@ -3914,6 +4160,11 @@ packages: deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. hasBin: true + precinct@12.3.2: + resolution: {integrity: sha512-JbJevI1K80z8e/WIyDt/4vUN/4qcfBSKKqOjJA4mosPPPb7zODKRJQV7YN7apVWN3k58nZYm/vEsLgEGYmnxwg==} + engines: {node: '>=18'} + hasBin: true + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -3931,6 +4182,10 @@ packages: resolution: {integrity: sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==} engines: {node: '>=6'} + pretty-ms@7.0.1: + resolution: {integrity: sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==} + engines: {node: '>=10'} + process@0.11.10: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} @@ -3984,6 +4239,9 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + quote-unquote@1.0.0: + resolution: {integrity: sha512-twwRO/ilhlG/FIgYeKGFqyHhoEhqgnKVkcmqMKi2r524gz3ZbDTcyFt38E9xjJI2vT+KbRNHVbnJ/e0I25Azwg==} + randombytes@2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} @@ -4029,9 +4287,31 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} + requirejs-config-file@4.0.0: + resolution: {integrity: sha512-jnIre8cbWOyvr8a5F2KuqBnY+SDA4NXr/hzEZJG79Mxm2WiFQz2dzhC8ibtPJS7zkmBEl1mxSwp5HhC1W4qpxw==} + engines: {node: '>=10.13.0'} + + requirejs@2.3.8: + resolution: {integrity: sha512-7/cTSLOdYkNBNJcDMWf+luFvMriVm7eYxp4BcFCsAX0wF421Vyce5SXP17c+Jd5otXKGNehIonFlyQXSowL6Mw==} + engines: {node: '>=0.4.0'} + hasBin: true + + resolve-dependency-path@4.0.1: + resolution: {integrity: sha512-YQftIIC4vzO9UMhO/sCgXukNyiwVRCVaxiWskCBy7Zpqkplm8kTAISZ8O1MoKW1ca6xzgLUBjZTcDgypXvXxiQ==} + engines: {node: '>=18'} + resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + resolve@1.22.12: + resolution: {integrity: sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==} + engines: {node: '>= 0.4'} + hasBin: true + + restore-cursor@3.1.0: + resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} + engines: {node: '>=8'} + restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -4080,6 +4360,11 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + sass-lookup@6.1.2: + resolution: {integrity: sha512-GjmndmKQBtlPil79RK72L7yc5kDXZPCQeH97bP8R8DcxtXQJO6vECExb3WP/m6+cxaV9h4ZxrSRvCkPG2v/VSw==} + engines: {node: '>=18'} + hasBin: true + saxes@6.0.0: resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} engines: {node: '>=v12.22.7'} @@ -4195,6 +4480,10 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + speakingurl@14.0.1: resolution: {integrity: sha512-1POYv7uv2gXoyGFpBCmpDVSNV74IfsWlDW216UPjbWufNf+bSU6GdbDsxdcxtfwb4xlI3yxzOTKClUosxARYrQ==} engines: {node: '>=0.10.0'} @@ -4222,6 +4511,13 @@ packages: std-env@4.0.0: resolution: {integrity: sha512-zUMPtQ/HBY3/50VbpkupYHbRroTRZJPRLvreamgErJVys0ceuzMkD44J/QjqhHjOzK42GQ3QZIeFG1OYfOtKqQ==} + stdin-discarder@0.3.2: + resolution: {integrity: sha512-eCPu1qRxPVkl5605OTWF8Wz40b4Mf45NY5LQmVPQ599knfs5QhASUm9GbJ5BDMDOXgrnh0wyEdvzmL//YMlw0A==} + engines: {node: '>=18'} + + stream-to-array@2.3.0: + resolution: {integrity: sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA==} + streamx@2.28.0: resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} @@ -4247,6 +4543,10 @@ packages: string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + stringify-object@3.3.0: + resolution: {integrity: sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==} + engines: {node: '>=4'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -4255,6 +4555,10 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} + strip-bom@3.0.0: + resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} + engines: {node: '>=4'} + strip-final-newline@2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} @@ -4274,6 +4578,11 @@ packages: style-inject@0.3.0: resolution: {integrity: sha512-IezA2qp+vcdlhJaVm5SOdPPTUu0FCEqfNSli2vRuSIBbu5Nq5UvygTk/VzeCqfLz2Atj3dVII5QBKGZRZ0edzw==} + stylus-lookup@6.1.2: + resolution: {integrity: sha512-O+Q/SJ8s1X2aMLh4213fQ9X/bND9M3dhSsyTRe+O1OXPcewGLiYmAtKCrnP7FDvDBaXB2ZHPkCt3zi4cJXBlCQ==} + engines: {node: '>=18'} + hasBin: true + superjson@2.2.6: resolution: {integrity: sha512-H+ue8Zo4vJmV2nRjpx86P35lzwDT3nItnIsocgumgr0hHMQ+ZGq5vrERg9kJBo5AWGmxZDhzDo+WVIJqkB0cGA==} engines: {node: '>=16'} @@ -4290,6 +4599,10 @@ packages: resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} engines: {node: '>=10'} + supports-preserve-symlinks-flag@1.0.0: + resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} + engines: {node: '>= 0.4'} + symbol-tree@3.2.4: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} @@ -4428,6 +4741,14 @@ packages: resolution: {integrity: sha512-H69ZwTw3rFHb5WYpQya40YAX2/w7Ut75uUECbgBIsLmM+BNuYnxsltfyyLMxy6sEeKxgijLTnQtLd0nKd6+IYw==} hasBin: true + ts-graphviz@2.1.6: + resolution: {integrity: sha512-XyLVuhBVvdJTJr2FJJV2L1pc4MwSjMhcunRVgDE9k4wbb2ee7ORYnPewxMWUav12vxyfUM686MSGsqnVRIInuw==} + engines: {node: '>=18'} + + tsconfig-paths@4.2.0: + resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} + engines: {node: '>=6'} + tslib@1.14.1: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} @@ -4463,6 +4784,11 @@ packages: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.1.0' + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -4706,6 +5032,13 @@ packages: resolution: {integrity: sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A==} engines: {node: 20 || >=22} + walkdir@0.4.1: + resolution: {integrity: sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==} + engines: {node: '>=6.0.0'} + + wcwidth@1.0.1: + resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -4818,10 +5151,18 @@ packages: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yargs@16.2.2: resolution: {integrity: sha512-Nt9ZJjXTv5R8MHbqby/wXQ6Gi0Bb3TcYZkR1bzuL4yB2OxWPkXknz513gEF0GoA6tn00UpbPvERW8rzCuWCA6w==} engines: {node: '>=10'} + yargs@18.1.0: + resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yauzl@3.4.0: resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==} engines: {node: '>=12'} @@ -4830,6 +5171,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yoctocolors@2.2.0: + resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} + engines: {node: '>=18'} + zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} @@ -4953,6 +5298,13 @@ snapshots: transitivePeerDependencies: - supports-color + '@dependents/detective-less@5.0.3': + dependencies: + gonzales-pe: 4.3.0 + node-source-walk: 7.0.2 + + '@discoveryjs/json-ext@1.1.0': {} + '@emnapi/core@1.10.0': dependencies: '@emnapi/wasi-threads': 1.2.1 @@ -5662,6 +6014,21 @@ snapshots: '@tokenizer/token@0.3.0': {} + '@ts-graphviz/adapter@2.0.6': + dependencies: + '@ts-graphviz/common': 2.1.5 + + '@ts-graphviz/ast@2.0.7': + dependencies: + '@ts-graphviz/common': 2.1.5 + + '@ts-graphviz/common@2.1.5': {} + + '@ts-graphviz/core@2.0.7': + dependencies: + '@ts-graphviz/ast': 2.0.7 + '@ts-graphviz/common': 2.1.5 + '@tsconfig/node20@20.1.9': {} '@tybys/wasm-util@0.10.3': @@ -5928,6 +6295,15 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/project-service@8.62.1(supports-color@8.1.1)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.62.1(typescript@5.9.3) + '@typescript-eslint/types': 8.62.1 + debug: 4.4.3(supports-color@8.1.1) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/project-service@8.62.1(supports-color@8.1.1)(typescript@6.0.3)': dependencies: '@typescript-eslint/tsconfig-utils': 8.62.1(typescript@6.0.3) @@ -5942,6 +6318,10 @@ snapshots: '@typescript-eslint/types': 8.62.1 '@typescript-eslint/visitor-keys': 8.62.1 + '@typescript-eslint/tsconfig-utils@8.62.1(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + '@typescript-eslint/tsconfig-utils@8.62.1(typescript@6.0.3)': dependencies: typescript: 6.0.3 @@ -5960,6 +6340,21 @@ snapshots: '@typescript-eslint/types@8.62.1': {} + '@typescript-eslint/typescript-estree@8.62.1(supports-color@8.1.1)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/project-service': 8.62.1(supports-color@8.1.1)(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.62.1(typescript@5.9.3) + '@typescript-eslint/types': 8.62.1 + '@typescript-eslint/visitor-keys': 8.62.1 + debug: 4.4.3(supports-color@8.1.1) + minimatch: 10.2.5 + semver: 7.8.5 + tinyglobby: 0.2.17 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/typescript-estree@8.62.1(supports-color@8.1.1)(typescript@6.0.3)': dependencies: '@typescript-eslint/project-service': 8.62.1(supports-color@8.1.1)(typescript@6.0.3) @@ -6298,11 +6693,15 @@ snapshots: ansi-styles@6.2.3: {} + any-promise@1.3.0: {} + anymatch@3.1.3: dependencies: normalize-path: 3.0.0 picomatch: 2.3.2 + app-module-path@2.2.0: {} + aproba@2.1.0: {} arch@2.2.0: {} @@ -6335,6 +6734,8 @@ snapshots: '@babel/parser': 7.29.7 pathe: 2.0.3 + ast-module-types@6.0.2: {} + ast-walker-scope@0.9.0: dependencies: '@babel/parser': 7.29.7 @@ -6501,6 +6902,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + chalk@5.6.2: {} + chart.js@4.5.1: dependencies: '@kurkle/color': 0.3.4 @@ -6529,10 +6932,18 @@ snapshots: ci-info@4.4.0: {} + cli-cursor@3.1.0: + dependencies: + restore-cursor: 3.1.0 + cli-cursor@5.0.0: dependencies: restore-cursor: 5.1.0 + cli-spinners@2.9.2: {} + + cli-spinners@3.4.0: {} + cli-table3@0.6.1: dependencies: string-width: 4.2.3 @@ -6550,6 +6961,14 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.2.0 + wrap-ansi: 9.0.2 + + clone@1.0.4: {} + color-convert@1.9.3: dependencies: color-name: 1.1.3 @@ -6601,12 +7020,16 @@ snapshots: commander@10.0.1: {} + commander@12.1.0: {} + commander@6.2.1: {} commander@7.2.0: {} common-tags@1.8.2: {} + commondir@1.0.1: {} + concat-map@0.0.1: {} confbox@0.1.8: {} @@ -6897,6 +7320,10 @@ snapshots: deep-is@0.1.4: {} + defaults@1.0.4: + dependencies: + clone: 1.0.4 + delaunator@5.1.0: dependencies: robust-predicates: 3.0.3 @@ -6907,13 +7334,89 @@ snapshots: depd@2.0.0: {} + dependency-tree@11.5.0(supports-color@8.1.1): + dependencies: + '@discoveryjs/json-ext': 1.1.0 + commander: 12.1.0 + filing-cabinet: 5.5.1 + precinct: 12.3.2(supports-color@8.1.1) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + detect-libc@2.1.2: {} + detective-amd@6.1.0: + dependencies: + ast-module-types: 6.0.2 + escodegen: 2.1.0 + get-amd-module-type: 6.0.2 + node-source-walk: 7.0.2 + + detective-cjs@6.1.1: + dependencies: + ast-module-types: 6.0.2 + node-source-walk: 7.0.2 + + detective-es6@5.0.2: + dependencies: + node-source-walk: 7.0.2 + + detective-postcss@8.0.4(postcss@8.5.16): + dependencies: + is-url-superb: 4.0.0 + postcss: 8.5.16 + postcss-values-parser: 6.0.2(postcss@8.5.16) + + detective-sass@6.0.2: + dependencies: + gonzales-pe: 4.3.0 + node-source-walk: 7.0.2 + + detective-scss@5.0.2: + dependencies: + gonzales-pe: 4.3.0 + node-source-walk: 7.0.2 + + detective-stylus@5.0.1: {} + + detective-typescript@14.1.2(supports-color@8.1.1)(typescript@5.9.3): + dependencies: + '@typescript-eslint/typescript-estree': 8.62.1(supports-color@8.1.1)(typescript@5.9.3) + ast-module-types: 6.0.2 + node-source-walk: 7.0.2 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + detective-vue2@2.3.0(supports-color@8.1.1)(typescript@5.9.3): + dependencies: + '@dependents/detective-less': 5.0.3 + '@vue/compiler-sfc': 3.5.38 + detective-es6: 5.0.2 + detective-sass: 6.0.2 + detective-scss: 5.0.2 + detective-stylus: 5.0.1 + detective-typescript: 14.1.2(supports-color@8.1.1)(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + domexception@4.0.0: dependencies: webidl-conversions: 7.0.0 optional: true + dpdm@4.3.0: + dependencies: + chalk: 5.6.2 + fs-extra: 11.4.0 + glob: 13.0.6 + ora: 9.4.1 + tslib: 2.8.1 + typescript: 5.9.3 + yargs: 18.1.0 + driver.js@1.4.0: {} dunder-proto@1.0.1: @@ -6996,6 +7499,14 @@ snapshots: escape-string-regexp@4.0.0: {} + escodegen@2.1.0: + dependencies: + esprima: 4.0.1 + estraverse: 5.3.0 + esutils: 2.0.3 + optionalDependencies: + source-map: 0.6.1 + eslint-config-prettier@10.1.8(eslint@10.0.3(jiti@2.7.0)(supports-color@8.1.1)): dependencies: eslint: 10.0.3(jiti@2.7.0)(supports-color@8.1.1) @@ -7081,6 +7592,8 @@ snapshots: acorn-jsx: 5.3.2(acorn@8.16.0) eslint-visitor-keys: 5.0.1 + esprima@4.0.1: {} + esquery@1.7.0: dependencies: estraverse: 5.3.0 @@ -7239,6 +7752,20 @@ snapshots: transitivePeerDependencies: - supports-color + filing-cabinet@5.5.1: + dependencies: + app-module-path: 2.2.0 + commander: 12.1.0 + enhanced-resolve: 5.21.6 + module-definition: 6.0.2 + module-lookup-amd: 9.1.3 + resolve: 1.22.12 + resolve-dependency-path: 4.0.1 + sass-lookup: 6.1.2 + stylus-lookup: 6.1.2 + tsconfig-paths: 4.2.0 + typescript: 5.9.3 + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -7301,6 +7828,12 @@ snapshots: fs-constants@1.0.0: {} + fs-extra@11.4.0: + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.2.1 + universalify: 2.0.1 + fs-extra@9.1.0: dependencies: at-least-node: 1.0.0 @@ -7331,6 +7864,11 @@ snapshots: strip-ansi: 6.0.1 wide-align: 1.1.5 + get-amd-module-type@6.0.2: + dependencies: + ast-module-types: 6.0.2 + node-source-walk: 7.0.2 + get-browser-rtc@1.1.0: {} get-caller-file@2.0.5: {} @@ -7350,6 +7888,8 @@ snapshots: hasown: 2.0.4 math-intrinsics: 1.1.0 + get-own-enumerable-property-symbols@3.0.2: {} + get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -7388,6 +7928,12 @@ snapshots: package-json-from-dist: 1.0.1 path-scurry: 1.11.1 + glob@13.0.6: + dependencies: + minimatch: 10.2.5 + minipass: 7.1.3 + path-scurry: 2.0.2 + glob@7.2.3: dependencies: fs.realpath: 1.0.0 @@ -7403,6 +7949,10 @@ snapshots: globals@17.4.0: {} + gonzales-pe@4.3.0: + dependencies: + minimist: 1.2.8 + google-protobuf@3.21.4: {} gopd@1.2.0: {} @@ -7513,6 +8063,10 @@ snapshots: dependencies: binary-extensions: 2.3.0 + is-core-module@2.16.2: + dependencies: + hasown: 2.0.4 + is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} @@ -7530,20 +8084,32 @@ snapshots: global-dirs: 3.0.1 is-path-inside: 3.0.3 + is-interactive@1.0.0: {} + + is-interactive@2.0.0: {} + is-number@7.0.0: {} + is-obj@1.0.1: {} + is-path-inside@3.0.3: {} is-potential-custom-element-name@1.0.1: {} is-promise@4.0.0: {} + is-regexp@1.0.0: {} + is-stream@2.0.1: {} is-typedarray@1.0.0: {} is-unicode-supported@0.1.0: {} + is-unicode-supported@2.1.0: {} + + is-url-superb@4.0.0: {} + is-what@5.5.0: {} isexe@2.0.0: {} @@ -7749,6 +8315,11 @@ snapshots: chalk: 4.1.2 is-unicode-supported: 0.1.0 + log-symbols@7.0.1: + dependencies: + is-unicode-supported: 2.1.0 + yoctocolors: 2.2.0 + log-update@6.1.0: dependencies: ansi-escapes: 7.3.0 @@ -7765,6 +8336,25 @@ snapshots: lru-cache@11.5.1: {} + madge@8.0.0(supports-color@8.1.1)(typescript@6.0.3): + dependencies: + chalk: 4.1.2 + commander: 7.2.0 + commondir: 1.0.1 + debug: 4.4.3(supports-color@8.1.1) + dependency-tree: 11.5.0(supports-color@8.1.1) + ora: 5.4.1 + pluralize: 8.0.0 + pretty-ms: 7.0.1 + rc: 1.2.8 + stream-to-array: 2.3.0 + ts-graphviz: 2.1.6 + walkdir: 0.4.1 + optionalDependencies: + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + magic-string-ast@1.0.3: dependencies: magic-string: 0.30.21 @@ -7854,6 +8444,17 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.4 + module-definition@6.0.2: + dependencies: + ast-module-types: 6.0.2 + node-source-walk: 7.0.2 + + module-lookup-amd@9.1.3: + dependencies: + commander: 12.1.0 + requirejs: 2.3.8 + requirejs-config-file: 4.0.0 + ms@2.1.3: {} muggle-string@0.4.1: {} @@ -7882,6 +8483,10 @@ snapshots: dependencies: whatwg-url: 5.0.0 + node-source-walk@7.0.2: + dependencies: + '@babel/parser': 7.29.7 + nodemon@3.1.14: dependencies: chokidar: 3.6.0 @@ -7971,6 +8576,29 @@ snapshots: type-check: 0.4.0 word-wrap: 1.2.5 + ora@5.4.1: + dependencies: + bl: 4.1.0 + chalk: 4.1.2 + cli-cursor: 3.1.0 + cli-spinners: 2.9.2 + is-interactive: 1.0.0 + is-unicode-supported: 0.1.0 + log-symbols: 4.1.0 + strip-ansi: 6.0.1 + wcwidth: 1.0.1 + + ora@9.4.1: + dependencies: + chalk: 5.6.2 + cli-cursor: 5.0.0 + cli-spinners: 3.4.0 + is-interactive: 2.0.0 + is-unicode-supported: 2.1.0 + log-symbols: 7.0.1 + stdin-discarder: 0.3.2 + string-width: 8.2.1 + ospath@1.2.2: {} outdent@0.8.0: {} @@ -8034,6 +8662,8 @@ snapshots: papaparse@5.5.3: {} + parse-ms@2.1.0: {} + parse-unit@1.0.1: {} parse5@7.3.0: @@ -8054,11 +8684,18 @@ snapshots: path-key@3.1.1: {} + path-parse@1.0.7: {} + path-scurry@1.11.1: dependencies: lru-cache: 10.4.3 minipass: 7.1.3 + path-scurry@2.0.2: + dependencies: + lru-cache: 11.5.1 + minipass: 7.1.3 + path-to-regexp@8.4.2: {} pathe@2.0.3: {} @@ -8107,11 +8744,20 @@ snapshots: platform@1.3.6: {} + pluralize@8.0.0: {} + postcss-selector-parser@7.1.1: dependencies: cssesc: 3.0.0 util-deprecate: 1.0.2 + postcss-values-parser@6.0.2(postcss@8.5.16): + dependencies: + color-name: 1.1.4 + is-url-superb: 4.0.0 + postcss: 8.5.16 + quote-unquote: 1.0.0 + postcss@8.5.16: dependencies: nanoid: 3.3.15 @@ -8133,6 +8779,26 @@ snapshots: tar-fs: 2.1.5 tunnel-agent: 0.6.0 + precinct@12.3.2(supports-color@8.1.1): + dependencies: + '@dependents/detective-less': 5.0.3 + commander: 12.1.0 + detective-amd: 6.1.0 + detective-cjs: 6.1.1 + detective-es6: 5.0.2 + detective-postcss: 8.0.4(postcss@8.5.16) + detective-sass: 6.0.2 + detective-scss: 5.0.2 + detective-stylus: 5.0.1 + detective-typescript: 14.1.2(supports-color@8.1.1)(typescript@5.9.3) + detective-vue2: 2.3.0(supports-color@8.1.1)(typescript@5.9.3) + module-definition: 6.0.2 + node-source-walk: 7.0.2 + postcss: 8.5.16 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + prelude-ls@1.2.1: {} prettier-linter-helpers@1.0.0: @@ -8143,6 +8809,10 @@ snapshots: pretty-bytes@5.6.0: {} + pretty-ms@7.0.1: + dependencies: + parse-ms: 2.1.0 + process@0.11.10: {} progress@2.0.3: {} @@ -8208,6 +8878,8 @@ snapshots: queue-microtask@1.2.3: {} + quote-unquote@1.0.0: {} + randombytes@2.1.0: dependencies: safe-buffer: 5.2.1 @@ -8252,8 +8924,29 @@ snapshots: require-from-string@2.0.2: {} + requirejs-config-file@4.0.0: + dependencies: + esprima: 4.0.1 + stringify-object: 3.3.0 + + requirejs@2.3.8: {} + + resolve-dependency-path@4.0.1: {} + resolve-pkg-maps@1.0.0: {} + resolve@1.22.12: + dependencies: + es-errors: 1.3.0 + is-core-module: 2.16.2 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + + restore-cursor@3.1.0: + dependencies: + onetime: 5.1.2 + signal-exit: 3.0.7 + restore-cursor@5.1.0: dependencies: onetime: 7.0.0 @@ -8318,6 +9011,11 @@ snapshots: safer-buffer@2.1.2: {} + sass-lookup@6.1.2: + dependencies: + commander: 12.1.0 + enhanced-resolve: 5.21.6 + saxes@6.0.0: dependencies: xmlchars: 2.2.0 @@ -8491,6 +9189,9 @@ snapshots: source-map-js@1.2.1: {} + source-map@0.6.1: + optional: true + speakingurl@14.0.1: {} sprintf-js@1.0.3: {} @@ -8526,6 +9227,12 @@ snapshots: std-env@4.0.0: {} + stdin-discarder@0.3.2: {} + + stream-to-array@2.3.0: + dependencies: + any-promise: 1.3.0 + streamx@2.28.0: dependencies: events-universal: 1.0.1 @@ -8564,6 +9271,12 @@ snapshots: dependencies: safe-buffer: 5.2.1 + stringify-object@3.3.0: + dependencies: + get-own-enumerable-property-symbols: 3.0.2 + is-obj: 1.0.1 + is-regexp: 1.0.0 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -8572,6 +9285,8 @@ snapshots: dependencies: ansi-regex: 6.2.2 + strip-bom@3.0.0: {} + strip-final-newline@2.0.0: {} strip-json-comments@2.0.1: {} @@ -8584,6 +9299,10 @@ snapshots: style-inject@0.3.0: {} + stylus-lookup@6.1.2: + dependencies: + commander: 12.1.0 + superjson@2.2.6: dependencies: copy-anything: 4.0.5 @@ -8600,6 +9319,8 @@ snapshots: dependencies: has-flag: 4.0.0 + supports-preserve-symlinks-flag@1.0.0: {} + symbol-tree@3.2.4: {} synckit@0.11.11: @@ -8746,6 +9467,10 @@ snapshots: tree-kill@1.2.2: {} + ts-api-utils@2.5.0(typescript@5.9.3): + dependencies: + typescript: 5.9.3 + ts-api-utils@2.5.0(typescript@6.0.3): dependencies: typescript: 6.0.3 @@ -8757,6 +9482,19 @@ snapshots: command-line-usage: 6.1.3 string-format: 2.0.0 + ts-graphviz@2.1.6: + dependencies: + '@ts-graphviz/adapter': 2.0.6 + '@ts-graphviz/ast': 2.0.7 + '@ts-graphviz/common': 2.1.5 + '@ts-graphviz/core': 2.0.7 + + tsconfig-paths@4.2.0: + dependencies: + json5: 2.2.3 + minimist: 1.2.8 + strip-bom: 3.0.0 + tslib@1.14.1: {} tslib@2.8.1: {} @@ -8792,6 +9530,8 @@ snapshots: transitivePeerDependencies: - supports-color + typescript@5.9.3: {} + typescript@6.0.3: {} typical@4.0.0: {} @@ -8992,6 +9732,12 @@ snapshots: walk-up-path@4.0.0: {} + walkdir@0.4.1: {} + + wcwidth@1.0.1: + dependencies: + defaults: 1.0.4 + webidl-conversions@3.0.1: {} webidl-conversions@7.0.0: @@ -9074,6 +9820,8 @@ snapshots: yargs-parser@20.2.9: {} + yargs-parser@22.0.0: {} + yargs@16.2.2: dependencies: cliui: 7.0.4 @@ -9084,12 +9832,23 @@ snapshots: y18n: 5.0.8 yargs-parser: 20.2.9 + yargs@18.1.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 8.2.1 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yauzl@3.4.0: dependencies: pend: 1.2.0 yocto-queue@0.1.0: {} + yoctocolors@2.2.0: {} + zod@3.25.76: {} zod@4.3.6: {} diff --git a/server/src/controllers/decentralized_controller.ts b/server/src/controllers/decentralized_controller.ts index 33bdc403a..08fb9983b 100644 --- a/server/src/controllers/decentralized_controller.ts +++ b/server/src/controllers/decentralized_controller.ts @@ -9,7 +9,7 @@ import { client, DataType } from "@epfml/discojs"; import { TrainingController } from "./training_controller.js"; import messages = client.decentralized.messages; -import MessageTypes = client.messages.type; +import MessageTypes = client.mtype.MType; const debug = createDebug("server:controllers:decentralized"); diff --git a/server/src/controllers/federated_controller.ts b/server/src/controllers/federated_controller.ts index 68d017bea..9c9784b58 100644 --- a/server/src/controllers/federated_controller.ts +++ b/server/src/controllers/federated_controller.ts @@ -12,7 +12,7 @@ import { import { TrainingController } from "./training_controller.js"; -import MessageTypes = client.messages.type; +import MessageTypes = client.mtype.MType; import FederatedMessages = client.federated.messages; const debug = createDebug("server:controllers:federated"); diff --git a/server/src/controllers/training_controller.ts b/server/src/controllers/training_controller.ts index ff3e52c6f..9e9c880ec 100644 --- a/server/src/controllers/training_controller.ts +++ b/server/src/controllers/training_controller.ts @@ -64,8 +64,8 @@ export abstract class TrainingController< "Sending enough-participant message to client [%s]", participantId.slice(0, 4), ); - const msg: client.messages.EnoughParticipants = { - type: client.messages.type.EnoughParticipants, + const msg: client.mtype.EnoughParticipants = { + type: client.mtype.MType.EnoughParticipants, nbOfParticipants: this.connections.size, }; participantWs.send(msgpack.encode(msg)); @@ -86,8 +86,8 @@ export abstract class TrainingController< "Telling remaining client [%s] to wait for participants", participantId.slice(0, 4), ); - const msg: client.messages.WaitingForMoreParticipants = { - type: client.messages.type.WaitingForMoreParticipants, + const msg: client.mtype.WaitingForMoreParticipants = { + type: client.mtype.MType.WaitingForMoreParticipants, nbOfParticipants: this.connections.size, }; participantWs.send(msgpack.encode(msg)); From 631b6469ec5c9dea7cec97c3ebc30012cdaccbc1 Mon Sep 17 00:00:00 2001 From: hugo Date: Sat, 8 Aug 2026 02:04:07 +0200 Subject: [PATCH 02/18] refactor: remove cycles part 2, plus useless discojs exports --- .knip.json | 3 - cli/src/benchmark_gpt.ts | 17 ++--- cli/src/cli.ts | 10 +-- cli/src/data.ts | 12 +-- cli/src/hellaswag_gpt.ts | 27 ++++--- cli/src/train_gpt.ts | 6 +- discojs-node/src/hellaswag.ts | 16 ++-- discojs-node/src/model_loader.ts | 12 +-- discojs-web/src/hellaswag.ts | 16 ++-- discojs/src/aggregator/aggregator.ts | 26 +++---- discojs/src/aggregator/byzantine.ts | 21 +++--- discojs/src/aggregator/get.ts | 14 ++-- discojs/src/aggregator/mean.ts | 16 ++-- discojs/src/aggregator/percentile_clipping.ts | 20 ++--- discojs/src/aggregator/secure.ts | 17 +++-- discojs/src/aggregator/secure_history.ts | 13 ++-- discojs/src/client/client.ts | 6 +- .../decentralized/decentralized_client.ts | 8 +- .../src/client/federated/federated_client.ts | 8 +- discojs/src/client/get_client.ts | 2 +- discojs/src/client/index.ts | 18 +++-- discojs/src/client/utils.ts | 42 ----------- discojs/src/dataset/index.ts | 3 +- discojs/src/dataset/types.ts | 3 +- discojs/src/index.ts | 74 ++++++++++++++----- discojs/src/processing/index.ts | 16 +++- discojs/src/serialization/index.ts | 9 ++- discojs/src/serialization/task.ts | 2 +- discojs/src/task/task_handler.ts | 21 +++--- discojs/src/training/disco.ts | 22 +----- discojs/src/training/index.ts | 3 +- discojs/src/training/trainer.ts | 2 +- discojs/src/training/types.ts | 19 +++++ discojs/src/weights/index.ts | 2 +- docs/examples/custom_task.ts | 4 +- docs/examples/wikitext.ts | 8 +- onnx-converter/src/convert_onnx.ts | 6 +- .../controllers/decentralized_controller.ts | 11 +-- .../src/controllers/federated_controller.ts | 39 +++++----- server/src/controllers/training_controller.ts | 14 ++-- server/src/model_set.ts | 16 ++-- server/src/routes/task_router.ts | 6 +- server/src/routes/training_router.ts | 10 +-- server/src/task_set.ts | 4 +- webapp/cypress/e2e/task-creation.cy.ts | 4 +- webapp/cypress/support/e2e.ts | 11 +-- .../task_creation_form/TaskCreationForm.vue | 4 +- .../src/components/testing/ModelLibrary.vue | 6 +- .../testing/__tests__/ModelLibrary.spec.ts | 13 +--- .../components/training/TrainerDashboard.vue | 6 +- .../__tests__/TrainerDashboard.spec.ts | 4 +- .../src/store/models/__tests__/models.spec.ts | 6 +- webapp/src/store/models/index.ts | 6 +- webapp/src/store/models/opfs.ts | 4 +- webapp/src/store/models/types.ts | 4 +- 55 files changed, 349 insertions(+), 343 deletions(-) diff --git a/.knip.json b/.knip.json index 151e9a13b..8336807d9 100644 --- a/.knip.json +++ b/.knip.json @@ -4,9 +4,6 @@ "ignore": ["docs/**"], "ignoreDependencies": ["typescript-eslint", "supports-color"] }, - "discojs": { - "ignoreExportsUsedInFile": true - }, "webapp": { "ignoreDependencies": ["buffer"] }, diff --git a/cli/src/benchmark_gpt.ts b/cli/src/benchmark_gpt.ts index 26fb71c75..1dcd7b362 100644 --- a/cli/src/benchmark_gpt.ts +++ b/cli/src/benchmark_gpt.ts @@ -4,11 +4,12 @@ import { parse } from "ts-command-line-args"; import type { Network, Task } from "@epfml/discojs"; import { - async_iterator, + gather, defaultTasks, defaultModels, fetchTasks, - models, + GPTConfig, + GPT, } from "@epfml/discojs"; import { loadModelFromDisk, loadText } from "@epfml/discojs-node"; @@ -109,8 +110,8 @@ async function main(args: Required): Promise { const epochsCount = 1; const iterationsPerEpoch = 10; - const config: models.GPTConfig = { - modelType: modelType as models.GPTConfig["modelType"], + const config: GPTConfig = { + modelType: modelType as GPTConfig["modelType"], maxIter: iterationsPerEpoch, lr: 0.0001, contextLength, @@ -130,16 +131,14 @@ async function main(args: Required): Promise { .batch(batchSize); // Init and train the model - const model = new models.GPT(config); + const model = new GPT(config); console.log( `\tmodel type ${modelType} \n\tbatch size ${batchSize} \n\tcontext length ${contextLength}`, ); let epochTime = performance.now(); for (let epochsCounter = 1; epochsCounter <= epochsCount; epochsCounter++) { - const [_, logs] = await async_iterator.gather( - model.train(preprocessedDataset), - ); + const [_, logs] = await gather(model.train(preprocessedDataset)); epochTime = performance.now() - epochTime; const msPerToken = epochTime / @@ -154,7 +153,7 @@ async function main(args: Required): Promise { */ } else { const model = await loadModelFromDisk(modelPath); - if (!(model instanceof models.GPT)) { + if (!(model instanceof GPT)) { throw new Error("Loaded model isn't a GPT model"); } diff --git a/cli/src/cli.ts b/cli/src/cli.ts index a2a586cc9..8ae0b7201 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -15,11 +15,7 @@ import type { TaskProvider, Network, } from "@epfml/discojs"; -import { - Disco, - aggregator as aggregators, - client as clients, -} from "@epfml/discojs"; +import { Disco, getAggregator, getClient } from "@epfml/discojs"; import { getTaskData } from "./data.js"; import { args } from "./args.js"; @@ -35,8 +31,8 @@ async function runUser( ): Promise> { // cast as typescript isn't good with generics const trainingScheme = task.trainingInformation.scheme as N; - const aggregator = aggregators.getAggregator(task); - const client = clients.getClient(trainingScheme, url, task, aggregator); + const aggregator = getAggregator(task); + const client = getClient(trainingScheme, url, task, aggregator); const disco = new Disco(task, client, { scheme: trainingScheme }); const dir = path.join(".", `${args.testID}`); diff --git a/cli/src/data.ts b/cli/src/data.ts index 9079d2475..c6ea091dc 100644 --- a/cli/src/data.ts +++ b/cli/src/data.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { Dataset, processing } from "@epfml/discojs"; +import { Dataset, extractColumn } from "@epfml/discojs"; import { DataFormat, DataType, Image, Task } from "@epfml/discojs"; import { loadCSV, loadImage, loadImagesInDir } from "@epfml/discojs-node"; import { Repeat } from "immutable"; @@ -31,10 +31,7 @@ function loadTinderDogData(split: number): Dataset { return loadCSV(path.join(folder, "labels.csv")) .map( (row) => - [ - processing.extractColumn(row, "filename"), - processing.extractColumn(row, "label"), - ] as const, + [extractColumn(row, "filename"), extractColumn(row, "label")] as const, ) .map(async ([filename, label]) => { try { @@ -58,10 +55,7 @@ function loadData( return loadCSV(path.join(folder, "labels.csv")) .map( (row) => - [ - processing.extractColumn(row, "filename"), - processing.extractColumn(row, "label"), - ] as const, + [extractColumn(row, "filename"), extractColumn(row, "label")] as const, ) .map(async ([filename, label]) => { try { diff --git a/cli/src/hellaswag_gpt.ts b/cli/src/hellaswag_gpt.ts index ba29a3221..0aafbf844 100644 --- a/cli/src/hellaswag_gpt.ts +++ b/cli/src/hellaswag_gpt.ts @@ -5,22 +5,25 @@ import { parse } from "ts-command-line-args"; import "@tensorflow/tfjs-node"; import path from "node:path"; -import { models, serialization, Tokenizer } from "@epfml/discojs"; +import { + GPT, + ONNXModel, + modelDecode, + Tokenizer, + evaluate_hellaswag, + HellaSwagDataset, +} from "@epfml/discojs"; import { loadHellaSwag } from "@epfml/discojs-node"; const __dirname = dirname(fileURLToPath(import.meta.url)); -async function evaluateModel( - model: models.GPT | models.ONNXModel, - numDataPoints = -1, -) { - const hellaswagDataset: models.HellaSwagDataset = - await loadHellaSwag(numDataPoints); +async function evaluateModel(model: GPT | ONNXModel, numDataPoints = -1) { + const hellaswagDataset: HellaSwagDataset = await loadHellaSwag(numDataPoints); const tokenizer = await Tokenizer.from_pretrained("Xenova/gpt2"); console.log("Starting the HellaSwag benchmark..."); const start = Date.now(); - const accuracy = await models.evaluate_hellaswag( + const accuracy = await evaluate_hellaswag( model, tokenizer, hellaswagDataset, @@ -91,15 +94,15 @@ async function main(): Promise { { helpArg: "help" }, ); - let model: models.GPT | models.ONNXModel | undefined; + let model: GPT | ONNXModel | undefined; switch (args.model) { case "onnx": console.log("Using ONNX pretrained model Xenova/gpt2"); - model = await models.ONNXModel.init_pretrained("Xenova/gpt2"); + model = await ONNXModel.init_pretrained("Xenova/gpt2"); break; case "gpt-tfjs-random": console.log("Using GPT-TFJS with random initialization"); - model = new models.GPT({ seed: 42 }); + model = new GPT({ seed: 42 }); break; case "gpt-tfjs-pretrained": console.log("Using GPT-TFJS with pretrained weights"); @@ -109,7 +112,7 @@ async function main(): Promise { ); } const encodedModel = await fsPromise.readFile(args.pretrainedModelPath); - model = (await serialization.model.decode(encodedModel)) as models.GPT; + model = (await modelDecode(encodedModel)) as GPT; break; } await evaluateModel(model, args.numDataPoints); diff --git a/cli/src/train_gpt.ts b/cli/src/train_gpt.ts index f6445f4ff..537e39bbb 100644 --- a/cli/src/train_gpt.ts +++ b/cli/src/train_gpt.ts @@ -1,12 +1,12 @@ import "@tensorflow/tfjs-node"; -import { models, Dataset, Tokenizer } from "@epfml/discojs"; +import { GPTConfig, GPT, Dataset, Tokenizer } from "@epfml/discojs"; import { List } from "immutable"; async function main(): Promise { const data = "Lorem ipsum dolor sit amet, consectetur adipis"; const seed = 42; - const config: models.GPTConfig = { + const config: GPTConfig = { modelType: "gpt-nano", lr: 0.01, maxIter: 50, @@ -26,7 +26,7 @@ async function main(): Promise { .repeat() .batch(8); - const model = new models.GPT(config); + const model = new GPT(config); for await (const logs of model.train(tokenDataset, undefined)) { console.log(logs); } diff --git a/discojs-node/src/hellaswag.ts b/discojs-node/src/hellaswag.ts index 77aa23998..04f5d6a6a 100644 --- a/discojs-node/src/hellaswag.ts +++ b/discojs-node/src/hellaswag.ts @@ -2,7 +2,11 @@ import path from "node:path"; import fetch from "node-fetch"; import fs from "node:fs/promises"; -import { models } from "@epfml/discojs"; +import { + HELLASWAG_URL, + HellaSwagExample, + HellaSwagDataset, +} from "@epfml/discojs"; import { dirname } from "path"; import { fileURLToPath } from "url"; @@ -17,7 +21,7 @@ const hellaswag_filepath = path.join(DATASET_DIR, "hellaswag_val.jsonl"); * @param limit - Maximum number of examples to load (-1 means all) * @returns A HellaSwagDataset containing the examples. */ -export async function load(limit = -1): Promise { +export async function load(limit = -1): Promise { let text: string; try { // Reads the file if it exists locally @@ -25,10 +29,10 @@ export async function load(limit = -1): Promise { } catch { console.log("Downloading the Hellaswag benchmark"); // Otherwise fetch it - const response = await fetch(models.HELLASWAG_URL); + const response = await fetch(HELLASWAG_URL); if (!response.ok) { throw new Error( - `Failed to fetch dataset from ${models.HELLASWAG_URL}: ${response.statusText}`, + `Failed to fetch dataset from ${HELLASWAG_URL}: ${response.statusText}`, ); } @@ -39,14 +43,14 @@ export async function load(limit = -1): Promise { const lines = text.split("\n"); - const dataset: models.HellaSwagDataset = []; + const dataset: HellaSwagDataset = []; let count = 0; for (const line of lines) { if (line.trim().length === 0) continue; if (limit !== -1 && count >= limit) break; try { - const data = JSON.parse(line.trim()) as models.HellaSwagExample; + const data = JSON.parse(line.trim()) as HellaSwagExample; dataset.push(data); count++; } catch (e) { diff --git a/discojs-node/src/model_loader.ts b/discojs-node/src/model_loader.ts index 1da84def0..879866dd5 100644 --- a/discojs-node/src/model_loader.ts +++ b/discojs-node/src/model_loader.ts @@ -1,14 +1,14 @@ import fs from "node:fs/promises"; -import type { models, DataType } from "@epfml/discojs"; -import { serialization } from "@epfml/discojs"; +import type { Model, DataType } from "@epfml/discojs"; +import { modelEncode, modelDecode } from "@epfml/discojs"; export async function saveModelToDisk( - model: models.Model, + model: Model, modelFolder: string, modelFileName: string, ): Promise { - const encoded = await serialization.model.encode(model); + const encoded = await modelEncode(model); await fs.mkdir(modelFolder, { recursive: true }); await fs.writeFile(`${modelFolder}/${modelFileName}`, encoded); @@ -16,8 +16,8 @@ export async function saveModelToDisk( export async function loadModelFromDisk( modelPath: string, -): Promise> { +): Promise> { const content = await fs.readFile(modelPath); - return await serialization.model.decode(content); + return await modelDecode(content); } diff --git a/discojs-web/src/hellaswag.ts b/discojs-web/src/hellaswag.ts index da96aac95..edefc6edb 100644 --- a/discojs-web/src/hellaswag.ts +++ b/discojs-web/src/hellaswag.ts @@ -1,4 +1,8 @@ -import { models } from "@epfml/discojs"; +import { + HELLASWAG_URL, + HellaSwagExample, + HellaSwagDataset, +} from "@epfml/discojs"; /** * Loads the HellaSwag dataset from the remote URL in the browser @@ -6,25 +10,25 @@ import { models } from "@epfml/discojs"; * @param limit - Maximum number of examples to load (-1 means all) * @returns A HellaSwagDataset containing the examples */ -export async function load(limit = -1): Promise { - const response = await fetch(models.HELLASWAG_URL); +export async function load(limit = -1): Promise { + const response = await fetch(HELLASWAG_URL); if (!response.ok) { throw new Error( - `Failed to fetch dataset from ${models.HELLASWAG_URL}: ${response.statusText}`, + `Failed to fetch dataset from ${HELLASWAG_URL}: ${response.statusText}`, ); } const text = await response.text(); const lines = text.split("\n"); - const dataset: models.HellaSwagDataset = []; + const dataset: HellaSwagDataset = []; let count = 0; for (const line of lines) { if (line.trim().length === 0) continue; if (limit !== -1 && count >= limit) break; try { - const data = JSON.parse(line.trim()) as models.HellaSwagExample; + const data = JSON.parse(line.trim()) as HellaSwagExample; dataset.push(data); count++; } catch (e) { diff --git a/discojs/src/aggregator/aggregator.ts b/discojs/src/aggregator/aggregator.ts index 404d26886..a48f88205 100644 --- a/discojs/src/aggregator/aggregator.ts +++ b/discojs/src/aggregator/aggregator.ts @@ -2,7 +2,7 @@ import createDebug from "debug"; import { Map, Set } from "immutable"; import type { WeightsContainer } from "#weights/index"; -import type * as client from "#client/index"; +import type { NodeID } from "#client/types"; import { EventEmitter } from "#utils/event_emitter"; @@ -28,14 +28,14 @@ export abstract class Aggregator extends EventEmitter<{ * Contains the ids of all active nodes, i.e. members of the aggregation group at * a given round. It is a subset of all the nodes available in the network. */ - protected _nodes: Set; + protected _nodes: Set; /** * Contains the contributions received from active nodes, accessible by node id. * It defines the effective aggregation group, which is possibly a subset * of all active nodes, depending on the aggregation scheme. */ // communication round -> NodeID -> WeightsContainer - protected contributions: Map>; + protected contributions: Map>; /** * The current aggregation round, used for assessing whether a node contribution is recent enough @@ -89,7 +89,7 @@ export abstract class Aggregator extends EventEmitter<{ * @param contribution The node's contribution */ add( - nodeId: client.NodeID, + nodeId: NodeID, contribution: WeightsContainer, aggregationRound: number, communicationRound?: number, @@ -126,7 +126,7 @@ export abstract class Aggregator extends EventEmitter<{ // Abstract method to be implemented by subclasses // Handles logging and adding the contribution to the list of the current round's contributions protected abstract _add( - nodeId: client.NodeID, + nodeId: NodeID, contribution: WeightsContainer, communicationRound?: number, ): void; @@ -138,7 +138,7 @@ export abstract class Aggregator extends EventEmitter<{ * @param nodeId the node id of the contribution to be added * @param round the aggregation round of the contribution to be added */ - isValidContribution(nodeId: client.NodeID, round: number): boolean { + isValidContribution(nodeId: NodeID, round: number): boolean { if (!this.nodes.has(nodeId)) { debug("Contribution rejected because node id is not registered"); return false; @@ -173,7 +173,7 @@ export abstract class Aggregator extends EventEmitter<{ * @param step The aggregation step * @param from The node which triggered the logging message */ - log(step: AggregationStep, from?: client.NodeID): void { + log(step: AggregationStep, from?: NodeID): void { switch (step) { case AggregationStep.ADD: debug( @@ -207,7 +207,7 @@ export abstract class Aggregator extends EventEmitter<{ * @param nodeId The node to be added * @returns True is the node wasn't already in the list of nodes, False if already included */ - registerNode(nodeId: client.NodeID): boolean { + registerNode(nodeId: NodeID): boolean { if (!this.nodes.has(nodeId)) { this._nodes = this._nodes.add(nodeId); return true; @@ -219,7 +219,7 @@ export abstract class Aggregator extends EventEmitter<{ * Remove a node's id from the set of active nodes. * @param nodeId The node to be removed */ - removeNode(nodeId: client.NodeID): void { + removeNode(nodeId: NodeID): void { this._nodes = this._nodes.delete(nodeId); } @@ -229,7 +229,7 @@ export abstract class Aggregator extends EventEmitter<{ * during this aggregation round. * @param nodeIds The new set of nodes */ - setNodes(nodeIds: Set): void { + setNodes(nodeIds: Set): void { this._nodes = nodeIds; } @@ -248,16 +248,14 @@ export abstract class Aggregator extends EventEmitter<{ * Constructs the payloads sent to other nodes as contribution. * @param base Object from which the payload is computed */ - abstract makePayloads( - base: WeightsContainer, - ): Map; + abstract makePayloads(base: WeightsContainer): Map; abstract isFull(): boolean; /** * The set of node ids, representing our neighbors within the network. */ - get nodes(): Set { + get nodes(): Set { return this._nodes; } diff --git a/discojs/src/aggregator/byzantine.ts b/discojs/src/aggregator/byzantine.ts index 0db000adc..a86d7e67d 100644 --- a/discojs/src/aggregator/byzantine.ts +++ b/discojs/src/aggregator/byzantine.ts @@ -1,11 +1,12 @@ import { Map } from "immutable"; import * as tf from "@tensorflow/tfjs"; -import { AggregationStep } from "./aggregator.js"; -import { MultiRoundAggregator, ThresholdType } from "./multiround.js"; -import { aggregation } from "#weights/index"; import type { WeightsContainer } from "#weights/index"; -import type * as client from "#client/index"; +import type { NodeID } from "#client/types"; +import { avg } from "#weights/index"; + +import { AggregationStep } from "#aggregator/aggregator"; +import { MultiRoundAggregator, ThresholdType } from "#aggregator/multiround"; /** * Byzantine-robust aggregator using Centered Clipping (CC), based on the @@ -43,7 +44,7 @@ export class ByzantineRobustAggregator extends MultiRoundAggregator { private readonly clippingRadius: number; private readonly maxIterations: number; private readonly beta: number; - private historyMomentums: Map = Map(); + private historyMomentums: Map = Map(); private prevAggregate: WeightsContainer | null = null; /** @@ -88,7 +89,7 @@ export class ByzantineRobustAggregator extends MultiRoundAggregator { this.beta = beta; } - override _add(nodeId: client.NodeID, contribution: WeightsContainer): void { + override _add(nodeId: NodeID, contribution: WeightsContainer): void { this.log( this.contributions.hasIn([0, nodeId]) ? AggregationStep.UPDATE @@ -116,7 +117,7 @@ export class ByzantineRobustAggregator extends MultiRoundAggregator { // If clipping radius is infinite, fall back to simple mean if (!isFinite(this.clippingRadius)) { - return aggregation.avg(currentContributions.values()); + return avg(currentContributions.values()); } // Step 1: Initialize v using previous aggregate or mean of contributions @@ -124,7 +125,7 @@ export class ByzantineRobustAggregator extends MultiRoundAggregator { if (this.prevAggregate) { v = this.prevAggregate.map((t) => tf.clone(t)); // Clone to avoid in-place modifications } else { - v = aggregation.avg(currentContributions.values()); + v = avg(currentContributions.values()); } const eps = tf.scalar(1e-12); @@ -153,7 +154,7 @@ export class ByzantineRobustAggregator extends MultiRoundAggregator { }, ); - const avgClip = aggregation.avg(clippedDiffs); + const avgClip = avg(clippedDiffs); const newV = v.add(avgClip); clippedDiffs.forEach((d) => d.dispose()); @@ -171,7 +172,7 @@ export class ByzantineRobustAggregator extends MultiRoundAggregator { override makePayloads( weights: WeightsContainer, - ): Map { + ): Map { // Communicate our local weights to every other node, be it a peer or a server return this.nodes.toMap().map(() => weights); } diff --git a/discojs/src/aggregator/get.ts b/discojs/src/aggregator/get.ts index e9c42ff91..0b2662290 100644 --- a/discojs/src/aggregator/get.ts +++ b/discojs/src/aggregator/get.ts @@ -1,7 +1,9 @@ import type { DataType, Network } from "#types/index"; import type { Task } from "#task/index"; -import * as aggregator from "#aggregator/index"; -import { ByzantineRobustAggregator } from "./byzantine.js"; +import { Aggregator } from "#aggregator/aggregator"; +import { MeanAggregator } from "#aggregator/mean"; +import { SecureAggregator } from "#aggregator/secure"; +import { ByzantineRobustAggregator } from "#aggregator/byzantine"; type AggregatorOptions = Partial<{ scheme: Task["trainingInformation"]["scheme"]; // if undefined, fallback on task.trainingInformation.scheme @@ -31,7 +33,7 @@ type AggregatorOptions = Partial<{ export function getAggregator( task: Task, options: AggregatorOptions = {}, -): aggregator.Aggregator { +): Aggregator { const scheme = options.scheme ?? task.trainingInformation.scheme; // If options are not specified, we default to expecting a contribution from all peers, so we set the threshold to 100% @@ -66,7 +68,7 @@ export function getAggregator( ); } case "mean": - return new aggregator.MeanAggregator( + return new MeanAggregator( networkOptions.roundCutOff, networkOptions.threshold, networkOptions.thresholdType, @@ -77,8 +79,6 @@ export function getAggregator( "secure aggregation is currently supported for decentralized only", ); } - return new aggregator.SecureAggregator( - task.trainingInformation.maxShareValue, - ); + return new SecureAggregator(task.trainingInformation.maxShareValue); } } diff --git a/discojs/src/aggregator/mean.ts b/discojs/src/aggregator/mean.ts index 58cedc82d..ac2dec8ae 100644 --- a/discojs/src/aggregator/mean.ts +++ b/discojs/src/aggregator/mean.ts @@ -1,9 +1,11 @@ import type { Map } from "immutable"; -import { AggregationStep } from "./aggregator.js"; -import { MultiRoundAggregator, ThresholdType } from "./multiround.js"; + import type { WeightsContainer } from "#weights/index"; -import type * as client from "#client/index"; -import { aggregation } from "#weights/index"; +import type { NodeID } from "#client/types"; +import { avg } from "#weights/index"; + +import { AggregationStep } from "#aggregator/aggregator"; +import { MultiRoundAggregator, ThresholdType } from "#aggregator/multiround"; /** * Mean aggregator whose aggregation step consists in computing the mean of the received weights. @@ -19,7 +21,7 @@ export class MeanAggregator extends MultiRoundAggregator { super(roundCutoff, threshold, thresholdType); } - override _add(nodeId: client.NodeID, contribution: WeightsContainer): void { + override _add(nodeId: NodeID, contribution: WeightsContainer): void { this.log( this.contributions.hasIn([0, nodeId]) ? AggregationStep.UPDATE @@ -36,13 +38,13 @@ export class MeanAggregator extends MultiRoundAggregator { this.log(AggregationStep.AGGREGATE); - const result = aggregation.avg(currentContributions.values()); + const result = avg(currentContributions.values()); return result; } override makePayloads( weights: WeightsContainer, - ): Map { + ): Map { // Communicate our local weights to every other node, be it a peer or a server return this.nodes.toMap().map(() => weights); } diff --git a/discojs/src/aggregator/percentile_clipping.ts b/discojs/src/aggregator/percentile_clipping.ts index 877a4bae3..2093433ed 100644 --- a/discojs/src/aggregator/percentile_clipping.ts +++ b/discojs/src/aggregator/percentile_clipping.ts @@ -1,9 +1,9 @@ import { Map } from "immutable"; import * as tf from "@tensorflow/tfjs"; -import { AggregationStep } from "./aggregator.js"; -import { MultiRoundAggregator, ThresholdType } from "./multiround.js"; -import { WeightsContainer, client } from "../index.js"; -import { aggregation } from "../index.js"; +import { AggregationStep } from "#aggregator/aggregator"; +import { MultiRoundAggregator, ThresholdType } from "#aggregator/multiround"; +import { NodeID } from "#client/index.js"; +import { WeightsContainer, avg } from "#weights/index"; /** * Percentile-based clipping aggregator. @@ -51,7 +51,7 @@ export class PercentileClippingAggregator extends MultiRoundAggregator { this.tauPercentile = tauPercentile; } - override _add(nodeId: client.NodeID, contribution: WeightsContainer): void { + override _add(nodeId: NodeID, contribution: WeightsContainer): void { this.log( this.contributions.hasIn([0, nodeId]) ? AggregationStep.UPDATE @@ -74,9 +74,9 @@ export class PercentileClippingAggregator extends MultiRoundAggregator { if (this.prevAggregate) { centerReference = this.prevAggregate.map((t) => tf.clone(t)); } else { - centerReference = aggregation - .avg(currentContributions.values()) - .map((t) => tf.clone(t)); + centerReference = avg(currentContributions.values()).map((t) => + tf.clone(t), + ); } // Step 2: Center the weights with respect to the reference @@ -103,7 +103,7 @@ export class PercentileClippingAggregator extends MultiRoundAggregator { centeredWeights.forEach((w) => w.dispose()); // Step 6: Average the clipped weights and add back the reference - const clippedAvg = aggregation.avg(clippedWeights); + const clippedAvg = avg(clippedWeights); const result = centerReference.add(clippedAvg); centerReference.dispose(); @@ -134,7 +134,7 @@ export class PercentileClippingAggregator extends MultiRoundAggregator { override makePayloads( weights: WeightsContainer, - ): Map { + ): Map { return this.nodes.toMap().map(() => weights); } } diff --git a/discojs/src/aggregator/secure.ts b/discojs/src/aggregator/secure.ts index 9f7abc1e0..949fe8e8b 100644 --- a/discojs/src/aggregator/secure.ts +++ b/discojs/src/aggregator/secure.ts @@ -1,10 +1,11 @@ import { Map, List, Range } from "immutable"; import * as tf from "@tensorflow/tfjs"; -import { AggregationStep, Aggregator } from "./aggregator.js"; import type { WeightsContainer } from "#weights/index"; -import { aggregation } from "#weights/index"; -import type * as client from "#client/index"; +import type { NodeID } from "#client/types"; +import { sum, avg } from "#weights/index"; + +import { AggregationStep, Aggregator } from "#aggregator/aggregator"; /** * Aggregator implementing secure multi-party computation for decentralized learning. @@ -28,7 +29,7 @@ export class SecureAggregator extends Aggregator { if (currentContributions === undefined) throw new Error("aggregating without any contribution"); - return aggregation.sum(currentContributions.values()); + return sum(currentContributions.values()); } // Average the received partial sums case 1: { @@ -36,7 +37,7 @@ export class SecureAggregator extends Aggregator { if (currentContributions === undefined) throw new Error("aggregating without any contribution"); - return aggregation.avg(currentContributions.values()); + return avg(currentContributions.values()); } default: throw new Error("communication round is out of bounds"); @@ -44,7 +45,7 @@ export class SecureAggregator extends Aggregator { } _add( - nodeId: client.NodeID, + nodeId: NodeID, contribution: WeightsContainer, communicationRound: number, ): void { @@ -78,7 +79,7 @@ export class SecureAggregator extends Aggregator { override makePayloads( weights: WeightsContainer, - ): Map { + ): Map { switch (this.communicationRound) { case 0: { const shares = this.generateAllShares(weights); @@ -106,7 +107,7 @@ export class SecureAggregator extends Aggregator { .toList(); // The last share completes the sum - return shares.push(secret.sub(aggregation.sum(shares))); + return shares.push(secret.sub(sum(shares))); } /** diff --git a/discojs/src/aggregator/secure_history.ts b/discojs/src/aggregator/secure_history.ts index 71c77b340..09b4cfef0 100644 --- a/discojs/src/aggregator/secure_history.ts +++ b/discojs/src/aggregator/secure_history.ts @@ -1,6 +1,6 @@ import type { WeightsContainer } from "../index.js"; import { SecureAggregator } from "./secure.js"; -import { aggregation } from "../index.js"; +import { avg } from "../index.js"; /** * Aggregator that implements secure multi-party computation with history-based momentum smoothing. @@ -43,15 +43,16 @@ export class SecureHistoryAggregator extends SecureAggregator { if (!currentContributions) throw new Error("aggregating without any contribution"); - const avg = aggregation.avg(currentContributions.values()); + const contribAvg = avg(currentContributions.values()); if (this.prevAggregate === null) { - this.prevAggregate = avg; - return avg; + this.prevAggregate = contribAvg; + return contribAvg; } - const updatedMomentum = this.prevAggregate.mapWith(avg, (prevT, currT) => - prevT.mul(this.beta).add(currT.mul(1 - this.beta)), + const updatedMomentum = this.prevAggregate.mapWith( + contribAvg, + (prevT, currT) => prevT.mul(this.beta).add(currT.mul(1 - this.beta)), ); // Dispose old tensors to avoid memory leaks diff --git a/discojs/src/client/client.ts b/discojs/src/client/client.ts index 667c58d19..1d23b559f 100644 --- a/discojs/src/client/client.ts +++ b/discojs/src/client/client.ts @@ -1,14 +1,14 @@ import createDebug from "debug"; import type { Model } from "#models/index"; -import type { RoundStatus } from "#training/index"; import type { DataType, Network } from "#types/index"; import type { Task } from "#task/index"; import type { WeightsContainer } from "#weights/index"; import type { Aggregator } from "#aggregator/index"; +import type { RoundStatus } from "#training/types"; import { EventEmitter } from "#utils/event_emitter"; -import * as serialization from "#serialization/index"; +import { modelDecode } from "#serialization/index"; import type { EventConnection } from "#client/event_connection"; import type { NodeID } from "#client/types"; @@ -200,7 +200,7 @@ export abstract class Client extends EventEmitter<{ if (!response.ok) throw new Error(`fetch: HTTP status ${response.status}`); const encoded = new Uint8Array(await response.arrayBuffer()); - return await serialization.model.decode(encoded); + return await modelDecode(encoded); } /** diff --git a/discojs/src/client/decentralized/decentralized_client.ts b/discojs/src/client/decentralized/decentralized_client.ts index bd71aece2..e8511eab0 100644 --- a/discojs/src/client/decentralized/decentralized_client.ts +++ b/discojs/src/client/decentralized/decentralized_client.ts @@ -4,9 +4,9 @@ import { Map, Set } from "immutable"; import type { WeightsContainer } from "#weights/index"; import type { Model } from "#models/index"; import type { DataType } from "#types/index"; -import * as serialization from "#serialization/index"; +import { weightsEncode, weightsDecode } from "#serialization/index"; import { Client, shortenId } from "#client/client"; -import { type NodeID } from "#client/index"; +import type { NodeID } from "#client/types"; import { MType, type ClientConnected } from "#client/mtype"; import { timeout } from "#client/utils"; import { @@ -256,7 +256,7 @@ export class DecentralizedClient extends Client<"decentralized"> { 60_000, "Timeout waiting for a contribution from peer " + peerId, ); - const decoded = serialization.weights.decode(message.payload); + const decoded = weightsDecode(message.payload); if ( !this.aggregator.isValidContribution( @@ -337,7 +337,7 @@ export class DecentralizedClient extends Client<"decentralized"> { // Send our payload to each peer const peer = connections.get(id); if (peer !== undefined) { - const encoded = await serialization.weights.encode(payload); + const encoded = await weightsEncode(payload); const msg: messages.PeerMessage = { type: MType.Payload, peer: id, diff --git a/discojs/src/client/federated/federated_client.ts b/discojs/src/client/federated/federated_client.ts index be2d08155..d8812a622 100644 --- a/discojs/src/client/federated/federated_client.ts +++ b/discojs/src/client/federated/federated_client.ts @@ -3,7 +3,7 @@ import createDebug from "debug"; import type { Model } from "#models/index"; import type { DataType } from "#types/index"; import type { WeightsContainer } from "#weights/index"; -import * as serialization from "#serialization/index"; +import { weightsEncode, weightsDecode } from "#serialization/index"; import { Client, shortenId } from "#client/client"; import { MType, type ClientConnected } from "#client/mtype"; import { waitMessage, WebSocketServer } from "#client/event_connection"; @@ -87,7 +87,7 @@ export class FederatedClient extends Client<"federated"> { `[${shortenId(this.ownId)}] upon connecting, wait for participant flag %o`, this.waitingForMoreParticipants, ); - model.weights = serialization.weights.decode(payload); + model.weights = weightsDecode(payload); return model; } @@ -142,7 +142,7 @@ export class FederatedClient extends Client<"federated"> { throw new Error("aggregator didn't make a payload for the server"); const msg: messages.SendPayload = { type: MType.SendPayload, - payload: await serialization.weights.encode(payloadToServer), + payload: await weightsEncode(payloadToServer), round: this.aggregator.round, }; @@ -161,7 +161,7 @@ export class FederatedClient extends Client<"federated"> { nbOfParticipants, } = await waitMessage(this.server, MType.ReceiveServerPayload); // Wait indefinitely for the server update this.nbOfParticipants = nbOfParticipants; // Save the current participants - const serverResult = serialization.weights.decode(payloadFromServer); + const serverResult = weightsDecode(payloadFromServer); this.aggregator.setRound(serverRound); return serverResult; diff --git a/discojs/src/client/get_client.ts b/discojs/src/client/get_client.ts index 9678c8dbe..7ee1455aa 100644 --- a/discojs/src/client/get_client.ts +++ b/discojs/src/client/get_client.ts @@ -4,7 +4,7 @@ import type * as aggregator from "#aggregator/index"; // import * as clients from "#client/index"; import { LocalClient } from "#client/local_client"; -import type { Client } from "#client/client"; +import { Client } from "#client/client"; import { DecentralizedClient } from "#client/decentralized/decentralized_client"; import { FederatedClient } from "#client/federated/federated_client"; diff --git a/discojs/src/client/index.ts b/discojs/src/client/index.ts index b6b8812e9..3d97c5f72 100644 --- a/discojs/src/client/index.ts +++ b/discojs/src/client/index.ts @@ -1,12 +1,18 @@ export { Client } from "./client.js"; -export * from "./types.js"; +export type { NodeID } from "./types.js"; +export { isNodeID } from "./types.js"; -export * as aggregator from "../aggregator/index.js"; -export * as decentralized from "./decentralized/index.js"; -export * as federated from "./federated/index.js"; -export * as messages from "./messages.js"; +export { + messages as decentralizedMessages, + DecentralizedClient, +} from "./decentralized/index.js"; +export { + messages as federatedMessages, + FederatedClient, +} from "./federated/index.js"; +export { Message, NarrowMessage } from "./messages.js"; export * as mtype from "./mtype.js"; -export { getClient, timeout } from "./utils.js"; +export { getClient } from "./get_client.js"; export { LocalClient } from "./local_client.js"; diff --git a/discojs/src/client/utils.ts b/discojs/src/client/utils.ts index a2b37f11b..20a5d8e5c 100644 --- a/discojs/src/client/utils.ts +++ b/discojs/src/client/utils.ts @@ -1,13 +1,3 @@ -import type { DataType, Network } from "#types/index"; -import type { Task } from "#task/index"; -import type * as aggregator from "#aggregator/index"; - -// import * as clients from "#client/index"; -import { LocalClient } from "#client/local_client"; -import { Client } from "#client/client"; -import { DecentralizedClient } from "#client/decentralized/decentralized_client"; -import { FederatedClient } from "#client/federated/federated_client"; - // Time to wait for the others in milliseconds. const MAX_WAIT_PER_ROUND = 15_000; @@ -21,35 +11,3 @@ export async function timeout( }, ms); }); } - -export function getClient( - scheme: N | "local", - serverURL: URL, - task: Task, - aggregator: aggregator.Aggregator, -): Client { - switch (scheme) { - case "decentralized": { - const t = task as Task; - t.trainingInformation.scheme = scheme; - - return new DecentralizedClient(serverURL, t, aggregator); - } - case "federated": { - const t = task as Task; - t.trainingInformation.scheme = scheme; - - return new FederatedClient(serverURL, t, aggregator); - } - case "local": { - const t = task as Task; - t.trainingInformation.scheme = scheme; - - return new LocalClient(serverURL, t, aggregator); - } - default: { - const _: never = scheme; - throw new Error("should never happen"); - } - } -} diff --git a/discojs/src/dataset/index.ts b/discojs/src/dataset/index.ts index 3214f4050..4bb673edd 100644 --- a/discojs/src/dataset/index.ts +++ b/discojs/src/dataset/index.ts @@ -1,2 +1,3 @@ export { Dataset } from "./dataset.js"; -export * from "./types.js"; +export { Image } from "./types.js"; +export type { Batched, Tabular, Text, TokenizedText } from "./types.js"; diff --git a/discojs/src/dataset/types.ts b/discojs/src/dataset/types.ts index 915cedc71..3b2b1fbd6 100644 --- a/discojs/src/dataset/types.ts +++ b/discojs/src/dataset/types.ts @@ -1,10 +1,9 @@ import { List } from "immutable"; -import { Image } from "./image.js"; +export { Image } from "./image.js"; export type Batched = List; -export { Image }; export type Tabular = Partial>; export type Text = string; export type TokenizedText = List; diff --git a/discojs/src/index.ts b/discojs/src/index.ts index 57a4711f0..9fe4a6ff4 100644 --- a/discojs/src/index.ts +++ b/discojs/src/index.ts @@ -1,41 +1,75 @@ -export * as data from "./dataset/index.js"; -export * as serialization from "./serialization/index.js"; -export * as training from "./training/index.js"; -export * as privacy from "./privacy.js"; +export { LocalClient, getClient } from "./client/index.js"; -export * as client from "./client/index.js"; -export * as aggregator from "./aggregator/index.js"; +export { + modelEncode, + modelDecode, + weightsEncode, + weightsDecode, + serializeTaskToJSON, + deserializeTaskFromJSON, + isEncoded, + Encoded, +} from "./serialization/index.js"; + +export { MeanAggregator, getAggregator } from "./aggregator/index.js"; + +export { + NodeID, + mtype, + federatedMessages, + decentralizedMessages, +} from "./client/index.js"; + +export { WeightsContainer, avg } from "./weights/index.js"; -export { WeightsContainer, aggregation } from "./weights/index.js"; -export { Logger, ConsoleLogger } from "./logging/index.js"; export { Disco, RoundLogs, RoundStatus, SummaryLogs, } from "./training/index.js"; + export { Validator } from "./validator.js"; -export { - Model, +export type { ModelCard, ModelCardInfo, BatchLogs, + ValidationMetrics, + HellaSwagDataset, +} from "./models/index.js"; + +export { + Model, EpochLogs, Tokenizer, - ValidationMetrics, fetchModels, + GPT, + GPTConfig, + TFJS, + ONNXModel, + HELLASWAG_URL, + HellaSwagExample, + evaluate_hellaswag, } from "./models/index.js"; -export * as models from "./models/index.js"; -export * from "./task/index.js"; -export * as defaultTasks from "./default_tasks/index.js"; -export * as defaultModels from "./models/cards/index.js"; - -export * as async_iterator from "./utils/async_iterator.js"; export { EventEmitter } from "./utils/event_emitter.js"; -export * from "./dataset/index.js"; -export * from "./types/index.js"; +export { Dataset, Image, Text, Tabular } from "./dataset/index.js"; + +export { split, gather } from "./utils/async_iterator.js"; -export * as processing from "./processing/index.js"; +export { + Task, + TaskProvider, + TrainingInformation, + pushTask, + fetchTasks, +} from "./task/index.js"; + +export type { DataType, Network, DataFormat } from "./types/index.js"; + +export { extractColumn } from "./processing/index.js"; + +export * as defaultTasks from "./default_tasks/index.js"; +export { cards as defaultModels } from "./models/index.js"; diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts index 3fed7348c..47ce26990 100644 --- a/discojs/src/processing/index.ts +++ b/discojs/src/processing/index.ts @@ -1,4 +1,14 @@ /** Dataset shapers, convenient to map with */ -export * from "./image.js"; -export * from "./tabular.js"; -export * from "./processing.js"; +export { + NormalizedImage, + resize, + normalize, + removeAlpha, + expandToMulticolor, +} from "./image.js"; +export { convertToNumber, extractColumn, indexInList } from "./tabular.js"; +export { + preprocess, + preprocessWithoutLabel, + postprocess, +} from "./processing.js"; diff --git a/discojs/src/serialization/index.ts b/discojs/src/serialization/index.ts index c59aaf689..02cb1e88e 100644 --- a/discojs/src/serialization/index.ts +++ b/discojs/src/serialization/index.ts @@ -1,6 +1,9 @@ -export * as model from "./model.js"; -export * as task from "./task.js"; -export * as weights from "./weights.js"; +export { encode as modelEncode, decode as modelDecode } from "./model.js"; +export { + serializeToJSON as serializeTaskToJSON, + deserializeFromJSON as deserializeTaskFromJSON, +} from "./task.js"; +export { encode as weightsEncode, decode as weightsDecode } from "./weights.js"; export type { Encoded } from "./coder.js"; export { isEncoded } from "./coder.js"; diff --git a/discojs/src/serialization/task.ts b/discojs/src/serialization/task.ts index 935c6d79c..f89db89e1 100644 --- a/discojs/src/serialization/task.ts +++ b/discojs/src/serialization/task.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import type { DataType, Network } from "#types/index"; -import { Task } from "#task/index"; +import { Task } from "#task/task"; import { Tokenizer } from "#models/index"; import type { JSONLike } from "./json_like.js"; diff --git a/discojs/src/task/task_handler.ts b/discojs/src/task/task_handler.ts index 2c8cc8b04..7b87555b4 100644 --- a/discojs/src/task/task_handler.ts +++ b/discojs/src/task/task_handler.ts @@ -2,7 +2,12 @@ import { Map, Seq } from "immutable"; import type { DataType, Network } from "#types/index"; import type { Model } from "#models/index"; -import * as serialization from "#serialization/index"; +import { + serializeTaskToJSON, + deserializeTaskFromJSON, + modelEncode, + JSONLike, +} from "#serialization/index"; import type { Task } from "./task.js"; @@ -27,11 +32,11 @@ export async function pushTask( method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - task: serialization.task.serializeToJSON(task), + task: serializeTaskToJSON(task), model: typeof model === "string" ? model - : [...(await serialization.model.encode(model))], + : [...(await modelEncode(model))], }), }); if (!response.ok) throw new Error(`fetch: HTTP status ${response.status}`); @@ -42,7 +47,7 @@ export async function fetchTasks( ): Promise>> { const response = await fetch(urlToTasks(base)); if (!response.ok) throw new Error(`fetch: HTTP status ${response.status}`); - const json = (await response.json()) as serialization.JSONLike; + const json = (await response.json()) as JSONLike; if (!Array.isArray(json)) throw new Error("invalid tasks response: expected a JSON array"); @@ -50,11 +55,9 @@ export async function fetchTasks( try { return Map( - Seq( - await Promise.all( - arr.map((t) => serialization.task.deserializeFromJSON(t)), - ), - ).map((t) => [t.id, t]), + Seq(await Promise.all(arr.map((t) => deserializeTaskFromJSON(t)))).map( + (t) => [t.id, t], + ), ); } catch (cause) { throw new Error("invalid tasks response: unable to parse all tasks", { diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index f0686394e..4672c5796 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -15,7 +15,8 @@ import * as clients from "#client/index"; import * as processing from "#processing/index"; import * as async_iterator from "#utils/async_iterator"; -import { RoundLogs, Trainer } from "./trainer.js"; +import { RoundLogs, Trainer } from "#training/trainer"; +import type { RoundStatus, SummaryLogs } from "#training/types"; interface DiscoConfig { scheme: N; @@ -31,25 +32,6 @@ interface DiscoConfig { preprocessOnce: boolean; } -export type SummaryLogs = { - round: number; - epoch: number; - trainingLoss: number; - trainingAccuracy: number; - peakMemory: number; - epochTime: number; - roundValidationLoss?: number; - roundValidationAccuracy?: number; - validationLoss?: number; - validationAccuracy?: number; -}; - -export type RoundStatus = - | "not enough participants" // Server notification to wait for more participants - | "updating model" // fetching/aggregating local updates into a global model - | "local training" // Training the model locally - | "connecting to peers"; // for decentralized only, fetch the server's list of participating peers - function buildSummaryLog( roundNum: number, epochNum: number, diff --git a/discojs/src/training/index.ts b/discojs/src/training/index.ts index 1c10a3fef..b58c14280 100644 --- a/discojs/src/training/index.ts +++ b/discojs/src/training/index.ts @@ -1,2 +1,3 @@ -export { Disco, RoundStatus, SummaryLogs } from "./disco.js"; +export { Disco } from "./disco.js"; +export type { RoundStatus, SummaryLogs } from "./types.js"; export { RoundLogs, Trainer } from "./trainer.js"; diff --git a/discojs/src/training/trainer.ts b/discojs/src/training/trainer.ts index 56bc3167d..ee7835399 100644 --- a/discojs/src/training/trainer.ts +++ b/discojs/src/training/trainer.ts @@ -7,7 +7,7 @@ import type { Batched } from "#dataset/index"; import type { Task } from "#task/index"; import { Dataset } from "#dataset/index"; -import { BatchLogs, EpochLogs, ValidationMetrics } from "#models/index"; +import type { BatchLogs, EpochLogs, ValidationMetrics } from "#models/index"; import { WeightsContainer } from "#weights/index"; import { Client } from "#client/index"; diff --git a/discojs/src/training/types.ts b/discojs/src/training/types.ts index e4ab2568a..a6e4290e6 100644 --- a/discojs/src/training/types.ts +++ b/discojs/src/training/types.ts @@ -1,3 +1,22 @@ import { List } from "immutable"; export type WeightNormHistory = List>; + +export type SummaryLogs = { + round: number; + epoch: number; + trainingLoss: number; + trainingAccuracy: number; + peakMemory: number; + epochTime: number; + roundValidationLoss?: number; + roundValidationAccuracy?: number; + validationLoss?: number; + validationAccuracy?: number; +}; + +export type RoundStatus = + | "not enough participants" // Server notification to wait for more participants + | "updating model" // fetching/aggregating local updates into a global model + | "local training" // Training the model locally + | "connecting to peers"; // for decentralized only, fetch the server's list of participating peers diff --git a/discojs/src/weights/index.ts b/discojs/src/weights/index.ts index b44515510..baaf74fa8 100644 --- a/discojs/src/weights/index.ts +++ b/discojs/src/weights/index.ts @@ -1,2 +1,2 @@ export { WeightsContainer } from "./weights_container.js"; -export * as aggregation from "./aggregation.js"; +export { sum, diff, avg } from "./aggregation.js"; diff --git a/docs/examples/custom_task.ts b/docs/examples/custom_task.ts index feb3f8e29..7e8074a8d 100644 --- a/docs/examples/custom_task.ts +++ b/docs/examples/custom_task.ts @@ -1,7 +1,7 @@ import tf from "@tensorflow/tfjs-node"; import type { TaskProvider, ModelCard } from "@epfml/discojs"; -import { defaultTasks, defaultModels, models } from "@epfml/discojs"; +import { defaultTasks, defaultModels, TFJS } from "@epfml/discojs"; import { Server as DiscoServer } from "server"; // Define your own model card @@ -32,7 +32,7 @@ const customModelCard: ModelCard<"tabular"> = { metrics: ["accuracy"], }); - return Promise.resolve(new models.TFJS("tabular", model)); + return Promise.resolve(new TFJS("tabular", model)); }, }; diff --git a/docs/examples/wikitext.ts b/docs/examples/wikitext.ts index 6bf3e06e8..aaf6d6a0e 100644 --- a/docs/examples/wikitext.ts +++ b/docs/examples/wikitext.ts @@ -1,6 +1,6 @@ import "@tensorflow/tfjs-node"; -import { Disco, fetchTasks, models, Task } from "@epfml/discojs"; +import { Disco, fetchTasks, GPT, Task } from "@epfml/discojs"; import { saveModelToDisk, loadModelFromDisk, @@ -37,14 +37,12 @@ async function main(): Promise { await disco.trainFully(dataset); // Get the model and save the trained model - model = disco.trainer.model as models.GPT; + model = disco.trainer.model as GPT; await saveModelToDisk(model, modelFolder, modelFileName); await disco.close(); } else { // Load the trained model - model = (await loadModelFromDisk( - `${modelFolder}/${modelFileName}`, - )) as models.GPT; + model = (await loadModelFromDisk(`${modelFolder}/${modelFileName}`)) as GPT; } // Preprocess prompt diff --git a/onnx-converter/src/convert_onnx.ts b/onnx-converter/src/convert_onnx.ts index fb00d0edd..505860bac 100644 --- a/onnx-converter/src/convert_onnx.ts +++ b/onnx-converter/src/convert_onnx.ts @@ -3,7 +3,7 @@ import { Map, Range } from "immutable"; import fsPromise from "node:fs/promises"; import * as tf from "@tensorflow/tfjs-node"; -import { models, serialization } from "@epfml/discojs"; +import { GPT, modelEncode } from "@epfml/discojs"; const OUTPUT_FILENAME = "model.json"; const GPT2_N_LAYER = 12; @@ -33,7 +33,7 @@ async function main() { // Init empty TF.js model // Context length value from https://huggingface.co/Xenova/gpt2/blob/main/config.json - const gptModel = new models.GPT({ modelType: "gpt2", contextLength: 1024 }); + const gptModel = new GPT({ modelType: "gpt2", contextLength: 1024 }); if (gptModel.config.nLayer != GPT2_N_LAYER) throw new Error( `ONNX conversion only supports GPT-2 with 12 layers, instead found ${gptModel.config.nLayer}.`, @@ -82,7 +82,7 @@ async function main() { gptLayersModel.setWeights(finalWeights); // shape or transpose mismatch will throw here - const encoded = await serialization.model.encode(gptModel); + const encoded = await modelEncode(gptModel); await fsPromise.writeFile(OUTPUT_FILENAME, encoded); console.log(`GPT-TFJS model saved to ${OUTPUT_FILENAME}`); } diff --git a/server/src/controllers/decentralized_controller.ts b/server/src/controllers/decentralized_controller.ts index 08fb9983b..f35672392 100644 --- a/server/src/controllers/decentralized_controller.ts +++ b/server/src/controllers/decentralized_controller.ts @@ -4,12 +4,13 @@ import * as msgpack from "@msgpack/msgpack"; import type WebSocket from "ws"; import { Map } from "immutable"; -import { client, DataType } from "@epfml/discojs"; +import type { DataType, NodeID } from "@epfml/discojs"; +import { mtype, decentralizedMessages } from "@epfml/discojs"; import { TrainingController } from "./training_controller.js"; -import messages = client.decentralized.messages; -import MessageTypes = client.mtype.MType; +import messages = decentralizedMessages; +import MessageTypes = mtype.MType; const debug = createDebug("server:controllers:decentralized"); @@ -20,7 +21,7 @@ export class DecentralizedController< // The boolean value indicates if the node is ready to exchange weight updates (i.e. // the node has already sent a PeerIsReady message) // We wait for all peers to be ready to exchange weight updates - #roundPeers = Map(); + #roundPeers = Map(); #aggregationRound = 0; handle(ws: WebSocket): void { @@ -143,7 +144,7 @@ export class DecentralizedController< debug("Sending peer list to: %o", id.slice(0, 4)); const encoded = msgpack.encode(readyPeerIDs); - return [id, encoded] as [client.NodeID, Buffer]; + return [id, encoded] as [NodeID, Buffer]; }) .map(([id, encoded]) => { const conn = this.connections.get(id); diff --git a/server/src/controllers/federated_controller.ts b/server/src/controllers/federated_controller.ts index 9c9784b58..e8e4871bf 100644 --- a/server/src/controllers/federated_controller.ts +++ b/server/src/controllers/federated_controller.ts @@ -5,15 +5,17 @@ import * as msgpack from "@msgpack/msgpack"; import type { DataType, Task } from "@epfml/discojs"; import { - aggregator as aggregators, - client, - serialization, + mtype, + federatedMessages, + Encoded, + weightsEncode, + weightsDecode, + MeanAggregator, } from "@epfml/discojs"; import { TrainingController } from "./training_controller.js"; -import MessageTypes = client.mtype.MType; -import FederatedMessages = client.federated.messages; +import MessageTypes = mtype.MType; const debug = createDebug("server:controllers:federated"); @@ -25,25 +27,24 @@ export class FederatedController extends TrainingController< * Aggregators for each hosted task. By default the server waits for 100% of the nodes to send their contributions before aggregating the updates */ - #aggregator = new aggregators.MeanAggregator(undefined, 1, "relative"); + #aggregator = new MeanAggregator(undefined, 1, "relative"); /** * The most up to date global weights. The model weights are already serialized and * can be sent to participants, before starting training, or when joining mid-training * or staled participants */ - #latestGlobalWeights: serialization.Encoded; + #latestGlobalWeights: Encoded; constructor( task: Task, - private readonly initialWeights: serialization.Encoded, + private readonly initialWeights: Encoded, ) { super(task); this.#latestGlobalWeights = this.initialWeights; // Save the latest weight updates to be able to send it to new or outdated clients this.#aggregator.on("aggregation", async (weightUpdate) => { - this.#latestGlobalWeights = - await serialization.weights.encode(weightUpdate); + this.#latestGlobalWeights = await weightsEncode(weightUpdate); }); } @@ -71,7 +72,7 @@ export class FederatedController extends TrainingController< // Setup callbacks triggered upon receiving the different client messages ws.on("message", (data: Buffer) => { const msg: unknown = msgpack.decode(data); - if (!FederatedMessages.isMessageFederated(msg)) { + if (!federatedMessages.isMessageFederated(msg)) { debug("invalid federated message received on WebSocket: %o", msg); return; // TODO send back error } @@ -87,7 +88,7 @@ export class FederatedController extends TrainingController< debug(`client [%s] joined ${this.task.id}`, shortId); this.connections = this.connections.set(clientId, ws); // add the new client - const msg: FederatedMessages.NewFederatedNodeInfo = { + const msg: federatedMessages.NewFederatedNodeInfo = { type: MessageTypes.NewFederatedNodeInfo, id: clientId, waitForMoreParticipants: @@ -107,7 +108,7 @@ export class FederatedController extends TrainingController< case MessageTypes.SendPayload: { const { payload, round } = msg; if (this.#aggregator.isValidContribution(clientId, round)) { - const weights = serialization.weights.decode(payload); + const weights = weightsDecode(payload); // Create a callback to send the aggregated weight to the client // when enough contributions are received @@ -117,10 +118,10 @@ export class FederatedController extends TrainingController< this.#aggregator.round, shortId, ); - const msg: FederatedMessages.ReceiveServerPayload = { + const msg: federatedMessages.ReceiveServerPayload = { type: MessageTypes.ReceiveServerPayload, round: this.#aggregator.round, // send the current round number after aggregation - payload: await serialization.weights.encode(weightUpdate), + payload: await weightsEncode(weightUpdate), nbOfParticipants: this.connections.size, }; ws.send(msgpack.encode(msg)); @@ -142,7 +143,7 @@ export class FederatedController extends TrainingController< // no latest model at the first round if (this.#latestGlobalWeights === undefined) return; - const msg: FederatedMessages.ReceiveServerPayload = { + const msg: federatedMessages.ReceiveServerPayload = { type: MessageTypes.ReceiveServerPayload, round: this.#aggregator.round - 1, // send the model from the previous round payload: this.#latestGlobalWeights, @@ -165,11 +166,7 @@ export class FederatedController extends TrainingController< // Reset the training session when all participants left if (this.connections.size === 0) { debug("All participants left. Resetting the training session"); - this.#aggregator = new aggregators.MeanAggregator( - undefined, - 1, - "relative", - ); + this.#aggregator = new MeanAggregator(undefined, 1, "relative"); this.#latestGlobalWeights = this.initialWeights; } diff --git a/server/src/controllers/training_controller.ts b/server/src/controllers/training_controller.ts index 9e9c880ec..60af7a491 100644 --- a/server/src/controllers/training_controller.ts +++ b/server/src/controllers/training_controller.ts @@ -3,7 +3,7 @@ import type WebSocket from "ws"; import { Map } from "immutable"; import * as msgpack from "@msgpack/msgpack"; -import { client } from "@epfml/discojs"; +import { NodeID, mtype } from "@epfml/discojs"; import type { DataType, Network, Task } from "@epfml/discojs"; const debug = createDebug("server:controllers"); @@ -37,7 +37,7 @@ export abstract class TrainingController< * the list allows updating participants about the training status * i.e. waiting for more participants or resuming training */ - protected connections = Map(); + protected connections = Map(); constructor(protected readonly task: Task) {} @@ -48,7 +48,7 @@ export abstract class TrainingController< * * @param currentId the id of the participant that just joined */ - protected sendEnoughParticipantsMsgIfNeeded(currentId: client.NodeID) { + protected sendEnoughParticipantsMsgIfNeeded(currentId: NodeID) { // If we are currently waiting for more participants to join and we now have enough, // broadcast to previously waiting participants that the training can start if ( @@ -64,8 +64,8 @@ export abstract class TrainingController< "Sending enough-participant message to client [%s]", participantId.slice(0, 4), ); - const msg: client.mtype.EnoughParticipants = { - type: client.mtype.MType.EnoughParticipants, + const msg: mtype.EnoughParticipants = { + type: mtype.MType.EnoughParticipants, nbOfParticipants: this.connections.size, }; participantWs.send(msgpack.encode(msg)); @@ -86,8 +86,8 @@ export abstract class TrainingController< "Telling remaining client [%s] to wait for participants", participantId.slice(0, 4), ); - const msg: client.mtype.WaitingForMoreParticipants = { - type: client.mtype.MType.WaitingForMoreParticipants, + const msg: mtype.WaitingForMoreParticipants = { + type: mtype.MType.WaitingForMoreParticipants, nbOfParticipants: this.connections.size, }; participantWs.send(msgpack.encode(msg)); diff --git a/server/src/model_set.ts b/server/src/model_set.ts index 021f1b7c2..d8cbad5b1 100644 --- a/server/src/model_set.ts +++ b/server/src/model_set.ts @@ -2,9 +2,15 @@ import { Map } from "immutable"; import "@tensorflow/tfjs-node"; import type { DataType, ModelCardInfo, ModelCard } from "@epfml/discojs"; -import { EventEmitter, Model, serialization } from "@epfml/discojs"; +import { + EventEmitter, + Model, + Encoded, + modelEncode, + isEncoded, +} from "@epfml/discojs"; -type EncodedModel = serialization.Encoded; +type EncodedModel = Encoded; type AvailableModel = [ModelCardInfo, EncodedModel]; /** @@ -77,13 +83,13 @@ export class ModelSet extends EventEmitter<{ let encodedModel: EncodedModel; if (!Array.isArray(newModel)) { const model = await newModel.getModel(); - encodedModel = await serialization.model.encode(model); + encodedModel = await modelEncode(model); } else { const model = newModel[1]; - if (serialization.isEncoded(model)) { + if (isEncoded(model)) { encodedModel = model; // don't do anything if already encoded } else { - encodedModel = await serialization.model.encode(model); + encodedModel = await modelEncode(model); } } diff --git a/server/src/routes/task_router.ts b/server/src/routes/task_router.ts index 13e2dab31..bdc17b10c 100644 --- a/server/src/routes/task_router.ts +++ b/server/src/routes/task_router.ts @@ -5,7 +5,7 @@ import { Set } from "immutable"; import type { DataType, ModelCardInfo, Network } from "@epfml/discojs"; import { Task } from "@epfml/discojs"; -import { serialization } from "@epfml/discojs"; +import { serializeTaskToJSON, deserializeTaskFromJSON } from "@epfml/discojs"; import type { TaskSet } from "../task_set.js"; import type { ModelSet } from "../model_set.js"; @@ -45,7 +45,7 @@ export class TaskRouter { res.status(200).send( this.#taskSet.tasks .valueSeq() - .map(([t, _]) => serialization.task.serializeToJSON(t)) + .map(([t, _]) => serializeTaskToJSON(t)) .toArray(), ); }); @@ -70,7 +70,7 @@ export class TaskRouter { z.string(), z.array(z.number()).transform((bytes) => Uint8Array.from(bytes)), ]), - task: z.any().transform(serialization.task.deserializeFromJSON), + task: z.any().transform(deserializeTaskFromJSON), }) .safeParseAsync(req.body); diff --git a/server/src/routes/training_router.ts b/server/src/routes/training_router.ts index 28da1e932..86ed1ef7c 100644 --- a/server/src/routes/training_router.ts +++ b/server/src/routes/training_router.ts @@ -1,7 +1,7 @@ import express from "express"; import type expressWS from "express-ws"; import type { Task, DataType, Network } from "@epfml/discojs"; -import { serialization } from "@epfml/discojs"; +import { Encoded, modelDecode, weightsEncode } from "@epfml/discojs"; import type { TaskSet } from "../task_set.js"; import { @@ -44,7 +44,7 @@ export class TrainingRouter> { // websocket connections private async onNewTask( task: Task, - encodedModel: serialization.Encoded, + encodedModel: Encoded, ): Promise { // The controller handles the actual logic of collaborative training // in its `handle` method. Each task has a dedicated controller which @@ -55,10 +55,8 @@ export class TrainingRouter> { // The federated controller takes the initial model weights at initialization // so that it can send it to new clients - const model = serialization.model.decode(encodedModel); - const encodedWeights = await serialization.weights.encode( - (await model).weights, - ); + const model = modelDecode(encodedModel); + const encodedWeights = await weightsEncode((await model).weights); taskController = new FederatedController(t, encodedWeights); } else { const t = task as Task; diff --git a/server/src/task_set.ts b/server/src/task_set.ts index 28ccfbaa4..fb02282e6 100644 --- a/server/src/task_set.ts +++ b/server/src/task_set.ts @@ -2,10 +2,10 @@ import { Map } from "immutable"; import "@tensorflow/tfjs-node"; import type { DataType, Network, Task } from "@epfml/discojs"; -import { EventEmitter, serialization } from "@epfml/discojs"; +import { EventEmitter, Encoded } from "@epfml/discojs"; import { ModelSet } from "./model_set.js"; -type EncodedModel = serialization.Encoded; +type EncodedModel = Encoded; type TaskAndModel = [Task, EncodedModel]; /** diff --git a/webapp/cypress/e2e/task-creation.cy.ts b/webapp/cypress/e2e/task-creation.cy.ts index 309404415..dbad26f4c 100644 --- a/webapp/cypress/e2e/task-creation.cy.ts +++ b/webapp/cypress/e2e/task-creation.cy.ts @@ -1,4 +1,4 @@ -import { serialization, type Task } from "@epfml/discojs"; +import { deserializeTaskFromJSON, type Task } from "@epfml/discojs"; import * as tf from "@tensorflow/tfjs"; @@ -64,7 +64,7 @@ it("submits with tabular task", () => { cy.wait("@posted") .its("request.body.task") - .then(serialization.task.deserializeFromJSON) + .then(deserializeTaskFromJSON) .should("deep.equal", { id: "id", dataType: "tabular", diff --git a/webapp/cypress/support/e2e.ts b/webapp/cypress/support/e2e.ts index 119b229bc..0a55ba067 100644 --- a/webapp/cypress/support/e2e.ts +++ b/webapp/cypress/support/e2e.ts @@ -5,8 +5,9 @@ import type { Task, TaskProvider, TrainingInformation, + Encoded, } from "@epfml/discojs"; -import { serialization } from "@epfml/discojs"; +import { serializeTaskToJSON, modelEncode } from "@epfml/discojs"; export function setupServerWith( ...providers: (Task | TaskProvider)[] @@ -23,9 +24,7 @@ export function setupServerWith( .as("taskAndModels"); cy.get, unknown]>>("@taskAndModels") - .then((taskAndModels) => - taskAndModels.map(([t]) => serialization.task.serializeToJSON(t)), - ) + .then((taskAndModels) => taskAndModels.map(([t]) => serializeTaskToJSON(t))) .then((tasks) => cy.intercept({ hostname: "server", pathname: "tasks" }, tasks), ); @@ -42,9 +41,7 @@ export function setupServerWith( { hostname: "server", pathname: `/tasks/${task.id}/model.json` }, { statusCode: 200 }, ); - cy.wrap, serialization.Encoded>( - serialization.model.encode(model), - ).then((encoded) => + cy.wrap, Encoded>(modelEncode(model)).then((encoded) => cy.intercept( { hostname: "server", pathname: `/tasks/${task.id}/model.json` }, (req) => diff --git a/webapp/src/components/task_creation_form/TaskCreationForm.vue b/webapp/src/components/task_creation_form/TaskCreationForm.vue index b45a8b026..fc6b0ce9d 100644 --- a/webapp/src/components/task_creation_form/TaskCreationForm.vue +++ b/webapp/src/components/task_creation_form/TaskCreationForm.vue @@ -616,7 +616,7 @@ import * as z from "zod"; import * as tf from "@tensorflow/tfjs"; import { - models, + TFJS, pushTask, Task, Tokenizer, @@ -941,7 +941,7 @@ async function onSubmit(form: unknown): Promise { loss, optimizer: tf.train[optimizer.name](optimizer.learningRate), }); - model = new models.TFJS(task.dataType, loaded); + model = new TFJS(task.dataType, loaded); break; } case "text": diff --git a/webapp/src/components/testing/ModelLibrary.vue b/webapp/src/components/testing/ModelLibrary.vue index 88cfca446..e58fe9ec4 100644 --- a/webapp/src/components/testing/ModelLibrary.vue +++ b/webapp/src/components/testing/ModelLibrary.vue @@ -154,7 +154,7 @@ import { RouterLink } from "vue-router"; import { VueSpinner } from "vue3-spinners"; import type { DataType, Model, Network, Task } from "@epfml/discojs"; -import { client as clients, aggregator } from "@epfml/discojs"; +import { LocalClient, getAggregator } from "@epfml/discojs"; import BinIcon from "@/assets/svg/BinIcon.vue"; import { useToaster } from "@/composables/toaster"; @@ -240,7 +240,7 @@ async function downloadModel(task: Task): Promise { try { toaster.info("Downloading model..."); - const client = new clients.LocalClient( + const client = new LocalClient( CONFIG.serverUrl, { ...task, @@ -249,7 +249,7 @@ async function downloadModel(task: Task): Promise { scheme: "local", }, } as Task, - aggregator.getAggregator(task), + getAggregator(task), ); const model = await client.getLatestModel(); diff --git a/webapp/src/components/testing/__tests__/ModelLibrary.spec.ts b/webapp/src/components/testing/__tests__/ModelLibrary.spec.ts index 5e92a8f63..c7c39211b 100644 --- a/webapp/src/components/testing/__tests__/ModelLibrary.spec.ts +++ b/webapp/src/components/testing/__tests__/ModelLibrary.spec.ts @@ -7,12 +7,7 @@ import { flushPromises, mount } from "@vue/test-utils"; import { createTestingPinia } from "@pinia/testing"; import { createPersistedStatePlugin } from "pinia-plugin-persistedstate-2"; -import { - models as discoModels, - serialization, - Task, - Tokenizer, -} from "@epfml/discojs"; +import { GPT, serializeTaskToJSON, Task, Tokenizer } from "@epfml/discojs"; import { CONFIG } from "@/config"; import { useModelsStore } from "@/store"; @@ -60,7 +55,7 @@ it("shows stored models", async () => { tasks.value = Map([[TASK.id, TASK]]); const models = useModelsStore(); - await models.add("task", new discoModels.GPT()); + await models.add("task", new GPT()); await nextTick(); expect(wrapper.get("div.text-xl").text()).to.equal("task title"); @@ -69,9 +64,7 @@ it("shows stored models", async () => { it("allows to download server's models", async () => { vi.stubGlobal("fetch", (url: string | URL) => { if (url.toString() === new URL("tasks", CONFIG.serverUrl).href) - return new Response( - JSON.stringify([serialization.task.serializeToJSON(TASK)]), - ); + return new Response(JSON.stringify([serializeTaskToJSON(TASK)])); throw new Error(`unhandled get: ${url}`); }); afterEach(() => { diff --git a/webapp/src/components/training/TrainerDashboard.vue b/webapp/src/components/training/TrainerDashboard.vue index dfd1d6c49..9324cdf00 100644 --- a/webapp/src/components/training/TrainerDashboard.vue +++ b/webapp/src/components/training/TrainerDashboard.vue @@ -151,7 +151,7 @@ import type { Task, Network, } from "@epfml/discojs"; -import { async_iterator, Disco } from "@epfml/discojs"; +import { split, Disco } from "@epfml/discojs"; import { useToaster } from "@/composables/toaster"; import TrainingInformation from "@/components/training/TrainingInformation.vue"; @@ -249,11 +249,11 @@ async function startTraining(): Promise { roundsLogs.value = List(); for await (const round of trainingGenerator.value) { - const [roundGen, roundLogs] = async_iterator.split(round); + const [roundGen, roundLogs] = split(round); roundGenerator.value = roundGen; for await (const epoch of roundGenerator.value) { - const [epochGen, epochLogs] = async_iterator.split(epoch); + const [epochGen, epochLogs] = split(epoch); epochGenerator.value = epochGen; for await (const batch of epochGenerator.value) diff --git a/webapp/src/components/training/__tests__/TrainerDashboard.spec.ts b/webapp/src/components/training/__tests__/TrainerDashboard.spec.ts index 2374369e2..a60d6aa59 100644 --- a/webapp/src/components/training/__tests__/TrainerDashboard.spec.ts +++ b/webapp/src/components/training/__tests__/TrainerDashboard.spec.ts @@ -1,6 +1,6 @@ import * as fs from "node:fs/promises"; import { CONFIG } from "@/config"; -import { defaultTasks, serialization } from "@epfml/discojs"; +import { defaultTasks, modelEncode } from "@epfml/discojs"; import { loadCSV } from "@epfml/discojs-web"; import { createTestingPinia } from "@pinia/testing"; import { mount } from "@vue/test-utils"; @@ -18,7 +18,7 @@ async function setupForTask() { new URL("tasks/titanic/model.json", CONFIG.serverUrl).href ) { const model = await provider.modelCard.getModel(); - const encoded = await serialization.model.encode(model); + const encoded = await modelEncode(model); return new Response(new Uint8Array(encoded)); } throw new Error(`unhandled get: ${url}`); diff --git a/webapp/src/store/models/__tests__/models.spec.ts b/webapp/src/store/models/__tests__/models.spec.ts index 175e3b2f7..b390ee06e 100644 --- a/webapp/src/store/models/__tests__/models.spec.ts +++ b/webapp/src/store/models/__tests__/models.spec.ts @@ -3,7 +3,7 @@ import { createPersistedStatePlugin } from "pinia-plugin-persistedstate-2"; import { beforeEach, expect, it } from "vitest"; import { createApp } from "vue"; -import { models as discoModels } from "@epfml/discojs"; +import { GPT } from "@epfml/discojs"; import { useModelsStore } from "../index"; @@ -17,7 +17,7 @@ beforeEach(() => { it("stores model", async () => { const models = useModelsStore(); - const id = await models.add("task id", new discoModels.GPT()); + const id = await models.add("task id", new GPT()); expect(models.infos.size).to.equal(1); - expect(await models.get(id)).to.be.an.instanceof(discoModels.GPT); + expect(await models.get(id)).to.be.an.instanceof(GPT); }); diff --git a/webapp/src/store/models/index.ts b/webapp/src/store/models/index.ts index aca613343..8ac462707 100644 --- a/webapp/src/store/models/index.ts +++ b/webapp/src/store/models/index.ts @@ -3,7 +3,7 @@ import { defineStore } from "pinia"; import { computed, shallowRef, toRaw } from "vue"; import type { DataType, Model } from "@epfml/discojs"; -import { serialization } from "@epfml/discojs"; +import { modelEncode, modelDecode } from "@epfml/discojs"; import { bestStorage } from "./storage"; import type { ModelID, State } from "./types"; @@ -29,7 +29,7 @@ export const useModelsStore = defineStore( const infos = idToModel.value.get(id); if (infos === undefined) return undefined; - return await serialization.model.decode(toRaw(infos.encoded)); + return await modelDecode(toRaw(infos.encoded)); } async function add( @@ -42,7 +42,7 @@ export const useModelsStore = defineStore( idToModel.value = idToModel.value.set(id, { taskID, dateSaved, - encoded: await serialization.model.encode(model), + encoded: await modelEncode(model), }); return id; diff --git a/webapp/src/store/models/opfs.ts b/webapp/src/store/models/opfs.ts index 32fbdf80f..51d50ef8d 100644 --- a/webapp/src/store/models/opfs.ts +++ b/webapp/src/store/models/opfs.ts @@ -2,7 +2,7 @@ import { Map } from "immutable"; import * as msgpack from "@msgpack/msgpack"; import type { IStorage } from "pinia-plugin-persistedstate-2"; -import { serialization } from "@epfml/discojs"; +import { isEncoded } from "@epfml/discojs"; import type { Storage } from "./storage"; import { UNSUPPORTED_STORAGE } from "./storage"; @@ -102,7 +102,7 @@ function isSerializedInfos(raw: unknown): raw is OPFS.Serialized[0][1] { if ( typeof taskID !== "string" || - !serialization.isEncoded(encoded) || + !isEncoded(encoded) || !(dateSaved instanceof Date) ) return false; diff --git a/webapp/src/store/models/types.ts b/webapp/src/store/models/types.ts index 1b15ca242..aa63c7214 100644 --- a/webapp/src/store/models/types.ts +++ b/webapp/src/store/models/types.ts @@ -1,13 +1,13 @@ import type { Map } from "immutable"; -import type { serialization } from "@epfml/discojs"; +import type { Encoded } from "@epfml/discojs"; export type ModelID = number; export interface Infos { taskID: string; dateSaved: Date; - encoded: serialization.Encoded; + encoded: Encoded; } // only `ref` is `idToModel` From d25e5d12d8195db8ccd49e967a023ed0a556ccf0 Mon Sep 17 00:00:00 2001 From: hugo Date: Sat, 8 Aug 2026 14:32:22 +0200 Subject: [PATCH 03/18] refactor: force type-only imports when possible --- cli/src/benchmark_gpt.ts | 4 ++-- cli/src/data.ts | 5 +++-- cli/src/hellaswag_gpt.ts | 5 +++-- cli/src/train_gpt.ts | 3 ++- cli/src/user_log.ts | 3 ++- discojs-node/src/hellaswag.ts | 7 ++++--- discojs-node/src/loaders/text.ts | 3 ++- discojs-web/src/hellaswag.spec.ts | 2 +- discojs-web/src/hellaswag.ts | 7 ++++--- discojs-web/src/loaders/text.ts | 3 ++- discojs/src/aggregator/byzantine.ts | 3 ++- discojs/src/aggregator/get.ts | 2 +- discojs/src/aggregator/mean.ts | 3 ++- discojs/src/aggregator/percentile_clipping.ts | 10 ++++++---- discojs/src/client/federated/messages.ts | 2 +- discojs/src/client/get_client.ts | 2 +- discojs/src/client/local_client.ts | 2 +- discojs/src/dataset/dataset.ts | 2 +- discojs/src/dataset/types.ts | 2 +- discojs/src/logging/console_logger.ts | 2 +- discojs/src/models/cards/CIFAR10Classifier.ts | 4 ++-- discojs/src/models/cards/LUSClassifier.ts | 4 ++-- discojs/src/models/cards/MNISTClassifier.ts | 4 ++-- discojs/src/models/cards/dogClassifier.ts | 4 ++-- discojs/src/models/cards/titanicClassifier.ts | 4 ++-- discojs/src/models/cards/wikitext.ts | 4 ++-- .../src/models/implementations/LUSClassifierModel.ts | 2 +- discojs/src/models/implementations/gpt/index.ts | 5 +++-- discojs/src/models/model.ts | 4 ++-- discojs/src/models/model_card.ts | 3 ++- discojs/src/models/onnx.ts | 4 ++-- discojs/src/models/tfjs.ts | 5 +++-- discojs/src/processing/tabular.ts | 2 +- discojs/src/serialization/model.ts | 7 ++++--- discojs/src/serialization/weights.ts | 2 +- discojs/src/task/task_handler.ts | 5 +++-- discojs/src/training/disco.ts | 8 +++++--- discojs/src/training/trainer.ts | 4 ++-- discojs/src/training/types.ts | 2 +- discojs/src/types/data_format.ts | 4 ++-- eslint.config.js | 1 + server/src/controllers/federated_controller.ts | 6 +++--- server/src/controllers/training_controller.ts | 4 ++-- server/src/model_set.ts | 6 +++--- server/src/routes/training_router.ts | 7 ++++--- server/src/task_set.ts | 6 +++--- webapp/src/components/containers/ButtonsCard.vue | 2 +- webapp/src/components/containers/TableLayout.vue | 2 +- webapp/src/components/dataset_input/FileSelection.vue | 3 ++- .../src/components/dataset_input/ImageDatasetInput.vue | 2 +- .../dataset_input/LabeledImageDatasetInput/ByCSV.vue | 3 ++- .../dataset_input/LabeledImageDatasetInput/ByGroup.vue | 3 ++- .../dataset_input/LabeledImageDatasetInput/index.vue | 2 +- .../components/dataset_input/TabularDatasetInput.vue | 2 +- .../src/components/dataset_input/TextDatasetInput.vue | 4 ++-- .../components/testing/__tests__/ModelLibrary.spec.ts | 3 ++- webapp/src/components/training/TrainingInformation.vue | 3 ++- webapp/src/store/tasks.ts | 3 ++- 58 files changed, 121 insertions(+), 94 deletions(-) diff --git a/cli/src/benchmark_gpt.ts b/cli/src/benchmark_gpt.ts index 1dcd7b362..cf647ad90 100644 --- a/cli/src/benchmark_gpt.ts +++ b/cli/src/benchmark_gpt.ts @@ -2,13 +2,13 @@ import "@tensorflow/tfjs-node"; import { List } from "immutable"; import { parse } from "ts-command-line-args"; -import type { Network, Task } from "@epfml/discojs"; +import type { Network, Task , + GPTConfig} from "@epfml/discojs"; import { gather, defaultTasks, defaultModels, fetchTasks, - GPTConfig, GPT, } from "@epfml/discojs"; import { loadModelFromDisk, loadText } from "@epfml/discojs-node"; diff --git a/cli/src/data.ts b/cli/src/data.ts index c6ea091dc..f20b67751 100644 --- a/cli/src/data.ts +++ b/cli/src/data.ts @@ -1,6 +1,7 @@ import path from "node:path"; -import { Dataset, extractColumn } from "@epfml/discojs"; -import { DataFormat, DataType, Image, Task } from "@epfml/discojs"; +import type { Dataset} from "@epfml/discojs"; +import { extractColumn } from "@epfml/discojs"; +import type { DataFormat, DataType, Image, Task } from "@epfml/discojs"; import { loadCSV, loadImage, loadImagesInDir } from "@epfml/discojs-node"; import { Repeat } from "immutable"; diff --git a/cli/src/hellaswag_gpt.ts b/cli/src/hellaswag_gpt.ts index 0aafbf844..369dc6f41 100644 --- a/cli/src/hellaswag_gpt.ts +++ b/cli/src/hellaswag_gpt.ts @@ -5,13 +5,14 @@ import { parse } from "ts-command-line-args"; import "@tensorflow/tfjs-node"; import path from "node:path"; +import type { + HellaSwagDataset} from "@epfml/discojs"; import { GPT, ONNXModel, modelDecode, Tokenizer, - evaluate_hellaswag, - HellaSwagDataset, + evaluate_hellaswag } from "@epfml/discojs"; import { loadHellaSwag } from "@epfml/discojs-node"; diff --git a/cli/src/train_gpt.ts b/cli/src/train_gpt.ts index 537e39bbb..965161212 100644 --- a/cli/src/train_gpt.ts +++ b/cli/src/train_gpt.ts @@ -1,5 +1,6 @@ import "@tensorflow/tfjs-node"; -import { GPTConfig, GPT, Dataset, Tokenizer } from "@epfml/discojs"; +import type { GPTConfig} from "@epfml/discojs"; +import { GPT, Dataset, Tokenizer } from "@epfml/discojs"; import { List } from "immutable"; async function main(): Promise { diff --git a/cli/src/user_log.ts b/cli/src/user_log.ts index 2b7fc7bca..cdb33f015 100644 --- a/cli/src/user_log.ts +++ b/cli/src/user_log.ts @@ -1,4 +1,5 @@ -import { args, BenchmarkArguments } from "./args.js"; +import type { BenchmarkArguments } from "./args.js"; +import { args } from "./args.js"; import type { SummaryLogs, DataType, Network, Task } from "@epfml/discojs"; type SerializableArguments = Omit & { diff --git a/discojs-node/src/hellaswag.ts b/discojs-node/src/hellaswag.ts index 04f5d6a6a..7d3faaae4 100644 --- a/discojs-node/src/hellaswag.ts +++ b/discojs-node/src/hellaswag.ts @@ -2,10 +2,11 @@ import path from "node:path"; import fetch from "node-fetch"; import fs from "node:fs/promises"; -import { - HELLASWAG_URL, +import type { HellaSwagExample, - HellaSwagDataset, + HellaSwagDataset} from "@epfml/discojs"; +import { + HELLASWAG_URL } from "@epfml/discojs"; import { dirname } from "path"; diff --git a/discojs-node/src/loaders/text.ts b/discojs-node/src/loaders/text.ts index cf5d22a48..c8c625e28 100644 --- a/discojs-node/src/loaders/text.ts +++ b/discojs-node/src/loaders/text.ts @@ -1,6 +1,7 @@ import createDebug from "debug"; import { createReadStream } from "node:fs"; -import { Dataset, Text } from "@epfml/discojs"; +import type { Text } from "@epfml/discojs"; +import { Dataset } from "@epfml/discojs"; const debug = createDebug("discojs-node:loaders:text"); diff --git a/discojs-web/src/hellaswag.spec.ts b/discojs-web/src/hellaswag.spec.ts index 76b5dbcef..64ad69d05 100644 --- a/discojs-web/src/hellaswag.spec.ts +++ b/discojs-web/src/hellaswag.spec.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { load as loadHellaSwag } from "./hellaswag.js"; -import { models } from "@epfml/discojs"; +import type { models } from "@epfml/discojs"; describe("hellaswag parser", () => { it("loads the whole hellaswag dataset", async () => { diff --git a/discojs-web/src/hellaswag.ts b/discojs-web/src/hellaswag.ts index edefc6edb..5050aa7e8 100644 --- a/discojs-web/src/hellaswag.ts +++ b/discojs-web/src/hellaswag.ts @@ -1,7 +1,8 @@ -import { - HELLASWAG_URL, +import type { HellaSwagExample, - HellaSwagDataset, + HellaSwagDataset} from "@epfml/discojs"; +import { + HELLASWAG_URL } from "@epfml/discojs"; /** diff --git a/discojs-web/src/loaders/text.ts b/discojs-web/src/loaders/text.ts index d9ad0f89e..0c84f0150 100644 --- a/discojs-web/src/loaders/text.ts +++ b/discojs-web/src/loaders/text.ts @@ -1,4 +1,5 @@ -import { Dataset, Text } from "@epfml/discojs"; +import type { Text } from "@epfml/discojs"; +import { Dataset } from "@epfml/discojs"; export function load(file: Blob): Dataset { return new Dataset(async function* () { diff --git a/discojs/src/aggregator/byzantine.ts b/discojs/src/aggregator/byzantine.ts index a86d7e67d..63b882669 100644 --- a/discojs/src/aggregator/byzantine.ts +++ b/discojs/src/aggregator/byzantine.ts @@ -6,7 +6,8 @@ import type { NodeID } from "#client/types"; import { avg } from "#weights/index"; import { AggregationStep } from "#aggregator/aggregator"; -import { MultiRoundAggregator, ThresholdType } from "#aggregator/multiround"; +import type { ThresholdType } from "#aggregator/multiround"; +import { MultiRoundAggregator } from "#aggregator/multiround"; /** * Byzantine-robust aggregator using Centered Clipping (CC), based on the diff --git a/discojs/src/aggregator/get.ts b/discojs/src/aggregator/get.ts index 0b2662290..af5919263 100644 --- a/discojs/src/aggregator/get.ts +++ b/discojs/src/aggregator/get.ts @@ -1,6 +1,6 @@ import type { DataType, Network } from "#types/index"; import type { Task } from "#task/index"; -import { Aggregator } from "#aggregator/aggregator"; +import type { Aggregator } from "#aggregator/aggregator"; import { MeanAggregator } from "#aggregator/mean"; import { SecureAggregator } from "#aggregator/secure"; import { ByzantineRobustAggregator } from "#aggregator/byzantine"; diff --git a/discojs/src/aggregator/mean.ts b/discojs/src/aggregator/mean.ts index ac2dec8ae..0f1faccfd 100644 --- a/discojs/src/aggregator/mean.ts +++ b/discojs/src/aggregator/mean.ts @@ -5,7 +5,8 @@ import type { NodeID } from "#client/types"; import { avg } from "#weights/index"; import { AggregationStep } from "#aggregator/aggregator"; -import { MultiRoundAggregator, ThresholdType } from "#aggregator/multiround"; +import type { ThresholdType } from "#aggregator/multiround"; +import { MultiRoundAggregator } from "#aggregator/multiround"; /** * Mean aggregator whose aggregation step consists in computing the mean of the received weights. diff --git a/discojs/src/aggregator/percentile_clipping.ts b/discojs/src/aggregator/percentile_clipping.ts index 2093433ed..81968a09f 100644 --- a/discojs/src/aggregator/percentile_clipping.ts +++ b/discojs/src/aggregator/percentile_clipping.ts @@ -1,9 +1,11 @@ -import { Map } from "immutable"; +import type { Map } from "immutable"; import * as tf from "@tensorflow/tfjs"; import { AggregationStep } from "#aggregator/aggregator"; -import { MultiRoundAggregator, ThresholdType } from "#aggregator/multiround"; -import { NodeID } from "#client/index.js"; -import { WeightsContainer, avg } from "#weights/index"; +import type { ThresholdType } from "#aggregator/multiround"; +import { MultiRoundAggregator } from "#aggregator/multiround"; +import type { NodeID } from "#client/index.js"; +import type { WeightsContainer} from "#weights/index"; +import { avg } from "#weights/index"; /** * Percentile-based clipping aggregator. diff --git a/discojs/src/client/federated/messages.ts b/discojs/src/client/federated/messages.ts index 99db9e13e..d45e7c542 100644 --- a/discojs/src/client/federated/messages.ts +++ b/discojs/src/client/federated/messages.ts @@ -1,4 +1,4 @@ -import * as serialization from "#serialization/index"; +import type * as serialization from "#serialization/index"; import type { NodeID } from "#client/types"; import { MType, hasMessageType } from "#client/mtype"; diff --git a/discojs/src/client/get_client.ts b/discojs/src/client/get_client.ts index 7ee1455aa..9678c8dbe 100644 --- a/discojs/src/client/get_client.ts +++ b/discojs/src/client/get_client.ts @@ -4,7 +4,7 @@ import type * as aggregator from "#aggregator/index"; // import * as clients from "#client/index"; import { LocalClient } from "#client/local_client"; -import { Client } from "#client/client"; +import type { Client } from "#client/client"; import { DecentralizedClient } from "#client/decentralized/decentralized_client"; import { FederatedClient } from "#client/federated/federated_client"; diff --git a/discojs/src/client/local_client.ts b/discojs/src/client/local_client.ts index 4e3559fa6..2d551b714 100644 --- a/discojs/src/client/local_client.ts +++ b/discojs/src/client/local_client.ts @@ -1,4 +1,4 @@ -import { WeightsContainer } from "#weights/index"; +import type { WeightsContainer } from "#weights/index"; import { Client } from "./client.js"; /** diff --git a/discojs/src/dataset/dataset.ts b/discojs/src/dataset/dataset.ts index be7085ef8..8cf43f049 100644 --- a/discojs/src/dataset/dataset.ts +++ b/discojs/src/dataset/dataset.ts @@ -1,7 +1,7 @@ import createDebug from "debug"; import { List, Range } from "immutable"; -import { Batched } from "./types.js"; +import type { Batched } from "./types.js"; const debug = createDebug("discojs:dataset"); diff --git a/discojs/src/dataset/types.ts b/discojs/src/dataset/types.ts index 3b2b1fbd6..857d36e73 100644 --- a/discojs/src/dataset/types.ts +++ b/discojs/src/dataset/types.ts @@ -1,4 +1,4 @@ -import { List } from "immutable"; +import type { List } from "immutable"; export { Image } from "./image.js"; diff --git a/discojs/src/logging/console_logger.ts b/discojs/src/logging/console_logger.ts index de83dfcef..dc6b9992b 100644 --- a/discojs/src/logging/console_logger.ts +++ b/discojs/src/logging/console_logger.ts @@ -1,5 +1,5 @@ import chalk from "chalk"; -import { Logger } from "./logger.js"; +import type { Logger } from "./logger.js"; /** * Same properties as Toaster but on the console diff --git a/discojs/src/models/cards/CIFAR10Classifier.ts b/discojs/src/models/cards/CIFAR10Classifier.ts index ddf9dd70f..e198c7b89 100644 --- a/discojs/src/models/cards/CIFAR10Classifier.ts +++ b/discojs/src/models/cards/CIFAR10Classifier.ts @@ -1,5 +1,5 @@ -import { Model } from "#models/model"; -import { ModelCard } from "#models/model_card"; +import type { Model } from "#models/model"; +import type { ModelCard } from "#models/model_card"; import { getModel } from "#models/implementations/CIFAR10ClassifierModel"; export const CIFAR10Classifier: ModelCard<"image"> = { diff --git a/discojs/src/models/cards/LUSClassifier.ts b/discojs/src/models/cards/LUSClassifier.ts index 0630adfbd..bbc0f0de0 100644 --- a/discojs/src/models/cards/LUSClassifier.ts +++ b/discojs/src/models/cards/LUSClassifier.ts @@ -1,5 +1,5 @@ -import { Model } from "#models/model"; -import { ModelCard } from "#models/model_card"; +import type { Model } from "#models/model"; +import type { ModelCard } from "#models/model_card"; import { model } from "#models/implementations/LUSClassifierModel"; export const LUSClassifier: ModelCard<"image"> = { diff --git a/discojs/src/models/cards/MNISTClassifier.ts b/discojs/src/models/cards/MNISTClassifier.ts index 8e7c264a2..96f911f45 100644 --- a/discojs/src/models/cards/MNISTClassifier.ts +++ b/discojs/src/models/cards/MNISTClassifier.ts @@ -1,5 +1,5 @@ -import { Model } from "#models/model"; -import { ModelCard } from "#models/model_card"; +import type { Model } from "#models/model"; +import type { ModelCard } from "#models/model_card"; import { model } from "#models/implementations/MNISTClassifierModel"; export const MNISTClassifier: ModelCard<"image"> = { diff --git a/discojs/src/models/cards/dogClassifier.ts b/discojs/src/models/cards/dogClassifier.ts index d37305bb2..a140e3865 100644 --- a/discojs/src/models/cards/dogClassifier.ts +++ b/discojs/src/models/cards/dogClassifier.ts @@ -1,5 +1,5 @@ -import { Model } from "#models/model"; -import { ModelCard } from "#models/model_card"; +import type { Model } from "#models/model"; +import type { ModelCard } from "#models/model_card"; import { model } from "#models/implementations/dogClassifierModel"; export const DogClassifier: ModelCard<"image"> = { diff --git a/discojs/src/models/cards/titanicClassifier.ts b/discojs/src/models/cards/titanicClassifier.ts index 15fae9af5..436639491 100644 --- a/discojs/src/models/cards/titanicClassifier.ts +++ b/discojs/src/models/cards/titanicClassifier.ts @@ -1,5 +1,5 @@ -import { Model } from "#models/model"; -import { ModelCard } from "#models/model_card"; +import type { Model } from "#models/model"; +import type { ModelCard } from "#models/model_card"; import { model } from "#models/implementations/titanicClassifierModel"; export const TitanicClassifier: ModelCard<"tabular"> = { diff --git a/discojs/src/models/cards/wikitext.ts b/discojs/src/models/cards/wikitext.ts index bb2de098a..a698ed5b7 100644 --- a/discojs/src/models/cards/wikitext.ts +++ b/discojs/src/models/cards/wikitext.ts @@ -1,5 +1,5 @@ -import { Model } from "#models/model"; -import { ModelCard } from "#models/model_card"; +import type { Model } from "#models/model"; +import type { ModelCard } from "#models/model_card"; import { GPT } from "#models/implementations/index"; export const Wikitext: ModelCard<"text"> = { diff --git a/discojs/src/models/implementations/LUSClassifierModel.ts b/discojs/src/models/implementations/LUSClassifierModel.ts index 96174b58b..2d34cd76b 100644 --- a/discojs/src/models/implementations/LUSClassifierModel.ts +++ b/discojs/src/models/implementations/LUSClassifierModel.ts @@ -1,6 +1,6 @@ import * as tf from "@tensorflow/tfjs"; -import { Model } from "#models/model"; +import type { Model } from "#models/model"; import { TFJS } from "#models/tfjs"; // Model architecture from tensorflow.js docs: diff --git a/discojs/src/models/implementations/gpt/index.ts b/discojs/src/models/implementations/gpt/index.ts index 8b9791ae2..c592ad69a 100644 --- a/discojs/src/models/implementations/gpt/index.ts +++ b/discojs/src/models/implementations/gpt/index.ts @@ -8,10 +8,11 @@ import { List, Range } from "immutable"; import * as tf from "@tensorflow/tfjs"; import { WeightsContainer } from "#weights/index"; -import { Dataset, Batched } from "#dataset/index"; +import type { Dataset, Batched } from "#dataset/index"; import type { DataFormat } from "#types/index"; -import { BatchLogs, EpochLogs } from "#models/logs"; +import type { BatchLogs} from "#models/logs"; +import { EpochLogs } from "#models/logs"; import { Model } from "#models/model"; import { GPTModel } from "./model.js"; import evaluate from "./evaluate.js"; diff --git a/discojs/src/models/model.ts b/discojs/src/models/model.ts index 596c6bb5d..ad4252c78 100644 --- a/discojs/src/models/model.ts +++ b/discojs/src/models/model.ts @@ -1,5 +1,5 @@ -import { WeightsContainer } from "#weights/index"; -import { Dataset, Batched } from "#dataset/index"; +import type { WeightsContainer } from "#weights/index"; +import type { Dataset, Batched } from "#dataset/index"; import type { DataFormat, DataType } from "#types/index"; import type { BatchLogs, EpochLogs, ValidationMetrics } from "./logs.js"; diff --git a/discojs/src/models/model_card.ts b/discojs/src/models/model_card.ts index 6bd3f975a..0df9d261b 100644 --- a/discojs/src/models/model_card.ts +++ b/discojs/src/models/model_card.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import type { Model } from "#models/model"; -import { DataType, dataTypeValues } from "#types/index"; +import type { DataType} from "#types/index"; +import { dataTypeValues } from "#types/index"; export namespace ModelCardInfo { export type ID = string; diff --git a/discojs/src/models/onnx.ts b/discojs/src/models/onnx.ts index e513f24ca..d45049402 100644 --- a/discojs/src/models/onnx.ts +++ b/discojs/src/models/onnx.ts @@ -1,7 +1,7 @@ -import type { CausalLMOutput } from "@xenova/transformers"; +import type { CausalLMOutput , + PreTrainedModel} from "@xenova/transformers"; import { AutoModelForCausalLM, - PreTrainedModel, Tensor, } from "@xenova/transformers"; import { List } from "immutable"; diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index a31c17e18..59b1b74a7 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -2,11 +2,12 @@ import { List, Map, Range } from "immutable"; import * as tf from "@tensorflow/tfjs"; import { WeightsContainer } from "#weights/index"; -import { Dataset, Batched } from "#dataset/index"; +import type { Dataset, Batched } from "#dataset/index"; import type { DataFormat, DataType } from "#types/index"; import { Model } from "./model.js"; -import { BatchLogs, EpochLogs } from "./logs.js"; +import type { BatchLogs} from "./logs.js"; +import { EpochLogs } from "./logs.js"; type Serialized = [D, tf.io.ModelArtifacts]; diff --git a/discojs/src/processing/tabular.ts b/discojs/src/processing/tabular.ts index 685baca03..9f9dcc58a 100644 --- a/discojs/src/processing/tabular.ts +++ b/discojs/src/processing/tabular.ts @@ -1,4 +1,4 @@ -import { List } from "immutable"; +import type { List } from "immutable"; /** * Convert a string to a number diff --git a/discojs/src/serialization/model.ts b/discojs/src/serialization/model.ts index 7a004d724..e8d4edc95 100644 --- a/discojs/src/serialization/model.ts +++ b/discojs/src/serialization/model.ts @@ -1,12 +1,13 @@ import type tf from "@tensorflow/tfjs"; import { encode as w_encode, decode as w_decode } from "#serialization/weights"; -import { GPTConfig, GPT, TFJS } from "#models/index"; -import type { Model } from "#models/index"; +import { GPT, TFJS } from "#models/index"; +import type { Model , GPTConfig} from "#models/index"; import type { DataType } from "#types/index"; import * as coder from "./coder.js"; -import { Encoded, isEncoded } from "./coder.js"; +import type { Encoded} from "./coder.js"; +import { isEncoded } from "./coder.js"; const Type = { TFJS: 0, diff --git a/discojs/src/serialization/weights.ts b/discojs/src/serialization/weights.ts index 5a40257ef..c7ee21190 100644 --- a/discojs/src/serialization/weights.ts +++ b/discojs/src/serialization/weights.ts @@ -2,7 +2,7 @@ import * as tf from "@tensorflow/tfjs"; import { WeightsContainer } from "#weights/index"; -import { Encoded } from "./coder.js"; +import type { Encoded } from "./coder.js"; import * as coder from "./coder.js"; type Serialized = { diff --git a/discojs/src/task/task_handler.ts b/discojs/src/task/task_handler.ts index 7b87555b4..d7f388da1 100644 --- a/discojs/src/task/task_handler.ts +++ b/discojs/src/task/task_handler.ts @@ -2,11 +2,12 @@ import { Map, Seq } from "immutable"; import type { DataType, Network } from "#types/index"; import type { Model } from "#models/index"; +import type { + JSONLike} from "#serialization/index"; import { serializeTaskToJSON, deserializeTaskFromJSON, - modelEncode, - JSONLike, + modelEncode } from "#serialization/index"; import type { Task } from "./task.js"; diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index 4672c5796..51418bee2 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -5,8 +5,9 @@ import type { Batched } from "#dataset/index"; import type { Aggregator } from "#aggregator/index"; import { Dataset } from "#dataset/index"; -import { ConsoleLogger, Logger } from "#logging/index"; -import { BatchLogs, EpochLogs } from "#models/index"; +import type { Logger } from "#logging/index"; +import { ConsoleLogger } from "#logging/index"; +import type { BatchLogs, EpochLogs } from "#models/index"; import { getAggregator } from "#aggregator/index"; import { enumerate, split } from "#utils/async_iterator"; import { EventEmitter } from "#utils/event_emitter"; @@ -15,7 +16,8 @@ import * as clients from "#client/index"; import * as processing from "#processing/index"; import * as async_iterator from "#utils/async_iterator"; -import { RoundLogs, Trainer } from "#training/trainer"; +import type { RoundLogs} from "#training/trainer"; +import { Trainer } from "#training/trainer"; import type { RoundStatus, SummaryLogs } from "#training/types"; interface DiscoConfig { diff --git a/discojs/src/training/trainer.ts b/discojs/src/training/trainer.ts index ee7835399..023b03766 100644 --- a/discojs/src/training/trainer.ts +++ b/discojs/src/training/trainer.ts @@ -6,10 +6,10 @@ import type { DataFormat, DataType, Network } from "#types/index"; import type { Batched } from "#dataset/index"; import type { Task } from "#task/index"; -import { Dataset } from "#dataset/index"; +import type { Dataset } from "#dataset/index"; import type { BatchLogs, EpochLogs, ValidationMetrics } from "#models/index"; import { WeightsContainer } from "#weights/index"; -import { Client } from "#client/index"; +import type { Client } from "#client/index"; import * as async_iterator from "#utils/async_iterator"; import * as privacy from "#root/privacy"; diff --git a/discojs/src/training/types.ts b/discojs/src/training/types.ts index a6e4290e6..87be9e6ad 100644 --- a/discojs/src/training/types.ts +++ b/discojs/src/training/types.ts @@ -1,4 +1,4 @@ -import { List } from "immutable"; +import type { List } from "immutable"; export type WeightNormHistory = List>; diff --git a/discojs/src/types/data_format.ts b/discojs/src/types/data_format.ts index 4927b4e73..6cce455aa 100644 --- a/discojs/src/types/data_format.ts +++ b/discojs/src/types/data_format.ts @@ -1,7 +1,7 @@ -import { List } from "immutable"; +import type { List } from "immutable"; import type { Image, Tabular, Text, TokenizedText } from "#dataset/index"; -import { NormalizedImage } from "#processing/image"; +import type { NormalizedImage } from "#processing/image"; /** * The data & label format goes through various stages. diff --git a/eslint.config.js b/eslint.config.js index 72b6e8cf2..ee752d053 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -50,6 +50,7 @@ export default defineConfigWithVueTs( typesToIgnore: ["Model", "DataType"], }, ], + "@typescript-eslint/consistent-type-imports": "error", }, }, { diff --git a/server/src/controllers/federated_controller.ts b/server/src/controllers/federated_controller.ts index e8e4871bf..acce39942 100644 --- a/server/src/controllers/federated_controller.ts +++ b/server/src/controllers/federated_controller.ts @@ -1,13 +1,13 @@ import createDebug from "debug"; -import WebSocket from "ws"; +import type WebSocket from "ws"; import { v4 as randomUUID } from "uuid"; import * as msgpack from "@msgpack/msgpack"; -import type { DataType, Task } from "@epfml/discojs"; +import type { DataType, Task , + Encoded} from "@epfml/discojs"; import { mtype, federatedMessages, - Encoded, weightsEncode, weightsDecode, MeanAggregator, diff --git a/server/src/controllers/training_controller.ts b/server/src/controllers/training_controller.ts index 60af7a491..d091af016 100644 --- a/server/src/controllers/training_controller.ts +++ b/server/src/controllers/training_controller.ts @@ -3,8 +3,8 @@ import type WebSocket from "ws"; import { Map } from "immutable"; import * as msgpack from "@msgpack/msgpack"; -import { NodeID, mtype } from "@epfml/discojs"; -import type { DataType, Network, Task } from "@epfml/discojs"; +import { mtype } from "@epfml/discojs"; +import type { DataType, Network, Task , NodeID} from "@epfml/discojs"; const debug = createDebug("server:controllers"); diff --git a/server/src/model_set.ts b/server/src/model_set.ts index d8cbad5b1..da5cdbd0d 100644 --- a/server/src/model_set.ts +++ b/server/src/model_set.ts @@ -1,11 +1,11 @@ import { Map } from "immutable"; import "@tensorflow/tfjs-node"; -import type { DataType, ModelCardInfo, ModelCard } from "@epfml/discojs"; +import type { DataType, ModelCardInfo, ModelCard , + Model, + Encoded} from "@epfml/discojs"; import { EventEmitter, - Model, - Encoded, modelEncode, isEncoded, } from "@epfml/discojs"; diff --git a/server/src/routes/training_router.ts b/server/src/routes/training_router.ts index 86ed1ef7c..f74ba6186 100644 --- a/server/src/routes/training_router.ts +++ b/server/src/routes/training_router.ts @@ -1,11 +1,12 @@ import express from "express"; import type expressWS from "express-ws"; -import type { Task, DataType, Network } from "@epfml/discojs"; -import { Encoded, modelDecode, weightsEncode } from "@epfml/discojs"; +import type { Task, DataType, Network , Encoded} from "@epfml/discojs"; +import { modelDecode, weightsEncode } from "@epfml/discojs"; import type { TaskSet } from "../task_set.js"; +import type { + TrainingController} from "../controllers/index.js"; import { - TrainingController, FederatedController, DecentralizedController, } from "../controllers/index.js"; diff --git a/server/src/task_set.ts b/server/src/task_set.ts index fb02282e6..3194b616c 100644 --- a/server/src/task_set.ts +++ b/server/src/task_set.ts @@ -1,9 +1,9 @@ import { Map } from "immutable"; import "@tensorflow/tfjs-node"; -import type { DataType, Network, Task } from "@epfml/discojs"; -import { EventEmitter, Encoded } from "@epfml/discojs"; -import { ModelSet } from "./model_set.js"; +import type { DataType, Network, Task , Encoded } from "@epfml/discojs"; +import { EventEmitter } from "@epfml/discojs"; +import type { ModelSet } from "./model_set.js"; type EncodedModel = Encoded; type TaskAndModel = [Task, EncodedModel]; diff --git a/webapp/src/components/containers/ButtonsCard.vue b/webapp/src/components/containers/ButtonsCard.vue index 65666e7df..aeb0552cc 100644 --- a/webapp/src/components/containers/ButtonsCard.vue +++ b/webapp/src/components/containers/ButtonsCard.vue @@ -36,7 +36,7 @@