diff --git a/__tests__/nested-resolution.ts b/__tests__/nested-resolution.ts index f817e71..830fcc9 100644 --- a/__tests__/nested-resolution.ts +++ b/__tests__/nested-resolution.ts @@ -13,7 +13,7 @@ import { RhizomeNode } from '../src/node'; import { Delta } from '../src/core'; import { DefaultSchemaRegistry } from '../src/schema'; import { SchemaBuilder, PrimitiveSchemas, ReferenceSchemas } from '../src/schema'; -import { CommonSchemas } from '../src/test-utils/schemas'; +import { CommonSchemas } from '../util/schemas'; import { TypedCollectionImpl } from '../src/collections'; describe('Nested Object Resolution', () => { diff --git a/__tests__/query.ts b/__tests__/query.ts index f89928b..28a42fc 100644 --- a/__tests__/query.ts +++ b/__tests__/query.ts @@ -2,7 +2,7 @@ import { QueryEngine } from '../src/query'; import { Lossless } from '../src/views'; import { DefaultSchemaRegistry } from '../src/schema'; import { SchemaBuilder, PrimitiveSchemas } from '../src/schema'; -import { CommonSchemas } from '../src/test-utils/schemas'; +import { CommonSchemas } from '../util/schemas'; import { Delta } from '../src/core'; import { RhizomeNode } from '../src/node'; diff --git a/__tests__/run/001-single-node-orchestrated.ts b/__tests__/run/001-single-node-orchestrated.ts new file mode 100644 index 0000000..e2c6eef --- /dev/null +++ b/__tests__/run/001-single-node-orchestrated.ts @@ -0,0 +1,60 @@ +import { createOrchestrator, type NodeConfig } from '../../src/orchestration'; + +describe('Run (Orchestrated)', () => { + const orchestrator = createOrchestrator('in-memory'); + let nodeHandle: any; + let apiUrl: string; + + beforeAll(async () => { + // Configure and start the node + const config: NodeConfig = { + id: 'app-001', + }; + + nodeHandle = await orchestrator.startNode(config); + apiUrl = nodeHandle.getApiUrl(); + }); + + afterAll(async () => { + // Stop the node + if (nodeHandle) { + await orchestrator.stopNode(nodeHandle); + } + }); + + it('can put a new user and fetch it', async () => { + // Create a new record + const createResponse = await fetch(`${apiUrl}/user`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 263, + }, + }), + }); + + const createdUser = await createResponse.json(); + expect(createdUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 263, + }, + }); + + // Read the created record + const getResponse = await fetch(`${apiUrl}/user/peon-1`); + const fetchedUser = await getResponse.json(); + + expect(fetchedUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 263, + }, + }); + }); +}); diff --git a/__tests__/run/002-two-nodes-orchestrated.ts b/__tests__/run/002-two-nodes-orchestrated.ts new file mode 100644 index 0000000..9f8ff8e --- /dev/null +++ b/__tests__/run/002-two-nodes-orchestrated.ts @@ -0,0 +1,117 @@ +import Debug from 'debug'; +import { createOrchestrator } from '../../src/orchestration'; +import type { NodeConfig, NodeHandle } from '../../src/orchestration'; + +const debug = Debug('test:two-orchestrated'); + +describe('Run (Two Nodes Orchestrated)', () => { + const orchestrator = createOrchestrator('in-memory'); + // Define a type that includes all required methods + type FullNodeHandle = NodeHandle & { + getRequestPort: () => number; + getApiUrl: () => string; + }; + + const nodes: FullNodeHandle[] = []; + const nodeIds = ['app-002-A', 'app-002-B']; + + beforeAll(async () => { + // Start first node + const node1Config: NodeConfig = { + id: nodeIds[0], + }; + + const node1 = (await orchestrator.startNode(node1Config)) as FullNodeHandle; + + // Start second node with first node as bootstrap peer + const node2Config: NodeConfig = { + id: nodeIds[1], + network: { + bootstrapPeers: [`localhost:${node1.getRequestPort()}`], + }, + }; + + const node2 = (await orchestrator.startNode(node2Config)) as FullNodeHandle; + + nodes.push(node1, node2); + + // Connect the nodes + await orchestrator.connectNodes(node1, node2); + }); + + afterAll(async () => { + // Stop all nodes in parallel + await Promise.all(nodes.map(node => node && orchestrator.stopNode(node))); + }); + + it('can create a record on node0 and read it from node1', async () => { + const [node0, node1] = nodes; + const node0Url = node0.getApiUrl(); + const node1Url = node1.getApiUrl(); + + debug(`Node 0 URL: ${node0Url}`); + debug(`Node 1 URL: ${node1Url}`); + + // Create a new record on node0 + const createResponse = await fetch(`${node0Url}/user`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 741, + }, + }), + }); + + const createdUser = await createResponse.json(); + expect(createdUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 741, + }, + }); + + // Small delay to allow for synchronization + await new Promise(resolve => setTimeout(resolve, 100)); + + // Read the record from node1 + const getResponse = await fetch(`${node1Url}/user/peon-1`); + const fetchedUser = await getResponse.json(); + + expect(fetchedUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 741, + }, + }); + }); + + it('can demonstrate network partitioning', async () => { + // This test shows how we can simulate network partitions + // For now, it's just a placeholder since we'd need to implement + // the actual partitioning logic in the InMemoryOrchestrator + const [node0, node1] = nodes; + + // Simulate partition (actual implementation would use orchestrator.partitionNetwork) + debug('Simulating network partition between nodes'); + // await orchestrator.partitionNetwork({ + // groups: [[node0.id], [node1.id]] + // }); + + // Test behavior during partition... + + // Heal partition + // await orchestrator.partitionNetwork({ + // groups: [[node0.id, node1.id]] + // }); + + // Test behavior after healing... + + // Mark test as passed (remove once actual test is implemented) + expect(true).toBe(true); + }); +}); diff --git a/__tests__/schema.ts b/__tests__/schema.ts index e83e25d..5ebc2ff 100644 --- a/__tests__/schema.ts +++ b/__tests__/schema.ts @@ -7,7 +7,7 @@ import { ObjectSchema } from '../src/schema'; import { DefaultSchemaRegistry } from '../src/schema'; -import { CommonSchemas } from '../src/test-utils/schemas'; +import { CommonSchemas } from '../util/schemas'; import { TypedCollectionImpl, SchemaValidationError } from '../src/collections'; import { RhizomeNode } from '../src/node'; import { Delta } from '../src/core'; diff --git a/markdown/007-investigation.md b/markdown/007-investigation.md new file mode 100644 index 0000000..fc36cb9 --- /dev/null +++ b/markdown/007-investigation.md @@ -0,0 +1,5 @@ +Network Layers: +- Gossip protocols +- Pub/sub +- RPC + diff --git a/package-lock.json b/package-lock.json index 6e1801b..305e1d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "version": "0.1.0", "license": "Unlicense", "dependencies": { + "@types/dockerode": "^3.3.40", "debug": "^4.4.0", + "dockerode": "^4.0.7", "express": "^4.21.2", "json-logic-js": "^2.0.5", "level": "^9.0.0", @@ -18,6 +20,7 @@ "object-hash": "^3.0.0", "showdown": "^2.1.0", "util": "./util/", + "uuid": "^9.0.0", "zeromq": "^6.1.2" }, "devDependencies": { @@ -31,6 +34,7 @@ "@types/node": "^22.10.2", "@types/object-hash": "^3.0.6", "@types/showdown": "^2.0.6", + "@types/uuid": "^10.0.0", "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", @@ -593,6 +597,12 @@ "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "license": "Apache-2.0" + }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", @@ -659,6 +669,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/@cypress/request/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", @@ -785,6 +804,37 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.4.tgz", + "integrity": "sha512-GsFaMXCkMqkKIvwCQjCrwH+GHbPKBjhwo/8ZuUkWHqbI73Kky9I+pQltrlT0+MWpedCoosda53lgjYfyEPgxBg==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.7.13", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -1339,6 +1389,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", @@ -1625,6 +1685,70 @@ "node": ">= 8" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@rtsao/scc": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", @@ -1764,6 +1888,27 @@ "@types/node": "*" } }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.40", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.40.tgz", + "integrity": "sha512-O1ckSFYbcYv/KcnAHMLCnKQYY8/5+6CRzpsOPcQIePHRX2jG4Gmz8uXPMCXIxTGN9OYkE5eox/L67l2sGY1UYg==", + "license": "MIT", + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -1966,6 +2111,30 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.111", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.111.tgz", + "integrity": "sha512-90sGdgA+QLJr1F9X79tQuEut0gEYIfkX9pydI4XGRgvFo9g2JWswefI+WUSUHPYVBHYSEfTEqBxA5hQvAZB3Mw==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -1973,6 +2142,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -2672,7 +2848,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -3124,6 +3299,41 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -3287,6 +3497,15 @@ "dev": true, "license": "MIT" }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3503,7 +3722,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -3536,7 +3754,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -3549,7 +3766,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, "node_modules/color-support": { @@ -3647,6 +3863,20 @@ "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==", "license": "MIT" }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/create-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", @@ -3944,6 +4174,52 @@ "node": ">=6" } }, + "node_modules/docker-modem": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", + "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.7.tgz", + "integrity": "sha512-R+rgrSRTRdU5mH14PZTCPZtW/zw3HDWNTS/1ZAQpL/5Upe/ye5K9WQkIysu4wBoiMwKynsz0a8qWuGsHgEvSAA==", + "license": "Apache-2.0", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.13", + "docker-modem": "^5.0.6", + "protobufjs": "^7.3.2", + "tar-fs": "~2.1.2", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/doctrine": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", @@ -4081,6 +4357,15 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -4230,7 +4515,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -5171,6 +5455,12 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -5292,7 +5582,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -7475,6 +7764,12 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -7489,6 +7784,12 @@ "dev": true, "license": "MIT" }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -7764,6 +8065,12 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/module-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", @@ -7819,6 +8126,13 @@ "url": "https://github.com/sponsors/raouldeheer" } }, + "node_modules/nan": { + "version": "2.22.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz", + "integrity": "sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==", + "license": "MIT", + "optional": true + }, "node_modules/napi-macros": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", @@ -8065,7 +8379,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -8487,6 +8800,30 @@ "node": ">= 6" } }, + "node_modules/protobufjs": { + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.3.tgz", + "integrity": "sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/protons-runtime": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/protons-runtime/-/protons-runtime-5.5.0.tgz", @@ -8510,6 +8847,16 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -8721,7 +9068,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9176,6 +9522,12 @@ "source-map": "^0.6.0" } }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "license": "ISC" + }, "node_modules/splitargs2": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/splitargs2/-/splitargs2-0.1.3.tgz", @@ -9189,6 +9541,23 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/ssh2": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", + "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.20.0" + } + }, "node_modules/sshpk": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", @@ -9432,6 +9801,40 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tar/node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", @@ -10008,9 +10411,13 @@ } }, "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "license": "MIT", "bin": { "uuid": "dist/bin/uuid" @@ -10223,7 +10630,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -10241,7 +10647,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/write-file-atomic": { @@ -10262,7 +10667,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -10278,7 +10682,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -10297,7 +10700,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "license": "ISC", "engines": { "node": ">=12" diff --git a/package.json b/package.json index 58f6883..c3ff91b 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,9 @@ "author": "Taliesin (Ladd) ", "license": "Unlicense", "dependencies": { + "@types/dockerode": "^3.3.40", "debug": "^4.4.0", + "dockerode": "^4.0.7", "express": "^4.21.2", "json-logic-js": "^2.0.5", "level": "^9.0.0", @@ -34,6 +36,7 @@ "object-hash": "^3.0.0", "showdown": "^2.1.0", "util": "./util/", + "uuid": "^9.0.0", "zeromq": "^6.1.2" }, "devDependencies": { @@ -47,6 +50,7 @@ "@types/node": "^22.10.2", "@types/object-hash": "^3.0.6", "@types/showdown": "^2.0.6", + "@types/uuid": "^10.0.0", "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", @@ -55,4 +59,4 @@ "typescript": "^5.7.2", "typescript-eslint": "^8.18.0" } -} \ No newline at end of file +} diff --git a/src/orchestration.ts b/src/orchestration.ts new file mode 100644 index 0000000..a5cb09b --- /dev/null +++ b/src/orchestration.ts @@ -0,0 +1,442 @@ +import Docker from 'dockerode'; +import { v4 as uuidv4 } from 'uuid'; +import { RhizomeNode, type RhizomeNodeConfig } from './node'; +import { PeerAddress } from './network'; +import { BasicCollection } from './collections/collection-basic'; + +const start = 5000; +const range = 5000; +const getRandomPort = () => Math.floor(start + range * Math.random()); + +/** + * Node Orchestration Layer + * + * Provides an abstraction for managing Rhizome nodes across different environments + * (local, containerized, cloud) with consistent interfaces for lifecycle management, + * network configuration, and resource allocation. + */ + +export interface NodeConfig { + /** Unique identifier for the node */ + id: string; + + /** Network configuration */ + network?: { + /** Port to listen on (0 = auto-select) */ + port?: number; + /** Port for request/reply communication */ + requestPort?: number; + /** Known peers to connect to */ + bootstrapPeers?: string[]; + }; + + /** Resource constraints */ + resources?: { + /** CPU shares (0-1024) */ + cpu?: number; + /** Memory limit in MB */ + memory?: number; + }; + + /** Storage configuration */ + storage?: { + /** Path to data directory */ + path?: string; + /** Maximum storage in MB */ + limit?: number; + }; + + /** Additional configuration options */ + [key: string]: any; +} + +export interface NodeStatus { + id: string; + status: 'starting' | 'running' | 'stopping' | 'stopped' | 'error'; + network?: { + address: string; + port: number; + peers: string[]; + }; + resources?: { + cpu: number; + memory: { + used: number; + total: number; + }; + }; + error?: string; +} + +export interface NodeHandle { + id: string; + config: NodeConfig; + status: () => Promise; + stop: () => Promise; + /** Get API URL if applicable */ + getApiUrl?: () => string; + getRequestPort: () => number; +} + +export interface NetworkPartition { + groups: string[][]; +} + +export interface NodeOrchestrator { + /** Start a new node with the given configuration */ + startNode(config: NodeConfig): Promise; + + /** Stop a running node */ + stopNode(handle: NodeHandle): Promise; + + /** Get status of a node */ + getNodeStatus(handle: NodeHandle): Promise; + + /** Connect two nodes */ + connectNodes(node1: NodeHandle, node2: NodeHandle): Promise; + + /** Create network partitions */ + partitionNetwork(partitions: NetworkPartition): Promise; + + /** Set resource limits for a node */ + setResourceLimits(handle: NodeHandle, limits: NonNullable>): Promise; +} + +/** + * In-memory implementation of NodeOrchestrator for testing + */ +export class InMemoryOrchestrator implements NodeOrchestrator { + private nodes: Map = new Map(); + + async startNode(config: NodeConfig): Promise { + const nodeId = config.id || `node-${Date.now()}`; + + // Create RhizomeNode configuration + const nodeConfig: Partial = { + peerId: nodeId, + httpEnable: true, + seedPeers: (config.network?.bootstrapPeers || []).map(peer => { + const [addr, port] = peer.split(':'); + return new PeerAddress(addr, parseInt(port)); + }), + creator: 'orchestrator', + publishBindPort: getRandomPort(), + requestBindPort: getRandomPort(), + httpPort: getRandomPort(), + }; + + // Create and start the RhizomeNode + const node = new RhizomeNode(nodeConfig); + + // Set up basic collections + const users = new BasicCollection("user"); + users.rhizomeConnect(node); + + // Start the node + await node.start(); + + const handle: NodeHandle = { + id: nodeId, + config: { + ...config, + id: nodeId, + }, + status: async () => ({ + id: nodeId, + status: 'running', + network: { + address: '127.0.0.1', + port: node.config.httpPort, + peers: [], + }, + resources: { + cpu: config.resources?.cpu || 0, + memory: { + used: 0, + total: config.resources?.memory || 0, + }, + }, + }), + stop: async () => { + await this.stopNode(handle); + }, + getApiUrl: () => `http://${node.config.httpAddr}:${node.config.httpPort}/api`, + getRequestPort: () => node.config.requestBindPort, + }; + + this.nodes.set(nodeId, { handle, node }); + return handle; + } + + async stopNode(handle: NodeHandle): Promise { + const nodeData = this.nodes.get(handle.id); + if (nodeData) { + await nodeData.node.stop(); + this.nodes.delete(handle.id); + } + } + + async getNodeStatus(handle: NodeHandle): Promise { + return handle.status(); + } + + async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { + // In-memory implementation would update peer lists + // Real implementation would establish network connection + } + + async partitionNetwork(partitions: NetworkPartition): Promise { + // In-memory implementation would update network topology + } + async setResourceLimits(handle: NodeHandle, limits: NonNullable>): Promise { + handle.config.resources = { + ...(handle.config.resources || {}), + ...(limits.memory !== undefined ? { memory: limits.memory } : {}), + ...(limits.cpu !== undefined ? { cpu: limits.cpu } : {}) + }; + } +} + +/** + * Docker-based implementation of NodeOrchestrator + */ +export class DockerOrchestrator implements NodeOrchestrator { + private docker: Docker; + private containers: Map = new Map(); + private networks: Map = new Map(); + private nodeHandles: Map = new Map(); + + constructor() { + this.docker = new Docker(); + } + + async startNode(config: NodeConfig): Promise { + const nodeId = config.id || `node-${uuidv4()}`; + const port = config.network?.port || 0; + const networkName = `rhizome-${uuidv4()}`; + + try { + // Create a Docker network for this node + const network = await this.docker.createNetwork({ + Name: networkName, + Driver: 'bridge', + CheckDuplicate: true, + }); + this.networks.set(nodeId, network); + + // Pull the latest image (you might want to pin to a specific version) + await new Promise((resolve, reject) => { + this.docker.pull('node:latest', (err: Error | null, stream: NodeJS.ReadableStream) => { + if (err) return reject(err); + + this.docker.modem.followProgress(stream, (err: Error | null) => { + if (err) return reject(err); + resolve(); + }); + }); + }); + + // Create and start the container + const container = await this.docker.createContainer({ + Image: 'node:latest', + name: `rhizome-${nodeId}`, + Cmd: ['sh', '-c', 'tail -f /dev/null'], // Keep container running + ExposedPorts: { + '3000/tcp': {} + }, + HostConfig: { + PortBindings: port ? { + '3000/tcp': [{ HostPort: port.toString() }] + } : {}, + NetworkMode: networkName, + Memory: config.resources?.memory ? config.resources.memory * 1024 * 1024 : undefined, + NanoCpus: config.resources?.cpu ? Math.floor(config.resources.cpu * 1e9) : undefined, + }, + Env: [ + `NODE_ID=${nodeId}`, + ...(config.network?.bootstrapPeers ? [`BOOTSTRAP_PEERS=${config.network.bootstrapPeers.join(',')}`] : []), + ], + }); + + // Start the container and store the container instance + const startedContainer = await container.start() + .then(() => container) // Return the container instance after starting + .catch(err => { + console.error(`Failed to start container: ${err.message}`); + throw new Error(`Failed to start container: ${err.message}`); + }); + + this.containers.set(nodeId, startedContainer); + + // Get container details + const inspect = await startedContainer.inspect(); + const networkInfo = inspect.NetworkSettings.Networks[networkName]; + + // Generate a random port for request port if not specified + const requestPort = getRandomPort(); + + const handle: NodeHandle = { + id: nodeId, + config: { + ...config, + network: { + ...config.network, + requestPort, + }, + }, + status: async () => { + const container = this.containers.get(nodeId); + if (!container) { + return { id: nodeId, status: 'stopped' }; + } + + const inspect = await container.inspect(); + const status: 'running' | 'stopped' | 'error' = + inspect.State.Running ? 'running' : + inspect.State.ExitCode === 0 ? 'stopped' : 'error'; + + return { + id: nodeId, + status, + network: { + address: networkInfo?.IPAddress || '127.0.0.1', + port: port || 3000, + requestPort, + peers: [], + containerId: container.id, + networkId: network.id + }, + resources: { + cpu: config.resources?.cpu || 0, + memory: { + used: inspect.State.Running ? inspect.State.Pid * 1024 * 1024 : 0, // Rough estimate + total: config.resources?.memory || 0 + } + }, + getApiUrl: () => `http://${networkInfo?.IPAddress || 'localhost'}:${port || 3000}`, + }; + }, + stop: async () => { + await this.stopNode(handle); + }, + getRequestPort: () => requestPort, + }; + + this.nodeHandles.set(nodeId, handle); + return handle; + } catch (error) { + // Cleanup on error + await this.cleanupNode(nodeId); + throw error; + } + } + + async stopNode(handle: NodeHandle): Promise { + await this.cleanupNode(handle.id); + } + + async getNodeStatus(handle: NodeHandle): Promise { + const nodeHandle = this.nodeHandles.get(handle.id); + if (!nodeHandle) { + return { id: handle.id, status: 'stopped' }; + } + return nodeHandle.status(); + } + + async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { + const container1 = this.containers.get(node1.id); + const container2 = this.containers.get(node2.id); + + if (!container1 || !container2) { + throw new Error('Both nodes must be running to connect them'); + } + + const network1 = this.networks.get(node1.id); + const network2 = this.networks.get(node2.id); + + if (network1 && network2) { + // Connect containers to each other's networks + await network1.connect({ Container: (await container2.inspect()).Id }); + await network2.connect({ Container: (await container1.inspect()).Id }); + } + } + + async partitionNetwork(partitions: NetworkPartition): Promise { + // For each partition group, create a new network and connect all containers in the group + for (const group of partitions.groups) { + const networkName = `partition-${uuidv4()}`; + const network = await this.docker.createNetwork({ + Name: networkName, + Driver: 'bridge' + }); + + for (const nodeId of group) { + const container = this.containers.get(nodeId); + if (container) { + await network.connect({ Container: container.id }); + } + } + } + } + + async setResourceLimits(handle: NodeHandle, limits: NonNullable>): Promise { + const container = this.containers.get(handle.id); + if (!container) { + throw new Error(`Container for node ${handle.id} not found`); + } + + // Update container resources + await container.update({ + Memory: limits.memory ? limits.memory * 1024 * 1024 : undefined, + NanoCPUs: limits.cpu ? limits.cpu * 1e9 : undefined, + }); + + // Update the handle's config + const nodeHandle = this.nodeHandles.get(handle.id); + if (nodeHandle) { + Object.assign(nodeHandle.config.resources ||= {}, limits); + } + } + + private async cleanupNode(nodeId: string): Promise { + const container = this.containers.get(nodeId); + const network = this.networks.get(nodeId); + + if (container) { + try { + await container.stop(); + await container.remove({ force: true }); + } catch (error) { + console.error(`Error cleaning up container ${nodeId}:`, error); + } + this.containers.delete(nodeId); + } + + if (network) { + try { + await network.remove(); + } catch (error) { + console.error(`Error cleaning up network for ${nodeId}:`, error); + } + this.networks.delete(nodeId); + } + + this.nodeHandles.delete(nodeId); + } +} + +/** + * Factory function to create an appropriate orchestrator based on environment + */ +export function createOrchestrator(type: 'in-memory' | 'docker' | 'kubernetes' = 'in-memory'): NodeOrchestrator { + switch (type) { + case 'in-memory': + return new InMemoryOrchestrator(); + case 'docker': + return new DockerOrchestrator(); + case 'kubernetes': + throw new Error(`Orchestrator type '${type}' not yet implemented`); + default: + throw new Error(`Unknown orchestrator type: ${type}`); + } +} diff --git a/src/test-utils/schemas.ts b/util/schemas.ts similarity index 97% rename from src/test-utils/schemas.ts rename to util/schemas.ts index 35f035e..c0cd46c 100644 --- a/src/test-utils/schemas.ts +++ b/util/schemas.ts @@ -1,4 +1,4 @@ -import { SchemaBuilder } from '../../src/schema'; +import { SchemaBuilder } from '../src/schema'; /** * Common schemas used for testing purposes.