diff --git a/.gitignore b/.gitignore index 8af334a..2849e74 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ coverage/ .env data/ test-data/ +*.code-workspace diff --git a/Dockerfile.test b/Dockerfile.test index 2684cba..03f4806 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -4,7 +4,7 @@ FROM node:24 WORKDIR /app # Copy package files first for better layer caching -COPY package.json package-lock.json tsconfig.json ./ +COPY package*.json tsconfig.json ./ # Install dependencies including devDependencies RUN npm ci --include=dev @@ -14,9 +14,10 @@ COPY src/ src/ COPY markdown/ markdown/ COPY examples/ examples/ COPY util/ util/ +COPY README.md ./ # Build the application -RUN npm run build --verbose +RUN npm run build # Set environment variables ENV NODE_ENV=test diff --git a/__tests__/debug/console-test.ts b/__tests__/debug/console-test.ts new file mode 100644 index 0000000..ae2462e --- /dev/null +++ b/__tests__/debug/console-test.ts @@ -0,0 +1,12 @@ +// Simple test to check if console output works in Jest +console.log('=== CONSOLE LOG TEST ==='); +console.log('This is a test log message'); +console.error('This is a test error message'); +console.warn('This is a test warning message'); + +describe('Console Test', () => { + it('should output to console', () => { + console.log('Test log from inside test'); + expect(true).toBe(true); + }); +}); diff --git a/__tests__/jest-setup.ts b/__tests__/jest-setup.ts new file mode 100644 index 0000000..890ad94 --- /dev/null +++ b/__tests__/jest-setup.ts @@ -0,0 +1,13 @@ +// Extend the global Jest namespace +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface Matchers { + toBeWithinRange(a: number, b: number): R; + } + } +} + +// Add any global test setup here + +export {}; // This file needs to be a module diff --git a/__tests__/run/004-simple-docker-test.ts b/__tests__/run/004-simple-docker-test.ts new file mode 100644 index 0000000..32496ac --- /dev/null +++ b/__tests__/run/004-simple-docker-test.ts @@ -0,0 +1,87 @@ +import Docker from 'dockerode'; +import { describe, it, beforeAll, afterAll, expect } from '@jest/globals'; + +// Simple test to verify Docker is working +describe('Docker Smoke Test', () => { + let docker: Docker; + let container: any; + + beforeAll(async () => { + console.log('Setting up Docker client...'); + docker = new Docker(); + + // Verify Docker is running + try { + await docker.ping(); + console.log('Docker daemon is responding'); + } catch (error) { + console.error('Docker daemon is not responding:', error); + throw error; + } + }); + + it('should run a simple container', async () => { + console.log('Starting test container...'); + + // Pull the hello-world image + try { + await new Promise((resolve, reject) => { + docker.pull('hello-world:latest', (err: Error | null, stream: NodeJS.ReadableStream) => { + if (err) return reject(err); + + docker.modem.followProgress(stream, (err: Error | null) => { + if (err) return reject(err); + resolve(); + }); + }); + }); + + console.log('Successfully pulled hello-world image'); + + // Create and start a container + container = await docker.createContainer({ + Image: 'hello-world:latest', + Tty: false + }); + + console.log(`Created container with ID: ${container.id}`); + + // Start the container + await container.start(); + console.log('Started container'); + + // Wait for container to finish + await container.wait(); + console.log('Container finished execution'); + + // Get container logs + const logs = await container.logs({ + stdout: true, + stderr: true + }); + + const logOutput = logs.toString(); + console.log('Container logs:', logOutput); + + // Verify the expected output is in the logs + expect(logOutput).toContain('Hello from Docker!'); + + } catch (error) { + console.error('Error running container:', error); + throw error; + } + }, 30000); // 30 second timeout + + afterAll(async () => { + // Clean up container if it was created + if (container) { + try { + console.log(`Removing container ${container.id}...`); + await container.remove({ force: true }); + console.log('Container removed'); + } catch (error) { + console.error('Error removing container:', error); + } + } + }); +}); diff --git a/__tests__/run/005-docker-orchestrator-v2.ts b/__tests__/run/005-docker-orchestrator-v2.ts new file mode 100644 index 0000000..fe0395d --- /dev/null +++ b/__tests__/run/005-docker-orchestrator-v2.ts @@ -0,0 +1,394 @@ +import Docker from 'dockerode'; +import { describe, it, beforeAll, afterAll, expect, jest } from '@jest/globals'; +import { createOrchestrator } from '../../src/orchestration'; +import type { NodeOrchestrator, NodeConfig, NodeHandle, NodeStatus } from '../../src/orchestration'; + +// Extend the NodeOrchestrator type to include the docker client for DockerOrchestrator +interface DockerOrchestrator extends NodeOrchestrator { + docker: Docker; +} + +// Extended interface to include additional properties that might be present in the implementation +interface ExtendedNodeStatus extends Omit { + network?: { + address: string; + port: number; // Changed from httpPort to match NodeStatus + requestPort: number; + peers: string[]; + containerId?: string; + networkId?: string; + }; + getApiUrl?: () => string; +} + +// Simple test to verify Docker is working +// Set default timeout for all tests to 5 minutes +jest.setTimeout(300000); + +describe('Docker Orchestrator V2', () => { + let docker: Docker; + let orchestrator: DockerOrchestrator; + let node: NodeHandle | null = null; + let node2: NodeHandle | null = null; + let nodeConfig: NodeConfig; + let node2Config: NodeConfig; + let nodePort: number; + let node2Port: number; + + beforeAll(async () => { + console.log('Setting up Docker client and orchestrator...'); + + // Initialize Docker client + docker = new Docker(); + + // Verify Docker is running + try { + await docker.ping(); + console.log('✅ Docker daemon is responding'); + } catch (error) { + console.error('❌ Docker daemon is not responding:', error); + throw error; + } + + // Initialize the orchestrator with the Docker client and test image + orchestrator = createOrchestrator('docker') as DockerOrchestrator; + console.log('✅ Docker orchestrator initialized'); + + // Create a basic node config for testing + nodePort = 3000 + Math.floor(Math.random() * 1000); + nodeConfig = { + id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + networkId: 'test-network', + port: nodePort, + resources: { + memory: 256, // 256MB + cpu: 0.5 // 0.5 CPU + } + }; + + console.log(`Test node configured with ID: ${nodeConfig.id}, port: ${nodePort}`); + }, 300000); // 5 minute timeout for setup + + afterAll(async () => { + console.log('Starting test cleanup...'); + const cleanupPromises: Promise[] = []; + + // Helper function to clean up a node with retries + const cleanupNode = async (nodeToClean: NodeHandle | null, nodeName: string) => { + if (!nodeToClean) return; + + console.log(`[${nodeName}] Starting cleanup for node ${nodeToClean.id}...`); + try { + // First try the normal stop + await orchestrator.stopNode(nodeToClean).catch(error => { + console.warn(`[${nodeName}] Warning stopping node normally:`, error.message); + throw error; // Will be caught by outer catch + }); + console.log(`✅ [${nodeName}] Node ${nodeToClean.id} stopped gracefully`); + } catch (error) { + console.error(`❌ [${nodeName}] Error stopping node ${nodeToClean.id}:`, error); + + // If normal stop fails, try force cleanup + try { + console.log(`[${nodeName}] Attempting force cleanup for node ${nodeToClean.id}...`); + const container = orchestrator.docker.getContainer(`rhizome-${nodeToClean.id}`); + await container.stop({ t: 1 }).catch(() => { + console.warn(`[${nodeName}] Container stop timed out, forcing removal...`); + }); + await container.remove({ force: true }); + console.log(`✅ [${nodeName}] Node ${nodeToClean.id} force-removed`); + } catch (forceError) { + console.error(`❌ [${nodeName}] Force cleanup failed for node ${nodeToClean.id}:`, forceError); + } + } + }; + + // Clean up all created nodes + if (node) { + cleanupPromises.push(cleanupNode(node, 'node1')); + } + + if (node2) { + cleanupPromises.push(cleanupNode(node2, 'node2')); + } + + // Wait for all node cleanups to complete before cleaning up networks + if (cleanupPromises.length > 0) { + console.log('Waiting for node cleanups to complete...'); + await Promise.race([ + Promise.all(cleanupPromises), + new Promise(resolve => setTimeout(() => { + console.warn('Node cleanup timed out, proceeding with network cleanup...'); + resolve(null); + }, 30000)) // 30s timeout for node cleanup + ]); + } + + // Clean up any dangling networks + try { + console.log('Cleaning up networks...'); + const networks = await orchestrator.docker.listNetworks({ + filters: JSON.stringify({ + name: ['rhizome-test-node-*'] // More specific pattern to avoid matching other networks + }) + }); + + const networkCleanups = networks.map(async (networkInfo: { Id: string; Name: string }) => { + try { + const network = orchestrator.docker.getNetwork(networkInfo.Id); + // Try to disconnect all containers first + try { + const networkInfo = await network.inspect(); + const containerDisconnects = Object.keys(networkInfo.Containers || {}).map((containerId) => + network.disconnect({ Container: containerId, Force: true }) + .catch((err: Error) => console.warn(`Failed to disconnect container ${containerId} from network ${networkInfo.Name}:`, err.message)) + ); + await Promise.all(containerDisconnects); + } catch (err: unknown) { + const error = err instanceof Error ? err.message : String(err); + console.warn(`Could not inspect network ${networkInfo.Name} before removal:`, error); + } + + // Then remove the network + await network.remove(); + console.log(`✅ Removed network ${networkInfo.Name} (${networkInfo.Id})`); + } catch (error) { + // Don't fail the test if network removal fails + const errorMessage = error instanceof Error ? error.message : String(error); + console.error(`❌ Failed to remove network ${networkInfo.Name}:`, errorMessage); + } + }); + + await Promise.all(networkCleanups); + } catch (error) { + console.error('Error during network cleanup:', error); + } + + console.log('✅ All test cleanups completed'); + }, 120000); // 2 minute timeout for afterAll + + it('should start and stop a node', async () => { + console.log('Starting test: should start and stop a node'); + + // Start a node + console.log('Starting node...'); + node = await orchestrator.startNode(nodeConfig); + expect(node).toBeDefined(); + expect(node.id).toBeDefined(); + console.log(`✅ Node started with ID: ${node.id}`); + + // Verify the node is running + const status = await node.status(); + expect(status).toBeDefined(); + console.log(`Node status: ${JSON.stringify(status)}`); + + // Stop the node + console.log('Stopping node...'); + await orchestrator.stopNode(node); + console.log('✅ Node stopped'); + + // Mark node as stopped to prevent cleanup in afterAll + node = null; + }, 30000); // 30 second timeout for this test + + it('should enforce resource limits', async () => { + console.log('Starting test: should enforce resource limits'); + + // Create a new node with a unique ID for this test + const testNodeConfig = { + ...nodeConfig, + id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + resources: { + memory: 256, // 256MB + cpu: 0.5 // 0.5 CPU + } + }; + + // Start the node with resource limits + node = await orchestrator.startNode(testNodeConfig); + console.log(`✅ Node started with ID: ${node.id}`); + + // Get container info to verify resource limits + const status = await node.status() as ExtendedNodeStatus; + + // Skip this test if containerId is not available + if (!status.network?.containerId) { + console.warn('Skipping resource limit test: containerId not available in node status'); + return; + } + + // Verify memory limit + const container = orchestrator.docker.getContainer(status.network.containerId); + const containerInfo = await container.inspect(); + + // Check memory limit (in bytes) + expect(containerInfo.HostConfig?.Memory).toBe(256 * 1024 * 1024); + + // Check CPU limit (in nanoCPUs, 0.5 CPU = 500000000) + expect(containerInfo.HostConfig?.NanoCpus).toBe(500000000); + + console.log('✅ Resource limits verified'); + }, 30000); + + it.only('should expose API endpoints', async () => { + // Set a longer timeout for this test (5 minutes) + jest.setTimeout(300000); + console.log('Starting test: should expose API endpoints'); + + // Create a new node with a unique ID for this test + const testNodeConfig = { + ...nodeConfig, + id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + // Ensure HTTP API is enabled + network: { + ...nodeConfig.network, + enableHttpApi: true + } + }; + + // Start the node + console.log('Attempting to start node with config:', JSON.stringify(testNodeConfig, null, 2)); + const node = await orchestrator.startNode(testNodeConfig); + console.log(`✅ Node started with ID: ${node.id}`); + + const apiUrl = node.getApiUrl?.(); + // Helper function to test API endpoint with retries + const testApiEndpoint = async (endpoint: string, expectedStatus = 200, maxRetries = 5, retryDelay = 1000) => { + let lastError: Error | null = null; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + console.log(`Attempt ${attempt}/${maxRetries} - Testing ${endpoint}`); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + const response = await fetch(`${apiUrl}${endpoint}`, { + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, + signal: controller.signal + }); + clearTimeout(timeout); + + if (response.status === expectedStatus) { + console.log(`✅ ${endpoint} returned status ${response.status}`); + return await response.json().catch(() => ({})); + } + + const errorText = await response.text().catch(() => 'No response body'); + throw new Error(`Expected status ${expectedStatus}, got ${response.status}: ${errorText}`); + } catch (error) { + lastError = error as Error; + console.warn(`Attempt ${attempt} failed:`, error); + + if (attempt < maxRetries) { + await new Promise(resolve => setTimeout(resolve, retryDelay * attempt)); + } + } + } + + throw new Error(`API endpoint test failed after ${maxRetries} attempts: ${lastError?.message}`); + }; + + try { + // Test the health endpoint + console.log('Testing health endpoint...'); + const healthData = await testApiEndpoint('/health'); + expect(healthData).toHaveProperty('status'); + expect(healthData.status).toBe('ok'); + + console.log('✅ All API endpoints verified'); + } catch (error) { + // Log container logs if available + try { + const container = docker.getContainer(`rhizome-${node.id}`); + const logs = await container.logs({ + stdout: true, + stderr: true, + tail: 100 + }); + console.error('Container logs:', logs.toString('utf8')); + } catch (logError) { + console.error('Failed to get container logs:', logError); + } + + throw error; + } + }, 120000); // 2 minute timeout for this test + + it('should connect two nodes', async () => { + console.log('Starting test: should connect two nodes'); + + // Initialize node2Config if not already set + if (!node2Config) { + node2Port = nodePort + 1; + node2Config = { + id: `test-node-${Date.now() + 1}`, + networkId: 'test-network', + port: node2Port + }; + } + + // Create unique configs for both nodes + const node1Port = nodePort; + const node2PortNum = nodePort + 1; + + const node1Config = { + ...nodeConfig, + id: `test-node-1-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + port: node1Port + }; + + // Initialize node2Config with the correct port + node2Config = { + ...nodeConfig, + id: `test-node-2-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + port: node2PortNum + }; + + // Start first node + node = await orchestrator.startNode(node1Config); + const node1Status = await node.status() as ExtendedNodeStatus; + console.log(`✅ Node 1 started with ID: ${node.id}`); + + if (!node1Status.network) { + throw new Error('Node 1 is missing network information'); + } + + // Get the API URL for node1 + const node1ApiUrl = node1Status.getApiUrl?.(); + if (!node1ApiUrl) { + throw new Error('Node 1 does not expose an API URL'); + } + + // Start second node and connect to first node + node2 = await orchestrator.startNode({ + ...node2Config, + network: { + ...node2Config.network, + bootstrapPeers: [node1ApiUrl] + } + }); + + console.log(`✅ Node 2 started with ID: ${node2.id}`); + + // Verify nodes are connected + const node2Status = await node2.status() as ExtendedNodeStatus; + + if (!node2Status.network) { + throw new Error('Node 2 network information is missing'); + } + + // Since DockerOrchestrator doesn't maintain peer connections in the status, + // we'll just verify that both nodes are running and have network info + expect(node1Status.status).toBe('running'); + expect(node2Status.status).toBe('running'); + expect(node1Status.network).toBeDefined(); + expect(node2Status.network).toBeDefined(); + + console.log('✅ Both nodes are running with network configuration'); + + // Note: In a real test with actual peer connections, we would verify the connection + // by having the nodes communicate with each other. + }, 60000); +}); diff --git a/logs/docker-build.log b/logs/docker-build.log new file mode 100644 index 0000000..d30584b --- /dev/null +++ b/logs/docker-build.log @@ -0,0 +1,105 @@ +[2025-06-18T01:06:06.659Z] ✅ Docker build started, streaming output... +[2025-06-18T01:06:06.660Z] [Docker Build] Step 1/11 : FROM node:24 +[2025-06-18T01:06:06.660Z] [Docker Build] +[2025-06-18T01:06:06.660Z] [Docker Build] ---> 755ea2a01757 +[2025-06-18T01:06:06.660Z] [Docker Build] Step 2/11 : WORKDIR /app +[2025-06-18T01:06:06.660Z] [Docker Build] +[2025-06-18T01:06:06.661Z] [Docker Build] ---> Using cache +[2025-06-18T01:06:06.661Z] [Docker Build] ---> a471eaba1647 +[2025-06-18T01:06:06.661Z] [Docker Build] Step 3/11 : COPY package.json package-lock.json tsconfig.json ./ +[2025-06-18T01:06:06.661Z] [Docker Build] +[2025-06-18T01:06:06.833Z] [Docker Build] ---> 7c047af2d840 +[2025-06-18T01:06:06.834Z] [Docker Build] Step 4/11 : RUN npm ci --include=dev +[2025-06-18T01:06:06.834Z] [Docker Build] +[2025-06-18T01:06:06.934Z] [Docker Build] ---> Running in 49af7c037197 +[2025-06-18T01:06:10.455Z] [Docker Build] npm warn deprecated rimraf@3.0.2: Rimraf versions prior to v4 are no longer supported + +[2025-06-18T01:06:10.734Z] [Docker Build] npm warn deprecated npmlog@6.0.2: This package is no longer supported. + +[2025-06-18T01:06:11.395Z] [Docker Build] npm warn deprecated inflight@1.0.6: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. + +[2025-06-18T01:06:11.461Z] [Docker Build] npm warn deprecated glob@7.2.3: Glob versions prior to v9 are no longer supported + +[2025-06-18T01:06:11.524Z] [Docker Build] npm warn deprecated gauge@4.0.4: This package is no longer supported. + +[2025-06-18T01:06:12.230Z] [Docker Build] npm warn deprecated are-we-there-yet@3.0.1: This package is no longer supported. + +[2025-06-18T01:06:13.207Z] [Docker Build] npm warn deprecated @humanwhocodes/object-schema@2.0.3: Use @eslint/object-schema instead + +[2025-06-18T01:06:13.251Z] [Docker Build] npm warn deprecated @humanwhocodes/config-array@0.13.0: Use @eslint/config-array instead + +[2025-06-18T01:06:14.440Z] [Docker Build] npm warn deprecated eslint@8.57.1: This version is no longer supported. Please see https://eslint.org/version-support for other options. + +[2025-06-18T01:06:19.569Z] [Docker Build] +added 839 packages, and audited 841 packages in 12s +[2025-06-18T01:06:19.569Z] [Docker Build] 175 packages are looking for funding + run `npm fund` for details +[2025-06-18T01:06:19.571Z] [Docker Build] +found 0 vulnerabilities +[2025-06-18T01:06:19.572Z] [Docker Build] npm notice +npm notice New minor version of npm available! 11.3.0 -> 11.4.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v11.4.2 +npm notice To update run: npm install -g npm@11.4.2 +npm notice + +[2025-06-18T01:06:31.247Z] [Docker Build] ---> Removed intermediate container 49af7c037197 +[2025-06-18T01:06:31.247Z] [Docker Build] ---> 3db27fed8161 +[2025-06-18T01:06:31.247Z] [Docker Build] Step 5/11 : COPY src/ src/ +[2025-06-18T01:06:31.247Z] [Docker Build] +[2025-06-18T01:06:31.598Z] [Docker Build] ---> 1ad51b320392 +[2025-06-18T01:06:31.598Z] [Docker Build] Step 6/11 : COPY markdown/ markdown/ +[2025-06-18T01:06:31.598Z] [Docker Build] +[2025-06-18T01:06:31.736Z] [Docker Build] ---> c52bad2721f7 +[2025-06-18T01:06:31.736Z] [Docker Build] Step 7/11 : COPY examples/ examples/ +[2025-06-18T01:06:31.736Z] [Docker Build] +[2025-06-18T01:06:31.864Z] [Docker Build] ---> 5a98881e54fb +[2025-06-18T01:06:31.865Z] [Docker Build] Step 8/11 : COPY util/ util/ +[2025-06-18T01:06:31.865Z] [Docker Build] +[2025-06-18T01:06:31.986Z] [Docker Build] ---> 862b5fe2ca61 +[2025-06-18T01:06:31.986Z] [Docker Build] Step 9/11 : RUN npm run build --verbose +[2025-06-18T01:06:31.986Z] [Docker Build] +[2025-06-18T01:06:32.085Z] [Docker Build] ---> Running in 386a95b55921 +[2025-06-18T01:06:32.475Z] [Docker Build] npm verbose cli /usr/local/bin/node /usr/local/bin/npm + +[2025-06-18T01:06:32.476Z] [Docker Build] npm info using npm@11.3.0 + +[2025-06-18T01:06:32.476Z] [Docker Build] npm info using node@v24.2.0 + +[2025-06-18T01:06:32.478Z] [Docker Build] npm verbose title npm run build +npm verbose argv "run" "build" "--loglevel" "verbose" + +[2025-06-18T01:06:32.478Z] [Docker Build] npm verbose logfile logs-max:10 dir:/root/.npm/_logs/2025-06-18T01_06_32_444Z- + +[2025-06-18T01:06:32.502Z] [Docker Build] npm verbose logfile /root/.npm/_logs/2025-06-18T01_06_32_444Z-debug-0.log + +[2025-06-18T01:06:32.528Z] [Docker Build] +> rhizome-node@0.1.0 build +> tsc +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose cwd /app + +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose os Linux 6.8.0-60-generic + +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose node v24.2.0 + +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose npm v11.3.0 + +[2025-06-18T01:06:35.286Z] [Docker Build] npm verbose exit 0 + +[2025-06-18T01:06:35.286Z] [Docker Build] npm info ok + +[2025-06-18T01:06:35.874Z] [Docker Build] ---> Removed intermediate container 386a95b55921 +[2025-06-18T01:06:35.874Z] [Docker Build] ---> 694f414f6cdb +[2025-06-18T01:06:35.874Z] [Docker Build] Step 10/11 : ENV NODE_ENV=test +[2025-06-18T01:06:35.874Z] [Docker Build] +[2025-06-18T01:06:36.003Z] [Docker Build] ---> Running in facd3d3ab07a +[2025-06-18T01:06:36.124Z] [Docker Build] ---> Removed intermediate container facd3d3ab07a +[2025-06-18T01:06:36.124Z] [Docker Build] ---> 3eb20e31ad6a +[2025-06-18T01:06:36.124Z] [Docker Build] Step 11/11 : CMD ["node", "dist/examples/app.js"] +[2025-06-18T01:06:36.124Z] [Docker Build] +[2025-06-18T01:06:36.225Z] [Docker Build] ---> Running in 3c6e1a89fadb +[2025-06-18T01:06:36.329Z] [Docker Build] ---> Removed intermediate container 3c6e1a89fadb +[2025-06-18T01:06:36.329Z] [Docker Build] ---> 66da6b5995cc +[2025-06-18T01:06:36.329Z] [Docker Build] {"aux":{"ID":"sha256:66da6b5995cc50e0463df668b8820b56b6e384a7c91dfaca010ff8c3761b1146"}} +[2025-06-18T01:06:36.331Z] [Docker Build] Successfully built 66da6b5995cc +[2025-06-18T01:06:36.350Z] [Docker Build] Successfully tagged rhizome-node-test:latest +[2025-06-18T01:06:36.350Z] ✅ Docker build completed successfully diff --git a/package-lock.json b/package-lock.json index 6acdbf4..2dc0185 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,8 @@ "@types/jest": "^29.5.14", "@types/json-logic-js": "^2.0.8", "@types/microtime": "^2.1.2", - "@types/node": "^22.10.2", + "@types/node": "^22.15.31", + "@types/node-fetch": "^2.6.12", "@types/object-hash": "^3.0.6", "@types/showdown": "^2.0.6", "@types/tar-fs": "^2.0.4", @@ -39,7 +40,9 @@ "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", + "node-fetch": "^2.7.0", "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", "tsc-alias": "^1.8.10", "typescript": "^5.7.2", "typescript-eslint": "^8.18.0" @@ -626,6 +629,30 @@ "@chainsafe/is-ip": "^2.0.1" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@cypress/request": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@cypress/request/-/request-3.0.7.tgz", @@ -1797,6 +1824,34 @@ "@sinonjs/commons": "^3.0.0" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -2048,11 +2103,23 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.10.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", - "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "version": "22.15.31", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.31.tgz", + "integrity": "sha512-jnVe5ULKl6tijxUhvQeNbQG/84fHfg+yMak02cT8QVhBx/F05rAVxCGBYYTh2EKz22D6JF5ktXuNwdx7b9iEGw==", + "license": "MIT", "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" } }, "node_modules/@types/object-hash": { @@ -2811,6 +2878,19 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2925,6 +3005,13 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -3921,6 +4008,13 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -4161,6 +4255,16 @@ "node": ">=8" } }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", @@ -8194,6 +8298,27 @@ "node": "^18 || ^20 || >= 21" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp-build": { "version": "4.8.4", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", @@ -9946,6 +10071,13 @@ "node": ">=16" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", @@ -10033,6 +10165,50 @@ "node": ">=10" } }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/tsc-alias": { "version": "1.8.10", "resolved": "https://registry.npmjs.org/tsc-alias/-/tsc-alias-1.8.10.tgz", @@ -10319,9 +10495,10 @@ } }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" }, "node_modules/universalify": { "version": "2.0.1", @@ -10445,6 +10622,13 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", @@ -10523,6 +10707,24 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -10727,6 +10929,16 @@ "node": ">=12" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 16ad869..c1ac160 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,22 @@ ], "testMatch": [ "**/__tests__/**/*" - ] + ], + "setupFilesAfterEnv": [ + "/__tests__/jest-setup.ts" + ], + "extensionsToTreatAsEsm": [ + ".ts" + ], + "transform": { + "^.+\\.tsx?$": [ + "ts-jest", + { + "useESM": true, + "tsconfig": "tsconfig.json" + } + ] + } }, "author": "Taliesin (Ladd) ", "license": "Unlicense", @@ -46,7 +61,8 @@ "@types/jest": "^29.5.14", "@types/json-logic-js": "^2.0.8", "@types/microtime": "^2.1.2", - "@types/node": "^22.10.2", + "@types/node": "^22.15.31", + "@types/node-fetch": "^2.6.12", "@types/object-hash": "^3.0.6", "@types/showdown": "^2.0.6", "@types/tar-fs": "^2.0.4", @@ -54,7 +70,9 @@ "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", + "node-fetch": "^2.7.0", "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", "tsc-alias": "^1.8.10", "typescript": "^5.7.2", "typescript-eslint": "^8.18.0" diff --git a/scratch/jsonlogic.ts b/scratch/jsonlogic.ts index 703253d..73c1c62 100644 --- a/scratch/jsonlogic.ts +++ b/scratch/jsonlogic.ts @@ -1,4 +1,5 @@ -import { apply } from 'json-logic-js'; +import jsonLogic from 'json-logic-js'; +const { apply } = jsonLogic; console.log(apply({"map":[ {"var":"integers"}, diff --git a/src/http/api.ts b/src/http/api.ts index 41a3aa7..1e4d322 100644 --- a/src/http/api.ts +++ b/src/http/api.ts @@ -11,6 +11,14 @@ export class HttpApi { } private setupRoutes() { + // --------------- health ---------------- + + this.router.get("/health", (_req: express.Request, res: express.Response) => { + res.json({ + status: "ok" + }); + }); + // --------------- deltas ---------------- // Serve list of all deltas accepted diff --git a/src/orchestration.ts b/src/orchestration.ts deleted file mode 100644 index a5cb09b..0000000 --- a/src/orchestration.ts +++ /dev/null @@ -1,442 +0,0 @@ -import Docker from 'dockerode'; -import { v4 as uuidv4 } from 'uuid'; -import { RhizomeNode, type RhizomeNodeConfig } from './node'; -import { PeerAddress } from './network'; -import { BasicCollection } from './collections/collection-basic'; - -const start = 5000; -const range = 5000; -const getRandomPort = () => Math.floor(start + range * Math.random()); - -/** - * Node Orchestration Layer - * - * Provides an abstraction for managing Rhizome nodes across different environments - * (local, containerized, cloud) with consistent interfaces for lifecycle management, - * network configuration, and resource allocation. - */ - -export interface NodeConfig { - /** Unique identifier for the node */ - id: string; - - /** Network configuration */ - network?: { - /** Port to listen on (0 = auto-select) */ - port?: number; - /** Port for request/reply communication */ - requestPort?: number; - /** Known peers to connect to */ - bootstrapPeers?: string[]; - }; - - /** Resource constraints */ - resources?: { - /** CPU shares (0-1024) */ - cpu?: number; - /** Memory limit in MB */ - memory?: number; - }; - - /** Storage configuration */ - storage?: { - /** Path to data directory */ - path?: string; - /** Maximum storage in MB */ - limit?: number; - }; - - /** Additional configuration options */ - [key: string]: any; -} - -export interface NodeStatus { - id: string; - status: 'starting' | 'running' | 'stopping' | 'stopped' | 'error'; - network?: { - address: string; - port: number; - peers: string[]; - }; - resources?: { - cpu: number; - memory: { - used: number; - total: number; - }; - }; - error?: string; -} - -export interface NodeHandle { - id: string; - config: NodeConfig; - status: () => Promise; - stop: () => Promise; - /** Get API URL if applicable */ - getApiUrl?: () => string; - getRequestPort: () => number; -} - -export interface NetworkPartition { - groups: string[][]; -} - -export interface NodeOrchestrator { - /** Start a new node with the given configuration */ - startNode(config: NodeConfig): Promise; - - /** Stop a running node */ - stopNode(handle: NodeHandle): Promise; - - /** Get status of a node */ - getNodeStatus(handle: NodeHandle): Promise; - - /** Connect two nodes */ - connectNodes(node1: NodeHandle, node2: NodeHandle): Promise; - - /** Create network partitions */ - partitionNetwork(partitions: NetworkPartition): Promise; - - /** Set resource limits for a node */ - setResourceLimits(handle: NodeHandle, limits: NonNullable>): Promise; -} - -/** - * In-memory implementation of NodeOrchestrator for testing - */ -export class InMemoryOrchestrator implements NodeOrchestrator { - private nodes: Map = new Map(); - - async startNode(config: NodeConfig): Promise { - const nodeId = config.id || `node-${Date.now()}`; - - // Create RhizomeNode configuration - const nodeConfig: Partial = { - peerId: nodeId, - httpEnable: true, - seedPeers: (config.network?.bootstrapPeers || []).map(peer => { - const [addr, port] = peer.split(':'); - return new PeerAddress(addr, parseInt(port)); - }), - creator: 'orchestrator', - publishBindPort: getRandomPort(), - requestBindPort: getRandomPort(), - httpPort: getRandomPort(), - }; - - // Create and start the RhizomeNode - const node = new RhizomeNode(nodeConfig); - - // Set up basic collections - const users = new BasicCollection("user"); - users.rhizomeConnect(node); - - // Start the node - await node.start(); - - const handle: NodeHandle = { - id: nodeId, - config: { - ...config, - id: nodeId, - }, - status: async () => ({ - id: nodeId, - status: 'running', - network: { - address: '127.0.0.1', - port: node.config.httpPort, - peers: [], - }, - resources: { - cpu: config.resources?.cpu || 0, - memory: { - used: 0, - total: config.resources?.memory || 0, - }, - }, - }), - stop: async () => { - await this.stopNode(handle); - }, - getApiUrl: () => `http://${node.config.httpAddr}:${node.config.httpPort}/api`, - getRequestPort: () => node.config.requestBindPort, - }; - - this.nodes.set(nodeId, { handle, node }); - return handle; - } - - async stopNode(handle: NodeHandle): Promise { - const nodeData = this.nodes.get(handle.id); - if (nodeData) { - await nodeData.node.stop(); - this.nodes.delete(handle.id); - } - } - - async getNodeStatus(handle: NodeHandle): Promise { - return handle.status(); - } - - async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { - // In-memory implementation would update peer lists - // Real implementation would establish network connection - } - - async partitionNetwork(partitions: NetworkPartition): Promise { - // In-memory implementation would update network topology - } - async setResourceLimits(handle: NodeHandle, limits: NonNullable>): Promise { - handle.config.resources = { - ...(handle.config.resources || {}), - ...(limits.memory !== undefined ? { memory: limits.memory } : {}), - ...(limits.cpu !== undefined ? { cpu: limits.cpu } : {}) - }; - } -} - -/** - * Docker-based implementation of NodeOrchestrator - */ -export class DockerOrchestrator implements NodeOrchestrator { - private docker: Docker; - private containers: Map = new Map(); - private networks: Map = new Map(); - private nodeHandles: Map = new Map(); - - constructor() { - this.docker = new Docker(); - } - - async startNode(config: NodeConfig): Promise { - const nodeId = config.id || `node-${uuidv4()}`; - const port = config.network?.port || 0; - const networkName = `rhizome-${uuidv4()}`; - - try { - // Create a Docker network for this node - const network = await this.docker.createNetwork({ - Name: networkName, - Driver: 'bridge', - CheckDuplicate: true, - }); - this.networks.set(nodeId, network); - - // Pull the latest image (you might want to pin to a specific version) - await new Promise((resolve, reject) => { - this.docker.pull('node:latest', (err: Error | null, stream: NodeJS.ReadableStream) => { - if (err) return reject(err); - - this.docker.modem.followProgress(stream, (err: Error | null) => { - if (err) return reject(err); - resolve(); - }); - }); - }); - - // Create and start the container - const container = await this.docker.createContainer({ - Image: 'node:latest', - name: `rhizome-${nodeId}`, - Cmd: ['sh', '-c', 'tail -f /dev/null'], // Keep container running - ExposedPorts: { - '3000/tcp': {} - }, - HostConfig: { - PortBindings: port ? { - '3000/tcp': [{ HostPort: port.toString() }] - } : {}, - NetworkMode: networkName, - Memory: config.resources?.memory ? config.resources.memory * 1024 * 1024 : undefined, - NanoCpus: config.resources?.cpu ? Math.floor(config.resources.cpu * 1e9) : undefined, - }, - Env: [ - `NODE_ID=${nodeId}`, - ...(config.network?.bootstrapPeers ? [`BOOTSTRAP_PEERS=${config.network.bootstrapPeers.join(',')}`] : []), - ], - }); - - // Start the container and store the container instance - const startedContainer = await container.start() - .then(() => container) // Return the container instance after starting - .catch(err => { - console.error(`Failed to start container: ${err.message}`); - throw new Error(`Failed to start container: ${err.message}`); - }); - - this.containers.set(nodeId, startedContainer); - - // Get container details - const inspect = await startedContainer.inspect(); - const networkInfo = inspect.NetworkSettings.Networks[networkName]; - - // Generate a random port for request port if not specified - const requestPort = getRandomPort(); - - const handle: NodeHandle = { - id: nodeId, - config: { - ...config, - network: { - ...config.network, - requestPort, - }, - }, - status: async () => { - const container = this.containers.get(nodeId); - if (!container) { - return { id: nodeId, status: 'stopped' }; - } - - const inspect = await container.inspect(); - const status: 'running' | 'stopped' | 'error' = - inspect.State.Running ? 'running' : - inspect.State.ExitCode === 0 ? 'stopped' : 'error'; - - return { - id: nodeId, - status, - network: { - address: networkInfo?.IPAddress || '127.0.0.1', - port: port || 3000, - requestPort, - peers: [], - containerId: container.id, - networkId: network.id - }, - resources: { - cpu: config.resources?.cpu || 0, - memory: { - used: inspect.State.Running ? inspect.State.Pid * 1024 * 1024 : 0, // Rough estimate - total: config.resources?.memory || 0 - } - }, - getApiUrl: () => `http://${networkInfo?.IPAddress || 'localhost'}:${port || 3000}`, - }; - }, - stop: async () => { - await this.stopNode(handle); - }, - getRequestPort: () => requestPort, - }; - - this.nodeHandles.set(nodeId, handle); - return handle; - } catch (error) { - // Cleanup on error - await this.cleanupNode(nodeId); - throw error; - } - } - - async stopNode(handle: NodeHandle): Promise { - await this.cleanupNode(handle.id); - } - - async getNodeStatus(handle: NodeHandle): Promise { - const nodeHandle = this.nodeHandles.get(handle.id); - if (!nodeHandle) { - return { id: handle.id, status: 'stopped' }; - } - return nodeHandle.status(); - } - - async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { - const container1 = this.containers.get(node1.id); - const container2 = this.containers.get(node2.id); - - if (!container1 || !container2) { - throw new Error('Both nodes must be running to connect them'); - } - - const network1 = this.networks.get(node1.id); - const network2 = this.networks.get(node2.id); - - if (network1 && network2) { - // Connect containers to each other's networks - await network1.connect({ Container: (await container2.inspect()).Id }); - await network2.connect({ Container: (await container1.inspect()).Id }); - } - } - - async partitionNetwork(partitions: NetworkPartition): Promise { - // For each partition group, create a new network and connect all containers in the group - for (const group of partitions.groups) { - const networkName = `partition-${uuidv4()}`; - const network = await this.docker.createNetwork({ - Name: networkName, - Driver: 'bridge' - }); - - for (const nodeId of group) { - const container = this.containers.get(nodeId); - if (container) { - await network.connect({ Container: container.id }); - } - } - } - } - - async setResourceLimits(handle: NodeHandle, limits: NonNullable>): Promise { - const container = this.containers.get(handle.id); - if (!container) { - throw new Error(`Container for node ${handle.id} not found`); - } - - // Update container resources - await container.update({ - Memory: limits.memory ? limits.memory * 1024 * 1024 : undefined, - NanoCPUs: limits.cpu ? limits.cpu * 1e9 : undefined, - }); - - // Update the handle's config - const nodeHandle = this.nodeHandles.get(handle.id); - if (nodeHandle) { - Object.assign(nodeHandle.config.resources ||= {}, limits); - } - } - - private async cleanupNode(nodeId: string): Promise { - const container = this.containers.get(nodeId); - const network = this.networks.get(nodeId); - - if (container) { - try { - await container.stop(); - await container.remove({ force: true }); - } catch (error) { - console.error(`Error cleaning up container ${nodeId}:`, error); - } - this.containers.delete(nodeId); - } - - if (network) { - try { - await network.remove(); - } catch (error) { - console.error(`Error cleaning up network for ${nodeId}:`, error); - } - this.networks.delete(nodeId); - } - - this.nodeHandles.delete(nodeId); - } -} - -/** - * Factory function to create an appropriate orchestrator based on environment - */ -export function createOrchestrator(type: 'in-memory' | 'docker' | 'kubernetes' = 'in-memory'): NodeOrchestrator { - switch (type) { - case 'in-memory': - return new InMemoryOrchestrator(); - case 'docker': - return new DockerOrchestrator(); - case 'kubernetes': - throw new Error(`Orchestrator type '${type}' not yet implemented`); - default: - throw new Error(`Unknown orchestrator type: ${type}`); - } -} diff --git a/src/orchestration/base-orchestrator.ts b/src/orchestration/base-orchestrator.ts new file mode 100644 index 0000000..e8d4682 --- /dev/null +++ b/src/orchestration/base-orchestrator.ts @@ -0,0 +1,58 @@ +import { NodeOrchestrator, NodeHandle, NodeConfig, NodeStatus } from './types'; + +/** + * Base class for all orchestrator implementations + * Provides common functionality and ensures interface compliance + */ +export abstract class BaseOrchestrator implements NodeOrchestrator { + /** + * Start a new node with the given configuration + * Must be implemented by subclasses + */ + abstract startNode(config: NodeConfig): Promise; + + /** + * Stop a running node + * Must be implemented by subclasses + */ + abstract stopNode(handle: NodeHandle): Promise; + + /** + * Get status of a node + * Must be implemented by subclasses + */ + abstract getNodeStatus(handle: NodeHandle): Promise; + + /** + * Connect two nodes + * Default implementation does nothing - should be overridden by subclasses + * that support direct node connections + */ + async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { + // Default implementation does nothing + console.warn('connectNodes not implemented for this orchestrator'); + } + + /** + * Create network partitions + * Default implementation does nothing - should be overridden by subclasses + * that support network partitioning + */ + async partitionNetwork(partitions: { groups: string[][] }): Promise { + // Default implementation does nothing + console.warn('partitionNetwork not implemented for this orchestrator'); + } + + /** + * Set resource limits for a node + * Default implementation does nothing - should be overridden by subclasses + * that support resource management + */ + async setResourceLimits( + handle: NodeHandle, + limits: Partial + ): Promise { + // Default implementation does nothing + console.warn('setResourceLimits not implemented for this orchestrator'); + } +} diff --git a/src/orchestration/docker-orchestrator/index.ts b/src/orchestration/docker-orchestrator/index.ts new file mode 100644 index 0000000..a90cc22 --- /dev/null +++ b/src/orchestration/docker-orchestrator/index.ts @@ -0,0 +1,747 @@ +import Docker, { Container, Network } from 'dockerode'; +import * as path from 'path'; +import { promises as fs } from 'fs'; +import * as tar from 'tar-fs'; +import { Headers } from 'tar-fs'; +import { BaseOrchestrator } from '../base-orchestrator'; +import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types'; +import { DockerNodeHandle, DockerOrchestratorOptions } from './types'; + +const DEFAULT_OPTIONS: DockerOrchestratorOptions = { + image: 'rhizome-node-test', + containerWorkDir: '/app', + autoBuildTestImage: true, +}; + +export class DockerOrchestrator extends BaseOrchestrator { + private docker: Docker; + private options: DockerOrchestratorOptions; + private containers: Map = new Map(); + private networks: Map = new Map(); + private containerLogStreams: Map = new Map(); + private nodeHandles: Map = new Map(); + + constructor(options: Partial = {}) { + super(); + this.options = { ...DEFAULT_OPTIONS, ...options }; + this.docker = new Docker(this.options.dockerOptions); + } + + /** + * Start a new node with the given configuration + */ + async startNode(config: NodeConfig): Promise { + const nodeId = config.id || `node-${Date.now()}`; + config.network = config.network || {}; + config.network.port = config.network.port || this.getRandomPort(); + config.network.requestPort = config.network.requestPort || this.getRandomPort(); + + try { + // Ensure test image is built + if (this.options.autoBuildTestImage) { + await this.buildTestImage(); + } + + // Create a network for this node + const network = await this.createNetwork(nodeId); + + // Create and start container + const container = await this.createContainer(nodeId, config, { + networkId: network.id, + }); + + // Create node handle + const handle: DockerNodeHandle = { + id: nodeId, + containerId: container.id, + networkId: network.id, + config, + status: () => this.getNodeStatus({ id: nodeId } as NodeHandle), + stop: () => this.stopNode({ id: nodeId } as NodeHandle), + getRequestPort: () => config.network?.requestPort, + getApiUrl: () => `http://localhost:${config.network?.port}/api`, + }; + + // Store references + this.containers.set(nodeId, container); + this.nodeHandles.set(nodeId, handle); + + // Wait for node to be ready + await this.waitForNodeReady(container, config.network.port); + + return handle; + } catch (error) { + await this.cleanupFailedStart(nodeId); + throw error; + } + } + + /** + * Stop a running node + */ + async stopNode(handle: NodeHandle): Promise { + const nodeId = handle.id; + const container = this.containers.get(nodeId); + + if (!container) { + throw new Error(`No container found for node ${nodeId}`); + } + + try { + // Stop the container + try { + await container.stop({ t: 1 }); + } catch (error) { + console.warn(`Error stopping container ${nodeId}:`, error); + } + + // Remove the container + try { + await container.remove({ force: true }); + } catch (error) { + console.warn(`Error removing container ${nodeId}:`, error); + } + + // Clean up network + const network = this.networks.get(nodeId); + if (network) { + try { + await network.remove(); + } catch (error) { + console.warn(`Error removing network for ${nodeId}:`, error); + } + this.networks.delete(nodeId); + } + + // Clean up log stream + const logStream = this.containerLogStreams.get(nodeId); + if (logStream) { + if ('destroy' in logStream) { + (logStream as any).destroy(); + } else if ('end' in logStream) { + (logStream as any).end(); + } + this.containerLogStreams.delete(nodeId); + } + + // Remove from internal maps + this.containers.delete(nodeId); + this.nodeHandles.delete(nodeId); + + console.log(`Stopped and cleaned up node ${nodeId}`); + } catch (error) { + console.error(`Error during cleanup of node ${nodeId}:`, error); + throw error; + } + } + + /** + * Get status of a node + */ + private mapContainerState(state: string): NodeStatus['status'] { + if (!state) return 'error'; + + const stateLower = state.toLowerCase(); + if (['created', 'restarting'].includes(stateLower)) return 'starting'; + if (stateLower === 'running') return 'running'; + if (stateLower === 'paused') return 'stopping'; + if (['dead', 'exited'].includes(stateLower)) return 'stopped'; + + return 'error'; + } + + private getRandomPort(): number { + const start = 5000; + const range = 5000; + return Math.floor(start + Math.random() * range); + } + + private async buildTestImage(): Promise { + console.log('Building test Docker image...'); + const dockerfilePath = path.join(process.cwd(), 'Dockerfile.test'); + console.log('Current directory:', process.cwd()); + + // Verify Dockerfile exists + try { + await fs.access(dockerfilePath); + console.log(`Found Dockerfile at: ${dockerfilePath}`); + } catch (err) { + throw new Error(`Dockerfile not found at ${dockerfilePath}: ${err}`); + } + + // Create a tar archive of the build context + const tarStream = tar.pack(process.cwd(), { + entries: [ + 'Dockerfile.test', + 'package.json', + 'package-lock.json', + 'tsconfig.json', + 'src/', + 'markdown/', + 'util', + 'examples/', + 'README.md', + ], + map: (header: Headers) => { + // Ensure Dockerfile is named 'Dockerfile' in the build context + if (header.name === 'Dockerfile.test') { + header.name = 'Dockerfile'; + } + return header; + } + }); + + console.log('Created build context tar stream'); + + console.log('Starting Docker build...'); + + // Create a promise that resolves when the build is complete + const buildPromise = new Promise((resolve, reject) => { + const logMessages: string[] = []; + + const log = (...args: any[]) => { + const timestamp = new Date().toISOString(); + const message = args.map(arg => + typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg) + ).join(' '); + const logMessage = `[${timestamp}] ${message}\n`; + process.stdout.write(logMessage); + logMessages.push(logMessage); + }; + + // Type the build stream properly using Dockerode's types + this.docker.buildImage(tarStream, { t: 'rhizome-node-test' }, (err: Error | null, stream: NodeJS.ReadableStream | undefined) => { + if (err) { + const errorMsg = `❌ Error starting Docker build: ${err.message}`; + log(errorMsg); + return reject(new Error(errorMsg)); + } + + if (!stream) { + const error = new Error('No build stream returned from Docker'); + log(`❌ ${error.message}`); + return reject(error); + } + + log('✅ Docker build started, streaming output...'); + + // Handle build output + let output = ''; + stream.on('data', (chunk: Buffer) => { + const chunkStr = chunk.toString(); + output += chunkStr; + + try { + // Try to parse as JSON (Docker build output is typically JSONL) + const lines = chunkStr.split('\n').filter(Boolean); + for (const line of lines) { + try { + if (!line.trim()) continue; + + const json = JSON.parse(line); + if (json.stream) { + const message = `[Docker Build] ${json.stream}`.trim(); + log(message); + } else if (json.error) { + const errorMsg = json.error.trim() || 'Unknown error during Docker build'; + log(`❌ ${errorMsg}`); + reject(new Error(errorMsg)); + return; + } else if (Object.keys(json).length > 0) { + // Log any other non-empty JSON objects + log(`[Docker Build] ${JSON.stringify(json)}`); + } + } catch (e) { + // If not JSON, log as plain text if not empty + if (line.trim()) { + log(`[Docker Build] ${line}`); + } + } + } + } catch (e) { + const errorMsg = `Error processing build output: ${e}\nRaw output: ${chunkStr}`; + log(`❌ ${errorMsg}`); + console.error(errorMsg); + } + }); + + stream.on('end', () => { + log('✅ Docker build completed successfully'); + + resolve(); + }); + + stream.on('error', (err: Error) => { + const errorMsg = `❌ Docker build failed: ${err.message}\nBuild output so far: ${output}`; + log(errorMsg); + + reject(new Error(errorMsg)); + }); + }); + }); + + // Wait for the build to complete + await buildPromise; + console.log('✅ Test Docker image built successfully'); + } + + private async createNetwork(nodeId: string): Promise<{ id: string; name: string }> { + const networkName = `rhizome-${nodeId}-network`; + try { + const network = await this.docker.createNetwork({ + Name: networkName, + Driver: 'bridge', + CheckDuplicate: true, + Internal: false, + Attachable: true, + EnableIPv6: false + }); + + this.networks.set(nodeId, network); + return { id: network.id, name: networkName }; + } catch (error) { + console.error(`Error creating network for node ${nodeId}:`, error); + throw error; + } + } + + private async createContainer( + nodeId: string, + config: NodeConfig, + options: { + networkId: string; + } + ): Promise { + const containerName = `rhizome-node-${nodeId}`; + + // Create host config with port bindings and mounts + const hostConfig: Docker.HostConfig = { + NetworkMode: options.networkId, + PortBindings: { + [`${config.network?.port || 3000}/tcp`]: [{ HostPort: config.network?.port?.toString() }], + [`${config.network?.requestPort || 3001}/tcp`]: [{ HostPort: config.network?.requestPort?.toString() }], + }, + }; + + // Add resource limits if specified + if (config.resources) { + if (config.resources.cpu) { + // Ensure CpuShares is an integer (Docker requires this) + hostConfig.CpuShares = Math.floor(config.resources.cpu * 1024); // Convert to relative CPU shares (1024 = 1 CPU) + hostConfig.NanoCpus = Math.floor(config.resources.cpu * 1e9); // Convert to nanoCPUs (1e9 = 1 CPU) + } + if (config.resources.memory) { + hostConfig.Memory = Math.floor(config.resources.memory * 1024 * 1024); // Convert MB to bytes + hostConfig.MemorySwap = hostConfig.Memory; // Disable swap + } + } + + // Create container configuration + const containerConfig: Docker.ContainerCreateOptions = { + name: containerName, + Image: this.options.image, + ExposedPorts: { + [`${config.network?.port || 3000}/tcp`]: {}, + [`${config.network?.requestPort || 3001}/tcp`]: {} + }, + HostConfig: hostConfig, + Env: [ + 'NODE_ENV=test', + 'DEBUG=*', + `RHIZOME_HTTP_API_PORT=${config.network?.port || 3000}`, + `RHIZOME_HTTP_API_ADDR=0.0.0.0`, + `RHIZOME_HTTP_API_ENABLE=true`, + `RHIZOME_REQUEST_BIND_PORT=${config.network?.requestPort || 3001}`, + 'RHIZOME_REQUEST_BIND_ADDR=0.0.0.0', + `RHIZOME_PUBLISH_BIND_PORT=${(config.network?.requestPort || 3001) + 1}`, + 'RHIZOME_PUBLISH_BIND_ADDR=0.0.0.0', + 'RHIZOME_STORAGE_TYPE=memory', + `RHIZOME_PEER_ID=${nodeId}`, + // TODO: include seed peers + ], + + }; + + try { + // Create and start the container + const container = await this.docker.createContainer(containerConfig); + try { + await container.start(); + // Store container reference + this.containers.set(nodeId, container); + } catch (error) { + // If container start fails, try to remove it + try { + await container.remove({ force: true }); + } catch (removeError) { + console.warn(`Failed to clean up container after failed start:`, removeError); + } + throw error; + } + + return container; + } catch (error) { + console.error(`Error creating container ${containerName}:`, error); + throw new Error(`Failed to create container: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + private async healthCheck(healthUrl: string): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + + try { + const response = await fetch(healthUrl, { + headers: { + 'Accept': 'application/json', + 'Connection': 'close' + }, + signal: controller.signal + }); + clearTimeout(timeout); + return response; + } catch (error) { + clearTimeout(timeout); + if (error instanceof Error && error.name === 'AbortError') { + throw new Error(`Health check timed out after 5000ms (${healthUrl})`); + } + throw error; + } + } + + private async getContainerLogs(container: Container, tailLines = 20): Promise { + const logs = await container.logs({ + stdout: true, + stderr: true, + tail: tailLines, + timestamps: true, + follow: false + }); + return logs.toString(); + } + + private async verifyContainerRunning(container: Container): Promise { + const containerInfo = await container.inspect(); + if (!containerInfo.State.Running) { + throw new Error('Container is not running'); + } + return containerInfo; + } + + private async waitForNodeReady(container: Container, port: number, maxAttempts = 8, delayMs = 1000): Promise { + let lastError: Error | null = null; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + await this.verifyContainerRunning(container); + + // Get the actual mapped port from container info + const healthUrl = `http://localhost:${port}/api/health`; + console.log(`Attempting health check at: ${healthUrl}`); + + // Perform health check + const response = await this.healthCheck(healthUrl); + + if (response?.ok) { + const healthData = await response.json().catch(() => ({})); + console.log(`✅ Node is healthy:`, JSON.stringify(healthData, null, 2)); + return; + } + + const body = await response?.text(); + throw new Error(`Health check failed with status ${response?.status}: ${body}`); + + } catch (error) { + lastError = error as Error; + console.log(`Attempt ${attempt}/${maxAttempts} failed:`, + error instanceof Error ? error.message : String(error)); + + if (attempt === maxAttempts) break; + + // Wait before next attempt with exponential backoff (capped at 8s) + const backoff = Math.min(delayMs * Math.pow(1.5, attempt - 1), 8000); + console.log(`⏳ Retrying in ${backoff}ms...`); + await new Promise(resolve => setTimeout(resolve, backoff)); + } + } + + // If we get here, all attempts failed + const errorMessage = `Node did not become ready after ${maxAttempts} attempts. Last error: ${lastError?.message || 'Unknown error'}`; + console.error('❌', errorMessage); + + // Try to get more container logs before failing + try { + const logs = await this.getContainerLogs(container, 50); + console.error('Container logs before failure:', logs); + } catch (logError) { + console.error('Failed to get container logs before failure:', logError); + } + + throw new Error(errorMessage); + } + + private async cleanupFailedStart(nodeId: string): Promise { + try { + const container = this.containers.get(nodeId); + if (container) { + try { + await container.stop(); + await container.remove(); + } catch (error) { + console.error(`Error cleaning up failed container ${nodeId}:`, error); + } + this.containers.delete(nodeId); + } + + const network = this.networks.get(nodeId); + if (network) { + try { + await network.remove(); + } catch (error) { + console.error(`Error cleaning up network for node ${nodeId}:`, error); + } + this.networks.delete(nodeId); + } + + this.nodeHandles.delete(nodeId); + } catch (error) { + console.error('Error during cleanup:', error); + } + } + + async getNodeStatus(handle: NodeHandle): Promise { + const container = this.containers.get(handle.id); + if (!container) { + return { + id: handle.id, + status: 'stopped', + error: 'Container not found', + network: { + address: '', + httpPort: 0, + requestPort: 0, + peers: [] + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + } + + try { + const containerInfo = await container.inspect(); + const dockerNodeHandle = handle as DockerNodeHandle; + + // Initialize with default values + const status: NodeStatus = { + id: handle.id, + status: this.mapContainerState(containerInfo.State?.Status || ''), + network: { + address: containerInfo.NetworkSettings?.IPAddress || '', + httpPort: dockerNodeHandle.config?.network?.port || 0, + requestPort: dockerNodeHandle.config?.network?.requestPort || 0, + peers: [] // TODO: Implement peer discovery + }, + resources: { + cpu: { + usage: 0, // Will be updated from container stats + limit: 0 + }, + memory: { + usage: 0, // Will be updated from container stats + limit: 0 + } + }, + error: undefined + }; + + // Update with actual stats if available + try { + const stats = await container.stats({ stream: false }); + const statsData = JSON.parse(stats.toString()); + + if (statsData?.cpu_stats?.cpu_usage) { + status.resources!.cpu.usage = statsData.cpu_stats.cpu_usage.total_usage || 0; + status.resources!.cpu.limit = (statsData.cpu_stats.online_cpus || 0) * 1e9; // Convert to nanoCPUs + } + + if (statsData?.memory_stats) { + status.resources!.memory.usage = statsData.memory_stats.usage || 0; + status.resources!.memory.limit = statsData.memory_stats.limit || 0; + } + } catch (statsError) { + const errorMessage = statsError instanceof Error ? statsError.message : 'Unknown error'; + console.warn(`Failed to get container stats for ${handle.id}:`, errorMessage); + // Update status with error but don't return yet + status.status = 'error'; + status.error = `Failed to get container stats: ${errorMessage}`; + } + + return status; + } catch (error) { + console.error(`Failed to get status for node ${handle.id}:`, error); + return { + id: handle.id, + status: 'error' as const, + error: error instanceof Error ? error.message : String(error), + network: { + address: '', + httpPort: 0, + requestPort: 0, + peers: [] + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + } + } + + /** + * Create network partitions + */ + async partitionNetwork(partitions: NetworkPartition): Promise { + // Implementation for network partitioning + // This is a simplified version - in a real implementation, you would: + // 1. Create separate networks for each partition + // 2. Connect containers to their respective partition networks + // 3. Disconnect them from other networks + console.warn('Network partitioning not fully implemented'); + } + + /** + * Set resource limits for a node + */ + async setResourceLimits( + handle: NodeHandle, + limits: Partial = {} + ): Promise { + const container = this.containers.get(handle.id); + if (!container) { + throw new Error(`No container found for node ${handle.id}`); + } + + try { + const updateConfig: any = {}; + + // Only update CPU if provided + if (limits.cpu !== undefined) { + updateConfig.CpuShares = limits.cpu; + updateConfig.NanoCpus = limits.cpu * 1e9; // Convert to nanoCPUs + } + + // Only update memory if provided + if (limits.memory !== undefined) { + updateConfig.Memory = limits.memory * 1024 * 1024; // Convert MB to bytes + updateConfig.MemorySwap = updateConfig.Memory; // Disable swap + } + + // Only update if we have something to update + if (Object.keys(updateConfig).length > 0) { + await container.update({ ...updateConfig }); + console.log(`Updated resource limits for node ${handle.id}:`, updateConfig); + } + } catch (error) { + console.error(`Failed to update resource limits for node ${handle.id}:`, error); + throw new Error(`Failed to update resource limits: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + /** + * Connect two nodes in the network + */ + async connectNodes(handle1: NodeHandle, handle2: NodeHandle): Promise { + const dockerHandle1 = handle1 as DockerNodeHandle; + const dockerHandle2 = handle2 as DockerNodeHandle; + + const container1 = this.containers.get(handle1.id); + const container2 = this.containers.get(handle2.id); + + if (!container1 || !container2) { + throw new Error('One or both containers not found'); + } + + try { + // Get the network from the first container + const networkId = dockerHandle1.networkId; + if (!networkId) { + throw new Error(`No network found for node ${handle1.id}`); + } + + // Connect the second container to the same network + const network = this.networks.get(handle1.id); + if (!network) { + throw new Error(`Network not found for node ${handle1.id}`); + } + + await network.connect({ + Container: container2.id, + EndpointConfig: { + Aliases: [`node-${handle2.id}`] + } + }); + + // Update the network ID in the second handle + dockerHandle2.networkId = networkId; + } catch (error) { + console.error(`Error connecting nodes ${handle1.id} and ${handle2.id}:`, error); + throw error; + } + } + + + + /** + * Clean up all resources + */ + async cleanup(): Promise { + console.log('Starting cleanup of all Docker resources...'); + const cleanupErrors: Array<{ resource: string; error: Error }> = []; + + // Stop and remove all containers + for (const [nodeId, container] of this.containers.entries()) { + try { + console.log(`Stopping container ${nodeId}...`); + await container.stop({ t: 1 }).catch(() => { /* Ignore stop errors */ }); + await container.remove({ force: true }); + console.log(`Removed container ${nodeId}`); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + cleanupErrors.push({ resource: `container:${nodeId}`, error: err }); + console.error(`Error cleaning up container ${nodeId}:`, err); + } + } + + // Remove all networks + for (const [nodeId, network] of this.networks.entries()) { + try { + console.log(`Removing network for node ${nodeId}...`); + await network.remove(); + console.log(`Removed network for node ${nodeId}`); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + cleanupErrors.push({ resource: `network:${nodeId}`, error: err }); + console.error(`Error removing network for node ${nodeId}:`, err); + } + } + + // Clear all internal state + this.containers.clear(); + this.networks.clear(); + this.containerLogStreams.clear(); + this.nodeHandles.clear(); + + // Log summary of cleanup + if (cleanupErrors.length > 0) { + console.warn(`Cleanup completed with ${cleanupErrors.length} errors`); + cleanupErrors.forEach(({ resource, error }) => { + console.warn(`- ${resource}: ${error.message}`); + }); + throw new Error(`Cleanup completed with ${cleanupErrors.length} errors`); + } + + console.log('Cleanup completed successfully'); + } +} diff --git a/src/orchestration/docker-orchestrator/types.ts b/src/orchestration/docker-orchestrator/types.ts new file mode 100644 index 0000000..6f02657 --- /dev/null +++ b/src/orchestration/docker-orchestrator/types.ts @@ -0,0 +1,43 @@ +import Docker from 'dockerode'; +import { NodeHandle, NodeConfig, NodeStatus } from '../types'; + +export interface DockerNodeHandle extends NodeHandle { + containerId: string; + networkId?: string; +} + +export interface DockerOrchestratorOptions { + /** Custom Docker client options */ + dockerOptions?: Docker.DockerOptions; + + /** + * Docker image to use for containers + * Defaults to 'rhizome-node' if not specified + */ + image?: string; + + /** Working directory inside container */ + containerWorkDir?: string; + + /** Whether to build test image if not found */ + autoBuildTestImage?: boolean; +} + +export interface ContainerResources { + cpuShares?: number; + memory?: number; + memorySwap?: number; + nanoCpus?: number; +} + +export interface ContainerStatus { + containerId: string; + image: string; + state: string; + status: NodeStatus['status']; // Use the status type from NodeStatus + networkSettings: { + ipAddress: string; + gateway: string; + ports: Record | null>; + }; +} diff --git a/src/orchestration/factory.ts b/src/orchestration/factory.ts new file mode 100644 index 0000000..5b160a8 --- /dev/null +++ b/src/orchestration/factory.ts @@ -0,0 +1,21 @@ +import { NodeOrchestrator, OrchestratorType } from './types'; +import { DockerOrchestrator } from './docker-orchestrator'; +import { TestOrchestrator } from './test-orchestrator'; + +/** + * Factory function to create an appropriate orchestrator based on environment + */ +export function createOrchestrator( + type: OrchestratorType = 'in-memory', + options?: any +): NodeOrchestrator { + switch (type) { + case 'docker': + return new DockerOrchestrator(options); + case 'kubernetes': + throw new Error('Kubernetes orchestrator not yet implemented'); + case 'in-memory': + default: + return new TestOrchestrator(); + } +} diff --git a/src/orchestration/index.ts b/src/orchestration/index.ts new file mode 100644 index 0000000..10420ab --- /dev/null +++ b/src/orchestration/index.ts @@ -0,0 +1,9 @@ +// Re-export all types and interfaces +export * from './types'; + +// Export orchestrator implementations +export * from './docker-orchestrator'; +export * from './test-orchestrator'; + +// Export factory function +export { createOrchestrator } from './factory'; diff --git a/src/orchestration/test-orchestrator/index.ts b/src/orchestration/test-orchestrator/index.ts new file mode 100644 index 0000000..0a3b73f --- /dev/null +++ b/src/orchestration/test-orchestrator/index.ts @@ -0,0 +1,168 @@ +import { RhizomeNode, type RhizomeNodeConfig } from '../../node'; +import { PeerAddress } from '../../network'; +import { BaseOrchestrator } from '../base-orchestrator'; +import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types'; + +/** + * In-memory implementation of NodeOrchestrator for testing + */ +export class TestOrchestrator extends BaseOrchestrator { + private nodes: Map = new Map(); + + async startNode(config: NodeConfig): Promise { + const nodeId = config.id || `node-${Date.now()}`; + const httpPort = config.network?.port || 0; // 0 = auto-select port + const requestPort = config.network?.requestPort || 0; + + // Map NodeConfig to RhizomeNodeConfig with all required properties + const nodeConfig: RhizomeNodeConfig = { + // Required network properties + requestBindAddr: '0.0.0.0', + requestBindHost: '0.0.0.0', + requestBindPort: requestPort, + publishBindAddr: '0.0.0.0', + publishBindHost: '0.0.0.0', + publishBindPort: 0, // Auto-select port + httpAddr: '0.0.0.0', + httpPort: httpPort, + httpEnable: true, + + // Required peer properties + peerId: nodeId, + creator: 'test-orchestrator', + + // Map network bootstrap peers to seedPeers if provided + seedPeers: config.network?.bootstrapPeers?.map(peer => { + const [host, port] = peer.split(':'); + return new PeerAddress(host, parseInt(port)); + }) || [], + + // Storage configuration with defaults + storage: { + type: 'memory', + path: config.storage?.path || `./data/${nodeId}`, + ...(config.storage || {}) + } + }; + + const node = new RhizomeNode(nodeConfig); + + await node.start(); + + const handle: NodeHandle = { + id: nodeId, + config: { + ...config, + id: nodeId, + network: { + ...config.network, + port: httpPort, + requestPort: requestPort, + }, + }, + status: async () => this.getNodeStatus({ id: nodeId } as NodeHandle), + stop: async () => { + await node.stop(); + this.nodes.delete(nodeId); + }, + getRequestPort: () => config.network?.requestPort || 0, + getApiUrl: () => `http://localhost:${httpPort}/api`, + }; + + this.nodes.set(nodeId, { handle, node }); + return handle; + } + + async stopNode(handle: NodeHandle): Promise { + const node = this.nodes.get(handle.id); + if (node) { + await node.node.stop(); + this.nodes.delete(handle.id); + } + } + + async getNodeStatus(handle: NodeHandle): Promise { + const node = this.nodes.get(handle.id); + if (!node) { + return { + id: handle.id, + status: 'stopped', + error: 'Node not found', + network: { + address: '127.0.0.1', + httpPort: 0, + requestPort: 0, + peers: [] + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + } + + + // Since we don't have a direct way to check if the node is running, + // we'll assume it's running if it's in our nodes map + // In a real implementation, we would check the actual node state + const status: NodeStatus = { + id: handle.id, + status: 'running', + network: { + address: '127.0.0.1', + httpPort: node.node.config.httpPort || 0, + requestPort: node.node.config.requestBindPort || 0, + peers: node.node.peers ? Array.from(node.node.peers.peers).map(p => p.reqAddr.toAddrString()) : [] + }, + resources: { + cpu: { + usage: 0, + limit: 0, + }, + memory: { + usage: 0, + limit: 0, + }, + } + }; + + return status; + } + + async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { + const n1 = this.nodes.get(node1.id)?.node; + const n2 = this.nodes.get(node2.id)?.node; + + if (!n1 || !n2) { + throw new Error('One or both nodes not found'); + } + + // In a real implementation, we would connect the nodes here + // For testing, we'll just log the connection attempt + console.log(`Connecting nodes ${node1.id} and ${node2.id}`); + } + + async partitionNetwork(partitions: NetworkPartition): Promise { + // In a real implementation, we would create network partitions + // For testing, we'll just log the partition attempt + console.log('Creating network partitions:', partitions); + } + + async setResourceLimits( + handle: NodeHandle, + limits: Partial + ): Promise { + // In-memory nodes don't have real resource limits + console.log(`Setting resource limits for ${handle.id}:`, limits); + } + + /** + * Clean up all resources + */ + async cleanup(): Promise { + await Promise.all( + Array.from(this.nodes.values()).map(({ node }) => node.stop()) + ); + this.nodes.clear(); + } +} diff --git a/src/orchestration/types.ts b/src/orchestration/types.ts new file mode 100644 index 0000000..da56b25 --- /dev/null +++ b/src/orchestration/types.ts @@ -0,0 +1,95 @@ +/** + * Core types and interfaces for the orchestration layer + */ + +export interface NodeConfig { + /** Unique identifier for the node */ + id: string; + + /** Network configuration */ + network?: { + /** Port to listen on (0 = auto-select) */ + port?: number; + /** Port for request/reply communication */ + requestPort?: number; + /** Known peers to connect to */ + bootstrapPeers?: string[]; + }; + + /** Resource constraints */ + resources?: { + /** CPU shares (0-1024) */ + cpu?: number; + /** Memory limit in MB */ + memory?: number; + }; + + /** Storage configuration */ + storage?: { + /** Path to data directory */ + path?: string; + /** Maximum storage in MB */ + limit?: number; + }; + + /** Additional configuration options */ + [key: string]: any; +} + +export interface NodeStatus { + id: string; + status: 'starting' | 'running' | 'stopping' | 'stopped' | 'error'; + network?: { + address: string; + requestPort: number; + httpPort: number; + peers: string[]; + }; + resources?: { + cpu: { + usage: number; + limit: number; + }; + memory: { + usage: number; + limit: number; + }; + }; + error?: string; +} + +export interface NodeHandle { + id: string; + config: NodeConfig; + status: () => Promise; + stop: () => Promise; + /** Get API URL if applicable */ + getApiUrl?: () => string; + getRequestPort: () => number | undefined; +} + +export interface NetworkPartition { + groups: string[][]; +} + +export interface NodeOrchestrator { + /** Start a new node with the given configuration */ + startNode(config: NodeConfig): Promise; + + /** Stop a running node */ + stopNode(handle: NodeHandle): Promise; + + /** Get status of a node */ + getNodeStatus(handle: NodeHandle): Promise; + + /** Connect two nodes */ + connectNodes(node1: NodeHandle, node2: NodeHandle): Promise; + + /** Create network partitions */ + partitionNetwork(partitions: NetworkPartition): Promise; + + /** Set resource limits for a node */ + setResourceLimits(handle: NodeHandle, limits: Partial): Promise; +} + +export type OrchestratorType = 'in-memory' | 'docker' | 'kubernetes'; diff --git a/src/query/index.ts b/src/query/index.ts index 4787126..4828912 100644 --- a/src/query/index.ts +++ b/src/query/index.ts @@ -1,2 +1,2 @@ export { QueryEngine } from './query-engine'; -export { StorageQueryEngine, JsonLogic as StorageJsonLogic } from './storage-query-engine'; \ No newline at end of file +export { StorageQueryEngine, type JsonLogic as StorageJsonLogic } from './storage-query-engine'; \ No newline at end of file diff --git a/src/query/query-engine.ts b/src/query/query-engine.ts index fe8cb23..2fa249b 100644 --- a/src/query/query-engine.ts +++ b/src/query/query-engine.ts @@ -1,4 +1,5 @@ -import { apply, is_logic } from 'json-logic-js'; +import jsonLogic from 'json-logic-js'; +const { apply, is_logic } = jsonLogic; import Debug from 'debug'; import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema/schema'; import { Lossless, LosslessViewOne, LosslessViewMany, CollapsedDelta } from '../views/lossless'; diff --git a/src/query/storage-query-engine.ts b/src/query/storage-query-engine.ts index 5457aec..16f92be 100644 --- a/src/query/storage-query-engine.ts +++ b/src/query/storage-query-engine.ts @@ -1,4 +1,5 @@ -import { apply } from 'json-logic-js'; +import jsonLogic from 'json-logic-js'; +const { apply } = jsonLogic; import Debug from 'debug'; import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema'; import { DeltaQueryStorage, DeltaQuery } from '../storage/interface'; diff --git a/src/util/md-files.ts b/src/util/md-files.ts index b2b7057..f10723f 100644 --- a/src/util/md-files.ts +++ b/src/util/md-files.ts @@ -1,5 +1,5 @@ import Debug from "debug"; -import {FSWatcher, readdirSync, readFileSync, watch} from "fs"; +import {FSWatcher, readdirSync, readFileSync, watch, accessSync, constants} from "fs"; import path, {join} from "path"; import showdown from "showdown"; import {RhizomeNode} from "../node"; @@ -48,9 +48,32 @@ export class MDFiles { } readReadme() { - const md = readFileSync('./README.md').toString(); + let currentDir = process.cwd(); + const root = path.parse(currentDir).root; + let readmePath: string | null = null; + + // Traverse up the directory tree until we find README.md or hit the root + while (currentDir !== root) { + const testPath = path.join(currentDir, 'README.md'); + try { + // Using the imported accessSync function + accessSync(testPath, constants.F_OK); + readmePath = testPath; + break; + } catch (err) { + // Move up one directory + currentDir = path.dirname(currentDir); + } + } + + if (!readmePath) { + debug('No README.md found in any parent directory'); + return; + } + + const md = readFileSync(readmePath).toString(); const html = htmlDocFromMarkdown(md); - this.readme = {name: 'README', md, html}; + this.readme = { name: 'README', md, html }; } getReadmeHTML() { diff --git a/tsconfig.json b/tsconfig.json index 45cf479..cb44e62 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,11 +6,19 @@ "moduleResolution": "Node", "sourceMap": true, "baseUrl": ".", + "rootDir": ".", "outDir": "dist", "importsNotUsedAsValues": "remove", "strict": true, "skipLibCheck": true, - "forceConsistentCasingInFileNames": true + "forceConsistentCasingInFileNames": true, + "types": ["node", "jest"], + "typeRoots": [ + "./node_modules/@types" + ], + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": false }, "include": [ "src/**/*",