diff --git a/.gitignore b/.gitignore index 8af334a..2849e74 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ coverage/ .env data/ test-data/ +*.code-workspace diff --git a/.windsurf/workflows/mr-summary.md b/.windsurf/workflows/mr-summary.md new file mode 100644 index 0000000..a9973c2 --- /dev/null +++ b/.windsurf/workflows/mr-summary.md @@ -0,0 +1,7 @@ +--- +description: Generate a merge request summary +--- + +- fetch origin/main +- compare the current branch to origin/main +- generate a merge request summary as an untracked file called merge-${branch}.md \ No newline at end of file diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..03f4806 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,26 @@ +FROM node:24 + +# Set working directory +WORKDIR /app + +# Copy package files first for better layer caching +COPY package*.json tsconfig.json ./ + +# Install dependencies including devDependencies +RUN npm ci --include=dev + +# Copy source files +COPY src/ src/ +COPY markdown/ markdown/ +COPY examples/ examples/ +COPY util/ util/ +COPY README.md ./ + +# Build the application +RUN npm run build + +# Set environment variables +ENV NODE_ENV=test + +# Command to run the application +CMD ["node", "dist/examples/app.js"] diff --git a/__tests__/debug/console-test.ts b/__tests__/debug/console-test.ts new file mode 100644 index 0000000..ee4214f --- /dev/null +++ b/__tests__/debug/console-test.ts @@ -0,0 +1,20 @@ +import Debug from 'debug'; + +// Set up debug instances for different log levels +const debug = Debug('rz:test:console'); +const debugError = Debug('rz:test:console:error'); +const debugWarn = Debug('rz:test:console:warn'); + +// Test debug output +// Note: These will only show if DEBUG=rz:* is set in the environment +debug('=== DEBUG LOG TEST ==='); +debug('This is a test debug message'); +debugError('This is a test error message'); +debugWarn('This is a test warning message'); + +describe('Debug Test', () => { + it('should output debug messages when DEBUG is enabled', () => { + debug('Test debug message from inside test'); + expect(true).toBe(true); + }); +}); diff --git a/__tests__/jest-setup.ts b/__tests__/jest-setup.ts new file mode 100644 index 0000000..0f33cc9 --- /dev/null +++ b/__tests__/jest-setup.ts @@ -0,0 +1,22 @@ +// Set up environment variables for tests +process.env.DEBUG = 'rz:*'; + +// Extend the global Jest namespace +declare global { + namespace jest { + interface Matchers { + toBeWithinRange(a: number, b: number): R; + } + } +} + +// Add any global test setup here + +// This is a placeholder test to satisfy Jest's requirement for at least one test +describe('Test Setup', () => { + it('should pass', () => { + expect(true).toBe(true); + }); +}); + +export {}; // This file needs to be a module diff --git a/__tests__/nested-resolution-performance.ts b/__tests__/nested-resolution-performance.ts index e446195..35b7e41 100644 --- a/__tests__/nested-resolution-performance.ts +++ b/__tests__/nested-resolution-performance.ts @@ -8,7 +8,10 @@ * - Circular reference handling at scale */ +import Debug from 'debug'; import { RhizomeNode } from '../src/node'; + +const debug = Debug('rz:test:nested-resolution-performance'); import { Delta } from '../src/core'; import { DefaultSchemaRegistry } from '../src/schema'; import { SchemaBuilder, PrimitiveSchemas, ReferenceSchemas, ArraySchemas } from '../src/schema'; @@ -124,7 +127,7 @@ describe('Nested Object Resolution Performance', () => { } const setupTime = performance.now() - startSetup; - console.log(`Setup time for ${userCount} users with relationships: ${setupTime.toFixed(2)}ms`); + debug(`Setup time for ${userCount} users with relationships: ${setupTime.toFixed(2)}ms`); // Test resolution performance for a user with many connections const testUserId = userIds[50]; // Pick a user in the middle @@ -141,7 +144,7 @@ describe('Nested Object Resolution Performance', () => { ); const resolutionTime = performance.now() - startResolution; - console.log(`Resolution time for user with many connections: ${resolutionTime.toFixed(2)}ms`); + debug(`Resolution time for user with many connections: ${resolutionTime.toFixed(2)}ms`); // Verify the resolution worked expect(nestedView.id).toBe(testUserId); @@ -155,7 +158,7 @@ describe('Nested Object Resolution Performance', () => { const totalNestedObjects = Object.values(nestedView.nestedObjects).reduce( (total, arr) => total + (arr?.length || 0), 0 ); - console.log('Total nested objects resolved:', totalNestedObjects); + debug('Total nested objects resolved: %o', totalNestedObjects); // The test user should have friends, followers, and possibly a mentor expect(Object.keys(nestedView.nestedObjects).length).toBeGreaterThan(0); @@ -218,7 +221,7 @@ describe('Nested Object Resolution Performance', () => { } const setupTime = performance.now() - startSetup; - console.log(`Setup time for chain of ${chainLength} users: ${setupTime.toFixed(2)}ms`); + debug(`Setup time for chain of ${chainLength} users: ${setupTime.toFixed(2)}ms`); // Test resolution from the start of the chain const firstUserId = userIds[0]; @@ -235,7 +238,7 @@ describe('Nested Object Resolution Performance', () => { ); const resolutionTime = performance.now() - startResolution; - console.log(`Resolution time for deep chain (maxDepth=5): ${resolutionTime.toFixed(2)}ms`); + debug(`Resolution time for deep chain (maxDepth=5): ${resolutionTime.toFixed(2)}ms`); // Verify the resolution worked and respected depth limits expect(nestedView.id).toBe(firstUserId); @@ -255,7 +258,7 @@ describe('Nested Object Resolution Performance', () => { } expect(depth).toBeLessThanOrEqual(5); - console.log(`Actual resolved depth: ${depth}`); + debug(`Actual resolved depth: ${depth}`); }); it('should handle circular references in large graphs without performance degradation', async () => { @@ -318,7 +321,7 @@ describe('Nested Object Resolution Performance', () => { } const setupTime = performance.now() - startSetup; - console.log(`Setup time for circular graph with ${userCount} users: ${setupTime.toFixed(2)}ms`); + debug(`Setup time for circular graph with ${userCount} users: ${setupTime.toFixed(2)}ms`); // Test resolution performance with circular references const testUserId = userIds[0]; @@ -335,7 +338,7 @@ describe('Nested Object Resolution Performance', () => { ); const resolutionTime = performance.now() - startResolution; - console.log(`Resolution time for circular graph (maxDepth=3): ${resolutionTime.toFixed(2)}ms`); + debug(`Resolution time for circular graph (maxDepth=3): ${resolutionTime.toFixed(2)}ms`); // Verify the resolution completed without hanging expect(nestedView.id).toBe(testUserId); @@ -352,7 +355,7 @@ describe('Nested Object Resolution Performance', () => { expect(nestedView.nestedObjects.connections.length).toBeLessThanOrEqual(3); } - console.log(`Connections resolved: ${nestedView.nestedObjects.connections?.length || 0}`); + debug(`Connections resolved: ${nestedView.nestedObjects.connections?.length || 0}`); }); }); }); \ No newline at end of file diff --git a/__tests__/nested-resolution.ts b/__tests__/nested-resolution.ts index f817e71..830fcc9 100644 --- a/__tests__/nested-resolution.ts +++ b/__tests__/nested-resolution.ts @@ -13,7 +13,7 @@ import { RhizomeNode } from '../src/node'; import { Delta } from '../src/core'; import { DefaultSchemaRegistry } from '../src/schema'; import { SchemaBuilder, PrimitiveSchemas, ReferenceSchemas } from '../src/schema'; -import { CommonSchemas } from '../src/test-utils/schemas'; +import { CommonSchemas } from '../util/schemas'; import { TypedCollectionImpl } from '../src/collections'; describe('Nested Object Resolution', () => { diff --git a/__tests__/query.ts b/__tests__/query.ts index f89928b..28a42fc 100644 --- a/__tests__/query.ts +++ b/__tests__/query.ts @@ -2,7 +2,7 @@ import { QueryEngine } from '../src/query'; import { Lossless } from '../src/views'; import { DefaultSchemaRegistry } from '../src/schema'; import { SchemaBuilder, PrimitiveSchemas } from '../src/schema'; -import { CommonSchemas } from '../src/test-utils/schemas'; +import { CommonSchemas } from '../util/schemas'; import { Delta } from '../src/core'; import { RhizomeNode } from '../src/node'; diff --git a/__tests__/run/001-single-node-orchestrated.ts b/__tests__/run/001-single-node-orchestrated.ts new file mode 100644 index 0000000..b75c3e1 --- /dev/null +++ b/__tests__/run/001-single-node-orchestrated.ts @@ -0,0 +1,72 @@ +import { createOrchestrator, type NodeConfig } from '../../src/orchestration'; + +// Increase test timeout to 30 seconds +jest.setTimeout(30000); + +describe('Run (Orchestrated)', () => { + const orchestrator = createOrchestrator('in-memory'); + let nodeHandle: any; + let apiUrl: string; + + beforeAll(async () => { + console.time('Test setup'); + console.time('Create config'); + // Configure and start the node + const config: NodeConfig = { + id: 'app-001', + }; + console.timeEnd('Create config'); + + console.time('Start node'); + nodeHandle = await orchestrator.startNode(config); + console.timeEnd('Start node'); + + console.time('Get API URL'); + apiUrl = nodeHandle.getApiUrl(); + console.timeEnd('Get API URL'); + console.timeEnd('Test setup'); + }, 60000); // Increase timeout to 60s for this hook + + afterAll(async () => { + // Stop the node + if (nodeHandle) { + await orchestrator.stopNode(nodeHandle); + } + }); + + it('can put a new user and fetch it', async () => { + // Create a new record + const createResponse = await fetch(`${apiUrl}/user`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 263, + }, + }), + }); + + const createdUser = await createResponse.json(); + expect(createdUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 263, + }, + }); + + // Read the created record + const getResponse = await fetch(`${apiUrl}/user/peon-1`); + const fetchedUser = await getResponse.json(); + + expect(fetchedUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 263, + }, + }); + }); +}); diff --git a/__tests__/run/002-two-nodes-orchestrated.ts b/__tests__/run/002-two-nodes-orchestrated.ts new file mode 100644 index 0000000..829fa9f --- /dev/null +++ b/__tests__/run/002-two-nodes-orchestrated.ts @@ -0,0 +1,134 @@ +import Debug from 'debug'; +import { createOrchestrator } from '../../src/orchestration'; +import type { NodeConfig, NodeHandle } from '../../src/orchestration'; + +// Increase test timeout to 30 seconds +jest.setTimeout(30000); + +const debug = Debug('test:two-orchestrated'); + +describe('Run (Two Nodes Orchestrated)', () => { + const orchestrator = createOrchestrator('in-memory'); + // Define a type that includes all required methods + type FullNodeHandle = NodeHandle & { + getRequestPort: () => number; + getApiUrl: () => string; + }; + + const nodes: FullNodeHandle[] = []; + const nodeIds = ['app-002-A', 'app-002-B']; + + beforeAll(async () => { + console.time('Test setup'); + + // Start first node + console.time('Create node1 config'); + const node1Config: NodeConfig = { + id: nodeIds[0], + }; + console.timeEnd('Create node1 config'); + + console.time('Start node1'); + const node1 = (await orchestrator.startNode(node1Config)) as FullNodeHandle; + console.timeEnd('Start node1'); + + // Start second node with first node as bootstrap peer + console.time('Create node2 config'); + const node2Config: NodeConfig = { + id: nodeIds[1], + network: { + bootstrapPeers: [`localhost:${node1.getRequestPort()}`], + }, + }; + console.timeEnd('Create node2 config'); + + console.time('Start node2'); + const node2 = (await orchestrator.startNode(node2Config)) as FullNodeHandle; + console.timeEnd('Start node2'); + + nodes.push(node1, node2); + + // Connect the nodes + console.time('Connect nodes'); + await orchestrator.connectNodes(node1, node2); + console.timeEnd('Connect nodes'); + + console.timeEnd('Test setup'); + }, 120000); // Increase timeout to 120s for this hook + + afterAll(async () => { + // Stop all nodes in parallel + await Promise.all(nodes.map(node => node && orchestrator.stopNode(node))); + }); + + it('can create a record on node0 and read it from node1', async () => { + const [node0, node1] = nodes; + const node0Url = node0.getApiUrl(); + const node1Url = node1.getApiUrl(); + + debug(`Node 0 URL: ${node0Url}`); + debug(`Node 1 URL: ${node1Url}`); + + // Create a new record on node0 + const createResponse = await fetch(`${node0Url}/user`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 741, + }, + }), + }); + + const createdUser = await createResponse.json(); + expect(createdUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 741, + }, + }); + + // Small delay to allow for synchronization + await new Promise(resolve => setTimeout(resolve, 100)); + + // Read the record from node1 + const getResponse = await fetch(`${node1Url}/user/peon-1`); + const fetchedUser = await getResponse.json(); + + expect(fetchedUser).toMatchObject({ + id: 'peon-1', + properties: { + name: 'Peon', + age: 741, + }, + }); + }); + + it.skip('can demonstrate network partitioning', async () => { + // This test shows how we can simulate network partitions + // For now, it's just a placeholder since we'd need to implement + // the actual partitioning logic in the InMemoryOrchestrator + const [node0, node1] = nodes; + + // Simulate partition (actual implementation would use orchestrator.partitionNetwork) + debug('Simulating network partition between nodes'); + // await orchestrator.partitionNetwork({ + // groups: [[node0.id], [node1.id]] + // }); + + // Test behavior during partition... + + // Heal partition + // await orchestrator.partitionNetwork({ + // groups: [[node0.id, node1.id]] + // }); + + // Test behavior after healing... + + // Mark test as passed (remove once actual test is implemented) + expect(true).toBe(true); + }); +}); diff --git a/__tests__/run/002-two-nodes.ts b/__tests__/run/002-two-nodes.ts index add6aaf..390ebf1 100644 --- a/__tests__/run/002-two-nodes.ts +++ b/__tests__/run/002-two-nodes.ts @@ -17,7 +17,7 @@ describe('Run', () => { apps[0].config.seedPeers.push(apps[1].myRequestAddr); apps[1].config.seedPeers.push(apps[0].myRequestAddr); - await Promise.all(apps.map((app) => app.start(false))); + await Promise.all(apps.map((app) => app.start())); }); afterAll(async () => { diff --git a/__tests__/run/005-docker-orchestrator.ts b/__tests__/run/005-docker-orchestrator.ts new file mode 100644 index 0000000..6a94ff4 --- /dev/null +++ b/__tests__/run/005-docker-orchestrator.ts @@ -0,0 +1,531 @@ +import Docker from 'dockerode'; +import { describe, it, beforeAll, afterAll, expect, jest } from '@jest/globals'; +import Debug from 'debug'; + +const debug = Debug('rz:test:docker-orchestrator-v2'); +import { createOrchestrator } from '../../src/orchestration'; +import type { NodeOrchestrator, NodeConfig, NodeHandle, NodeStatus } from '../../src/orchestration'; +import { ImageManager } from '../../src/orchestration/docker-orchestrator/managers/image-manager'; + +// Extend the NodeOrchestrator type to include the docker client for DockerOrchestrator +interface DockerOrchestrator extends NodeOrchestrator { + docker: Docker; +} + +// Extended interface to include additional properties that might be present in the implementation +interface ExtendedNodeStatus extends Omit { + network?: { + address: string; + port: number; // Changed from httpPort to match NodeStatus + requestPort: number; + peers: string[]; + bootstrapPeers?: string[]; + containerId?: string; + networkId?: string; + }; + getApiUrl?: () => string; +} + +// Simple test to verify Docker is working +// Set default timeout for all tests to 5 minutes +jest.setTimeout(300000); + +describe('Docker Orchestrator V2', () => { + let docker: Docker; + let orchestrator: DockerOrchestrator; + let node: NodeHandle | null = null; + let node2: NodeHandle | null = null; + let nodeConfig: NodeConfig; + let node2Config: NodeConfig; + let nodePort: number; + let node2Port: number; + + beforeAll(async () => { + debug('Setting up Docker client and orchestrator...'); + + // Initialize Docker client + docker = new Docker(); + + // Verify Docker is running + try { + await docker.ping(); + debug('Docker daemon is responding'); + } catch (error) { + debug('Docker daemon is not responding: %o', error); + throw error; + } + + // Initialize the orchestrator with the Docker client and test image + orchestrator = createOrchestrator('docker') as DockerOrchestrator; + debug('Docker orchestrator initialized'); + + // Create a basic node config for testing + nodePort = 3000 + Math.floor(Math.random() * 1000); + nodeConfig = { + id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + networkId: 'test-network', + port: nodePort, + resources: { + memory: 256, // 256MB + cpu: 0.5 // 0.5 CPU + } + }; + + debug(`Test node configured with ID: ${nodeConfig.id}, port: ${nodePort}`); + + const imageManager = new ImageManager(); + await imageManager.buildTestImage(); + }); // 30 second timeout + + afterAll(async () => { + debug('Starting test cleanup...'); + const cleanupPromises: Promise[] = []; + + // Helper function to clean up a node with retries + const cleanupNode = async (nodeToClean: NodeHandle | null, nodeName: string) => { + if (!nodeToClean) return; + + debug(`[${nodeName}] Starting cleanup for node ${nodeToClean.id}...`); + try { + // First try the normal stop + await orchestrator.stopNode(nodeToClean).catch(error => { + debug(`[${nodeName}] Warning stopping node normally: %s`, error.message); + throw error; // Will be caught by outer catch + }); + debug(`[${nodeName}] Node ${nodeToClean.id} stopped gracefully`); + } catch (error) { + debug(`[${nodeName}] Error stopping node ${nodeToClean.id}: %o`, error); + + // If normal stop fails, try force cleanup + try { + debug(`[${nodeName}] Attempting force cleanup for node ${nodeToClean.id}...`); + const container = orchestrator.docker.getContainer(`rhizome-${nodeToClean.id}`); + await container.stop({ t: 1 }).catch(() => { + debug(`[${nodeName}] Container stop timed out, forcing removal...`); + }); + await container.remove({ force: true }); + debug(`[${nodeName}] Node ${nodeToClean.id} force-removed`); + } catch (forceError) { + debug(`[${nodeName}] Force cleanup failed for node ${nodeToClean.id}: %o`, forceError); + } + } + }; + + // Clean up all created nodes + if (node) { + cleanupPromises.push(cleanupNode(node, 'node1')); + } + + if (node2) { + cleanupPromises.push(cleanupNode(node2, 'node2')); + } + + // Wait for all node cleanups to complete before cleaning up networks + if (cleanupPromises.length > 0) { + debug('Waiting for node cleanups to complete...'); + await Promise.race([ + Promise.all(cleanupPromises), + new Promise(resolve => setTimeout(() => { + debug('Node cleanup timed out, proceeding with network cleanup...'); + resolve(null); + }, 30000)) // 30s timeout for node cleanup + ]); + } + + // Clean up any dangling networks using NetworkManager + try { + debug('Cleaning up networks...'); + // Get the network manager from the orchestrator + const networkManager = (orchestrator as any).networkManager; + if (!networkManager) { + debug('Network manager not available for cleanup'); + return; + } + + // Get all networks managed by this test + const networks = Array.from((orchestrator as any).networks.entries() || []); + + const cleanupResults = await networkManager.cleanupNetworks((orchestrator as any).networks); + + // Log any cleanup errors + cleanupResults.forEach(({ resource, error }: { resource: string; error: Error }) => { + if (error) { + debug(`Failed to clean up network ${resource || 'unknown'}: %s`, error.message); + } else { + debug(`Successfully cleaned up network ${resource || 'unknown'}`); + } + }); + } catch (error) { + debug('Error during network cleanup: %o', error); + } + + debug('All test cleanups completed'); + }, 120000); // 2 minute timeout for afterAll + + it('should start and stop a node', async () => { + debug('Starting test: should start and stop a node'); + + // Create a new config with a unique ID for this test + const testNodeConfig = { + ...nodeConfig, + id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + network: { + ...nodeConfig.network, + enableHttpApi: true + } + }; + + // Start a node + debug('Starting node...'); + const testNode = await orchestrator.startNode(testNodeConfig); + expect(testNode).toBeDefined(); + expect(testNode.id).toBeDefined(); + debug(`✅ Node started with ID: ${testNode.id}`); + + try { + // Verify the node is running + const status = await testNode.status(); + expect(status).toBeDefined(); + debug('Node status: %o', status); + + // Verify we can access the health endpoint + const apiUrl = testNode.getApiUrl?.(); + if (apiUrl) { + const response = await fetch(`${apiUrl}/health`); + expect(response.ok).toBe(true); + const health = await response.json(); + expect(health).toHaveProperty('status', 'ok'); + } + + // Stop the node + debug('Stopping node...'); + await orchestrator.stopNode(testNode); + debug('Node stopped'); + } finally { + // Ensure node is cleaned up even if test fails + try { + await orchestrator.stopNode(testNode).catch(() => {}); + } catch (e) { + debug('Error during node cleanup: %o', e); + } + } + }, 30000); // 30 second timeout for this test + + it('should enforce resource limits', async () => { + debug('Starting test: should enforce resource limits'); + + // Create a new node with a unique ID for this test + const testNodeConfig = { + ...nodeConfig, + id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + resources: { + memory: 256, // 256MB + cpu: 0.5 // 0.5 CPU + }, + network: { + ...nodeConfig.network, + enableHttpApi: true + } + }; + + let testNode: NodeHandle | null = null; + + try { + // Start the node with resource limits + testNode = await orchestrator.startNode(testNodeConfig); + debug(`Node started with ID: ${testNode.id}`); + + // Get container info to verify resource limits + const status = await testNode.status() as ExtendedNodeStatus; + + // Verify container ID is available at the root level + if (!status.containerId) { + throw new Error('Container ID not available in node status'); + } + + // Get the container ID from the node status + if (!status.containerId) { + throw new Error('Container ID not available in node status'); + } + + // Get container info using ContainerManager + const container = await (orchestrator as any).containerManager.getContainer(status.containerId); + if (!container) { + throw new Error('Container not found'); + } + + // Get container info + const containerInfo = await container.inspect(); + + // Log container info for debugging + debug('Container info: %o', { + Memory: containerInfo.HostConfig?.Memory, + NanoCpus: containerInfo.HostConfig?.NanoCpus, + CpuQuota: containerInfo.HostConfig?.CpuQuota, + CpuPeriod: containerInfo.HostConfig?.CpuPeriod + }); + + // Check memory limit (in bytes) + expect(containerInfo.HostConfig?.Memory).toBe(256 * 1024 * 1024); + + // Check CPU limit (can be set as NanoCpus or CpuQuota/CpuPeriod) + const expectedCpuNano = 0.5 * 1e9; // 0.5 CPU in nanoCPUs + const actualCpuNano = containerInfo.HostConfig?.NanoCpus; + + // Some Docker versions use CpuQuota/CpuPeriod instead of NanoCpus + if (actualCpuNano === undefined && containerInfo.HostConfig?.CpuQuota && containerInfo.HostConfig?.CpuPeriod) { + const cpuQuota = containerInfo.HostConfig.CpuQuota; + const cpuPeriod = containerInfo.HostConfig.CpuPeriod; + const calculatedCpu = (cpuQuota / cpuPeriod) * 1e9; + expect(Math.round(calculatedCpu)).toBeCloseTo(Math.round(expectedCpuNano), -8); // Allow for small rounding differences + } else { + expect(actualCpuNano).toBe(expectedCpuNano); + } + + debug('Resource limits verified'); + } finally { + // Clean up the test node + if (testNode) { + try { + await orchestrator.stopNode(testNode); + } catch (e) { + debug('Error cleaning up test node: %o', e); + } + } + } + }, 30000); + + it('should expose API endpoints', async () => { + // Set a longer timeout for this test (5 minutes) + jest.setTimeout(300000); + debug('Starting test: should expose API endpoints'); + + // Create a new node with a unique ID for this test + const testNodeConfig = { + ...nodeConfig, + id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + // Ensure HTTP API is enabled + network: { + ...nodeConfig.network, + enableHttpApi: true + } + }; + + // Start the node + debug('Attempting to start node with config: %o', testNodeConfig); + const node = await orchestrator.startNode(testNodeConfig); + debug(`Node started with ID: ${node.id}`); + + const apiUrl = node.getApiUrl?.(); + // Helper function to test API endpoint with retries + const testApiEndpoint = async (endpoint: string, expectedStatus = 200, maxRetries = 5, retryDelay = 1000) => { + let lastError: Error | null = null; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + debug(`Attempt ${attempt}/${maxRetries} - Testing ${endpoint}`); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + const response = await fetch(`${apiUrl}${endpoint}`, { + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, + signal: controller.signal + }); + clearTimeout(timeout); + + if (response.status === expectedStatus) { + debug(`${endpoint} returned status ${response.status}`); + return await response.json().catch(() => ({})); + } + + const errorText = await response.text().catch(() => 'No response body'); + throw new Error(`Expected status ${expectedStatus}, got ${response.status}: ${errorText}`); + } catch (error) { + lastError = error as Error; + debug(`Attempt ${attempt} failed: %o`, error); + + if (attempt < maxRetries) { + await new Promise(resolve => setTimeout(resolve, retryDelay * attempt)); + } + } + } + + throw new Error(`API endpoint test failed after ${maxRetries} attempts: ${lastError?.message}`); + }; + + try { + // Test the health endpoint + debug('Testing health endpoint...'); + const healthData = await testApiEndpoint('/health'); + expect(healthData).toHaveProperty('status'); + expect(healthData.status).toBe('ok'); + + debug('All API endpoints verified'); + } catch (error) { + // Log container logs if available + try { + const container = docker.getContainer(`rhizome-${node.id}`); + const logs = await container.logs({ + stdout: true, + stderr: true, + tail: 100 + }); + debug('Container logs: %s', logs.toString('utf8')); + } catch (logError) { + debug('Failed to get container logs: %o', logError); + } + + throw error; + } + }); + + it.skip('should connect two nodes', async () => { + debug('Starting test: should connect two nodes'); + + // Create unique configs for both nodes + const node1Port = 3000 + Math.floor(Math.random() * 1000); + const node2Port = node1Port + 1; + const networkId = `test-network-${Date.now()}`; + + const node1Config: NodeConfig = { + id: `test-node-1-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + networkId, + network: { + port: node1Port, + requestPort: node1Port + 1000, // Different port for request API + bootstrapPeers: [] + }, + resources: { + memory: 256, + cpu: 0.5 + } + }; + + const node2Config: NodeConfig = { + id: `test-node-2-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + networkId, + network: { + port: node2Port, + requestPort: node2Port + 1000, // Different port for request API + bootstrapPeers: [`/ip4/127.0.0.1/tcp/${node1Port + 1000}`] + }, + resources: { + memory: 256, + cpu: 0.5 + } + }; + + let node1: NodeHandle | null = null; + let node2: NodeHandle | null = null; + + try { + // Start first node + debug('Starting node 1...'); + node1 = await orchestrator.startNode(node1Config); + debug(`Node 1 started with ID: ${node1.id}`); + + // Get node 1's status and API URL + const status1 = await node1.status() as ExtendedNodeStatus; + const node1ApiUrl = node1.getApiUrl?.(); + + // Update node 2's config with node 1's actual address if available + if (status1.network?.address && node2Config.network) { + // This assumes the address is in a format like /ip4/127.0.0.1/tcp/3001 + node2Config.network.bootstrapPeers = [status1.network.address]; + } + + // Start second node + debug('Starting node 2...'); + node2 = await orchestrator.startNode(node2Config); + debug(`Node 2 started with ID: ${node2.id}`); + + // Get node 2's status + const status2 = await node2.status() as ExtendedNodeStatus; + const node2ApiUrl = node2.getApiUrl?.(); + + // Verify both nodes are running + expect(status1).toBeDefined(); + expect(status2).toBeDefined(); + // TODO: this status check is inadequate + debug('Both nodes are running'); + + // Helper function to wait for peers + const waitForPeers = async (nodeHandle: NodeHandle, expectedPeerCount = 1, maxAttempts = 10) => { + for (let i = 0; i < maxAttempts; i++) { + const status = await nodeHandle.status() as ExtendedNodeStatus; + const peerCount = status.network?.peers?.length || 0; + + if (peerCount >= expectedPeerCount) { + debug(`Found ${peerCount} peers after ${i + 1} attempts`); + return true; + } + + debug(`Waiting for peers... (attempt ${i + 1}/${maxAttempts})`); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + return false; + }; + + // Wait for nodes to discover each other + debug('Waiting for nodes to discover each other...'); + const node1Discovered = await waitForPeers(node1); + const node2Discovered = await waitForPeers(node2); + + // Final status check + const finalStatus1 = await node1.status() as ExtendedNodeStatus; + const finalStatus2 = await node2.status() as ExtendedNodeStatus; + + // Log peer information + debug('Node 1 discovered: %o', node1Discovered); + debug('Node 2 discovered: %o', node2Discovered); + debug('Node 1 peers: %o', finalStatus1.network?.peers || 'none'); + debug('Node 2 peers: %o', finalStatus2.network?.peers || 'none'); + debug('Node 1 bootstrapPeers: %o', finalStatus1.network?.bootstrapPeers || 'none'); + debug('Node 2 bootstrapPeers: %o', finalStatus2.network?.bootstrapPeers || 'none'); + + // Log the addresses for debugging + debug('Node 1 address: %o', finalStatus1.network?.address); + debug('Node 2 address: %o', finalStatus2.network?.address); + + // Verify both nodes have network configuration + expect(finalStatus1.network).toBeDefined(); + expect(finalStatus2.network).toBeDefined(); + expect(finalStatus1.network?.address).toBeDefined(); + expect(finalStatus2.network?.address).toBeDefined(); + + // For now, we'll just verify that both nodes are running and have network info + // In a real test, you would want to verify actual communication between nodes + debug('✅ Both nodes are running with network configuration'); + + } finally { + // Clean up nodes + const cleanupPromises = []; + + if (node1) { + debug('Stopping node 1...'); + cleanupPromises.push( + orchestrator.stopNode(node1).catch(e => + debug('Error stopping node 1: %o', e) + ) + ); + } + + if (node2) { + debug('Stopping node 2...'); + cleanupPromises.push( + orchestrator.stopNode(node2).catch(e => + debug('Error stopping node 2: %o', e) + ) + ); + } + + await Promise.all(cleanupPromises); + debug('✅ Both nodes stopped'); + } + + // Note: In a real test with actual peer connections, we would verify the connection + // by having the nodes communicate with each other. + }, 60000); +}); diff --git a/__tests__/schema.ts b/__tests__/schema.ts index e83e25d..5ebc2ff 100644 --- a/__tests__/schema.ts +++ b/__tests__/schema.ts @@ -7,7 +7,7 @@ import { ObjectSchema } from '../src/schema'; import { DefaultSchemaRegistry } from '../src/schema'; -import { CommonSchemas } from '../src/test-utils/schemas'; +import { CommonSchemas } from '../util/schemas'; import { TypedCollectionImpl, SchemaValidationError } from '../src/collections'; import { RhizomeNode } from '../src/node'; import { Delta } from '../src/core'; diff --git a/__tests__/test-utils.ts b/__tests__/test-utils.ts new file mode 100644 index 0000000..590f315 --- /dev/null +++ b/__tests__/test-utils.ts @@ -0,0 +1,98 @@ +import { createOrchestrator } from '../src/orchestration/factory'; +import { NodeConfig, NodeOrchestrator } from '../src/orchestration/types'; +import Debug from 'debug'; + +const debug = Debug('rz:test-utils'); + +// Global test orchestrator instance +let testOrchestrator: NodeOrchestrator; + +// Default test node configuration +const DEFAULT_TEST_NODE_CONFIG: Partial = { + network: { + // Use default ports that will be overridden by getRandomPort() in the orchestrator + port: 0, + }, + storage: { + type: 'memory', + path: '/data', + }, +}; + +/** + * Set up the test environment before all tests run + */ +export const setupTestEnvironment = async () => { + debug('Setting up Docker test environment...'); + + try { + // Create a Docker orchestrator instance + testOrchestrator = createOrchestrator('docker', { + // Enable auto-building of test images + autoBuildTestImage: true, + // Use a specific test image name + image: 'rhizome-node-test', + }); + + debug('Docker test environment setup complete'); + } catch (error) { + debug('Error setting up Docker test environment:', error); + throw error; + } +}; + +/** + * Clean up the test environment after all tests complete + */ +export const teardownTestEnvironment = async () => { + debug('Tearing down Docker test environment...'); + + if (testOrchestrator) { + try { + // Clean up all containers and networks + await testOrchestrator.cleanup(); + debug('Docker resources cleaned up successfully'); + } catch (error) { + debug('Error during Docker environment teardown:', error); + // Don't throw to allow tests to complete + } + } + + debug('Docker test environment teardown complete'); +}; + +/** + * Get the test orchestrator instance + */ +export const getTestOrchestrator = (): NodeOrchestrator => { + if (!testOrchestrator) { + throw new Error('Test orchestrator not initialized. Call setupTestEnvironment() first.'); + } + return testOrchestrator; +}; + +/** + * Create a test node with the given configuration + */ +export const createTestNode = async (config: Partial = {}) => { + const orchestrator = getTestOrchestrator(); + + // Merge default config with provided config + const nodeConfig: NodeConfig = { + ...DEFAULT_TEST_NODE_CONFIG, + ...config, + // Ensure we have a unique ID for each node + id: config.id || `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`, + }; + + debug(`Creating test node with ID: ${nodeConfig.id}`); + + try { + const nodeHandle = await orchestrator.startNode(nodeConfig); + debug(`Test node ${nodeConfig.id} created successfully`); + return nodeHandle; + } catch (error) { + debug(`Error creating test node ${nodeConfig.id}:`, error); + throw error; + } +}; diff --git a/logs/docker-build.log b/logs/docker-build.log new file mode 100644 index 0000000..d30584b --- /dev/null +++ b/logs/docker-build.log @@ -0,0 +1,105 @@ +[2025-06-18T01:06:06.659Z] ✅ Docker build started, streaming output... +[2025-06-18T01:06:06.660Z] [Docker Build] Step 1/11 : FROM node:24 +[2025-06-18T01:06:06.660Z] [Docker Build] +[2025-06-18T01:06:06.660Z] [Docker Build] ---> 755ea2a01757 +[2025-06-18T01:06:06.660Z] [Docker Build] Step 2/11 : WORKDIR /app +[2025-06-18T01:06:06.660Z] [Docker Build] +[2025-06-18T01:06:06.661Z] [Docker Build] ---> Using cache +[2025-06-18T01:06:06.661Z] [Docker Build] ---> a471eaba1647 +[2025-06-18T01:06:06.661Z] [Docker Build] Step 3/11 : COPY package.json package-lock.json tsconfig.json ./ +[2025-06-18T01:06:06.661Z] [Docker Build] +[2025-06-18T01:06:06.833Z] [Docker Build] ---> 7c047af2d840 +[2025-06-18T01:06:06.834Z] [Docker Build] Step 4/11 : RUN npm ci --include=dev +[2025-06-18T01:06:06.834Z] [Docker Build] +[2025-06-18T01:06:06.934Z] [Docker Build] ---> Running in 49af7c037197 +[2025-06-18T01:06:10.455Z] [Docker Build] npm warn deprecated rimraf@3.0.2: Rimraf versions prior to v4 are no longer supported + +[2025-06-18T01:06:10.734Z] [Docker Build] npm warn deprecated npmlog@6.0.2: This package is no longer supported. + +[2025-06-18T01:06:11.395Z] [Docker Build] npm warn deprecated inflight@1.0.6: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. + +[2025-06-18T01:06:11.461Z] [Docker Build] npm warn deprecated glob@7.2.3: Glob versions prior to v9 are no longer supported + +[2025-06-18T01:06:11.524Z] [Docker Build] npm warn deprecated gauge@4.0.4: This package is no longer supported. + +[2025-06-18T01:06:12.230Z] [Docker Build] npm warn deprecated are-we-there-yet@3.0.1: This package is no longer supported. + +[2025-06-18T01:06:13.207Z] [Docker Build] npm warn deprecated @humanwhocodes/object-schema@2.0.3: Use @eslint/object-schema instead + +[2025-06-18T01:06:13.251Z] [Docker Build] npm warn deprecated @humanwhocodes/config-array@0.13.0: Use @eslint/config-array instead + +[2025-06-18T01:06:14.440Z] [Docker Build] npm warn deprecated eslint@8.57.1: This version is no longer supported. Please see https://eslint.org/version-support for other options. + +[2025-06-18T01:06:19.569Z] [Docker Build] +added 839 packages, and audited 841 packages in 12s +[2025-06-18T01:06:19.569Z] [Docker Build] 175 packages are looking for funding + run `npm fund` for details +[2025-06-18T01:06:19.571Z] [Docker Build] +found 0 vulnerabilities +[2025-06-18T01:06:19.572Z] [Docker Build] npm notice +npm notice New minor version of npm available! 11.3.0 -> 11.4.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v11.4.2 +npm notice To update run: npm install -g npm@11.4.2 +npm notice + +[2025-06-18T01:06:31.247Z] [Docker Build] ---> Removed intermediate container 49af7c037197 +[2025-06-18T01:06:31.247Z] [Docker Build] ---> 3db27fed8161 +[2025-06-18T01:06:31.247Z] [Docker Build] Step 5/11 : COPY src/ src/ +[2025-06-18T01:06:31.247Z] [Docker Build] +[2025-06-18T01:06:31.598Z] [Docker Build] ---> 1ad51b320392 +[2025-06-18T01:06:31.598Z] [Docker Build] Step 6/11 : COPY markdown/ markdown/ +[2025-06-18T01:06:31.598Z] [Docker Build] +[2025-06-18T01:06:31.736Z] [Docker Build] ---> c52bad2721f7 +[2025-06-18T01:06:31.736Z] [Docker Build] Step 7/11 : COPY examples/ examples/ +[2025-06-18T01:06:31.736Z] [Docker Build] +[2025-06-18T01:06:31.864Z] [Docker Build] ---> 5a98881e54fb +[2025-06-18T01:06:31.865Z] [Docker Build] Step 8/11 : COPY util/ util/ +[2025-06-18T01:06:31.865Z] [Docker Build] +[2025-06-18T01:06:31.986Z] [Docker Build] ---> 862b5fe2ca61 +[2025-06-18T01:06:31.986Z] [Docker Build] Step 9/11 : RUN npm run build --verbose +[2025-06-18T01:06:31.986Z] [Docker Build] +[2025-06-18T01:06:32.085Z] [Docker Build] ---> Running in 386a95b55921 +[2025-06-18T01:06:32.475Z] [Docker Build] npm verbose cli /usr/local/bin/node /usr/local/bin/npm + +[2025-06-18T01:06:32.476Z] [Docker Build] npm info using npm@11.3.0 + +[2025-06-18T01:06:32.476Z] [Docker Build] npm info using node@v24.2.0 + +[2025-06-18T01:06:32.478Z] [Docker Build] npm verbose title npm run build +npm verbose argv "run" "build" "--loglevel" "verbose" + +[2025-06-18T01:06:32.478Z] [Docker Build] npm verbose logfile logs-max:10 dir:/root/.npm/_logs/2025-06-18T01_06_32_444Z- + +[2025-06-18T01:06:32.502Z] [Docker Build] npm verbose logfile /root/.npm/_logs/2025-06-18T01_06_32_444Z-debug-0.log + +[2025-06-18T01:06:32.528Z] [Docker Build] +> rhizome-node@0.1.0 build +> tsc +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose cwd /app + +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose os Linux 6.8.0-60-generic + +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose node v24.2.0 + +[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose npm v11.3.0 + +[2025-06-18T01:06:35.286Z] [Docker Build] npm verbose exit 0 + +[2025-06-18T01:06:35.286Z] [Docker Build] npm info ok + +[2025-06-18T01:06:35.874Z] [Docker Build] ---> Removed intermediate container 386a95b55921 +[2025-06-18T01:06:35.874Z] [Docker Build] ---> 694f414f6cdb +[2025-06-18T01:06:35.874Z] [Docker Build] Step 10/11 : ENV NODE_ENV=test +[2025-06-18T01:06:35.874Z] [Docker Build] +[2025-06-18T01:06:36.003Z] [Docker Build] ---> Running in facd3d3ab07a +[2025-06-18T01:06:36.124Z] [Docker Build] ---> Removed intermediate container facd3d3ab07a +[2025-06-18T01:06:36.124Z] [Docker Build] ---> 3eb20e31ad6a +[2025-06-18T01:06:36.124Z] [Docker Build] Step 11/11 : CMD ["node", "dist/examples/app.js"] +[2025-06-18T01:06:36.124Z] [Docker Build] +[2025-06-18T01:06:36.225Z] [Docker Build] ---> Running in 3c6e1a89fadb +[2025-06-18T01:06:36.329Z] [Docker Build] ---> Removed intermediate container 3c6e1a89fadb +[2025-06-18T01:06:36.329Z] [Docker Build] ---> 66da6b5995cc +[2025-06-18T01:06:36.329Z] [Docker Build] {"aux":{"ID":"sha256:66da6b5995cc50e0463df668b8820b56b6e384a7c91dfaca010ff8c3761b1146"}} +[2025-06-18T01:06:36.331Z] [Docker Build] Successfully built 66da6b5995cc +[2025-06-18T01:06:36.350Z] [Docker Build] Successfully tagged rhizome-node-test:latest +[2025-06-18T01:06:36.350Z] ✅ Docker build completed successfully diff --git a/markdown/007-investigation.md b/markdown/007-investigation.md new file mode 100644 index 0000000..fc36cb9 --- /dev/null +++ b/markdown/007-investigation.md @@ -0,0 +1,5 @@ +Network Layers: +- Gossip protocols +- Pub/sub +- RPC + diff --git a/package-lock.json b/package-lock.json index 3f8906b..2dc0185 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "version": "0.1.0", "license": "Unlicense", "dependencies": { + "@types/dockerode": "^3.3.40", "debug": "^4.4.0", + "dockerode": "^4.0.7", "express": "^4.21.2", "json-logic-js": "^2.0.5", "level": "^9.0.0", @@ -18,6 +20,7 @@ "object-hash": "^3.0.0", "showdown": "^2.1.0", "util": "./util/", + "uuid": "^9.0.0", "zeromq": "^6.1.2" }, "devDependencies": { @@ -28,13 +31,18 @@ "@types/jest": "^29.5.14", "@types/json-logic-js": "^2.0.8", "@types/microtime": "^2.1.2", - "@types/node": "^22.10.2", + "@types/node": "^22.15.31", + "@types/node-fetch": "^2.6.12", "@types/object-hash": "^3.0.6", "@types/showdown": "^2.0.6", + "@types/tar-fs": "^2.0.4", + "@types/uuid": "^10.0.0", "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", + "node-fetch": "^2.7.0", "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", "tsc-alias": "^1.8.10", "typescript": "^5.7.2", "typescript-eslint": "^8.18.0" @@ -92,15 +100,15 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", - "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.25.9", + "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" + "picocolors": "^1.1.1" }, "engines": { "node": ">=6.9.0" @@ -237,9 +245,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", - "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", "dev": true, "license": "MIT", "engines": { @@ -247,9 +255,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", - "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", "dev": true, "license": "MIT", "engines": { @@ -267,27 +275,27 @@ } }, "node_modules/@babel/helpers": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", - "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", + "version": "7.27.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.6.tgz", + "integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.25.9", - "@babel/types": "^7.26.0" + "@babel/template": "^7.27.2", + "@babel/types": "^7.27.6" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.3.tgz", - "integrity": "sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==", + "version": "7.27.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.5.tgz", + "integrity": "sha512-OsQd175SxWkGlzbny8J3K8TnnDD0N3lrIUtB92xwyRpzaenGZhxDvxN/JgU00U3CDZNj9tPuDJ5H0WS4Nt3vKg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.26.3" + "@babel/types": "^7.27.3" }, "bin": { "parser": "bin/babel-parser.js" @@ -536,15 +544,15 @@ } }, "node_modules/@babel/template": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", - "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.25.9", - "@babel/parser": "^7.25.9", - "@babel/types": "^7.25.9" + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -580,19 +588,25 @@ } }, "node_modules/@babel/types": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz", - "integrity": "sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==", + "version": "7.27.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.6.tgz", + "integrity": "sha512-ETyHEk2VHHvl9b9jZP5IHPavHYk57EhanlRRuae9XCpb/j5bDCbPPMOBfCWhnl/7EDJz0jEMCi/RhccCE8r1+Q==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.25.9", - "@babel/helper-validator-identifier": "^7.25.9" + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "license": "Apache-2.0" + }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", @@ -615,6 +629,30 @@ "@chainsafe/is-ip": "^2.0.1" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@cypress/request": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@cypress/request/-/request-3.0.7.tgz", @@ -659,6 +697,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/@cypress/request/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", @@ -785,6 +832,37 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.4.tgz", + "integrity": "sha512-GsFaMXCkMqkKIvwCQjCrwH+GHbPKBjhwo/8ZuUkWHqbI73Kky9I+pQltrlT0+MWpedCoosda53lgjYfyEPgxBg==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.7.13", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -1339,6 +1417,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", @@ -1625,6 +1713,70 @@ "node": ">= 8" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@rtsao/scc": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", @@ -1672,6 +1824,34 @@ "@sinonjs/commons": "^3.0.0" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -1764,6 +1944,27 @@ "@types/node": "*" } }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.41", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.41.tgz", + "integrity": "sha512-5kOi6bcnEjqfJ68ZNV/bBvSMLNIucc0XbRmBO4hg5OoFCoP99eSRcbMysjkzV7ZxQEmmc/zMnv4A7odwuKFzDA==", + "license": "MIT", + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -1902,11 +2103,23 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.10.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", - "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "version": "22.15.31", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.31.tgz", + "integrity": "sha512-jnVe5ULKl6tijxUhvQeNbQG/84fHfg+yMak02cT8QVhBx/F05rAVxCGBYYTh2EKz22D6JF5ktXuNwdx7b9iEGw==", + "license": "MIT", "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" } }, "node_modules/@types/object-hash": { @@ -1966,6 +2179,30 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.112", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.112.tgz", + "integrity": "sha512-i+Vukt9POdS/MBI7YrrkkI5fMfwFtOjphSmt4WXYLfwqsfr6z/HdCx7LqT9M7JktGob8WNgj8nFB4TbGNE4Cog==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -1973,6 +2210,34 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/tar-fs": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/tar-fs/-/tar-fs-2.0.4.tgz", + "integrity": "sha512-ipPec0CjTmVDWE+QKr9cTmIIoTl7dFG/yARCM5MqK8i6CNLIG1P8x4kwDsOQY1ChZOZjH0wO9nvfgBvWl4R3kA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tar-stream": "*" + } + }, + "node_modules/@types/tar-stream": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-3.1.4.tgz", + "integrity": "sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -2164,10 +2429,11 @@ } }, "node_modules/@typescript-eslint/parser/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -2297,10 +2563,11 @@ } }, "node_modules/@typescript-eslint/type-utils/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -2484,10 +2751,11 @@ } }, "node_modules/@typescript-eslint/utils/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -2610,6 +2878,19 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2669,7 +2950,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -2725,6 +3005,13 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -3121,6 +3408,41 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -3166,9 +3488,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -3284,6 +3606,15 @@ "dev": true, "license": "MIT" }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3500,7 +3831,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -3533,7 +3863,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -3546,7 +3875,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, "node_modules/color-support": { @@ -3644,6 +3972,20 @@ "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==", "license": "MIT" }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/create-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", @@ -3666,6 +4008,13 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3906,6 +4255,16 @@ "node": ">=8" } }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", @@ -3941,6 +4300,52 @@ "node": ">=6" } }, + "node_modules/docker-modem": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", + "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.7.tgz", + "integrity": "sha512-R+rgrSRTRdU5mH14PZTCPZtW/zw3HDWNTS/1ZAQpL/5Upe/ye5K9WQkIysu4wBoiMwKynsz0a8qWuGsHgEvSAA==", + "license": "Apache-2.0", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.13", + "docker-modem": "^5.0.6", + "protobufjs": "^7.3.2", + "tar-fs": "~2.1.2", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/doctrine": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", @@ -4078,6 +4483,15 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -4227,7 +4641,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -5014,9 +5427,9 @@ } }, "node_modules/filelist/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -5168,6 +5581,12 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -5289,7 +5708,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -7472,6 +7890,12 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -7486,6 +7910,12 @@ "dev": true, "license": "MIT" }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -7761,6 +8191,12 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/module-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", @@ -7816,6 +8252,13 @@ "url": "https://github.com/sponsors/raouldeheer" } }, + "node_modules/nan": { + "version": "2.22.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz", + "integrity": "sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==", + "license": "MIT", + "optional": true + }, "node_modules/napi-macros": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", @@ -7855,6 +8298,27 @@ "node": "^18 || ^20 || >= 21" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp-build": { "version": "4.8.4", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", @@ -8062,7 +8526,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -8484,6 +8947,30 @@ "node": ">= 6" } }, + "node_modules/protobufjs": { + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.3.tgz", + "integrity": "sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/protons-runtime": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/protons-runtime/-/protons-runtime-5.5.0.tgz", @@ -8507,6 +8994,16 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -8718,7 +9215,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9173,6 +9669,12 @@ "source-map": "^0.6.0" } }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "license": "ISC" + }, "node_modules/splitargs2": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/splitargs2/-/splitargs2-0.1.3.tgz", @@ -9186,6 +9688,23 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/ssh2": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", + "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.20.0" + } + }, "node_modules/sshpk": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", @@ -9429,6 +9948,40 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tar/node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", @@ -9518,6 +10071,13 @@ "node": ">=16" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", @@ -9605,6 +10165,50 @@ "node": ">=10" } }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/tsc-alias": { "version": "1.8.10", "resolved": "https://registry.npmjs.org/tsc-alias/-/tsc-alias-1.8.10.tgz", @@ -9891,9 +10495,10 @@ } }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" }, "node_modules/universalify": { "version": "2.0.1", @@ -10005,14 +10610,25 @@ } }, "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", @@ -10091,6 +10707,24 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -10220,7 +10854,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -10238,7 +10871,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/write-file-atomic": { @@ -10259,7 +10891,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -10275,7 +10906,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -10294,12 +10924,21 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "license": "ISC", "engines": { "node": ">=12" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 58f6883..0c23c35 100644 --- a/package.json +++ b/package.json @@ -2,30 +2,49 @@ "name": "rhizome-node", "version": "0.1.0", "description": "Rhizomatic database engine node", - "type": "module", "scripts": { "build": "tsc", "build:watch": "tsc --watch", "lint": "eslint", - "test": "node --experimental-vm-modules node_modules/.bin/jest", + "test": "jest", "coverage": "./scripts/coverage.sh", "coverage-report": "npm run test -- --coverage --coverageDirectory=coverage", "example-app": "node dist/examples/app.js" }, "jest": { "testEnvironment": "node", - "preset": "ts-jest", + "preset": "ts-jest/presets/default", "roots": [ "./__tests__/" ], "testMatch": [ "**/__tests__/**/*" - ] + ], + "setupFilesAfterEnv": [ + "/__tests__/jest-setup.ts" + ], + "transform": { + "^\\.tsx?$": [ + "ts-jest", + { + "tsconfig": "tsconfig.json" + } + ] + }, + "detectOpenHandles": true, + "forceExit": true, + "verbose": true, + "testEnvironmentOptions": { + "NODE_ENV": "test", + "DEBUG": "rz:*" + } }, "author": "Taliesin (Ladd) ", "license": "Unlicense", "dependencies": { + "@types/dockerode": "^3.3.40", "debug": "^4.4.0", + "dockerode": "^4.0.7", "express": "^4.21.2", "json-logic-js": "^2.0.5", "level": "^9.0.0", @@ -34,6 +53,7 @@ "object-hash": "^3.0.0", "showdown": "^2.1.0", "util": "./util/", + "uuid": "^9.0.0", "zeromq": "^6.1.2" }, "devDependencies": { @@ -44,15 +64,20 @@ "@types/jest": "^29.5.14", "@types/json-logic-js": "^2.0.8", "@types/microtime": "^2.1.2", - "@types/node": "^22.10.2", + "@types/node": "^22.15.31", + "@types/node-fetch": "^2.6.12", "@types/object-hash": "^3.0.6", "@types/showdown": "^2.0.6", + "@types/tar-fs": "^2.0.4", + "@types/uuid": "^10.0.0", "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", + "node-fetch": "^2.7.0", "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", "tsc-alias": "^1.8.10", "typescript": "^5.7.2", "typescript-eslint": "^8.18.0" } -} \ No newline at end of file +} diff --git a/scratch/jsonlogic.ts b/scratch/jsonlogic.ts index 703253d..eb12446 100644 --- a/scratch/jsonlogic.ts +++ b/scratch/jsonlogic.ts @@ -1,12 +1,20 @@ -import { apply } from 'json-logic-js'; +import Debug from 'debug'; +import jsonLogic from 'json-logic-js'; -console.log(apply({"map":[ +const debug = Debug('rz:scratch:jsonlogic'); +const { apply } = jsonLogic; + +// Example of using jsonLogic's map operation +const mapResult = apply({"map":[ {"var":"integers"}, {"*":[{"var":""},2]} -]}, {"integers":[1,2,3,4,5]})); +]}, {"integers":[1,2,3,4,5]}); +debug('Map result: %o', mapResult); -console.log(apply({"reduce":[ +// Example of using jsonLogic's reduce operation +const reduceResult = apply({"reduce":[ {"var":"integers"}, {"+":[{"var":"current"}, {"var":"accumulator"}]}, 0 -]}, {"integers":[1,2,3,4,5]})); +]}, {"integers":[1,2,3,4,5]}); +debug('Reduce result: %o', reduceResult); diff --git a/src/http/api.ts b/src/http/api.ts index 41a3aa7..6eb03e1 100644 --- a/src/http/api.ts +++ b/src/http/api.ts @@ -1,8 +1,11 @@ +import Debug from 'debug'; import express, {Router} from "express"; import {Collection} from "../collections"; import {Delta, DeltaFilter} from "../core"; import {RhizomeNode} from "../node"; +const debug = Debug('rz:http:api'); + export class HttpApi { router = Router(); @@ -11,6 +14,14 @@ export class HttpApi { } private setupRoutes() { + // --------------- health ---------------- + + this.router.get("/health", (_req: express.Request, res: express.Response) => { + res.json({ + status: "ok" + }); + }); + // --------------- deltas ---------------- // Serve list of all deltas accepted @@ -161,7 +172,7 @@ export class HttpApi { if (maxResults) options.maxResults = maxResults; if (deltaFilter) { // Note: deltaFilter would need to be serialized/deserialized properly in a real implementation - console.warn('deltaFilter not supported in HTTP API yet'); + debug('deltaFilter not supported in HTTP API yet'); } const result = await this.rhizomeNode.queryEngine.query(schemaId, filter, options); diff --git a/src/http/index.ts b/src/http/index.ts index 23f2e15..35fec13 100644 --- a/src/http/index.ts +++ b/src/http/index.ts @@ -21,15 +21,81 @@ export class HttpServer { this.app.use('/api', this.httpApi.router); } + /** + * Start the HTTP server + */ start() { const {httpAddr, httpPort} = this.rhizomeNode.config; - this.httpHtml.start(); - this.server = this.app.listen({ - port: httpPort, - host: httpAddr, - exclusive: true - }, () => { - debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP API bound to ${httpAddr}:${httpPort}`); + debug(`[${this.rhizomeNode.config.peerId}]`, `Starting HTTP server on ${httpAddr}:${httpPort}...`); + + try { + this.httpHtml.start(); + + // Create the server + this.server = this.app.listen({ + port: httpPort, + host: httpAddr, + exclusive: true + }); + + // Add error handler + this.server.on('error', (error) => { + debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP server error:`, error); + }); + + // Add callback for logging + this.server.on('listening', () => { + const address = this.server?.address(); + const actualPort = typeof address === 'string' ? httpPort : address?.port; + debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP server bound to ${httpAddr}:${actualPort}`); + }); + + debug(`[${this.rhizomeNode.config.peerId}]`, 'HTTP server start initiated'); + } catch (error) { + debug(`[${this.rhizomeNode.config.peerId}]`, 'Error starting HTTP server:', error); + throw error; + } + } + + /** + * Start the HTTP server and return a promise that resolves when the server is listening + */ + async startAndWait(): Promise { + // If server is already listening, resolve immediately + if (this.server?.listening) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error(`HTTP server failed to start within 10 seconds`)); + }, 10000); + + const onListening = () => { + cleanup(); + resolve(); + }; + + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + + const cleanup = () => { + clearTimeout(timeout); + this.server?.off('listening', onListening); + this.server?.off('error', onError); + }; + + // Start the server if not already started + if (!this.server) { + this.start(); + } + + // Add event listeners + this.server?.on('listening', onListening); + this.server?.on('error', onError); }); } diff --git a/src/network/pub-sub.ts b/src/network/pub-sub.ts index a6ef366..e477790 100644 --- a/src/network/pub-sub.ts +++ b/src/network/pub-sub.ts @@ -8,7 +8,8 @@ export type SubscribedMessageHandler = (sender: PeerAddress, msg: string) => voi // TODO: Allow subscribing to multiple topics on one socket export class Subscription { - sock = new Subscriber(); + private sock: Subscriber; + private isRunning = false; topic: string; publishAddr: PeerAddress; publishAddrStr: string; @@ -20,6 +21,7 @@ export class Subscription { topic: string, cb: SubscribedMessageHandler, ) { + this.sock = new Subscriber(); this.cb = cb; this.topic = topic; this.publishAddr = publishAddr; @@ -27,20 +29,60 @@ export class Subscription { } async start() { + if (this.isRunning) return; + this.isRunning = true; + this.sock.connect(this.publishAddrStr); this.sock.subscribe(this.topic); debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Subscribing to ${this.topic} topic on ZeroMQ ${this.publishAddrStr}`); - // Wait for ZeroMQ messages. - // This will block indefinitely. - for await (const [, sender, msg] of this.sock) { - const senderStr = PeerAddress.fromString(sender.toString()); - const msgStr = msg.toString(); - debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `ZeroMQ subscribtion received msg: ${msgStr}`); - this.cb(senderStr, msgStr); + // Set up message handler + const processMessage = async () => { + try { + if (!this.isRunning) return; + + // Use a promise race to handle both messages and the stop signal + const [topic, sender, msg] = await Promise.race([ + this.sock.receive(), + new Promise<[Buffer, Buffer, Buffer]>(() => {}).then(() => { + if (!this.isRunning) throw new Error('Subscription stopped'); + return [Buffer.alloc(0), Buffer.alloc(0), Buffer.alloc(0)]; + }) + ]); + + if (!this.isRunning) return; + + const senderStr = PeerAddress.fromString(sender.toString()); + const msgStr = msg.toString(); + debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `ZeroMQ subscription received msg: ${msgStr}`); + this.cb(senderStr, msgStr); + + // Process next message + process.nextTick(processMessage); + } catch (error) { + if (this.isRunning) { + debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Error in subscription:`, error); + // Attempt to restart the message processing + if (this.isRunning) { + process.nextTick(processMessage); + } + } + } + }; + + // Start processing messages + process.nextTick(processMessage); + } + + close() { + if (!this.isRunning) return; + this.isRunning = false; + try { + this.sock.close(); + debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Closed subscription for topic ${this.topic}`); + } catch (error) { + debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Error closing subscription:`, error); } - - debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Done waiting for subscription socket for topic ${this.topic}`); } } @@ -53,8 +95,8 @@ export class PubSub { constructor(rhizomeNode: RhizomeNode) { this.rhizomeNode = rhizomeNode; - const {publishBindAddr, publishBindPort} = this.rhizomeNode.config; - this.publishAddrStr = `tcp://${publishBindAddr}:${publishBindPort}`; + const {publishBindHost, publishBindPort} = this.rhizomeNode.config; + this.publishAddrStr = `tcp://${publishBindHost}:${publishBindPort}`; } async startZmq() { @@ -85,16 +127,33 @@ export class PubSub { return subscription; } - async stop() { - if (this.publishSock) { - await this.publishSock.unbind(this.publishAddrStr); - this.publishSock.close(); - // Free the memory by taking the old object out of scope. - this.publishSock = undefined; - } + /** + * Check if the PubSub is running + * @returns boolean indicating if the publisher socket is active + */ + isRunning(): boolean { + return !!this.publishSock; + } + async stop() { + // First close all subscriptions for (const subscription of this.subscriptions) { - subscription.sock.close(); + subscription.close(); + } + this.subscriptions = []; + + // Then close the publisher socket + if (this.publishSock) { + try { + await this.publishSock.unbind(this.publishAddrStr); + this.publishSock.close(); + debug(`[${this.rhizomeNode.config.peerId}]`, 'Unbound and closed publisher socket'); + } catch (error) { + debug(`[${this.rhizomeNode.config.peerId}]`, 'Error closing publisher socket:', error); + } finally { + // Free the memory by taking the old object out of scope. + this.publishSock = undefined; + } } } } diff --git a/src/network/request-reply.ts b/src/network/request-reply.ts index da0edc3..fc594ed 100644 --- a/src/network/request-reply.ts +++ b/src/network/request-reply.ts @@ -74,8 +74,8 @@ export class RequestReply { constructor(rhizomeNode: RhizomeNode) { this.rhizomeNode = rhizomeNode; - const {requestBindAddr, requestBindPort} = this.rhizomeNode.config; - this.requestBindAddrStr = `tcp://${requestBindAddr}:${requestBindPort}`; + const {requestBindHost, requestBindPort} = this.rhizomeNode.config; + this.requestBindAddrStr = `tcp://${requestBindHost}:${requestBindPort}`; } // Listen for incoming requests diff --git a/src/node.ts b/src/node.ts index 75d2f8d..9d5859e 100644 --- a/src/node.ts +++ b/src/node.ts @@ -9,10 +9,8 @@ import {DeltaQueryStorage, StorageFactory, StorageConfig} from './storage'; const debug = Debug('rz:rhizome-node'); export type RhizomeNodeConfig = { - requestBindAddr: string; requestBindHost: string; requestBindPort: number; - publishBindAddr: string; publishBindHost: string; publishBindPort: number; httpAddr: string; @@ -42,10 +40,8 @@ export class RhizomeNode { constructor(config?: Partial) { this.config = { - requestBindAddr: REQUEST_BIND_ADDR, requestBindHost: REQUEST_BIND_HOST, requestBindPort: REQUEST_BIND_PORT, - publishBindAddr: PUBLISH_BIND_ADDR, publishBindHost: PUBLISH_BIND_HOST, publishBindPort: PUBLISH_BIND_PORT, httpAddr: HTTP_API_ADDR, @@ -85,7 +81,16 @@ export class RhizomeNode { this.storageQueryEngine = new StorageQueryEngine(this.deltaStorage, this.schemaRegistry); } - async start(syncOnStart = false) { + /** + * Start the node components + * @param options.syncOnStart Whether to sync with peers on startup (default: false) + * @returns Promise that resolves when the node is fully started and ready + */ + async start({ + syncOnStart = false + }: { syncOnStart?: boolean } = {}): Promise { + debug(`[${this.config.peerId}]`, 'Starting node (waiting for ready)...'); + // Connect our lossless view to the delta stream this.deltaStream.subscribeDeltas(async (delta) => { // Ingest into lossless view @@ -100,44 +105,38 @@ export class RhizomeNode { }); // Bind ZeroMQ publish socket - // TODO: Config option to enable zmq pubsub await this.pubSub.startZmq(); - + // Bind ZeroMQ request socket - // TODO: request/reply via libp2p? - // TODO: config options to enable request/reply, or configure available commands this.requestReply.start(); - - // Start HTTP server - if (this.config.httpEnable) { - this.httpServer.start(); + + // Start HTTP server if enabled + if (this.config.httpEnable && this.httpServer) { + await this.httpServer.startAndWait(); } - { - // Wait a short time for sockets to initialize - await new Promise((resolve) => setTimeout(resolve, 500)); - - // Subscribe to seed peers - this.peers.subscribeToSeeds(); - - // Wait a short time for sockets to initialize - // await new Promise((resolve) => setTimeout(resolve, 500)); - } + // Initialize network components + await new Promise((resolve) => setTimeout(resolve, 500)); + this.peers.subscribeToSeeds(); if (syncOnStart) { // Ask all peers for all deltas this.peers.askAllPeersForDeltas(); - - // Wait to receive all deltas await new Promise((resolve) => setTimeout(resolve, 1000)); } + + debug(`[${this.config.peerId}]`, 'Node started and ready'); } async stop() { this.peers.stop(); await this.pubSub.stop(); await this.requestReply.stop(); - await this.httpServer.stop(); + + // Stop the HTTP server if it was started + if (this.config.httpEnable && this.httpServer) { + await this.httpServer.stop(); + } // Close storage try { diff --git a/src/orchestration/base-orchestrator.ts b/src/orchestration/base-orchestrator.ts new file mode 100644 index 0000000..5340233 --- /dev/null +++ b/src/orchestration/base-orchestrator.ts @@ -0,0 +1,68 @@ +import { NodeOrchestrator, NodeHandle, NodeConfig, NodeStatus } from './types'; + +/** + * Base class for all orchestrator implementations + * Provides common functionality and ensures interface compliance + */ +export abstract class BaseOrchestrator implements NodeOrchestrator { + /** + * Start a new node with the given configuration + * Must be implemented by subclasses + */ + abstract startNode(config: NodeConfig): Promise; + + /** + * Stop a running node + * Must be implemented by subclasses + */ + abstract stopNode(handle: NodeHandle): Promise; + + /** + * Get status of a node + * Must be implemented by subclasses + */ + abstract getNodeStatus(handle: NodeHandle): Promise; + + /** + * Connect two nodes + * Default implementation does nothing - should be overridden by subclasses + * that support direct node connections + */ + async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { + // Default implementation does nothing + console.warn('connectNodes not implemented for this orchestrator'); + } + + /** + * Create network partitions + * Default implementation does nothing - should be overridden by subclasses + * that support network partitioning + */ + async partitionNetwork(partitions: { groups: string[][] }): Promise { + // Default implementation does nothing + console.warn('partitionNetwork not implemented for this orchestrator'); + } + + /** + * Set resource limits for a node + * Default implementation does nothing - should be overridden by subclasses + * that support resource management + */ + async setResourceLimits( + handle: NodeHandle, + limits: Partial + ): Promise { + // Default implementation does nothing + console.warn('setResourceLimits not implemented for this orchestrator'); + } + + /** + * Clean up all resources + * Default implementation does nothing - should be overridden by subclasses + * that need to clean up resources + */ + async cleanup(): Promise { + // Default implementation does nothing + console.warn('cleanup not implemented for this orchestrator'); + } +} diff --git a/src/orchestration/docker-orchestrator/index.ts b/src/orchestration/docker-orchestrator/index.ts new file mode 100644 index 0000000..d6f5f25 --- /dev/null +++ b/src/orchestration/docker-orchestrator/index.ts @@ -0,0 +1,462 @@ +import { Container, Network } from 'dockerode'; +import { BaseOrchestrator } from '../base-orchestrator'; +import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types'; +import { DockerNodeHandle, DockerOrchestratorOptions } from './types'; +import { ContainerManager } from './managers/container-manager'; +import { NetworkManager } from './managers/network-manager'; +import { ResourceManager } from './managers/resource-manager'; +import { StatusManager } from './managers/status-manager'; +import { ImageManager } from './managers/image-manager'; +import { getRandomPort } from './utils/port-utils'; +import Debug from 'debug'; + +const debug = Debug('rz:docker:orchestrator'); + +const DEFAULT_OPTIONS: DockerOrchestratorOptions = { + image: 'rhizome-node-test', + containerWorkDir: '/app', + autoBuildTestImage: true, +}; + +export class DockerOrchestrator extends BaseOrchestrator { + private options: DockerOrchestratorOptions; + private containers: Map = new Map(); + private networks: Map = new Map(); + private containerLogStreams: Map = new Map(); + private nodeHandles: Map = new Map(); + + // Managers + private readonly containerManager: ContainerManager; + private readonly networkManager: NetworkManager; + private readonly resourceManager: ResourceManager; + private readonly statusManager: StatusManager; + private readonly imageManager: ImageManager; + + constructor(options: Partial = {}) { + super(); + this.options = { ...DEFAULT_OPTIONS, ...options }; + + // Initialize managers + this.containerManager = new ContainerManager(); + this.networkManager = new NetworkManager(); + this.resourceManager = new ResourceManager(); + this.statusManager = new StatusManager(); + this.imageManager = new ImageManager(); + } + + /** + * Start a new node with the given configuration + */ + async startNode(config: NodeConfig): Promise { + const nodeId = config.id || `node-${Date.now()}`; + config.network = config.network || {}; + config.network.port = config.network.port || getRandomPort(); + config.network.requestPort = config.network.requestPort || getRandomPort(); + + try { + // Ensure test image is built + if (this.options.autoBuildTestImage) { + await this.imageManager.buildTestImage(this.options.image); + } + + // Create a network for this node using NetworkManager + const network = await this.networkManager.createNetwork(nodeId); + this.networks.set(nodeId, network); + + // Create container using ContainerManager + const container = await this.containerManager.createContainer( + nodeId, + config, + network.id + ); + + // Store container reference before starting it + this.containers.set(nodeId, container); + + // Start the container + await this.containerManager.startContainer(container); + + // Create node handle + const handle: DockerNodeHandle = { + id: nodeId, + containerId: container.id, + networkId: network.id, + config, + status: () => this.getNodeStatus({ id: nodeId } as NodeHandle), + stop: () => this.stopNode({ id: nodeId } as NodeHandle), + getRequestPort: () => config.network?.requestPort, + getApiUrl: () => `http://localhost:${config.network?.port}/api`, + }; + + // Store handle + this.nodeHandles.set(nodeId, handle); + + // Wait for node to be ready using StatusManager + await this.statusManager.waitForNodeReady( container, config.network.port); + + return handle; + } catch (error) { + await this.cleanupFailedStart(nodeId); + throw error; + } + } + + /** + * Stop a running node + */ + async stopNode(handle: NodeHandle): Promise { + const nodeId = handle.id; + const container = this.containers.get(nodeId); + + if (!container) { + throw new Error(`No container found for node ${nodeId}`); + } + + try { + // Stop and remove the container using ContainerManager + try { + await this.containerManager.stopContainer(container); + await this.containerManager.removeContainer(container); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug(`Error managing container ${nodeId}: %s`, errorMessage); + // Continue with cleanup even if container operations fail + } + + // Clean up network using NetworkManager + const network = this.networks.get(nodeId); + if (network) { + try { + await this.networkManager.removeNetwork(network.id); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug(`Error removing network for node ${nodeId}: %s`, errorMessage); + } finally { + this.networks.delete(nodeId); + } + } + + // Clean up log stream + this.cleanupLogStream(nodeId); + + // Remove from internal maps + this.containers.delete(nodeId); + this.nodeHandles.delete(nodeId); + + debug(`Stopped and cleaned up node ${nodeId}`); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug(`Error during cleanup of node ${nodeId}: %s`, errorMessage); + throw new Error(`Failed to stop node ${nodeId}: ${errorMessage}`); + } + } + + /** + * Clean up log stream for a node + * @private + */ + private cleanupLogStream(nodeId: string): void { + const logStream = this.containerLogStreams.get(nodeId); + if (!logStream) return; + + try { + if ('destroy' in logStream) { + (logStream as { destroy: () => void }).destroy(); + } else if ('end' in logStream) { + (logStream as { end: () => void }).end(); + } + } catch (error) { + debug(`Error cleaning up log stream for node ${nodeId}: %o`, error); + } finally { + this.containerLogStreams.delete(nodeId); + } + } + + /** + * Get status of a node + */ + async getNodeStatus(handle: NodeHandle): Promise { + const container = this.containers.get(handle.id); + + // If container not found, return stopped status + if (!container) { + return { + id: handle.id, + status: 'stopped', + error: 'Container not found', + network: { + address: '', + httpPort: 0, + requestPort: 0, + peers: [] + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + } + + try { + // Delegate to StatusManager to get the node status + return await this.statusManager.getNodeStatus(handle, container); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug(`Error getting status for node ${handle.id}: %s`, errorMessage); + + return { + id: handle.id, + status: 'error', + error: errorMessage, + network: { + address: '', + httpPort: 0, + requestPort: 0, + peers: [] + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + } + } + + /** + * Create network partitions + */ + async partitionNetwork(partitions: NetworkPartition): Promise { + // Implementation for network partitioning + // This is a simplified version - in a real implementation, you would: + // 1. Create separate networks for each partition + // 2. Connect containers to their respective partition networks + // 3. Disconnect them from other networks + debug('Network partitioning not fully implemented'); + } + + /** + * Set resource limits for a node + */ + async setResourceLimits( + handle: NodeHandle, + limits: Partial = {} + ): Promise { + const container = this.containers.get(handle.id); + if (!container) { + throw new Error(`No container found for node ${handle.id}`); + } + + try { + // Delegate to ResourceManager + await this.resourceManager.setResourceLimits(container, { + cpu: limits.cpu, + memory: limits.memory, + memorySwap: limits.memory // Default to same as memory limit if not specified + }); + + debug(`Updated resource limits for node %s: %o`, handle.id, limits); + } catch (error) { + debug(`Failed to update resource limits for node ${handle.id}: %o`, error); + throw new Error(`Failed to update resource limits: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + /** + * Connect two nodes in the network + */ + async connectNodes(handle1: NodeHandle, handle2: NodeHandle): Promise { + const dockerHandle1 = handle1 as DockerNodeHandle; + const dockerHandle2 = handle2 as DockerNodeHandle; + + const container1 = this.containers.get(handle1.id); + const container2 = this.containers.get(handle2.id); + + if (!container1 || !container2) { + throw new Error('One or both containers not found'); + } + + try { + // Get the network from the first container + const networkId = dockerHandle1.networkId; + if (!networkId) { + throw new Error(`No network found for node ${handle1.id}`); + } + + // Connect the second container to the same network + const network = this.networks.get(handle1.id); + if (!network) { + throw new Error(`Network not found for node ${handle1.id}`); + } + + await network.connect({ + Container: container2.id, + EndpointConfig: { + Aliases: [`node-${handle2.id}`] + } + }); + + // Update the network ID in the second handle + dockerHandle2.networkId = networkId; + } catch (error) { + debug(`Error connecting nodes ${handle1.id} and ${handle2.id}: %o`, error); + throw error; + } + } + + + /** + * Cleans up resources for a specific node that failed to start properly. + * + * This method is automatically called when a node fails to start during the `startNode` process. + * It handles cleanup of both the container and network resources associated with the failed node, + * and ensures all internal state is properly cleaned up. + * + * @remarks + * - Runs container and network cleanup in parallel for efficiency + * - Handles errors gracefully by logging them without rethrowing + * - Cleans up internal state for just the specified node + * - Used internally by the orchestrator during error handling + * + * @param nodeId - The unique identifier of the node that failed to start + * @private + */ + private async cleanupFailedStart(nodeId: string): Promise { + debug(`Cleaning up failed start for node ${nodeId}...`); + + // Get references to resources before starting cleanup + const container = this.containers.get(nodeId); + const network = this.networks.get(nodeId); + + // Create a map of containers to clean up + const containersToCleanup = new Map(); + if (container) { + containersToCleanup.set(nodeId, container); + } + + // Create a map of networks to clean up + const networksToCleanup = new Map(); + if (network) { + networksToCleanup.set(nodeId, network); + } + + try { + // Run container and network cleanup in parallel + const [containerErrors, networkErrors] = await Promise.all([ + // Clean up containers using ContainerManager + this.containerManager.cleanupContainers(containersToCleanup), + // Clean up networks using NetworkManager + this.networkManager.cleanupNetworks(networksToCleanup) + ]); + + // Log any errors that occurred during cleanup + if (containerErrors.length > 0) { + debug(`Encountered ${containerErrors.length} error(s) while cleaning up containers for node ${nodeId}:`); + containerErrors.forEach(({ resource, error }) => { + console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error'); + }); + } + + if (networkErrors.length > 0) { + debug(`Encountered ${networkErrors.length} error(s) while cleaning up networks for node ${nodeId}:`); + networkErrors.forEach(({ resource, error }) => { + console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error'); + }); + } + + debug(`Completed cleanup for node ${nodeId}`); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug(`Unexpected error during cleanup of node ${nodeId}: %s`, errorMessage); + } finally { + // Always clean up internal state, even if errors occurred + this.containers.delete(nodeId); + this.networks.delete(nodeId); + this.nodeHandles.delete(nodeId); + this.containerLogStreams.delete(nodeId); + } + } + + /** + * Get a container by ID + * @param containerId The ID of the container to retrieve + * @returns The container instance or undefined if not found + */ + async getContainer(containerId: string): Promise { + // First try to get from our containers map + const container = this.containers.get(containerId); + if (container) { + return container; + } + + // If not found, try to get it from the container manager + try { + return await this.containerManager.getContainer(containerId); + } catch (error) { + debug(`Failed to get container ${containerId}: %o`, error); + return undefined; + } + } + + /** + * Cleans up all resources managed by this orchestrator. + * + * This method should be called during shutdown or when you need to completely tear down + * all containers and networks created by this orchestrator instance. + * + * @remarks + * - Stops and removes all containers first + * - Then removes all networks (sequential execution) + * - Clears all internal state including node handles and log streams + * - Throws any errors that occur during cleanup + * - Should be called when the orchestrator is being shut down + * + * @throws {Error} If any error occurs during the cleanup process + */ + async cleanup(): Promise { + debug('Starting cleanup of all resources...'); + + // Create copies of the maps to avoid modification during iteration + const containersToCleanup = new Map(this.containers); + const networksToCleanup = new Map(this.networks); + + try { + // First, clean up all containers + debug('Stopping and removing all containers...'); + const containerErrors = await this.containerManager.cleanupContainers(containersToCleanup); + + // Wait a short time to ensure all container cleanup is complete + await new Promise(resolve => setTimeout(resolve, 1000)); + + // Then clean up all networks + debug('Removing all networks...'); + const networkErrors = await this.networkManager.cleanupNetworks(networksToCleanup); + + // Log any errors that occurred during cleanup + if (containerErrors.length > 0) { + debug(`Encountered ${containerErrors.length} error(s) while cleaning up containers:`); + containerErrors.forEach(({ resource, error }) => { + console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error'); + }); + } + + if (networkErrors.length > 0) { + debug(`Encountered ${networkErrors.length} error(s) while cleaning up networks:`); + networkErrors.forEach(({ resource, error }) => { + console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error'); + }); + } + + debug('Completed cleanup of all resources'); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug('Unexpected error during cleanup: %s', errorMessage); + throw error; // Re-throw to allow callers to handle the error + } finally { + // Always clear internal state, even if errors occurred + this.containers.clear(); + this.networks.clear(); + this.nodeHandles.clear(); + this.containerLogStreams.clear(); + } + } +} diff --git a/src/orchestration/docker-orchestrator/managers/container-manager.ts b/src/orchestration/docker-orchestrator/managers/container-manager.ts new file mode 100644 index 0000000..0ac98a6 --- /dev/null +++ b/src/orchestration/docker-orchestrator/managers/container-manager.ts @@ -0,0 +1,249 @@ +import Debug from 'debug'; +import Docker, { Container, DockerOptions } from 'dockerode'; +import { IContainerManager } from './interfaces'; +import { NodeConfig, NodeStatus } from '../../types'; + +const debug = Debug('rz:docker:container-manager'); + +export class ContainerManager implements IContainerManager { + private docker: Docker; + + constructor() { + this.docker = new Docker(); + } + + async createContainer( + nodeId: string, + config: NodeConfig, + networkId: string + ): Promise { + const containerName = `rhizome-node-${nodeId}`; + + // Create host config with port bindings and mounts + const hostConfig: Docker.HostConfig = { + NetworkMode: networkId, + PortBindings: { + [`${config.network?.port || 3000}/tcp`]: [{ HostPort: config.network?.port?.toString() }], + [`${config.network?.requestPort || 3001}/tcp`]: [{ HostPort: config.network?.requestPort?.toString() }], + }, + }; + + // Add resource limits if specified + if (config.resources) { + if (config.resources.cpu) { + // Convert CPU cores to nanoCPUs (1 CPU = 1e9 nanoCPUs) + hostConfig.NanoCpus = config.resources.cpu * 1e9; + } + + if (config.resources.memory) { + hostConfig.Memory = config.resources.memory * 1024 * 1024; // Convert MB to bytes + hostConfig.MemorySwap = config.resources.memorySwap + ? config.resources.memorySwap * 1024 * 1024 + : config.resources.memory * 2 * 1024 * 1024; // Default swap to 2x memory + } + } + + // Increase file descriptor limits + hostConfig.Ulimits = [ + { + Name: 'nofile', + Soft: 65536, + Hard: 65536 + } + ]; + + // Set environment variables to optimize performance and disable file watching + const envVars = [ + // Node.js and memory settings + `NODE_OPTIONS=--max-old-space-size=${Math.floor((config.resources?.memory || 512) * 0.8)}`, + 'NODE_ENV=test', + + // Network configuration + `RHIZOME_HTTP_BIND_PORT=${config.network?.port || 3000}`, + 'RHIZOME_HTTP_BIND_ADDR=0.0.0.0', + `RHIZOME_REQUEST_BIND_PORT=${config.network?.requestPort || 3001}`, + 'RHIZOME_REQUEST_BIND_ADDR=0.0.0.0', + `RHIZOME_PUBLISH_BIND_PORT=${(config.network?.requestPort || 3001) + 1}`, + 'RHIZOME_PUBLISH_BIND_ADDR=0.0.0.0', + + // Application settings + 'RHIZOME_STORAGE_TYPE=memory', + 'RHIZOME_HTTP_API_ENABLE=true', + `RHIZOME_PEER_ID=${nodeId}`, + + // Disable unnecessary features for testing + 'DISABLE_HTTP_HTML=true', + 'DISABLE_MARKDOWN=true', + + // Debug settings + 'DEBUG=rz:*,rhizome:*,docker:*', + 'DEBUG_COLORS=true' + ]; + + // Create container configuration with all environment variables + const containerConfig: Docker.ContainerCreateOptions = { + name: containerName, + Image: 'rhizome-node-test', + ExposedPorts: { + [`${config.network?.port || 3000}/tcp`]: {}, + [`${config.network?.requestPort || 3001}/tcp`]: {} + }, + HostConfig: hostConfig, + Env: [ + ...envVars, + 'NODE_ENV=test', + 'DEBUG=*', + `RHIZOME_HTTP_API_PORT=${config.network?.port || 3000}`, + 'RHIZOME_HTTP_API_ADDR=0.0.0.0', + 'RHIZOME_HTTP_API_ENABLE=true' + ] + }; + + try { + const container = await this.docker.createContainer(containerConfig); + + return container; + } catch (error) { + throw new Error(`Failed to create container: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async startContainer(container: Container): Promise { + try { + debug(`Starting container ${container.id}`); + await container.start(); + + // Verify the container is actually running + const containerInfo = await this.verifyContainerRunning(container); + debug(`Container ${container.id} started with status:`, containerInfo.State); + + } catch (error) { + // Get container logs for debugging + let logs = ''; + try { + logs = await this.getContainerLogs(container); + } catch (logError) { + debug('Failed to get container logs:', logError); + } + + throw new Error( + `Failed to start container: ${error instanceof Error ? error.message : 'Unknown error'}\n` + + `Container logs:\n${logs}` + ); + } + } + + async stopContainer(container: Container): Promise { + try { + await container.stop({ t: 1 }); + } catch (error) { + debug('Error stopping container: %o', error); + throw error; + } + } + + async removeContainer(container: Container): Promise { + try { + await container.remove({ force: true }); + } catch (error) { + debug('Error removing container: %o', error); + throw error; + } + } + + async getContainerLogs(container: Container, tailLines = 20): Promise { + const logs = await container.logs({ + stdout: true, + stderr: true, + tail: tailLines, + timestamps: true, + follow: false, + }); + return logs.toString(); + } + + /** + * Get a container by ID + * @param containerId The ID of the container to retrieve + * @returns The container instance + * @throws Error if the container cannot be found + */ + async getContainer(containerId: string): Promise { + try { + const container = this.docker.getContainer(containerId); + // Verify the container exists by inspecting it + await container.inspect(); + return container; + } catch (error) { + throw new Error(`Failed to get container ${containerId}: ${error instanceof Error ? error.message : String(error)}`); + } + } + + async verifyContainerRunning(container: Container): Promise { + const containerInfo = await container.inspect(); + if (!containerInfo.State.Running) { + throw new Error('Container is not running'); + } + return containerInfo; + } + + mapContainerState(state: string): NodeStatus['status'] { + if (!state) return 'error'; + + const stateLower = state.toLowerCase(); + if (['created', 'restarting'].includes(stateLower)) return 'starting'; + if (stateLower === 'running') return 'running'; + if (stateLower === 'paused') return 'stopping'; + if (['dead', 'exited'].includes(stateLower)) return 'stopped'; + + return 'error'; + } + + async cleanupContainers(containers: Map): Promise> { + const cleanupErrors: Array<{ resource: string; error: Error }> = []; + + // Process containers in sequence to avoid overwhelming the Docker daemon + for (const [nodeId, container] of containers.entries()) { + try { + debug(`[Cleanup] Stopping container ${nodeId}...`); + + try { + // First, try to stop the container gracefully + await this.stopContainer(container); + debug(`[Cleanup] Successfully stopped container ${nodeId}`); + } catch (stopError) { + debug(`[Cleanup] Failed to stop container ${nodeId}: %o`, stopError); + // Continue with force removal even if stop failed + } + + // Now remove the container + debug(`[Cleanup] Removing container ${nodeId}...`); + await this.removeContainer(container); + debug(`[Cleanup] Successfully removed container ${nodeId}`); + + // Verify the container is actually gone + try { + const containerInfo = await container.inspect(); + debug(`[Cleanup] Container ${nodeId} still exists after removal: %s`, containerInfo.State?.Status); + cleanupErrors.push({ + resource: `container:${nodeId}`, + error: new Error(`Container still exists after removal: ${containerInfo.State?.Status}`) + }); + } catch (inspectError) { + // Expected - container should not exist anymore + debug(`[Cleanup] Verified container ${nodeId} has been removed`); + } + + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + debug(`[Cleanup] Error cleaning up container ${nodeId}: %o`, err); + cleanupErrors.push({ resource: `container:${nodeId}`, error: err }); + } + + // Add a small delay between container cleanups + await new Promise(resolve => setTimeout(resolve, 500)); + } + + return cleanupErrors; + } +} diff --git a/src/orchestration/docker-orchestrator/managers/image-manager.ts b/src/orchestration/docker-orchestrator/managers/image-manager.ts new file mode 100644 index 0000000..5d59802 --- /dev/null +++ b/src/orchestration/docker-orchestrator/managers/image-manager.ts @@ -0,0 +1,159 @@ +import Docker, { DockerOptions } from 'dockerode'; +import * as path from 'path'; +import { promises as fs } from 'fs'; +import * as tar from 'tar-fs'; +import { Headers } from 'tar-fs'; +import { IImageManager } from './interfaces'; +import Debug from 'debug'; + +const debug = Debug('rz:docker:image-manager'); + +// Global promise to track test image build +let testImageBuildPromise: Promise | null = null; + +export class ImageManager implements IImageManager { + private docker: Docker; + + constructor() { + this.docker = new Docker(); + } + + /** + * Build a test Docker image if it doesn't exist + */ + async buildTestImage(imageName: string = 'rhizome-node-test'): Promise { + if (testImageBuildPromise) { + debug('Test image build in progress, reusing existing build promise...'); + return testImageBuildPromise; + } + + debug('Building test Docker image...'); + const dockerfilePath = path.join(process.cwd(), 'Dockerfile.test'); + + // Verify Dockerfile exists + try { + await fs.access(dockerfilePath); + debug(`Found Dockerfile at: %s`, dockerfilePath); + } catch (err) { + throw new Error(`Dockerfile not found at ${dockerfilePath}: ${err}`); + } + + // Create a tar archive of the build context + const tarStream = tar.pack(process.cwd(), { + entries: [ + 'Dockerfile.test', + 'package.json', + 'package-lock.json', + 'tsconfig.json', + 'src/', + 'markdown/', + 'util', + 'examples/', + 'README.md', + ], + map: (header: Headers) => { + // Ensure Dockerfile is named 'Dockerfile' in the build context + if (header.name === 'Dockerfile.test') { + header.name = 'Dockerfile'; + } + return header; + } + }); + + debug('Created build context tar stream'); + + testImageBuildPromise = new Promise((resolve, reject) => { + const logMessages: string[] = []; + + const log = (...args: any[]) => { + const timestamp = new Date().toISOString(); + const message = args.map(arg => + typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg) + ).join(' '); + const logMessage = `[${timestamp}] ${message}\n`; + process.stdout.write(logMessage); + logMessages.push(logMessage); + }; + + this.docker.buildImage(tarStream, { t: imageName }, (err, stream) => { + if (err) { + const errorMsg = `❌ Error starting Docker build: ${err.message}`; + log(errorMsg); + return reject(new Error(errorMsg)); + } + + if (!stream) { + const error = new Error('No build stream returned from Docker'); + log(`❌ ${error.message}`); + return reject(error); + } + + log('✅ Docker build started, streaming output...'); + + // Handle build output + let output = ''; + stream.on('data', (chunk: Buffer) => { + const chunkStr = chunk.toString(); + output += chunkStr; + + try { + // Try to parse as JSON (Docker build output is typically JSONL) + const lines = chunkStr.split('\n').filter(Boolean); + for (const line of lines) { + try { + if (!line.trim()) continue; + + const json = JSON.parse(line); + if (json.stream) { + const message = `[Docker Build] ${json.stream}`.trim(); + log(message); + } else if (json.error) { + const errorMsg = json.error.trim() || 'Unknown error during Docker build'; + log(`❌ ${errorMsg}`); + reject(new Error(errorMsg)); + return; + } else if (Object.keys(json).length > 0) { + // Log any other non-empty JSON objects + log(`[Docker Build] ${JSON.stringify(json)}`); + } + } catch (e) { + // If not JSON, log as plain text if not empty + if (line.trim()) { + log(`[Docker Build] ${line}`); + } + } + } + } catch (e) { + const errorMsg = `Error processing build output: ${e}\nRaw output: ${chunkStr}`; + log(`❌ ${errorMsg}`); + debug('Docker build error: %s', errorMsg); + } + }); + + stream.on('end', () => { + log('✅ Docker build completed successfully'); + resolve(); + }); + + stream.on('error', (err: Error) => { + const errorMsg = `❌ Docker build failed: ${err.message}\nBuild output so far: ${output}`; + log(errorMsg); + reject(new Error(errorMsg)); + }); + }); + }); + } + + /** + * Check if an image exists locally + */ + async imageExists(imageName: string): Promise { + try { + const image = this.docker.getImage(imageName); + await image.inspect(); + return true; + } catch (error) { + return false; + } + } +} diff --git a/src/orchestration/docker-orchestrator/managers/index.ts b/src/orchestration/docker-orchestrator/managers/index.ts new file mode 100644 index 0000000..bdfc5ab --- /dev/null +++ b/src/orchestration/docker-orchestrator/managers/index.ts @@ -0,0 +1,5 @@ +export * from './interfaces'; +export * from './container-manager'; +export * from './network-manager'; +export * from './resource-manager'; +export * from './status-manager'; diff --git a/src/orchestration/docker-orchestrator/managers/interfaces.ts b/src/orchestration/docker-orchestrator/managers/interfaces.ts new file mode 100644 index 0000000..49bb0ea --- /dev/null +++ b/src/orchestration/docker-orchestrator/managers/interfaces.ts @@ -0,0 +1,69 @@ +import Docker, { Container, Network, NetworkInspectInfo } from 'dockerode'; +import { NodeConfig, NodeHandle, NodeStatus } from '../../types'; + +export interface IContainerManager { + createContainer( + nodeId: string, + config: NodeConfig, + networkId: string + ): Promise; + + startContainer(container: Container): Promise; + stopContainer(container: Container): Promise; + removeContainer(container: Container): Promise; + getContainerLogs(container: Container, tailLines?: number): Promise; + getContainer(containerId: string): Promise; + verifyContainerRunning(container: Container): Promise; + mapContainerState(state: string): NodeStatus['status']; + cleanupContainers(containers: Map): Promise>; +} + +export interface INetworkManager { + createNetwork(nodeId: string): Promise; + removeNetwork(networkId: string): Promise; + connectToNetwork(containerId: string, networkId: string, aliases?: string[]): Promise; + disconnectFromNetwork(containerId: string, networkId: string): Promise; + setupPortBindings(ports: Record): Docker.HostConfig['PortBindings']; + getNetworkInfo(networkId: string): Promise; + cleanupNetworks(networks: Map): Promise>; +} + +export interface IResourceManager { + setResourceLimits( + container: Container, + limits: Partial + ): Promise; + + getResourceUsage(container: Container): Promise<{ + cpu: { usage: number; limit: number }; + memory: { usage: number; limit: number }; + }>; +} + +export interface IImageManager { + /** + * Build a test Docker image if it doesn't exist + * @param imageName The name to give to the built image + */ + buildTestImage(imageName: string): Promise; +} + +export interface IStatusManager { + waitForNodeReady( + container: Container, + port: number, + maxAttempts?: number, + delayMs?: number + ): Promise; + + healthCheck(healthUrl: string): Promise<{ ok: boolean; status: number }>; + mapContainerState(state: string): NodeStatus['status']; + + /** + * Get the status of a node including container status, network info, and resource usage + * @param handle The node handle containing node metadata + * @param container The Docker container instance + * @returns A promise that resolves to the node status + */ + getNodeStatus(handle: NodeHandle, container: Container): Promise; +} diff --git a/src/orchestration/docker-orchestrator/managers/network-manager.ts b/src/orchestration/docker-orchestrator/managers/network-manager.ts new file mode 100644 index 0000000..e876fbb --- /dev/null +++ b/src/orchestration/docker-orchestrator/managers/network-manager.ts @@ -0,0 +1,167 @@ +import Debug from 'debug'; +import Docker, { Network, NetworkInspectInfo } from 'dockerode'; +import { INetworkManager } from './interfaces'; + +const debug = Debug('rz:docker:network-manager'); + +export class NetworkManager implements INetworkManager { + private networks: Map = new Map(); + private docker: Docker; + + constructor() { + this.docker = new Docker(); + } + + async createNetwork(nodeId: string): Promise { + const networkName = `rhizome-${nodeId}-network`; + + try { + const network = await this.docker.createNetwork({ + Name: networkName, + Driver: 'bridge', + CheckDuplicate: true, + Internal: false, + Attachable: true, + EnableIPv6: false + }); + + this.networks.set(nodeId, network); + return network; + } catch (error) { + debug(`Error creating network for node ${nodeId}: %o`, error); + throw error; + } + } + + async removeNetwork(networkId: string): Promise { + try { + const network = this.docker.getNetwork(networkId); + await network.remove(); + + // Remove from our tracking map + for (const [nodeId, net] of this.networks.entries()) { + if (net.id === networkId) { + this.networks.delete(nodeId); + break; + } + } + } catch (error) { + debug(`Failed to remove network ${networkId}: %o`, error); + throw error; + } + } + + async connectToNetwork( + containerId: string, + networkId: string, + aliases: string[] = [] + ): Promise { + try { + const network = this.docker.getNetwork(networkId); + await network.connect({ + Container: containerId, + EndpointConfig: { + Aliases: aliases + } + }); + } catch (error) { + debug(`Failed to connect container ${containerId} to network ${networkId}: %o`, error); + throw error; + } + } + + async disconnectFromNetwork(containerId: string, networkId: string): Promise { + try { + const network = this.docker.getNetwork(networkId); + await network.disconnect({ Container: containerId }); + } catch (error) { + debug(`Failed to disconnect container ${containerId} from network ${networkId}: %o`, error); + throw error; + } + } + + setupPortBindings(ports: Record): Docker.HostConfig['PortBindings'] { + const portBindings: Docker.HostConfig['PortBindings'] = {}; + + for (const [containerPort, hostPort] of Object.entries(ports)) { + const [port, protocol = 'tcp'] = containerPort.split('/'); + portBindings[`${port}/${protocol}`] = [{ HostPort: hostPort.toString() }]; + } + + return portBindings; + } + + async getNetworkInfo(networkId: string): Promise { + try { + const network = this.docker.getNetwork(networkId); + return await network.inspect(); + } catch (error) { + debug(`Failed to get network info for ${networkId}: %o`, error); + throw error; + } + } + + async cleanupNetworks(networks: Map): Promise> { + const cleanupErrors: Array<{ resource: string; error: Error }> = []; + + // Process networks in sequence to avoid overwhelming the Docker daemon + for (const [nodeId, network] of networks.entries()) { + try { + debug(`[Cleanup] Removing network for node ${nodeId}...`); + + // First, inspect the network to see if it has any connected containers + try { + const networkInfo = await this.getNetworkInfo(network.id); + if (networkInfo.Containers && Object.keys(networkInfo.Containers).length > 0) { + debug(`[Cleanup] Network ${nodeId} still has ${Object.keys(networkInfo.Containers).length} connected containers`); + + // Try to disconnect all containers from the network first + for (const containerId of Object.keys(networkInfo.Containers)) { + try { + debug(`[Cleanup] Disconnecting container ${containerId} from network ${nodeId}...`); + await this.disconnectFromNetwork(containerId, network.id); + debug(`[Cleanup] Successfully disconnected container ${containerId} from network ${nodeId}`); + } catch (disconnectError) { + debug(`[Cleanup] Failed to disconnect container ${containerId} from network ${nodeId}: %o`, disconnectError); + // Continue with network removal even if disconnect failed + } + + // Add a small delay between disconnects + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + } catch (inspectError) { + debug(`[Cleanup] Failed to inspect network ${nodeId} before removal: %o`, inspectError); + // Continue with removal even if inspect failed + } + + // Now remove the network + await this.removeNetwork(network.id); + debug(`[Cleanup] Successfully removed network for node ${nodeId}`); + + // Verify the network is actually gone + try { + await this.getNetworkInfo(network.id); + debug(`[Cleanup] Network ${nodeId} still exists after removal`); + cleanupErrors.push({ + resource: `network:${nodeId}`, + error: new Error('Network still exists after removal') + }); + } catch (inspectError) { + // Expected - network should not exist anymore + debug(`[Cleanup] Verified network ${nodeId} has been removed`); + } + + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + debug(`[Cleanup] Error cleaning up network ${nodeId}: %o`, err); + cleanupErrors.push({ resource: `network:${nodeId}`, error: err }); + } + + // Add a small delay between network cleanups + await new Promise(resolve => setTimeout(resolve, 500)); + } + + return cleanupErrors; + } +} diff --git a/src/orchestration/docker-orchestrator/managers/resource-manager.ts b/src/orchestration/docker-orchestrator/managers/resource-manager.ts new file mode 100644 index 0000000..3988225 --- /dev/null +++ b/src/orchestration/docker-orchestrator/managers/resource-manager.ts @@ -0,0 +1,269 @@ +import Debug from 'debug'; +import { Container } from 'dockerode'; +import { IResourceManager } from './interfaces'; + +const debug = Debug('rz:docker:resource-manager'); + +// Define the structure of the Docker stats object +interface ContainerStats { + cpu_stats: { + cpu_usage: { + total_usage: number; + usage_in_kernelmode?: number; + usage_in_usermode?: number; + }; + system_cpu_usage: number; + online_cpus?: number; + throttling_data?: Record; + }; + precpu_stats: { + cpu_usage: { + total_usage: number; + usage_in_kernelmode?: number; + usage_in_usermode?: number; + }; + system_cpu_usage: number; + online_cpus?: number; + throttling_data?: Record; + }; + memory_stats: { + usage?: number; + max_usage?: number; + limit?: number; + stats?: { + total_rss?: number; + [key: string]: unknown; + }; + usage_in_bytes?: number; + limit_in_bytes?: number; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +// Type guard to check if an object is a Node.js ReadableStream +function isReadableStream(obj: unknown): obj is NodeJS.ReadableStream { + return ( + obj !== null && + typeof obj === 'object' && + typeof (obj as any).pipe === 'function' && + typeof (obj as any).on === 'function' + ); +} + +export class ResourceManager implements IResourceManager { + private debug = debug.extend('ResourceManager'); + + constructor() { + this.debug('ResourceManager initialized'); + } + + async setResourceLimits( + container: Container, + limits: { + cpu?: number; + memory?: number; + memorySwap?: number; + } = {} + ): Promise { + try { + const updateConfig: any = {}; + + if (limits.cpu !== undefined) { + updateConfig.CpuShares = limits.cpu; + updateConfig.NanoCpus = limits.cpu * 1e9; // Convert to nanoCPUs + } + + if (limits.memory !== undefined) { + updateConfig.Memory = limits.memory * 1024 * 1024; // Convert MB to bytes + updateConfig.MemorySwap = limits.memorySwap !== undefined + ? limits.memorySwap * 1024 * 1024 + : updateConfig.Memory; // Default to same as memory if not specified + } + + if (Object.keys(updateConfig).length > 0) { + await container.update(updateConfig); + } + } catch (error) { + throw new Error(`Failed to set resource limits: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + async getResourceUsage(container: Container): Promise<{ + cpu: { usage: number; limit: number }; + memory: { usage: number; limit: number }; + }> { + try { + this.debug('Getting container stats...'); + + // Get container stats with stream:false to get a single stats object + const stats = await container.stats({ stream: false }); + + // Log the raw stats type and constructor for debugging + this.debug('Raw stats type: %s', typeof stats); + this.debug('Raw stats constructor: %s', stats?.constructor?.name); + + // Handle the response based on its type + let statsData: ContainerStats; + + if (typeof stats === 'string') { + // If it's a string, parse it as JSON + this.debug('Stats is a string, parsing JSON'); + try { + statsData = JSON.parse(stats) as ContainerStats; + } catch (error) { + this.debug('Failed to parse stats JSON: %o', error); + throw new Error(`Failed to parse stats JSON: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } else if (stats && typeof stats === 'object') { + // Check if it's a Node.js stream using our type guard + if (isReadableStream(stats)) { + this.debug('Stats is a stream, reading data...'); + // Convert the stream to a string and parse as JSON + const statsString = await this.streamToString(stats); + try { + statsData = JSON.parse(statsString) as ContainerStats; + this.debug('Successfully parsed streamed stats'); + } catch (error) { + this.debug('Failed to parse streamed stats: %o', error); + throw new Error(`Failed to parse streamed stats: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } else { + // If it's already an object, use it directly + this.debug('Stats is a plain object'); + statsData = stats as unknown as ContainerStats; + } + } else { + throw new Error(`Unexpected stats type: ${typeof stats}`); + } + + // Calculate and return the resource usage + return this.calculateResourceUsage(statsData); + } catch (error: unknown) { + this.debug('Error in getResourceUsage: %o', error); + // Return default values on error + return { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 }, + }; + } + } + + /** + * Convert a ReadableStream to a string + */ + private streamToString(stream: NodeJS.ReadableStream): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + + stream.on('data', (chunk: unknown) => { + if (Buffer.isBuffer(chunk)) { + chunks.push(chunk); + } else if (typeof chunk === 'string') { + chunks.push(Buffer.from(chunk, 'utf8')); + } else if (chunk instanceof Uint8Array) { + chunks.push(Buffer.from(chunk)); + } else { + this.debug('Unexpected chunk type: %s', typeof chunk); + reject(new Error(`Unexpected chunk type: ${typeof chunk}`)); + } + }); + + stream.on('end', () => { + try { + const result = Buffer.concat(chunks).toString('utf8'); + resolve(result); + } catch (error) { + this.debug('Error concatenating chunks: %o', error); + reject(error); + } + }); + + stream.on('error', (error: Error) => { + this.debug('Error reading stream: %o', error); + reject(error); + }); + }); + } + + /** + * Calculate resource usage from stats data + */ + private calculateResourceUsage(statsData: ContainerStats): { + cpu: { usage: number; limit: number }; + memory: { usage: number; limit: number }; + } { + this.debug('Calculating resource usage from stats data'); + + // Log detailed CPU stats if available + if (statsData.cpu_stats) { + this.debug('CPU stats: %o', { + online_cpus: statsData.cpu_stats.online_cpus, + cpu_usage: statsData.cpu_stats.cpu_usage, + system_cpu_usage: statsData.cpu_stats.system_cpu_usage, + }); + } else { + this.debug('No CPU stats available'); + } + + // Log detailed memory stats if available + if (statsData.memory_stats) { + this.debug('Memory stats: %o', { + usage: statsData.memory_stats.usage, + max_usage: statsData.memory_stats.max_usage, + limit: statsData.memory_stats.limit, + stats: statsData.memory_stats.stats, + }); + } else { + this.debug('No memory stats available'); + } + + // Calculate CPU usage percentage + let cpuPercent = 0; + const cpuCores = statsData.cpu_stats?.online_cpus || 1; + + // Check if we have the necessary data for CPU calculation + if (statsData.cpu_stats?.cpu_usage?.total_usage !== undefined && + statsData.precpu_stats?.cpu_usage?.total_usage !== undefined) { + const cpuDelta = statsData.cpu_stats.cpu_usage.total_usage - + (statsData.precpu_stats.cpu_usage.total_usage || 0); + const systemDelta = statsData.cpu_stats.system_cpu_usage - + (statsData.precpu_stats.system_cpu_usage || 0); + + this.debug('CPU delta: %d, System delta: %d', cpuDelta, systemDelta); + + if (systemDelta > 0 && cpuDelta > 0) { + cpuPercent = (cpuDelta / systemDelta) * cpuCores * 100.0; + this.debug('Calculated CPU percent: %d%%', cpuPercent); + } + } else { + this.debug('Insufficient CPU stats data for calculation'); + this.debug('Available CPU stats: %o', statsData.cpu_stats); + this.debug('Available precpu_stats: %o', statsData.precpu_stats); + } + + // Get memory usage with fallbacks + const memoryUsage = statsData.memory_stats?.usage || + statsData.memory_stats?.stats?.total_rss || + statsData.memory_stats?.usage_in_bytes || + 0; + + const memoryLimit = statsData.memory_stats?.limit || + statsData.memory_stats?.max_usage || + statsData.memory_stats?.limit_in_bytes || + 0; + + this.debug('Memory usage: %d / %d bytes', memoryUsage, memoryLimit); + + return { + cpu: { + usage: cpuPercent, + limit: 100, // 100% CPU limit as a percentage + }, + memory: { + usage: memoryUsage, + limit: memoryLimit || 0, // Ensure we don't return undefined + }, + }; + } +} diff --git a/src/orchestration/docker-orchestrator/managers/status-manager.ts b/src/orchestration/docker-orchestrator/managers/status-manager.ts new file mode 100644 index 0000000..fafe851 --- /dev/null +++ b/src/orchestration/docker-orchestrator/managers/status-manager.ts @@ -0,0 +1,311 @@ +import Docker, { Container } from 'dockerode'; +import { IStatusManager } from './interfaces'; +import { NodeHandle, NodeStatus } from '../../types'; +import Debug from 'debug'; + +const debug = Debug('rz:docker:status-manager'); + +const DEFAULT_MAX_ATTEMPTS = 8; +const DEFAULT_DELAY_MS = 1000; +const MAX_BACKOFF_MS = 30000; // 30 seconds max backoff + +export class StatusManager implements IStatusManager { + async waitForNodeReady( + container: Container, + port: number, + maxAttempts: number = DEFAULT_MAX_ATTEMPTS, + initialDelayMs: number = DEFAULT_DELAY_MS + ): Promise { + debug(`[waitForNodeReady] Starting with port ${port}, maxAttempts: ${maxAttempts}, initialDelayMs: ${initialDelayMs}`); + let lastError: Error | null = null; + let attempt = 0; + let delay = initialDelayMs; + + while (attempt < maxAttempts) { + attempt++; + const attemptStartTime = Date.now(); + + try { + debug(`[Attempt ${attempt}/${maxAttempts}] Verifying container is running...`); + + // Add timeout to verifyContainerRunning + const verifyPromise = this.verifyContainerRunning(container); + const timeoutPromise = new Promise((_, reject) => + setTimeout(() => reject(new Error('verifyContainerRunning timed out')), 10000) + ); + + await Promise.race([verifyPromise, timeoutPromise]); + debug(`[Attempt ${attempt}/${maxAttempts}] Container is running`); + + const healthUrl = `http://localhost:${port}/api/health`; + debug(`[Attempt ${attempt}/${maxAttempts}] Checking health at: ${healthUrl}`); + + // Add timeout to health check + const healthCheckPromise = this.healthCheck(healthUrl); + const healthCheckTimeout = new Promise((_, reject) => + setTimeout(() => reject(new Error('Health check timed out')), 10000) + ); + + const response = await Promise.race([healthCheckPromise, healthCheckTimeout]); + + if (response.ok) { + debug(`✅ Node is ready! (Attempt ${attempt}/${maxAttempts})`); + return; // Success! + } + + throw new Error(`Health check failed with status: ${response.status}`); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + lastError = error instanceof Error ? error : new Error(errorMessage); + + const attemptDuration = Date.now() - attemptStartTime; + debug(`[Attempt ${attempt}/${maxAttempts}] Failed after ${attemptDuration}ms: %s`, errorMessage); + + // Log container state on error + try { + const containerInfo = await container.inspect(); + debug(`[Container State] Status: ${containerInfo.State.Status}, Running: ${containerInfo.State.Running}, ExitCode: ${containerInfo.State.ExitCode}`); + + // Log recent container logs on error + if (containerInfo.State.Running) { + try { + const logs = await container.logs({ + stdout: true, + stderr: true, + tail: 20, + timestamps: true, + }); + debug(`[Container Logs] Last 20 lines:\n${logs.toString()}`); + } catch (logError) { + debug('Failed to get container logs: %o', logError); + } + } + } catch (inspectError) { + debug('Failed to inspect container: %o', inspectError); + } + + // Exponential backoff with jitter, but don't wait if we're out of attempts + if (attempt < maxAttempts) { + const jitter = Math.random() * 1000; // Add up to 1s of jitter + const backoff = Math.min(delay + jitter, MAX_BACKOFF_MS); + debug(`[Backoff] Waiting ${Math.round(backoff)}ms before next attempt...`); + await new Promise(resolve => setTimeout(resolve, backoff)); + delay = Math.min(delay * 2, MAX_BACKOFF_MS); // Double the delay for next time, up to max + } + } + } + + // If we get here, all attempts failed + const errorMessage = `Node did not become ready after ${maxAttempts} attempts. Last error: ${lastError?.message || 'Unknown error'}`; + debug('❌ %s', errorMessage); + + // Final attempt to get container logs before failing + try { + const logs = await container.logs({ + stdout: true, + stderr: true, + tail: 100, + timestamps: true, + follow: false + }); + debug('=== FINAL CONTAINER LOGS ===\n%s\n=== END CONTAINER LOGS ===', logs.toString()); + } catch (logError) { + debug('Failed to get final container logs: %o', logError); + } + + throw new Error(errorMessage); + } + + async healthCheck(healthUrl: string): Promise<{ ok: boolean; status: number }> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + + try { + const response = await fetch(healthUrl, { + headers: { + 'Accept': 'application/json', + 'Connection': 'close' + }, + signal: controller.signal + }); + clearTimeout(timeout); + return { + ok: response.ok, + status: response.status + }; + } catch (error) { + clearTimeout(timeout); + if (error instanceof Error && error.name === 'AbortError') { + throw new Error(`Health check timed out after 5000ms (${healthUrl})`); + } + throw error; + } + } + + mapContainerState(state: string): NodeStatus['status'] { + if (!state) return 'error'; + + const stateLower = state.toLowerCase(); + if (['created', 'restarting'].includes(stateLower)) return 'starting'; + if (stateLower === 'running') return 'running'; + if (stateLower === 'paused') return 'stopping'; + if (['dead', 'exited', 'stopped'].includes(stateLower)) return 'stopped'; + + return 'error'; + } + + private async verifyContainerRunning(container: Container): Promise { + debug('[verifyContainerRunning] Checking container status...'); + + try { + const data = await container.inspect(); + debug('[verifyContainerRunning] Container inspect data:', JSON.stringify({ + Id: data.Id, + Name: data.Name, + State: data.State, + Config: { + Image: data.Config?.Image, + Env: data.Config?.Env?.filter(env => env.startsWith('NODE_') || env.startsWith('DEBUG')), + Cmd: data.Config?.Cmd + }, + HostConfig: { + Memory: data.HostConfig?.Memory, + NanoCpus: data.HostConfig?.NanoCpus, + NetworkMode: data.HostConfig?.NetworkMode + } + }, null, 2)); + + if (!data.State.Running) { + const errorMessage = `Container is not running. Status: ${data.State.Status}, ExitCode: ${data.State.ExitCode}, Error: ${data.State.Error}`; + debug(`[verifyContainerRunning] ${errorMessage}`); + + // Try to get container logs for more context + try { + const logs = await container.logs({ + stdout: true, + stderr: true, + tail: 50 // Get last 50 lines of logs + }); + debug('[verifyContainerRunning] Container logs:', logs.toString()); + } catch (logError) { + debug('[verifyContainerRunning] Failed to get container logs:', logError); + } + + throw new Error(errorMessage); + } + + debug('[verifyContainerRunning] Container is running'); + } catch (error) { + debug('[verifyContainerRunning] Error checking container status:', error); + throw error; + } + } + + /** + * Get the status of a node including container status, network info, and resource usage + * @param handle The node handle containing node metadata + * @param container The Docker container instance + * @returns A promise that resolves to the node status + */ + async getNodeStatus(handle: NodeHandle, container: Container): Promise { + // Default error status for when container is not found or other errors occur + const errorStatus: NodeStatus = { + id: handle.id, + status: 'error', + error: 'Failed to get node status', + network: { + address: '', + httpPort: 0, + requestPort: 0, + peers: [] + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + + try { + // Get container info + const containerInfo = await container.inspect(); + + // Get request port once since we use it multiple times + const requestPort = handle.getRequestPort?.() || 0; + + // Initialize with default values + const status: NodeStatus = { + id: handle.id, // Use the node ID from handle + containerId: container.id, + status: this.mapContainerState(containerInfo.State?.Status || ''), + network: { + address: containerInfo.NetworkSettings?.IPAddress || '', + httpPort: requestPort, + requestPort: requestPort, + peers: [], + networkId: '' + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + + // Update network info if available + if (containerInfo.NetworkSettings?.Networks) { + const network = Object.values(containerInfo.NetworkSettings.Networks)[0]; + if (network) { + // Ensure we have existing network values or use defaults + const currentNetwork = status.network || { + address: '', + httpPort: 0, + requestPort: 0, + peers: [] + }; + + // Create a new network object with all required properties + status.network = { + address: network.IPAddress || currentNetwork.address, + httpPort: currentNetwork.httpPort, + requestPort: currentNetwork.requestPort, + peers: currentNetwork.peers, + networkId: network.NetworkID || '' + }; + } + } + + // Get container stats for resource usage + try { + const stats = await container.stats({ stream: false }); + const statsData = JSON.parse(stats.toString()); + + if (statsData?.cpu_stats?.cpu_usage) { + status.resources!.cpu.usage = statsData.cpu_stats.cpu_usage.total_usage || 0; + status.resources!.cpu.limit = (statsData.cpu_stats.online_cpus || 0) * 1e9; // Convert to nanoCPUs + } + + if (statsData?.memory_stats) { + status.resources!.memory.usage = statsData.memory_stats.usage || 0; + status.resources!.memory.limit = statsData.memory_stats.limit || 0; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug(`Failed to get container stats for ${container.id}: %s`, errorMessage); + // Update status with error but don't return yet + status.status = 'error'; + status.error = `Failed to get container stats: ${errorMessage}`; + } + + return status; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + debug(`Error getting node status for ${handle.id}: %s`, errorMessage); + + return { + ...errorStatus, + id: handle.id, + error: errorMessage, + status: 'error' + }; + } + } +} diff --git a/src/orchestration/docker-orchestrator/types.ts b/src/orchestration/docker-orchestrator/types.ts new file mode 100644 index 0000000..8b47325 --- /dev/null +++ b/src/orchestration/docker-orchestrator/types.ts @@ -0,0 +1,40 @@ +import Docker from 'dockerode'; +import { NodeHandle, NodeConfig, NodeStatus } from '../types'; + +export interface DockerNodeHandle extends NodeHandle { + containerId: string; + networkId?: string; +} + +export interface DockerOrchestratorOptions { + /** + * Docker image to use for containers + * Defaults to 'rhizome-node' if not specified + */ + image?: string; + + /** Working directory inside container */ + containerWorkDir?: string; + + /** Whether to build test image if not found */ + autoBuildTestImage?: boolean; +} + +export interface ContainerResources { + cpuShares?: number; + memory?: number; + memorySwap?: number; + nanoCpus?: number; +} + +export interface ContainerStatus { + containerId: string; + image: string; + state: string; + status: NodeStatus['status']; // Use the status type from NodeStatus + networkSettings: { + ipAddress: string; + gateway: string; + ports: Record | null>; + }; +} diff --git a/src/orchestration/docker-orchestrator/utils/port-utils.ts b/src/orchestration/docker-orchestrator/utils/port-utils.ts new file mode 100644 index 0000000..f99b875 --- /dev/null +++ b/src/orchestration/docker-orchestrator/utils/port-utils.ts @@ -0,0 +1,46 @@ +/** + * Get a random available port in the range 30000-50000 + * @returns A random port number + */ +export function getRandomPort(): number { + return Math.floor(30000 + Math.random() * 20000); +} + +/** + * Check if a port is available + * @param port Port number to check + * @returns True if the port is available, false otherwise + */ +export async function isPortAvailable(port: number): Promise { + const net = await import('net'); + return new Promise((resolve) => { + const server = net.createServer(); + server.once('error', () => resolve(false)); + server.once('listening', () => { + server.close(() => resolve(true)); + }); + server.listen(port); + }); +} + +/** + * Get an available port, optionally starting from a specific port + * @param startPort Optional starting port (default: 30000) + * @returns A promise that resolves to an available port + */ +export async function getAvailablePort(startPort: number = 30000): Promise { + let port = startPort; + while (port <= 65535) { + if (await isPortAvailable(port)) { + return port; + } + port++; + } + throw new Error('No available ports found'); +} + +export default { + getRandomPort, + isPortAvailable, + getAvailablePort +}; diff --git a/src/orchestration/factory.ts b/src/orchestration/factory.ts new file mode 100644 index 0000000..5b160a8 --- /dev/null +++ b/src/orchestration/factory.ts @@ -0,0 +1,21 @@ +import { NodeOrchestrator, OrchestratorType } from './types'; +import { DockerOrchestrator } from './docker-orchestrator'; +import { TestOrchestrator } from './test-orchestrator'; + +/** + * Factory function to create an appropriate orchestrator based on environment + */ +export function createOrchestrator( + type: OrchestratorType = 'in-memory', + options?: any +): NodeOrchestrator { + switch (type) { + case 'docker': + return new DockerOrchestrator(options); + case 'kubernetes': + throw new Error('Kubernetes orchestrator not yet implemented'); + case 'in-memory': + default: + return new TestOrchestrator(); + } +} diff --git a/src/orchestration/index.ts b/src/orchestration/index.ts new file mode 100644 index 0000000..10420ab --- /dev/null +++ b/src/orchestration/index.ts @@ -0,0 +1,9 @@ +// Re-export all types and interfaces +export * from './types'; + +// Export orchestrator implementations +export * from './docker-orchestrator'; +export * from './test-orchestrator'; + +// Export factory function +export { createOrchestrator } from './factory'; diff --git a/src/orchestration/test-orchestrator/index.ts b/src/orchestration/test-orchestrator/index.ts new file mode 100644 index 0000000..38f5280 --- /dev/null +++ b/src/orchestration/test-orchestrator/index.ts @@ -0,0 +1,198 @@ +import { RhizomeNode, type RhizomeNodeConfig } from '../../node'; +import { PeerAddress } from '../../network'; +import { BaseOrchestrator } from '../base-orchestrator'; +import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types'; +import { getRandomPort } from '../docker-orchestrator/utils/port-utils'; +import { BasicCollection } from '../../collections/collection-basic'; +import Debug from 'debug'; + +const debug = Debug('rz:test-orchestrator'); + +/** + * In-memory implementation of NodeOrchestrator for testing + */ +export class TestOrchestrator extends BaseOrchestrator { + private nodes: Map = new Map(); + + async startNode(config: NodeConfig): Promise { + const nodeId = config.id || `node-${Date.now()}`; + // Use getRandomPort instead of 0 for auto-selection + const httpPort = config.network?.port || getRandomPort(); + const requestPort = config.network?.requestPort || getRandomPort(); + + // Map NodeConfig to RhizomeNodeConfig with all required properties + const nodeConfig: RhizomeNodeConfig = { + // Required network properties + requestBindHost: '0.0.0.0', + requestBindPort: requestPort, + publishBindHost: '0.0.0.0', + publishBindPort: getRandomPort(), // Use a random port for publish socket + httpAddr: '0.0.0.0', + httpPort: httpPort, + httpEnable: true, + + // Required peer properties + peerId: nodeId, + creator: 'test-orchestrator', + + // Map network bootstrap peers to seedPeers if provided + seedPeers: config.network?.bootstrapPeers?.map(peer => { + const [host, port] = peer.split(':'); + return new PeerAddress(host, parseInt(port)); + }) || [], + + // Storage configuration with defaults + storage: { + type: 'memory', + path: config.storage?.path || `./data/${nodeId}`, + ...(config.storage || {}) + } + }; + + const node = new RhizomeNode(nodeConfig); + + // Create and connect a user collection + const userCollection = new BasicCollection('user'); + // Connect the collection to the node before serving it + userCollection.rhizomeConnect(node); + // Now serve the collection through the HTTP API + node.httpServer.httpApi.serveCollection(userCollection); + + // Start the node and wait for all components to be ready + debug(`[${nodeId}] Starting node and waiting for it to be fully ready...`); + try { + await node.start(); + debug(`[${nodeId}] Node is fully started and ready`); + } catch (error) { + debug(`[${nodeId}] Error starting node:`, error); + throw error; + } + + // Get the actual port the server is using + const serverAddress = node.httpServer.server?.address(); + let actualPort = httpPort; + + // Handle different address types (string or AddressInfo) + if (serverAddress) { + actualPort = typeof serverAddress === 'string' + ? httpPort + : serverAddress.port || httpPort; + } + + const handle: NodeHandle = { + id: nodeId, + config: { + ...config, + id: nodeId, + network: { + ...config.network, + port: actualPort, + requestPort: requestPort + } + }, + status: async () => this.getNodeStatus(handle), + getApiUrl: () => `http://localhost:${actualPort}/api`, + stop: async () => { + await node.stop(); + this.nodes.delete(nodeId); + }, + getRequestPort: () => requestPort, + }; + + this.nodes.set(nodeId, { handle, node }); + return handle; + } + + async stopNode(handle: NodeHandle): Promise { + const node = this.nodes.get(handle.id); + if (node) { + await node.node.stop(); + this.nodes.delete(handle.id); + } + } + + async getNodeStatus(handle: NodeHandle): Promise { + const node = this.nodes.get(handle.id); + if (!node) { + return { + id: handle.id, + status: 'stopped', + error: 'Node not found', + network: { + address: '127.0.0.1', + httpPort: 0, + requestPort: 0, + peers: [] + }, + resources: { + cpu: { usage: 0, limit: 0 }, + memory: { usage: 0, limit: 0 } + } + }; + } + + + // Since we don't have a direct way to check if the node is running, + // we'll assume it's running if it's in our nodes map + // In a real implementation, we would check the actual node state + const status: NodeStatus = { + id: handle.id, + status: 'running', + network: { + address: '127.0.0.1', + httpPort: node.node.config.httpPort || 0, + requestPort: node.node.config.requestBindPort || 0, + peers: node.node.peers ? Array.from(node.node.peers.peers).map(p => p.reqAddr.toAddrString()) : [] + }, + resources: { + cpu: { + usage: 0, + limit: 0, + }, + memory: { + usage: 0, + limit: 0, + }, + } + }; + + return status; + } + + async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise { + const n1 = this.nodes.get(node1.id)?.node; + const n2 = this.nodes.get(node2.id)?.node; + + if (!n1 || !n2) { + throw new Error('One or both nodes not found'); + } + + // In a real implementation, we would connect the nodes here + // For testing, we'll just log the connection attempt + console.log(`Connecting nodes ${node1.id} and ${node2.id}`); + } + + async partitionNetwork(partitions: NetworkPartition): Promise { + // In a real implementation, we would create network partitions + // For testing, we'll just log the partition attempt + console.log('Creating network partitions:', partitions); + } + + async setResourceLimits( + handle: NodeHandle, + limits: Partial + ): Promise { + // In-memory nodes don't have real resource limits + console.log(`Setting resource limits for ${handle.id}:`, limits); + } + + /** + * Clean up all resources + */ + async cleanup(): Promise { + await Promise.all( + Array.from(this.nodes.values()).map(({ node }) => node.stop()) + ); + this.nodes.clear(); + } +} diff --git a/src/orchestration/types.ts b/src/orchestration/types.ts new file mode 100644 index 0000000..51b2c7b --- /dev/null +++ b/src/orchestration/types.ts @@ -0,0 +1,104 @@ +/** + * Core types and interfaces for the orchestration layer + */ + +export interface NodeConfig { + /** Unique identifier for the node */ + id: string; + + /** Network configuration */ + network?: { + /** Port to listen on (0 = auto-select) */ + port?: number; + /** Port for request/reply communication */ + requestPort?: number; + /** Known peers to connect to */ + bootstrapPeers?: string[]; + }; + + /** Resource constraints */ + resources?: { + /** CPU shares (0-1024) */ + cpu?: number; + /** Memory limit in MB */ + memory?: number; + /** Memory swap limit in MB (defaults to 2x memory if not specified) */ + memorySwap?: number; + }; + + /** Storage configuration */ + storage?: { + /** Storage type */ + type?: 'memory' | 'leveldb' | 'sqlite' | 'postgres'; + /** Path to data directory */ + path?: string; + /** Maximum storage in MB */ + limit?: number; + }; + + /** Additional configuration options */ + [key: string]: any; +} + +export interface NodeStatus { + id: string; + containerId?: string; + status: 'starting' | 'running' | 'stopping' | 'stopped' | 'error'; + network?: { + address: string; + requestPort: number; + httpPort: number; + peers: string[]; + networkId?: string; + }; + resources?: { + cpu: { + usage: number; + limit: number; + }; + memory: { + usage: number; + limit: number; + }; + }; + error?: string; +} + +export interface NodeHandle { + id: string; + config: NodeConfig; + status: () => Promise; + stop: () => Promise; + /** Get API URL if applicable */ + getApiUrl?: () => string; + getRequestPort: () => number | undefined; +} + +export interface NetworkPartition { + groups: string[][]; +} + +export interface NodeOrchestrator { + /** Start a new node with the given configuration */ + startNode(config: NodeConfig): Promise; + + /** Stop a running node */ + stopNode(handle: NodeHandle): Promise; + + /** Get status of a node */ + getNodeStatus(handle: NodeHandle): Promise; + + /** Connect two nodes */ + connectNodes(node1: NodeHandle, node2: NodeHandle): Promise; + + /** Create network partitions */ + partitionNetwork(partitions: NetworkPartition): Promise; + + /** Set resource limits for a node */ + setResourceLimits(handle: NodeHandle, limits: Partial): Promise; + + /** Clean up all resources */ + cleanup(): Promise; +} + +export type OrchestratorType = 'in-memory' | 'docker' | 'kubernetes'; diff --git a/src/query/index.ts b/src/query/index.ts index 4787126..4828912 100644 --- a/src/query/index.ts +++ b/src/query/index.ts @@ -1,2 +1,2 @@ export { QueryEngine } from './query-engine'; -export { StorageQueryEngine, JsonLogic as StorageJsonLogic } from './storage-query-engine'; \ No newline at end of file +export { StorageQueryEngine, type JsonLogic as StorageJsonLogic } from './storage-query-engine'; \ No newline at end of file diff --git a/src/query/query-engine.ts b/src/query/query-engine.ts index fe8cb23..2fa249b 100644 --- a/src/query/query-engine.ts +++ b/src/query/query-engine.ts @@ -1,4 +1,5 @@ -import { apply, is_logic } from 'json-logic-js'; +import jsonLogic from 'json-logic-js'; +const { apply, is_logic } = jsonLogic; import Debug from 'debug'; import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema/schema'; import { Lossless, LosslessViewOne, LosslessViewMany, CollapsedDelta } from '../views/lossless'; diff --git a/src/query/storage-query-engine.ts b/src/query/storage-query-engine.ts index 5457aec..16f92be 100644 --- a/src/query/storage-query-engine.ts +++ b/src/query/storage-query-engine.ts @@ -1,4 +1,5 @@ -import { apply } from 'json-logic-js'; +import jsonLogic from 'json-logic-js'; +const { apply } = jsonLogic; import Debug from 'debug'; import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema'; import { DeltaQueryStorage, DeltaQuery } from '../storage/interface'; diff --git a/src/storage/factory.ts b/src/storage/factory.ts index 7b854e7..5b50a48 100644 --- a/src/storage/factory.ts +++ b/src/storage/factory.ts @@ -1,7 +1,10 @@ +import Debug from 'debug'; import { DeltaStorage, DeltaQueryStorage, StorageConfig } from './interface'; import { MemoryDeltaStorage } from './memory'; import { LevelDBDeltaStorage } from './leveldb'; +const debug = Debug('rz:storage:factory'); + /** * Factory for creating delta storage instances based on configuration */ @@ -56,10 +59,10 @@ export class StorageFactory { ): Promise { const batchSize = options.batchSize || 1000; - console.log('Starting storage migration...'); + debug('Starting storage migration...'); const allDeltas = await source.getAllDeltas(); - console.log(`Found ${allDeltas.length} deltas to migrate`); + debug(`Found %d deltas to migrate`, allDeltas.length); // Migrate in batches to avoid memory issues for (let i = 0; i < allDeltas.length; i += batchSize) { @@ -69,19 +72,21 @@ export class StorageFactory { await target.storeDelta(delta); } - console.log(`Migrated ${Math.min(i + batchSize, allDeltas.length)} / ${allDeltas.length} deltas`); + debug('Migrated %d / %d deltas', Math.min(i + batchSize, allDeltas.length), allDeltas.length); } - console.log('Migration completed successfully'); + debug('Migration completed successfully'); // Verify migration const sourceStats = await source.getStats(); const targetStats = await target.getStats(); if (sourceStats.totalDeltas !== targetStats.totalDeltas) { - throw new Error(`Migration verification failed: source has ${sourceStats.totalDeltas} deltas, target has ${targetStats.totalDeltas}`); + const errorMsg = `Migration verification failed: source has ${sourceStats.totalDeltas} deltas, target has ${targetStats.totalDeltas}`; + debug(errorMsg); + throw new Error(errorMsg); } - console.log(`Migration verified: ${targetStats.totalDeltas} deltas migrated successfully`); + debug('Migration verified: %d deltas migrated successfully', targetStats.totalDeltas); } } \ No newline at end of file diff --git a/src/util/md-files.ts b/src/util/md-files.ts index b2b7057..f10723f 100644 --- a/src/util/md-files.ts +++ b/src/util/md-files.ts @@ -1,5 +1,5 @@ import Debug from "debug"; -import {FSWatcher, readdirSync, readFileSync, watch} from "fs"; +import {FSWatcher, readdirSync, readFileSync, watch, accessSync, constants} from "fs"; import path, {join} from "path"; import showdown from "showdown"; import {RhizomeNode} from "../node"; @@ -48,9 +48,32 @@ export class MDFiles { } readReadme() { - const md = readFileSync('./README.md').toString(); + let currentDir = process.cwd(); + const root = path.parse(currentDir).root; + let readmePath: string | null = null; + + // Traverse up the directory tree until we find README.md or hit the root + while (currentDir !== root) { + const testPath = path.join(currentDir, 'README.md'); + try { + // Using the imported accessSync function + accessSync(testPath, constants.F_OK); + readmePath = testPath; + break; + } catch (err) { + // Move up one directory + currentDir = path.dirname(currentDir); + } + } + + if (!readmePath) { + debug('No README.md found in any parent directory'); + return; + } + + const md = readFileSync(readmePath).toString(); const html = htmlDocFromMarkdown(md); - this.readme = {name: 'README', md, html}; + this.readme = { name: 'README', md, html }; } getReadmeHTML() { diff --git a/src/views/lossless.ts b/src/views/lossless.ts index 6b1fb02..1bf01b3 100644 --- a/src/views/lossless.ts +++ b/src/views/lossless.ts @@ -172,7 +172,6 @@ export class Lossless { viewSpecific(entityId: DomainEntityID, deltaIds: DeltaID[], deltaFilter?: DeltaFilter): LosslessViewOne | undefined { const combinedFilter = (delta: Delta) => { if (!deltaIds.includes(delta.id)) { - debug(`[${this.rhizomeNode.config.peerId}]`, `Excluding delta ${delta.id} because it's not in the requested list of deltas`); return false; } if (!deltaFilter) return true; diff --git a/tsconfig.json b/tsconfig.json index 888dcfa..cb44e62 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,16 +1,24 @@ { "compilerOptions": { - "target": "ES2022", - "module": "ESNext", + "target": "ES6", + "module": "CommonJS", "esModuleInterop": true, - "allowSyntheticDefaultImports": true, "moduleResolution": "Node", "sourceMap": true, "baseUrl": ".", + "rootDir": ".", "outDir": "dist", + "importsNotUsedAsValues": "remove", "strict": true, "skipLibCheck": true, - "forceConsistentCasingInFileNames": true + "forceConsistentCasingInFileNames": true, + "types": ["node", "jest"], + "typeRoots": [ + "./node_modules/@types" + ], + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": false }, "include": [ "src/**/*", diff --git a/src/test-utils/schemas.ts b/util/schemas.ts similarity index 97% rename from src/test-utils/schemas.ts rename to util/schemas.ts index 35f035e..c0cd46c 100644 --- a/src/test-utils/schemas.ts +++ b/util/schemas.ts @@ -1,4 +1,4 @@ -import { SchemaBuilder } from '../../src/schema'; +import { SchemaBuilder } from '../src/schema'; /** * Common schemas used for testing purposes.