Merge pull request 'feature/docker-orchestrator-tests' (#3) from feature/docker-orchestrator-tests into main

Reviewed-on: #3

.gitignore

@@ -6,3 +6,4 @@ coverage/
 .env
 data/
 test-data/
+*.code-workspace

.windsurf/workflows/mr-summary.md

@@ -0,0 +1,7 @@
+---
+description: Generate a merge request summary
+---
+
+- fetch origin/main
+- compare the current branch to origin/main
+- generate a merge request summary as an untracked file called merge-${branch}.md

Dockerfile.test

@@ -0,0 +1,26 @@
+FROM node:24
+
+# Set working directory
+WORKDIR /app
+
+# Copy package files first for better layer caching
+COPY package*.json tsconfig.json ./
+
+# Install dependencies including devDependencies
+RUN npm ci --include=dev
+
+# Copy source files
+COPY src/ src/
+COPY markdown/ markdown/
+COPY examples/ examples/
+COPY util/ util/
+COPY README.md ./
+
+# Build the application
+RUN npm run build 
+
+# Set environment variables
+ENV NODE_ENV=test
+
+# Command to run the application
+CMD ["node", "dist/examples/app.js"]

__tests__/debug/console-test.ts

@@ -0,0 +1,20 @@
+import Debug from 'debug';
+
+// Set up debug instances for different log levels
+const debug = Debug('rz:test:console');
+const debugError = Debug('rz:test:console:error');
+const debugWarn = Debug('rz:test:console:warn');
+
+// Test debug output
+// Note: These will only show if DEBUG=rz:* is set in the environment
+debug('=== DEBUG LOG TEST ===');
+debug('This is a test debug message');
+debugError('This is a test error message');
+debugWarn('This is a test warning message');
+
+describe('Debug Test', () => {
+  it('should output debug messages when DEBUG is enabled', () => {
+    debug('Test debug message from inside test');
+    expect(true).toBe(true);
+  });
+});

__tests__/jest-setup.ts

@@ -0,0 +1,22 @@
+// Set up environment variables for tests
+process.env.DEBUG = 'rz:*';
+
+// Extend the global Jest namespace
+declare global {
+  namespace jest {
+    interface Matchers<R> {
+      toBeWithinRange(a: number, b: number): R;
+    }
+  }
+}
+
+// Add any global test setup here
+
+// This is a placeholder test to satisfy Jest's requirement for at least one test
+describe('Test Setup', () => {
+  it('should pass', () => {
+    expect(true).toBe(true);
+  });
+});
+
+export {}; // This file needs to be a module

__tests__/nested-resolution-performance.ts

@@ -8,7 +8,10 @@
  * - Circular reference handling at scale
  */
 
+import Debug from 'debug';
 import { RhizomeNode } from '../src/node';
+
+const debug = Debug('rz:test:nested-resolution-performance');
 import { Delta } from '../src/core';
 import { DefaultSchemaRegistry } from '../src/schema';
 import { SchemaBuilder, PrimitiveSchemas, ReferenceSchemas, ArraySchemas } from '../src/schema';
@@ -124,7 +127,7 @@ describe('Nested Object Resolution Performance', () => {
       }
 
       const setupTime = performance.now() - startSetup;
-      console.log(`Setup time for ${userCount} users with relationships: ${setupTime.toFixed(2)}ms`);
+      debug(`Setup time for ${userCount} users with relationships: ${setupTime.toFixed(2)}ms`);
 
       // Test resolution performance for a user with many connections
       const testUserId = userIds[50]; // Pick a user in the middle
@@ -141,7 +144,7 @@ describe('Nested Object Resolution Performance', () => {
       );
 
       const resolutionTime = performance.now() - startResolution;
-      console.log(`Resolution time for user with many connections: ${resolutionTime.toFixed(2)}ms`);
+      debug(`Resolution time for user with many connections: ${resolutionTime.toFixed(2)}ms`);
 
       // Verify the resolution worked
       expect(nestedView.id).toBe(testUserId);
@@ -155,7 +158,7 @@ describe('Nested Object Resolution Performance', () => {
       const totalNestedObjects = Object.values(nestedView.nestedObjects).reduce(
         (total, arr) => total + (arr?.length || 0), 0
       );
-      console.log('Total nested objects resolved:', totalNestedObjects);
+      debug('Total nested objects resolved: %o', totalNestedObjects);
       
       // The test user should have friends, followers, and possibly a mentor
       expect(Object.keys(nestedView.nestedObjects).length).toBeGreaterThan(0);
@@ -218,7 +221,7 @@ describe('Nested Object Resolution Performance', () => {
       }
 
       const setupTime = performance.now() - startSetup;
-      console.log(`Setup time for chain of ${chainLength} users: ${setupTime.toFixed(2)}ms`);
+      debug(`Setup time for chain of ${chainLength} users: ${setupTime.toFixed(2)}ms`);
 
       // Test resolution from the start of the chain
       const firstUserId = userIds[0];
@@ -235,7 +238,7 @@ describe('Nested Object Resolution Performance', () => {
       );
 
       const resolutionTime = performance.now() - startResolution;
-      console.log(`Resolution time for deep chain (maxDepth=5): ${resolutionTime.toFixed(2)}ms`);
+      debug(`Resolution time for deep chain (maxDepth=5): ${resolutionTime.toFixed(2)}ms`);
 
       // Verify the resolution worked and respected depth limits
       expect(nestedView.id).toBe(firstUserId);
@@ -255,7 +258,7 @@ describe('Nested Object Resolution Performance', () => {
       }
       
       expect(depth).toBeLessThanOrEqual(5);
-      console.log(`Actual resolved depth: ${depth}`);
+      debug(`Actual resolved depth: ${depth}`);
     });
 
     it('should handle circular references in large graphs without performance degradation', async () => {
@@ -318,7 +321,7 @@ describe('Nested Object Resolution Performance', () => {
       }
 
       const setupTime = performance.now() - startSetup;
-      console.log(`Setup time for circular graph with ${userCount} users: ${setupTime.toFixed(2)}ms`);
+      debug(`Setup time for circular graph with ${userCount} users: ${setupTime.toFixed(2)}ms`);
 
       // Test resolution performance with circular references
       const testUserId = userIds[0];
@@ -335,7 +338,7 @@ describe('Nested Object Resolution Performance', () => {
       );
 
       const resolutionTime = performance.now() - startResolution;
-      console.log(`Resolution time for circular graph (maxDepth=3): ${resolutionTime.toFixed(2)}ms`);
+      debug(`Resolution time for circular graph (maxDepth=3): ${resolutionTime.toFixed(2)}ms`);
 
       // Verify the resolution completed without hanging
       expect(nestedView.id).toBe(testUserId);
@@ -352,7 +355,7 @@ describe('Nested Object Resolution Performance', () => {
         expect(nestedView.nestedObjects.connections.length).toBeLessThanOrEqual(3);
       }
       
-      console.log(`Connections resolved: ${nestedView.nestedObjects.connections?.length || 0}`);
+      debug(`Connections resolved: ${nestedView.nestedObjects.connections?.length || 0}`);
     });
   });
 });

__tests__/nested-resolution.ts

@@ -13,7 +13,7 @@ import { RhizomeNode } from '../src/node';
 import { Delta } from '../src/core';
 import { DefaultSchemaRegistry } from '../src/schema';
 import { SchemaBuilder, PrimitiveSchemas, ReferenceSchemas } from '../src/schema';
-import { CommonSchemas } from '../src/test-utils/schemas';
+import { CommonSchemas } from '../util/schemas';
 import { TypedCollectionImpl } from '../src/collections';
 
 describe('Nested Object Resolution', () => {

__tests__/query.ts

@@ -2,7 +2,7 @@ import { QueryEngine } from '../src/query';
 import { Lossless } from '../src/views';
 import { DefaultSchemaRegistry } from '../src/schema';
 import { SchemaBuilder, PrimitiveSchemas } from '../src/schema';
-import { CommonSchemas } from '../src/test-utils/schemas';
+import { CommonSchemas } from '../util/schemas';
 import { Delta } from '../src/core';
 import { RhizomeNode } from '../src/node';
 

__tests__/run/001-single-node-orchestrated.ts

@@ -0,0 +1,72 @@
+import { createOrchestrator, type NodeConfig } from '../../src/orchestration';
+
+// Increase test timeout to 30 seconds
+jest.setTimeout(30000);
+
+describe('Run (Orchestrated)', () => {
+  const orchestrator = createOrchestrator('in-memory');
+  let nodeHandle: any;
+  let apiUrl: string;
+
+  beforeAll(async () => {
+    console.time('Test setup');
+    console.time('Create config');
+    // Configure and start the node
+    const config: NodeConfig = {
+      id: 'app-001',
+    };
+    console.timeEnd('Create config');
+
+    console.time('Start node');
+    nodeHandle = await orchestrator.startNode(config);
+    console.timeEnd('Start node');
+
+    console.time('Get API URL');
+    apiUrl = nodeHandle.getApiUrl();
+    console.timeEnd('Get API URL');
+    console.timeEnd('Test setup');
+  }, 60000); // Increase timeout to 60s for this hook
+
+  afterAll(async () => {
+    // Stop the node
+    if (nodeHandle) {
+      await orchestrator.stopNode(nodeHandle);
+    }
+  });
+
+  it('can put a new user and fetch it', async () => {
+    // Create a new record
+    const createResponse = await fetch(`${apiUrl}/user`, {
+      method: 'PUT',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        id: 'peon-1',
+        properties: {
+          name: 'Peon',
+          age: 263,
+        },
+      }),
+    });
+
+    const createdUser = await createResponse.json();
+    expect(createdUser).toMatchObject({
+      id: 'peon-1',
+      properties: {
+        name: 'Peon',
+        age: 263,
+      },
+    });
+
+    // Read the created record
+    const getResponse = await fetch(`${apiUrl}/user/peon-1`);
+    const fetchedUser = await getResponse.json();
+    
+    expect(fetchedUser).toMatchObject({
+      id: 'peon-1',
+      properties: {
+        name: 'Peon',
+        age: 263,
+      },
+    });
+  });
+});

__tests__/run/002-two-nodes-orchestrated.ts

@@ -0,0 +1,134 @@
+import Debug from 'debug';
+import { createOrchestrator } from '../../src/orchestration';
+import type { NodeConfig, NodeHandle } from '../../src/orchestration';
+
+// Increase test timeout to 30 seconds
+jest.setTimeout(30000);
+
+const debug = Debug('test:two-orchestrated');
+
+describe('Run (Two Nodes Orchestrated)', () => {
+  const orchestrator = createOrchestrator('in-memory');
+  // Define a type that includes all required methods
+  type FullNodeHandle = NodeHandle & {
+    getRequestPort: () => number;
+    getApiUrl: () => string;
+  };
+  
+  const nodes: FullNodeHandle[] = [];
+  const nodeIds = ['app-002-A', 'app-002-B'];
+
+  beforeAll(async () => {
+    console.time('Test setup');
+    
+    // Start first node
+    console.time('Create node1 config');
+    const node1Config: NodeConfig = {
+      id: nodeIds[0],
+    };
+    console.timeEnd('Create node1 config');
+
+    console.time('Start node1');
+    const node1 = (await orchestrator.startNode(node1Config)) as FullNodeHandle;
+    console.timeEnd('Start node1');
+
+    // Start second node with first node as bootstrap peer
+    console.time('Create node2 config');
+    const node2Config: NodeConfig = {
+      id: nodeIds[1],
+      network: {
+        bootstrapPeers: [`localhost:${node1.getRequestPort()}`],
+      },
+    };
+    console.timeEnd('Create node2 config');
+
+    console.time('Start node2');
+    const node2 = (await orchestrator.startNode(node2Config)) as FullNodeHandle;
+    console.timeEnd('Start node2');
+
+    nodes.push(node1, node2);
+
+    // Connect the nodes
+    console.time('Connect nodes');
+    await orchestrator.connectNodes(node1, node2);
+    console.timeEnd('Connect nodes');
+    
+    console.timeEnd('Test setup');
+  }, 120000); // Increase timeout to 120s for this hook
+
+  afterAll(async () => {
+    // Stop all nodes in parallel
+    await Promise.all(nodes.map(node => node && orchestrator.stopNode(node)));
+  });
+
+  it('can create a record on node0 and read it from node1', async () => {
+    const [node0, node1] = nodes;
+    const node0Url = node0.getApiUrl();
+    const node1Url = node1.getApiUrl();
+
+    debug(`Node 0 URL: ${node0Url}`);
+    debug(`Node 1 URL: ${node1Url}`);
+
+    // Create a new record on node0
+    const createResponse = await fetch(`${node0Url}/user`, {
+      method: 'PUT',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        id: 'peon-1',
+        properties: {
+          name: 'Peon',
+          age: 741,
+        },
+      }),
+    });
+
+    const createdUser = await createResponse.json();
+    expect(createdUser).toMatchObject({
+      id: 'peon-1',
+      properties: {
+        name: 'Peon',
+        age: 741,
+      },
+    });
+
+    // Small delay to allow for synchronization
+    await new Promise(resolve => setTimeout(resolve, 100));
+
+    // Read the record from node1
+    const getResponse = await fetch(`${node1Url}/user/peon-1`);
+    const fetchedUser = await getResponse.json();
+    
+    expect(fetchedUser).toMatchObject({
+      id: 'peon-1',
+      properties: {
+        name: 'Peon',
+        age: 741,
+      },
+    });
+  });
+
+  it.skip('can demonstrate network partitioning', async () => {
+    // This test shows how we can simulate network partitions
+    // For now, it's just a placeholder since we'd need to implement
+    // the actual partitioning logic in the InMemoryOrchestrator
+    const [node0, node1] = nodes;
+    
+    // Simulate partition (actual implementation would use orchestrator.partitionNetwork)
+    debug('Simulating network partition between nodes');
+    // await orchestrator.partitionNetwork({
+    //   groups: [[node0.id], [node1.id]]
+    // });
+    
+    // Test behavior during partition...
+    
+    // Heal partition
+    // await orchestrator.partitionNetwork({
+    //   groups: [[node0.id, node1.id]]
+    // });
+    
+    // Test behavior after healing...
+    
+    // Mark test as passed (remove once actual test is implemented)
+    expect(true).toBe(true);
+  });
+});

__tests__/run/002-two-nodes.ts

@@ -17,7 +17,7 @@ describe('Run', () => {
     apps[0].config.seedPeers.push(apps[1].myRequestAddr);
     apps[1].config.seedPeers.push(apps[0].myRequestAddr);
 
-    await Promise.all(apps.map((app) => app.start(false)));
+    await Promise.all(apps.map((app) => app.start()));
   });
 
   afterAll(async () => {

__tests__/run/005-docker-orchestrator.ts

@@ -0,0 +1,531 @@
+import Docker from 'dockerode';
+import { describe, it, beforeAll, afterAll, expect, jest } from '@jest/globals';
+import Debug from 'debug';
+
+const debug = Debug('rz:test:docker-orchestrator-v2');
+import { createOrchestrator } from '../../src/orchestration';
+import type { NodeOrchestrator, NodeConfig, NodeHandle, NodeStatus } from '../../src/orchestration';
+import { ImageManager } from '../../src/orchestration/docker-orchestrator/managers/image-manager';
+
+// Extend the NodeOrchestrator type to include the docker client for DockerOrchestrator
+interface DockerOrchestrator extends NodeOrchestrator {
+  docker: Docker;
+}
+
+// Extended interface to include additional properties that might be present in the implementation
+interface ExtendedNodeStatus extends Omit<NodeStatus, 'network'> {
+  network?: {
+    address: string;
+    port: number;  // Changed from httpPort to match NodeStatus
+    requestPort: number;
+    peers: string[];
+    bootstrapPeers?: string[];
+    containerId?: string;
+    networkId?: string;
+  };
+  getApiUrl?: () => string;
+}
+
+// Simple test to verify Docker is working
+// Set default timeout for all tests to 5 minutes
+jest.setTimeout(300000);
+
+describe('Docker Orchestrator V2', () => {
+  let docker: Docker;
+  let orchestrator: DockerOrchestrator;
+  let node: NodeHandle | null = null;
+  let node2: NodeHandle | null = null;
+  let nodeConfig: NodeConfig;
+  let node2Config: NodeConfig;
+  let nodePort: number;
+  let node2Port: number;
+
+  beforeAll(async () => {
+    debug('Setting up Docker client and orchestrator...');
+    
+    // Initialize Docker client
+    docker = new Docker();
+    
+    // Verify Docker is running
+    try {
+      await docker.ping();
+      debug('Docker daemon is responding');
+    } catch (error) {
+      debug('Docker daemon is not responding: %o', error);
+      throw error;
+    }
+    
+    // Initialize the orchestrator with the Docker client and test image
+    orchestrator = createOrchestrator('docker') as DockerOrchestrator;
+    debug('Docker orchestrator initialized');
+    
+    // Create a basic node config for testing
+    nodePort = 3000 + Math.floor(Math.random() * 1000);
+    nodeConfig = {
+      id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+      networkId: 'test-network',
+      port: nodePort,
+      resources: {
+        memory: 256, // 256MB
+        cpu: 0.5    // 0.5 CPU
+      }
+    };
+    
+    debug(`Test node configured with ID: ${nodeConfig.id}, port: ${nodePort}`);
+
+    const imageManager = new ImageManager();
+    await imageManager.buildTestImage();
+  }); // 30 second timeout
+
+  afterAll(async () => {
+    debug('Starting test cleanup...');
+    const cleanupPromises: Promise<unknown>[] = [];
+    
+    // Helper function to clean up a node with retries
+    const cleanupNode = async (nodeToClean: NodeHandle | null, nodeName: string) => {
+      if (!nodeToClean) return;
+      
+      debug(`[${nodeName}] Starting cleanup for node ${nodeToClean.id}...`);
+      try {
+        // First try the normal stop
+        await orchestrator.stopNode(nodeToClean).catch(error => {
+          debug(`[${nodeName}] Warning stopping node normally: %s`, error.message);
+          throw error; // Will be caught by outer catch
+        });
+        debug(`[${nodeName}] Node ${nodeToClean.id} stopped gracefully`);
+      } catch (error) {
+        debug(`[${nodeName}] Error stopping node ${nodeToClean.id}: %o`, error);
+        
+        // If normal stop fails, try force cleanup
+        try {
+          debug(`[${nodeName}] Attempting force cleanup for node ${nodeToClean.id}...`);
+          const container = orchestrator.docker.getContainer(`rhizome-${nodeToClean.id}`);
+          await container.stop({ t: 1 }).catch(() => {
+            debug(`[${nodeName}] Container stop timed out, forcing removal...`);
+          });
+          await container.remove({ force: true });
+          debug(`[${nodeName}] Node ${nodeToClean.id} force-removed`);
+        } catch (forceError) {
+          debug(`[${nodeName}] Force cleanup failed for node ${nodeToClean.id}: %o`, forceError);
+        }
+      }
+    };
+    
+    // Clean up all created nodes
+    if (node) {
+      cleanupPromises.push(cleanupNode(node, 'node1'));
+    }
+    
+    if (node2) {
+      cleanupPromises.push(cleanupNode(node2, 'node2'));
+    }
+
+    // Wait for all node cleanups to complete before cleaning up networks
+    if (cleanupPromises.length > 0) {
+      debug('Waiting for node cleanups to complete...');
+      await Promise.race([
+        Promise.all(cleanupPromises),
+        new Promise(resolve => setTimeout(() => {
+          debug('Node cleanup timed out, proceeding with network cleanup...');
+          resolve(null);
+        }, 30000)) // 30s timeout for node cleanup
+      ]);
+    }
+    
+    // Clean up any dangling networks using NetworkManager
+    try {
+      debug('Cleaning up networks...');
+      // Get the network manager from the orchestrator
+      const networkManager = (orchestrator as any).networkManager;
+      if (!networkManager) {
+        debug('Network manager not available for cleanup');
+        return;
+      }
+      
+      // Get all networks managed by this test
+      const networks = Array.from((orchestrator as any).networks.entries() || []);
+      
+      const cleanupResults = await networkManager.cleanupNetworks((orchestrator as any).networks);
+      
+      // Log any cleanup errors
+      cleanupResults.forEach(({ resource, error }: { resource: string; error: Error }) => {
+        if (error) {
+          debug(`Failed to clean up network ${resource || 'unknown'}: %s`, error.message);
+        } else {
+          debug(`Successfully cleaned up network ${resource || 'unknown'}`);
+        }
+      });
+    } catch (error) {
+      debug('Error during network cleanup: %o', error);
+    }
+    
+    debug('All test cleanups completed');
+  }, 120000); // 2 minute timeout for afterAll
+
+  it('should start and stop a node', async () => {
+    debug('Starting test: should start and stop a node');
+    
+    // Create a new config with a unique ID for this test
+    const testNodeConfig = {
+      ...nodeConfig,
+      id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+      network: {
+        ...nodeConfig.network,
+        enableHttpApi: true
+      }
+    };
+    
+    // Start a node
+    debug('Starting node...');
+    const testNode = await orchestrator.startNode(testNodeConfig);
+    expect(testNode).toBeDefined();
+    expect(testNode.id).toBeDefined();
+    debug(`✅ Node started with ID: ${testNode.id}`);
+    
+    try {
+      // Verify the node is running
+      const status = await testNode.status();
+      expect(status).toBeDefined();
+      debug('Node status: %o', status);
+      
+      // Verify we can access the health endpoint
+      const apiUrl = testNode.getApiUrl?.();
+      if (apiUrl) {
+        const response = await fetch(`${apiUrl}/health`);
+        expect(response.ok).toBe(true);
+        const health = await response.json();
+        expect(health).toHaveProperty('status', 'ok');
+      }
+      
+      // Stop the node
+      debug('Stopping node...');
+      await orchestrator.stopNode(testNode);
+      debug('Node stopped');
+    } finally {
+      // Ensure node is cleaned up even if test fails
+      try {
+        await orchestrator.stopNode(testNode).catch(() => {});
+      } catch (e) {
+        debug('Error during node cleanup: %o', e);
+      }
+    }
+  }, 30000); // 30 second timeout for this test
+
+  it('should enforce resource limits', async () => {
+    debug('Starting test: should enforce resource limits');
+    
+    // Create a new node with a unique ID for this test
+    const testNodeConfig = {
+      ...nodeConfig,
+      id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+      resources: {
+        memory: 256, // 256MB
+        cpu: 0.5    // 0.5 CPU
+      },
+      network: {
+        ...nodeConfig.network,
+        enableHttpApi: true
+      }
+    };
+    
+    let testNode: NodeHandle | null = null;
+    
+    try {
+      // Start the node with resource limits
+      testNode = await orchestrator.startNode(testNodeConfig);
+      debug(`Node started with ID: ${testNode.id}`);
+      
+      // Get container info to verify resource limits
+      const status = await testNode.status() as ExtendedNodeStatus;
+      
+      // Verify container ID is available at the root level
+      if (!status.containerId) {
+        throw new Error('Container ID not available in node status');
+      }
+      
+      // Get the container ID from the node status
+      if (!status.containerId) {
+        throw new Error('Container ID not available in node status');
+      }
+      
+      // Get container info using ContainerManager
+      const container = await (orchestrator as any).containerManager.getContainer(status.containerId);
+      if (!container) {
+        throw new Error('Container not found');
+      }
+      
+      // Get container info
+      const containerInfo = await container.inspect();
+      
+      // Log container info for debugging
+      debug('Container info: %o', {
+        Memory: containerInfo.HostConfig?.Memory,
+        NanoCpus: containerInfo.HostConfig?.NanoCpus,
+        CpuQuota: containerInfo.HostConfig?.CpuQuota,
+        CpuPeriod: containerInfo.HostConfig?.CpuPeriod
+      });
+      
+      // Check memory limit (in bytes)
+      expect(containerInfo.HostConfig?.Memory).toBe(256 * 1024 * 1024);
+      
+      // Check CPU limit (can be set as NanoCpus or CpuQuota/CpuPeriod)
+      const expectedCpuNano = 0.5 * 1e9; // 0.5 CPU in nanoCPUs
+      const actualCpuNano = containerInfo.HostConfig?.NanoCpus;
+      
+      // Some Docker versions use CpuQuota/CpuPeriod instead of NanoCpus
+      if (actualCpuNano === undefined && containerInfo.HostConfig?.CpuQuota && containerInfo.HostConfig?.CpuPeriod) {
+        const cpuQuota = containerInfo.HostConfig.CpuQuota;
+        const cpuPeriod = containerInfo.HostConfig.CpuPeriod;
+        const calculatedCpu = (cpuQuota / cpuPeriod) * 1e9;
+        expect(Math.round(calculatedCpu)).toBeCloseTo(Math.round(expectedCpuNano), -8); // Allow for small rounding differences
+      } else {
+        expect(actualCpuNano).toBe(expectedCpuNano);
+      }
+      
+      debug('Resource limits verified');
+    } finally {
+      // Clean up the test node
+      if (testNode) {
+        try {
+          await orchestrator.stopNode(testNode);
+        } catch (e) {
+          debug('Error cleaning up test node: %o', e);
+        }
+      }
+    }
+  }, 30000);
+
+  it('should expose API endpoints', async () => {
+    // Set a longer timeout for this test (5 minutes)
+    jest.setTimeout(300000);
+    debug('Starting test: should expose API endpoints');
+    
+    // Create a new node with a unique ID for this test
+    const testNodeConfig = {
+      ...nodeConfig,
+      id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+      // Ensure HTTP API is enabled
+      network: {
+        ...nodeConfig.network,
+        enableHttpApi: true
+      }
+    };
+    
+    // Start the node
+    debug('Attempting to start node with config: %o', testNodeConfig);
+    const node = await orchestrator.startNode(testNodeConfig);
+    debug(`Node started with ID: ${node.id}`);
+    
+    const apiUrl = node.getApiUrl?.();
+    // Helper function to test API endpoint with retries
+    const testApiEndpoint = async (endpoint: string, expectedStatus = 200, maxRetries = 5, retryDelay = 1000) => {
+      let lastError: Error | null = null;
+      
+      for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+          debug(`Attempt ${attempt}/${maxRetries} - Testing ${endpoint}`);
+          const controller = new AbortController();
+          const timeout = setTimeout(() => controller.abort(), 5000);
+          const response = await fetch(`${apiUrl}${endpoint}`, {
+            headers: {
+              'Accept': 'application/json',
+              'Content-Type': 'application/json'
+            },
+            signal: controller.signal
+          });
+          clearTimeout(timeout);
+          
+          if (response.status === expectedStatus) {
+            debug(`${endpoint} returned status ${response.status}`);
+            return await response.json().catch(() => ({}));
+          }
+          
+          const errorText = await response.text().catch(() => 'No response body');
+          throw new Error(`Expected status ${expectedStatus}, got ${response.status}: ${errorText}`);
+        } catch (error) {
+          lastError = error as Error;
+          debug(`Attempt ${attempt} failed: %o`, error);
+          
+          if (attempt < maxRetries) {
+            await new Promise(resolve => setTimeout(resolve, retryDelay * attempt));
+          }
+        }
+      }
+      
+      throw new Error(`API endpoint test failed after ${maxRetries} attempts: ${lastError?.message}`);
+    };
+    
+    try {
+      // Test the health endpoint
+      debug('Testing health endpoint...');
+      const healthData = await testApiEndpoint('/health');
+      expect(healthData).toHaveProperty('status');
+      expect(healthData.status).toBe('ok');
+      
+      debug('All API endpoints verified');
+    } catch (error) {
+      // Log container logs if available
+      try {
+        const container = docker.getContainer(`rhizome-${node.id}`);
+        const logs = await container.logs({
+          stdout: true,
+          stderr: true,
+          tail: 100
+        });
+        debug('Container logs: %s', logs.toString('utf8'));
+      } catch (logError) {
+        debug('Failed to get container logs: %o', logError);
+      }
+      
+      throw error;
+    }
+  });
+
+  it.skip('should connect two nodes', async () => {
+    debug('Starting test: should connect two nodes');
+    
+    // Create unique configs for both nodes
+    const node1Port = 3000 + Math.floor(Math.random() * 1000);
+    const node2Port = node1Port + 1;
+    const networkId = `test-network-${Date.now()}`;
+    
+    const node1Config: NodeConfig = {
+      id: `test-node-1-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+      networkId,
+      network: {
+        port: node1Port,
+        requestPort: node1Port + 1000, // Different port for request API
+        bootstrapPeers: []
+      },
+      resources: {
+        memory: 256,
+        cpu: 0.5
+      }
+    };
+    
+    const node2Config: NodeConfig = {
+      id: `test-node-2-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+      networkId,
+      network: {
+        port: node2Port,
+        requestPort: node2Port + 1000, // Different port for request API
+        bootstrapPeers: [`/ip4/127.0.0.1/tcp/${node1Port + 1000}`]
+      },
+      resources: {
+        memory: 256,
+        cpu: 0.5
+      }
+    };
+    
+    let node1: NodeHandle | null = null;
+    let node2: NodeHandle | null = null;
+    
+    try {
+      // Start first node
+      debug('Starting node 1...');
+      node1 = await orchestrator.startNode(node1Config);
+      debug(`Node 1 started with ID: ${node1.id}`);
+      
+      // Get node 1's status and API URL
+      const status1 = await node1.status() as ExtendedNodeStatus;
+      const node1ApiUrl = node1.getApiUrl?.();
+      
+      // Update node 2's config with node 1's actual address if available
+      if (status1.network?.address && node2Config.network) {
+        // This assumes the address is in a format like /ip4/127.0.0.1/tcp/3001
+        node2Config.network.bootstrapPeers = [status1.network.address];
+      }
+      
+      // Start second node
+      debug('Starting node 2...');
+      node2 = await orchestrator.startNode(node2Config);
+      debug(`Node 2 started with ID: ${node2.id}`);
+      
+      // Get node 2's status
+      const status2 = await node2.status() as ExtendedNodeStatus;
+      const node2ApiUrl = node2.getApiUrl?.();
+      
+      // Verify both nodes are running
+      expect(status1).toBeDefined();
+      expect(status2).toBeDefined();
+      // TODO: this status check is inadequate
+      debug('Both nodes are running');
+      
+      // Helper function to wait for peers
+      const waitForPeers = async (nodeHandle: NodeHandle, expectedPeerCount = 1, maxAttempts = 10) => {
+        for (let i = 0; i < maxAttempts; i++) {
+          const status = await nodeHandle.status() as ExtendedNodeStatus;
+          const peerCount = status.network?.peers?.length || 0;
+          
+          if (peerCount >= expectedPeerCount) {
+            debug(`Found ${peerCount} peers after ${i + 1} attempts`);
+            return true;
+          }
+          
+          debug(`Waiting for peers... (attempt ${i + 1}/${maxAttempts})`);
+          await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+        return false;
+      };
+      
+      // Wait for nodes to discover each other
+      debug('Waiting for nodes to discover each other...');
+      const node1Discovered = await waitForPeers(node1);
+      const node2Discovered = await waitForPeers(node2);
+      
+      // Final status check
+      const finalStatus1 = await node1.status() as ExtendedNodeStatus;
+      const finalStatus2 = await node2.status() as ExtendedNodeStatus;
+      
+      // Log peer information
+      debug('Node 1 discovered: %o', node1Discovered);
+      debug('Node 2 discovered: %o', node2Discovered);
+      debug('Node 1 peers: %o', finalStatus1.network?.peers || 'none');
+      debug('Node 2 peers: %o', finalStatus2.network?.peers || 'none');
+      debug('Node 1 bootstrapPeers: %o', finalStatus1.network?.bootstrapPeers || 'none');
+      debug('Node 2 bootstrapPeers: %o', finalStatus2.network?.bootstrapPeers || 'none');
+      
+      // Log the addresses for debugging
+      debug('Node 1 address: %o', finalStatus1.network?.address);
+      debug('Node 2 address: %o', finalStatus2.network?.address);
+      
+      // Verify both nodes have network configuration
+      expect(finalStatus1.network).toBeDefined();
+      expect(finalStatus2.network).toBeDefined();
+      expect(finalStatus1.network?.address).toBeDefined();
+      expect(finalStatus2.network?.address).toBeDefined();
+      
+      // For now, we'll just verify that both nodes are running and have network info
+      // In a real test, you would want to verify actual communication between nodes
+      debug('✅ Both nodes are running with network configuration');
+      
+    } finally {
+      // Clean up nodes
+      const cleanupPromises = [];
+      
+      if (node1) {
+        debug('Stopping node 1...');
+        cleanupPromises.push(
+          orchestrator.stopNode(node1).catch(e => 
+            debug('Error stopping node 1: %o', e)
+          )
+        );
+      }
+      
+      if (node2) {
+        debug('Stopping node 2...');
+        cleanupPromises.push(
+          orchestrator.stopNode(node2).catch(e => 
+            debug('Error stopping node 2: %o', e)
+          )
+        );
+      }
+      
+      await Promise.all(cleanupPromises);
+      debug('✅ Both nodes stopped');
+    }
+    
+    // Note: In a real test with actual peer connections, we would verify the connection
+    // by having the nodes communicate with each other.
+  }, 60000);
+});

__tests__/schema.ts

@@ -7,7 +7,7 @@ import {
   ObjectSchema
 } from '../src/schema';
 import { DefaultSchemaRegistry } from '../src/schema';
-import { CommonSchemas } from '../src/test-utils/schemas';
+import { CommonSchemas } from '../util/schemas';
 import { TypedCollectionImpl, SchemaValidationError } from '../src/collections';
 import { RhizomeNode } from '../src/node';
 import { Delta } from '../src/core';

__tests__/test-utils.ts

@@ -0,0 +1,98 @@
+import { createOrchestrator } from '../src/orchestration/factory';
+import { NodeConfig, NodeOrchestrator } from '../src/orchestration/types';
+import Debug from 'debug';
+
+const debug = Debug('rz:test-utils');
+
+// Global test orchestrator instance
+let testOrchestrator: NodeOrchestrator;
+
+// Default test node configuration
+const DEFAULT_TEST_NODE_CONFIG: Partial<NodeConfig> = {
+  network: {
+    // Use default ports that will be overridden by getRandomPort() in the orchestrator
+    port: 0,
+  },
+  storage: {
+    type: 'memory',
+    path: '/data',
+  },
+};
+
+/**
+ * Set up the test environment before all tests run
+ */
+export const setupTestEnvironment = async () => {
+  debug('Setting up Docker test environment...');
+  
+  try {
+    // Create a Docker orchestrator instance
+    testOrchestrator = createOrchestrator('docker', {
+      // Enable auto-building of test images
+      autoBuildTestImage: true,
+      // Use a specific test image name
+      image: 'rhizome-node-test',
+    });
+    
+    debug('Docker test environment setup complete');
+  } catch (error) {
+    debug('Error setting up Docker test environment:', error);
+    throw error;
+  }
+};
+
+/**
+ * Clean up the test environment after all tests complete
+ */
+export const teardownTestEnvironment = async () => {
+  debug('Tearing down Docker test environment...');
+  
+  if (testOrchestrator) {
+    try {
+      // Clean up all containers and networks
+      await testOrchestrator.cleanup();
+      debug('Docker resources cleaned up successfully');
+    } catch (error) {
+      debug('Error during Docker environment teardown:', error);
+      // Don't throw to allow tests to complete
+    }
+  }
+  
+  debug('Docker test environment teardown complete');
+};
+
+/**
+ * Get the test orchestrator instance
+ */
+export const getTestOrchestrator = (): NodeOrchestrator => {
+  if (!testOrchestrator) {
+    throw new Error('Test orchestrator not initialized. Call setupTestEnvironment() first.');
+  }
+  return testOrchestrator;
+};
+
+/**
+ * Create a test node with the given configuration
+ */
+export const createTestNode = async (config: Partial<NodeConfig> = {}) => {
+  const orchestrator = getTestOrchestrator();
+  
+  // Merge default config with provided config
+  const nodeConfig: NodeConfig = {
+    ...DEFAULT_TEST_NODE_CONFIG,
+    ...config,
+    // Ensure we have a unique ID for each node
+    id: config.id || `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+  };
+  
+  debug(`Creating test node with ID: ${nodeConfig.id}`);
+  
+  try {
+    const nodeHandle = await orchestrator.startNode(nodeConfig);
+    debug(`Test node ${nodeConfig.id} created successfully`);
+    return nodeHandle;
+  } catch (error) {
+    debug(`Error creating test node ${nodeConfig.id}:`, error);
+    throw error;
+  }
+};

logs/docker-build.log

@@ -0,0 +1,105 @@
+[2025-06-18T01:06:06.659Z] ✅ Docker build started, streaming output...
+[2025-06-18T01:06:06.660Z] [Docker Build] Step 1/11 : FROM node:24
+[2025-06-18T01:06:06.660Z] [Docker Build]
+[2025-06-18T01:06:06.660Z] [Docker Build]  ---> 755ea2a01757
+[2025-06-18T01:06:06.660Z] [Docker Build] Step 2/11 : WORKDIR /app
+[2025-06-18T01:06:06.660Z] [Docker Build]
+[2025-06-18T01:06:06.661Z] [Docker Build]  ---> Using cache
+[2025-06-18T01:06:06.661Z] [Docker Build]  ---> a471eaba1647
+[2025-06-18T01:06:06.661Z] [Docker Build] Step 3/11 : COPY package.json package-lock.json tsconfig.json ./
+[2025-06-18T01:06:06.661Z] [Docker Build]
+[2025-06-18T01:06:06.833Z] [Docker Build]  ---> 7c047af2d840
+[2025-06-18T01:06:06.834Z] [Docker Build] Step 4/11 : RUN npm ci --include=dev
+[2025-06-18T01:06:06.834Z] [Docker Build]
+[2025-06-18T01:06:06.934Z] [Docker Build]  ---> Running in 49af7c037197
+[2025-06-18T01:06:10.455Z] [Docker Build] npm warn deprecated rimraf@3.0.2: Rimraf versions prior to v4 are no longer supported
+
+[2025-06-18T01:06:10.734Z] [Docker Build] npm warn deprecated npmlog@6.0.2: This package is no longer supported.
+
+[2025-06-18T01:06:11.395Z] [Docker Build] npm warn deprecated inflight@1.0.6: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.
+
+[2025-06-18T01:06:11.461Z] [Docker Build] npm warn deprecated glob@7.2.3: Glob versions prior to v9 are no longer supported
+
+[2025-06-18T01:06:11.524Z] [Docker Build] npm warn deprecated gauge@4.0.4: This package is no longer supported.
+
+[2025-06-18T01:06:12.230Z] [Docker Build] npm warn deprecated are-we-there-yet@3.0.1: This package is no longer supported.
+
+[2025-06-18T01:06:13.207Z] [Docker Build] npm warn deprecated @humanwhocodes/object-schema@2.0.3: Use @eslint/object-schema instead
+
+[2025-06-18T01:06:13.251Z] [Docker Build] npm warn deprecated @humanwhocodes/config-array@0.13.0: Use @eslint/config-array instead
+
+[2025-06-18T01:06:14.440Z] [Docker Build] npm warn deprecated eslint@8.57.1: This version is no longer supported. Please see https://eslint.org/version-support for other options.
+
+[2025-06-18T01:06:19.569Z] [Docker Build] 
+added 839 packages, and audited 841 packages in 12s
+[2025-06-18T01:06:19.569Z] [Docker Build] 175 packages are looking for funding
+  run `npm fund` for details
+[2025-06-18T01:06:19.571Z] [Docker Build] 
+found 0 vulnerabilities
+[2025-06-18T01:06:19.572Z] [Docker Build] npm notice
+npm notice New minor version of npm available! 11.3.0 -> 11.4.2
+npm notice Changelog: https://github.com/npm/cli/releases/tag/v11.4.2
+npm notice To update run: npm install -g npm@11.4.2
+npm notice
+
+[2025-06-18T01:06:31.247Z] [Docker Build]  ---> Removed intermediate container 49af7c037197
+[2025-06-18T01:06:31.247Z] [Docker Build]  ---> 3db27fed8161
+[2025-06-18T01:06:31.247Z] [Docker Build] Step 5/11 : COPY src/ src/
+[2025-06-18T01:06:31.247Z] [Docker Build]
+[2025-06-18T01:06:31.598Z] [Docker Build]  ---> 1ad51b320392
+[2025-06-18T01:06:31.598Z] [Docker Build] Step 6/11 : COPY markdown/ markdown/
+[2025-06-18T01:06:31.598Z] [Docker Build]
+[2025-06-18T01:06:31.736Z] [Docker Build]  ---> c52bad2721f7
+[2025-06-18T01:06:31.736Z] [Docker Build] Step 7/11 : COPY examples/ examples/
+[2025-06-18T01:06:31.736Z] [Docker Build]
+[2025-06-18T01:06:31.864Z] [Docker Build]  ---> 5a98881e54fb
+[2025-06-18T01:06:31.865Z] [Docker Build] Step 8/11 : COPY util/ util/
+[2025-06-18T01:06:31.865Z] [Docker Build]
+[2025-06-18T01:06:31.986Z] [Docker Build]  ---> 862b5fe2ca61
+[2025-06-18T01:06:31.986Z] [Docker Build] Step 9/11 : RUN npm run build --verbose
+[2025-06-18T01:06:31.986Z] [Docker Build]
+[2025-06-18T01:06:32.085Z] [Docker Build]  ---> Running in 386a95b55921
+[2025-06-18T01:06:32.475Z] [Docker Build] npm verbose cli /usr/local/bin/node /usr/local/bin/npm
+
+[2025-06-18T01:06:32.476Z] [Docker Build] npm info using npm@11.3.0
+
+[2025-06-18T01:06:32.476Z] [Docker Build] npm info using node@v24.2.0
+
+[2025-06-18T01:06:32.478Z] [Docker Build] npm verbose title npm run build
+npm verbose argv "run" "build" "--loglevel" "verbose"
+
+[2025-06-18T01:06:32.478Z] [Docker Build] npm verbose logfile logs-max:10 dir:/root/.npm/_logs/2025-06-18T01_06_32_444Z-
+
+[2025-06-18T01:06:32.502Z] [Docker Build] npm verbose logfile /root/.npm/_logs/2025-06-18T01_06_32_444Z-debug-0.log
+
+[2025-06-18T01:06:32.528Z] [Docker Build] 
+> rhizome-node@0.1.0 build
+> tsc
+[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose cwd /app
+
+[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose os Linux 6.8.0-60-generic
+
+[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose node v24.2.0
+
+[2025-06-18T01:06:35.285Z] [Docker Build] npm verbose npm  v11.3.0
+
+[2025-06-18T01:06:35.286Z] [Docker Build] npm verbose exit 0
+
+[2025-06-18T01:06:35.286Z] [Docker Build] npm info ok
+
+[2025-06-18T01:06:35.874Z] [Docker Build]  ---> Removed intermediate container 386a95b55921
+[2025-06-18T01:06:35.874Z] [Docker Build]  ---> 694f414f6cdb
+[2025-06-18T01:06:35.874Z] [Docker Build] Step 10/11 : ENV NODE_ENV=test
+[2025-06-18T01:06:35.874Z] [Docker Build]
+[2025-06-18T01:06:36.003Z] [Docker Build]  ---> Running in facd3d3ab07a
+[2025-06-18T01:06:36.124Z] [Docker Build]  ---> Removed intermediate container facd3d3ab07a
+[2025-06-18T01:06:36.124Z] [Docker Build]  ---> 3eb20e31ad6a
+[2025-06-18T01:06:36.124Z] [Docker Build] Step 11/11 : CMD ["node", "dist/examples/app.js"]
+[2025-06-18T01:06:36.124Z] [Docker Build]
+[2025-06-18T01:06:36.225Z] [Docker Build]  ---> Running in 3c6e1a89fadb
+[2025-06-18T01:06:36.329Z] [Docker Build]  ---> Removed intermediate container 3c6e1a89fadb
+[2025-06-18T01:06:36.329Z] [Docker Build]  ---> 66da6b5995cc
+[2025-06-18T01:06:36.329Z] [Docker Build] {"aux":{"ID":"sha256:66da6b5995cc50e0463df668b8820b56b6e384a7c91dfaca010ff8c3761b1146"}}
+[2025-06-18T01:06:36.331Z] [Docker Build] Successfully built 66da6b5995cc
+[2025-06-18T01:06:36.350Z] [Docker Build] Successfully tagged rhizome-node-test:latest
+[2025-06-18T01:06:36.350Z] ✅ Docker build completed successfully

markdown/007-investigation.md

@@ -0,0 +1,5 @@
+Network Layers:
+- Gossip protocols
+- Pub/sub
+- RPC
+

package.json

@@ -2,30 +2,49 @@
   "name": "rhizome-node",
   "version": "0.1.0",
   "description": "Rhizomatic database engine node",
-  "type": "module",
   "scripts": {
     "build": "tsc",
     "build:watch": "tsc --watch",
     "lint": "eslint",
-    "test": "node --experimental-vm-modules node_modules/.bin/jest",
+    "test": "jest",
     "coverage": "./scripts/coverage.sh",
     "coverage-report": "npm run test -- --coverage --coverageDirectory=coverage",
     "example-app": "node dist/examples/app.js"
   },
   "jest": {
     "testEnvironment": "node",
-    "preset": "ts-jest",
+    "preset": "ts-jest/presets/default",
     "roots": [
       "./__tests__/"
     ],
     "testMatch": [
       "**/__tests__/**/*"
-    ]
+    ],
+    "setupFilesAfterEnv": [
+      "<rootDir>/__tests__/jest-setup.ts"
+    ],
+    "transform": {
+      "^\\.tsx?$": [
+        "ts-jest",
+        {
+          "tsconfig": "tsconfig.json"
+        }
+      ]
+    },
+    "detectOpenHandles": true,
+    "forceExit": true,
+    "verbose": true,
+    "testEnvironmentOptions": {
+      "NODE_ENV": "test",
+      "DEBUG": "rz:*"
+    }
   },
   "author": "Taliesin (Ladd) <ladd@dgov.io>",
   "license": "Unlicense",
   "dependencies": {
+    "@types/dockerode": "^3.3.40",
     "debug": "^4.4.0",
+    "dockerode": "^4.0.7",
     "express": "^4.21.2",
     "json-logic-js": "^2.0.5",
     "level": "^9.0.0",
@@ -34,6 +53,7 @@
     "object-hash": "^3.0.0",
     "showdown": "^2.1.0",
     "util": "./util/",
+    "uuid": "^9.0.0",
     "zeromq": "^6.1.2"
   },
   "devDependencies": {
@@ -44,13 +64,18 @@
     "@types/jest": "^29.5.14",
     "@types/json-logic-js": "^2.0.8",
     "@types/microtime": "^2.1.2",
-    "@types/node": "^22.10.2",
+    "@types/node": "^22.15.31",
+    "@types/node-fetch": "^2.6.12",
     "@types/object-hash": "^3.0.6",
     "@types/showdown": "^2.0.6",
+    "@types/tar-fs": "^2.0.4",
+    "@types/uuid": "^10.0.0",
     "eslint": "^9.17.0",
     "eslint-config-airbnb-base-typescript": "^1.1.0",
     "jest": "^29.7.0",
+    "node-fetch": "^2.7.0",
     "ts-jest": "^29.2.5",
+    "ts-node": "^10.9.2",
     "tsc-alias": "^1.8.10",
     "typescript": "^5.7.2",
     "typescript-eslint": "^8.18.0"

scratch/jsonlogic.ts

@@ -1,12 +1,20 @@
-import { apply } from 'json-logic-js';
+import Debug from 'debug';
+import jsonLogic from 'json-logic-js';
 
-console.log(apply({"map":[
+const debug = Debug('rz:scratch:jsonlogic');
+const { apply } = jsonLogic;
+
+// Example of using jsonLogic's map operation
+const mapResult = apply({"map":[
   {"var":"integers"},
   {"*":[{"var":""},2]}
-]}, {"integers":[1,2,3,4,5]}));
+]}, {"integers":[1,2,3,4,5]});
+debug('Map result: %o', mapResult);
 
-console.log(apply({"reduce":[
+// Example of using jsonLogic's reduce operation
+const reduceResult = apply({"reduce":[
     {"var":"integers"},
     {"+":[{"var":"current"}, {"var":"accumulator"}]},
     0
-]}, {"integers":[1,2,3,4,5]}));
+]}, {"integers":[1,2,3,4,5]});
+debug('Reduce result: %o', reduceResult);

src/http/api.ts

@@ -1,8 +1,11 @@
+import Debug from 'debug';
 import express, {Router} from "express";
 import {Collection} from "../collections";
 import {Delta, DeltaFilter} from "../core";
 import {RhizomeNode} from "../node";
 
+const debug = Debug('rz:http:api');
+
 export class HttpApi {
   router = Router();
 
@@ -11,6 +14,14 @@ export class HttpApi {
   }
 
   private setupRoutes() {
+    // --------------- health ----------------
+
+    this.router.get("/health", (_req: express.Request, res: express.Response) => {
+      res.json({
+        status: "ok"
+      });
+    });
+
     // --------------- deltas ----------------
 
     // Serve list of all deltas accepted
@@ -161,7 +172,7 @@ export class HttpApi {
         if (maxResults) options.maxResults = maxResults;
         if (deltaFilter) {
           // Note: deltaFilter would need to be serialized/deserialized properly in a real implementation
-          console.warn('deltaFilter not supported in HTTP API yet');
+          debug('deltaFilter not supported in HTTP API yet');
         }
 
         const result = await this.rhizomeNode.queryEngine.query(schemaId, filter, options);

src/http/index.ts

@@ -21,15 +21,81 @@ export class HttpServer {
     this.app.use('/api', this.httpApi.router);
   }
 
+  /**
+   * Start the HTTP server
+   */
   start() {
     const {httpAddr, httpPort} = this.rhizomeNode.config;
-    this.httpHtml.start();
+    debug(`[${this.rhizomeNode.config.peerId}]`, `Starting HTTP server on ${httpAddr}:${httpPort}...`);
-    this.server = this.app.listen({
+    
-      port: httpPort,
+    try {
-      host: httpAddr,
+      this.httpHtml.start();
-      exclusive: true
+      
-    }, () => {
+      // Create the server
-      debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP API bound to ${httpAddr}:${httpPort}`);
+      this.server = this.app.listen({
+        port: httpPort,
+        host: httpAddr,
+        exclusive: true
+      });
+      
+      // Add error handler
+      this.server.on('error', (error) => {
+        debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP server error:`, error);
+      });
+      
+      // Add callback for logging
+      this.server.on('listening', () => {
+        const address = this.server?.address();
+        const actualPort = typeof address === 'string' ? httpPort : address?.port;
+        debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP server bound to ${httpAddr}:${actualPort}`);
+      });
+      
+      debug(`[${this.rhizomeNode.config.peerId}]`, 'HTTP server start initiated');
+    } catch (error) {
+      debug(`[${this.rhizomeNode.config.peerId}]`, 'Error starting HTTP server:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Start the HTTP server and return a promise that resolves when the server is listening
+   */
+  async startAndWait(): Promise<void> {
+    // If server is already listening, resolve immediately
+    if (this.server?.listening) {
+      return Promise.resolve();
+    }
+
+    return new Promise<void>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        cleanup();
+        reject(new Error(`HTTP server failed to start within 10 seconds`));
+      }, 10000);
+
+      const onListening = () => {
+        cleanup();
+        resolve();
+      };
+
+      const onError = (error: Error) => {
+        cleanup();
+        reject(error);
+      };
+
+      const cleanup = () => {
+        clearTimeout(timeout);
+        this.server?.off('listening', onListening);
+        this.server?.off('error', onError);
+      };
+
+      // Start the server if not already started
+      if (!this.server) {
+        this.start();
+      }
+
+      // Add event listeners
+      this.server?.on('listening', onListening);
+      this.server?.on('error', onError);
     });
   }
 

src/network/pub-sub.ts

@@ -8,7 +8,8 @@ export type SubscribedMessageHandler = (sender: PeerAddress, msg: string) => voi
 
 // TODO: Allow subscribing to multiple topics on one socket
 export class Subscription {
-  sock = new Subscriber();
+  private sock: Subscriber;
+  private isRunning = false;
   topic: string;
   publishAddr: PeerAddress;
   publishAddrStr: string;
@@ -20,6 +21,7 @@ export class Subscription {
     topic: string,
     cb: SubscribedMessageHandler,
   ) {
+    this.sock = new Subscriber();
     this.cb = cb;
     this.topic = topic;
     this.publishAddr = publishAddr;
@@ -27,20 +29,60 @@ export class Subscription {
   }
 
   async start() {
+    if (this.isRunning) return;
+    this.isRunning = true;
+    
     this.sock.connect(this.publishAddrStr);
     this.sock.subscribe(this.topic);
     debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Subscribing to ${this.topic} topic on ZeroMQ ${this.publishAddrStr}`);
 
-    // Wait for ZeroMQ messages.
+    // Set up message handler
-    // This will block indefinitely.
+    const processMessage = async () => {
-    for await (const [, sender, msg] of this.sock) {
+      try {
-      const senderStr = PeerAddress.fromString(sender.toString());
+        if (!this.isRunning) return;
-      const msgStr = msg.toString();
-      debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `ZeroMQ subscribtion received msg: ${msgStr}`);
-      this.cb(senderStr, msgStr);
-    }
         
-    debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Done waiting for subscription socket for topic ${this.topic}`);
+        // Use a promise race to handle both messages and the stop signal
+        const [topic, sender, msg] = await Promise.race([
+          this.sock.receive(),
+          new Promise<[Buffer, Buffer, Buffer]>(() => {}).then(() => { 
+            if (!this.isRunning) throw new Error('Subscription stopped'); 
+            return [Buffer.alloc(0), Buffer.alloc(0), Buffer.alloc(0)]; 
+          })
+        ]);
+        
+        if (!this.isRunning) return;
+        
+        const senderStr = PeerAddress.fromString(sender.toString());
+        const msgStr = msg.toString();
+        debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `ZeroMQ subscription received msg: ${msgStr}`);
+        this.cb(senderStr, msgStr);
+        
+        // Process next message
+        process.nextTick(processMessage);
+      } catch (error) {
+        if (this.isRunning) {
+          debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Error in subscription:`, error);
+          // Attempt to restart the message processing
+          if (this.isRunning) {
+            process.nextTick(processMessage);
+          }
+        }
+      }
+    };
+    
+    // Start processing messages
+    process.nextTick(processMessage);
+  }
+  
+  close() {
+    if (!this.isRunning) return;
+    this.isRunning = false;
+    try {
+      this.sock.close();
+      debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Closed subscription for topic ${this.topic}`);
+    } catch (error) {
+      debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Error closing subscription:`, error);
+    }
   }
 }
 
@@ -53,8 +95,8 @@ export class PubSub {
   constructor(rhizomeNode: RhizomeNode) {
     this.rhizomeNode = rhizomeNode;
 
-    const {publishBindAddr, publishBindPort} = this.rhizomeNode.config;
+    const {publishBindHost, publishBindPort} = this.rhizomeNode.config;
-    this.publishAddrStr = `tcp://${publishBindAddr}:${publishBindPort}`;
+    this.publishAddrStr = `tcp://${publishBindHost}:${publishBindPort}`;
   }
 
   async startZmq() {
@@ -85,16 +127,33 @@ export class PubSub {
     return subscription;
   }
 
-  async stop() {
+  /**
-    if (this.publishSock) {
+   * Check if the PubSub is running
-      await this.publishSock.unbind(this.publishAddrStr);
+   * @returns boolean indicating if the publisher socket is active
-      this.publishSock.close();
+   */
-      // Free the memory by taking the old object out of scope.
+  isRunning(): boolean {
-      this.publishSock = undefined;
+    return !!this.publishSock;
-    }
+  }
 
+  async stop() {
+    // First close all subscriptions
     for (const subscription of this.subscriptions) {
-      subscription.sock.close();
+      subscription.close();
+    }
+    this.subscriptions = [];
+    
+    // Then close the publisher socket
+    if (this.publishSock) {
+      try {
+        await this.publishSock.unbind(this.publishAddrStr);
+        this.publishSock.close();
+        debug(`[${this.rhizomeNode.config.peerId}]`, 'Unbound and closed publisher socket');
+      } catch (error) {
+        debug(`[${this.rhizomeNode.config.peerId}]`, 'Error closing publisher socket:', error);
+      } finally {
+        // Free the memory by taking the old object out of scope.
+        this.publishSock = undefined;
+      }
     }
   }
 }

src/network/request-reply.ts

@@ -74,8 +74,8 @@ export class RequestReply {
 
   constructor(rhizomeNode: RhizomeNode) {
     this.rhizomeNode = rhizomeNode;
-    const {requestBindAddr, requestBindPort} = this.rhizomeNode.config;
+    const {requestBindHost, requestBindPort} = this.rhizomeNode.config;
-    this.requestBindAddrStr = `tcp://${requestBindAddr}:${requestBindPort}`;
+    this.requestBindAddrStr = `tcp://${requestBindHost}:${requestBindPort}`;
   }
 
   // Listen for incoming requests

src/node.ts

@@ -9,10 +9,8 @@ import {DeltaQueryStorage, StorageFactory, StorageConfig} from './storage';
 const debug = Debug('rz:rhizome-node');
 
 export type RhizomeNodeConfig = {
-  requestBindAddr: string;
   requestBindHost: string;
   requestBindPort: number;
-  publishBindAddr: string;
   publishBindHost: string;
   publishBindPort: number;
   httpAddr: string;
@@ -42,10 +40,8 @@ export class RhizomeNode {
 
   constructor(config?: Partial<RhizomeNodeConfig>) {
     this.config = {
-      requestBindAddr: REQUEST_BIND_ADDR,
       requestBindHost: REQUEST_BIND_HOST,
       requestBindPort: REQUEST_BIND_PORT,
-      publishBindAddr: PUBLISH_BIND_ADDR,
       publishBindHost: PUBLISH_BIND_HOST,
       publishBindPort: PUBLISH_BIND_PORT,
       httpAddr: HTTP_API_ADDR,
@@ -85,7 +81,16 @@ export class RhizomeNode {
     this.storageQueryEngine = new StorageQueryEngine(this.deltaStorage, this.schemaRegistry);
   }
 
-  async start(syncOnStart = false) {
+  /**
+   * Start the node components
+   * @param options.syncOnStart Whether to sync with peers on startup (default: false)
+   * @returns Promise that resolves when the node is fully started and ready
+   */
+  async start({
+    syncOnStart = false
+  }: { syncOnStart?: boolean } = {}): Promise<void> {
+    debug(`[${this.config.peerId}]`, 'Starting node (waiting for ready)...');
+    
     // Connect our lossless view to the delta stream
     this.deltaStream.subscribeDeltas(async (delta) => {
       // Ingest into lossless view
@@ -100,44 +105,38 @@ export class RhizomeNode {
     });
 
     // Bind ZeroMQ publish socket
-    // TODO: Config option to enable zmq pubsub
     await this.pubSub.startZmq();
     
     // Bind ZeroMQ request socket
-    // TODO: request/reply via libp2p?
-    // TODO: config options to enable request/reply, or configure available commands
     this.requestReply.start();
     
-    // Start HTTP server
+    // Start HTTP server if enabled
-    if (this.config.httpEnable) {
+    if (this.config.httpEnable && this.httpServer) {
-      this.httpServer.start();
+      await this.httpServer.startAndWait();
     }
 
-    {
+    // Initialize network components
-      // Wait a short time for sockets to initialize
+    await new Promise((resolve) => setTimeout(resolve, 500));
-      await new Promise((resolve) => setTimeout(resolve, 500));
+    this.peers.subscribeToSeeds();
-
-      // Subscribe to seed peers
-      this.peers.subscribeToSeeds();
-
-      // Wait a short time for sockets to initialize
-      // await new Promise((resolve) => setTimeout(resolve, 500));
-    }
 
     if (syncOnStart) {
       // Ask all peers for all deltas
       this.peers.askAllPeersForDeltas();
-
-      // Wait to receive all deltas
       await new Promise((resolve) => setTimeout(resolve, 1000));
     }
+    
+    debug(`[${this.config.peerId}]`, 'Node started and ready');
   }
 
   async stop() {
     this.peers.stop();
     await this.pubSub.stop();
     await this.requestReply.stop();
-    await this.httpServer.stop();
+    
+    // Stop the HTTP server if it was started
+    if (this.config.httpEnable && this.httpServer) {
+      await this.httpServer.stop();
+    }
     
     // Close storage
     try {

src/orchestration/base-orchestrator.ts

@@ -0,0 +1,68 @@
+import { NodeOrchestrator, NodeHandle, NodeConfig, NodeStatus } from './types';
+
+/**
+ * Base class for all orchestrator implementations
+ * Provides common functionality and ensures interface compliance
+ */
+export abstract class BaseOrchestrator implements NodeOrchestrator {
+  /**
+   * Start a new node with the given configuration
+   * Must be implemented by subclasses
+   */
+  abstract startNode(config: NodeConfig): Promise<NodeHandle>;
+
+  /**
+   * Stop a running node
+   * Must be implemented by subclasses
+   */
+  abstract stopNode(handle: NodeHandle): Promise<void>;
+
+  /**
+   * Get status of a node
+   * Must be implemented by subclasses
+   */
+  abstract getNodeStatus(handle: NodeHandle): Promise<NodeStatus>;
+
+  /**
+   * Connect two nodes
+   * Default implementation does nothing - should be overridden by subclasses
+   * that support direct node connections
+   */
+  async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise<void> {
+    // Default implementation does nothing
+    console.warn('connectNodes not implemented for this orchestrator');
+  }
+
+  /**
+   * Create network partitions
+   * Default implementation does nothing - should be overridden by subclasses
+   * that support network partitioning
+   */
+  async partitionNetwork(partitions: { groups: string[][] }): Promise<void> {
+    // Default implementation does nothing
+    console.warn('partitionNetwork not implemented for this orchestrator');
+  }
+
+  /**
+   * Set resource limits for a node
+   * Default implementation does nothing - should be overridden by subclasses
+   * that support resource management
+   */
+  async setResourceLimits(
+    handle: NodeHandle,
+    limits: Partial<NodeConfig['resources']>
+  ): Promise<void> {
+    // Default implementation does nothing
+    console.warn('setResourceLimits not implemented for this orchestrator');
+  }
+
+  /**
+   * Clean up all resources
+   * Default implementation does nothing - should be overridden by subclasses
+   * that need to clean up resources
+   */
+  async cleanup(): Promise<void> {
+    // Default implementation does nothing
+    console.warn('cleanup not implemented for this orchestrator');
+  }
+}

src/orchestration/docker-orchestrator/index.ts

@@ -0,0 +1,462 @@
+import { Container, Network } from 'dockerode';
+import { BaseOrchestrator } from '../base-orchestrator';
+import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types';
+import { DockerNodeHandle, DockerOrchestratorOptions } from './types';
+import { ContainerManager } from './managers/container-manager';
+import { NetworkManager } from './managers/network-manager';
+import { ResourceManager } from './managers/resource-manager';
+import { StatusManager } from './managers/status-manager';
+import { ImageManager } from './managers/image-manager';
+import { getRandomPort } from './utils/port-utils';
+import Debug from 'debug';
+
+const debug = Debug('rz:docker:orchestrator');
+
+const DEFAULT_OPTIONS: DockerOrchestratorOptions = {
+  image: 'rhizome-node-test',
+  containerWorkDir: '/app',
+  autoBuildTestImage: true,
+};
+
+export class DockerOrchestrator extends BaseOrchestrator {
+  private options: DockerOrchestratorOptions;
+  private containers: Map<string, Container> = new Map();
+  private networks: Map<string, Network> = new Map();
+  private containerLogStreams: Map<string, NodeJS.ReadableStream> = new Map();
+  private nodeHandles: Map<string, DockerNodeHandle> = new Map();
+  
+  // Managers
+  private readonly containerManager: ContainerManager;
+  private readonly networkManager: NetworkManager;
+  private readonly resourceManager: ResourceManager;
+  private readonly statusManager: StatusManager;
+  private readonly imageManager: ImageManager;
+
+  constructor(options: Partial<DockerOrchestratorOptions> = {}) {
+    super();
+    this.options = { ...DEFAULT_OPTIONS, ...options };
+    
+    // Initialize managers
+    this.containerManager = new ContainerManager();
+    this.networkManager = new NetworkManager();
+    this.resourceManager = new ResourceManager();
+    this.statusManager = new StatusManager();
+    this.imageManager = new ImageManager();
+  }
+
+  /**
+   * Start a new node with the given configuration
+   */
+  async startNode(config: NodeConfig): Promise<NodeHandle> {
+    const nodeId = config.id || `node-${Date.now()}`;
+    config.network = config.network || {};
+    config.network.port = config.network.port || getRandomPort();
+    config.network.requestPort = config.network.requestPort || getRandomPort();
+
+    try {
+      // Ensure test image is built
+      if (this.options.autoBuildTestImage) {
+        await this.imageManager.buildTestImage(this.options.image);
+      }
+      
+      // Create a network for this node using NetworkManager
+      const network = await this.networkManager.createNetwork(nodeId);
+      this.networks.set(nodeId, network);
+      
+      // Create container using ContainerManager
+      const container = await this.containerManager.createContainer(
+        nodeId,
+        config,
+        network.id
+      );
+
+      // Store container reference before starting it
+      this.containers.set(nodeId, container);
+
+      // Start the container
+      await this.containerManager.startContainer(container);
+
+      // Create node handle
+      const handle: DockerNodeHandle = {
+        id: nodeId,
+        containerId: container.id,
+        networkId: network.id,
+        config,
+        status: () => this.getNodeStatus({ id: nodeId } as NodeHandle),
+        stop: () => this.stopNode({ id: nodeId } as NodeHandle),
+        getRequestPort: () => config.network?.requestPort,
+        getApiUrl: () => `http://localhost:${config.network?.port}/api`,
+      };
+
+      // Store handle
+      this.nodeHandles.set(nodeId, handle);
+
+      // Wait for node to be ready using StatusManager
+      await this.statusManager.waitForNodeReady( container, config.network.port);
+
+      return handle;
+    } catch (error) {
+      await this.cleanupFailedStart(nodeId);
+      throw error;
+    }
+  }
+
+  /**
+   * Stop a running node
+   */
+  async stopNode(handle: NodeHandle): Promise<void> {
+    const nodeId = handle.id;
+    const container = this.containers.get(nodeId);
+    
+    if (!container) {
+      throw new Error(`No container found for node ${nodeId}`);
+    }
+
+    try {
+      // Stop and remove the container using ContainerManager
+      try {
+        await this.containerManager.stopContainer(container);
+        await this.containerManager.removeContainer(container);
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+        debug(`Error managing container ${nodeId}: %s`, errorMessage);
+        // Continue with cleanup even if container operations fail
+      }
+      
+      // Clean up network using NetworkManager
+      const network = this.networks.get(nodeId);
+      if (network) {
+        try {
+          await this.networkManager.removeNetwork(network.id);
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+          debug(`Error removing network for node ${nodeId}: %s`, errorMessage);
+        } finally {
+          this.networks.delete(nodeId);
+        }
+      }
+      
+      // Clean up log stream
+      this.cleanupLogStream(nodeId);
+      
+      // Remove from internal maps
+      this.containers.delete(nodeId);
+      this.nodeHandles.delete(nodeId);
+      
+      debug(`Stopped and cleaned up node ${nodeId}`);
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+      debug(`Error during cleanup of node ${nodeId}: %s`, errorMessage);
+      throw new Error(`Failed to stop node ${nodeId}: ${errorMessage}`);
+    }
+  }
+
+  /**
+   * Clean up log stream for a node
+   * @private
+   */
+  private cleanupLogStream(nodeId: string): void {
+    const logStream = this.containerLogStreams.get(nodeId);
+    if (!logStream) return;
+    
+    try {
+      if ('destroy' in logStream) {
+        (logStream as { destroy: () => void }).destroy();
+      } else if ('end' in logStream) {
+        (logStream as { end: () => void }).end();
+      }
+    } catch (error) {
+      debug(`Error cleaning up log stream for node ${nodeId}: %o`, error);
+    } finally {
+      this.containerLogStreams.delete(nodeId);
+    }
+  }
+
+  /**
+   * Get status of a node
+   */
+  async getNodeStatus(handle: NodeHandle): Promise<NodeStatus> {
+    const container = this.containers.get(handle.id);
+    
+    // If container not found, return stopped status
+    if (!container) {
+      return {
+        id: handle.id,
+        status: 'stopped',
+        error: 'Container not found',
+        network: {
+          address: '',
+          httpPort: 0,
+          requestPort: 0,
+          peers: []
+        },
+        resources: {
+          cpu: { usage: 0, limit: 0 },
+          memory: { usage: 0, limit: 0 }
+        }
+      };
+    }
+
+    try {
+      // Delegate to StatusManager to get the node status
+      return await this.statusManager.getNodeStatus(handle, container);
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+      debug(`Error getting status for node ${handle.id}: %s`, errorMessage);
+      
+      return {
+        id: handle.id,
+        status: 'error',
+        error: errorMessage,
+        network: {
+          address: '',
+          httpPort: 0,
+          requestPort: 0,
+          peers: []
+        },
+        resources: {
+          cpu: { usage: 0, limit: 0 },
+          memory: { usage: 0, limit: 0 }
+        }
+      };
+    }
+  }
+
+  /**
+   * Create network partitions
+   */
+  async partitionNetwork(partitions: NetworkPartition): Promise<void> {
+    // Implementation for network partitioning
+    // This is a simplified version - in a real implementation, you would:
+    // 1. Create separate networks for each partition
+    // 2. Connect containers to their respective partition networks
+    // 3. Disconnect them from other networks
+    debug('Network partitioning not fully implemented');
+  }
+
+  /**
+   * Set resource limits for a node
+   */
+  async setResourceLimits(
+    handle: NodeHandle,
+    limits: Partial<NodeConfig['resources']> = {}
+  ): Promise<void> {
+    const container = this.containers.get(handle.id);
+    if (!container) {
+      throw new Error(`No container found for node ${handle.id}`);
+    }
+
+    try {
+      // Delegate to ResourceManager
+      await this.resourceManager.setResourceLimits(container, {
+        cpu: limits.cpu,
+        memory: limits.memory,
+        memorySwap: limits.memory // Default to same as memory limit if not specified
+      });
+      
+      debug(`Updated resource limits for node %s: %o`, handle.id, limits);
+    } catch (error) {
+      debug(`Failed to update resource limits for node ${handle.id}: %o`, error);
+      throw new Error(`Failed to update resource limits: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+
+  /**
+   * Connect two nodes in the network
+   */
+  async connectNodes(handle1: NodeHandle, handle2: NodeHandle): Promise<void> {
+    const dockerHandle1 = handle1 as DockerNodeHandle;
+    const dockerHandle2 = handle2 as DockerNodeHandle;
+    
+    const container1 = this.containers.get(handle1.id);
+    const container2 = this.containers.get(handle2.id);
+
+    if (!container1 || !container2) {
+      throw new Error('One or both containers not found');
+    }
+
+    try {
+      // Get the network from the first container
+      const networkId = dockerHandle1.networkId;
+      if (!networkId) {
+        throw new Error(`No network found for node ${handle1.id}`);
+      }
+
+      // Connect the second container to the same network
+      const network = this.networks.get(handle1.id);
+      if (!network) {
+        throw new Error(`Network not found for node ${handle1.id}`);
+      }
+
+      await network.connect({
+        Container: container2.id,
+        EndpointConfig: {
+          Aliases: [`node-${handle2.id}`]
+        }
+      });
+
+      // Update the network ID in the second handle
+      dockerHandle2.networkId = networkId;
+    } catch (error) {
+      debug(`Error connecting nodes ${handle1.id} and ${handle2.id}: %o`, error);
+      throw error;
+    }
+  }
+
+
+  /**
+   * Cleans up resources for a specific node that failed to start properly.
+   * 
+   * This method is automatically called when a node fails to start during the `startNode` process.
+   * It handles cleanup of both the container and network resources associated with the failed node,
+   * and ensures all internal state is properly cleaned up.
+   * 
+   * @remarks
+   * - Runs container and network cleanup in parallel for efficiency
+   * - Handles errors gracefully by logging them without rethrowing
+   * - Cleans up internal state for just the specified node
+   * - Used internally by the orchestrator during error handling
+   * 
+   * @param nodeId - The unique identifier of the node that failed to start
+   * @private
+   */
+  private async cleanupFailedStart(nodeId: string): Promise<void> {
+    debug(`Cleaning up failed start for node ${nodeId}...`);
+    
+    // Get references to resources before starting cleanup
+    const container = this.containers.get(nodeId);
+    const network = this.networks.get(nodeId);
+    
+    // Create a map of containers to clean up
+    const containersToCleanup = new Map<string, Container>();
+    if (container) {
+      containersToCleanup.set(nodeId, container);
+    }
+    
+    // Create a map of networks to clean up
+    const networksToCleanup = new Map<string, Network>();
+    if (network) {
+      networksToCleanup.set(nodeId, network);
+    }
+    
+    try {
+      // Run container and network cleanup in parallel
+      const [containerErrors, networkErrors] = await Promise.all([
+        // Clean up containers using ContainerManager
+        this.containerManager.cleanupContainers(containersToCleanup),
+        // Clean up networks using NetworkManager
+        this.networkManager.cleanupNetworks(networksToCleanup)
+      ]);
+      
+      // Log any errors that occurred during cleanup
+      if (containerErrors.length > 0) {
+        debug(`Encountered ${containerErrors.length} error(s) while cleaning up containers for node ${nodeId}:`);
+        containerErrors.forEach(({ resource, error }) => {
+          console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
+        });
+      }
+      
+      if (networkErrors.length > 0) {
+        debug(`Encountered ${networkErrors.length} error(s) while cleaning up networks for node ${nodeId}:`);
+        networkErrors.forEach(({ resource, error }) => {
+          console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
+        });
+      }
+      
+      debug(`Completed cleanup for node ${nodeId}`);
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+      debug(`Unexpected error during cleanup of node ${nodeId}: %s`, errorMessage);
+    } finally {
+      // Always clean up internal state, even if errors occurred
+      this.containers.delete(nodeId);
+      this.networks.delete(nodeId);
+      this.nodeHandles.delete(nodeId);
+      this.containerLogStreams.delete(nodeId);
+    }
+  }
+
+  /**
+   * Get a container by ID
+   * @param containerId The ID of the container to retrieve
+   * @returns The container instance or undefined if not found
+   */
+  async getContainer(containerId: string): Promise<Container | undefined> {
+    // First try to get from our containers map
+    const container = this.containers.get(containerId);
+    if (container) {
+      return container;
+    }
+    
+    // If not found, try to get it from the container manager
+    try {
+      return await this.containerManager.getContainer(containerId);
+    } catch (error) {
+      debug(`Failed to get container ${containerId}: %o`, error);
+      return undefined;
+    }
+  }
+
+  /**
+   * Cleans up all resources managed by this orchestrator.
+   * 
+   * This method should be called during shutdown or when you need to completely tear down
+   * all containers and networks created by this orchestrator instance.
+   * 
+   * @remarks
+   * - Stops and removes all containers first
+   * - Then removes all networks (sequential execution)
+   * - Clears all internal state including node handles and log streams
+   * - Throws any errors that occur during cleanup
+   * - Should be called when the orchestrator is being shut down
+   * 
+   * @throws {Error} If any error occurs during the cleanup process
+   */
+  async cleanup(): Promise<void> {
+    debug('Starting cleanup of all resources...');
+    
+    // Create copies of the maps to avoid modification during iteration
+    const containersToCleanup = new Map(this.containers);
+    const networksToCleanup = new Map(this.networks);
+    
+    try {
+      // First, clean up all containers
+      debug('Stopping and removing all containers...');
+      const containerErrors = await this.containerManager.cleanupContainers(containersToCleanup);
+      
+      // Wait a short time to ensure all container cleanup is complete
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      
+      // Then clean up all networks
+      debug('Removing all networks...');
+      const networkErrors = await this.networkManager.cleanupNetworks(networksToCleanup);
+      
+      // Log any errors that occurred during cleanup
+      if (containerErrors.length > 0) {
+        debug(`Encountered ${containerErrors.length} error(s) while cleaning up containers:`);
+        containerErrors.forEach(({ resource, error }) => {
+          console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
+        });
+      }
+      
+      if (networkErrors.length > 0) {
+        debug(`Encountered ${networkErrors.length} error(s) while cleaning up networks:`);
+        networkErrors.forEach(({ resource, error }) => {
+          console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
+        });
+      }
+      
+      debug('Completed cleanup of all resources');
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+      debug('Unexpected error during cleanup: %s', errorMessage);
+      throw error; // Re-throw to allow callers to handle the error
+    } finally {
+      // Always clear internal state, even if errors occurred
+      this.containers.clear();
+      this.networks.clear();
+      this.nodeHandles.clear();
+      this.containerLogStreams.clear();
+    }
+  }
+}

src/orchestration/docker-orchestrator/managers/container-manager.ts

@@ -0,0 +1,249 @@
+import Debug from 'debug';
+import Docker, { Container, DockerOptions } from 'dockerode';
+import { IContainerManager } from './interfaces';
+import { NodeConfig, NodeStatus } from '../../types';
+
+const debug = Debug('rz:docker:container-manager');
+
+export class ContainerManager implements IContainerManager {
+  private docker: Docker;
+
+  constructor() {
+    this.docker = new Docker();
+  }
+
+  async createContainer(
+    nodeId: string,
+    config: NodeConfig,
+    networkId: string
+  ): Promise<Container> {
+    const containerName = `rhizome-node-${nodeId}`;
+    
+    // Create host config with port bindings and mounts
+    const hostConfig: Docker.HostConfig = {
+      NetworkMode: networkId,
+      PortBindings: {
+        [`${config.network?.port || 3000}/tcp`]: [{ HostPort: config.network?.port?.toString() }],
+        [`${config.network?.requestPort || 3001}/tcp`]: [{ HostPort: config.network?.requestPort?.toString() }],
+      },
+    };
+
+    // Add resource limits if specified
+    if (config.resources) {
+      if (config.resources.cpu) {
+        // Convert CPU cores to nanoCPUs (1 CPU = 1e9 nanoCPUs)
+        hostConfig.NanoCpus = config.resources.cpu * 1e9;
+      }
+
+      if (config.resources.memory) {
+        hostConfig.Memory = config.resources.memory * 1024 * 1024; // Convert MB to bytes
+        hostConfig.MemorySwap = config.resources.memorySwap 
+          ? config.resources.memorySwap * 1024 * 1024 
+          : config.resources.memory * 2 * 1024 * 1024; // Default swap to 2x memory
+      }
+    }
+  
+    // Increase file descriptor limits
+    hostConfig.Ulimits = [
+      {
+        Name: 'nofile',
+        Soft: 65536,
+        Hard: 65536
+      }
+    ];
+  
+    // Set environment variables to optimize performance and disable file watching
+    const envVars = [
+      // Node.js and memory settings
+      `NODE_OPTIONS=--max-old-space-size=${Math.floor((config.resources?.memory || 512) * 0.8)}`,
+      'NODE_ENV=test',
+      
+      // Network configuration
+      `RHIZOME_HTTP_BIND_PORT=${config.network?.port || 3000}`,
+      'RHIZOME_HTTP_BIND_ADDR=0.0.0.0',
+      `RHIZOME_REQUEST_BIND_PORT=${config.network?.requestPort || 3001}`,
+      'RHIZOME_REQUEST_BIND_ADDR=0.0.0.0',
+      `RHIZOME_PUBLISH_BIND_PORT=${(config.network?.requestPort || 3001) + 1}`,
+      'RHIZOME_PUBLISH_BIND_ADDR=0.0.0.0',
+      
+      // Application settings
+      'RHIZOME_STORAGE_TYPE=memory',
+      'RHIZOME_HTTP_API_ENABLE=true',
+      `RHIZOME_PEER_ID=${nodeId}`,
+      
+      // Disable unnecessary features for testing
+      'DISABLE_HTTP_HTML=true',
+      'DISABLE_MARKDOWN=true',
+      
+      // Debug settings
+      'DEBUG=rz:*,rhizome:*,docker:*',
+      'DEBUG_COLORS=true'
+    ];
+
+    // Create container configuration with all environment variables
+    const containerConfig: Docker.ContainerCreateOptions = {
+      name: containerName,
+      Image: 'rhizome-node-test',
+      ExposedPorts: {
+        [`${config.network?.port || 3000}/tcp`]: {},
+        [`${config.network?.requestPort || 3001}/tcp`]: {}
+      },
+      HostConfig: hostConfig,
+      Env: [
+        ...envVars,
+        'NODE_ENV=test',
+        'DEBUG=*',
+        `RHIZOME_HTTP_API_PORT=${config.network?.port || 3000}`,
+        'RHIZOME_HTTP_API_ADDR=0.0.0.0',
+        'RHIZOME_HTTP_API_ENABLE=true'
+      ]
+    };
+
+    try {
+      const container = await this.docker.createContainer(containerConfig);
+
+      return container;
+    } catch (error) {
+      throw new Error(`Failed to create container: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+
+  async startContainer(container: Container): Promise<void> {
+    try {
+      debug(`Starting container ${container.id}`);
+      await container.start();
+      
+      // Verify the container is actually running
+      const containerInfo = await this.verifyContainerRunning(container);
+      debug(`Container ${container.id} started with status:`, containerInfo.State);
+      
+    } catch (error) {
+      // Get container logs for debugging
+      let logs = '';
+      try {
+        logs = await this.getContainerLogs(container);
+      } catch (logError) {
+        debug('Failed to get container logs:', logError);
+      }
+      
+      throw new Error(
+        `Failed to start container: ${error instanceof Error ? error.message : 'Unknown error'}\n` +
+        `Container logs:\n${logs}`
+      );
+    }
+  }
+
+  async stopContainer(container: Container): Promise<void> {
+    try {
+      await container.stop({ t: 1 });
+    } catch (error) {
+      debug('Error stopping container: %o', error);
+      throw error;
+    }
+  }
+
+  async removeContainer(container: Container): Promise<void> {
+    try {
+      await container.remove({ force: true });
+    } catch (error) {
+      debug('Error removing container: %o', error);
+      throw error;
+    }
+  }
+
+  async getContainerLogs(container: Container, tailLines = 20): Promise<string> {
+    const logs = await container.logs({
+      stdout: true,
+      stderr: true,
+      tail: tailLines,
+      timestamps: true,
+      follow: false,
+    });
+    return logs.toString();
+  }
+
+  /**
+   * Get a container by ID
+   * @param containerId The ID of the container to retrieve
+   * @returns The container instance
+   * @throws Error if the container cannot be found
+   */
+  async getContainer(containerId: string): Promise<Container> {
+    try {
+      const container = this.docker.getContainer(containerId);
+      // Verify the container exists by inspecting it
+      await container.inspect();
+      return container;
+    } catch (error) {
+      throw new Error(`Failed to get container ${containerId}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async verifyContainerRunning(container: Container): Promise<Docker.ContainerInspectInfo> {
+    const containerInfo = await container.inspect();
+    if (!containerInfo.State.Running) {
+      throw new Error('Container is not running');
+    }
+    return containerInfo;
+  }
+
+  mapContainerState(state: string): NodeStatus['status'] {
+    if (!state) return 'error';
+    
+    const stateLower = state.toLowerCase();
+    if (['created', 'restarting'].includes(stateLower)) return 'starting';
+    if (stateLower === 'running') return 'running';
+    if (stateLower === 'paused') return 'stopping';
+    if (['dead', 'exited'].includes(stateLower)) return 'stopped';
+    
+    return 'error';
+  }
+
+  async cleanupContainers(containers: Map<string, Container>): Promise<Array<{ resource: string; error: Error }>> {
+    const cleanupErrors: Array<{ resource: string; error: Error }> = [];
+    
+    // Process containers in sequence to avoid overwhelming the Docker daemon
+    for (const [nodeId, container] of containers.entries()) {
+      try {
+        debug(`[Cleanup] Stopping container ${nodeId}...`);
+        
+        try {
+          // First, try to stop the container gracefully
+          await this.stopContainer(container);
+          debug(`[Cleanup] Successfully stopped container ${nodeId}`);
+        } catch (stopError) {
+          debug(`[Cleanup] Failed to stop container ${nodeId}: %o`, stopError);
+          // Continue with force removal even if stop failed
+        }
+        
+        // Now remove the container
+        debug(`[Cleanup] Removing container ${nodeId}...`);
+        await this.removeContainer(container);
+        debug(`[Cleanup] Successfully removed container ${nodeId}`);
+        
+        // Verify the container is actually gone
+        try {
+          const containerInfo = await container.inspect();
+          debug(`[Cleanup] Container ${nodeId} still exists after removal: %s`, containerInfo.State?.Status);
+          cleanupErrors.push({ 
+            resource: `container:${nodeId}`, 
+            error: new Error(`Container still exists after removal: ${containerInfo.State?.Status}`) 
+          });
+        } catch (inspectError) {
+          // Expected - container should not exist anymore
+          debug(`[Cleanup] Verified container ${nodeId} has been removed`);
+        }
+        
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        debug(`[Cleanup] Error cleaning up container ${nodeId}: %o`, err);
+        cleanupErrors.push({ resource: `container:${nodeId}`, error: err });
+      }
+      
+      // Add a small delay between container cleanups
+      await new Promise(resolve => setTimeout(resolve, 500));
+    }
+    
+    return cleanupErrors;
+  }
+}

src/orchestration/docker-orchestrator/managers/image-manager.ts

@@ -0,0 +1,159 @@
+import Docker, { DockerOptions } from 'dockerode';
+import * as path from 'path';
+import { promises as fs } from 'fs';
+import * as tar from 'tar-fs';
+import { Headers } from 'tar-fs';
+import { IImageManager } from './interfaces';
+import Debug from 'debug';
+
+const debug = Debug('rz:docker:image-manager');
+
+// Global promise to track test image build
+let testImageBuildPromise: Promise<void> | null = null;
+
+export class ImageManager implements IImageManager {
+  private docker: Docker;
+
+  constructor() {
+    this.docker = new Docker();
+  }
+
+  /**
+   * Build a test Docker image if it doesn't exist
+   */
+  async buildTestImage(imageName: string = 'rhizome-node-test'): Promise<void> {
+    if (testImageBuildPromise) {
+      debug('Test image build in progress, reusing existing build promise...');
+      return testImageBuildPromise;
+    }
+
+    debug('Building test Docker image...');
+    const dockerfilePath = path.join(process.cwd(), 'Dockerfile.test');
+    
+    // Verify Dockerfile exists
+    try {
+      await fs.access(dockerfilePath);
+      debug(`Found Dockerfile at: %s`, dockerfilePath);
+    } catch (err) {
+      throw new Error(`Dockerfile not found at ${dockerfilePath}: ${err}`);
+    }
+    
+    // Create a tar archive of the build context
+    const tarStream = tar.pack(process.cwd(), {
+      entries: [
+        'Dockerfile.test', 
+        'package.json',
+        'package-lock.json',
+        'tsconfig.json',
+        'src/',
+        'markdown/',
+        'util',
+        'examples/',
+        'README.md',
+      ],
+      map: (header: Headers) => {
+        // Ensure Dockerfile is named 'Dockerfile' in the build context
+        if (header.name === 'Dockerfile.test') {
+          header.name = 'Dockerfile';
+        }
+        return header;
+      }
+    });
+    
+    debug('Created build context tar stream');
+    
+    testImageBuildPromise = new Promise<void>((resolve, reject) => {
+      const logMessages: string[] = [];
+      
+      const log = (...args: any[]) => {
+        const timestamp = new Date().toISOString();
+        const message = args.map(arg => 
+          typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
+        ).join(' ');
+        const logMessage = `[${timestamp}] ${message}\n`;
+        process.stdout.write(logMessage);
+        logMessages.push(logMessage);
+      };
+      
+      this.docker.buildImage(tarStream, { t: imageName }, (err, stream) => {
+        if (err) {
+          const errorMsg = `❌ Error starting Docker build: ${err.message}`;
+          log(errorMsg);
+          return reject(new Error(errorMsg));
+        }
+        
+        if (!stream) {
+          const error = new Error('No build stream returned from Docker');
+          log(`❌ ${error.message}`);
+          return reject(error);
+        }
+        
+        log('✅ Docker build started, streaming output...');
+        
+        // Handle build output
+        let output = '';
+        stream.on('data', (chunk: Buffer) => {
+          const chunkStr = chunk.toString();
+          output += chunkStr;
+          
+          try {
+            // Try to parse as JSON (Docker build output is typically JSONL)
+            const lines = chunkStr.split('\n').filter(Boolean);
+            for (const line of lines) {
+              try {
+                if (!line.trim()) continue;
+                
+                const json = JSON.parse(line);
+                if (json.stream) {
+                  const message = `[Docker Build] ${json.stream}`.trim();
+                  log(message);
+                } else if (json.error) {
+                  const errorMsg = json.error.trim() || 'Unknown error during Docker build';
+                  log(`❌ ${errorMsg}`);
+                  reject(new Error(errorMsg));
+                  return;
+                } else if (Object.keys(json).length > 0) {
+                  // Log any other non-empty JSON objects
+                  log(`[Docker Build] ${JSON.stringify(json)}`);
+                }
+              } catch (e) {
+                // If not JSON, log as plain text if not empty
+                if (line.trim()) {
+                  log(`[Docker Build] ${line}`);
+                }
+              }
+            }
+          } catch (e) {
+            const errorMsg = `Error processing build output: ${e}\nRaw output: ${chunkStr}`;
+            log(`❌ ${errorMsg}`);
+            debug('Docker build error: %s', errorMsg);
+          }
+        });
+        
+        stream.on('end', () => {
+          log('✅ Docker build completed successfully');
+          resolve();
+        });
+        
+        stream.on('error', (err: Error) => {
+          const errorMsg = `❌ Docker build failed: ${err.message}\nBuild output so far: ${output}`;
+          log(errorMsg);
+          reject(new Error(errorMsg));
+        });
+      });
+    });
+  }
+
+  /**
+   * Check if an image exists locally
+   */
+  async imageExists(imageName: string): Promise<boolean> {
+    try {
+      const image = this.docker.getImage(imageName);
+      await image.inspect();
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+}

src/orchestration/docker-orchestrator/managers/index.ts

@@ -0,0 +1,5 @@
+export * from './interfaces';
+export * from './container-manager';
+export * from './network-manager';
+export * from './resource-manager';
+export * from './status-manager';

src/orchestration/docker-orchestrator/managers/interfaces.ts

@@ -0,0 +1,69 @@
+import Docker, { Container, Network, NetworkInspectInfo } from 'dockerode';
+import { NodeConfig, NodeHandle, NodeStatus } from '../../types';
+
+export interface IContainerManager {
+  createContainer(
+    nodeId: string,
+    config: NodeConfig,
+    networkId: string
+  ): Promise<Container>;
+  
+  startContainer(container: Container): Promise<void>;
+  stopContainer(container: Container): Promise<void>;
+  removeContainer(container: Container): Promise<void>;
+  getContainerLogs(container: Container, tailLines?: number): Promise<string>;
+  getContainer(containerId: string): Promise<Container>;
+  verifyContainerRunning(container: Container): Promise<Docker.ContainerInspectInfo>;
+  mapContainerState(state: string): NodeStatus['status'];
+  cleanupContainers(containers: Map<string, Container>): Promise<Array<{ resource: string; error: Error }>>;
+}
+
+export interface INetworkManager {
+  createNetwork(nodeId: string): Promise<Network>;
+  removeNetwork(networkId: string): Promise<void>;
+  connectToNetwork(containerId: string, networkId: string, aliases?: string[]): Promise<void>;
+  disconnectFromNetwork(containerId: string, networkId: string): Promise<void>;
+  setupPortBindings(ports: Record<string, any>): Docker.HostConfig['PortBindings'];
+  getNetworkInfo(networkId: string): Promise<NetworkInspectInfo>;
+  cleanupNetworks(networks: Map<string, Network>): Promise<Array<{ resource: string; error: Error }>>;
+}
+
+export interface IResourceManager {
+  setResourceLimits(
+    container: Container,
+    limits: Partial<NodeConfig['resources']>
+  ): Promise<void>;
+  
+  getResourceUsage(container: Container): Promise<{
+    cpu: { usage: number; limit: number };
+    memory: { usage: number; limit: number };
+  }>;
+}
+
+export interface IImageManager {
+  /**
+   * Build a test Docker image if it doesn't exist
+   * @param imageName The name to give to the built image
+   */
+  buildTestImage(imageName: string): Promise<void>;
+}
+
+export interface IStatusManager {
+  waitForNodeReady(
+    container: Container,
+    port: number,
+    maxAttempts?: number,
+    delayMs?: number
+  ): Promise<void>;
+  
+  healthCheck(healthUrl: string): Promise<{ ok: boolean; status: number }>;
+  mapContainerState(state: string): NodeStatus['status'];
+  
+  /**
+   * Get the status of a node including container status, network info, and resource usage
+   * @param handle The node handle containing node metadata
+   * @param container The Docker container instance
+   * @returns A promise that resolves to the node status
+   */
+  getNodeStatus(handle: NodeHandle, container: Container): Promise<NodeStatus>;
+}

src/orchestration/docker-orchestrator/managers/network-manager.ts

@@ -0,0 +1,167 @@
+import Debug from 'debug';
+import Docker, { Network, NetworkInspectInfo } from 'dockerode';
+import { INetworkManager } from './interfaces';
+
+const debug = Debug('rz:docker:network-manager');
+
+export class NetworkManager implements INetworkManager {
+  private networks: Map<string, Network> = new Map();
+  private docker: Docker;
+
+  constructor() {
+    this.docker = new Docker();
+  }
+
+  async createNetwork(nodeId: string): Promise<Network> {
+    const networkName = `rhizome-${nodeId}-network`;
+    
+    try {
+      const network = await this.docker.createNetwork({
+        Name: networkName,
+        Driver: 'bridge',
+        CheckDuplicate: true,
+        Internal: false,
+        Attachable: true,
+        EnableIPv6: false
+      });
+      
+      this.networks.set(nodeId, network);
+      return network;
+    } catch (error) {
+      debug(`Error creating network for node ${nodeId}: %o`, error);
+      throw error;
+    }
+  }
+
+  async removeNetwork(networkId: string): Promise<void> {
+    try {
+      const network = this.docker.getNetwork(networkId);
+      await network.remove();
+      
+      // Remove from our tracking map
+      for (const [nodeId, net] of this.networks.entries()) {
+        if (net.id === networkId) {
+          this.networks.delete(nodeId);
+          break;
+        }
+      }
+    } catch (error) {
+      debug(`Failed to remove network ${networkId}: %o`, error);
+      throw error;
+    }
+  }
+
+  async connectToNetwork(
+    containerId: string,
+    networkId: string,
+    aliases: string[] = []
+  ): Promise<void> {
+    try {
+      const network = this.docker.getNetwork(networkId);
+      await network.connect({
+        Container: containerId,
+        EndpointConfig: {
+          Aliases: aliases
+        }
+      });
+    } catch (error) {
+      debug(`Failed to connect container ${containerId} to network ${networkId}: %o`, error);
+      throw error;
+    }
+  }
+
+  async disconnectFromNetwork(containerId: string, networkId: string): Promise<void> {
+    try {
+      const network = this.docker.getNetwork(networkId);
+      await network.disconnect({ Container: containerId });
+    } catch (error) {
+      debug(`Failed to disconnect container ${containerId} from network ${networkId}: %o`, error);
+      throw error;
+    }
+  }
+
+  setupPortBindings(ports: Record<string, any>): Docker.HostConfig['PortBindings'] {
+    const portBindings: Docker.HostConfig['PortBindings'] = {};
+    
+    for (const [containerPort, hostPort] of Object.entries(ports)) {
+      const [port, protocol = 'tcp'] = containerPort.split('/');
+      portBindings[`${port}/${protocol}`] = [{ HostPort: hostPort.toString() }];
+    }
+    
+    return portBindings;
+  }
+
+  async getNetworkInfo(networkId: string): Promise<NetworkInspectInfo> {
+    try {
+      const network = this.docker.getNetwork(networkId);
+      return await network.inspect();
+    } catch (error) {
+      debug(`Failed to get network info for ${networkId}: %o`, error);
+      throw error;
+    }
+  }
+
+  async cleanupNetworks(networks: Map<string, Network>): Promise<Array<{ resource: string; error: Error }>> {
+    const cleanupErrors: Array<{ resource: string; error: Error }> = [];
+    
+    // Process networks in sequence to avoid overwhelming the Docker daemon
+    for (const [nodeId, network] of networks.entries()) {
+      try {
+        debug(`[Cleanup] Removing network for node ${nodeId}...`);
+        
+        // First, inspect the network to see if it has any connected containers
+        try {
+          const networkInfo = await this.getNetworkInfo(network.id);
+          if (networkInfo.Containers && Object.keys(networkInfo.Containers).length > 0) {
+            debug(`[Cleanup] Network ${nodeId} still has ${Object.keys(networkInfo.Containers).length} connected containers`);
+            
+            // Try to disconnect all containers from the network first
+            for (const containerId of Object.keys(networkInfo.Containers)) {
+              try {
+                debug(`[Cleanup] Disconnecting container ${containerId} from network ${nodeId}...`);
+                await this.disconnectFromNetwork(containerId, network.id);
+                debug(`[Cleanup] Successfully disconnected container ${containerId} from network ${nodeId}`);
+              } catch (disconnectError) {
+                debug(`[Cleanup] Failed to disconnect container ${containerId} from network ${nodeId}: %o`, disconnectError);
+                // Continue with network removal even if disconnect failed
+              }
+              
+              // Add a small delay between disconnects
+              await new Promise(resolve => setTimeout(resolve, 100));
+            }
+          }
+        } catch (inspectError) {
+          debug(`[Cleanup] Failed to inspect network ${nodeId} before removal: %o`, inspectError);
+          // Continue with removal even if inspect failed
+        }
+        
+        // Now remove the network
+        await this.removeNetwork(network.id);
+        debug(`[Cleanup] Successfully removed network for node ${nodeId}`);
+        
+        // Verify the network is actually gone
+        try {
+          await this.getNetworkInfo(network.id);
+          debug(`[Cleanup] Network ${nodeId} still exists after removal`);
+          cleanupErrors.push({ 
+            resource: `network:${nodeId}`, 
+            error: new Error('Network still exists after removal') 
+          });
+        } catch (inspectError) {
+          // Expected - network should not exist anymore
+          debug(`[Cleanup] Verified network ${nodeId} has been removed`);
+        }
+        
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        debug(`[Cleanup] Error cleaning up network ${nodeId}: %o`, err);
+        cleanupErrors.push({ resource: `network:${nodeId}`, error: err });
+      }
+      
+      // Add a small delay between network cleanups
+      await new Promise(resolve => setTimeout(resolve, 500));
+    }
+    
+    return cleanupErrors;
+  }
+}

src/orchestration/docker-orchestrator/managers/resource-manager.ts

@@ -0,0 +1,269 @@
+import Debug from 'debug';
+import { Container } from 'dockerode';
+import { IResourceManager } from './interfaces';
+
+const debug = Debug('rz:docker:resource-manager');
+
+// Define the structure of the Docker stats object
+interface ContainerStats {
+  cpu_stats: {
+    cpu_usage: {
+      total_usage: number;
+      usage_in_kernelmode?: number;
+      usage_in_usermode?: number;
+    };
+    system_cpu_usage: number;
+    online_cpus?: number;
+    throttling_data?: Record<string, unknown>;
+  };
+  precpu_stats: {
+    cpu_usage: {
+      total_usage: number;
+      usage_in_kernelmode?: number;
+      usage_in_usermode?: number;
+    };
+    system_cpu_usage: number;
+    online_cpus?: number;
+    throttling_data?: Record<string, unknown>;
+  };
+  memory_stats: {
+    usage?: number;
+    max_usage?: number;
+    limit?: number;
+    stats?: {
+      total_rss?: number;
+      [key: string]: unknown;
+    };
+    usage_in_bytes?: number;
+    limit_in_bytes?: number;
+    [key: string]: unknown;
+  };
+  [key: string]: unknown;
+}
+
+// Type guard to check if an object is a Node.js ReadableStream
+function isReadableStream(obj: unknown): obj is NodeJS.ReadableStream {
+  return (
+    obj !== null &&
+    typeof obj === 'object' &&
+    typeof (obj as any).pipe === 'function' &&
+    typeof (obj as any).on === 'function'
+  );
+}
+
+export class ResourceManager implements IResourceManager {
+  private debug = debug.extend('ResourceManager');
+
+  constructor() {
+    this.debug('ResourceManager initialized');
+  }
+
+  async setResourceLimits(
+    container: Container,
+    limits: {
+      cpu?: number;
+      memory?: number;
+      memorySwap?: number;
+    } = {}
+  ): Promise<void> {
+    try {
+      const updateConfig: any = {};
+
+      if (limits.cpu !== undefined) {
+        updateConfig.CpuShares = limits.cpu;
+        updateConfig.NanoCpus = limits.cpu * 1e9; // Convert to nanoCPUs
+      }
+
+      if (limits.memory !== undefined) {
+        updateConfig.Memory = limits.memory * 1024 * 1024; // Convert MB to bytes
+        updateConfig.MemorySwap = limits.memorySwap !== undefined
+          ? limits.memorySwap * 1024 * 1024
+          : updateConfig.Memory; // Default to same as memory if not specified
+      }
+
+      if (Object.keys(updateConfig).length > 0) {
+        await container.update(updateConfig);
+      }
+    } catch (error) {
+      throw new Error(`Failed to set resource limits: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+
+  async getResourceUsage(container: Container): Promise<{
+    cpu: { usage: number; limit: number };
+    memory: { usage: number; limit: number };
+  }> {
+    try {
+      this.debug('Getting container stats...');
+
+      // Get container stats with stream:false to get a single stats object
+      const stats = await container.stats({ stream: false });
+
+      // Log the raw stats type and constructor for debugging
+      this.debug('Raw stats type: %s', typeof stats);
+      this.debug('Raw stats constructor: %s', stats?.constructor?.name);
+
+      // Handle the response based on its type
+      let statsData: ContainerStats;
+
+      if (typeof stats === 'string') {
+        // If it's a string, parse it as JSON
+        this.debug('Stats is a string, parsing JSON');
+        try {
+          statsData = JSON.parse(stats) as ContainerStats;
+        } catch (error) {
+          this.debug('Failed to parse stats JSON: %o', error);
+          throw new Error(`Failed to parse stats JSON: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+      } else if (stats && typeof stats === 'object') {
+        // Check if it's a Node.js stream using our type guard
+        if (isReadableStream(stats)) {
+          this.debug('Stats is a stream, reading data...');
+          // Convert the stream to a string and parse as JSON
+          const statsString = await this.streamToString(stats);
+          try {
+            statsData = JSON.parse(statsString) as ContainerStats;
+            this.debug('Successfully parsed streamed stats');
+          } catch (error) {
+            this.debug('Failed to parse streamed stats: %o', error);
+            throw new Error(`Failed to parse streamed stats: ${error instanceof Error ? error.message : 'Unknown error'}`);
+          }
+        } else {
+          // If it's already an object, use it directly
+          this.debug('Stats is a plain object');
+          statsData = stats as unknown as ContainerStats;
+        }
+      } else {
+        throw new Error(`Unexpected stats type: ${typeof stats}`);
+      }
+
+      // Calculate and return the resource usage
+      return this.calculateResourceUsage(statsData);
+    } catch (error: unknown) {
+      this.debug('Error in getResourceUsage: %o', error);
+      // Return default values on error
+      return {
+        cpu: { usage: 0, limit: 0 },
+        memory: { usage: 0, limit: 0 },
+      };
+    }
+  }
+
+  /**
+   * Convert a ReadableStream to a string
+   */
+  private streamToString(stream: NodeJS.ReadableStream): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const chunks: Buffer[] = [];
+
+      stream.on('data', (chunk: unknown) => {
+        if (Buffer.isBuffer(chunk)) {
+          chunks.push(chunk);
+        } else if (typeof chunk === 'string') {
+          chunks.push(Buffer.from(chunk, 'utf8'));
+        } else if (chunk instanceof Uint8Array) {
+          chunks.push(Buffer.from(chunk));
+        } else {
+          this.debug('Unexpected chunk type: %s', typeof chunk);
+          reject(new Error(`Unexpected chunk type: ${typeof chunk}`));
+        }
+      });
+
+      stream.on('end', () => {
+        try {
+          const result = Buffer.concat(chunks).toString('utf8');
+          resolve(result);
+        } catch (error) {
+          this.debug('Error concatenating chunks: %o', error);
+          reject(error);
+        }
+      });
+
+      stream.on('error', (error: Error) => {
+        this.debug('Error reading stream: %o', error);
+        reject(error);
+      });
+    });
+  }
+
+  /**
+   * Calculate resource usage from stats data
+   */
+  private calculateResourceUsage(statsData: ContainerStats): {
+    cpu: { usage: number; limit: number };
+    memory: { usage: number; limit: number };
+  } {
+    this.debug('Calculating resource usage from stats data');
+
+    // Log detailed CPU stats if available
+    if (statsData.cpu_stats) {
+      this.debug('CPU stats: %o', {
+        online_cpus: statsData.cpu_stats.online_cpus,
+        cpu_usage: statsData.cpu_stats.cpu_usage,
+        system_cpu_usage: statsData.cpu_stats.system_cpu_usage,
+      });
+    } else {
+      this.debug('No CPU stats available');
+    }
+
+    // Log detailed memory stats if available
+    if (statsData.memory_stats) {
+      this.debug('Memory stats: %o', {
+        usage: statsData.memory_stats.usage,
+        max_usage: statsData.memory_stats.max_usage,
+        limit: statsData.memory_stats.limit,
+        stats: statsData.memory_stats.stats,
+      });
+    } else {
+      this.debug('No memory stats available');
+    }
+
+    // Calculate CPU usage percentage
+    let cpuPercent = 0;
+    const cpuCores = statsData.cpu_stats?.online_cpus || 1;
+
+    // Check if we have the necessary data for CPU calculation
+    if (statsData.cpu_stats?.cpu_usage?.total_usage !== undefined &&
+      statsData.precpu_stats?.cpu_usage?.total_usage !== undefined) {
+      const cpuDelta = statsData.cpu_stats.cpu_usage.total_usage -
+        (statsData.precpu_stats.cpu_usage.total_usage || 0);
+      const systemDelta = statsData.cpu_stats.system_cpu_usage -
+        (statsData.precpu_stats.system_cpu_usage || 0);
+
+      this.debug('CPU delta: %d, System delta: %d', cpuDelta, systemDelta);
+
+      if (systemDelta > 0 && cpuDelta > 0) {
+        cpuPercent = (cpuDelta / systemDelta) * cpuCores * 100.0;
+        this.debug('Calculated CPU percent: %d%%', cpuPercent);
+      }
+    } else {
+      this.debug('Insufficient CPU stats data for calculation');
+      this.debug('Available CPU stats: %o', statsData.cpu_stats);
+      this.debug('Available precpu_stats: %o', statsData.precpu_stats);
+    }
+
+    // Get memory usage with fallbacks
+    const memoryUsage = statsData.memory_stats?.usage ||
+      statsData.memory_stats?.stats?.total_rss ||
+      statsData.memory_stats?.usage_in_bytes ||
+      0;
+
+    const memoryLimit = statsData.memory_stats?.limit ||
+      statsData.memory_stats?.max_usage ||
+      statsData.memory_stats?.limit_in_bytes ||
+      0;
+
+    this.debug('Memory usage: %d / %d bytes', memoryUsage, memoryLimit);
+
+    return {
+      cpu: {
+        usage: cpuPercent,
+        limit: 100, // 100% CPU limit as a percentage
+      },
+      memory: {
+        usage: memoryUsage,
+        limit: memoryLimit || 0, // Ensure we don't return undefined
+      },
+    };
+  }
+}

src/orchestration/docker-orchestrator/managers/status-manager.ts

@@ -0,0 +1,311 @@
+import Docker, { Container } from 'dockerode';
+import { IStatusManager } from './interfaces';
+import { NodeHandle, NodeStatus } from '../../types';
+import Debug from 'debug';
+
+const debug = Debug('rz:docker:status-manager');
+
+const DEFAULT_MAX_ATTEMPTS = 8;
+const DEFAULT_DELAY_MS = 1000;
+const MAX_BACKOFF_MS = 30000; // 30 seconds max backoff
+
+export class StatusManager implements IStatusManager {
+  async waitForNodeReady(
+    container: Container,
+    port: number,
+    maxAttempts: number = DEFAULT_MAX_ATTEMPTS,
+    initialDelayMs: number = DEFAULT_DELAY_MS
+  ): Promise<void> {
+    debug(`[waitForNodeReady] Starting with port ${port}, maxAttempts: ${maxAttempts}, initialDelayMs: ${initialDelayMs}`);
+    let lastError: Error | null = null;
+    let attempt = 0;
+    let delay = initialDelayMs;
+
+    while (attempt < maxAttempts) {
+      attempt++;
+      const attemptStartTime = Date.now();
+      
+      try {
+        debug(`[Attempt ${attempt}/${maxAttempts}] Verifying container is running...`);
+        
+        // Add timeout to verifyContainerRunning
+        const verifyPromise = this.verifyContainerRunning(container);
+        const timeoutPromise = new Promise((_, reject) => 
+          setTimeout(() => reject(new Error('verifyContainerRunning timed out')), 10000)
+        );
+        
+        await Promise.race([verifyPromise, timeoutPromise]);
+        debug(`[Attempt ${attempt}/${maxAttempts}] Container is running`);
+        
+        const healthUrl = `http://localhost:${port}/api/health`;
+        debug(`[Attempt ${attempt}/${maxAttempts}] Checking health at: ${healthUrl}`);
+        
+        // Add timeout to health check
+        const healthCheckPromise = this.healthCheck(healthUrl);
+        const healthCheckTimeout = new Promise<never>((_, reject) =>
+          setTimeout(() => reject(new Error('Health check timed out')), 10000)
+        );
+        
+        const response = await Promise.race([healthCheckPromise, healthCheckTimeout]);
+        
+        if (response.ok) {
+          debug(`✅ Node is ready! (Attempt ${attempt}/${maxAttempts})`);
+          return; // Success!
+        }
+        
+        throw new Error(`Health check failed with status: ${response.status}`);
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        lastError = error instanceof Error ? error : new Error(errorMessage);
+        
+        const attemptDuration = Date.now() - attemptStartTime;
+        debug(`[Attempt ${attempt}/${maxAttempts}] Failed after ${attemptDuration}ms: %s`, errorMessage);
+        
+        // Log container state on error
+        try {
+          const containerInfo = await container.inspect();
+          debug(`[Container State] Status: ${containerInfo.State.Status}, Running: ${containerInfo.State.Running}, ExitCode: ${containerInfo.State.ExitCode}`);
+          
+          // Log recent container logs on error
+          if (containerInfo.State.Running) {
+            try {
+              const logs = await container.logs({
+                stdout: true,
+                stderr: true,
+                tail: 20,
+                timestamps: true,
+              });
+              debug(`[Container Logs] Last 20 lines:\n${logs.toString()}`);
+            } catch (logError) {
+              debug('Failed to get container logs: %o', logError);
+            }
+          }
+        } catch (inspectError) {
+          debug('Failed to inspect container: %o', inspectError);
+        }
+        
+        // Exponential backoff with jitter, but don't wait if we're out of attempts
+        if (attempt < maxAttempts) {
+          const jitter = Math.random() * 1000; // Add up to 1s of jitter
+          const backoff = Math.min(delay + jitter, MAX_BACKOFF_MS);
+          debug(`[Backoff] Waiting ${Math.round(backoff)}ms before next attempt...`);
+          await new Promise(resolve => setTimeout(resolve, backoff));
+          delay = Math.min(delay * 2, MAX_BACKOFF_MS); // Double the delay for next time, up to max
+        }
+      }
+    }
+    
+    // If we get here, all attempts failed
+    const errorMessage = `Node did not become ready after ${maxAttempts} attempts. Last error: ${lastError?.message || 'Unknown error'}`;
+    debug('❌ %s', errorMessage);
+    
+    // Final attempt to get container logs before failing
+    try {
+      const logs = await container.logs({
+        stdout: true,
+        stderr: true,
+        tail: 100,
+        timestamps: true,
+        follow: false
+      });
+      debug('=== FINAL CONTAINER LOGS ===\n%s\n=== END CONTAINER LOGS ===', logs.toString());
+    } catch (logError) {
+      debug('Failed to get final container logs: %o', logError);
+    }
+    
+    throw new Error(errorMessage);
+  }
+
+  async healthCheck(healthUrl: string): Promise<{ ok: boolean; status: number }> {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 5000);
+    
+    try {
+      const response = await fetch(healthUrl, {
+        headers: { 
+          'Accept': 'application/json',
+          'Connection': 'close'
+        },
+        signal: controller.signal
+      });
+      clearTimeout(timeout);
+      return {
+        ok: response.ok,
+        status: response.status
+      };
+    } catch (error) {
+      clearTimeout(timeout);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error(`Health check timed out after 5000ms (${healthUrl})`);
+      }
+      throw error;
+    }
+  }
+
+  mapContainerState(state: string): NodeStatus['status'] {
+    if (!state) return 'error';
+    
+    const stateLower = state.toLowerCase();
+    if (['created', 'restarting'].includes(stateLower)) return 'starting';
+    if (stateLower === 'running') return 'running';
+    if (stateLower === 'paused') return 'stopping';
+    if (['dead', 'exited', 'stopped'].includes(stateLower)) return 'stopped';
+    
+    return 'error';
+  }
+
+  private async verifyContainerRunning(container: Container): Promise<void> {
+    debug('[verifyContainerRunning] Checking container status...');
+    
+    try {
+      const data = await container.inspect();
+      debug('[verifyContainerRunning] Container inspect data:', JSON.stringify({
+        Id: data.Id,
+        Name: data.Name,
+        State: data.State,
+        Config: {
+          Image: data.Config?.Image,
+          Env: data.Config?.Env?.filter(env => env.startsWith('NODE_') || env.startsWith('DEBUG')),
+          Cmd: data.Config?.Cmd
+        },
+        HostConfig: {
+          Memory: data.HostConfig?.Memory,
+          NanoCpus: data.HostConfig?.NanoCpus,
+          NetworkMode: data.HostConfig?.NetworkMode
+        }
+      }, null, 2));
+      
+      if (!data.State.Running) {
+        const errorMessage = `Container is not running. Status: ${data.State.Status}, ExitCode: ${data.State.ExitCode}, Error: ${data.State.Error}`;
+        debug(`[verifyContainerRunning] ${errorMessage}`);
+        
+        // Try to get container logs for more context
+        try {
+          const logs = await container.logs({
+            stdout: true,
+            stderr: true,
+            tail: 50 // Get last 50 lines of logs
+          });
+          debug('[verifyContainerRunning] Container logs:', logs.toString());
+        } catch (logError) {
+          debug('[verifyContainerRunning] Failed to get container logs:', logError);
+        }
+        
+        throw new Error(errorMessage);
+      }
+      
+      debug('[verifyContainerRunning] Container is running');
+    } catch (error) {
+      debug('[verifyContainerRunning] Error checking container status:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Get the status of a node including container status, network info, and resource usage
+   * @param handle The node handle containing node metadata
+   * @param container The Docker container instance
+   * @returns A promise that resolves to the node status
+   */
+  async getNodeStatus(handle: NodeHandle, container: Container): Promise<NodeStatus> {
+    // Default error status for when container is not found or other errors occur
+    const errorStatus: NodeStatus = {
+      id: handle.id,
+      status: 'error',
+      error: 'Failed to get node status',
+      network: {
+        address: '',
+        httpPort: 0,
+        requestPort: 0,
+        peers: []
+      },
+      resources: {
+        cpu: { usage: 0, limit: 0 },
+        memory: { usage: 0, limit: 0 }
+      }
+    };
+
+    try {
+      // Get container info
+      const containerInfo = await container.inspect();
+      
+      // Get request port once since we use it multiple times
+      const requestPort = handle.getRequestPort?.() || 0;
+      
+      // Initialize with default values
+      const status: NodeStatus = {
+        id: handle.id, // Use the node ID from handle
+        containerId: container.id,
+        status: this.mapContainerState(containerInfo.State?.Status || ''),
+        network: {
+          address: containerInfo.NetworkSettings?.IPAddress || '',
+          httpPort: requestPort,
+          requestPort: requestPort,
+          peers: [],
+          networkId: ''
+        },
+        resources: {
+          cpu: { usage: 0, limit: 0 },
+          memory: { usage: 0, limit: 0 }
+        }
+      };
+
+      // Update network info if available
+      if (containerInfo.NetworkSettings?.Networks) {
+        const network = Object.values(containerInfo.NetworkSettings.Networks)[0];
+        if (network) {
+          // Ensure we have existing network values or use defaults
+          const currentNetwork = status.network || {
+            address: '',
+            httpPort: 0,
+            requestPort: 0,
+            peers: []
+          };
+          
+          // Create a new network object with all required properties
+          status.network = {
+            address: network.IPAddress || currentNetwork.address,
+            httpPort: currentNetwork.httpPort,
+            requestPort: currentNetwork.requestPort,
+            peers: currentNetwork.peers,
+            networkId: network.NetworkID || ''
+          };
+        }
+      }
+
+      // Get container stats for resource usage
+      try {
+        const stats = await container.stats({ stream: false });
+        const statsData = JSON.parse(stats.toString());
+        
+        if (statsData?.cpu_stats?.cpu_usage) {
+          status.resources!.cpu.usage = statsData.cpu_stats.cpu_usage.total_usage || 0;
+          status.resources!.cpu.limit = (statsData.cpu_stats.online_cpus || 0) * 1e9; // Convert to nanoCPUs
+        }
+        
+        if (statsData?.memory_stats) {
+          status.resources!.memory.usage = statsData.memory_stats.usage || 0;
+          status.resources!.memory.limit = statsData.memory_stats.limit || 0;
+        }
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+        debug(`Failed to get container stats for ${container.id}: %s`, errorMessage);
+        // Update status with error but don't return yet
+        status.status = 'error';
+        status.error = `Failed to get container stats: ${errorMessage}`;
+      }
+      
+      return status;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+      debug(`Error getting node status for ${handle.id}: %s`, errorMessage);
+      
+      return {
+        ...errorStatus,
+        id: handle.id,
+        error: errorMessage,
+        status: 'error'
+      };
+    }
+  }
+}

src/orchestration/docker-orchestrator/types.ts

@@ -0,0 +1,40 @@
+import Docker from 'dockerode';
+import { NodeHandle, NodeConfig, NodeStatus } from '../types';
+
+export interface DockerNodeHandle extends NodeHandle {
+  containerId: string;
+  networkId?: string;
+}
+
+export interface DockerOrchestratorOptions {
+  /** 
+   * Docker image to use for containers
+   * Defaults to 'rhizome-node' if not specified
+   */
+  image?: string;
+  
+  /** Working directory inside container */
+  containerWorkDir?: string;
+  
+  /** Whether to build test image if not found */
+  autoBuildTestImage?: boolean;
+}
+
+export interface ContainerResources {
+  cpuShares?: number;
+  memory?: number;
+  memorySwap?: number;
+  nanoCpus?: number;
+}
+
+export interface ContainerStatus {
+  containerId: string;
+  image: string;
+  state: string;
+  status: NodeStatus['status']; // Use the status type from NodeStatus
+  networkSettings: {
+    ipAddress: string;
+    gateway: string;
+    ports: Record<string, Array<{ hostIp: string; hostPort: string }> | null>;
+  };
+}

src/orchestration/docker-orchestrator/utils/port-utils.ts

@@ -0,0 +1,46 @@
+/**
+ * Get a random available port in the range 30000-50000
+ * @returns A random port number
+ */
+export function getRandomPort(): number {
+  return Math.floor(30000 + Math.random() * 20000);
+}
+
+/**
+ * Check if a port is available
+ * @param port Port number to check
+ * @returns True if the port is available, false otherwise
+ */
+export async function isPortAvailable(port: number): Promise<boolean> {
+  const net = await import('net');
+  return new Promise((resolve) => {
+    const server = net.createServer();
+    server.once('error', () => resolve(false));
+    server.once('listening', () => {
+      server.close(() => resolve(true));
+    });
+    server.listen(port);
+  });
+}
+
+/**
+ * Get an available port, optionally starting from a specific port
+ * @param startPort Optional starting port (default: 30000)
+ * @returns A promise that resolves to an available port
+ */
+export async function getAvailablePort(startPort: number = 30000): Promise<number> {
+  let port = startPort;
+  while (port <= 65535) {
+    if (await isPortAvailable(port)) {
+      return port;
+    }
+    port++;
+  }
+  throw new Error('No available ports found');
+}
+
+export default {
+  getRandomPort,
+  isPortAvailable,
+  getAvailablePort
+};

src/orchestration/factory.ts

@@ -0,0 +1,21 @@
+import { NodeOrchestrator, OrchestratorType } from './types';
+import { DockerOrchestrator } from './docker-orchestrator';
+import { TestOrchestrator } from './test-orchestrator';
+
+/**
+ * Factory function to create an appropriate orchestrator based on environment
+ */
+export function createOrchestrator(
+  type: OrchestratorType = 'in-memory',
+  options?: any
+): NodeOrchestrator {
+  switch (type) {
+    case 'docker':
+      return new DockerOrchestrator(options);
+    case 'kubernetes':
+      throw new Error('Kubernetes orchestrator not yet implemented');
+    case 'in-memory':
+    default:
+      return new TestOrchestrator();
+  }
+}

src/orchestration/index.ts

@@ -0,0 +1,9 @@
+// Re-export all types and interfaces
+export * from './types';
+
+// Export orchestrator implementations
+export * from './docker-orchestrator';
+export * from './test-orchestrator';
+
+// Export factory function
+export { createOrchestrator } from './factory';

src/orchestration/test-orchestrator/index.ts

@@ -0,0 +1,198 @@
+import { RhizomeNode, type RhizomeNodeConfig } from '../../node';
+import { PeerAddress } from '../../network';
+import { BaseOrchestrator } from '../base-orchestrator';
+import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types';
+import { getRandomPort } from '../docker-orchestrator/utils/port-utils';
+import { BasicCollection } from '../../collections/collection-basic';
+import Debug from 'debug';
+
+const debug = Debug('rz:test-orchestrator');
+
+/**
+ * In-memory implementation of NodeOrchestrator for testing
+ */
+export class TestOrchestrator extends BaseOrchestrator {
+  private nodes: Map<string, { handle: NodeHandle; node: RhizomeNode }> = new Map();
+
+  async startNode(config: NodeConfig): Promise<NodeHandle> {
+    const nodeId = config.id || `node-${Date.now()}`;
+    // Use getRandomPort instead of 0 for auto-selection
+    const httpPort = config.network?.port || getRandomPort();
+    const requestPort = config.network?.requestPort || getRandomPort();
+    
+    // Map NodeConfig to RhizomeNodeConfig with all required properties
+    const nodeConfig: RhizomeNodeConfig = {
+      // Required network properties
+      requestBindHost: '0.0.0.0',
+      requestBindPort: requestPort,
+      publishBindHost: '0.0.0.0',
+      publishBindPort: getRandomPort(), // Use a random port for publish socket
+      httpAddr: '0.0.0.0',
+      httpPort: httpPort,
+      httpEnable: true,
+      
+      // Required peer properties
+      peerId: nodeId,
+      creator: 'test-orchestrator',
+      
+      // Map network bootstrap peers to seedPeers if provided
+      seedPeers: config.network?.bootstrapPeers?.map(peer => {
+        const [host, port] = peer.split(':');
+        return new PeerAddress(host, parseInt(port));
+      }) || [],
+      
+      // Storage configuration with defaults
+      storage: {
+        type: 'memory',
+        path: config.storage?.path || `./data/${nodeId}`,
+        ...(config.storage || {})
+      }
+    };
+
+    const node = new RhizomeNode(nodeConfig);
+    
+    // Create and connect a user collection
+    const userCollection = new BasicCollection('user');
+    // Connect the collection to the node before serving it
+    userCollection.rhizomeConnect(node);
+    // Now serve the collection through the HTTP API
+    node.httpServer.httpApi.serveCollection(userCollection);
+    
+    // Start the node and wait for all components to be ready
+    debug(`[${nodeId}] Starting node and waiting for it to be fully ready...`);
+    try {
+      await node.start();
+      debug(`[${nodeId}] Node is fully started and ready`);
+    } catch (error) {
+      debug(`[${nodeId}] Error starting node:`, error);
+      throw error;
+    }
+
+    // Get the actual port the server is using
+    const serverAddress = node.httpServer.server?.address();
+    let actualPort = httpPort;
+    
+    // Handle different address types (string or AddressInfo)
+    if (serverAddress) {
+      actualPort = typeof serverAddress === 'string' 
+        ? httpPort 
+        : serverAddress.port || httpPort;
+    }
+    
+    const handle: NodeHandle = {
+      id: nodeId,
+      config: {
+        ...config,
+        id: nodeId,
+        network: {
+          ...config.network,
+          port: actualPort,
+          requestPort: requestPort
+        }
+      },
+      status: async () => this.getNodeStatus(handle),
+      getApiUrl: () => `http://localhost:${actualPort}/api`,
+      stop: async () => {
+        await node.stop();
+        this.nodes.delete(nodeId);
+      },
+      getRequestPort: () => requestPort,
+    };
+
+    this.nodes.set(nodeId, { handle, node });
+    return handle;
+  }
+
+  async stopNode(handle: NodeHandle): Promise<void> {
+    const node = this.nodes.get(handle.id);
+    if (node) {
+      await node.node.stop();
+      this.nodes.delete(handle.id);
+    }
+  }
+
+  async getNodeStatus(handle: NodeHandle): Promise<NodeStatus> {
+    const node = this.nodes.get(handle.id);
+    if (!node) {
+      return {
+        id: handle.id,
+        status: 'stopped',
+        error: 'Node not found',
+        network: {
+          address: '127.0.0.1',
+          httpPort: 0,
+          requestPort: 0,
+          peers: []
+        },
+        resources: {
+          cpu: { usage: 0, limit: 0 },
+          memory: { usage: 0, limit: 0 }
+        }
+      };
+    }
+
+
+    // Since we don't have a direct way to check if the node is running,
+    // we'll assume it's running if it's in our nodes map
+    // In a real implementation, we would check the actual node state
+    const status: NodeStatus = {
+      id: handle.id,
+      status: 'running',
+      network: {
+        address: '127.0.0.1',
+        httpPort: node.node.config.httpPort || 0,
+        requestPort: node.node.config.requestBindPort || 0,
+        peers: node.node.peers ? Array.from(node.node.peers.peers).map(p => p.reqAddr.toAddrString()) : []
+      },
+      resources: {
+        cpu: {
+          usage: 0,
+          limit: 0,
+        },
+        memory: {
+          usage: 0,
+          limit: 0,
+        },
+      }
+    };
+
+    return status;
+  }
+
+  async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise<void> {
+    const n1 = this.nodes.get(node1.id)?.node;
+    const n2 = this.nodes.get(node2.id)?.node;
+
+    if (!n1 || !n2) {
+      throw new Error('One or both nodes not found');
+    }
+
+    // In a real implementation, we would connect the nodes here
+    // For testing, we'll just log the connection attempt
+    console.log(`Connecting nodes ${node1.id} and ${node2.id}`);
+  }
+
+  async partitionNetwork(partitions: NetworkPartition): Promise<void> {
+    // In a real implementation, we would create network partitions
+    // For testing, we'll just log the partition attempt
+    console.log('Creating network partitions:', partitions);
+  }
+
+  async setResourceLimits(
+    handle: NodeHandle,
+    limits: Partial<NodeConfig['resources']>
+  ): Promise<void> {
+    // In-memory nodes don't have real resource limits
+    console.log(`Setting resource limits for ${handle.id}:`, limits);
+  }
+
+  /**
+   * Clean up all resources
+   */
+  async cleanup(): Promise<void> {
+    await Promise.all(
+      Array.from(this.nodes.values()).map(({ node }) => node.stop())
+    );
+    this.nodes.clear();
+  }
+}

src/orchestration/types.ts

@@ -0,0 +1,104 @@
+/**
+ * Core types and interfaces for the orchestration layer
+ */
+
+export interface NodeConfig {
+  /** Unique identifier for the node */
+  id: string;
+  
+  /** Network configuration */
+  network?: {
+    /** Port to listen on (0 = auto-select) */
+    port?: number;
+    /** Port for request/reply communication */
+    requestPort?: number;
+    /** Known peers to connect to */
+    bootstrapPeers?: string[];
+  };
+  
+  /** Resource constraints */
+  resources?: {
+    /** CPU shares (0-1024) */
+    cpu?: number;
+    /** Memory limit in MB */
+    memory?: number;
+    /** Memory swap limit in MB (defaults to 2x memory if not specified) */
+    memorySwap?: number;
+  };
+  
+  /** Storage configuration */
+  storage?: {
+    /** Storage type */
+    type?: 'memory' | 'leveldb' | 'sqlite' | 'postgres';
+    /** Path to data directory */
+    path?: string;
+    /** Maximum storage in MB */
+    limit?: number;
+  };
+  
+  /** Additional configuration options */
+  [key: string]: any;
+}
+
+export interface NodeStatus {
+  id: string;
+  containerId?: string;
+  status: 'starting' | 'running' | 'stopping' | 'stopped' | 'error';
+  network?: {
+    address: string;
+    requestPort: number;
+    httpPort: number;
+    peers: string[];
+    networkId?: string;
+  };
+  resources?: {
+    cpu: {
+      usage: number;
+      limit: number;
+    };
+    memory: {
+      usage: number;
+      limit: number;
+    };
+  };
+  error?: string;
+}
+
+export interface NodeHandle {
+  id: string;
+  config: NodeConfig;
+  status: () => Promise<NodeStatus>;
+  stop: () => Promise<void>;
+  /** Get API URL if applicable */
+  getApiUrl?: () => string;
+  getRequestPort: () => number | undefined;
+}
+
+export interface NetworkPartition {
+  groups: string[][];
+}
+
+export interface NodeOrchestrator {
+  /** Start a new node with the given configuration */
+  startNode(config: NodeConfig): Promise<NodeHandle>;
+  
+  /** Stop a running node */
+  stopNode(handle: NodeHandle): Promise<void>;
+  
+  /** Get status of a node */
+  getNodeStatus(handle: NodeHandle): Promise<NodeStatus>;
+  
+  /** Connect two nodes */
+  connectNodes(node1: NodeHandle, node2: NodeHandle): Promise<void>;
+  
+  /** Create network partitions */
+  partitionNetwork(partitions: NetworkPartition): Promise<void>;
+  
+  /** Set resource limits for a node */
+  setResourceLimits(handle: NodeHandle, limits: Partial<NodeConfig['resources']>): Promise<void>;
+  
+  /** Clean up all resources */
+  cleanup(): Promise<void>;
+}
+
+export type OrchestratorType = 'in-memory' | 'docker' | 'kubernetes';

src/query/index.ts

@@ -1,2 +1,2 @@
 export { QueryEngine } from './query-engine';
-export { StorageQueryEngine, JsonLogic as StorageJsonLogic } from './storage-query-engine';
+export { StorageQueryEngine, type JsonLogic as StorageJsonLogic } from './storage-query-engine';

src/query/query-engine.ts

@@ -1,4 +1,5 @@
-import { apply, is_logic } from 'json-logic-js';
+import jsonLogic from 'json-logic-js';
+const { apply, is_logic } = jsonLogic;
 import Debug from 'debug';
 import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema/schema';
 import { Lossless, LosslessViewOne, LosslessViewMany, CollapsedDelta } from '../views/lossless';

src/query/storage-query-engine.ts

@@ -1,4 +1,5 @@
-import { apply } from 'json-logic-js';
+import jsonLogic from 'json-logic-js';
+const { apply } = jsonLogic;
 import Debug from 'debug';
 import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema';
 import { DeltaQueryStorage, DeltaQuery } from '../storage/interface';

src/storage/factory.ts

@@ -1,7 +1,10 @@
+import Debug from 'debug';
 import { DeltaStorage, DeltaQueryStorage, StorageConfig } from './interface';
 import { MemoryDeltaStorage } from './memory';
 import { LevelDBDeltaStorage } from './leveldb';
 
+const debug = Debug('rz:storage:factory');
+
 /**
  * Factory for creating delta storage instances based on configuration
  */
@@ -56,10 +59,10 @@ export class StorageFactory {
   ): Promise<void> {
     const batchSize = options.batchSize || 1000;
     
-    console.log('Starting storage migration...');
+    debug('Starting storage migration...');
     
     const allDeltas = await source.getAllDeltas();
-    console.log(`Found ${allDeltas.length} deltas to migrate`);
+    debug(`Found %d deltas to migrate`, allDeltas.length);
     
     // Migrate in batches to avoid memory issues
     for (let i = 0; i < allDeltas.length; i += batchSize) {
@@ -69,19 +72,21 @@ export class StorageFactory {
         await target.storeDelta(delta);
       }
       
-      console.log(`Migrated ${Math.min(i + batchSize, allDeltas.length)} / ${allDeltas.length} deltas`);
+      debug('Migrated %d / %d deltas', Math.min(i + batchSize, allDeltas.length), allDeltas.length);
     }
     
-    console.log('Migration completed successfully');
+    debug('Migration completed successfully');
     
     // Verify migration
     const sourceStats = await source.getStats();
     const targetStats = await target.getStats();
     
     if (sourceStats.totalDeltas !== targetStats.totalDeltas) {
-      throw new Error(`Migration verification failed: source has ${sourceStats.totalDeltas} deltas, target has ${targetStats.totalDeltas}`);
+      const errorMsg = `Migration verification failed: source has ${sourceStats.totalDeltas} deltas, target has ${targetStats.totalDeltas}`;
+      debug(errorMsg);
+      throw new Error(errorMsg);
     }
     
-    console.log(`Migration verified: ${targetStats.totalDeltas} deltas migrated successfully`);
+    debug('Migration verified: %d deltas migrated successfully', targetStats.totalDeltas);
   }
 }

src/util/md-files.ts

@@ -1,5 +1,5 @@
 import Debug from "debug";
-import {FSWatcher, readdirSync, readFileSync, watch} from "fs";
+import {FSWatcher, readdirSync, readFileSync, watch, accessSync, constants} from "fs";
 import path, {join} from "path";
 import showdown from "showdown";
 import {RhizomeNode} from "../node";
@@ -48,9 +48,32 @@ export class MDFiles {
   }
 
   readReadme() {
-    const md = readFileSync('./README.md').toString();
+    let currentDir = process.cwd();
+    const root = path.parse(currentDir).root;
+    let readmePath: string | null = null;
+  
+    // Traverse up the directory tree until we find README.md or hit the root
+    while (currentDir !== root) {
+      const testPath = path.join(currentDir, 'README.md');
+      try {
+        // Using the imported accessSync function
+        accessSync(testPath, constants.F_OK);
+        readmePath = testPath;
+        break;
+      } catch (err) {
+        // Move up one directory
+        currentDir = path.dirname(currentDir);
+      }
+    }
+  
+    if (!readmePath) {
+      debug('No README.md found in any parent directory');
+      return;
+    }
+  
+    const md = readFileSync(readmePath).toString();
     const html = htmlDocFromMarkdown(md);
-    this.readme = {name: 'README', md, html};
+    this.readme = { name: 'README', md, html };
   }
 
   getReadmeHTML() {

src/views/lossless.ts

@@ -172,7 +172,6 @@ export class Lossless {
   viewSpecific(entityId: DomainEntityID, deltaIds: DeltaID[], deltaFilter?: DeltaFilter): LosslessViewOne | undefined {
     const combinedFilter = (delta: Delta) => {
       if (!deltaIds.includes(delta.id)) {
-        debug(`[${this.rhizomeNode.config.peerId}]`, `Excluding delta ${delta.id} because it's not in the requested list of deltas`);
         return false;
       }
       if (!deltaFilter) return true;

tsconfig.json

@@ -1,16 +1,24 @@
 {
   "compilerOptions": {
-    "target": "ES2022",
+    "target": "ES6",
-    "module": "ESNext",
+    "module": "CommonJS",
     "esModuleInterop": true,
-    "allowSyntheticDefaultImports": true,
     "moduleResolution": "Node",
     "sourceMap": true,
     "baseUrl": ".",
+    "rootDir": ".",
     "outDir": "dist",
+    "importsNotUsedAsValues": "remove",
     "strict": true,
     "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true
+    "forceConsistentCasingInFileNames": true,
+    "types": ["node", "jest"],
+    "typeRoots": [
+      "./node_modules/@types"
+    ],
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": false
   },
   "include": [
     "src/**/*",

util/schemas.ts

@@ -1,4 +1,4 @@
-import { SchemaBuilder } from '../../src/schema';
+import { SchemaBuilder } from '../src/schema';
 
 /**
  * Common schemas used for testing purposes.