Merge pull request 'feature/docker-orchestrator-tests' (#3) from feature/docker-orchestrator-tests into main
Reviewed-on: #3
This commit is contained in:
commit
3a6191a2a2
1
.gitignore
vendored
1
.gitignore
vendored
@ -6,3 +6,4 @@ coverage/
|
||||
.env
|
||||
data/
|
||||
test-data/
|
||||
*.code-workspace
|
||||
|
||||
7
.windsurf/workflows/mr-summary.md
Normal file
7
.windsurf/workflows/mr-summary.md
Normal file
@ -0,0 +1,7 @@
|
||||
---
|
||||
description: Generate a merge request summary
|
||||
---
|
||||
|
||||
- fetch origin/main
|
||||
- compare the current branch to origin/main
|
||||
- generate a merge request summary as an untracked file called merge-${branch}.md
|
||||
26
Dockerfile.test
Normal file
26
Dockerfile.test
Normal file
@ -0,0 +1,26 @@
|
||||
FROM node:24
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files first for better layer caching
|
||||
COPY package*.json tsconfig.json ./
|
||||
|
||||
# Install dependencies including devDependencies
|
||||
RUN npm ci --include=dev
|
||||
|
||||
# Copy source files
|
||||
COPY src/ src/
|
||||
COPY markdown/ markdown/
|
||||
COPY examples/ examples/
|
||||
COPY util/ util/
|
||||
COPY README.md ./
|
||||
|
||||
# Build the application
|
||||
RUN npm run build
|
||||
|
||||
# Set environment variables
|
||||
ENV NODE_ENV=test
|
||||
|
||||
# Command to run the application
|
||||
CMD ["node", "dist/examples/app.js"]
|
||||
20
__tests__/debug/console-test.ts
Normal file
20
__tests__/debug/console-test.ts
Normal file
@ -0,0 +1,20 @@
|
||||
import Debug from 'debug';
|
||||
|
||||
// Set up debug instances for different log levels
|
||||
const debug = Debug('rz:test:console');
|
||||
const debugError = Debug('rz:test:console:error');
|
||||
const debugWarn = Debug('rz:test:console:warn');
|
||||
|
||||
// Test debug output
|
||||
// Note: These will only show if DEBUG=rz:* is set in the environment
|
||||
debug('=== DEBUG LOG TEST ===');
|
||||
debug('This is a test debug message');
|
||||
debugError('This is a test error message');
|
||||
debugWarn('This is a test warning message');
|
||||
|
||||
describe('Debug Test', () => {
|
||||
it('should output debug messages when DEBUG is enabled', () => {
|
||||
debug('Test debug message from inside test');
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
22
__tests__/jest-setup.ts
Normal file
22
__tests__/jest-setup.ts
Normal file
@ -0,0 +1,22 @@
|
||||
// Set up environment variables for tests
|
||||
process.env.DEBUG = 'rz:*';
|
||||
|
||||
// Extend the global Jest namespace
|
||||
declare global {
|
||||
namespace jest {
|
||||
interface Matchers<R> {
|
||||
toBeWithinRange(a: number, b: number): R;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add any global test setup here
|
||||
|
||||
// This is a placeholder test to satisfy Jest's requirement for at least one test
|
||||
describe('Test Setup', () => {
|
||||
it('should pass', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
export {}; // This file needs to be a module
|
||||
@ -8,7 +8,10 @@
|
||||
* - Circular reference handling at scale
|
||||
*/
|
||||
|
||||
import Debug from 'debug';
|
||||
import { RhizomeNode } from '../src/node';
|
||||
|
||||
const debug = Debug('rz:test:nested-resolution-performance');
|
||||
import { Delta } from '../src/core';
|
||||
import { DefaultSchemaRegistry } from '../src/schema';
|
||||
import { SchemaBuilder, PrimitiveSchemas, ReferenceSchemas, ArraySchemas } from '../src/schema';
|
||||
@ -124,7 +127,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
}
|
||||
|
||||
const setupTime = performance.now() - startSetup;
|
||||
console.log(`Setup time for ${userCount} users with relationships: ${setupTime.toFixed(2)}ms`);
|
||||
debug(`Setup time for ${userCount} users with relationships: ${setupTime.toFixed(2)}ms`);
|
||||
|
||||
// Test resolution performance for a user with many connections
|
||||
const testUserId = userIds[50]; // Pick a user in the middle
|
||||
@ -141,7 +144,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
);
|
||||
|
||||
const resolutionTime = performance.now() - startResolution;
|
||||
console.log(`Resolution time for user with many connections: ${resolutionTime.toFixed(2)}ms`);
|
||||
debug(`Resolution time for user with many connections: ${resolutionTime.toFixed(2)}ms`);
|
||||
|
||||
// Verify the resolution worked
|
||||
expect(nestedView.id).toBe(testUserId);
|
||||
@ -155,7 +158,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
const totalNestedObjects = Object.values(nestedView.nestedObjects).reduce(
|
||||
(total, arr) => total + (arr?.length || 0), 0
|
||||
);
|
||||
console.log('Total nested objects resolved:', totalNestedObjects);
|
||||
debug('Total nested objects resolved: %o', totalNestedObjects);
|
||||
|
||||
// The test user should have friends, followers, and possibly a mentor
|
||||
expect(Object.keys(nestedView.nestedObjects).length).toBeGreaterThan(0);
|
||||
@ -218,7 +221,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
}
|
||||
|
||||
const setupTime = performance.now() - startSetup;
|
||||
console.log(`Setup time for chain of ${chainLength} users: ${setupTime.toFixed(2)}ms`);
|
||||
debug(`Setup time for chain of ${chainLength} users: ${setupTime.toFixed(2)}ms`);
|
||||
|
||||
// Test resolution from the start of the chain
|
||||
const firstUserId = userIds[0];
|
||||
@ -235,7 +238,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
);
|
||||
|
||||
const resolutionTime = performance.now() - startResolution;
|
||||
console.log(`Resolution time for deep chain (maxDepth=5): ${resolutionTime.toFixed(2)}ms`);
|
||||
debug(`Resolution time for deep chain (maxDepth=5): ${resolutionTime.toFixed(2)}ms`);
|
||||
|
||||
// Verify the resolution worked and respected depth limits
|
||||
expect(nestedView.id).toBe(firstUserId);
|
||||
@ -255,7 +258,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
}
|
||||
|
||||
expect(depth).toBeLessThanOrEqual(5);
|
||||
console.log(`Actual resolved depth: ${depth}`);
|
||||
debug(`Actual resolved depth: ${depth}`);
|
||||
});
|
||||
|
||||
it('should handle circular references in large graphs without performance degradation', async () => {
|
||||
@ -318,7 +321,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
}
|
||||
|
||||
const setupTime = performance.now() - startSetup;
|
||||
console.log(`Setup time for circular graph with ${userCount} users: ${setupTime.toFixed(2)}ms`);
|
||||
debug(`Setup time for circular graph with ${userCount} users: ${setupTime.toFixed(2)}ms`);
|
||||
|
||||
// Test resolution performance with circular references
|
||||
const testUserId = userIds[0];
|
||||
@ -335,7 +338,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
);
|
||||
|
||||
const resolutionTime = performance.now() - startResolution;
|
||||
console.log(`Resolution time for circular graph (maxDepth=3): ${resolutionTime.toFixed(2)}ms`);
|
||||
debug(`Resolution time for circular graph (maxDepth=3): ${resolutionTime.toFixed(2)}ms`);
|
||||
|
||||
// Verify the resolution completed without hanging
|
||||
expect(nestedView.id).toBe(testUserId);
|
||||
@ -352,7 +355,7 @@ describe('Nested Object Resolution Performance', () => {
|
||||
expect(nestedView.nestedObjects.connections.length).toBeLessThanOrEqual(3);
|
||||
}
|
||||
|
||||
console.log(`Connections resolved: ${nestedView.nestedObjects.connections?.length || 0}`);
|
||||
debug(`Connections resolved: ${nestedView.nestedObjects.connections?.length || 0}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -13,7 +13,7 @@ import { RhizomeNode } from '../src/node';
|
||||
import { Delta } from '../src/core';
|
||||
import { DefaultSchemaRegistry } from '../src/schema';
|
||||
import { SchemaBuilder, PrimitiveSchemas, ReferenceSchemas } from '../src/schema';
|
||||
import { CommonSchemas } from '../src/test-utils/schemas';
|
||||
import { CommonSchemas } from '../util/schemas';
|
||||
import { TypedCollectionImpl } from '../src/collections';
|
||||
|
||||
describe('Nested Object Resolution', () => {
|
||||
|
||||
@ -2,7 +2,7 @@ import { QueryEngine } from '../src/query';
|
||||
import { Lossless } from '../src/views';
|
||||
import { DefaultSchemaRegistry } from '../src/schema';
|
||||
import { SchemaBuilder, PrimitiveSchemas } from '../src/schema';
|
||||
import { CommonSchemas } from '../src/test-utils/schemas';
|
||||
import { CommonSchemas } from '../util/schemas';
|
||||
import { Delta } from '../src/core';
|
||||
import { RhizomeNode } from '../src/node';
|
||||
|
||||
|
||||
72
__tests__/run/001-single-node-orchestrated.ts
Normal file
72
__tests__/run/001-single-node-orchestrated.ts
Normal file
@ -0,0 +1,72 @@
|
||||
import { createOrchestrator, type NodeConfig } from '../../src/orchestration';
|
||||
|
||||
// Increase test timeout to 30 seconds
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('Run (Orchestrated)', () => {
|
||||
const orchestrator = createOrchestrator('in-memory');
|
||||
let nodeHandle: any;
|
||||
let apiUrl: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
console.time('Test setup');
|
||||
console.time('Create config');
|
||||
// Configure and start the node
|
||||
const config: NodeConfig = {
|
||||
id: 'app-001',
|
||||
};
|
||||
console.timeEnd('Create config');
|
||||
|
||||
console.time('Start node');
|
||||
nodeHandle = await orchestrator.startNode(config);
|
||||
console.timeEnd('Start node');
|
||||
|
||||
console.time('Get API URL');
|
||||
apiUrl = nodeHandle.getApiUrl();
|
||||
console.timeEnd('Get API URL');
|
||||
console.timeEnd('Test setup');
|
||||
}, 60000); // Increase timeout to 60s for this hook
|
||||
|
||||
afterAll(async () => {
|
||||
// Stop the node
|
||||
if (nodeHandle) {
|
||||
await orchestrator.stopNode(nodeHandle);
|
||||
}
|
||||
});
|
||||
|
||||
it('can put a new user and fetch it', async () => {
|
||||
// Create a new record
|
||||
const createResponse = await fetch(`${apiUrl}/user`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
id: 'peon-1',
|
||||
properties: {
|
||||
name: 'Peon',
|
||||
age: 263,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const createdUser = await createResponse.json();
|
||||
expect(createdUser).toMatchObject({
|
||||
id: 'peon-1',
|
||||
properties: {
|
||||
name: 'Peon',
|
||||
age: 263,
|
||||
},
|
||||
});
|
||||
|
||||
// Read the created record
|
||||
const getResponse = await fetch(`${apiUrl}/user/peon-1`);
|
||||
const fetchedUser = await getResponse.json();
|
||||
|
||||
expect(fetchedUser).toMatchObject({
|
||||
id: 'peon-1',
|
||||
properties: {
|
||||
name: 'Peon',
|
||||
age: 263,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
134
__tests__/run/002-two-nodes-orchestrated.ts
Normal file
134
__tests__/run/002-two-nodes-orchestrated.ts
Normal file
@ -0,0 +1,134 @@
|
||||
import Debug from 'debug';
|
||||
import { createOrchestrator } from '../../src/orchestration';
|
||||
import type { NodeConfig, NodeHandle } from '../../src/orchestration';
|
||||
|
||||
// Increase test timeout to 30 seconds
|
||||
jest.setTimeout(30000);
|
||||
|
||||
const debug = Debug('test:two-orchestrated');
|
||||
|
||||
describe('Run (Two Nodes Orchestrated)', () => {
|
||||
const orchestrator = createOrchestrator('in-memory');
|
||||
// Define a type that includes all required methods
|
||||
type FullNodeHandle = NodeHandle & {
|
||||
getRequestPort: () => number;
|
||||
getApiUrl: () => string;
|
||||
};
|
||||
|
||||
const nodes: FullNodeHandle[] = [];
|
||||
const nodeIds = ['app-002-A', 'app-002-B'];
|
||||
|
||||
beforeAll(async () => {
|
||||
console.time('Test setup');
|
||||
|
||||
// Start first node
|
||||
console.time('Create node1 config');
|
||||
const node1Config: NodeConfig = {
|
||||
id: nodeIds[0],
|
||||
};
|
||||
console.timeEnd('Create node1 config');
|
||||
|
||||
console.time('Start node1');
|
||||
const node1 = (await orchestrator.startNode(node1Config)) as FullNodeHandle;
|
||||
console.timeEnd('Start node1');
|
||||
|
||||
// Start second node with first node as bootstrap peer
|
||||
console.time('Create node2 config');
|
||||
const node2Config: NodeConfig = {
|
||||
id: nodeIds[1],
|
||||
network: {
|
||||
bootstrapPeers: [`localhost:${node1.getRequestPort()}`],
|
||||
},
|
||||
};
|
||||
console.timeEnd('Create node2 config');
|
||||
|
||||
console.time('Start node2');
|
||||
const node2 = (await orchestrator.startNode(node2Config)) as FullNodeHandle;
|
||||
console.timeEnd('Start node2');
|
||||
|
||||
nodes.push(node1, node2);
|
||||
|
||||
// Connect the nodes
|
||||
console.time('Connect nodes');
|
||||
await orchestrator.connectNodes(node1, node2);
|
||||
console.timeEnd('Connect nodes');
|
||||
|
||||
console.timeEnd('Test setup');
|
||||
}, 120000); // Increase timeout to 120s for this hook
|
||||
|
||||
afterAll(async () => {
|
||||
// Stop all nodes in parallel
|
||||
await Promise.all(nodes.map(node => node && orchestrator.stopNode(node)));
|
||||
});
|
||||
|
||||
it('can create a record on node0 and read it from node1', async () => {
|
||||
const [node0, node1] = nodes;
|
||||
const node0Url = node0.getApiUrl();
|
||||
const node1Url = node1.getApiUrl();
|
||||
|
||||
debug(`Node 0 URL: ${node0Url}`);
|
||||
debug(`Node 1 URL: ${node1Url}`);
|
||||
|
||||
// Create a new record on node0
|
||||
const createResponse = await fetch(`${node0Url}/user`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
id: 'peon-1',
|
||||
properties: {
|
||||
name: 'Peon',
|
||||
age: 741,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const createdUser = await createResponse.json();
|
||||
expect(createdUser).toMatchObject({
|
||||
id: 'peon-1',
|
||||
properties: {
|
||||
name: 'Peon',
|
||||
age: 741,
|
||||
},
|
||||
});
|
||||
|
||||
// Small delay to allow for synchronization
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
// Read the record from node1
|
||||
const getResponse = await fetch(`${node1Url}/user/peon-1`);
|
||||
const fetchedUser = await getResponse.json();
|
||||
|
||||
expect(fetchedUser).toMatchObject({
|
||||
id: 'peon-1',
|
||||
properties: {
|
||||
name: 'Peon',
|
||||
age: 741,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.skip('can demonstrate network partitioning', async () => {
|
||||
// This test shows how we can simulate network partitions
|
||||
// For now, it's just a placeholder since we'd need to implement
|
||||
// the actual partitioning logic in the InMemoryOrchestrator
|
||||
const [node0, node1] = nodes;
|
||||
|
||||
// Simulate partition (actual implementation would use orchestrator.partitionNetwork)
|
||||
debug('Simulating network partition between nodes');
|
||||
// await orchestrator.partitionNetwork({
|
||||
// groups: [[node0.id], [node1.id]]
|
||||
// });
|
||||
|
||||
// Test behavior during partition...
|
||||
|
||||
// Heal partition
|
||||
// await orchestrator.partitionNetwork({
|
||||
// groups: [[node0.id, node1.id]]
|
||||
// });
|
||||
|
||||
// Test behavior after healing...
|
||||
|
||||
// Mark test as passed (remove once actual test is implemented)
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@ -17,7 +17,7 @@ describe('Run', () => {
|
||||
apps[0].config.seedPeers.push(apps[1].myRequestAddr);
|
||||
apps[1].config.seedPeers.push(apps[0].myRequestAddr);
|
||||
|
||||
await Promise.all(apps.map((app) => app.start(false)));
|
||||
await Promise.all(apps.map((app) => app.start()));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
||||
531
__tests__/run/005-docker-orchestrator.ts
Normal file
531
__tests__/run/005-docker-orchestrator.ts
Normal file
@ -0,0 +1,531 @@
|
||||
import Docker from 'dockerode';
|
||||
import { describe, it, beforeAll, afterAll, expect, jest } from '@jest/globals';
|
||||
import Debug from 'debug';
|
||||
|
||||
const debug = Debug('rz:test:docker-orchestrator-v2');
|
||||
import { createOrchestrator } from '../../src/orchestration';
|
||||
import type { NodeOrchestrator, NodeConfig, NodeHandle, NodeStatus } from '../../src/orchestration';
|
||||
import { ImageManager } from '../../src/orchestration/docker-orchestrator/managers/image-manager';
|
||||
|
||||
// Extend the NodeOrchestrator type to include the docker client for DockerOrchestrator
|
||||
interface DockerOrchestrator extends NodeOrchestrator {
|
||||
docker: Docker;
|
||||
}
|
||||
|
||||
// Extended interface to include additional properties that might be present in the implementation
|
||||
interface ExtendedNodeStatus extends Omit<NodeStatus, 'network'> {
|
||||
network?: {
|
||||
address: string;
|
||||
port: number; // Changed from httpPort to match NodeStatus
|
||||
requestPort: number;
|
||||
peers: string[];
|
||||
bootstrapPeers?: string[];
|
||||
containerId?: string;
|
||||
networkId?: string;
|
||||
};
|
||||
getApiUrl?: () => string;
|
||||
}
|
||||
|
||||
// Simple test to verify Docker is working
|
||||
// Set default timeout for all tests to 5 minutes
|
||||
jest.setTimeout(300000);
|
||||
|
||||
describe('Docker Orchestrator V2', () => {
|
||||
let docker: Docker;
|
||||
let orchestrator: DockerOrchestrator;
|
||||
let node: NodeHandle | null = null;
|
||||
let node2: NodeHandle | null = null;
|
||||
let nodeConfig: NodeConfig;
|
||||
let node2Config: NodeConfig;
|
||||
let nodePort: number;
|
||||
let node2Port: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
debug('Setting up Docker client and orchestrator...');
|
||||
|
||||
// Initialize Docker client
|
||||
docker = new Docker();
|
||||
|
||||
// Verify Docker is running
|
||||
try {
|
||||
await docker.ping();
|
||||
debug('Docker daemon is responding');
|
||||
} catch (error) {
|
||||
debug('Docker daemon is not responding: %o', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Initialize the orchestrator with the Docker client and test image
|
||||
orchestrator = createOrchestrator('docker') as DockerOrchestrator;
|
||||
debug('Docker orchestrator initialized');
|
||||
|
||||
// Create a basic node config for testing
|
||||
nodePort = 3000 + Math.floor(Math.random() * 1000);
|
||||
nodeConfig = {
|
||||
id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
|
||||
networkId: 'test-network',
|
||||
port: nodePort,
|
||||
resources: {
|
||||
memory: 256, // 256MB
|
||||
cpu: 0.5 // 0.5 CPU
|
||||
}
|
||||
};
|
||||
|
||||
debug(`Test node configured with ID: ${nodeConfig.id}, port: ${nodePort}`);
|
||||
|
||||
const imageManager = new ImageManager();
|
||||
await imageManager.buildTestImage();
|
||||
}); // 30 second timeout
|
||||
|
||||
afterAll(async () => {
|
||||
debug('Starting test cleanup...');
|
||||
const cleanupPromises: Promise<unknown>[] = [];
|
||||
|
||||
// Helper function to clean up a node with retries
|
||||
const cleanupNode = async (nodeToClean: NodeHandle | null, nodeName: string) => {
|
||||
if (!nodeToClean) return;
|
||||
|
||||
debug(`[${nodeName}] Starting cleanup for node ${nodeToClean.id}...`);
|
||||
try {
|
||||
// First try the normal stop
|
||||
await orchestrator.stopNode(nodeToClean).catch(error => {
|
||||
debug(`[${nodeName}] Warning stopping node normally: %s`, error.message);
|
||||
throw error; // Will be caught by outer catch
|
||||
});
|
||||
debug(`[${nodeName}] Node ${nodeToClean.id} stopped gracefully`);
|
||||
} catch (error) {
|
||||
debug(`[${nodeName}] Error stopping node ${nodeToClean.id}: %o`, error);
|
||||
|
||||
// If normal stop fails, try force cleanup
|
||||
try {
|
||||
debug(`[${nodeName}] Attempting force cleanup for node ${nodeToClean.id}...`);
|
||||
const container = orchestrator.docker.getContainer(`rhizome-${nodeToClean.id}`);
|
||||
await container.stop({ t: 1 }).catch(() => {
|
||||
debug(`[${nodeName}] Container stop timed out, forcing removal...`);
|
||||
});
|
||||
await container.remove({ force: true });
|
||||
debug(`[${nodeName}] Node ${nodeToClean.id} force-removed`);
|
||||
} catch (forceError) {
|
||||
debug(`[${nodeName}] Force cleanup failed for node ${nodeToClean.id}: %o`, forceError);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Clean up all created nodes
|
||||
if (node) {
|
||||
cleanupPromises.push(cleanupNode(node, 'node1'));
|
||||
}
|
||||
|
||||
if (node2) {
|
||||
cleanupPromises.push(cleanupNode(node2, 'node2'));
|
||||
}
|
||||
|
||||
// Wait for all node cleanups to complete before cleaning up networks
|
||||
if (cleanupPromises.length > 0) {
|
||||
debug('Waiting for node cleanups to complete...');
|
||||
await Promise.race([
|
||||
Promise.all(cleanupPromises),
|
||||
new Promise(resolve => setTimeout(() => {
|
||||
debug('Node cleanup timed out, proceeding with network cleanup...');
|
||||
resolve(null);
|
||||
}, 30000)) // 30s timeout for node cleanup
|
||||
]);
|
||||
}
|
||||
|
||||
// Clean up any dangling networks using NetworkManager
|
||||
try {
|
||||
debug('Cleaning up networks...');
|
||||
// Get the network manager from the orchestrator
|
||||
const networkManager = (orchestrator as any).networkManager;
|
||||
if (!networkManager) {
|
||||
debug('Network manager not available for cleanup');
|
||||
return;
|
||||
}
|
||||
|
||||
// Get all networks managed by this test
|
||||
const networks = Array.from((orchestrator as any).networks.entries() || []);
|
||||
|
||||
const cleanupResults = await networkManager.cleanupNetworks((orchestrator as any).networks);
|
||||
|
||||
// Log any cleanup errors
|
||||
cleanupResults.forEach(({ resource, error }: { resource: string; error: Error }) => {
|
||||
if (error) {
|
||||
debug(`Failed to clean up network ${resource || 'unknown'}: %s`, error.message);
|
||||
} else {
|
||||
debug(`Successfully cleaned up network ${resource || 'unknown'}`);
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
debug('Error during network cleanup: %o', error);
|
||||
}
|
||||
|
||||
debug('All test cleanups completed');
|
||||
}, 120000); // 2 minute timeout for afterAll
|
||||
|
||||
it('should start and stop a node', async () => {
|
||||
debug('Starting test: should start and stop a node');
|
||||
|
||||
// Create a new config with a unique ID for this test
|
||||
const testNodeConfig = {
|
||||
...nodeConfig,
|
||||
id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
|
||||
network: {
|
||||
...nodeConfig.network,
|
||||
enableHttpApi: true
|
||||
}
|
||||
};
|
||||
|
||||
// Start a node
|
||||
debug('Starting node...');
|
||||
const testNode = await orchestrator.startNode(testNodeConfig);
|
||||
expect(testNode).toBeDefined();
|
||||
expect(testNode.id).toBeDefined();
|
||||
debug(`✅ Node started with ID: ${testNode.id}`);
|
||||
|
||||
try {
|
||||
// Verify the node is running
|
||||
const status = await testNode.status();
|
||||
expect(status).toBeDefined();
|
||||
debug('Node status: %o', status);
|
||||
|
||||
// Verify we can access the health endpoint
|
||||
const apiUrl = testNode.getApiUrl?.();
|
||||
if (apiUrl) {
|
||||
const response = await fetch(`${apiUrl}/health`);
|
||||
expect(response.ok).toBe(true);
|
||||
const health = await response.json();
|
||||
expect(health).toHaveProperty('status', 'ok');
|
||||
}
|
||||
|
||||
// Stop the node
|
||||
debug('Stopping node...');
|
||||
await orchestrator.stopNode(testNode);
|
||||
debug('Node stopped');
|
||||
} finally {
|
||||
// Ensure node is cleaned up even if test fails
|
||||
try {
|
||||
await orchestrator.stopNode(testNode).catch(() => {});
|
||||
} catch (e) {
|
||||
debug('Error during node cleanup: %o', e);
|
||||
}
|
||||
}
|
||||
}, 30000); // 30 second timeout for this test
|
||||
|
||||
it('should enforce resource limits', async () => {
|
||||
debug('Starting test: should enforce resource limits');
|
||||
|
||||
// Create a new node with a unique ID for this test
|
||||
const testNodeConfig = {
|
||||
...nodeConfig,
|
||||
id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
|
||||
resources: {
|
||||
memory: 256, // 256MB
|
||||
cpu: 0.5 // 0.5 CPU
|
||||
},
|
||||
network: {
|
||||
...nodeConfig.network,
|
||||
enableHttpApi: true
|
||||
}
|
||||
};
|
||||
|
||||
let testNode: NodeHandle | null = null;
|
||||
|
||||
try {
|
||||
// Start the node with resource limits
|
||||
testNode = await orchestrator.startNode(testNodeConfig);
|
||||
debug(`Node started with ID: ${testNode.id}`);
|
||||
|
||||
// Get container info to verify resource limits
|
||||
const status = await testNode.status() as ExtendedNodeStatus;
|
||||
|
||||
// Verify container ID is available at the root level
|
||||
if (!status.containerId) {
|
||||
throw new Error('Container ID not available in node status');
|
||||
}
|
||||
|
||||
// Get the container ID from the node status
|
||||
if (!status.containerId) {
|
||||
throw new Error('Container ID not available in node status');
|
||||
}
|
||||
|
||||
// Get container info using ContainerManager
|
||||
const container = await (orchestrator as any).containerManager.getContainer(status.containerId);
|
||||
if (!container) {
|
||||
throw new Error('Container not found');
|
||||
}
|
||||
|
||||
// Get container info
|
||||
const containerInfo = await container.inspect();
|
||||
|
||||
// Log container info for debugging
|
||||
debug('Container info: %o', {
|
||||
Memory: containerInfo.HostConfig?.Memory,
|
||||
NanoCpus: containerInfo.HostConfig?.NanoCpus,
|
||||
CpuQuota: containerInfo.HostConfig?.CpuQuota,
|
||||
CpuPeriod: containerInfo.HostConfig?.CpuPeriod
|
||||
});
|
||||
|
||||
// Check memory limit (in bytes)
|
||||
expect(containerInfo.HostConfig?.Memory).toBe(256 * 1024 * 1024);
|
||||
|
||||
// Check CPU limit (can be set as NanoCpus or CpuQuota/CpuPeriod)
|
||||
const expectedCpuNano = 0.5 * 1e9; // 0.5 CPU in nanoCPUs
|
||||
const actualCpuNano = containerInfo.HostConfig?.NanoCpus;
|
||||
|
||||
// Some Docker versions use CpuQuota/CpuPeriod instead of NanoCpus
|
||||
if (actualCpuNano === undefined && containerInfo.HostConfig?.CpuQuota && containerInfo.HostConfig?.CpuPeriod) {
|
||||
const cpuQuota = containerInfo.HostConfig.CpuQuota;
|
||||
const cpuPeriod = containerInfo.HostConfig.CpuPeriod;
|
||||
const calculatedCpu = (cpuQuota / cpuPeriod) * 1e9;
|
||||
expect(Math.round(calculatedCpu)).toBeCloseTo(Math.round(expectedCpuNano), -8); // Allow for small rounding differences
|
||||
} else {
|
||||
expect(actualCpuNano).toBe(expectedCpuNano);
|
||||
}
|
||||
|
||||
debug('Resource limits verified');
|
||||
} finally {
|
||||
// Clean up the test node
|
||||
if (testNode) {
|
||||
try {
|
||||
await orchestrator.stopNode(testNode);
|
||||
} catch (e) {
|
||||
debug('Error cleaning up test node: %o', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
it('should expose API endpoints', async () => {
|
||||
// Set a longer timeout for this test (5 minutes)
|
||||
jest.setTimeout(300000);
|
||||
debug('Starting test: should expose API endpoints');
|
||||
|
||||
// Create a new node with a unique ID for this test
|
||||
const testNodeConfig = {
|
||||
...nodeConfig,
|
||||
id: `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
|
||||
// Ensure HTTP API is enabled
|
||||
network: {
|
||||
...nodeConfig.network,
|
||||
enableHttpApi: true
|
||||
}
|
||||
};
|
||||
|
||||
// Start the node
|
||||
debug('Attempting to start node with config: %o', testNodeConfig);
|
||||
const node = await orchestrator.startNode(testNodeConfig);
|
||||
debug(`Node started with ID: ${node.id}`);
|
||||
|
||||
const apiUrl = node.getApiUrl?.();
|
||||
// Helper function to test API endpoint with retries
|
||||
const testApiEndpoint = async (endpoint: string, expectedStatus = 200, maxRetries = 5, retryDelay = 1000) => {
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
debug(`Attempt ${attempt}/${maxRetries} - Testing ${endpoint}`);
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 5000);
|
||||
const response = await fetch(`${apiUrl}${endpoint}`, {
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
signal: controller.signal
|
||||
});
|
||||
clearTimeout(timeout);
|
||||
|
||||
if (response.status === expectedStatus) {
|
||||
debug(`${endpoint} returned status ${response.status}`);
|
||||
return await response.json().catch(() => ({}));
|
||||
}
|
||||
|
||||
const errorText = await response.text().catch(() => 'No response body');
|
||||
throw new Error(`Expected status ${expectedStatus}, got ${response.status}: ${errorText}`);
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
debug(`Attempt ${attempt} failed: %o`, error);
|
||||
|
||||
if (attempt < maxRetries) {
|
||||
await new Promise(resolve => setTimeout(resolve, retryDelay * attempt));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`API endpoint test failed after ${maxRetries} attempts: ${lastError?.message}`);
|
||||
};
|
||||
|
||||
try {
|
||||
// Test the health endpoint
|
||||
debug('Testing health endpoint...');
|
||||
const healthData = await testApiEndpoint('/health');
|
||||
expect(healthData).toHaveProperty('status');
|
||||
expect(healthData.status).toBe('ok');
|
||||
|
||||
debug('All API endpoints verified');
|
||||
} catch (error) {
|
||||
// Log container logs if available
|
||||
try {
|
||||
const container = docker.getContainer(`rhizome-${node.id}`);
|
||||
const logs = await container.logs({
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
tail: 100
|
||||
});
|
||||
debug('Container logs: %s', logs.toString('utf8'));
|
||||
} catch (logError) {
|
||||
debug('Failed to get container logs: %o', logError);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
it.skip('should connect two nodes', async () => {
|
||||
debug('Starting test: should connect two nodes');
|
||||
|
||||
// Create unique configs for both nodes
|
||||
const node1Port = 3000 + Math.floor(Math.random() * 1000);
|
||||
const node2Port = node1Port + 1;
|
||||
const networkId = `test-network-${Date.now()}`;
|
||||
|
||||
const node1Config: NodeConfig = {
|
||||
id: `test-node-1-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
|
||||
networkId,
|
||||
network: {
|
||||
port: node1Port,
|
||||
requestPort: node1Port + 1000, // Different port for request API
|
||||
bootstrapPeers: []
|
||||
},
|
||||
resources: {
|
||||
memory: 256,
|
||||
cpu: 0.5
|
||||
}
|
||||
};
|
||||
|
||||
const node2Config: NodeConfig = {
|
||||
id: `test-node-2-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
|
||||
networkId,
|
||||
network: {
|
||||
port: node2Port,
|
||||
requestPort: node2Port + 1000, // Different port for request API
|
||||
bootstrapPeers: [`/ip4/127.0.0.1/tcp/${node1Port + 1000}`]
|
||||
},
|
||||
resources: {
|
||||
memory: 256,
|
||||
cpu: 0.5
|
||||
}
|
||||
};
|
||||
|
||||
let node1: NodeHandle | null = null;
|
||||
let node2: NodeHandle | null = null;
|
||||
|
||||
try {
|
||||
// Start first node
|
||||
debug('Starting node 1...');
|
||||
node1 = await orchestrator.startNode(node1Config);
|
||||
debug(`Node 1 started with ID: ${node1.id}`);
|
||||
|
||||
// Get node 1's status and API URL
|
||||
const status1 = await node1.status() as ExtendedNodeStatus;
|
||||
const node1ApiUrl = node1.getApiUrl?.();
|
||||
|
||||
// Update node 2's config with node 1's actual address if available
|
||||
if (status1.network?.address && node2Config.network) {
|
||||
// This assumes the address is in a format like /ip4/127.0.0.1/tcp/3001
|
||||
node2Config.network.bootstrapPeers = [status1.network.address];
|
||||
}
|
||||
|
||||
// Start second node
|
||||
debug('Starting node 2...');
|
||||
node2 = await orchestrator.startNode(node2Config);
|
||||
debug(`Node 2 started with ID: ${node2.id}`);
|
||||
|
||||
// Get node 2's status
|
||||
const status2 = await node2.status() as ExtendedNodeStatus;
|
||||
const node2ApiUrl = node2.getApiUrl?.();
|
||||
|
||||
// Verify both nodes are running
|
||||
expect(status1).toBeDefined();
|
||||
expect(status2).toBeDefined();
|
||||
// TODO: this status check is inadequate
|
||||
debug('Both nodes are running');
|
||||
|
||||
// Helper function to wait for peers
|
||||
const waitForPeers = async (nodeHandle: NodeHandle, expectedPeerCount = 1, maxAttempts = 10) => {
|
||||
for (let i = 0; i < maxAttempts; i++) {
|
||||
const status = await nodeHandle.status() as ExtendedNodeStatus;
|
||||
const peerCount = status.network?.peers?.length || 0;
|
||||
|
||||
if (peerCount >= expectedPeerCount) {
|
||||
debug(`Found ${peerCount} peers after ${i + 1} attempts`);
|
||||
return true;
|
||||
}
|
||||
|
||||
debug(`Waiting for peers... (attempt ${i + 1}/${maxAttempts})`);
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// Wait for nodes to discover each other
|
||||
debug('Waiting for nodes to discover each other...');
|
||||
const node1Discovered = await waitForPeers(node1);
|
||||
const node2Discovered = await waitForPeers(node2);
|
||||
|
||||
// Final status check
|
||||
const finalStatus1 = await node1.status() as ExtendedNodeStatus;
|
||||
const finalStatus2 = await node2.status() as ExtendedNodeStatus;
|
||||
|
||||
// Log peer information
|
||||
debug('Node 1 discovered: %o', node1Discovered);
|
||||
debug('Node 2 discovered: %o', node2Discovered);
|
||||
debug('Node 1 peers: %o', finalStatus1.network?.peers || 'none');
|
||||
debug('Node 2 peers: %o', finalStatus2.network?.peers || 'none');
|
||||
debug('Node 1 bootstrapPeers: %o', finalStatus1.network?.bootstrapPeers || 'none');
|
||||
debug('Node 2 bootstrapPeers: %o', finalStatus2.network?.bootstrapPeers || 'none');
|
||||
|
||||
// Log the addresses for debugging
|
||||
debug('Node 1 address: %o', finalStatus1.network?.address);
|
||||
debug('Node 2 address: %o', finalStatus2.network?.address);
|
||||
|
||||
// Verify both nodes have network configuration
|
||||
expect(finalStatus1.network).toBeDefined();
|
||||
expect(finalStatus2.network).toBeDefined();
|
||||
expect(finalStatus1.network?.address).toBeDefined();
|
||||
expect(finalStatus2.network?.address).toBeDefined();
|
||||
|
||||
// For now, we'll just verify that both nodes are running and have network info
|
||||
// In a real test, you would want to verify actual communication between nodes
|
||||
debug('✅ Both nodes are running with network configuration');
|
||||
|
||||
} finally {
|
||||
// Clean up nodes
|
||||
const cleanupPromises = [];
|
||||
|
||||
if (node1) {
|
||||
debug('Stopping node 1...');
|
||||
cleanupPromises.push(
|
||||
orchestrator.stopNode(node1).catch(e =>
|
||||
debug('Error stopping node 1: %o', e)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (node2) {
|
||||
debug('Stopping node 2...');
|
||||
cleanupPromises.push(
|
||||
orchestrator.stopNode(node2).catch(e =>
|
||||
debug('Error stopping node 2: %o', e)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
await Promise.all(cleanupPromises);
|
||||
debug('✅ Both nodes stopped');
|
||||
}
|
||||
|
||||
// Note: In a real test with actual peer connections, we would verify the connection
|
||||
// by having the nodes communicate with each other.
|
||||
}, 60000);
|
||||
});
|
||||
@ -7,7 +7,7 @@ import {
|
||||
ObjectSchema
|
||||
} from '../src/schema';
|
||||
import { DefaultSchemaRegistry } from '../src/schema';
|
||||
import { CommonSchemas } from '../src/test-utils/schemas';
|
||||
import { CommonSchemas } from '../util/schemas';
|
||||
import { TypedCollectionImpl, SchemaValidationError } from '../src/collections';
|
||||
import { RhizomeNode } from '../src/node';
|
||||
import { Delta } from '../src/core';
|
||||
|
||||
98
__tests__/test-utils.ts
Normal file
98
__tests__/test-utils.ts
Normal file
@ -0,0 +1,98 @@
|
||||
import { createOrchestrator } from '../src/orchestration/factory';
|
||||
import { NodeConfig, NodeOrchestrator } from '../src/orchestration/types';
|
||||
import Debug from 'debug';
|
||||
|
||||
const debug = Debug('rz:test-utils');
|
||||
|
||||
// Global test orchestrator instance
|
||||
let testOrchestrator: NodeOrchestrator;
|
||||
|
||||
// Default test node configuration
|
||||
const DEFAULT_TEST_NODE_CONFIG: Partial<NodeConfig> = {
|
||||
network: {
|
||||
// Use default ports that will be overridden by getRandomPort() in the orchestrator
|
||||
port: 0,
|
||||
},
|
||||
storage: {
|
||||
type: 'memory',
|
||||
path: '/data',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Set up the test environment before all tests run
|
||||
*/
|
||||
export const setupTestEnvironment = async () => {
|
||||
debug('Setting up Docker test environment...');
|
||||
|
||||
try {
|
||||
// Create a Docker orchestrator instance
|
||||
testOrchestrator = createOrchestrator('docker', {
|
||||
// Enable auto-building of test images
|
||||
autoBuildTestImage: true,
|
||||
// Use a specific test image name
|
||||
image: 'rhizome-node-test',
|
||||
});
|
||||
|
||||
debug('Docker test environment setup complete');
|
||||
} catch (error) {
|
||||
debug('Error setting up Docker test environment:', error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Clean up the test environment after all tests complete
|
||||
*/
|
||||
export const teardownTestEnvironment = async () => {
|
||||
debug('Tearing down Docker test environment...');
|
||||
|
||||
if (testOrchestrator) {
|
||||
try {
|
||||
// Clean up all containers and networks
|
||||
await testOrchestrator.cleanup();
|
||||
debug('Docker resources cleaned up successfully');
|
||||
} catch (error) {
|
||||
debug('Error during Docker environment teardown:', error);
|
||||
// Don't throw to allow tests to complete
|
||||
}
|
||||
}
|
||||
|
||||
debug('Docker test environment teardown complete');
|
||||
};
|
||||
|
||||
/**
|
||||
* Get the test orchestrator instance
|
||||
*/
|
||||
export const getTestOrchestrator = (): NodeOrchestrator => {
|
||||
if (!testOrchestrator) {
|
||||
throw new Error('Test orchestrator not initialized. Call setupTestEnvironment() first.');
|
||||
}
|
||||
return testOrchestrator;
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a test node with the given configuration
|
||||
*/
|
||||
export const createTestNode = async (config: Partial<NodeConfig> = {}) => {
|
||||
const orchestrator = getTestOrchestrator();
|
||||
|
||||
// Merge default config with provided config
|
||||
const nodeConfig: NodeConfig = {
|
||||
...DEFAULT_TEST_NODE_CONFIG,
|
||||
...config,
|
||||
// Ensure we have a unique ID for each node
|
||||
id: config.id || `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
|
||||
};
|
||||
|
||||
debug(`Creating test node with ID: ${nodeConfig.id}`);
|
||||
|
||||
try {
|
||||
const nodeHandle = await orchestrator.startNode(nodeConfig);
|
||||
debug(`Test node ${nodeConfig.id} created successfully`);
|
||||
return nodeHandle;
|
||||
} catch (error) {
|
||||
debug(`Error creating test node ${nodeConfig.id}:`, error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
105
logs/docker-build.log
Normal file
105
logs/docker-build.log
Normal file
@ -0,0 +1,105 @@
|
||||
[2025-06-18T01:06:06.659Z] ✅ Docker build started, streaming output...
|
||||
[2025-06-18T01:06:06.660Z] [Docker Build] Step 1/11 : FROM node:24
|
||||
[2025-06-18T01:06:06.660Z] [Docker Build]
|
||||
[2025-06-18T01:06:06.660Z] [Docker Build] ---> 755ea2a01757
|
||||
[2025-06-18T01:06:06.660Z] [Docker Build] Step 2/11 : WORKDIR /app
|
||||
[2025-06-18T01:06:06.660Z] [Docker Build]
|
||||
[2025-06-18T01:06:06.661Z] [Docker Build] ---> Using cache
|
||||
[2025-06-18T01:06:06.661Z] [Docker Build] ---> a471eaba1647
|
||||
[2025-06-18T01:06:06.661Z] [Docker Build] Step 3/11 : COPY package.json package-lock.json tsconfig.json ./
|
||||
[2025-06-18T01:06:06.661Z] [Docker Build]
|
||||
[2025-06-18T01:06:06.833Z] [Docker Build] ---> 7c047af2d840
|
||||
[2025-06-18T01:06:06.834Z] [Docker Build] Step 4/11 : RUN npm ci --include=dev
|
||||
[2025-06-18T01:06:06.834Z] [Docker Build]
|
||||
[2025-06-18T01:06:06.934Z] [Docker Build] ---> Running in 49af7c037197
|
||||
[2025-06-18T01:06:10.455Z] [Docker Build] [91mnpm warn deprecated rimraf@3.0.2: Rimraf versions prior to v4 are no longer supported
|
||||
[0m
|
||||
[2025-06-18T01:06:10.734Z] [Docker Build] [91mnpm warn deprecated npmlog@6.0.2: This package is no longer supported.
|
||||
[0m
|
||||
[2025-06-18T01:06:11.395Z] [Docker Build] [91mnpm warn deprecated inflight@1.0.6: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.
|
||||
[0m
|
||||
[2025-06-18T01:06:11.461Z] [Docker Build] [91mnpm warn deprecated glob@7.2.3: Glob versions prior to v9 are no longer supported
|
||||
[0m
|
||||
[2025-06-18T01:06:11.524Z] [Docker Build] [91mnpm warn deprecated gauge@4.0.4: This package is no longer supported.
|
||||
[0m
|
||||
[2025-06-18T01:06:12.230Z] [Docker Build] [91mnpm warn deprecated are-we-there-yet@3.0.1: This package is no longer supported.
|
||||
[0m
|
||||
[2025-06-18T01:06:13.207Z] [Docker Build] [91mnpm warn deprecated @humanwhocodes/object-schema@2.0.3: Use @eslint/object-schema instead
|
||||
[0m
|
||||
[2025-06-18T01:06:13.251Z] [Docker Build] [91mnpm warn deprecated @humanwhocodes/config-array@0.13.0: Use @eslint/config-array instead
|
||||
[0m
|
||||
[2025-06-18T01:06:14.440Z] [Docker Build] [91mnpm warn deprecated eslint@8.57.1: This version is no longer supported. Please see https://eslint.org/version-support for other options.
|
||||
[0m
|
||||
[2025-06-18T01:06:19.569Z] [Docker Build]
|
||||
added 839 packages, and audited 841 packages in 12s
|
||||
[2025-06-18T01:06:19.569Z] [Docker Build] 175 packages are looking for funding
|
||||
run `npm fund` for details
|
||||
[2025-06-18T01:06:19.571Z] [Docker Build]
|
||||
found 0 vulnerabilities
|
||||
[2025-06-18T01:06:19.572Z] [Docker Build] [91mnpm notice
|
||||
npm notice New minor version of npm available! 11.3.0 -> 11.4.2
|
||||
npm notice Changelog: https://github.com/npm/cli/releases/tag/v11.4.2
|
||||
npm notice To update run: npm install -g npm@11.4.2
|
||||
npm notice
|
||||
[0m
|
||||
[2025-06-18T01:06:31.247Z] [Docker Build] ---> Removed intermediate container 49af7c037197
|
||||
[2025-06-18T01:06:31.247Z] [Docker Build] ---> 3db27fed8161
|
||||
[2025-06-18T01:06:31.247Z] [Docker Build] Step 5/11 : COPY src/ src/
|
||||
[2025-06-18T01:06:31.247Z] [Docker Build]
|
||||
[2025-06-18T01:06:31.598Z] [Docker Build] ---> 1ad51b320392
|
||||
[2025-06-18T01:06:31.598Z] [Docker Build] Step 6/11 : COPY markdown/ markdown/
|
||||
[2025-06-18T01:06:31.598Z] [Docker Build]
|
||||
[2025-06-18T01:06:31.736Z] [Docker Build] ---> c52bad2721f7
|
||||
[2025-06-18T01:06:31.736Z] [Docker Build] Step 7/11 : COPY examples/ examples/
|
||||
[2025-06-18T01:06:31.736Z] [Docker Build]
|
||||
[2025-06-18T01:06:31.864Z] [Docker Build] ---> 5a98881e54fb
|
||||
[2025-06-18T01:06:31.865Z] [Docker Build] Step 8/11 : COPY util/ util/
|
||||
[2025-06-18T01:06:31.865Z] [Docker Build]
|
||||
[2025-06-18T01:06:31.986Z] [Docker Build] ---> 862b5fe2ca61
|
||||
[2025-06-18T01:06:31.986Z] [Docker Build] Step 9/11 : RUN npm run build --verbose
|
||||
[2025-06-18T01:06:31.986Z] [Docker Build]
|
||||
[2025-06-18T01:06:32.085Z] [Docker Build] ---> Running in 386a95b55921
|
||||
[2025-06-18T01:06:32.475Z] [Docker Build] [91mnpm verbose cli /usr/local/bin/node /usr/local/bin/npm
|
||||
[0m
|
||||
[2025-06-18T01:06:32.476Z] [Docker Build] [91mnpm info using npm@11.3.0
|
||||
[0m
|
||||
[2025-06-18T01:06:32.476Z] [Docker Build] [91mnpm info using node@v24.2.0
|
||||
[0m
|
||||
[2025-06-18T01:06:32.478Z] [Docker Build] [91mnpm verbose title npm run build
|
||||
npm verbose argv "run" "build" "--loglevel" "verbose"
|
||||
[0m
|
||||
[2025-06-18T01:06:32.478Z] [Docker Build] [91mnpm verbose logfile logs-max:10 dir:/root/.npm/_logs/2025-06-18T01_06_32_444Z-
|
||||
[0m
|
||||
[2025-06-18T01:06:32.502Z] [Docker Build] [91mnpm verbose logfile /root/.npm/_logs/2025-06-18T01_06_32_444Z-debug-0.log
|
||||
[0m
|
||||
[2025-06-18T01:06:32.528Z] [Docker Build]
|
||||
> rhizome-node@0.1.0 build
|
||||
> tsc
|
||||
[2025-06-18T01:06:35.285Z] [Docker Build] [91mnpm verbose cwd /app
|
||||
[0m
|
||||
[2025-06-18T01:06:35.285Z] [Docker Build] [91mnpm verbose os Linux 6.8.0-60-generic
|
||||
[0m
|
||||
[2025-06-18T01:06:35.285Z] [Docker Build] [91mnpm verbose node v24.2.0
|
||||
[0m
|
||||
[2025-06-18T01:06:35.285Z] [Docker Build] [91mnpm verbose npm v11.3.0
|
||||
[0m
|
||||
[2025-06-18T01:06:35.286Z] [Docker Build] [91mnpm verbose exit 0
|
||||
[0m
|
||||
[2025-06-18T01:06:35.286Z] [Docker Build] [91mnpm info ok
|
||||
[0m
|
||||
[2025-06-18T01:06:35.874Z] [Docker Build] ---> Removed intermediate container 386a95b55921
|
||||
[2025-06-18T01:06:35.874Z] [Docker Build] ---> 694f414f6cdb
|
||||
[2025-06-18T01:06:35.874Z] [Docker Build] Step 10/11 : ENV NODE_ENV=test
|
||||
[2025-06-18T01:06:35.874Z] [Docker Build]
|
||||
[2025-06-18T01:06:36.003Z] [Docker Build] ---> Running in facd3d3ab07a
|
||||
[2025-06-18T01:06:36.124Z] [Docker Build] ---> Removed intermediate container facd3d3ab07a
|
||||
[2025-06-18T01:06:36.124Z] [Docker Build] ---> 3eb20e31ad6a
|
||||
[2025-06-18T01:06:36.124Z] [Docker Build] Step 11/11 : CMD ["node", "dist/examples/app.js"]
|
||||
[2025-06-18T01:06:36.124Z] [Docker Build]
|
||||
[2025-06-18T01:06:36.225Z] [Docker Build] ---> Running in 3c6e1a89fadb
|
||||
[2025-06-18T01:06:36.329Z] [Docker Build] ---> Removed intermediate container 3c6e1a89fadb
|
||||
[2025-06-18T01:06:36.329Z] [Docker Build] ---> 66da6b5995cc
|
||||
[2025-06-18T01:06:36.329Z] [Docker Build] {"aux":{"ID":"sha256:66da6b5995cc50e0463df668b8820b56b6e384a7c91dfaca010ff8c3761b1146"}}
|
||||
[2025-06-18T01:06:36.331Z] [Docker Build] Successfully built 66da6b5995cc
|
||||
[2025-06-18T01:06:36.350Z] [Docker Build] Successfully tagged rhizome-node-test:latest
|
||||
[2025-06-18T01:06:36.350Z] ✅ Docker build completed successfully
|
||||
5
markdown/007-investigation.md
Normal file
5
markdown/007-investigation.md
Normal file
@ -0,0 +1,5 @@
|
||||
Network Layers:
|
||||
- Gossip protocols
|
||||
- Pub/sub
|
||||
- RPC
|
||||
|
||||
779
package-lock.json
generated
779
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
35
package.json
35
package.json
@ -2,30 +2,49 @@
|
||||
"name": "rhizome-node",
|
||||
"version": "0.1.0",
|
||||
"description": "Rhizomatic database engine node",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"build:watch": "tsc --watch",
|
||||
"lint": "eslint",
|
||||
"test": "node --experimental-vm-modules node_modules/.bin/jest",
|
||||
"test": "jest",
|
||||
"coverage": "./scripts/coverage.sh",
|
||||
"coverage-report": "npm run test -- --coverage --coverageDirectory=coverage",
|
||||
"example-app": "node dist/examples/app.js"
|
||||
},
|
||||
"jest": {
|
||||
"testEnvironment": "node",
|
||||
"preset": "ts-jest",
|
||||
"preset": "ts-jest/presets/default",
|
||||
"roots": [
|
||||
"./__tests__/"
|
||||
],
|
||||
"testMatch": [
|
||||
"**/__tests__/**/*"
|
||||
]
|
||||
],
|
||||
"setupFilesAfterEnv": [
|
||||
"<rootDir>/__tests__/jest-setup.ts"
|
||||
],
|
||||
"transform": {
|
||||
"^\\.tsx?$": [
|
||||
"ts-jest",
|
||||
{
|
||||
"tsconfig": "tsconfig.json"
|
||||
}
|
||||
]
|
||||
},
|
||||
"detectOpenHandles": true,
|
||||
"forceExit": true,
|
||||
"verbose": true,
|
||||
"testEnvironmentOptions": {
|
||||
"NODE_ENV": "test",
|
||||
"DEBUG": "rz:*"
|
||||
}
|
||||
},
|
||||
"author": "Taliesin (Ladd) <ladd@dgov.io>",
|
||||
"license": "Unlicense",
|
||||
"dependencies": {
|
||||
"@types/dockerode": "^3.3.40",
|
||||
"debug": "^4.4.0",
|
||||
"dockerode": "^4.0.7",
|
||||
"express": "^4.21.2",
|
||||
"json-logic-js": "^2.0.5",
|
||||
"level": "^9.0.0",
|
||||
@ -34,6 +53,7 @@
|
||||
"object-hash": "^3.0.0",
|
||||
"showdown": "^2.1.0",
|
||||
"util": "./util/",
|
||||
"uuid": "^9.0.0",
|
||||
"zeromq": "^6.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
@ -44,13 +64,18 @@
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/json-logic-js": "^2.0.8",
|
||||
"@types/microtime": "^2.1.2",
|
||||
"@types/node": "^22.10.2",
|
||||
"@types/node": "^22.15.31",
|
||||
"@types/node-fetch": "^2.6.12",
|
||||
"@types/object-hash": "^3.0.6",
|
||||
"@types/showdown": "^2.0.6",
|
||||
"@types/tar-fs": "^2.0.4",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.17.0",
|
||||
"eslint-config-airbnb-base-typescript": "^1.1.0",
|
||||
"jest": "^29.7.0",
|
||||
"node-fetch": "^2.7.0",
|
||||
"ts-jest": "^29.2.5",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsc-alias": "^1.8.10",
|
||||
"typescript": "^5.7.2",
|
||||
"typescript-eslint": "^8.18.0"
|
||||
|
||||
@ -1,12 +1,20 @@
|
||||
import { apply } from 'json-logic-js';
|
||||
import Debug from 'debug';
|
||||
import jsonLogic from 'json-logic-js';
|
||||
|
||||
console.log(apply({"map":[
|
||||
const debug = Debug('rz:scratch:jsonlogic');
|
||||
const { apply } = jsonLogic;
|
||||
|
||||
// Example of using jsonLogic's map operation
|
||||
const mapResult = apply({"map":[
|
||||
{"var":"integers"},
|
||||
{"*":[{"var":""},2]}
|
||||
]}, {"integers":[1,2,3,4,5]}));
|
||||
]}, {"integers":[1,2,3,4,5]});
|
||||
debug('Map result: %o', mapResult);
|
||||
|
||||
console.log(apply({"reduce":[
|
||||
// Example of using jsonLogic's reduce operation
|
||||
const reduceResult = apply({"reduce":[
|
||||
{"var":"integers"},
|
||||
{"+":[{"var":"current"}, {"var":"accumulator"}]},
|
||||
0
|
||||
]}, {"integers":[1,2,3,4,5]}));
|
||||
]}, {"integers":[1,2,3,4,5]});
|
||||
debug('Reduce result: %o', reduceResult);
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
import Debug from 'debug';
|
||||
import express, {Router} from "express";
|
||||
import {Collection} from "../collections";
|
||||
import {Delta, DeltaFilter} from "../core";
|
||||
import {RhizomeNode} from "../node";
|
||||
|
||||
const debug = Debug('rz:http:api');
|
||||
|
||||
export class HttpApi {
|
||||
router = Router();
|
||||
|
||||
@ -11,6 +14,14 @@ export class HttpApi {
|
||||
}
|
||||
|
||||
private setupRoutes() {
|
||||
// --------------- health ----------------
|
||||
|
||||
this.router.get("/health", (_req: express.Request, res: express.Response) => {
|
||||
res.json({
|
||||
status: "ok"
|
||||
});
|
||||
});
|
||||
|
||||
// --------------- deltas ----------------
|
||||
|
||||
// Serve list of all deltas accepted
|
||||
@ -161,7 +172,7 @@ export class HttpApi {
|
||||
if (maxResults) options.maxResults = maxResults;
|
||||
if (deltaFilter) {
|
||||
// Note: deltaFilter would need to be serialized/deserialized properly in a real implementation
|
||||
console.warn('deltaFilter not supported in HTTP API yet');
|
||||
debug('deltaFilter not supported in HTTP API yet');
|
||||
}
|
||||
|
||||
const result = await this.rhizomeNode.queryEngine.query(schemaId, filter, options);
|
||||
|
||||
@ -21,15 +21,81 @@ export class HttpServer {
|
||||
this.app.use('/api', this.httpApi.router);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the HTTP server
|
||||
*/
|
||||
start() {
|
||||
const {httpAddr, httpPort} = this.rhizomeNode.config;
|
||||
this.httpHtml.start();
|
||||
this.server = this.app.listen({
|
||||
port: httpPort,
|
||||
host: httpAddr,
|
||||
exclusive: true
|
||||
}, () => {
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP API bound to ${httpAddr}:${httpPort}`);
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, `Starting HTTP server on ${httpAddr}:${httpPort}...`);
|
||||
|
||||
try {
|
||||
this.httpHtml.start();
|
||||
|
||||
// Create the server
|
||||
this.server = this.app.listen({
|
||||
port: httpPort,
|
||||
host: httpAddr,
|
||||
exclusive: true
|
||||
});
|
||||
|
||||
// Add error handler
|
||||
this.server.on('error', (error) => {
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP server error:`, error);
|
||||
});
|
||||
|
||||
// Add callback for logging
|
||||
this.server.on('listening', () => {
|
||||
const address = this.server?.address();
|
||||
const actualPort = typeof address === 'string' ? httpPort : address?.port;
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, `HTTP server bound to ${httpAddr}:${actualPort}`);
|
||||
});
|
||||
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, 'HTTP server start initiated');
|
||||
} catch (error) {
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, 'Error starting HTTP server:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the HTTP server and return a promise that resolves when the server is listening
|
||||
*/
|
||||
async startAndWait(): Promise<void> {
|
||||
// If server is already listening, resolve immediately
|
||||
if (this.server?.listening) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error(`HTTP server failed to start within 10 seconds`));
|
||||
}, 10000);
|
||||
|
||||
const onListening = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
this.server?.off('listening', onListening);
|
||||
this.server?.off('error', onError);
|
||||
};
|
||||
|
||||
// Start the server if not already started
|
||||
if (!this.server) {
|
||||
this.start();
|
||||
}
|
||||
|
||||
// Add event listeners
|
||||
this.server?.on('listening', onListening);
|
||||
this.server?.on('error', onError);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -8,7 +8,8 @@ export type SubscribedMessageHandler = (sender: PeerAddress, msg: string) => voi
|
||||
|
||||
// TODO: Allow subscribing to multiple topics on one socket
|
||||
export class Subscription {
|
||||
sock = new Subscriber();
|
||||
private sock: Subscriber;
|
||||
private isRunning = false;
|
||||
topic: string;
|
||||
publishAddr: PeerAddress;
|
||||
publishAddrStr: string;
|
||||
@ -20,6 +21,7 @@ export class Subscription {
|
||||
topic: string,
|
||||
cb: SubscribedMessageHandler,
|
||||
) {
|
||||
this.sock = new Subscriber();
|
||||
this.cb = cb;
|
||||
this.topic = topic;
|
||||
this.publishAddr = publishAddr;
|
||||
@ -27,20 +29,60 @@ export class Subscription {
|
||||
}
|
||||
|
||||
async start() {
|
||||
if (this.isRunning) return;
|
||||
this.isRunning = true;
|
||||
|
||||
this.sock.connect(this.publishAddrStr);
|
||||
this.sock.subscribe(this.topic);
|
||||
debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Subscribing to ${this.topic} topic on ZeroMQ ${this.publishAddrStr}`);
|
||||
|
||||
// Wait for ZeroMQ messages.
|
||||
// This will block indefinitely.
|
||||
for await (const [, sender, msg] of this.sock) {
|
||||
const senderStr = PeerAddress.fromString(sender.toString());
|
||||
const msgStr = msg.toString();
|
||||
debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `ZeroMQ subscribtion received msg: ${msgStr}`);
|
||||
this.cb(senderStr, msgStr);
|
||||
}
|
||||
// Set up message handler
|
||||
const processMessage = async () => {
|
||||
try {
|
||||
if (!this.isRunning) return;
|
||||
|
||||
debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Done waiting for subscription socket for topic ${this.topic}`);
|
||||
// Use a promise race to handle both messages and the stop signal
|
||||
const [topic, sender, msg] = await Promise.race([
|
||||
this.sock.receive(),
|
||||
new Promise<[Buffer, Buffer, Buffer]>(() => {}).then(() => {
|
||||
if (!this.isRunning) throw new Error('Subscription stopped');
|
||||
return [Buffer.alloc(0), Buffer.alloc(0), Buffer.alloc(0)];
|
||||
})
|
||||
]);
|
||||
|
||||
if (!this.isRunning) return;
|
||||
|
||||
const senderStr = PeerAddress.fromString(sender.toString());
|
||||
const msgStr = msg.toString();
|
||||
debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `ZeroMQ subscription received msg: ${msgStr}`);
|
||||
this.cb(senderStr, msgStr);
|
||||
|
||||
// Process next message
|
||||
process.nextTick(processMessage);
|
||||
} catch (error) {
|
||||
if (this.isRunning) {
|
||||
debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Error in subscription:`, error);
|
||||
// Attempt to restart the message processing
|
||||
if (this.isRunning) {
|
||||
process.nextTick(processMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Start processing messages
|
||||
process.nextTick(processMessage);
|
||||
}
|
||||
|
||||
close() {
|
||||
if (!this.isRunning) return;
|
||||
this.isRunning = false;
|
||||
try {
|
||||
this.sock.close();
|
||||
debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Closed subscription for topic ${this.topic}`);
|
||||
} catch (error) {
|
||||
debug(`[${this.pubSub.rhizomeNode.config.peerId}]`, `Error closing subscription:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -53,8 +95,8 @@ export class PubSub {
|
||||
constructor(rhizomeNode: RhizomeNode) {
|
||||
this.rhizomeNode = rhizomeNode;
|
||||
|
||||
const {publishBindAddr, publishBindPort} = this.rhizomeNode.config;
|
||||
this.publishAddrStr = `tcp://${publishBindAddr}:${publishBindPort}`;
|
||||
const {publishBindHost, publishBindPort} = this.rhizomeNode.config;
|
||||
this.publishAddrStr = `tcp://${publishBindHost}:${publishBindPort}`;
|
||||
}
|
||||
|
||||
async startZmq() {
|
||||
@ -85,16 +127,33 @@ export class PubSub {
|
||||
return subscription;
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (this.publishSock) {
|
||||
await this.publishSock.unbind(this.publishAddrStr);
|
||||
this.publishSock.close();
|
||||
// Free the memory by taking the old object out of scope.
|
||||
this.publishSock = undefined;
|
||||
}
|
||||
/**
|
||||
* Check if the PubSub is running
|
||||
* @returns boolean indicating if the publisher socket is active
|
||||
*/
|
||||
isRunning(): boolean {
|
||||
return !!this.publishSock;
|
||||
}
|
||||
|
||||
async stop() {
|
||||
// First close all subscriptions
|
||||
for (const subscription of this.subscriptions) {
|
||||
subscription.sock.close();
|
||||
subscription.close();
|
||||
}
|
||||
this.subscriptions = [];
|
||||
|
||||
// Then close the publisher socket
|
||||
if (this.publishSock) {
|
||||
try {
|
||||
await this.publishSock.unbind(this.publishAddrStr);
|
||||
this.publishSock.close();
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, 'Unbound and closed publisher socket');
|
||||
} catch (error) {
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, 'Error closing publisher socket:', error);
|
||||
} finally {
|
||||
// Free the memory by taking the old object out of scope.
|
||||
this.publishSock = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -74,8 +74,8 @@ export class RequestReply {
|
||||
|
||||
constructor(rhizomeNode: RhizomeNode) {
|
||||
this.rhizomeNode = rhizomeNode;
|
||||
const {requestBindAddr, requestBindPort} = this.rhizomeNode.config;
|
||||
this.requestBindAddrStr = `tcp://${requestBindAddr}:${requestBindPort}`;
|
||||
const {requestBindHost, requestBindPort} = this.rhizomeNode.config;
|
||||
this.requestBindAddrStr = `tcp://${requestBindHost}:${requestBindPort}`;
|
||||
}
|
||||
|
||||
// Listen for incoming requests
|
||||
|
||||
47
src/node.ts
47
src/node.ts
@ -9,10 +9,8 @@ import {DeltaQueryStorage, StorageFactory, StorageConfig} from './storage';
|
||||
const debug = Debug('rz:rhizome-node');
|
||||
|
||||
export type RhizomeNodeConfig = {
|
||||
requestBindAddr: string;
|
||||
requestBindHost: string;
|
||||
requestBindPort: number;
|
||||
publishBindAddr: string;
|
||||
publishBindHost: string;
|
||||
publishBindPort: number;
|
||||
httpAddr: string;
|
||||
@ -42,10 +40,8 @@ export class RhizomeNode {
|
||||
|
||||
constructor(config?: Partial<RhizomeNodeConfig>) {
|
||||
this.config = {
|
||||
requestBindAddr: REQUEST_BIND_ADDR,
|
||||
requestBindHost: REQUEST_BIND_HOST,
|
||||
requestBindPort: REQUEST_BIND_PORT,
|
||||
publishBindAddr: PUBLISH_BIND_ADDR,
|
||||
publishBindHost: PUBLISH_BIND_HOST,
|
||||
publishBindPort: PUBLISH_BIND_PORT,
|
||||
httpAddr: HTTP_API_ADDR,
|
||||
@ -85,7 +81,16 @@ export class RhizomeNode {
|
||||
this.storageQueryEngine = new StorageQueryEngine(this.deltaStorage, this.schemaRegistry);
|
||||
}
|
||||
|
||||
async start(syncOnStart = false) {
|
||||
/**
|
||||
* Start the node components
|
||||
* @param options.syncOnStart Whether to sync with peers on startup (default: false)
|
||||
* @returns Promise that resolves when the node is fully started and ready
|
||||
*/
|
||||
async start({
|
||||
syncOnStart = false
|
||||
}: { syncOnStart?: boolean } = {}): Promise<void> {
|
||||
debug(`[${this.config.peerId}]`, 'Starting node (waiting for ready)...');
|
||||
|
||||
// Connect our lossless view to the delta stream
|
||||
this.deltaStream.subscribeDeltas(async (delta) => {
|
||||
// Ingest into lossless view
|
||||
@ -100,44 +105,38 @@ export class RhizomeNode {
|
||||
});
|
||||
|
||||
// Bind ZeroMQ publish socket
|
||||
// TODO: Config option to enable zmq pubsub
|
||||
await this.pubSub.startZmq();
|
||||
|
||||
// Bind ZeroMQ request socket
|
||||
// TODO: request/reply via libp2p?
|
||||
// TODO: config options to enable request/reply, or configure available commands
|
||||
this.requestReply.start();
|
||||
|
||||
// Start HTTP server
|
||||
if (this.config.httpEnable) {
|
||||
this.httpServer.start();
|
||||
// Start HTTP server if enabled
|
||||
if (this.config.httpEnable && this.httpServer) {
|
||||
await this.httpServer.startAndWait();
|
||||
}
|
||||
|
||||
{
|
||||
// Wait a short time for sockets to initialize
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
|
||||
// Subscribe to seed peers
|
||||
this.peers.subscribeToSeeds();
|
||||
|
||||
// Wait a short time for sockets to initialize
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
// Initialize network components
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
this.peers.subscribeToSeeds();
|
||||
|
||||
if (syncOnStart) {
|
||||
// Ask all peers for all deltas
|
||||
this.peers.askAllPeersForDeltas();
|
||||
|
||||
// Wait to receive all deltas
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
debug(`[${this.config.peerId}]`, 'Node started and ready');
|
||||
}
|
||||
|
||||
async stop() {
|
||||
this.peers.stop();
|
||||
await this.pubSub.stop();
|
||||
await this.requestReply.stop();
|
||||
await this.httpServer.stop();
|
||||
|
||||
// Stop the HTTP server if it was started
|
||||
if (this.config.httpEnable && this.httpServer) {
|
||||
await this.httpServer.stop();
|
||||
}
|
||||
|
||||
// Close storage
|
||||
try {
|
||||
|
||||
68
src/orchestration/base-orchestrator.ts
Normal file
68
src/orchestration/base-orchestrator.ts
Normal file
@ -0,0 +1,68 @@
|
||||
import { NodeOrchestrator, NodeHandle, NodeConfig, NodeStatus } from './types';
|
||||
|
||||
/**
|
||||
* Base class for all orchestrator implementations
|
||||
* Provides common functionality and ensures interface compliance
|
||||
*/
|
||||
export abstract class BaseOrchestrator implements NodeOrchestrator {
|
||||
/**
|
||||
* Start a new node with the given configuration
|
||||
* Must be implemented by subclasses
|
||||
*/
|
||||
abstract startNode(config: NodeConfig): Promise<NodeHandle>;
|
||||
|
||||
/**
|
||||
* Stop a running node
|
||||
* Must be implemented by subclasses
|
||||
*/
|
||||
abstract stopNode(handle: NodeHandle): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get status of a node
|
||||
* Must be implemented by subclasses
|
||||
*/
|
||||
abstract getNodeStatus(handle: NodeHandle): Promise<NodeStatus>;
|
||||
|
||||
/**
|
||||
* Connect two nodes
|
||||
* Default implementation does nothing - should be overridden by subclasses
|
||||
* that support direct node connections
|
||||
*/
|
||||
async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise<void> {
|
||||
// Default implementation does nothing
|
||||
console.warn('connectNodes not implemented for this orchestrator');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create network partitions
|
||||
* Default implementation does nothing - should be overridden by subclasses
|
||||
* that support network partitioning
|
||||
*/
|
||||
async partitionNetwork(partitions: { groups: string[][] }): Promise<void> {
|
||||
// Default implementation does nothing
|
||||
console.warn('partitionNetwork not implemented for this orchestrator');
|
||||
}
|
||||
|
||||
/**
|
||||
* Set resource limits for a node
|
||||
* Default implementation does nothing - should be overridden by subclasses
|
||||
* that support resource management
|
||||
*/
|
||||
async setResourceLimits(
|
||||
handle: NodeHandle,
|
||||
limits: Partial<NodeConfig['resources']>
|
||||
): Promise<void> {
|
||||
// Default implementation does nothing
|
||||
console.warn('setResourceLimits not implemented for this orchestrator');
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all resources
|
||||
* Default implementation does nothing - should be overridden by subclasses
|
||||
* that need to clean up resources
|
||||
*/
|
||||
async cleanup(): Promise<void> {
|
||||
// Default implementation does nothing
|
||||
console.warn('cleanup not implemented for this orchestrator');
|
||||
}
|
||||
}
|
||||
462
src/orchestration/docker-orchestrator/index.ts
Normal file
462
src/orchestration/docker-orchestrator/index.ts
Normal file
@ -0,0 +1,462 @@
|
||||
import { Container, Network } from 'dockerode';
|
||||
import { BaseOrchestrator } from '../base-orchestrator';
|
||||
import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types';
|
||||
import { DockerNodeHandle, DockerOrchestratorOptions } from './types';
|
||||
import { ContainerManager } from './managers/container-manager';
|
||||
import { NetworkManager } from './managers/network-manager';
|
||||
import { ResourceManager } from './managers/resource-manager';
|
||||
import { StatusManager } from './managers/status-manager';
|
||||
import { ImageManager } from './managers/image-manager';
|
||||
import { getRandomPort } from './utils/port-utils';
|
||||
import Debug from 'debug';
|
||||
|
||||
const debug = Debug('rz:docker:orchestrator');
|
||||
|
||||
const DEFAULT_OPTIONS: DockerOrchestratorOptions = {
|
||||
image: 'rhizome-node-test',
|
||||
containerWorkDir: '/app',
|
||||
autoBuildTestImage: true,
|
||||
};
|
||||
|
||||
export class DockerOrchestrator extends BaseOrchestrator {
|
||||
private options: DockerOrchestratorOptions;
|
||||
private containers: Map<string, Container> = new Map();
|
||||
private networks: Map<string, Network> = new Map();
|
||||
private containerLogStreams: Map<string, NodeJS.ReadableStream> = new Map();
|
||||
private nodeHandles: Map<string, DockerNodeHandle> = new Map();
|
||||
|
||||
// Managers
|
||||
private readonly containerManager: ContainerManager;
|
||||
private readonly networkManager: NetworkManager;
|
||||
private readonly resourceManager: ResourceManager;
|
||||
private readonly statusManager: StatusManager;
|
||||
private readonly imageManager: ImageManager;
|
||||
|
||||
constructor(options: Partial<DockerOrchestratorOptions> = {}) {
|
||||
super();
|
||||
this.options = { ...DEFAULT_OPTIONS, ...options };
|
||||
|
||||
// Initialize managers
|
||||
this.containerManager = new ContainerManager();
|
||||
this.networkManager = new NetworkManager();
|
||||
this.resourceManager = new ResourceManager();
|
||||
this.statusManager = new StatusManager();
|
||||
this.imageManager = new ImageManager();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a new node with the given configuration
|
||||
*/
|
||||
async startNode(config: NodeConfig): Promise<NodeHandle> {
|
||||
const nodeId = config.id || `node-${Date.now()}`;
|
||||
config.network = config.network || {};
|
||||
config.network.port = config.network.port || getRandomPort();
|
||||
config.network.requestPort = config.network.requestPort || getRandomPort();
|
||||
|
||||
try {
|
||||
// Ensure test image is built
|
||||
if (this.options.autoBuildTestImage) {
|
||||
await this.imageManager.buildTestImage(this.options.image);
|
||||
}
|
||||
|
||||
// Create a network for this node using NetworkManager
|
||||
const network = await this.networkManager.createNetwork(nodeId);
|
||||
this.networks.set(nodeId, network);
|
||||
|
||||
// Create container using ContainerManager
|
||||
const container = await this.containerManager.createContainer(
|
||||
nodeId,
|
||||
config,
|
||||
network.id
|
||||
);
|
||||
|
||||
// Store container reference before starting it
|
||||
this.containers.set(nodeId, container);
|
||||
|
||||
// Start the container
|
||||
await this.containerManager.startContainer(container);
|
||||
|
||||
// Create node handle
|
||||
const handle: DockerNodeHandle = {
|
||||
id: nodeId,
|
||||
containerId: container.id,
|
||||
networkId: network.id,
|
||||
config,
|
||||
status: () => this.getNodeStatus({ id: nodeId } as NodeHandle),
|
||||
stop: () => this.stopNode({ id: nodeId } as NodeHandle),
|
||||
getRequestPort: () => config.network?.requestPort,
|
||||
getApiUrl: () => `http://localhost:${config.network?.port}/api`,
|
||||
};
|
||||
|
||||
// Store handle
|
||||
this.nodeHandles.set(nodeId, handle);
|
||||
|
||||
// Wait for node to be ready using StatusManager
|
||||
await this.statusManager.waitForNodeReady( container, config.network.port);
|
||||
|
||||
return handle;
|
||||
} catch (error) {
|
||||
await this.cleanupFailedStart(nodeId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop a running node
|
||||
*/
|
||||
async stopNode(handle: NodeHandle): Promise<void> {
|
||||
const nodeId = handle.id;
|
||||
const container = this.containers.get(nodeId);
|
||||
|
||||
if (!container) {
|
||||
throw new Error(`No container found for node ${nodeId}`);
|
||||
}
|
||||
|
||||
try {
|
||||
// Stop and remove the container using ContainerManager
|
||||
try {
|
||||
await this.containerManager.stopContainer(container);
|
||||
await this.containerManager.removeContainer(container);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug(`Error managing container ${nodeId}: %s`, errorMessage);
|
||||
// Continue with cleanup even if container operations fail
|
||||
}
|
||||
|
||||
// Clean up network using NetworkManager
|
||||
const network = this.networks.get(nodeId);
|
||||
if (network) {
|
||||
try {
|
||||
await this.networkManager.removeNetwork(network.id);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug(`Error removing network for node ${nodeId}: %s`, errorMessage);
|
||||
} finally {
|
||||
this.networks.delete(nodeId);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up log stream
|
||||
this.cleanupLogStream(nodeId);
|
||||
|
||||
// Remove from internal maps
|
||||
this.containers.delete(nodeId);
|
||||
this.nodeHandles.delete(nodeId);
|
||||
|
||||
debug(`Stopped and cleaned up node ${nodeId}`);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug(`Error during cleanup of node ${nodeId}: %s`, errorMessage);
|
||||
throw new Error(`Failed to stop node ${nodeId}: ${errorMessage}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up log stream for a node
|
||||
* @private
|
||||
*/
|
||||
private cleanupLogStream(nodeId: string): void {
|
||||
const logStream = this.containerLogStreams.get(nodeId);
|
||||
if (!logStream) return;
|
||||
|
||||
try {
|
||||
if ('destroy' in logStream) {
|
||||
(logStream as { destroy: () => void }).destroy();
|
||||
} else if ('end' in logStream) {
|
||||
(logStream as { end: () => void }).end();
|
||||
}
|
||||
} catch (error) {
|
||||
debug(`Error cleaning up log stream for node ${nodeId}: %o`, error);
|
||||
} finally {
|
||||
this.containerLogStreams.delete(nodeId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get status of a node
|
||||
*/
|
||||
async getNodeStatus(handle: NodeHandle): Promise<NodeStatus> {
|
||||
const container = this.containers.get(handle.id);
|
||||
|
||||
// If container not found, return stopped status
|
||||
if (!container) {
|
||||
return {
|
||||
id: handle.id,
|
||||
status: 'stopped',
|
||||
error: 'Container not found',
|
||||
network: {
|
||||
address: '',
|
||||
httpPort: 0,
|
||||
requestPort: 0,
|
||||
peers: []
|
||||
},
|
||||
resources: {
|
||||
cpu: { usage: 0, limit: 0 },
|
||||
memory: { usage: 0, limit: 0 }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// Delegate to StatusManager to get the node status
|
||||
return await this.statusManager.getNodeStatus(handle, container);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug(`Error getting status for node ${handle.id}: %s`, errorMessage);
|
||||
|
||||
return {
|
||||
id: handle.id,
|
||||
status: 'error',
|
||||
error: errorMessage,
|
||||
network: {
|
||||
address: '',
|
||||
httpPort: 0,
|
||||
requestPort: 0,
|
||||
peers: []
|
||||
},
|
||||
resources: {
|
||||
cpu: { usage: 0, limit: 0 },
|
||||
memory: { usage: 0, limit: 0 }
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create network partitions
|
||||
*/
|
||||
async partitionNetwork(partitions: NetworkPartition): Promise<void> {
|
||||
// Implementation for network partitioning
|
||||
// This is a simplified version - in a real implementation, you would:
|
||||
// 1. Create separate networks for each partition
|
||||
// 2. Connect containers to their respective partition networks
|
||||
// 3. Disconnect them from other networks
|
||||
debug('Network partitioning not fully implemented');
|
||||
}
|
||||
|
||||
/**
|
||||
* Set resource limits for a node
|
||||
*/
|
||||
async setResourceLimits(
|
||||
handle: NodeHandle,
|
||||
limits: Partial<NodeConfig['resources']> = {}
|
||||
): Promise<void> {
|
||||
const container = this.containers.get(handle.id);
|
||||
if (!container) {
|
||||
throw new Error(`No container found for node ${handle.id}`);
|
||||
}
|
||||
|
||||
try {
|
||||
// Delegate to ResourceManager
|
||||
await this.resourceManager.setResourceLimits(container, {
|
||||
cpu: limits.cpu,
|
||||
memory: limits.memory,
|
||||
memorySwap: limits.memory // Default to same as memory limit if not specified
|
||||
});
|
||||
|
||||
debug(`Updated resource limits for node %s: %o`, handle.id, limits);
|
||||
} catch (error) {
|
||||
debug(`Failed to update resource limits for node ${handle.id}: %o`, error);
|
||||
throw new Error(`Failed to update resource limits: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect two nodes in the network
|
||||
*/
|
||||
async connectNodes(handle1: NodeHandle, handle2: NodeHandle): Promise<void> {
|
||||
const dockerHandle1 = handle1 as DockerNodeHandle;
|
||||
const dockerHandle2 = handle2 as DockerNodeHandle;
|
||||
|
||||
const container1 = this.containers.get(handle1.id);
|
||||
const container2 = this.containers.get(handle2.id);
|
||||
|
||||
if (!container1 || !container2) {
|
||||
throw new Error('One or both containers not found');
|
||||
}
|
||||
|
||||
try {
|
||||
// Get the network from the first container
|
||||
const networkId = dockerHandle1.networkId;
|
||||
if (!networkId) {
|
||||
throw new Error(`No network found for node ${handle1.id}`);
|
||||
}
|
||||
|
||||
// Connect the second container to the same network
|
||||
const network = this.networks.get(handle1.id);
|
||||
if (!network) {
|
||||
throw new Error(`Network not found for node ${handle1.id}`);
|
||||
}
|
||||
|
||||
await network.connect({
|
||||
Container: container2.id,
|
||||
EndpointConfig: {
|
||||
Aliases: [`node-${handle2.id}`]
|
||||
}
|
||||
});
|
||||
|
||||
// Update the network ID in the second handle
|
||||
dockerHandle2.networkId = networkId;
|
||||
} catch (error) {
|
||||
debug(`Error connecting nodes ${handle1.id} and ${handle2.id}: %o`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Cleans up resources for a specific node that failed to start properly.
|
||||
*
|
||||
* This method is automatically called when a node fails to start during the `startNode` process.
|
||||
* It handles cleanup of both the container and network resources associated with the failed node,
|
||||
* and ensures all internal state is properly cleaned up.
|
||||
*
|
||||
* @remarks
|
||||
* - Runs container and network cleanup in parallel for efficiency
|
||||
* - Handles errors gracefully by logging them without rethrowing
|
||||
* - Cleans up internal state for just the specified node
|
||||
* - Used internally by the orchestrator during error handling
|
||||
*
|
||||
* @param nodeId - The unique identifier of the node that failed to start
|
||||
* @private
|
||||
*/
|
||||
private async cleanupFailedStart(nodeId: string): Promise<void> {
|
||||
debug(`Cleaning up failed start for node ${nodeId}...`);
|
||||
|
||||
// Get references to resources before starting cleanup
|
||||
const container = this.containers.get(nodeId);
|
||||
const network = this.networks.get(nodeId);
|
||||
|
||||
// Create a map of containers to clean up
|
||||
const containersToCleanup = new Map<string, Container>();
|
||||
if (container) {
|
||||
containersToCleanup.set(nodeId, container);
|
||||
}
|
||||
|
||||
// Create a map of networks to clean up
|
||||
const networksToCleanup = new Map<string, Network>();
|
||||
if (network) {
|
||||
networksToCleanup.set(nodeId, network);
|
||||
}
|
||||
|
||||
try {
|
||||
// Run container and network cleanup in parallel
|
||||
const [containerErrors, networkErrors] = await Promise.all([
|
||||
// Clean up containers using ContainerManager
|
||||
this.containerManager.cleanupContainers(containersToCleanup),
|
||||
// Clean up networks using NetworkManager
|
||||
this.networkManager.cleanupNetworks(networksToCleanup)
|
||||
]);
|
||||
|
||||
// Log any errors that occurred during cleanup
|
||||
if (containerErrors.length > 0) {
|
||||
debug(`Encountered ${containerErrors.length} error(s) while cleaning up containers for node ${nodeId}:`);
|
||||
containerErrors.forEach(({ resource, error }) => {
|
||||
console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
|
||||
});
|
||||
}
|
||||
|
||||
if (networkErrors.length > 0) {
|
||||
debug(`Encountered ${networkErrors.length} error(s) while cleaning up networks for node ${nodeId}:`);
|
||||
networkErrors.forEach(({ resource, error }) => {
|
||||
console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
|
||||
});
|
||||
}
|
||||
|
||||
debug(`Completed cleanup for node ${nodeId}`);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug(`Unexpected error during cleanup of node ${nodeId}: %s`, errorMessage);
|
||||
} finally {
|
||||
// Always clean up internal state, even if errors occurred
|
||||
this.containers.delete(nodeId);
|
||||
this.networks.delete(nodeId);
|
||||
this.nodeHandles.delete(nodeId);
|
||||
this.containerLogStreams.delete(nodeId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a container by ID
|
||||
* @param containerId The ID of the container to retrieve
|
||||
* @returns The container instance or undefined if not found
|
||||
*/
|
||||
async getContainer(containerId: string): Promise<Container | undefined> {
|
||||
// First try to get from our containers map
|
||||
const container = this.containers.get(containerId);
|
||||
if (container) {
|
||||
return container;
|
||||
}
|
||||
|
||||
// If not found, try to get it from the container manager
|
||||
try {
|
||||
return await this.containerManager.getContainer(containerId);
|
||||
} catch (error) {
|
||||
debug(`Failed to get container ${containerId}: %o`, error);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up all resources managed by this orchestrator.
|
||||
*
|
||||
* This method should be called during shutdown or when you need to completely tear down
|
||||
* all containers and networks created by this orchestrator instance.
|
||||
*
|
||||
* @remarks
|
||||
* - Stops and removes all containers first
|
||||
* - Then removes all networks (sequential execution)
|
||||
* - Clears all internal state including node handles and log streams
|
||||
* - Throws any errors that occur during cleanup
|
||||
* - Should be called when the orchestrator is being shut down
|
||||
*
|
||||
* @throws {Error} If any error occurs during the cleanup process
|
||||
*/
|
||||
async cleanup(): Promise<void> {
|
||||
debug('Starting cleanup of all resources...');
|
||||
|
||||
// Create copies of the maps to avoid modification during iteration
|
||||
const containersToCleanup = new Map(this.containers);
|
||||
const networksToCleanup = new Map(this.networks);
|
||||
|
||||
try {
|
||||
// First, clean up all containers
|
||||
debug('Stopping and removing all containers...');
|
||||
const containerErrors = await this.containerManager.cleanupContainers(containersToCleanup);
|
||||
|
||||
// Wait a short time to ensure all container cleanup is complete
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
// Then clean up all networks
|
||||
debug('Removing all networks...');
|
||||
const networkErrors = await this.networkManager.cleanupNetworks(networksToCleanup);
|
||||
|
||||
// Log any errors that occurred during cleanup
|
||||
if (containerErrors.length > 0) {
|
||||
debug(`Encountered ${containerErrors.length} error(s) while cleaning up containers:`);
|
||||
containerErrors.forEach(({ resource, error }) => {
|
||||
console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
|
||||
});
|
||||
}
|
||||
|
||||
if (networkErrors.length > 0) {
|
||||
debug(`Encountered ${networkErrors.length} error(s) while cleaning up networks:`);
|
||||
networkErrors.forEach(({ resource, error }) => {
|
||||
console.warn(`- ${resource}:`, error instanceof Error ? error.message : 'Unknown error');
|
||||
});
|
||||
}
|
||||
|
||||
debug('Completed cleanup of all resources');
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug('Unexpected error during cleanup: %s', errorMessage);
|
||||
throw error; // Re-throw to allow callers to handle the error
|
||||
} finally {
|
||||
// Always clear internal state, even if errors occurred
|
||||
this.containers.clear();
|
||||
this.networks.clear();
|
||||
this.nodeHandles.clear();
|
||||
this.containerLogStreams.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,249 @@
|
||||
import Debug from 'debug';
|
||||
import Docker, { Container, DockerOptions } from 'dockerode';
|
||||
import { IContainerManager } from './interfaces';
|
||||
import { NodeConfig, NodeStatus } from '../../types';
|
||||
|
||||
const debug = Debug('rz:docker:container-manager');
|
||||
|
||||
export class ContainerManager implements IContainerManager {
|
||||
private docker: Docker;
|
||||
|
||||
constructor() {
|
||||
this.docker = new Docker();
|
||||
}
|
||||
|
||||
async createContainer(
|
||||
nodeId: string,
|
||||
config: NodeConfig,
|
||||
networkId: string
|
||||
): Promise<Container> {
|
||||
const containerName = `rhizome-node-${nodeId}`;
|
||||
|
||||
// Create host config with port bindings and mounts
|
||||
const hostConfig: Docker.HostConfig = {
|
||||
NetworkMode: networkId,
|
||||
PortBindings: {
|
||||
[`${config.network?.port || 3000}/tcp`]: [{ HostPort: config.network?.port?.toString() }],
|
||||
[`${config.network?.requestPort || 3001}/tcp`]: [{ HostPort: config.network?.requestPort?.toString() }],
|
||||
},
|
||||
};
|
||||
|
||||
// Add resource limits if specified
|
||||
if (config.resources) {
|
||||
if (config.resources.cpu) {
|
||||
// Convert CPU cores to nanoCPUs (1 CPU = 1e9 nanoCPUs)
|
||||
hostConfig.NanoCpus = config.resources.cpu * 1e9;
|
||||
}
|
||||
|
||||
if (config.resources.memory) {
|
||||
hostConfig.Memory = config.resources.memory * 1024 * 1024; // Convert MB to bytes
|
||||
hostConfig.MemorySwap = config.resources.memorySwap
|
||||
? config.resources.memorySwap * 1024 * 1024
|
||||
: config.resources.memory * 2 * 1024 * 1024; // Default swap to 2x memory
|
||||
}
|
||||
}
|
||||
|
||||
// Increase file descriptor limits
|
||||
hostConfig.Ulimits = [
|
||||
{
|
||||
Name: 'nofile',
|
||||
Soft: 65536,
|
||||
Hard: 65536
|
||||
}
|
||||
];
|
||||
|
||||
// Set environment variables to optimize performance and disable file watching
|
||||
const envVars = [
|
||||
// Node.js and memory settings
|
||||
`NODE_OPTIONS=--max-old-space-size=${Math.floor((config.resources?.memory || 512) * 0.8)}`,
|
||||
'NODE_ENV=test',
|
||||
|
||||
// Network configuration
|
||||
`RHIZOME_HTTP_BIND_PORT=${config.network?.port || 3000}`,
|
||||
'RHIZOME_HTTP_BIND_ADDR=0.0.0.0',
|
||||
`RHIZOME_REQUEST_BIND_PORT=${config.network?.requestPort || 3001}`,
|
||||
'RHIZOME_REQUEST_BIND_ADDR=0.0.0.0',
|
||||
`RHIZOME_PUBLISH_BIND_PORT=${(config.network?.requestPort || 3001) + 1}`,
|
||||
'RHIZOME_PUBLISH_BIND_ADDR=0.0.0.0',
|
||||
|
||||
// Application settings
|
||||
'RHIZOME_STORAGE_TYPE=memory',
|
||||
'RHIZOME_HTTP_API_ENABLE=true',
|
||||
`RHIZOME_PEER_ID=${nodeId}`,
|
||||
|
||||
// Disable unnecessary features for testing
|
||||
'DISABLE_HTTP_HTML=true',
|
||||
'DISABLE_MARKDOWN=true',
|
||||
|
||||
// Debug settings
|
||||
'DEBUG=rz:*,rhizome:*,docker:*',
|
||||
'DEBUG_COLORS=true'
|
||||
];
|
||||
|
||||
// Create container configuration with all environment variables
|
||||
const containerConfig: Docker.ContainerCreateOptions = {
|
||||
name: containerName,
|
||||
Image: 'rhizome-node-test',
|
||||
ExposedPorts: {
|
||||
[`${config.network?.port || 3000}/tcp`]: {},
|
||||
[`${config.network?.requestPort || 3001}/tcp`]: {}
|
||||
},
|
||||
HostConfig: hostConfig,
|
||||
Env: [
|
||||
...envVars,
|
||||
'NODE_ENV=test',
|
||||
'DEBUG=*',
|
||||
`RHIZOME_HTTP_API_PORT=${config.network?.port || 3000}`,
|
||||
'RHIZOME_HTTP_API_ADDR=0.0.0.0',
|
||||
'RHIZOME_HTTP_API_ENABLE=true'
|
||||
]
|
||||
};
|
||||
|
||||
try {
|
||||
const container = await this.docker.createContainer(containerConfig);
|
||||
|
||||
return container;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to create container: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
async startContainer(container: Container): Promise<void> {
|
||||
try {
|
||||
debug(`Starting container ${container.id}`);
|
||||
await container.start();
|
||||
|
||||
// Verify the container is actually running
|
||||
const containerInfo = await this.verifyContainerRunning(container);
|
||||
debug(`Container ${container.id} started with status:`, containerInfo.State);
|
||||
|
||||
} catch (error) {
|
||||
// Get container logs for debugging
|
||||
let logs = '';
|
||||
try {
|
||||
logs = await this.getContainerLogs(container);
|
||||
} catch (logError) {
|
||||
debug('Failed to get container logs:', logError);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to start container: ${error instanceof Error ? error.message : 'Unknown error'}\n` +
|
||||
`Container logs:\n${logs}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async stopContainer(container: Container): Promise<void> {
|
||||
try {
|
||||
await container.stop({ t: 1 });
|
||||
} catch (error) {
|
||||
debug('Error stopping container: %o', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async removeContainer(container: Container): Promise<void> {
|
||||
try {
|
||||
await container.remove({ force: true });
|
||||
} catch (error) {
|
||||
debug('Error removing container: %o', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getContainerLogs(container: Container, tailLines = 20): Promise<string> {
|
||||
const logs = await container.logs({
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
tail: tailLines,
|
||||
timestamps: true,
|
||||
follow: false,
|
||||
});
|
||||
return logs.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a container by ID
|
||||
* @param containerId The ID of the container to retrieve
|
||||
* @returns The container instance
|
||||
* @throws Error if the container cannot be found
|
||||
*/
|
||||
async getContainer(containerId: string): Promise<Container> {
|
||||
try {
|
||||
const container = this.docker.getContainer(containerId);
|
||||
// Verify the container exists by inspecting it
|
||||
await container.inspect();
|
||||
return container;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get container ${containerId}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async verifyContainerRunning(container: Container): Promise<Docker.ContainerInspectInfo> {
|
||||
const containerInfo = await container.inspect();
|
||||
if (!containerInfo.State.Running) {
|
||||
throw new Error('Container is not running');
|
||||
}
|
||||
return containerInfo;
|
||||
}
|
||||
|
||||
mapContainerState(state: string): NodeStatus['status'] {
|
||||
if (!state) return 'error';
|
||||
|
||||
const stateLower = state.toLowerCase();
|
||||
if (['created', 'restarting'].includes(stateLower)) return 'starting';
|
||||
if (stateLower === 'running') return 'running';
|
||||
if (stateLower === 'paused') return 'stopping';
|
||||
if (['dead', 'exited'].includes(stateLower)) return 'stopped';
|
||||
|
||||
return 'error';
|
||||
}
|
||||
|
||||
async cleanupContainers(containers: Map<string, Container>): Promise<Array<{ resource: string; error: Error }>> {
|
||||
const cleanupErrors: Array<{ resource: string; error: Error }> = [];
|
||||
|
||||
// Process containers in sequence to avoid overwhelming the Docker daemon
|
||||
for (const [nodeId, container] of containers.entries()) {
|
||||
try {
|
||||
debug(`[Cleanup] Stopping container ${nodeId}...`);
|
||||
|
||||
try {
|
||||
// First, try to stop the container gracefully
|
||||
await this.stopContainer(container);
|
||||
debug(`[Cleanup] Successfully stopped container ${nodeId}`);
|
||||
} catch (stopError) {
|
||||
debug(`[Cleanup] Failed to stop container ${nodeId}: %o`, stopError);
|
||||
// Continue with force removal even if stop failed
|
||||
}
|
||||
|
||||
// Now remove the container
|
||||
debug(`[Cleanup] Removing container ${nodeId}...`);
|
||||
await this.removeContainer(container);
|
||||
debug(`[Cleanup] Successfully removed container ${nodeId}`);
|
||||
|
||||
// Verify the container is actually gone
|
||||
try {
|
||||
const containerInfo = await container.inspect();
|
||||
debug(`[Cleanup] Container ${nodeId} still exists after removal: %s`, containerInfo.State?.Status);
|
||||
cleanupErrors.push({
|
||||
resource: `container:${nodeId}`,
|
||||
error: new Error(`Container still exists after removal: ${containerInfo.State?.Status}`)
|
||||
});
|
||||
} catch (inspectError) {
|
||||
// Expected - container should not exist anymore
|
||||
debug(`[Cleanup] Verified container ${nodeId} has been removed`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
debug(`[Cleanup] Error cleaning up container ${nodeId}: %o`, err);
|
||||
cleanupErrors.push({ resource: `container:${nodeId}`, error: err });
|
||||
}
|
||||
|
||||
// Add a small delay between container cleanups
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
return cleanupErrors;
|
||||
}
|
||||
}
|
||||
159
src/orchestration/docker-orchestrator/managers/image-manager.ts
Normal file
159
src/orchestration/docker-orchestrator/managers/image-manager.ts
Normal file
@ -0,0 +1,159 @@
|
||||
import Docker, { DockerOptions } from 'dockerode';
|
||||
import * as path from 'path';
|
||||
import { promises as fs } from 'fs';
|
||||
import * as tar from 'tar-fs';
|
||||
import { Headers } from 'tar-fs';
|
||||
import { IImageManager } from './interfaces';
|
||||
import Debug from 'debug';
|
||||
|
||||
const debug = Debug('rz:docker:image-manager');
|
||||
|
||||
// Global promise to track test image build
|
||||
let testImageBuildPromise: Promise<void> | null = null;
|
||||
|
||||
export class ImageManager implements IImageManager {
|
||||
private docker: Docker;
|
||||
|
||||
constructor() {
|
||||
this.docker = new Docker();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a test Docker image if it doesn't exist
|
||||
*/
|
||||
async buildTestImage(imageName: string = 'rhizome-node-test'): Promise<void> {
|
||||
if (testImageBuildPromise) {
|
||||
debug('Test image build in progress, reusing existing build promise...');
|
||||
return testImageBuildPromise;
|
||||
}
|
||||
|
||||
debug('Building test Docker image...');
|
||||
const dockerfilePath = path.join(process.cwd(), 'Dockerfile.test');
|
||||
|
||||
// Verify Dockerfile exists
|
||||
try {
|
||||
await fs.access(dockerfilePath);
|
||||
debug(`Found Dockerfile at: %s`, dockerfilePath);
|
||||
} catch (err) {
|
||||
throw new Error(`Dockerfile not found at ${dockerfilePath}: ${err}`);
|
||||
}
|
||||
|
||||
// Create a tar archive of the build context
|
||||
const tarStream = tar.pack(process.cwd(), {
|
||||
entries: [
|
||||
'Dockerfile.test',
|
||||
'package.json',
|
||||
'package-lock.json',
|
||||
'tsconfig.json',
|
||||
'src/',
|
||||
'markdown/',
|
||||
'util',
|
||||
'examples/',
|
||||
'README.md',
|
||||
],
|
||||
map: (header: Headers) => {
|
||||
// Ensure Dockerfile is named 'Dockerfile' in the build context
|
||||
if (header.name === 'Dockerfile.test') {
|
||||
header.name = 'Dockerfile';
|
||||
}
|
||||
return header;
|
||||
}
|
||||
});
|
||||
|
||||
debug('Created build context tar stream');
|
||||
|
||||
testImageBuildPromise = new Promise<void>((resolve, reject) => {
|
||||
const logMessages: string[] = [];
|
||||
|
||||
const log = (...args: any[]) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
const message = args.map(arg =>
|
||||
typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
|
||||
).join(' ');
|
||||
const logMessage = `[${timestamp}] ${message}\n`;
|
||||
process.stdout.write(logMessage);
|
||||
logMessages.push(logMessage);
|
||||
};
|
||||
|
||||
this.docker.buildImage(tarStream, { t: imageName }, (err, stream) => {
|
||||
if (err) {
|
||||
const errorMsg = `❌ Error starting Docker build: ${err.message}`;
|
||||
log(errorMsg);
|
||||
return reject(new Error(errorMsg));
|
||||
}
|
||||
|
||||
if (!stream) {
|
||||
const error = new Error('No build stream returned from Docker');
|
||||
log(`❌ ${error.message}`);
|
||||
return reject(error);
|
||||
}
|
||||
|
||||
log('✅ Docker build started, streaming output...');
|
||||
|
||||
// Handle build output
|
||||
let output = '';
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
const chunkStr = chunk.toString();
|
||||
output += chunkStr;
|
||||
|
||||
try {
|
||||
// Try to parse as JSON (Docker build output is typically JSONL)
|
||||
const lines = chunkStr.split('\n').filter(Boolean);
|
||||
for (const line of lines) {
|
||||
try {
|
||||
if (!line.trim()) continue;
|
||||
|
||||
const json = JSON.parse(line);
|
||||
if (json.stream) {
|
||||
const message = `[Docker Build] ${json.stream}`.trim();
|
||||
log(message);
|
||||
} else if (json.error) {
|
||||
const errorMsg = json.error.trim() || 'Unknown error during Docker build';
|
||||
log(`❌ ${errorMsg}`);
|
||||
reject(new Error(errorMsg));
|
||||
return;
|
||||
} else if (Object.keys(json).length > 0) {
|
||||
// Log any other non-empty JSON objects
|
||||
log(`[Docker Build] ${JSON.stringify(json)}`);
|
||||
}
|
||||
} catch (e) {
|
||||
// If not JSON, log as plain text if not empty
|
||||
if (line.trim()) {
|
||||
log(`[Docker Build] ${line}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
const errorMsg = `Error processing build output: ${e}\nRaw output: ${chunkStr}`;
|
||||
log(`❌ ${errorMsg}`);
|
||||
debug('Docker build error: %s', errorMsg);
|
||||
}
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
log('✅ Docker build completed successfully');
|
||||
resolve();
|
||||
});
|
||||
|
||||
stream.on('error', (err: Error) => {
|
||||
const errorMsg = `❌ Docker build failed: ${err.message}\nBuild output so far: ${output}`;
|
||||
log(errorMsg);
|
||||
reject(new Error(errorMsg));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an image exists locally
|
||||
*/
|
||||
async imageExists(imageName: string): Promise<boolean> {
|
||||
try {
|
||||
const image = this.docker.getImage(imageName);
|
||||
await image.inspect();
|
||||
return true;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
5
src/orchestration/docker-orchestrator/managers/index.ts
Normal file
5
src/orchestration/docker-orchestrator/managers/index.ts
Normal file
@ -0,0 +1,5 @@
|
||||
export * from './interfaces';
|
||||
export * from './container-manager';
|
||||
export * from './network-manager';
|
||||
export * from './resource-manager';
|
||||
export * from './status-manager';
|
||||
69
src/orchestration/docker-orchestrator/managers/interfaces.ts
Normal file
69
src/orchestration/docker-orchestrator/managers/interfaces.ts
Normal file
@ -0,0 +1,69 @@
|
||||
import Docker, { Container, Network, NetworkInspectInfo } from 'dockerode';
|
||||
import { NodeConfig, NodeHandle, NodeStatus } from '../../types';
|
||||
|
||||
export interface IContainerManager {
|
||||
createContainer(
|
||||
nodeId: string,
|
||||
config: NodeConfig,
|
||||
networkId: string
|
||||
): Promise<Container>;
|
||||
|
||||
startContainer(container: Container): Promise<void>;
|
||||
stopContainer(container: Container): Promise<void>;
|
||||
removeContainer(container: Container): Promise<void>;
|
||||
getContainerLogs(container: Container, tailLines?: number): Promise<string>;
|
||||
getContainer(containerId: string): Promise<Container>;
|
||||
verifyContainerRunning(container: Container): Promise<Docker.ContainerInspectInfo>;
|
||||
mapContainerState(state: string): NodeStatus['status'];
|
||||
cleanupContainers(containers: Map<string, Container>): Promise<Array<{ resource: string; error: Error }>>;
|
||||
}
|
||||
|
||||
export interface INetworkManager {
|
||||
createNetwork(nodeId: string): Promise<Network>;
|
||||
removeNetwork(networkId: string): Promise<void>;
|
||||
connectToNetwork(containerId: string, networkId: string, aliases?: string[]): Promise<void>;
|
||||
disconnectFromNetwork(containerId: string, networkId: string): Promise<void>;
|
||||
setupPortBindings(ports: Record<string, any>): Docker.HostConfig['PortBindings'];
|
||||
getNetworkInfo(networkId: string): Promise<NetworkInspectInfo>;
|
||||
cleanupNetworks(networks: Map<string, Network>): Promise<Array<{ resource: string; error: Error }>>;
|
||||
}
|
||||
|
||||
export interface IResourceManager {
|
||||
setResourceLimits(
|
||||
container: Container,
|
||||
limits: Partial<NodeConfig['resources']>
|
||||
): Promise<void>;
|
||||
|
||||
getResourceUsage(container: Container): Promise<{
|
||||
cpu: { usage: number; limit: number };
|
||||
memory: { usage: number; limit: number };
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface IImageManager {
|
||||
/**
|
||||
* Build a test Docker image if it doesn't exist
|
||||
* @param imageName The name to give to the built image
|
||||
*/
|
||||
buildTestImage(imageName: string): Promise<void>;
|
||||
}
|
||||
|
||||
export interface IStatusManager {
|
||||
waitForNodeReady(
|
||||
container: Container,
|
||||
port: number,
|
||||
maxAttempts?: number,
|
||||
delayMs?: number
|
||||
): Promise<void>;
|
||||
|
||||
healthCheck(healthUrl: string): Promise<{ ok: boolean; status: number }>;
|
||||
mapContainerState(state: string): NodeStatus['status'];
|
||||
|
||||
/**
|
||||
* Get the status of a node including container status, network info, and resource usage
|
||||
* @param handle The node handle containing node metadata
|
||||
* @param container The Docker container instance
|
||||
* @returns A promise that resolves to the node status
|
||||
*/
|
||||
getNodeStatus(handle: NodeHandle, container: Container): Promise<NodeStatus>;
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
import Debug from 'debug';
|
||||
import Docker, { Network, NetworkInspectInfo } from 'dockerode';
|
||||
import { INetworkManager } from './interfaces';
|
||||
|
||||
const debug = Debug('rz:docker:network-manager');
|
||||
|
||||
export class NetworkManager implements INetworkManager {
|
||||
private networks: Map<string, Network> = new Map();
|
||||
private docker: Docker;
|
||||
|
||||
constructor() {
|
||||
this.docker = new Docker();
|
||||
}
|
||||
|
||||
async createNetwork(nodeId: string): Promise<Network> {
|
||||
const networkName = `rhizome-${nodeId}-network`;
|
||||
|
||||
try {
|
||||
const network = await this.docker.createNetwork({
|
||||
Name: networkName,
|
||||
Driver: 'bridge',
|
||||
CheckDuplicate: true,
|
||||
Internal: false,
|
||||
Attachable: true,
|
||||
EnableIPv6: false
|
||||
});
|
||||
|
||||
this.networks.set(nodeId, network);
|
||||
return network;
|
||||
} catch (error) {
|
||||
debug(`Error creating network for node ${nodeId}: %o`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async removeNetwork(networkId: string): Promise<void> {
|
||||
try {
|
||||
const network = this.docker.getNetwork(networkId);
|
||||
await network.remove();
|
||||
|
||||
// Remove from our tracking map
|
||||
for (const [nodeId, net] of this.networks.entries()) {
|
||||
if (net.id === networkId) {
|
||||
this.networks.delete(nodeId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
debug(`Failed to remove network ${networkId}: %o`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async connectToNetwork(
|
||||
containerId: string,
|
||||
networkId: string,
|
||||
aliases: string[] = []
|
||||
): Promise<void> {
|
||||
try {
|
||||
const network = this.docker.getNetwork(networkId);
|
||||
await network.connect({
|
||||
Container: containerId,
|
||||
EndpointConfig: {
|
||||
Aliases: aliases
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
debug(`Failed to connect container ${containerId} to network ${networkId}: %o`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async disconnectFromNetwork(containerId: string, networkId: string): Promise<void> {
|
||||
try {
|
||||
const network = this.docker.getNetwork(networkId);
|
||||
await network.disconnect({ Container: containerId });
|
||||
} catch (error) {
|
||||
debug(`Failed to disconnect container ${containerId} from network ${networkId}: %o`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
setupPortBindings(ports: Record<string, any>): Docker.HostConfig['PortBindings'] {
|
||||
const portBindings: Docker.HostConfig['PortBindings'] = {};
|
||||
|
||||
for (const [containerPort, hostPort] of Object.entries(ports)) {
|
||||
const [port, protocol = 'tcp'] = containerPort.split('/');
|
||||
portBindings[`${port}/${protocol}`] = [{ HostPort: hostPort.toString() }];
|
||||
}
|
||||
|
||||
return portBindings;
|
||||
}
|
||||
|
||||
async getNetworkInfo(networkId: string): Promise<NetworkInspectInfo> {
|
||||
try {
|
||||
const network = this.docker.getNetwork(networkId);
|
||||
return await network.inspect();
|
||||
} catch (error) {
|
||||
debug(`Failed to get network info for ${networkId}: %o`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async cleanupNetworks(networks: Map<string, Network>): Promise<Array<{ resource: string; error: Error }>> {
|
||||
const cleanupErrors: Array<{ resource: string; error: Error }> = [];
|
||||
|
||||
// Process networks in sequence to avoid overwhelming the Docker daemon
|
||||
for (const [nodeId, network] of networks.entries()) {
|
||||
try {
|
||||
debug(`[Cleanup] Removing network for node ${nodeId}...`);
|
||||
|
||||
// First, inspect the network to see if it has any connected containers
|
||||
try {
|
||||
const networkInfo = await this.getNetworkInfo(network.id);
|
||||
if (networkInfo.Containers && Object.keys(networkInfo.Containers).length > 0) {
|
||||
debug(`[Cleanup] Network ${nodeId} still has ${Object.keys(networkInfo.Containers).length} connected containers`);
|
||||
|
||||
// Try to disconnect all containers from the network first
|
||||
for (const containerId of Object.keys(networkInfo.Containers)) {
|
||||
try {
|
||||
debug(`[Cleanup] Disconnecting container ${containerId} from network ${nodeId}...`);
|
||||
await this.disconnectFromNetwork(containerId, network.id);
|
||||
debug(`[Cleanup] Successfully disconnected container ${containerId} from network ${nodeId}`);
|
||||
} catch (disconnectError) {
|
||||
debug(`[Cleanup] Failed to disconnect container ${containerId} from network ${nodeId}: %o`, disconnectError);
|
||||
// Continue with network removal even if disconnect failed
|
||||
}
|
||||
|
||||
// Add a small delay between disconnects
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
}
|
||||
} catch (inspectError) {
|
||||
debug(`[Cleanup] Failed to inspect network ${nodeId} before removal: %o`, inspectError);
|
||||
// Continue with removal even if inspect failed
|
||||
}
|
||||
|
||||
// Now remove the network
|
||||
await this.removeNetwork(network.id);
|
||||
debug(`[Cleanup] Successfully removed network for node ${nodeId}`);
|
||||
|
||||
// Verify the network is actually gone
|
||||
try {
|
||||
await this.getNetworkInfo(network.id);
|
||||
debug(`[Cleanup] Network ${nodeId} still exists after removal`);
|
||||
cleanupErrors.push({
|
||||
resource: `network:${nodeId}`,
|
||||
error: new Error('Network still exists after removal')
|
||||
});
|
||||
} catch (inspectError) {
|
||||
// Expected - network should not exist anymore
|
||||
debug(`[Cleanup] Verified network ${nodeId} has been removed`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
debug(`[Cleanup] Error cleaning up network ${nodeId}: %o`, err);
|
||||
cleanupErrors.push({ resource: `network:${nodeId}`, error: err });
|
||||
}
|
||||
|
||||
// Add a small delay between network cleanups
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
return cleanupErrors;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,269 @@
|
||||
import Debug from 'debug';
|
||||
import { Container } from 'dockerode';
|
||||
import { IResourceManager } from './interfaces';
|
||||
|
||||
const debug = Debug('rz:docker:resource-manager');
|
||||
|
||||
// Define the structure of the Docker stats object
|
||||
interface ContainerStats {
|
||||
cpu_stats: {
|
||||
cpu_usage: {
|
||||
total_usage: number;
|
||||
usage_in_kernelmode?: number;
|
||||
usage_in_usermode?: number;
|
||||
};
|
||||
system_cpu_usage: number;
|
||||
online_cpus?: number;
|
||||
throttling_data?: Record<string, unknown>;
|
||||
};
|
||||
precpu_stats: {
|
||||
cpu_usage: {
|
||||
total_usage: number;
|
||||
usage_in_kernelmode?: number;
|
||||
usage_in_usermode?: number;
|
||||
};
|
||||
system_cpu_usage: number;
|
||||
online_cpus?: number;
|
||||
throttling_data?: Record<string, unknown>;
|
||||
};
|
||||
memory_stats: {
|
||||
usage?: number;
|
||||
max_usage?: number;
|
||||
limit?: number;
|
||||
stats?: {
|
||||
total_rss?: number;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
usage_in_bytes?: number;
|
||||
limit_in_bytes?: number;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// Type guard to check if an object is a Node.js ReadableStream
|
||||
function isReadableStream(obj: unknown): obj is NodeJS.ReadableStream {
|
||||
return (
|
||||
obj !== null &&
|
||||
typeof obj === 'object' &&
|
||||
typeof (obj as any).pipe === 'function' &&
|
||||
typeof (obj as any).on === 'function'
|
||||
);
|
||||
}
|
||||
|
||||
export class ResourceManager implements IResourceManager {
|
||||
private debug = debug.extend('ResourceManager');
|
||||
|
||||
constructor() {
|
||||
this.debug('ResourceManager initialized');
|
||||
}
|
||||
|
||||
async setResourceLimits(
|
||||
container: Container,
|
||||
limits: {
|
||||
cpu?: number;
|
||||
memory?: number;
|
||||
memorySwap?: number;
|
||||
} = {}
|
||||
): Promise<void> {
|
||||
try {
|
||||
const updateConfig: any = {};
|
||||
|
||||
if (limits.cpu !== undefined) {
|
||||
updateConfig.CpuShares = limits.cpu;
|
||||
updateConfig.NanoCpus = limits.cpu * 1e9; // Convert to nanoCPUs
|
||||
}
|
||||
|
||||
if (limits.memory !== undefined) {
|
||||
updateConfig.Memory = limits.memory * 1024 * 1024; // Convert MB to bytes
|
||||
updateConfig.MemorySwap = limits.memorySwap !== undefined
|
||||
? limits.memorySwap * 1024 * 1024
|
||||
: updateConfig.Memory; // Default to same as memory if not specified
|
||||
}
|
||||
|
||||
if (Object.keys(updateConfig).length > 0) {
|
||||
await container.update(updateConfig);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to set resource limits: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getResourceUsage(container: Container): Promise<{
|
||||
cpu: { usage: number; limit: number };
|
||||
memory: { usage: number; limit: number };
|
||||
}> {
|
||||
try {
|
||||
this.debug('Getting container stats...');
|
||||
|
||||
// Get container stats with stream:false to get a single stats object
|
||||
const stats = await container.stats({ stream: false });
|
||||
|
||||
// Log the raw stats type and constructor for debugging
|
||||
this.debug('Raw stats type: %s', typeof stats);
|
||||
this.debug('Raw stats constructor: %s', stats?.constructor?.name);
|
||||
|
||||
// Handle the response based on its type
|
||||
let statsData: ContainerStats;
|
||||
|
||||
if (typeof stats === 'string') {
|
||||
// If it's a string, parse it as JSON
|
||||
this.debug('Stats is a string, parsing JSON');
|
||||
try {
|
||||
statsData = JSON.parse(stats) as ContainerStats;
|
||||
} catch (error) {
|
||||
this.debug('Failed to parse stats JSON: %o', error);
|
||||
throw new Error(`Failed to parse stats JSON: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
} else if (stats && typeof stats === 'object') {
|
||||
// Check if it's a Node.js stream using our type guard
|
||||
if (isReadableStream(stats)) {
|
||||
this.debug('Stats is a stream, reading data...');
|
||||
// Convert the stream to a string and parse as JSON
|
||||
const statsString = await this.streamToString(stats);
|
||||
try {
|
||||
statsData = JSON.parse(statsString) as ContainerStats;
|
||||
this.debug('Successfully parsed streamed stats');
|
||||
} catch (error) {
|
||||
this.debug('Failed to parse streamed stats: %o', error);
|
||||
throw new Error(`Failed to parse streamed stats: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
} else {
|
||||
// If it's already an object, use it directly
|
||||
this.debug('Stats is a plain object');
|
||||
statsData = stats as unknown as ContainerStats;
|
||||
}
|
||||
} else {
|
||||
throw new Error(`Unexpected stats type: ${typeof stats}`);
|
||||
}
|
||||
|
||||
// Calculate and return the resource usage
|
||||
return this.calculateResourceUsage(statsData);
|
||||
} catch (error: unknown) {
|
||||
this.debug('Error in getResourceUsage: %o', error);
|
||||
// Return default values on error
|
||||
return {
|
||||
cpu: { usage: 0, limit: 0 },
|
||||
memory: { usage: 0, limit: 0 },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a ReadableStream to a string
|
||||
*/
|
||||
private streamToString(stream: NodeJS.ReadableStream): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
stream.on('data', (chunk: unknown) => {
|
||||
if (Buffer.isBuffer(chunk)) {
|
||||
chunks.push(chunk);
|
||||
} else if (typeof chunk === 'string') {
|
||||
chunks.push(Buffer.from(chunk, 'utf8'));
|
||||
} else if (chunk instanceof Uint8Array) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
} else {
|
||||
this.debug('Unexpected chunk type: %s', typeof chunk);
|
||||
reject(new Error(`Unexpected chunk type: ${typeof chunk}`));
|
||||
}
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
try {
|
||||
const result = Buffer.concat(chunks).toString('utf8');
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
this.debug('Error concatenating chunks: %o', error);
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
||||
stream.on('error', (error: Error) => {
|
||||
this.debug('Error reading stream: %o', error);
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate resource usage from stats data
|
||||
*/
|
||||
private calculateResourceUsage(statsData: ContainerStats): {
|
||||
cpu: { usage: number; limit: number };
|
||||
memory: { usage: number; limit: number };
|
||||
} {
|
||||
this.debug('Calculating resource usage from stats data');
|
||||
|
||||
// Log detailed CPU stats if available
|
||||
if (statsData.cpu_stats) {
|
||||
this.debug('CPU stats: %o', {
|
||||
online_cpus: statsData.cpu_stats.online_cpus,
|
||||
cpu_usage: statsData.cpu_stats.cpu_usage,
|
||||
system_cpu_usage: statsData.cpu_stats.system_cpu_usage,
|
||||
});
|
||||
} else {
|
||||
this.debug('No CPU stats available');
|
||||
}
|
||||
|
||||
// Log detailed memory stats if available
|
||||
if (statsData.memory_stats) {
|
||||
this.debug('Memory stats: %o', {
|
||||
usage: statsData.memory_stats.usage,
|
||||
max_usage: statsData.memory_stats.max_usage,
|
||||
limit: statsData.memory_stats.limit,
|
||||
stats: statsData.memory_stats.stats,
|
||||
});
|
||||
} else {
|
||||
this.debug('No memory stats available');
|
||||
}
|
||||
|
||||
// Calculate CPU usage percentage
|
||||
let cpuPercent = 0;
|
||||
const cpuCores = statsData.cpu_stats?.online_cpus || 1;
|
||||
|
||||
// Check if we have the necessary data for CPU calculation
|
||||
if (statsData.cpu_stats?.cpu_usage?.total_usage !== undefined &&
|
||||
statsData.precpu_stats?.cpu_usage?.total_usage !== undefined) {
|
||||
const cpuDelta = statsData.cpu_stats.cpu_usage.total_usage -
|
||||
(statsData.precpu_stats.cpu_usage.total_usage || 0);
|
||||
const systemDelta = statsData.cpu_stats.system_cpu_usage -
|
||||
(statsData.precpu_stats.system_cpu_usage || 0);
|
||||
|
||||
this.debug('CPU delta: %d, System delta: %d', cpuDelta, systemDelta);
|
||||
|
||||
if (systemDelta > 0 && cpuDelta > 0) {
|
||||
cpuPercent = (cpuDelta / systemDelta) * cpuCores * 100.0;
|
||||
this.debug('Calculated CPU percent: %d%%', cpuPercent);
|
||||
}
|
||||
} else {
|
||||
this.debug('Insufficient CPU stats data for calculation');
|
||||
this.debug('Available CPU stats: %o', statsData.cpu_stats);
|
||||
this.debug('Available precpu_stats: %o', statsData.precpu_stats);
|
||||
}
|
||||
|
||||
// Get memory usage with fallbacks
|
||||
const memoryUsage = statsData.memory_stats?.usage ||
|
||||
statsData.memory_stats?.stats?.total_rss ||
|
||||
statsData.memory_stats?.usage_in_bytes ||
|
||||
0;
|
||||
|
||||
const memoryLimit = statsData.memory_stats?.limit ||
|
||||
statsData.memory_stats?.max_usage ||
|
||||
statsData.memory_stats?.limit_in_bytes ||
|
||||
0;
|
||||
|
||||
this.debug('Memory usage: %d / %d bytes', memoryUsage, memoryLimit);
|
||||
|
||||
return {
|
||||
cpu: {
|
||||
usage: cpuPercent,
|
||||
limit: 100, // 100% CPU limit as a percentage
|
||||
},
|
||||
memory: {
|
||||
usage: memoryUsage,
|
||||
limit: memoryLimit || 0, // Ensure we don't return undefined
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
311
src/orchestration/docker-orchestrator/managers/status-manager.ts
Normal file
311
src/orchestration/docker-orchestrator/managers/status-manager.ts
Normal file
@ -0,0 +1,311 @@
|
||||
import Docker, { Container } from 'dockerode';
|
||||
import { IStatusManager } from './interfaces';
|
||||
import { NodeHandle, NodeStatus } from '../../types';
|
||||
import Debug from 'debug';
|
||||
|
||||
const debug = Debug('rz:docker:status-manager');
|
||||
|
||||
const DEFAULT_MAX_ATTEMPTS = 8;
|
||||
const DEFAULT_DELAY_MS = 1000;
|
||||
const MAX_BACKOFF_MS = 30000; // 30 seconds max backoff
|
||||
|
||||
export class StatusManager implements IStatusManager {
|
||||
async waitForNodeReady(
|
||||
container: Container,
|
||||
port: number,
|
||||
maxAttempts: number = DEFAULT_MAX_ATTEMPTS,
|
||||
initialDelayMs: number = DEFAULT_DELAY_MS
|
||||
): Promise<void> {
|
||||
debug(`[waitForNodeReady] Starting with port ${port}, maxAttempts: ${maxAttempts}, initialDelayMs: ${initialDelayMs}`);
|
||||
let lastError: Error | null = null;
|
||||
let attempt = 0;
|
||||
let delay = initialDelayMs;
|
||||
|
||||
while (attempt < maxAttempts) {
|
||||
attempt++;
|
||||
const attemptStartTime = Date.now();
|
||||
|
||||
try {
|
||||
debug(`[Attempt ${attempt}/${maxAttempts}] Verifying container is running...`);
|
||||
|
||||
// Add timeout to verifyContainerRunning
|
||||
const verifyPromise = this.verifyContainerRunning(container);
|
||||
const timeoutPromise = new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('verifyContainerRunning timed out')), 10000)
|
||||
);
|
||||
|
||||
await Promise.race([verifyPromise, timeoutPromise]);
|
||||
debug(`[Attempt ${attempt}/${maxAttempts}] Container is running`);
|
||||
|
||||
const healthUrl = `http://localhost:${port}/api/health`;
|
||||
debug(`[Attempt ${attempt}/${maxAttempts}] Checking health at: ${healthUrl}`);
|
||||
|
||||
// Add timeout to health check
|
||||
const healthCheckPromise = this.healthCheck(healthUrl);
|
||||
const healthCheckTimeout = new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Health check timed out')), 10000)
|
||||
);
|
||||
|
||||
const response = await Promise.race([healthCheckPromise, healthCheckTimeout]);
|
||||
|
||||
if (response.ok) {
|
||||
debug(`✅ Node is ready! (Attempt ${attempt}/${maxAttempts})`);
|
||||
return; // Success!
|
||||
}
|
||||
|
||||
throw new Error(`Health check failed with status: ${response.status}`);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
lastError = error instanceof Error ? error : new Error(errorMessage);
|
||||
|
||||
const attemptDuration = Date.now() - attemptStartTime;
|
||||
debug(`[Attempt ${attempt}/${maxAttempts}] Failed after ${attemptDuration}ms: %s`, errorMessage);
|
||||
|
||||
// Log container state on error
|
||||
try {
|
||||
const containerInfo = await container.inspect();
|
||||
debug(`[Container State] Status: ${containerInfo.State.Status}, Running: ${containerInfo.State.Running}, ExitCode: ${containerInfo.State.ExitCode}`);
|
||||
|
||||
// Log recent container logs on error
|
||||
if (containerInfo.State.Running) {
|
||||
try {
|
||||
const logs = await container.logs({
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
tail: 20,
|
||||
timestamps: true,
|
||||
});
|
||||
debug(`[Container Logs] Last 20 lines:\n${logs.toString()}`);
|
||||
} catch (logError) {
|
||||
debug('Failed to get container logs: %o', logError);
|
||||
}
|
||||
}
|
||||
} catch (inspectError) {
|
||||
debug('Failed to inspect container: %o', inspectError);
|
||||
}
|
||||
|
||||
// Exponential backoff with jitter, but don't wait if we're out of attempts
|
||||
if (attempt < maxAttempts) {
|
||||
const jitter = Math.random() * 1000; // Add up to 1s of jitter
|
||||
const backoff = Math.min(delay + jitter, MAX_BACKOFF_MS);
|
||||
debug(`[Backoff] Waiting ${Math.round(backoff)}ms before next attempt...`);
|
||||
await new Promise(resolve => setTimeout(resolve, backoff));
|
||||
delay = Math.min(delay * 2, MAX_BACKOFF_MS); // Double the delay for next time, up to max
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here, all attempts failed
|
||||
const errorMessage = `Node did not become ready after ${maxAttempts} attempts. Last error: ${lastError?.message || 'Unknown error'}`;
|
||||
debug('❌ %s', errorMessage);
|
||||
|
||||
// Final attempt to get container logs before failing
|
||||
try {
|
||||
const logs = await container.logs({
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
tail: 100,
|
||||
timestamps: true,
|
||||
follow: false
|
||||
});
|
||||
debug('=== FINAL CONTAINER LOGS ===\n%s\n=== END CONTAINER LOGS ===', logs.toString());
|
||||
} catch (logError) {
|
||||
debug('Failed to get final container logs: %o', logError);
|
||||
}
|
||||
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
async healthCheck(healthUrl: string): Promise<{ ok: boolean; status: number }> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 5000);
|
||||
|
||||
try {
|
||||
const response = await fetch(healthUrl, {
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Connection': 'close'
|
||||
},
|
||||
signal: controller.signal
|
||||
});
|
||||
clearTimeout(timeout);
|
||||
return {
|
||||
ok: response.ok,
|
||||
status: response.status
|
||||
};
|
||||
} catch (error) {
|
||||
clearTimeout(timeout);
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
throw new Error(`Health check timed out after 5000ms (${healthUrl})`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
mapContainerState(state: string): NodeStatus['status'] {
|
||||
if (!state) return 'error';
|
||||
|
||||
const stateLower = state.toLowerCase();
|
||||
if (['created', 'restarting'].includes(stateLower)) return 'starting';
|
||||
if (stateLower === 'running') return 'running';
|
||||
if (stateLower === 'paused') return 'stopping';
|
||||
if (['dead', 'exited', 'stopped'].includes(stateLower)) return 'stopped';
|
||||
|
||||
return 'error';
|
||||
}
|
||||
|
||||
private async verifyContainerRunning(container: Container): Promise<void> {
|
||||
debug('[verifyContainerRunning] Checking container status...');
|
||||
|
||||
try {
|
||||
const data = await container.inspect();
|
||||
debug('[verifyContainerRunning] Container inspect data:', JSON.stringify({
|
||||
Id: data.Id,
|
||||
Name: data.Name,
|
||||
State: data.State,
|
||||
Config: {
|
||||
Image: data.Config?.Image,
|
||||
Env: data.Config?.Env?.filter(env => env.startsWith('NODE_') || env.startsWith('DEBUG')),
|
||||
Cmd: data.Config?.Cmd
|
||||
},
|
||||
HostConfig: {
|
||||
Memory: data.HostConfig?.Memory,
|
||||
NanoCpus: data.HostConfig?.NanoCpus,
|
||||
NetworkMode: data.HostConfig?.NetworkMode
|
||||
}
|
||||
}, null, 2));
|
||||
|
||||
if (!data.State.Running) {
|
||||
const errorMessage = `Container is not running. Status: ${data.State.Status}, ExitCode: ${data.State.ExitCode}, Error: ${data.State.Error}`;
|
||||
debug(`[verifyContainerRunning] ${errorMessage}`);
|
||||
|
||||
// Try to get container logs for more context
|
||||
try {
|
||||
const logs = await container.logs({
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
tail: 50 // Get last 50 lines of logs
|
||||
});
|
||||
debug('[verifyContainerRunning] Container logs:', logs.toString());
|
||||
} catch (logError) {
|
||||
debug('[verifyContainerRunning] Failed to get container logs:', logError);
|
||||
}
|
||||
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
debug('[verifyContainerRunning] Container is running');
|
||||
} catch (error) {
|
||||
debug('[verifyContainerRunning] Error checking container status:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the status of a node including container status, network info, and resource usage
|
||||
* @param handle The node handle containing node metadata
|
||||
* @param container The Docker container instance
|
||||
* @returns A promise that resolves to the node status
|
||||
*/
|
||||
async getNodeStatus(handle: NodeHandle, container: Container): Promise<NodeStatus> {
|
||||
// Default error status for when container is not found or other errors occur
|
||||
const errorStatus: NodeStatus = {
|
||||
id: handle.id,
|
||||
status: 'error',
|
||||
error: 'Failed to get node status',
|
||||
network: {
|
||||
address: '',
|
||||
httpPort: 0,
|
||||
requestPort: 0,
|
||||
peers: []
|
||||
},
|
||||
resources: {
|
||||
cpu: { usage: 0, limit: 0 },
|
||||
memory: { usage: 0, limit: 0 }
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
// Get container info
|
||||
const containerInfo = await container.inspect();
|
||||
|
||||
// Get request port once since we use it multiple times
|
||||
const requestPort = handle.getRequestPort?.() || 0;
|
||||
|
||||
// Initialize with default values
|
||||
const status: NodeStatus = {
|
||||
id: handle.id, // Use the node ID from handle
|
||||
containerId: container.id,
|
||||
status: this.mapContainerState(containerInfo.State?.Status || ''),
|
||||
network: {
|
||||
address: containerInfo.NetworkSettings?.IPAddress || '',
|
||||
httpPort: requestPort,
|
||||
requestPort: requestPort,
|
||||
peers: [],
|
||||
networkId: ''
|
||||
},
|
||||
resources: {
|
||||
cpu: { usage: 0, limit: 0 },
|
||||
memory: { usage: 0, limit: 0 }
|
||||
}
|
||||
};
|
||||
|
||||
// Update network info if available
|
||||
if (containerInfo.NetworkSettings?.Networks) {
|
||||
const network = Object.values(containerInfo.NetworkSettings.Networks)[0];
|
||||
if (network) {
|
||||
// Ensure we have existing network values or use defaults
|
||||
const currentNetwork = status.network || {
|
||||
address: '',
|
||||
httpPort: 0,
|
||||
requestPort: 0,
|
||||
peers: []
|
||||
};
|
||||
|
||||
// Create a new network object with all required properties
|
||||
status.network = {
|
||||
address: network.IPAddress || currentNetwork.address,
|
||||
httpPort: currentNetwork.httpPort,
|
||||
requestPort: currentNetwork.requestPort,
|
||||
peers: currentNetwork.peers,
|
||||
networkId: network.NetworkID || ''
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Get container stats for resource usage
|
||||
try {
|
||||
const stats = await container.stats({ stream: false });
|
||||
const statsData = JSON.parse(stats.toString());
|
||||
|
||||
if (statsData?.cpu_stats?.cpu_usage) {
|
||||
status.resources!.cpu.usage = statsData.cpu_stats.cpu_usage.total_usage || 0;
|
||||
status.resources!.cpu.limit = (statsData.cpu_stats.online_cpus || 0) * 1e9; // Convert to nanoCPUs
|
||||
}
|
||||
|
||||
if (statsData?.memory_stats) {
|
||||
status.resources!.memory.usage = statsData.memory_stats.usage || 0;
|
||||
status.resources!.memory.limit = statsData.memory_stats.limit || 0;
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug(`Failed to get container stats for ${container.id}: %s`, errorMessage);
|
||||
// Update status with error but don't return yet
|
||||
status.status = 'error';
|
||||
status.error = `Failed to get container stats: ${errorMessage}`;
|
||||
}
|
||||
|
||||
return status;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
debug(`Error getting node status for ${handle.id}: %s`, errorMessage);
|
||||
|
||||
return {
|
||||
...errorStatus,
|
||||
id: handle.id,
|
||||
error: errorMessage,
|
||||
status: 'error'
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
40
src/orchestration/docker-orchestrator/types.ts
Normal file
40
src/orchestration/docker-orchestrator/types.ts
Normal file
@ -0,0 +1,40 @@
|
||||
import Docker from 'dockerode';
|
||||
import { NodeHandle, NodeConfig, NodeStatus } from '../types';
|
||||
|
||||
export interface DockerNodeHandle extends NodeHandle {
|
||||
containerId: string;
|
||||
networkId?: string;
|
||||
}
|
||||
|
||||
export interface DockerOrchestratorOptions {
|
||||
/**
|
||||
* Docker image to use for containers
|
||||
* Defaults to 'rhizome-node' if not specified
|
||||
*/
|
||||
image?: string;
|
||||
|
||||
/** Working directory inside container */
|
||||
containerWorkDir?: string;
|
||||
|
||||
/** Whether to build test image if not found */
|
||||
autoBuildTestImage?: boolean;
|
||||
}
|
||||
|
||||
export interface ContainerResources {
|
||||
cpuShares?: number;
|
||||
memory?: number;
|
||||
memorySwap?: number;
|
||||
nanoCpus?: number;
|
||||
}
|
||||
|
||||
export interface ContainerStatus {
|
||||
containerId: string;
|
||||
image: string;
|
||||
state: string;
|
||||
status: NodeStatus['status']; // Use the status type from NodeStatus
|
||||
networkSettings: {
|
||||
ipAddress: string;
|
||||
gateway: string;
|
||||
ports: Record<string, Array<{ hostIp: string; hostPort: string }> | null>;
|
||||
};
|
||||
}
|
||||
46
src/orchestration/docker-orchestrator/utils/port-utils.ts
Normal file
46
src/orchestration/docker-orchestrator/utils/port-utils.ts
Normal file
@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Get a random available port in the range 30000-50000
|
||||
* @returns A random port number
|
||||
*/
|
||||
export function getRandomPort(): number {
|
||||
return Math.floor(30000 + Math.random() * 20000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a port is available
|
||||
* @param port Port number to check
|
||||
* @returns True if the port is available, false otherwise
|
||||
*/
|
||||
export async function isPortAvailable(port: number): Promise<boolean> {
|
||||
const net = await import('net');
|
||||
return new Promise((resolve) => {
|
||||
const server = net.createServer();
|
||||
server.once('error', () => resolve(false));
|
||||
server.once('listening', () => {
|
||||
server.close(() => resolve(true));
|
||||
});
|
||||
server.listen(port);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an available port, optionally starting from a specific port
|
||||
* @param startPort Optional starting port (default: 30000)
|
||||
* @returns A promise that resolves to an available port
|
||||
*/
|
||||
export async function getAvailablePort(startPort: number = 30000): Promise<number> {
|
||||
let port = startPort;
|
||||
while (port <= 65535) {
|
||||
if (await isPortAvailable(port)) {
|
||||
return port;
|
||||
}
|
||||
port++;
|
||||
}
|
||||
throw new Error('No available ports found');
|
||||
}
|
||||
|
||||
export default {
|
||||
getRandomPort,
|
||||
isPortAvailable,
|
||||
getAvailablePort
|
||||
};
|
||||
21
src/orchestration/factory.ts
Normal file
21
src/orchestration/factory.ts
Normal file
@ -0,0 +1,21 @@
|
||||
import { NodeOrchestrator, OrchestratorType } from './types';
|
||||
import { DockerOrchestrator } from './docker-orchestrator';
|
||||
import { TestOrchestrator } from './test-orchestrator';
|
||||
|
||||
/**
|
||||
* Factory function to create an appropriate orchestrator based on environment
|
||||
*/
|
||||
export function createOrchestrator(
|
||||
type: OrchestratorType = 'in-memory',
|
||||
options?: any
|
||||
): NodeOrchestrator {
|
||||
switch (type) {
|
||||
case 'docker':
|
||||
return new DockerOrchestrator(options);
|
||||
case 'kubernetes':
|
||||
throw new Error('Kubernetes orchestrator not yet implemented');
|
||||
case 'in-memory':
|
||||
default:
|
||||
return new TestOrchestrator();
|
||||
}
|
||||
}
|
||||
9
src/orchestration/index.ts
Normal file
9
src/orchestration/index.ts
Normal file
@ -0,0 +1,9 @@
|
||||
// Re-export all types and interfaces
|
||||
export * from './types';
|
||||
|
||||
// Export orchestrator implementations
|
||||
export * from './docker-orchestrator';
|
||||
export * from './test-orchestrator';
|
||||
|
||||
// Export factory function
|
||||
export { createOrchestrator } from './factory';
|
||||
198
src/orchestration/test-orchestrator/index.ts
Normal file
198
src/orchestration/test-orchestrator/index.ts
Normal file
@ -0,0 +1,198 @@
|
||||
import { RhizomeNode, type RhizomeNodeConfig } from '../../node';
|
||||
import { PeerAddress } from '../../network';
|
||||
import { BaseOrchestrator } from '../base-orchestrator';
|
||||
import { NodeConfig, NodeHandle, NodeStatus, NetworkPartition } from '../types';
|
||||
import { getRandomPort } from '../docker-orchestrator/utils/port-utils';
|
||||
import { BasicCollection } from '../../collections/collection-basic';
|
||||
import Debug from 'debug';
|
||||
|
||||
const debug = Debug('rz:test-orchestrator');
|
||||
|
||||
/**
|
||||
* In-memory implementation of NodeOrchestrator for testing
|
||||
*/
|
||||
export class TestOrchestrator extends BaseOrchestrator {
|
||||
private nodes: Map<string, { handle: NodeHandle; node: RhizomeNode }> = new Map();
|
||||
|
||||
async startNode(config: NodeConfig): Promise<NodeHandle> {
|
||||
const nodeId = config.id || `node-${Date.now()}`;
|
||||
// Use getRandomPort instead of 0 for auto-selection
|
||||
const httpPort = config.network?.port || getRandomPort();
|
||||
const requestPort = config.network?.requestPort || getRandomPort();
|
||||
|
||||
// Map NodeConfig to RhizomeNodeConfig with all required properties
|
||||
const nodeConfig: RhizomeNodeConfig = {
|
||||
// Required network properties
|
||||
requestBindHost: '0.0.0.0',
|
||||
requestBindPort: requestPort,
|
||||
publishBindHost: '0.0.0.0',
|
||||
publishBindPort: getRandomPort(), // Use a random port for publish socket
|
||||
httpAddr: '0.0.0.0',
|
||||
httpPort: httpPort,
|
||||
httpEnable: true,
|
||||
|
||||
// Required peer properties
|
||||
peerId: nodeId,
|
||||
creator: 'test-orchestrator',
|
||||
|
||||
// Map network bootstrap peers to seedPeers if provided
|
||||
seedPeers: config.network?.bootstrapPeers?.map(peer => {
|
||||
const [host, port] = peer.split(':');
|
||||
return new PeerAddress(host, parseInt(port));
|
||||
}) || [],
|
||||
|
||||
// Storage configuration with defaults
|
||||
storage: {
|
||||
type: 'memory',
|
||||
path: config.storage?.path || `./data/${nodeId}`,
|
||||
...(config.storage || {})
|
||||
}
|
||||
};
|
||||
|
||||
const node = new RhizomeNode(nodeConfig);
|
||||
|
||||
// Create and connect a user collection
|
||||
const userCollection = new BasicCollection('user');
|
||||
// Connect the collection to the node before serving it
|
||||
userCollection.rhizomeConnect(node);
|
||||
// Now serve the collection through the HTTP API
|
||||
node.httpServer.httpApi.serveCollection(userCollection);
|
||||
|
||||
// Start the node and wait for all components to be ready
|
||||
debug(`[${nodeId}] Starting node and waiting for it to be fully ready...`);
|
||||
try {
|
||||
await node.start();
|
||||
debug(`[${nodeId}] Node is fully started and ready`);
|
||||
} catch (error) {
|
||||
debug(`[${nodeId}] Error starting node:`, error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Get the actual port the server is using
|
||||
const serverAddress = node.httpServer.server?.address();
|
||||
let actualPort = httpPort;
|
||||
|
||||
// Handle different address types (string or AddressInfo)
|
||||
if (serverAddress) {
|
||||
actualPort = typeof serverAddress === 'string'
|
||||
? httpPort
|
||||
: serverAddress.port || httpPort;
|
||||
}
|
||||
|
||||
const handle: NodeHandle = {
|
||||
id: nodeId,
|
||||
config: {
|
||||
...config,
|
||||
id: nodeId,
|
||||
network: {
|
||||
...config.network,
|
||||
port: actualPort,
|
||||
requestPort: requestPort
|
||||
}
|
||||
},
|
||||
status: async () => this.getNodeStatus(handle),
|
||||
getApiUrl: () => `http://localhost:${actualPort}/api`,
|
||||
stop: async () => {
|
||||
await node.stop();
|
||||
this.nodes.delete(nodeId);
|
||||
},
|
||||
getRequestPort: () => requestPort,
|
||||
};
|
||||
|
||||
this.nodes.set(nodeId, { handle, node });
|
||||
return handle;
|
||||
}
|
||||
|
||||
async stopNode(handle: NodeHandle): Promise<void> {
|
||||
const node = this.nodes.get(handle.id);
|
||||
if (node) {
|
||||
await node.node.stop();
|
||||
this.nodes.delete(handle.id);
|
||||
}
|
||||
}
|
||||
|
||||
async getNodeStatus(handle: NodeHandle): Promise<NodeStatus> {
|
||||
const node = this.nodes.get(handle.id);
|
||||
if (!node) {
|
||||
return {
|
||||
id: handle.id,
|
||||
status: 'stopped',
|
||||
error: 'Node not found',
|
||||
network: {
|
||||
address: '127.0.0.1',
|
||||
httpPort: 0,
|
||||
requestPort: 0,
|
||||
peers: []
|
||||
},
|
||||
resources: {
|
||||
cpu: { usage: 0, limit: 0 },
|
||||
memory: { usage: 0, limit: 0 }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// Since we don't have a direct way to check if the node is running,
|
||||
// we'll assume it's running if it's in our nodes map
|
||||
// In a real implementation, we would check the actual node state
|
||||
const status: NodeStatus = {
|
||||
id: handle.id,
|
||||
status: 'running',
|
||||
network: {
|
||||
address: '127.0.0.1',
|
||||
httpPort: node.node.config.httpPort || 0,
|
||||
requestPort: node.node.config.requestBindPort || 0,
|
||||
peers: node.node.peers ? Array.from(node.node.peers.peers).map(p => p.reqAddr.toAddrString()) : []
|
||||
},
|
||||
resources: {
|
||||
cpu: {
|
||||
usage: 0,
|
||||
limit: 0,
|
||||
},
|
||||
memory: {
|
||||
usage: 0,
|
||||
limit: 0,
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
async connectNodes(node1: NodeHandle, node2: NodeHandle): Promise<void> {
|
||||
const n1 = this.nodes.get(node1.id)?.node;
|
||||
const n2 = this.nodes.get(node2.id)?.node;
|
||||
|
||||
if (!n1 || !n2) {
|
||||
throw new Error('One or both nodes not found');
|
||||
}
|
||||
|
||||
// In a real implementation, we would connect the nodes here
|
||||
// For testing, we'll just log the connection attempt
|
||||
console.log(`Connecting nodes ${node1.id} and ${node2.id}`);
|
||||
}
|
||||
|
||||
async partitionNetwork(partitions: NetworkPartition): Promise<void> {
|
||||
// In a real implementation, we would create network partitions
|
||||
// For testing, we'll just log the partition attempt
|
||||
console.log('Creating network partitions:', partitions);
|
||||
}
|
||||
|
||||
async setResourceLimits(
|
||||
handle: NodeHandle,
|
||||
limits: Partial<NodeConfig['resources']>
|
||||
): Promise<void> {
|
||||
// In-memory nodes don't have real resource limits
|
||||
console.log(`Setting resource limits for ${handle.id}:`, limits);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all resources
|
||||
*/
|
||||
async cleanup(): Promise<void> {
|
||||
await Promise.all(
|
||||
Array.from(this.nodes.values()).map(({ node }) => node.stop())
|
||||
);
|
||||
this.nodes.clear();
|
||||
}
|
||||
}
|
||||
104
src/orchestration/types.ts
Normal file
104
src/orchestration/types.ts
Normal file
@ -0,0 +1,104 @@
|
||||
/**
|
||||
* Core types and interfaces for the orchestration layer
|
||||
*/
|
||||
|
||||
export interface NodeConfig {
|
||||
/** Unique identifier for the node */
|
||||
id: string;
|
||||
|
||||
/** Network configuration */
|
||||
network?: {
|
||||
/** Port to listen on (0 = auto-select) */
|
||||
port?: number;
|
||||
/** Port for request/reply communication */
|
||||
requestPort?: number;
|
||||
/** Known peers to connect to */
|
||||
bootstrapPeers?: string[];
|
||||
};
|
||||
|
||||
/** Resource constraints */
|
||||
resources?: {
|
||||
/** CPU shares (0-1024) */
|
||||
cpu?: number;
|
||||
/** Memory limit in MB */
|
||||
memory?: number;
|
||||
/** Memory swap limit in MB (defaults to 2x memory if not specified) */
|
||||
memorySwap?: number;
|
||||
};
|
||||
|
||||
/** Storage configuration */
|
||||
storage?: {
|
||||
/** Storage type */
|
||||
type?: 'memory' | 'leveldb' | 'sqlite' | 'postgres';
|
||||
/** Path to data directory */
|
||||
path?: string;
|
||||
/** Maximum storage in MB */
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
/** Additional configuration options */
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface NodeStatus {
|
||||
id: string;
|
||||
containerId?: string;
|
||||
status: 'starting' | 'running' | 'stopping' | 'stopped' | 'error';
|
||||
network?: {
|
||||
address: string;
|
||||
requestPort: number;
|
||||
httpPort: number;
|
||||
peers: string[];
|
||||
networkId?: string;
|
||||
};
|
||||
resources?: {
|
||||
cpu: {
|
||||
usage: number;
|
||||
limit: number;
|
||||
};
|
||||
memory: {
|
||||
usage: number;
|
||||
limit: number;
|
||||
};
|
||||
};
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface NodeHandle {
|
||||
id: string;
|
||||
config: NodeConfig;
|
||||
status: () => Promise<NodeStatus>;
|
||||
stop: () => Promise<void>;
|
||||
/** Get API URL if applicable */
|
||||
getApiUrl?: () => string;
|
||||
getRequestPort: () => number | undefined;
|
||||
}
|
||||
|
||||
export interface NetworkPartition {
|
||||
groups: string[][];
|
||||
}
|
||||
|
||||
export interface NodeOrchestrator {
|
||||
/** Start a new node with the given configuration */
|
||||
startNode(config: NodeConfig): Promise<NodeHandle>;
|
||||
|
||||
/** Stop a running node */
|
||||
stopNode(handle: NodeHandle): Promise<void>;
|
||||
|
||||
/** Get status of a node */
|
||||
getNodeStatus(handle: NodeHandle): Promise<NodeStatus>;
|
||||
|
||||
/** Connect two nodes */
|
||||
connectNodes(node1: NodeHandle, node2: NodeHandle): Promise<void>;
|
||||
|
||||
/** Create network partitions */
|
||||
partitionNetwork(partitions: NetworkPartition): Promise<void>;
|
||||
|
||||
/** Set resource limits for a node */
|
||||
setResourceLimits(handle: NodeHandle, limits: Partial<NodeConfig['resources']>): Promise<void>;
|
||||
|
||||
/** Clean up all resources */
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
export type OrchestratorType = 'in-memory' | 'docker' | 'kubernetes';
|
||||
@ -1,2 +1,2 @@
|
||||
export { QueryEngine } from './query-engine';
|
||||
export { StorageQueryEngine, JsonLogic as StorageJsonLogic } from './storage-query-engine';
|
||||
export { StorageQueryEngine, type JsonLogic as StorageJsonLogic } from './storage-query-engine';
|
||||
@ -1,4 +1,5 @@
|
||||
import { apply, is_logic } from 'json-logic-js';
|
||||
import jsonLogic from 'json-logic-js';
|
||||
const { apply, is_logic } = jsonLogic;
|
||||
import Debug from 'debug';
|
||||
import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema/schema';
|
||||
import { Lossless, LosslessViewOne, LosslessViewMany, CollapsedDelta } from '../views/lossless';
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { apply } from 'json-logic-js';
|
||||
import jsonLogic from 'json-logic-js';
|
||||
const { apply } = jsonLogic;
|
||||
import Debug from 'debug';
|
||||
import { SchemaRegistry, SchemaID, ObjectSchema } from '../schema';
|
||||
import { DeltaQueryStorage, DeltaQuery } from '../storage/interface';
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
import Debug from 'debug';
|
||||
import { DeltaStorage, DeltaQueryStorage, StorageConfig } from './interface';
|
||||
import { MemoryDeltaStorage } from './memory';
|
||||
import { LevelDBDeltaStorage } from './leveldb';
|
||||
|
||||
const debug = Debug('rz:storage:factory');
|
||||
|
||||
/**
|
||||
* Factory for creating delta storage instances based on configuration
|
||||
*/
|
||||
@ -56,10 +59,10 @@ export class StorageFactory {
|
||||
): Promise<void> {
|
||||
const batchSize = options.batchSize || 1000;
|
||||
|
||||
console.log('Starting storage migration...');
|
||||
debug('Starting storage migration...');
|
||||
|
||||
const allDeltas = await source.getAllDeltas();
|
||||
console.log(`Found ${allDeltas.length} deltas to migrate`);
|
||||
debug(`Found %d deltas to migrate`, allDeltas.length);
|
||||
|
||||
// Migrate in batches to avoid memory issues
|
||||
for (let i = 0; i < allDeltas.length; i += batchSize) {
|
||||
@ -69,19 +72,21 @@ export class StorageFactory {
|
||||
await target.storeDelta(delta);
|
||||
}
|
||||
|
||||
console.log(`Migrated ${Math.min(i + batchSize, allDeltas.length)} / ${allDeltas.length} deltas`);
|
||||
debug('Migrated %d / %d deltas', Math.min(i + batchSize, allDeltas.length), allDeltas.length);
|
||||
}
|
||||
|
||||
console.log('Migration completed successfully');
|
||||
debug('Migration completed successfully');
|
||||
|
||||
// Verify migration
|
||||
const sourceStats = await source.getStats();
|
||||
const targetStats = await target.getStats();
|
||||
|
||||
if (sourceStats.totalDeltas !== targetStats.totalDeltas) {
|
||||
throw new Error(`Migration verification failed: source has ${sourceStats.totalDeltas} deltas, target has ${targetStats.totalDeltas}`);
|
||||
const errorMsg = `Migration verification failed: source has ${sourceStats.totalDeltas} deltas, target has ${targetStats.totalDeltas}`;
|
||||
debug(errorMsg);
|
||||
throw new Error(errorMsg);
|
||||
}
|
||||
|
||||
console.log(`Migration verified: ${targetStats.totalDeltas} deltas migrated successfully`);
|
||||
debug('Migration verified: %d deltas migrated successfully', targetStats.totalDeltas);
|
||||
}
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
import Debug from "debug";
|
||||
import {FSWatcher, readdirSync, readFileSync, watch} from "fs";
|
||||
import {FSWatcher, readdirSync, readFileSync, watch, accessSync, constants} from "fs";
|
||||
import path, {join} from "path";
|
||||
import showdown from "showdown";
|
||||
import {RhizomeNode} from "../node";
|
||||
@ -48,9 +48,32 @@ export class MDFiles {
|
||||
}
|
||||
|
||||
readReadme() {
|
||||
const md = readFileSync('./README.md').toString();
|
||||
let currentDir = process.cwd();
|
||||
const root = path.parse(currentDir).root;
|
||||
let readmePath: string | null = null;
|
||||
|
||||
// Traverse up the directory tree until we find README.md or hit the root
|
||||
while (currentDir !== root) {
|
||||
const testPath = path.join(currentDir, 'README.md');
|
||||
try {
|
||||
// Using the imported accessSync function
|
||||
accessSync(testPath, constants.F_OK);
|
||||
readmePath = testPath;
|
||||
break;
|
||||
} catch (err) {
|
||||
// Move up one directory
|
||||
currentDir = path.dirname(currentDir);
|
||||
}
|
||||
}
|
||||
|
||||
if (!readmePath) {
|
||||
debug('No README.md found in any parent directory');
|
||||
return;
|
||||
}
|
||||
|
||||
const md = readFileSync(readmePath).toString();
|
||||
const html = htmlDocFromMarkdown(md);
|
||||
this.readme = {name: 'README', md, html};
|
||||
this.readme = { name: 'README', md, html };
|
||||
}
|
||||
|
||||
getReadmeHTML() {
|
||||
|
||||
@ -172,7 +172,6 @@ export class Lossless {
|
||||
viewSpecific(entityId: DomainEntityID, deltaIds: DeltaID[], deltaFilter?: DeltaFilter): LosslessViewOne | undefined {
|
||||
const combinedFilter = (delta: Delta) => {
|
||||
if (!deltaIds.includes(delta.id)) {
|
||||
debug(`[${this.rhizomeNode.config.peerId}]`, `Excluding delta ${delta.id} because it's not in the requested list of deltas`);
|
||||
return false;
|
||||
}
|
||||
if (!deltaFilter) return true;
|
||||
|
||||
@ -1,16 +1,24 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"target": "ES6",
|
||||
"module": "CommonJS",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"moduleResolution": "Node",
|
||||
"sourceMap": true,
|
||||
"baseUrl": ".",
|
||||
"rootDir": ".",
|
||||
"outDir": "dist",
|
||||
"importsNotUsedAsValues": "remove",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"types": ["node", "jest"],
|
||||
"typeRoots": [
|
||||
"./node_modules/@types"
|
||||
],
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": false
|
||||
},
|
||||
"include": [
|
||||
"src/**/*",
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { SchemaBuilder } from '../../src/schema';
|
||||
import { SchemaBuilder } from '../src/schema';
|
||||
|
||||
/**
|
||||
* Common schemas used for testing purposes.
|
||||
Loading…
x
Reference in New Issue
Block a user