JSON AST spike

- Remove NegationHelper.createNegation in favor of using DeltaBuilder's fluent API
- Update all test cases to use createDelta().negate().buildV1() pattern
- Update documentation to reflect the preferred way to create negation deltas
- Remove unused isNegationDeltaById helper method

__tests__/jest-setup.ts

@@ -7,22 +7,9 @@ if (!process.env.DEBUG && !process.env.NO_DEBUG) {
   process.env.DEBUG = 'rz:*';
 }
 
-// Extend the global Jest namespace
-declare global {
-  namespace jest {
-    interface Matchers<R> {
-      toBeWithinRange(a: number, b: number): R;
-    }
-  }
-}
-
-// Add any global test setup here
-
 // This is a placeholder test to satisfy Jest's requirement for at least one test
 describe('Test Setup', () => {
   it('should pass', () => {
     expect(true).toBe(true);
   });
 });
-
-export {}; // This file needs to be a module

__tests__/json-ast.test.ts

@@ -0,0 +1,172 @@
+import { jsonToAst } from '../src/util/json-ast';
+import { JsonNode } from '../src/util/json-ast/types';
+
+describe('jsonToAst', () => {
+  it('should handle primitive values', () => {
+    expect(jsonToAst(42)).toMatchObject({
+      type: 'number',
+      value: 42
+    });
+
+    expect(jsonToAst('test')).toMatchObject({
+      type: 'string',
+      value: 'test'
+    });
+
+    expect(jsonToAst(true)).toMatchObject({
+      type: 'boolean',
+      value: true
+    });
+
+    expect(jsonToAst(null)).toMatchObject({
+      type: 'null',
+      value: null
+    });
+  });
+
+  it('should handle empty objects and arrays', () => {
+    const emptyObj = jsonToAst({});
+    expect(emptyObj).toMatchObject({
+      type: 'object',
+      children: []
+    });
+
+    const emptyArray = jsonToAst([]);
+    expect(emptyArray).toMatchObject({
+      type: 'array',
+      children: []
+    });
+  });
+
+  it('should handle nested objects', () => {
+    const ast = jsonToAst({
+      name: 'test',
+      nested: { value: 42 }
+    });
+
+    expect(ast.type).toBe('object');
+    expect(ast.children).toHaveLength(2);
+    
+    const nameNode = ast.children?.[0];
+    const nestedNode = ast.children?.[1];
+    
+    expect(nameNode).toMatchObject({
+      type: 'string',
+      key: 'name',
+      value: 'test'
+    });
+
+    expect(nestedNode).toMatchObject({
+      type: 'object',
+      key: 'nested'
+    });
+
+    expect(nestedNode?.children?.[0]).toMatchObject({
+      type: 'number',
+      key: 'value',
+      value: 42
+    });
+  });
+
+  it('should handle arrays', () => {
+    const ast = jsonToAst([1, 'two', true]);
+    
+    expect(ast.type).toBe('array');
+    expect(ast.children).toHaveLength(3);
+    
+    expect(ast.children?.[0]).toMatchObject({
+      type: 'number',
+      value: 1
+    });
+    
+    expect(ast.children?.[1]).toMatchObject({
+      type: 'string',
+      value: 'two'
+    });
+    
+    expect(ast.children?.[2]).toMatchObject({
+      type: 'boolean',
+      value: true
+    });
+  });
+
+  it('should include paths when includePath is true', () => {
+    const ast = jsonToAst({
+      user: {
+        name: 'test',
+        roles: ['admin', 'user']
+      }
+    }, { includePath: true });
+
+    const findNode = (node: JsonNode, key: string): JsonNode | undefined => {
+      if (node.key === key) return node;
+      if (!node.children) return undefined;
+      for (const child of node.children) {
+        const found = findNode(child, key);
+        if (found) return found;
+      }
+      return undefined;
+    };
+
+    const nameNode = findNode(ast, 'name');
+    const rolesNode = findNode(ast, 'roles');
+    
+    expect(nameNode?.path).toBe('user.name');
+    expect(rolesNode?.path).toBe('user.roles');
+    expect(rolesNode?.children?.[0].path).toBe('user.roles[0]');
+  });
+
+  it('should respect maxDepth option', () => {
+    const deepObject = {
+      level1: {
+        level2: {
+          level3: {
+            value: 'too deep'
+          }
+        }
+      }
+    };
+
+    const ast = jsonToAst(deepObject, { 
+      maxDepth: 2,
+      includePath: true 
+    });
+
+    const level2 = ast.children?.[0].children?.[0];
+    expect(level2?.type).toBe('object');
+    // The implementation sets value to undefined when max depth is exceeded
+    expect(level2?.value).toBeUndefined();
+    expect(level2?.path).toBe('level1.level2');
+  });
+
+  it('should apply filter function when provided', () => {
+    const data = {
+      name: 'test',
+      age: 42,
+      active: true,
+      address: {
+        street: '123 Main St',
+        city: 'Anytown'
+      }
+    };
+
+    // Only include string and number values
+    const ast = jsonToAst(data, {
+      filter: (node: JsonNode) => 
+        node.type === 'string' || 
+        node.type === 'number' ||
+        node.type === 'object' // Keep objects to maintain structure
+    });
+
+    // Should have filtered out the boolean 'active' field
+    expect(ast.children).toHaveLength(3);
+    expect(ast.children?.some((c: any) => c.key === 'active')).toBe(false);
+    
+    // Nested object should only have string properties
+    const addressNode = ast.children?.find((c: any) => c.key === 'address');
+    expect(addressNode?.children).toHaveLength(2);
+    expect(addressNode?.children?.every((c: any) => 
+      c.type === 'string' || c.key === 'city' || c.key === 'street'
+    )).toBe(true);
+  });
+});

__tests__/negation.ts

@@ -21,11 +21,9 @@ describe('Negation System', () => {
         .setProperty('entity1', 'name', 'Alice')
         .buildV1();
 
-      const negationDelta = NegationHelper.createNegation(
+      const negationDelta = createDelta('moderator', 'host1')
-        originalDelta.id,
+        .negate(originalDelta.id)
-        'moderator',
+        .buildV1();
-        'host1'
-      );
 
       expect(negationDelta.creator).toBe('moderator');
       expect(negationDelta.pointers).toHaveLength(1);
@@ -42,11 +40,9 @@ describe('Negation System', () => {
         .setProperty('entity1', 'name', 'Entity 1')
         .buildV1();
 
-      const negationDelta = NegationHelper.createNegation(
+      const negationDelta = createDelta('moderator', 'host1')
-        'delta-to-negate',
+        .negate('delta-to-negate')
-        'moderator',
+        .buildV1();
-        'host1'
-      );
 
       expect(NegationHelper.isNegationDelta(regularDelta)).toBe(false);
       expect(NegationHelper.isNegationDelta(negationDelta)).toBe(true);
@@ -54,11 +50,9 @@ describe('Negation System', () => {
 
     it('should extract negated delta ID', () => {
       const targetDeltaId = 'target-delta-123';
-      const negationDelta = NegationHelper.createNegation(
+      const negationDelta = createDelta('moderator', 'host1')
-        targetDeltaId,
+        .negate(targetDeltaId)
-        'moderator',
+        .buildV1();
-        'host1'
-      );
 
       const extractedId = NegationHelper.getNegatedDeltaId(negationDelta);
       expect(extractedId).toBe(targetDeltaId);
@@ -79,9 +73,9 @@ describe('Negation System', () => {
         .setProperty('entity1', 'age', 25)
         .buildV1();
 
-      const negation1 = NegationHelper.createNegation(delta1.id, 'mod1', 'host1');
+      const negation1 = createDelta('mod1', 'host1').negate(delta1.id).buildV1();
-      const negation2 = NegationHelper.createNegation(delta1.id, 'mod2', 'host1');
+      const negation2 = createDelta('mod2', 'host1').negate(delta1.id).buildV1();
-      const negation3 = NegationHelper.createNegation(delta2.id, 'mod1', 'host1');
+      const negation3 = createDelta('mod1', 'host1').negate(delta2.id).buildV1();
 
       const allDeltas = [delta1, delta2, negation1, negation2, negation3];
 
@@ -104,7 +98,7 @@ describe('Negation System', () => {
         .setProperty('entity1', 'age', 25)
         .buildV1();
 
-      const negation1 = NegationHelper.createNegation(delta1.id, 'mod1', 'host1');
+      const negation1 = createDelta('mod1', 'host1').negate(delta1.id).buildV1();
       const allDeltas = [delta1, delta2, negation1];
 
       expect(NegationHelper.isDeltaNegated(delta1.id, allDeltas)).toBe(true);
@@ -124,8 +118,8 @@ describe('Negation System', () => {
         .setProperty('entity1', 'email', 'entity1@example.com')
         .buildV1();
 
-      const negation1 = NegationHelper.createNegation(delta1.id, 'mod1', 'host1');
+      const negation1 = createDelta('mod1', 'host1').negate(delta1.id).buildV1();
-      const negation2 = NegationHelper.createNegation(delta2.id, 'mod2', 'host1');
+      const negation2 = createDelta('mod2', 'host1').negate(delta2.id).buildV1();
 
       const allDeltas = [delta1, delta2, delta3, negation1, negation2];
       const filtered = NegationHelper.filterNegatedDeltas(allDeltas);
@@ -144,7 +138,7 @@ describe('Negation System', () => {
         .setProperty('entity1', 'age', 25)
         .buildV1();
 
-      const negation1 = NegationHelper.createNegation(delta1.id, 'mod1', 'host1');
+      const negation1 = createDelta('mod1', 'host1').negate(delta1.id).buildV1();
       const allDeltas = [delta1, delta2, negation1];
 
       const stats = NegationHelper.getNegationStats(allDeltas);
@@ -166,7 +160,7 @@ describe('Negation System', () => {
         .setProperty('entity1', 'status', 'active')
         .buildV1();
 
-      const negation1 = NegationHelper.createNegation(delta1.id, 'mod1', 'host1');
+      const negation1 = createDelta('mod1', 'host1').negate(delta1.id).buildV1();
       negation1.timeCreated = baseTime + 1000; // 1 second later
 
       const delta2 = createDelta('user1', 'host1')
@@ -174,7 +168,7 @@ describe('Negation System', () => {
         .setProperty('entity1', 'status', 'inactive')
         .buildV1();
 
-      const negation2 = NegationHelper.createNegation(delta2.id, 'mod1', 'host1');
+      const negation2 = createDelta('mod1', 'host1').negate(delta2.id).buildV1();
       negation2.timeCreated = baseTime + 3000; // 3 seconds later
 
       const allDeltas = [delta1, negation1, delta2, negation2];
@@ -193,11 +187,9 @@ describe('Negation System', () => {
         .buildV1();
 
       // Create negation delta
-      const negationDelta = NegationHelper.createNegation(
+      const negationDelta = createDelta('moderator', 'host1')
-        originalDelta.id,
+        .negate(originalDelta.id)
-        'moderator',
+        .buildV1();
-        'host1'
-      );
       
 
       // Create another non-negated delta
@@ -225,8 +217,8 @@ describe('Negation System', () => {
         .setProperty('post1', 'content', 'Original content')
         .buildV1();
 
-      const negation1 = NegationHelper.createNegation(originalDelta.id, 'mod1', 'host1');
+      const negation1 = createDelta('mod1', 'host1').negate(originalDelta.id).buildV1();
-      const negation2 = NegationHelper.createNegation(originalDelta.id, 'mod2', 'host1');
+      const negation2 = createDelta('mod2', 'host1').negate(originalDelta.id).buildV1();
 
       lossless.ingestDelta(originalDelta);
       lossless.ingestDelta(negation1);
@@ -247,7 +239,7 @@ describe('Negation System', () => {
         .setProperty('article1', 'content', 'Article content')
         .buildV1();
 
-      const negation1 = NegationHelper.createNegation(delta1.id, 'mod1', 'host1');
+      const negation1 = createDelta('mod1', 'host1').negate(delta1.id).buildV1();
 
       lossless.ingestDelta(delta1);
       lossless.ingestDelta(delta2);
@@ -268,11 +260,7 @@ describe('Negation System', () => {
         .setProperty('task1', 'status', 'pending')
         .buildV1();
 
-      const negationDelta = NegationHelper.createNegation(
+      const negationDelta = createDelta('admin', 'host1').negate(originalDelta.id).buildV1();
-        originalDelta.id,
-        'admin',
-        'host1'
-      );
 
       lossless.ingestDelta(originalDelta);
       lossless.ingestDelta(negationDelta);
@@ -299,7 +287,7 @@ describe('Negation System', () => {
         .buildV1();
 
       // Create negation delta in same transaction
-      const negationDelta = NegationHelper.createNegation(originalDelta.id, 'moderator', 'host1');
+      const negationDelta = createDelta('moderator', 'host1').negate(originalDelta.id).buildV1();
       negationDelta.pointers.unshift({
         localContext: '_transaction',
         target: transactionId,
@@ -324,7 +312,7 @@ describe('Negation System', () => {
         .buildV1();
 
       // Moderator negates it
-      const negationDelta = NegationHelper.createNegation(postDelta.id, 'moderator', 'host1');
+      const negationDelta = createDelta('moderator', 'host1').negate(postDelta.id).buildV1();
       negationDelta.timeCreated = baseTime + 1000;
 
       // User edits content (after negation)
@@ -351,11 +339,7 @@ describe('Negation System', () => {
 
   describe('Edge Cases', () => {
     it('should handle negation of non-existent deltas', () => {
-      const negationDelta = NegationHelper.createNegation(
+      const negationDelta = createDelta('moderator', 'host1').negate('non-existent-delta-id').buildV1();
-        'non-existent-delta-id',
-        'moderator',
-        'host1'
-      );
 
       lossless.ingestDelta(negationDelta);
 
@@ -371,7 +355,7 @@ describe('Negation System', () => {
         .setProperty('node1', 'child', 'node1') // Self-reference
         .buildV1();
 
-      const negationDelta = NegationHelper.createNegation(selfRefDelta.id, 'admin', 'host1');
+      const negationDelta = createDelta('admin', 'host1').negate(selfRefDelta.id).buildV1();
 
       lossless.ingestDelta(selfRefDelta);
       lossless.ingestDelta(negationDelta);
@@ -390,8 +374,8 @@ describe('Negation System', () => {
         .buildV1();
 
       // Create two negations of the same delta
-      const negation1 = NegationHelper.createNegation(originalDelta.id, 'user2', 'host1');
+      const negation1 = createDelta('user2', 'host1').negate(originalDelta.id).buildV1();
-      const negation2 = NegationHelper.createNegation(originalDelta.id, 'user3', 'host1');
+      const negation2 = createDelta('user3', 'host1').negate(originalDelta.id).buildV1();
 
       // Process all deltas
       testLossless.ingestDelta(originalDelta);
@@ -421,9 +405,9 @@ describe('Negation System', () => {
         .buildV1();
 
       // Create a chain of negations: B negates A, C negates B, D negates C
-      const deltaB = NegationHelper.createNegation(deltaA.id, 'user2', 'host1');
+      const deltaB = createDelta('user2', 'host1').negate(deltaA.id).buildV1();
-      const deltaC = NegationHelper.createNegation(deltaB.id, 'user3', 'host1');
+      const deltaC = createDelta('user3', 'host1').negate(deltaB.id).buildV1();
-      const deltaD = NegationHelper.createNegation(deltaC.id, 'user4', 'host1');
+      const deltaD = createDelta('user4', 'host1').negate(deltaC.id).buildV1();
 
       debug('Delta A (original): %s', deltaA.id);
       debug('Delta B (negates A): %s', deltaB.id);
@@ -498,8 +482,8 @@ describe('Negation System', () => {
         .buildV1();
 
       // Create negations for both deltas
-      const negation1 = NegationHelper.createNegation(delta1.id, 'user3', 'host1');
+      const negation1 = createDelta('user3', 'host1').negate(delta1.id).buildV1();
-      const negation2 = NegationHelper.createNegation(delta2.id, 'user4', 'host1');
+      const negation2 = createDelta('user4', 'host1').negate(delta2.id).buildV1();
 
       // Process all deltas
       testLossless.ingestDelta(delta1);

__tests__/schema.ts

@@ -3,7 +3,6 @@ import {
   PrimitiveSchemas, 
   ReferenceSchemas, 
   ArraySchemas,
-  // CommonSchemas has been moved to ./test-utils/schemas
   ObjectSchema
 } from '../src/schema';
 import { DefaultSchemaRegistry } from '../src/schema';

__tests__/test-utils.ts

@@ -1,98 +0,0 @@
-import { createOrchestrator } from '../src/orchestration/factory';
-import { NodeConfig, NodeOrchestrator } from '../src/orchestration/types';
-import Debug from 'debug';
-
-const debug = Debug('rz:test-utils');
-
-// Global test orchestrator instance
-let testOrchestrator: NodeOrchestrator;
-
-// Default test node configuration
-const DEFAULT_TEST_NODE_CONFIG: Partial<NodeConfig> = {
-  network: {
-    // Use default ports that will be overridden by getRandomPort() in the orchestrator
-    port: 0,
-  },
-  storage: {
-    type: 'memory',
-    path: '/data',
-  },
-};
-
-/**
- * Set up the test environment before all tests run
- */
-export const setupTestEnvironment = async () => {
-  debug('Setting up Docker test environment...');
-  
-  try {
-    // Create a Docker orchestrator instance
-    testOrchestrator = createOrchestrator('docker', {
-      // Enable auto-building of test images
-      autoBuildTestImage: true,
-      // Use a specific test image name
-      image: 'rhizome-node-test',
-    });
-    
-    debug('Docker test environment setup complete');
-  } catch (error) {
-    debug('Error setting up Docker test environment:', error);
-    throw error;
-  }
-};
-
-/**
- * Clean up the test environment after all tests complete
- */
-export const teardownTestEnvironment = async () => {
-  debug('Tearing down Docker test environment...');
-  
-  if (testOrchestrator) {
-    try {
-      // Clean up all containers and networks
-      await testOrchestrator.cleanup();
-      debug('Docker resources cleaned up successfully');
-    } catch (error) {
-      debug('Error during Docker environment teardown:', error);
-      // Don't throw to allow tests to complete
-    }
-  }
-  
-  debug('Docker test environment teardown complete');
-};
-
-/**
- * Get the test orchestrator instance
- */
-export const getTestOrchestrator = (): NodeOrchestrator => {
-  if (!testOrchestrator) {
-    throw new Error('Test orchestrator not initialized. Call setupTestEnvironment() first.');
-  }
-  return testOrchestrator;
-};
-
-/**
- * Create a test node with the given configuration
- */
-export const createTestNode = async (config: Partial<NodeConfig> = {}) => {
-  const orchestrator = getTestOrchestrator();
-  
-  // Merge default config with provided config
-  const nodeConfig: NodeConfig = {
-    ...DEFAULT_TEST_NODE_CONFIG,
-    ...config,
-    // Ensure we have a unique ID for each node
-    id: config.id || `test-node-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
-  };
-  
-  debug(`Creating test node with ID: ${nodeConfig.id}`);
-  
-  try {
-    const nodeHandle = await orchestrator.startNode(nodeConfig);
-    debug(`Test node ${nodeConfig.id} created successfully`);
-    return nodeHandle;
-  } catch (error) {
-    debug(`Error creating test node ${nodeConfig.id}:`, error);
-    throw error;
-  }
-};

delta-patterns/summary.md

@@ -0,0 +1,83 @@
+# Delta Patterns in Rhizome-Node
+
+This document outlines the distinct delta patterns identified in the Rhizome-Node test suite.
+
+## 1. Basic Entity Creation
+```typescript
+createDelta('creator', 'host')
+  .setProperty('entity1', 'name', 'Alice', 'user')
+  .buildV1();
+```
+
+## 2. Relationship Creation
+```typescript
+createDelta('creator', 'host')
+  .addPointer('users', 'alice', 'friends')
+  .addPointer('friend', 'bob')
+  .addPointer('type', 'friendship')
+  .buildV1();
+```
+
+## 3. Transaction-Enabled Deltas
+```typescript
+createDelta('user1', 'host1')
+  .inTransaction('tx123')
+  .setProperty('doc1', 'status', 'draft')
+  .buildV1();
+```
+
+## 4. Negation Deltas
+```typescript
+// Creating a negation delta
+const delta = createDelta('user1', 'host1').buildV1();
+const negation = createDelta('moderator', 'host1').negate(delta.id).buildV1();
+```
+
+## 5. Temporal Deltas
+```typescript
+createDelta('user1', 'host1')
+  .withTimestamp(1624233600000)
+  .setProperty('entity1', 'score', 100, 'game')
+  .buildV1();
+```
+
+## 6. Multi-Property Deltas
+```typescript
+createDelta('user1', 'host1')
+  .setProperty('entity1', 'title', 'Hello World', 'post')
+  .setProperty('entity1', 'content', 'This is a test', 'post')
+  .setProperty('entity1', 'published', true, 'post')
+  .buildV1();
+```
+
+## 7. Reference-Only Deltas
+```typescript
+createDelta('system', 'host1')
+  .addPointer('posts', 'post1', 'recent')
+  .buildV1();
+```
+
+## 8. Bulk Operation Deltas
+```typescript
+// Multiple entities in a single delta
+createDelta('batch', 'host1')
+  .setProperty('user1', 'status', 'active', 'user')
+  .setProperty('user2', 'status', 'inactive', 'user')
+  .buildV1();
+```
+
+## 9. Versioned Deltas
+```typescript
+// V1 format
+createDelta('a', 'h').buildV1();
+// V2 format
+createDelta('a', 'h').buildV2();
+```
+
+## Key Observations
+- Most deltas follow a fluent builder pattern
+- Deltas can be composed of multiple operations (setProperty, addPointer, etc.)
+- Support for both V1 and V2 delta formats
+- Strong typing and schema validation is commonly used
+- Transaction support is built into the delta creation process
+- Temporal aspects can be explicitly controlled

docs/json-ast.md

@@ -0,0 +1,191 @@
+# JSON AST (Abstract Syntax Tree)
+
+JSON AST is a representation of a JSON object as a tree of nodes.
+
+The following is obtained from running
+
+    ts-node examples/json-ast.ts
+
+## Original JSON:
+```json
+{
+  "name": "John Doe",
+  "age": 42,
+  "active": true,
+  "scores": [
+    95,
+    87,
+    92
+  ],
+  "address": {
+    "street": "123 Main St",
+    "city": "Anytown",
+    "coordinates": {
+      "lat": 42.1234,
+      "lng": -71.2345
+    }
+  },
+  "tags": [
+    "admin",
+    "user",
+    "premium"
+  ],
+  "metadata": {
+    "createdAt": "2023-01-01T00:00:00Z",
+    "updatedAt": "2023-06-21T12:34:56Z"
+  }
+}
+```
+
+## AST:
+```json
+{
+  "type": "object",
+  "children": [
+    {
+      "type": "string",
+      "value": "John Doe",
+      "path": "name",
+      "key": "name"
+    },
+    {
+      "type": "number",
+      "value": 42,
+      "path": "age",
+      "key": "age"
+    },
+    {
+      "type": "boolean",
+      "value": true,
+      "path": "active",
+      "key": "active"
+    },
+    {
+      "type": "array",
+      "children": [
+        {
+          "type": "number",
+          "value": 95,
+          "path": "scores[0]"
+        },
+        {
+          "type": "number",
+          "value": 87,
+          "path": "scores[1]"
+        },
+        {
+          "type": "number",
+          "value": 92,
+          "path": "scores[2]"
+        }
+      ],
+      "path": "scores",
+      "key": "scores"
+    },
+    {
+      "type": "object",
+      "children": [
+        {
+          "type": "string",
+          "value": "123 Main St",
+          "path": "address.street",
+          "key": "street"
+        },
+        {
+          "type": "string",
+          "value": "Anytown",
+          "path": "address.city",
+          "key": "city"
+        },
+        {
+          "type": "object",
+          "children": [
+            {
+              "type": "number",
+              "value": 42.1234,
+              "path": "address.coordinates.lat",
+              "key": "lat"
+            },
+            {
+              "type": "number",
+              "value": -71.2345,
+              "path": "address.coordinates.lng",
+              "key": "lng"
+            }
+          ],
+          "path": "address.coordinates",
+          "key": "coordinates"
+        }
+      ],
+      "path": "address",
+      "key": "address"
+    },
+    {
+      "type": "array",
+      "children": [
+        {
+          "type": "string",
+          "value": "admin",
+          "path": "tags[0]"
+        },
+        {
+          "type": "string",
+          "value": "user",
+          "path": "tags[1]"
+        },
+        {
+          "type": "string",
+          "value": "premium",
+          "path": "tags[2]"
+        }
+      ],
+      "path": "tags",
+      "key": "tags"
+    },
+    {
+      "type": "object",
+      "children": [
+        {
+          "type": "string",
+          "value": "2023-01-01T00:00:00Z",
+          "path": "metadata.createdAt",
+          "key": "createdAt"
+        },
+        {
+          "type": "string",
+          "value": "2023-06-21T12:34:56Z",
+          "path": "metadata.updatedAt",
+          "key": "updatedAt"
+        }
+      ],
+      "path": "metadata",
+      "key": "metadata"
+    }
+  ]
+}
+```
+
+## Traversed AST:
+```text
+OBJECT
+    STRING: "John Doe" [name]
+    NUMBER: 42 [age]
+    BOOLEAN: true [active]
+    ARRAY [scores]
+        NUMBER: 95 [scores[0]]
+        NUMBER: 87 [scores[1]]
+        NUMBER: 92 [scores[2]]
+    OBJECT [address]
+        STRING: "123 Main St" [address.street]
+        STRING: "Anytown" [address.city]
+        OBJECT [address.coordinates]
+            NUMBER: 42.1234 [address.coordinates.lat]
+            NUMBER: -71.2345 [address.coordinates.lng]
+    ARRAY [tags]
+        STRING: "admin" [tags[0]]
+        STRING: "user" [tags[1]]
+        STRING: "premium" [tags[2]]
+    OBJECT [metadata]
+        STRING: "2023-01-01T00:00:00Z" [metadata.createdAt]
+        STRING: "2023-06-21T12:34:56Z" [metadata.updatedAt]
+```

docs/schema-validation.md

@@ -0,0 +1,90 @@
+# Schema Validation in Rhizome-Node
+
+This document explains how schema validation works with deltas in Rhizome-Node.
+
+## Overview
+
+Schema validation in Rhizome-Node is enforced at the `TypedCollection` level when using the `put` method, which validates data before creating deltas. This means:
+
+1. **Local Changes**: When you use `collection.put()`, the data is validated against the schema before any deltas are created and ingested.
+2. **Peer Changes**: Deltas received from other peers are ingested without validation by default, which means invalid data can enter the system.
+3. **Validation Tracking**: The system tracks which entities are valid/invalid after ingestion.
+
+## Example Usage
+
+```typescript
+// 1. Define a schema for users
+const userSchema = SchemaBuilder
+  .create('user')
+  .name('User')
+  .property('name', PrimitiveSchemas.requiredString())
+  .property('email', PrimitiveSchemas.email())
+  .property('age', PrimitiveSchemas.integer({ minimum: 0 }))
+  .required('name')
+  .build();
+
+// 2. Create a typed collection with strict validation
+const collection = new TypedCollectionImpl<{
+  name: string;
+  email?: string;
+  age?: number;
+}>('users', userSchema, schemaRegistry, {
+  strictValidation: true // Enable strict validation
+});
+
+// Connect to the node
+collection.rhizomeConnect(node);
+
+// 3. Local changes - validated on put()
+// Valid usage - will pass schema validation
+await collection.put('user1', { 
+  name: 'Alice', 
+  email: 'alice@example.com',
+  age: 30
+});
+
+// Invalid usage - will throw SchemaValidationError
+await expect(collection.put('user2', {
+  email: 'invalid-email', // Invalid email format
+  age: -5                 // Negative age
+})).rejects.toThrow(SchemaValidationError);
+
+// 4. Peer data - ingested without validation by default
+const unsafeDelta = createDelta('peer1', 'peer1')
+  .setProperty('user3', 'name', 'Bob', 'users')
+  .setProperty('user3', 'age', 'not-a-number', 'users')
+  .buildV1();
+
+// This will be ingested without validation
+node.lossless.ingestDelta(unsafeDelta);
+
+// 5. Check validation status after the fact
+const stats = collection.getValidationStats();
+console.log(`Valid: ${stats.validEntities}, Invalid: ${stats.invalidEntities}`);
+
+// Get details about invalid entities
+const invalidUsers = collection.getInvalidEntities();
+invalidUsers.forEach(user => {
+  console.log(`User ${user.entityId} is invalid:`, user.errors);
+});
+```
+
+## Key Points
+
+### Validation Timing
+- Schema validation happens in `TypedCollection.put()` before deltas are created
+- Deltas from peers are ingested without validation by default
+
+### Validation Modes
+- `strictValidation: true`: Throws errors on invalid data (recommended for local changes)
+- `strictValidation: false`: Allows invalid data but tracks it (default)
+
+### Monitoring
+- Use `getValidationStats()` to get counts of valid/invalid entities
+- Use `getInvalidEntities()` to get detailed error information
+
+### Best Practices
+- Always validate data before creating deltas when accepting external input
+- Use `strictValidation: true` for collections where data integrity is critical
+- Monitor validation statistics in production to detect data quality issues
+- Consider implementing a validation layer for peer data if needed

examples/json-ast.ts

@@ -0,0 +1,51 @@
+import { jsonToAst } from '../src/util/json-ast/index';
+
+// Example JSON data
+const exampleJson = {
+  name: "John Doe",
+  age: 42,
+  active: true,
+  scores: [95, 87, 92],
+  address: {
+    street: "123 Main St",
+    city: "Anytown",
+    coordinates: {
+      lat: 42.1234,
+      lng: -71.2345
+    }
+  },
+  tags: ["admin", "user", "premium"],
+  metadata: {
+    createdAt: "2023-01-01T00:00:00Z",
+    updatedAt: "2023-06-21T12:34:56Z"
+  }
+};
+
+// Convert JSON to AST with path information
+const ast = jsonToAst(exampleJson, { 
+  includePath: true,
+  maxDepth: 10,
+  // Optional filter - only include nodes with paths that include 'address'
+  // filter: (node) => !node.path || node.path.includes('address')
+});
+
+console.log("Original JSON:", JSON.stringify(exampleJson, null, 2));
+
+console.log("\nAST:", JSON.stringify(ast, null, 2));
+
+// Example of traversing the AST
+function traverse(node: any, indent = 0) {
+  const padding = '  '.repeat(indent);
+  const type = node.type.toUpperCase();
+  const value = node.value !== undefined ? `: ${JSON.stringify(node.value)}` : '';
+  const path = node.path ? ` [${node.path}]` : '';
+  
+  console.log(`${padding}${type}${value}${path}`);
+  
+  if (node.children) {
+    node.children.forEach((child: any) => traverse(child, indent + 2));
+  }
+}
+
+console.log('\nTraversed AST:');
+traverse(ast);

markdown/coverage_report.md

@@ -1,41 +0,0 @@
-
-> rhizome-node@0.1.0 test
-> jest --coverage
-
-PASS __tests__/peer-address.ts
-PASS __tests__/lossy.ts
-PASS __tests__/lossless.ts
-PASS __tests__/run/001-single-node.ts
-PASS __tests__/run/002-two-nodes.ts
--------------------|---------|----------|---------|---------|----------------------------------------------------
-File               | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s
--------------------|---------|----------|---------|---------|----------------------------------------------------
-All files          |   81.01 |    72.28 |    77.4 |   85.03 |
- src               |   86.71 |    79.89 |   83.67 |    91.2 |
-  collection.ts    |   79.54 |    68.18 |      76 |   82.92 | 10,60,69,113-120,125,128,139,142,146,173,180
-  config.ts        |      95 |    89.65 |      50 |      95 | 24
-  delta.ts         |      95 |    83.33 |     100 |      95 | 31
-  deltas.ts        |   68.65 |    70.96 |   78.26 |   72.58 | 6,43-46,57-61,69-70,77-85
-  lossless.ts      |   93.02 |    90.24 |   92.85 |   93.02 | 36-40,56,113
-  lossy.ts         |   97.29 |    81.81 |     100 |   97.29 | 36
-  node.ts          |   91.07 |       85 |   88.23 |   97.87 | 6
-  peers.ts         |   91.86 |    88.46 |   93.54 |    96.1 | 6,121-122
-  pub-sub.ts       |   81.81 |    69.44 |   73.07 |   93.65 | 6,15-16,51
-  request-reply.ts |   81.17 |    68.42 |   75.86 |   91.54 | 6,15-16,58,72,100
-  transactions.ts  |   98.11 |    95.55 |   93.33 |      98 | 99
-  types.ts         |     100 |      100 |     100 |     100 |
- src/http          |   55.72 |    47.45 |      50 |   59.66 |
-  api.ts           |   43.66 |    31.42 |    42.3 |   45.31 | 6,22,26,31-48,53,67-68,81-120
-  html.ts          |      60 |        0 |      40 |      60 | 17-18,22-29,33
-  index.ts         |   77.14 |    77.27 |   69.23 |      90 | 3,5-6
- src/util          |   72.16 |    53.52 |      75 |   72.94 |
-  md-files.ts      |   72.16 |    53.52 |      75 |   72.94 | 10-11,16,21-23,74-78,92-95,108,110,114-118,131-138
- util              |     100 |      100 |     100 |     100 |
-  app.ts           |     100 |      100 |     100 |     100 |
--------------------|---------|----------|---------|---------|----------------------------------------------------
-
-Test Suites: 1 skipped, 5 passed, 5 of 6 total
-Tests:       2 skipped, 7 passed, 9 total
-Snapshots:   0 total
-Time:        4.51 s, estimated 5 s
-Ran all test suites.

package.json

@@ -7,8 +7,7 @@
     "build:watch": "tsc --watch",
     "lint": "eslint",
     "test": "jest",
-    "coverage": "./scripts/coverage.sh",
+    "coverage": "npm run test -- --coverage --coverageDirectory=coverage",
-    "coverage-report": "npm run test -- --coverage --coverageDirectory=coverage",
     "example-app": "node dist/examples/app.js",
     "stop-all": "docker ps -a --filter \"name=^/rhizome-node-\" --format {{.Names}} | xargs -r docker stop",
     "build-test-image": "docker build -t rhizome-node-test -f Dockerfile.test ."

plans/README.md

@@ -0,0 +1,18 @@
+# Project Initiatives
+
+This directory contains planning documents and proposals for Rhizome Node development initiatives.
+
+## Purpose
+
+- Document project goals and roadmaps
+- Propose and discuss new features
+- Plan research and development efforts
+
+## Naming Conventions
+
+- Use kebab-case for all filenames (e.g., `distributed-sync-research.md`)
+- Include dates in filenames for time-sensitive documents (e.g., `2025-06-peer-discovery-proposal.md`)
+
+## Contributing
+
+1. Create a new markdown file for your proposal or research

scripts/coverage.sh

@@ -1,17 +0,0 @@
-#!/bin/env bash
-
-force=false
-
-while [[ -n "$1" ]]; do
-  case "$1" in
-    -f | --force)
-      force=true
-      ;;
-  esac
-  shift
-done
-
-dest="./markdown/coverage_report.md"
-
-npm run test -- --coverage 2>&1 | tee "$dest"
-sed -i 's/\s*$//' "$dest"

src/features/negation.ts

@@ -14,23 +14,6 @@ export interface NegationPointer {
 
 // Helper functions for creating and identifying negation deltas
 export class NegationHelper {
-  
-  /**
-   * Create a negation delta that negates another delta
-   */
-  static createNegation(
-    deltaToNegate: DeltaID,
-    creator: CreatorID,
-    host: HostID
-  ): Delta {
-    const negationDelta = createDelta(creator, host)
-      .negate(deltaToNegate)
-      .buildV1();
-
-    debug(`Created negation delta ${negationDelta.id} negating ${deltaToNegate}`);
-    return negationDelta;
-  }
-
   /**
    * Check if a delta is a negation delta
    */
@@ -361,14 +344,6 @@ export class NegationHelper {
     };
   }
   
-  /**
-   * Helper to check if a delta with the given ID is a negation delta
-   */
-  private static isNegationDeltaById(deltaId: DeltaID, deltas: Delta[]): boolean {
-    const delta = deltas.find(d => d.id === deltaId);
-    return delta ? this.isNegationDelta(delta) : false;
-  }
-
   /**
    * Apply negations to a delta stream in chronological order
    * Later negations can override earlier ones

src/schema/schema.ts

@@ -198,9 +198,6 @@ export class SchemaBuilder {
   }
 }
 
-// Common schema patterns have been moved to __tests__/test-utils/schemas.ts
-// since they are only used for testing purposes.
-
 /**
  * Context for tracking resolution state during nested object resolution
  * Prevents circular references and manages depth tracking

src/util/json-ast/index.ts

@@ -0,0 +1,114 @@
+import { JsonValue, JsonNode, JsonAstOptions } from './types';
+
+/**
+ * Convert a JSON value to an Abstract Syntax Tree (AST)
+ * @param json The JSON value to convert
+ * @param options Configuration options
+ * @param currentPath Internal use: current path in the JSON structure
+ * @param depth Internal use: current depth in the JSON structure
+ * @returns The root node of the AST
+ */
+export function jsonToAst(
+  json: JsonValue,
+  options: JsonAstOptions = {},
+  currentPath: string = '',
+  depth: number = 0
+): JsonNode {
+  const { includePath = true, maxDepth = 100, filter } = options;
+  
+  // Handle max depth
+  if (depth > maxDepth) {
+    return { 
+      type: typeof json === 'object' && json !== null ? 'object' : typeof json as any,
+      value: '[Max depth exceeded]',
+      ...(includePath && currentPath ? { path: currentPath } : {})
+    };
+  }
+
+  // Handle null
+  if (json === null) {
+    return createNode('null', null, currentPath, includePath);
+  }
+
+  // Handle primitive types
+  const type = typeof json as 'string' | 'number' | 'boolean' | 'object';
+  if (type !== 'object') {
+    return createNode(type, json, currentPath, includePath);
+  }
+
+  // Handle arrays
+  if (Array.isArray(json)) {
+    const node: JsonNode = {
+      type: 'array',
+      children: json
+        .map((item, index) => {
+          const childPath = includePath ? `${currentPath}[${index}]` : '';
+          return jsonToAst(item, options, childPath, depth + 1);
+        })
+    };
+    
+    if (includePath && currentPath) {
+      node.path = currentPath;
+    }
+    
+    return node;
+  }
+
+  // Handle objects
+  const children: JsonNode[] = [];
+  for (const [key, value] of Object.entries(json)) {
+    const childPath = includePath 
+      ? currentPath ? `${currentPath}.${key}` : key 
+      : '';
+      
+    const childNode = jsonToAst(value, options, childPath, depth + 1);
+    childNode.key = key;
+    children.push(childNode);
+  }
+
+  const node: JsonNode = { 
+    type: 'object',
+    children: filter ? children.filter(filter) : children 
+  };
+  
+  if (includePath && currentPath) {
+    node.path = currentPath;
+  }
+  
+  return node;
+}
+
+/**
+ * Create a new AST node with the given properties
+ */
+function createNode(
+  type: JsonNode['type'],
+  value: any,
+  path: string = '',
+  includePath: boolean = true
+): JsonNode {
+  const node: JsonNode = { type, value };
+  if (includePath && path) {
+    node.path = path;
+  }
+  return node;
+}
+
+/**
+ * Example usage of the JSON to AST converter
+ */
+function exampleUsage() {
+  const exampleJson = {
+    name: "John",
+    age: 30,
+    active: true,
+    tags: ["admin", "user"],
+    address: {
+      street: "123 Main St",
+      city: "Anytown"
+    }
+  };
+
+  const ast = jsonToAst(exampleJson, { includePath: true });
+  console.log(JSON.stringify(ast, null, 2));
+}

src/util/json-ast/types.ts

@@ -0,0 +1,21 @@
+export type JsonValue = 
+  | string 
+  | number 
+  | boolean 
+  | null 
+  | JsonValue[] 
+  | { [key: string]: JsonValue };
+
+export interface JsonNode {
+  type: 'object' | 'array' | 'string' | 'number' | 'boolean' | 'null';
+  value?: any;
+  children?: JsonNode[];
+  key?: string;
+  path?: string; // Path to this node in the JSON (e.g., 'address.city')
+}
+
+export interface JsonAstOptions {
+  includePath?: boolean;  // Whether to include path information in nodes
+  maxDepth?: number;      // Maximum depth to traverse
+  filter?: (node: JsonNode) => boolean; // Optional filter function
+}

summary.md

@@ -0,0 +1,108 @@
+# Rhizome-Node Repository Analysis
+
+## Core Architecture
+
+### 1. Delta System
+- **Delta Types**: Implements V1 (array-based) and V2 (object-based) delta formats
+- **Delta Lifecycle**: 
+  - Creation via `DeltaBuilder`
+  - Propagation through `DeltaStream`
+  - Storage in `Lossless` view
+  - Transformation in `Lossy` views
+
+### 2. Network Layer
+- **Communication**:
+  - Pub/Sub system for delta propagation
+  - Request/Reply pattern for direct communication
+  - Peer management
+- **Delta Propagation**:
+  - Deduplication using content hashing
+  - Policy-based acceptance/rejection
+  - Queuing for deferred processing
+
+### 3. Storage
+- **In-Memory Storage**:
+  - `Lossless` view maintains complete delta history
+  - `Lossy` views provide optimized access patterns
+- **Persistence**:
+  - LevelDB integration
+  - Delta compaction strategies
+
+### 4. Schema System
+- **Type Definitions**:
+  - Support for primitives, references, and arrays
+  - Validation rules
+- **Schema Registry**:
+  - Central schema management
+  - Versioning support
+
+## Key Components
+
+1. **Core**:
+   - `delta.ts`: Core delta implementation
+   - `delta-builder.ts`: Fluent API for delta creation
+   - `entity.ts`: Base entity definitions
+
+2. **Network**:
+   - `delta-stream.ts`: Delta propagation and management
+   - `pub-sub.ts`: Publish/subscribe functionality
+   - `request-reply.ts`: Direct node communication
+
+3. **Views**:
+   - `lossless.ts`: Complete delta history
+   - `lossy.ts`: Derived, optimized views
+
+4. **Schema**:
+   - `schema.ts`: Type definitions
+   - `schema-registry.ts`: Schema management
+
+## Strengths
+
+1. **Flexible Data Model**: Hypergraph structure supports complex relationships
+2. **Extensible**: Plugin architecture for storage and networking
+3. **Type Safety**: Comprehensive TypeScript types
+4. **Incremental Processing**: Efficient updates with `Lossy` views
+
+## Areas for Improvement
+
+1. **Documentation**:
+   - Limited inline documentation
+   - Need for architectural overview
+   - Example implementations
+
+2. **Testing**:
+   - Incomplete test coverage
+   - Need for integration tests
+   - Performance benchmarking
+
+3. **Scalability**:
+   - In-memory storage limits
+   - Delta compaction strategy
+   - Sharding support
+
+4. **Security**:
+   - Authentication/authorization
+   - Delta signing/verification
+   - Encryption
+
+## Recommended Next Steps
+
+1. **Documentation**:
+   - Create architecture diagrams
+   - Add usage examples
+   - Document extension points
+
+2. **Testing**:
+   - Increase test coverage
+   - Add performance benchmarks
+   - Test at scale
+
+3. **Features**:
+   - Implement delta compression
+   - Add conflict resolution strategies
+   - Support for offline operation
+
+4. **Tooling**:
+   - CLI for administration
+   - Monitoring/metrics
+   - Debugging tools