Compare commits

..

No commits in common. "76e00f38afcbb146cd6e7ba65ad6aeb0540c0608" and "54a50a9c222ce7a1de69bd8b13a9da8d92bf694a" have entirely different histories.

13 changed files with 86 additions and 376 deletions

1
.gitignore vendored
View File

@ -7,4 +7,3 @@ coverage/
data/ data/
test-data/ test-data/
*.code-workspace *.code-workspace
local-notes/

View File

@ -2,11 +2,13 @@ import { createDelta } from '../src/core/delta-builder';
import { import {
RhizomeNode, RhizomeNode,
Lossless, Lossless,
Delta,
LastWriteWins,
TimestampResolver,
SumResolver, SumResolver,
CustomResolver, CustomResolver,
LastWriteWinsPlugin, LastWriteWinsPlugin,
MajorityVotePlugin, MajorityVotePlugin
TimestampResolver
} from "../src"; } from "../src";
describe('Concurrent Write Scenarios', () => { describe('Concurrent Write Scenarios', () => {
@ -37,7 +39,7 @@ describe('Concurrent Write Scenarios', () => {
.buildV1() .buildV1()
); );
const resolver = new TimestampResolver(lossless); const resolver = new LastWriteWins(lossless);
const result = resolver.resolve(); const result = resolver.resolve();
expect(result).toBeDefined(); expect(result).toBeDefined();
@ -121,7 +123,7 @@ describe('Concurrent Write Scenarios', () => {
.buildV1() .buildV1()
); );
const resolver = new TimestampResolver(lossless); const resolver = new LastWriteWins(lossless);
const result = resolver.resolve(); const result = resolver.resolve();
expect(result).toBeDefined(); expect(result).toBeDefined();
@ -247,7 +249,7 @@ describe('Concurrent Write Scenarios', () => {
); );
} }
const resolver = new TimestampResolver(lossless); const resolver = new LastWriteWins(lossless);
const result = resolver.resolve(); const result = resolver.resolve();
expect(result).toBeDefined(); expect(result).toBeDefined();

View File

@ -2,7 +2,7 @@ import { createDelta } from '../src/core/delta-builder';
import { DeltaV1, DeltaV2 } from '../src/core/delta'; import { DeltaV1, DeltaV2 } from '../src/core/delta';
import { Lossless } from '../src/views/lossless'; import { Lossless } from '../src/views/lossless';
import { RhizomeNode } from '../src/node'; import { RhizomeNode } from '../src/node';
import { TimestampResolver } from '../src/views/resolvers/timestamp-resolvers'; import { LastWriteWins } from '../src/views/resolvers/last-write-wins';
describe('DeltaBuilder', () => { describe('DeltaBuilder', () => {
const creator = 'creator-123'; const creator = 'creator-123';
@ -47,7 +47,7 @@ describe('DeltaBuilder', () => {
// Verify that the entity property resolves correctly // Verify that the entity property resolves correctly
const lossless = new Lossless(node); const lossless = new Lossless(node);
lossless.ingestDelta(delta); lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless); const lossy = new LastWriteWins(lossless);
const result = lossy.resolve(); const result = lossy.resolve();
expect(result).toBeDefined(); expect(result).toBeDefined();
expect(result!['entity-1'].properties.name).toBe('Test Entity'); expect(result!['entity-1'].properties.name).toBe('Test Entity');
@ -72,7 +72,7 @@ describe('DeltaBuilder', () => {
// Verify that the entity property resolves correctly // Verify that the entity property resolves correctly
const lossless = new Lossless(node); const lossless = new Lossless(node);
lossless.ingestDelta(delta); lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless); const lossy = new LastWriteWins(lossless);
const result = lossy.resolve(); const result = lossy.resolve();
expect(result).toBeDefined(); expect(result).toBeDefined();
expect(result!['entity-1'].properties.name).toBe('Test Entity'); expect(result!['entity-1'].properties.name).toBe('Test Entity');
@ -80,55 +80,18 @@ describe('DeltaBuilder', () => {
it('should create a V1 delta with relationships', () => { it('should create a V1 delta with relationships', () => {
const delta = createDelta(creator, host) const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows') .relate('user-1', 'follows', 'user-2')
.buildV1(); .buildV1();
// This delta sets values on a new relationship entity
expect(delta.pointers).toContainEqual({ expect(delta.pointers).toContainEqual({
localContext: '_rel_target', localContext: 'follows',
target: expect.any(String), target: 'user-2',
targetContext: 'target' targetContext: 'follows'
});
const relId = delta.pointers.find(p => p.localContext === '_rel_target')?.target;
expect(delta.pointers).toContainEqual({
localContext: '_rel_source',
target: relId,
targetContext: 'source'
}); });
expect(delta.pointers).toContainEqual({ expect(delta.pointers).toContainEqual({
localContext: '_rel_type', localContext: 'source',
target: relId, target: 'user-1',
targetContext: 'type' targetContext: 'follows'
});
});
it('should create a V1 delta with relationships and properties', () => {
const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows', { version: 1})
.buildV1();
// This delta sets values on a new relationship entity
expect(delta.pointers).toContainEqual({
localContext: '_rel_target',
target: expect.any(String),
targetContext: 'target'
});
const relId = delta.pointers.find(p => p.localContext === '_rel_target')?.target;
expect(delta.pointers).toContainEqual({
localContext: '_rel_source',
target: relId,
targetContext: 'source'
});
expect(delta.pointers).toContainEqual({
localContext: '_rel_type',
target: relId,
targetContext: 'type'
});
expect(delta.pointers).toContainEqual({
localContext: '_rel_version',
target: relId,
targetContext: 'version'
}); });
}); });
}); });
@ -158,61 +121,11 @@ describe('DeltaBuilder', () => {
it('should create a V2 delta with relationships', () => { it('should create a V2 delta with relationships', () => {
const delta = createDelta(creator, host) const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows') .relate('user-1', 'follows', 'user-2')
.buildV2(); .buildV2();
expect(delta.pointers).toHaveProperty('_rel_source'); expect(delta.pointers).toHaveProperty('follows', { 'user-2': 'follows' });
const [relId] = Object.entries(delta.pointers._rel_source!)[0]; expect(delta.pointers).toHaveProperty('source', { 'user-1': 'follows' });
expect(delta.pointers).toHaveProperty('_rel_source', { [relId]: 'source' });
expect(delta.pointers).toHaveProperty('_rel_target', { [relId]: 'target' });
expect(delta.pointers).toHaveProperty('_rel_type', { [relId]: 'type' });
expect(delta.pointers).toHaveProperty('source', 'user-1');
expect(delta.pointers).toHaveProperty('target', 'user-2');
expect(delta.pointers).toHaveProperty('type', 'follows');
const lossless = new Lossless(node);
lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless);
const result = lossy.resolve([relId]);
expect(result).toBeDefined();
expect(result![relId]).toMatchObject({
properties: {
source: 'user-1',
target: 'user-2',
type: 'follows'
}
});
});
it('should create a V2 delta with relationships and properties', () => {
const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows', { version: 1})
.buildV2();
expect(delta.pointers).toHaveProperty('_rel_source');
const [relId] = Object.entries(delta.pointers._rel_source!)[0];
expect(delta.pointers).toHaveProperty('_rel_source', { [relId]: 'source' });
expect(delta.pointers).toHaveProperty('_rel_target', { [relId]: 'target' });
expect(delta.pointers).toHaveProperty('_rel_type', { [relId]: 'type' });
expect(delta.pointers).toHaveProperty('_rel_version', { [relId]: 'version' });
expect(delta.pointers).toHaveProperty('source', 'user-1');
expect(delta.pointers).toHaveProperty('target', 'user-2');
expect(delta.pointers).toHaveProperty('type', 'follows');
expect(delta.pointers).toHaveProperty('version', 1);
const lossless = new Lossless(node);
lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless);
const result = lossy.resolve([relId]);
expect(result).toBeDefined();
expect(result![relId]).toMatchObject({
properties: {
source: 'user-1',
target: 'user-2',
type: 'follows',
version: 1
}
});
}); });
}); });

View File

@ -4,7 +4,7 @@
// NO_DEBUG=true npm test // will not set DEBUG // NO_DEBUG=true npm test // will not set DEBUG
// DEBUG=other npm test // will set DEBUG=other // DEBUG=other npm test // will set DEBUG=other
if (!process.env.DEBUG && !process.env.NO_DEBUG) { if (!process.env.DEBUG && !process.env.NO_DEBUG) {
// process.env.DEBUG = 'rz:*'; process.env.DEBUG = 'rz:*';
} }
// This is a placeholder test to satisfy Jest's requirement for at least one test // This is a placeholder test to satisfy Jest's requirement for at least one test

View File

@ -1,19 +1,15 @@
import Debug from "debug"; import Debug from "debug";
import { createDelta } from '../src/core/delta-builder'; import { createDelta } from '../src/core/delta-builder';
import {Lossless, RhizomeNode} from "../src"; import {Delta, LastWriteWins, Lossless, RhizomeNode} from "../src";
import {TimestampResolver} from "../src/views/resolvers/timestamp-resolvers";
const debug = Debug('test:last-write-wins'); const debug = Debug('test:last-write-wins');
// This was initially written to test a LastWriteWins resolver, but that has been
// superceded by the TimestampResolver.
describe('Last write wins', () => { describe('Last write wins', () => {
describe('given that two separate writes occur', () => { describe('given that two separate writes occur', () => {
const node = new RhizomeNode(); const node = new RhizomeNode();
const lossless = new Lossless(node); const lossless = new Lossless(node);
const lossy = new TimestampResolver(lossless); const lossy = new LastWriteWins(lossless);
beforeAll(() => { beforeAll(() => {
lossless.ingestDelta(createDelta('a', 'h') lossless.ingestDelta(createDelta('a', 'h')

View File

@ -1,4 +1,3 @@
import { LosslessViewOne } from '../src/views/lossless';
import { import {
SchemaBuilder, SchemaBuilder,
PrimitiveSchemas, PrimitiveSchemas,
@ -156,7 +155,7 @@ describe('Schema System', () => {
schemaRegistry.register(userSchema); schemaRegistry.register(userSchema);
// Create a valid lossless view // Create a valid lossless view
const validView: LosslessViewOne = { const validView = {
id: 'user123', id: 'user123',
referencedAs: ['user'], referencedAs: ['user'],
propertyDeltas: { propertyDeltas: {
@ -174,7 +173,7 @@ describe('Schema System', () => {
creator: 'creator1', creator: 'creator1',
pointers: [{ age: 25 }] pointers: [{ age: 25 }]
}] }]
}, }
}; };
const result = schemaRegistry.validate('user123', 'user', validView); const result = schemaRegistry.validate('user123', 'user', validView);
@ -182,7 +181,7 @@ describe('Schema System', () => {
expect(result.errors).toHaveLength(0); expect(result.errors).toHaveLength(0);
// Test invalid view (missing required property) // Test invalid view (missing required property)
const invalidView: LosslessViewOne = { const invalidView = {
id: 'user456', id: 'user456',
referencedAs: ['user'], referencedAs: ['user'],
propertyDeltas: { propertyDeltas: {
@ -218,7 +217,7 @@ describe('Schema System', () => {
schemaRegistry.register(schema); schemaRegistry.register(schema);
// Valid types // Valid types
const validView: LosslessViewOne = { const validView = {
id: 'test1', id: 'test1',
referencedAs: [], referencedAs: [],
propertyDeltas: { propertyDeltas: {
@ -232,7 +231,7 @@ describe('Schema System', () => {
expect(validResult.valid).toBe(true); expect(validResult.valid).toBe(true);
// Invalid types // Invalid types
const invalidView: LosslessViewOne = { const invalidView = {
id: 'test2', id: 'test2',
referencedAs: [], referencedAs: [],
propertyDeltas: { propertyDeltas: {

View File

@ -1,222 +0,0 @@
# Entity Relationship Graph Implementation Plan
## Overview
This document outlines the plan to implement entity relationship tracking in the rhizome-node system. The implementation treats relationships as first-class entities, each with their own identity and properties.
## Core Design
### Relationship as First-Class Entities
- Each relationship is a domain entity with its own unique ID
- Relationships have standard properties: `source`, `target`, and `type`
- Additional properties can be added to relationships
- Relationships are created using the `relate()` method in `DeltaBuilder`
### Delta Structure for Relationships
```typescript
// Creating a relationship
createDelta(creator, host)
.relate(
sourceId, // ID of the source entity
targetId, // ID of the target entity
'REL_TYPE', // Relationship type
{ // Optional properties
prop1: 'value1',
prop2: 'value2'
}
)
.build();
```
### Data Structures
#### `LosslessEntity` Updates
```typescript
class LosslessEntity {
// Existing properties
properties = new Map<PropertyID, Set<Delta>>();
// Track relationships where this entity is the source
outboundRelationships = new Map<string, Set<string>>(); // relationshipType -> Set<relationshipId>
// Track relationships where this entity is the target
inboundRelationships = new Map<string, Set<string>>(); // relationshipType -> Set<relationshipId>
// ... rest of the class
}
```
#### `LosslessViewOne` Extension
```typescript
type RelationshipView = {
id: string; // Relationship ID
type: string; // Relationship type
direction: 'inbound' | 'outbound';
target: string; // Target entity ID
properties: Record<string, any>; // Relationship properties
};
type LosslessViewOne = {
id: DomainEntityID;
// ... existing fields ...
relationships?: {
outbound: RelationshipView[];
inbound: RelationshipView[];
};
};
```
## Implementation Steps
### Phase 1: Core Data Structures
1. [x] Update `DeltaBuilder.relate()` to create relationship entities
2. [ ] Update `LosslessEntity` to track relationship IDs
3. [ ] Extend `LosslessViewOne` type to include relationships
### Phase 2: Relationship Management
1. [ ] Implement relationship tracking in `Lossless` class
- Track all relationships by ID
- Maintain source/target indexes
2. [ ] Implement methods for querying relationships
- Get relationships for an entity
- Filter by type and direction
- Support pagination
### Phase 3: Delta Processing
1. [ ] Update `ingestDelta` to handle relationship deltas
- Extract relationship information from deltas
- Update relationship indexes
- Handle relationship updates and deletions
2. [ ] Add conflict resolution for concurrent relationship updates
### Phase 4: View Generation
1. [ ] Update `view` method to include relationships
- Option to include/exclude relationships
- Support for filtering relationships
- Handle circular references
### Phase 5: Performance Optimization
1. [ ] Add indexing for relationship lookups
2. [ ] Implement lazy loading for large relationship sets
3. [ ] Add caching for frequently accessed relationships
## API Extensions
### Get Entity with Relationships
```typescript
// Get an entity with its relationships
GET /entities/{id}?include=relationships
// Response
{
"id": "entity1",
"properties": { /* ... */ },
"relationships": {
"outbound": [
{
"id": "rel-123",
"type": "OWNS",
"target": "entity2",
"direction": "outbound",
"properties": {
"since": "2023-01-01"
}
}
],
"inbound": []
}
}
```
### Query Relationships
```typescript
// Get relationships for an entity
GET /entities/{id}/relationships?type=OWNS&direction=outbound
// Response
{
"relationships": [
{
"id": "rel-123",
"type": "OWNS",
"source": "entity1",
"target": "entity2",
"properties": {
"since": "2023-01-01"
}
}
]
}
```
### Create Relationship
```typescript
// Create a new relationship
POST /relationships
{
"source": "entity1",
"target": "entity2",
"type": "OWNS",
"properties": {
"since": "2023-01-01"
}
}
// Response
{
"id": "rel-123",
"source": "entity1",
"target": "entity2",
"type": "OWNS",
"properties": {
"since": "2023-01-01"
}
}
```
## Performance Considerations
1. **Memory Usage**:
- Store only relationship IDs in entity maps
- Use lazy loading for relationship properties
- Consider weak references if memory becomes an issue
2. **Query Performance**:
- Add indexes for common relationship queries
- Cache frequently accessed relationships
- Support pagination for large relationship sets
3. **Delta Processing**:
- Batch process relationship updates
- Optimize delta application for relationship-heavy workloads
## Future Enhancements
1. **Advanced Querying**:
- GraphQL support for complex relationship queries
- Support for recursive relationship traversal
2. **Schema Validation**:
- Define relationship schemas with property validation
- Support for required/optional properties
- Default values for relationship properties
3. **Indexing**:
- Add support for indexing relationship properties
- Implement efficient querying of relationships by property values
## Testing Strategy
1. **Unit Tests**:
- Test relationship creation and deletion
- Verify relationship queries with various filters
- Test delta processing for relationships
2. **Integration Tests**:
- Test relationship persistence across restarts
- Verify concurrent relationship updates
- Test with large numbers of relationships
3. **Performance Tests**:
- Measure memory usage with large relationship graphs
- Test query performance with complex relationship patterns
- Benchmark delta processing speed for relationship operations

View File

@ -1,5 +0,0 @@
# Housekeeping tasks
- [] Organize tests?
- [] More documentation in docs/
- [] Rename/consolidate, lossless view() and compose() --> composeView()

View File

@ -4,15 +4,14 @@
// It could then be further extended with e.g. table semantics like filter, sort, join // It could then be further extended with e.g. table semantics like filter, sort, join
import {Collection} from '../collections/collection-abstract'; import {Collection} from '../collections/collection-abstract';
import {ResolvedViewOne} from '../views/resolvers/last-write-wins'; import {LastWriteWins, ResolvedViewOne} from '../views/resolvers/last-write-wins';
import {TimestampResolver} from '../views/resolvers/timestamp-resolvers';
export class BasicCollection extends Collection<TimestampResolver> { export class BasicCollection extends Collection<LastWriteWins> {
declare lossy?: TimestampResolver; declare lossy?: LastWriteWins;
initializeView() { initializeView() {
if (!this.rhizomeNode) throw new Error('not connected to rhizome'); if (!this.rhizomeNode) throw new Error('not connected to rhizome');
this.lossy = new TimestampResolver(this.rhizomeNode.lossless); this.lossy = new LastWriteWins(this.rhizomeNode.lossless);
} }
resolve( resolve(

View File

@ -1,8 +1,7 @@
import {Collection} from "./collection-abstract"; import {Collection} from "./collection-abstract";
import {ResolvedViewOne} from "../views/resolvers/last-write-wins"; import {LastWriteWins, ResolvedViewOne} from "../views/resolvers/last-write-wins";
import {TimestampResolver} from "../views/resolvers/timestamp-resolvers";
class RelationalView extends TimestampResolver { class RelationalView extends LastWriteWins {
} }
export class RelationalCollection extends Collection<RelationalView> { export class RelationalCollection extends Collection<RelationalView> {

View File

@ -1,7 +1,6 @@
import Debug from 'debug'; import Debug from 'debug';
import { Collection } from '../collections/collection-abstract'; import { Collection } from '../collections/collection-abstract';
import { ResolvedViewOne } from '../views/resolvers/last-write-wins'; import { LastWriteWins, ResolvedViewOne } from '../views/resolvers/last-write-wins';
import { TimestampResolver } from '../views/resolvers/timestamp-resolvers'
import { import {
ObjectSchema, ObjectSchema,
SchemaValidationResult, SchemaValidationResult,
@ -24,7 +23,7 @@ export class SchemaValidationError extends Error {
} }
export class TypedCollectionImpl<T extends Record<string, unknown>> export class TypedCollectionImpl<T extends Record<string, unknown>>
extends Collection<TimestampResolver> extends Collection<LastWriteWins>
implements TypedCollection<T> { implements TypedCollection<T> {
schema: ObjectSchema; schema: ObjectSchema;
@ -57,7 +56,7 @@ export class TypedCollectionImpl<T extends Record<string, unknown>>
initializeView(): void { initializeView(): void {
if (!this.rhizomeNode) throw new Error('not connected to rhizome'); if (!this.rhizomeNode) throw new Error('not connected to rhizome');
this.lossy = new TimestampResolver(this.rhizomeNode.lossless); this.lossy = new LastWriteWins(this.rhizomeNode.lossless);
} }
resolve(id: string): ResolvedViewOne | undefined { resolve(id: string): ResolvedViewOne | undefined {
@ -74,7 +73,7 @@ export class TypedCollectionImpl<T extends Record<string, unknown>>
const mockLosslessView: LosslessViewOne = { const mockLosslessView: LosslessViewOne = {
id: 'validation-mock', id: 'validation-mock',
referencedAs: [], referencedAs: [],
propertyDeltas: {}, propertyDeltas: {}
}; };
// Create mock deltas for each property // Create mock deltas for each property

View File

@ -87,7 +87,6 @@ export class DeltaBuilder {
* Set a property on an entity * Set a property on an entity
*/ */
setProperty(entityId: string, property: string, value: string | number | boolean | null, entityLabel = "entity"): this { setProperty(entityId: string, property: string, value: string | number | boolean | null, entityLabel = "entity"): this {
// Note that entityLabe and property each need to be unique within a given delta
this.addPointer(entityLabel, entityId, property) this.addPointer(entityLabel, entityId, property)
this.addPointer(property, value); this.addPointer(property, value);
return this; return this;
@ -95,22 +94,10 @@ export class DeltaBuilder {
/** /**
* Create a relationship between two entities * Create a relationship between two entities
* @param sourceId The ID of the source entity
* @param targetId The ID of the target entity
* @param relationship The type of relationship
* @param properties Optional properties for the relationship
*/ */
relate(sourceId: string, targetId: string, relationship: string, properties?: Record<string, any>): this { relate(sourceId: string, relationship: string, targetId: string): this {
const relId = randomUUID(); this.pointers[relationship] = { [targetId]: relationship };
this.setProperty(relId, 'source', sourceId, '_rel_source'); this.pointers.source = { [sourceId]: relationship };
this.setProperty(relId, 'target', targetId, '_rel_target');
this.setProperty(relId, 'type', relationship, '_rel_type');
if (properties) {
for (const [key, value] of Object.entries(properties)) {
this.setProperty(relId, key, value, `_rel_${key}`);
}
}
return this; return this;
} }

View File

@ -1,6 +1,7 @@
// import Debug from 'debug'; // import Debug from 'debug';
import {EntityProperties} from "../../core/entity"; import {EntityProperties} from "../../core/entity";
import {CollapsedDelta} from "../lossless"; import {CollapsedDelta, LosslessViewOne} from "../lossless";
import {Lossy} from '../lossy';
import {DomainEntityID, PropertyID, PropertyTypes, Timestamp, ViewMany} from "../../core/types"; import {DomainEntityID, PropertyID, PropertyTypes, Timestamp, ViewMany} from "../../core/types";
// const debug = Debug('rz:lossy:last-write-wins'); // const debug = Debug('rz:lossy:last-write-wins');
@ -67,3 +68,46 @@ export function lastValueFromDeltas(
return res; return res;
} }
export class LastWriteWins extends Lossy<Accumulator, Result> {
initializer(view: LosslessViewOne): Accumulator {
return {
[view.id]: { id: view.id, properties: {} }
};
}
reducer(acc: Accumulator, cur: LosslessViewOne): Accumulator {
if (!acc[cur.id]) {
acc[cur.id] = { id: cur.id, properties: {} };
}
for (const [key, deltas] of Object.entries(cur.propertyDeltas)) {
const { value, timeUpdated } = lastValueFromDeltas(key, deltas) || {};
if (!value || timeUpdated === undefined) continue;
const currentTime = acc[cur.id].properties[key]?.timeUpdated || 0;
if (timeUpdated > currentTime) {
acc[cur.id].properties[key] = { value, timeUpdated };
}
}
return acc;
}
resolver(cur: Accumulator): Result {
const result: Result = {};
for (const [id, entity] of Object.entries(cur)) {
result[id] = {
id,
properties: Object.fromEntries(
Object.entries(entity.properties)
.map(([key, { value }]) => [key, value])
)
};
}
return result;
}
}