lentil/rhizome-node
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: lentil:76e00f38afcbb146cd6e7ba65ad6aeb0540c0608
Branches Tags
lentil:main
..
compare: lentil:54a50a9c222ce7a1de69bd8b13a9da8d92bf694a
Branches Tags
lentil:main

No commits in common. "76e00f38afcbb146cd6e7ba65ad6aeb0540c0608" and "54a50a9c222ce7a1de69bd8b13a9da8d92bf694a" have entirely different histories.
76e00f38af ... 54a50a9c22

13 changed files with 86 additions and 376 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -7,4 +7,3 @@ coverage/
data/
test-data/
*.code-workspace
local-notes/

12
__tests__/concurrent-writes.ts
View File

@ -2,11 +2,13 @@ import { createDelta } from '../src/core/delta-builder';
import {
RhizomeNode,
Lossless,
Delta,
LastWriteWins,
TimestampResolver,
SumResolver,
CustomResolver,
LastWriteWinsPlugin,
MajorityVotePlugin,
TimestampResolver
MajorityVotePlugin
} from "../src";
describe('Concurrent Write Scenarios', () => {
@ -37,7 +39,7 @@ describe('Concurrent Write Scenarios', () => {
.buildV1()
);
const resolver = new TimestampResolver(lossless);
const resolver = new LastWriteWins(lossless);
const result = resolver.resolve();
expect(result).toBeDefined();
@ -121,7 +123,7 @@ describe('Concurrent Write Scenarios', () => {
.buildV1()
);
const resolver = new TimestampResolver(lossless);
const resolver = new LastWriteWins(lossless);
const result = resolver.resolve();
expect(result).toBeDefined();
@ -247,7 +249,7 @@ describe('Concurrent Write Scenarios', () => {
);
}
const resolver = new TimestampResolver(lossless);
const resolver = new LastWriteWins(lossless);
const result = resolver.resolve();
expect(result).toBeDefined();

113
__tests__/delta-builder.test.ts
View File

@ -2,7 +2,7 @@ import { createDelta } from '../src/core/delta-builder';
import { DeltaV1, DeltaV2 } from '../src/core/delta';
import { Lossless } from '../src/views/lossless';
import { RhizomeNode } from '../src/node';
import { TimestampResolver } from '../src/views/resolvers/timestamp-resolvers';
import { LastWriteWins } from '../src/views/resolvers/last-write-wins';
describe('DeltaBuilder', () => {
const creator = 'creator-123';
@ -47,7 +47,7 @@ describe('DeltaBuilder', () => {
// Verify that the entity property resolves correctly
const lossless = new Lossless(node);
lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless);
const lossy = new LastWriteWins(lossless);
const result = lossy.resolve();
expect(result).toBeDefined();
expect(result!['entity-1'].properties.name).toBe('Test Entity');
@ -72,7 +72,7 @@ describe('DeltaBuilder', () => {
// Verify that the entity property resolves correctly
const lossless = new Lossless(node);
lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless);
const lossy = new LastWriteWins(lossless);
const result = lossy.resolve();
expect(result).toBeDefined();
expect(result!['entity-1'].properties.name).toBe('Test Entity');
@ -80,55 +80,18 @@ describe('DeltaBuilder', () => {
it('should create a V1 delta with relationships', () => {
const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows')
.relate('user-1', 'follows', 'user-2')
.buildV1();
// This delta sets values on a new relationship entity
expect(delta.pointers).toContainEqual({
localContext: '_rel_target',
target: expect.any(String),
targetContext: 'target'
});
const relId = delta.pointers.find(p => p.localContext === '_rel_target')?.target;
expect(delta.pointers).toContainEqual({
localContext: '_rel_source',
target: relId,
targetContext: 'source'
localContext: 'follows',
target: 'user-2',
targetContext: 'follows'
});
expect(delta.pointers).toContainEqual({
localContext: '_rel_type',
target: relId,
targetContext: 'type'
});
});
it('should create a V1 delta with relationships and properties', () => {
const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows', { version: 1})
.buildV1();
// This delta sets values on a new relationship entity
expect(delta.pointers).toContainEqual({
localContext: '_rel_target',
target: expect.any(String),
targetContext: 'target'
});
const relId = delta.pointers.find(p => p.localContext === '_rel_target')?.target;
expect(delta.pointers).toContainEqual({
localContext: '_rel_source',
target: relId,
targetContext: 'source'
});
expect(delta.pointers).toContainEqual({
localContext: '_rel_type',
target: relId,
targetContext: 'type'
});
expect(delta.pointers).toContainEqual({
localContext: '_rel_version',
target: relId,
targetContext: 'version'
localContext: 'source',
target: 'user-1',
targetContext: 'follows'
});
});
});
@ -158,61 +121,11 @@ describe('DeltaBuilder', () => {
it('should create a V2 delta with relationships', () => {
const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows')
.relate('user-1', 'follows', 'user-2')
.buildV2();
expect(delta.pointers).toHaveProperty('_rel_source');
const [relId] = Object.entries(delta.pointers._rel_source!)[0];
expect(delta.pointers).toHaveProperty('_rel_source', { [relId]: 'source' });
expect(delta.pointers).toHaveProperty('_rel_target', { [relId]: 'target' });
expect(delta.pointers).toHaveProperty('_rel_type', { [relId]: 'type' });
expect(delta.pointers).toHaveProperty('source', 'user-1');
expect(delta.pointers).toHaveProperty('target', 'user-2');
expect(delta.pointers).toHaveProperty('type', 'follows');
const lossless = new Lossless(node);
lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless);
const result = lossy.resolve([relId]);
expect(result).toBeDefined();
expect(result![relId]).toMatchObject({
properties: {
source: 'user-1',
target: 'user-2',
type: 'follows'
}
});
});
it('should create a V2 delta with relationships and properties', () => {
const delta = createDelta(creator, host)
.relate('user-1', 'user-2', 'follows', { version: 1})
.buildV2();
expect(delta.pointers).toHaveProperty('_rel_source');
const [relId] = Object.entries(delta.pointers._rel_source!)[0];
expect(delta.pointers).toHaveProperty('_rel_source', { [relId]: 'source' });
expect(delta.pointers).toHaveProperty('_rel_target', { [relId]: 'target' });
expect(delta.pointers).toHaveProperty('_rel_type', { [relId]: 'type' });
expect(delta.pointers).toHaveProperty('_rel_version', { [relId]: 'version' });
expect(delta.pointers).toHaveProperty('source', 'user-1');
expect(delta.pointers).toHaveProperty('target', 'user-2');
expect(delta.pointers).toHaveProperty('type', 'follows');
expect(delta.pointers).toHaveProperty('version', 1);
const lossless = new Lossless(node);
lossless.ingestDelta(delta);
const lossy = new TimestampResolver(lossless);
const result = lossy.resolve([relId]);
expect(result).toBeDefined();
expect(result![relId]).toMatchObject({
properties: {
source: 'user-1',
target: 'user-2',
type: 'follows',
version: 1
}
});
expect(delta.pointers).toHaveProperty('follows', { 'user-2': 'follows' });
expect(delta.pointers).toHaveProperty('source', { 'user-1': 'follows' });
});
});

2
__tests__/jest-setup.ts
View File

@ -4,7 +4,7 @@
// NO_DEBUG=true npm test // will not set DEBUG
// DEBUG=other npm test // will set DEBUG=other
if (!process.env.DEBUG && !process.env.NO_DEBUG) {
// process.env.DEBUG = 'rz:*';
process.env.DEBUG = 'rz:*';
}
// This is a placeholder test to satisfy Jest's requirement for at least one test

8
__tests__/last-write-wins.ts
View File

@ -1,19 +1,15 @@
import Debug from "debug";
import { createDelta } from '../src/core/delta-builder';
import {Lossless, RhizomeNode} from "../src";
import {TimestampResolver} from "../src/views/resolvers/timestamp-resolvers";
import {Delta, LastWriteWins, Lossless, RhizomeNode} from "../src";
const debug = Debug('test:last-write-wins');
// This was initially written to test a LastWriteWins resolver, but that has been
// superceded by the TimestampResolver.
describe('Last write wins', () => {
describe('given that two separate writes occur', () => {
const node = new RhizomeNode();
const lossless = new Lossless(node);
const lossy = new TimestampResolver(lossless);
const lossy = new LastWriteWins(lossless);
beforeAll(() => {
lossless.ingestDelta(createDelta('a', 'h')

11
__tests__/schema.ts
View File

@ -1,4 +1,3 @@
import { LosslessViewOne } from '../src/views/lossless';
import {
SchemaBuilder,
PrimitiveSchemas,
@ -156,7 +155,7 @@ describe('Schema System', () => {
schemaRegistry.register(userSchema);
// Create a valid lossless view
const validView: LosslessViewOne = {
const validView = {
id: 'user123',
referencedAs: ['user'],
propertyDeltas: {
@ -174,7 +173,7 @@ describe('Schema System', () => {
creator: 'creator1',
pointers: [{ age: 25 }]
}]
},
}
};
const result = schemaRegistry.validate('user123', 'user', validView);
@ -182,7 +181,7 @@ describe('Schema System', () => {
expect(result.errors).toHaveLength(0);
// Test invalid view (missing required property)
const invalidView: LosslessViewOne = {
const invalidView = {
id: 'user456',
referencedAs: ['user'],
propertyDeltas: {
@ -218,7 +217,7 @@ describe('Schema System', () => {
schemaRegistry.register(schema);
// Valid types
const validView: LosslessViewOne = {
const validView = {
id: 'test1',
referencedAs: [],
propertyDeltas: {
@ -232,7 +231,7 @@ describe('Schema System', () => {
expect(validResult.valid).toBe(true);
// Invalid types
const invalidView: LosslessViewOne = {
const invalidView = {
id: 'test2',
referencedAs: [],
propertyDeltas: {

222
plans/ent-rel-graph.md
View File

@ -1,222 +0,0 @@
# Entity Relationship Graph Implementation Plan
## Overview
This document outlines the plan to implement entity relationship tracking in the rhizome-node system. The implementation treats relationships as first-class entities, each with their own identity and properties.
## Core Design
### Relationship as First-Class Entities
- Each relationship is a domain entity with its own unique ID
- Relationships have standard properties: `source`, `target`, and `type`
- Additional properties can be added to relationships
- Relationships are created using the `relate()` method in `DeltaBuilder`
### Delta Structure for Relationships
```typescript
// Creating a relationship
createDelta(creator, host)
.relate(
sourceId, // ID of the source entity
targetId, // ID of the target entity
'REL_TYPE', // Relationship type
{ // Optional properties
prop1: 'value1',
prop2: 'value2'
}
)
.build();
```
### Data Structures
#### `LosslessEntity` Updates
```typescript
class LosslessEntity {
// Existing properties
properties = new Map<PropertyID, Set<Delta>>();
// Track relationships where this entity is the source
outboundRelationships = new Map<string, Set<string>>(); // relationshipType -> Set<relationshipId>
// Track relationships where this entity is the target
inboundRelationships = new Map<string, Set<string>>(); // relationshipType -> Set<relationshipId>
// ... rest of the class
}
```
#### `LosslessViewOne` Extension
```typescript
type RelationshipView = {
id: string; // Relationship ID
type: string; // Relationship type
direction: 'inbound' | 'outbound';
target: string; // Target entity ID
properties: Record<string, any>; // Relationship properties
};
type LosslessViewOne = {
id: DomainEntityID;
// ... existing fields ...
relationships?: {
outbound: RelationshipView[];
inbound: RelationshipView[];
};
};
```
## Implementation Steps
### Phase 1: Core Data Structures
1. [x] Update `DeltaBuilder.relate()` to create relationship entities
2. [ ] Update `LosslessEntity` to track relationship IDs
3. [ ] Extend `LosslessViewOne` type to include relationships
### Phase 2: Relationship Management
1. [ ] Implement relationship tracking in `Lossless` class
- Track all relationships by ID
- Maintain source/target indexes
2. [ ] Implement methods for querying relationships
- Get relationships for an entity
- Filter by type and direction
- Support pagination
### Phase 3: Delta Processing
1. [ ] Update `ingestDelta` to handle relationship deltas
- Extract relationship information from deltas
- Update relationship indexes
- Handle relationship updates and deletions
2. [ ] Add conflict resolution for concurrent relationship updates
### Phase 4: View Generation
1. [ ] Update `view` method to include relationships
- Option to include/exclude relationships
- Support for filtering relationships
- Handle circular references
### Phase 5: Performance Optimization
1. [ ] Add indexing for relationship lookups
2. [ ] Implement lazy loading for large relationship sets
3. [ ] Add caching for frequently accessed relationships
## API Extensions
### Get Entity with Relationships
```typescript
// Get an entity with its relationships
GET /entities/{id}?include=relationships
// Response
{
"id": "entity1",
"properties": { /* ... */ },
"relationships": {
"outbound": [
{
"id": "rel-123",
"type": "OWNS",
"target": "entity2",
"direction": "outbound",
"properties": {
"since": "2023-01-01"
}
}
],
"inbound": []
}
}
```
### Query Relationships
```typescript
// Get relationships for an entity
GET /entities/{id}/relationships?type=OWNS&direction=outbound
// Response
{
"relationships": [
{
"id": "rel-123",
"type": "OWNS",
"source": "entity1",
"target": "entity2",
"properties": {
"since": "2023-01-01"
}
}
]
}
```
### Create Relationship
```typescript
// Create a new relationship
POST /relationships
{
"source": "entity1",
"target": "entity2",
"type": "OWNS",
"properties": {
"since": "2023-01-01"
}
}
// Response
{
"id": "rel-123",
"source": "entity1",
"target": "entity2",
"type": "OWNS",
"properties": {
"since": "2023-01-01"
}
}
```
## Performance Considerations
1. **Memory Usage**:
- Store only relationship IDs in entity maps
- Use lazy loading for relationship properties
- Consider weak references if memory becomes an issue
2. **Query Performance**:
- Add indexes for common relationship queries
- Cache frequently accessed relationships
- Support pagination for large relationship sets
3. **Delta Processing**:
- Batch process relationship updates
- Optimize delta application for relationship-heavy workloads
## Future Enhancements
1. **Advanced Querying**:
- GraphQL support for complex relationship queries
- Support for recursive relationship traversal
2. **Schema Validation**:
- Define relationship schemas with property validation
- Support for required/optional properties
- Default values for relationship properties
3. **Indexing**:
- Add support for indexing relationship properties
- Implement efficient querying of relationships by property values
## Testing Strategy
1. **Unit Tests**:
- Test relationship creation and deletion
- Verify relationship queries with various filters
- Test delta processing for relationships
2. **Integration Tests**:
- Test relationship persistence across restarts
- Verify concurrent relationship updates
- Test with large numbers of relationships
3. **Performance Tests**:
- Measure memory usage with large relationship graphs
- Test query performance with complex relationship patterns
- Benchmark delta processing speed for relationship operations

5
plans/housekeeping.md
View File

@ -1,5 +0,0 @@
# Housekeeping tasks
- [] Organize tests?
- [] More documentation in docs/
- [] Rename/consolidate, lossless view() and compose() --> composeView()

9
src/collections/collection-basic.ts
View File

@ -4,15 +4,14 @@
// It could then be further extended with e.g. table semantics like filter, sort, join
import {Collection} from '../collections/collection-abstract';
import {ResolvedViewOne} from '../views/resolvers/last-write-wins';
import {TimestampResolver} from '../views/resolvers/timestamp-resolvers';
import {LastWriteWins, ResolvedViewOne} from '../views/resolvers/last-write-wins';
export class BasicCollection extends Collection<TimestampResolver> {
declare lossy?: TimestampResolver;
export class BasicCollection extends Collection<LastWriteWins> {
declare lossy?: LastWriteWins;
initializeView() {
if (!this.rhizomeNode) throw new Error('not connected to rhizome');
this.lossy = new TimestampResolver(this.rhizomeNode.lossless);
this.lossy = new LastWriteWins(this.rhizomeNode.lossless);
}
resolve(

5
src/collections/collection-relational.ts
View File

@ -1,8 +1,7 @@
import {Collection} from "./collection-abstract";
import {ResolvedViewOne} from "../views/resolvers/last-write-wins";
import {TimestampResolver} from "../views/resolvers/timestamp-resolvers";
import {LastWriteWins, ResolvedViewOne} from "../views/resolvers/last-write-wins";
class RelationalView extends TimestampResolver {
class RelationalView extends LastWriteWins {
}
export class RelationalCollection extends Collection<RelationalView> {

9
src/collections/collection-typed.ts
View File

@ -1,7 +1,6 @@
import Debug from 'debug';
import { Collection } from '../collections/collection-abstract';
import { ResolvedViewOne } from '../views/resolvers/last-write-wins';
import { TimestampResolver } from '../views/resolvers/timestamp-resolvers'
import { LastWriteWins, ResolvedViewOne } from '../views/resolvers/last-write-wins';
import {
ObjectSchema,
SchemaValidationResult,
@ -24,7 +23,7 @@ export class SchemaValidationError extends Error {
}
export class TypedCollectionImpl<T extends Record<string, unknown>>
extends Collection<TimestampResolver>
extends Collection<LastWriteWins>
implements TypedCollection<T> {
schema: ObjectSchema;
@ -57,7 +56,7 @@ export class TypedCollectionImpl<T extends Record<string, unknown>>
initializeView(): void {
if (!this.rhizomeNode) throw new Error('not connected to rhizome');
this.lossy = new TimestampResolver(this.rhizomeNode.lossless);
this.lossy = new LastWriteWins(this.rhizomeNode.lossless);
}
resolve(id: string): ResolvedViewOne | undefined {
@ -74,7 +73,7 @@ export class TypedCollectionImpl<T extends Record<string, unknown>>
const mockLosslessView: LosslessViewOne = {
id: 'validation-mock',
referencedAs: [],
propertyDeltas: {},
propertyDeltas: {}
};
// Create mock deltas for each property

19
src/core/delta-builder.ts
View File

@ -87,7 +87,6 @@ export class DeltaBuilder {
* Set a property on an entity
*/
setProperty(entityId: string, property: string, value: string | number | boolean | null, entityLabel = "entity"): this {
// Note that entityLabe and property each need to be unique within a given delta
this.addPointer(entityLabel, entityId, property)
this.addPointer(property, value);
return this;
@ -95,22 +94,10 @@ export class DeltaBuilder {
/**
* Create a relationship between two entities
* @param sourceId The ID of the source entity
* @param targetId The ID of the target entity
* @param relationship The type of relationship
* @param properties Optional properties for the relationship
*/
relate(sourceId: string, targetId: string, relationship: string, properties?: Record<string, any>): this {
const relId = randomUUID();
this.setProperty(relId, 'source', sourceId, '_rel_source');
this.setProperty(relId, 'target', targetId, '_rel_target');
this.setProperty(relId, 'type', relationship, '_rel_type');
if (properties) {
for (const [key, value] of Object.entries(properties)) {
this.setProperty(relId, key, value, `_rel_${key}`);
}
}
relate(sourceId: string, relationship: string, targetId: string): this {
this.pointers[relationship] = { [targetId]: relationship };
this.pointers.source = { [sourceId]: relationship };
return this;
}

46
src/views/resolvers/last-write-wins.ts
View File

@ -1,6 +1,7 @@
// import Debug from 'debug';
import {EntityProperties} from "../../core/entity";
import {CollapsedDelta} from "../lossless";
import {CollapsedDelta, LosslessViewOne} from "../lossless";
import {Lossy} from '../lossy';
import {DomainEntityID, PropertyID, PropertyTypes, Timestamp, ViewMany} from "../../core/types";
// const debug = Debug('rz:lossy:last-write-wins');
@ -67,3 +68,46 @@ export function lastValueFromDeltas(
return res;
}
export class LastWriteWins extends Lossy<Accumulator, Result> {
initializer(view: LosslessViewOne): Accumulator {
return {
[view.id]: { id: view.id, properties: {} }
};
}
reducer(acc: Accumulator, cur: LosslessViewOne): Accumulator {
if (!acc[cur.id]) {
acc[cur.id] = { id: cur.id, properties: {} };
}
for (const [key, deltas] of Object.entries(cur.propertyDeltas)) {
const { value, timeUpdated } = lastValueFromDeltas(key, deltas) || {};
if (!value || timeUpdated === undefined) continue;
const currentTime = acc[cur.id].properties[key]?.timeUpdated || 0;
if (timeUpdated > currentTime) {
acc[cur.id].properties[key] = { value, timeUpdated };
}
}
return acc;
}
resolver(cur: Accumulator): Result {
const result: Result = {};
for (const [id, entity] of Object.entries(cur)) {
result[id] = {
id,
properties: Object.fromEntries(
Object.entries(entity.properties)
.map(([key, { value }]) => [key, value])
)
};
}
return result;
}
}