From dc59dcc7782eda15f96c3f0fdd59a04ef154bc82 Mon Sep 17 00:00:00 2001 From: Mykola Bilokonsky Date: Mon, 9 Jun 2025 13:21:52 -0400 Subject: [PATCH] schema foundation support complete --- __tests__/delta-validation.ts | 4 +- __tests__/schema.ts | 421 ++++++++++++++++++++++++++++++++++ __tests__/transactions.ts | 2 +- src/collection-typed.ts | 248 ++++++++++++++++++++ src/delta-validation.ts | 6 +- src/delta.ts | 4 +- src/schema-registry.ts | 398 ++++++++++++++++++++++++++++++++ src/schema.ts | 233 +++++++++++++++++++ src/types.ts | 2 +- todo.md | 12 +- 10 files changed, 1314 insertions(+), 16 deletions(-) create mode 100644 __tests__/schema.ts create mode 100644 src/collection-typed.ts create mode 100644 src/schema-registry.ts create mode 100644 src/schema.ts diff --git a/__tests__/delta-validation.ts b/__tests__/delta-validation.ts index 2055ec0..812ae0b 100644 --- a/__tests__/delta-validation.ts +++ b/__tests__/delta-validation.ts @@ -157,9 +157,7 @@ describe("Delta Validation", () => { it("should throw error for invalid pointer values", () => { const invalidPointers = [ { key: undefined }, - { key: [] }, - { key: true }, - { key: false } + { key: [] } ]; invalidPointers.forEach(pointers => { diff --git a/__tests__/schema.ts b/__tests__/schema.ts new file mode 100644 index 0000000..da5eb3c --- /dev/null +++ b/__tests__/schema.ts @@ -0,0 +1,421 @@ +import { + SchemaBuilder, + PrimitiveSchemas, + ReferenceSchemas, + ArraySchemas, + CommonSchemas, + ObjectSchema +} from '../src/schema'; +import { DefaultSchemaRegistry } from '../src/schema-registry'; +import { TypedCollectionImpl, SchemaValidationError } from '../src/collection-typed'; +import { RhizomeNode } from '../src/node'; +import { Delta } from '../src/delta'; + +describe('Schema System', () => { + let schemaRegistry: DefaultSchemaRegistry; + let node: RhizomeNode; + + beforeEach(() => { + schemaRegistry = new DefaultSchemaRegistry(); + node = new RhizomeNode(); + }); + + describe('Schema Builder', () => { + it('should create a basic schema', () => { + const schema = SchemaBuilder + .create('user') + .name('User') + .description('A user entity') + .property('name', PrimitiveSchemas.requiredString()) + .property('age', PrimitiveSchemas.number()) + .property('active', PrimitiveSchemas.boolean()) + .required('name') + .build(); + + expect(schema.id).toBe('user'); + expect(schema.name).toBe('User'); + expect(schema.description).toBe('A user entity'); + expect(schema.properties.name).toEqual({ + type: 'primitive', + primitiveType: 'string', + required: true + }); + expect(schema.requiredProperties).toContain('name'); + }); + + it('should create schema with references', () => { + const schema = SchemaBuilder + .create('post') + .name('Post') + .property('title', PrimitiveSchemas.requiredString()) + .property('author', ReferenceSchemas.required('user')) + .property('tags', ArraySchemas.of(PrimitiveSchemas.string())) + .build(); + + expect(schema.properties.author).toEqual({ + type: 'reference', + targetSchema: 'user', + maxDepth: 3, + required: true + }); + expect(schema.properties.tags).toEqual({ + type: 'array', + itemSchema: { type: 'primitive', primitiveType: 'string' } + }); + }); + + it('should enforce required fields', () => { + expect(() => { + SchemaBuilder.create('').build(); + }).toThrow('Schema must have id and name'); + + expect(() => { + SchemaBuilder.create('test').build(); + }).toThrow('Schema must have id and name'); + }); + }); + + describe('Schema Registry', () => { + it('should register and retrieve schemas', () => { + const schema = CommonSchemas.User(); + schemaRegistry.register(schema); + + const retrieved = schemaRegistry.get('user'); + expect(retrieved).toEqual(schema); + + const all = schemaRegistry.list(); + expect(all).toContain(schema); + }); + + it('should validate schema structure', () => { + const invalidSchema = { + id: 'invalid', + name: 'Invalid', + properties: { + badProperty: { type: 'unknown' } as never + } + }; + + expect(() => { + schemaRegistry.register(invalidSchema as ObjectSchema); + }).toThrow('Unknown schema type'); + }); + + it('should validate required properties exist', () => { + const schema = SchemaBuilder + .create('test') + .name('Test') + .property('name', PrimitiveSchemas.string()) + .required('name', 'nonexistent') + .build(); + + expect(() => { + schemaRegistry.register(schema); + }).toThrow("Required property 'nonexistent' not found"); + }); + + it('should detect circular dependencies', () => { + // Create schemas with circular references + const userSchema = SchemaBuilder + .create('user') + .name('User') + .property('name', PrimitiveSchemas.string()) + .property('bestFriend', ReferenceSchemas.to('user')) + .build(); + + schemaRegistry.register(userSchema); + + // This is circular (self-reference) + expect(schemaRegistry.hasCircularDependencies()).toBe(true); + + // Create actual circular dependency + const groupSchema = SchemaBuilder + .create('group') + .name('Group') + .property('owner', ReferenceSchemas.to('user')) + .build(); + + const userWithGroupSchema = SchemaBuilder + .create('user-with-group') + .name('User With Group') + .property('group', ReferenceSchemas.to('group')) + .build(); + + schemaRegistry.register(groupSchema); + schemaRegistry.register(userWithGroupSchema); + + // Still circular due to the self-referencing user schema + expect(schemaRegistry.hasCircularDependencies()).toBe(true); + }); + + it('should validate lossless views against schemas', () => { + const userSchema = CommonSchemas.User(); + schemaRegistry.register(userSchema); + + // Create a valid lossless view + const validView = { + id: 'user123', + referencedAs: ['user'], + propertyDeltas: { + name: [{ + id: 'delta1', + timeCreated: 123, + host: 'host1', + creator: 'creator1', + pointers: [{ name: 'Alice' }] + }], + age: [{ + id: 'delta2', + timeCreated: 124, + host: 'host1', + creator: 'creator1', + pointers: [{ age: 25 }] + }] + } + }; + + const result = schemaRegistry.validate('user123', 'user', validView); + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + + // Test invalid view (missing required property) + const invalidView = { + id: 'user456', + referencedAs: ['user'], + propertyDeltas: { + age: [{ + id: 'delta3', + timeCreated: 125, + host: 'host1', + creator: 'creator1', + pointers: [{ age: 30 }] + }] + } + }; + + const invalidResult = schemaRegistry.validate('user456', 'user', invalidView); + expect(invalidResult.valid).toBe(false); + expect(invalidResult.errors).toContainEqual( + expect.objectContaining({ + property: 'name', + message: expect.stringContaining('Required property') + }) + ); + }); + + it('should validate primitive types', () => { + const schema = SchemaBuilder + .create('test') + .name('Test') + .property('stringProp', PrimitiveSchemas.string()) + .property('numberProp', PrimitiveSchemas.number()) + .property('booleanProp', PrimitiveSchemas.boolean()) + .build(); + + schemaRegistry.register(schema); + + // Valid types + const validView = { + id: 'test1', + referencedAs: [], + propertyDeltas: { + stringProp: [{ id: 'd1', timeCreated: 1, host: 'h', creator: 'c', pointers: [{ stringProp: 'hello' }] }], + numberProp: [{ id: 'd2', timeCreated: 1, host: 'h', creator: 'c', pointers: [{ numberProp: 42 }] }], + booleanProp: [{ id: 'd3', timeCreated: 1, host: 'h', creator: 'c', pointers: [{ booleanProp: true }] }] + } + }; + + const validResult = schemaRegistry.validate('test1', 'test', validView); + expect(validResult.valid).toBe(true); + + // Invalid types + const invalidView = { + id: 'test2', + referencedAs: [], + propertyDeltas: { + stringProp: [{ id: 'd4', timeCreated: 1, host: 'h', creator: 'c', pointers: [{ stringProp: 123 as never }] }], + numberProp: [{ id: 'd5', timeCreated: 1, host: 'h', creator: 'c', pointers: [{ numberProp: 'not-number' as never }] }] + } + }; + + const invalidResult = schemaRegistry.validate('test2', 'test', invalidView); + expect(invalidResult.valid).toBe(false); + expect(invalidResult.errors).toHaveLength(2); + }); + }); + + describe('Typed Collection', () => { + it('should create typed collection with schema validation', () => { + const userSchema = CommonSchemas.User(); + const collection = new TypedCollectionImpl<{ + name: string; + email?: string; + age?: number; + active?: boolean; + }>('users', userSchema, schemaRegistry); + + expect(collection.schema).toEqual(userSchema); + expect(collection.name).toBe('users'); + }); + + it('should validate entities against schema', () => { + const userSchema = CommonSchemas.User(); + const collection = new TypedCollectionImpl<{ + name: string; + email?: string; + age?: number; + }>('users', userSchema, schemaRegistry); + + // Valid entity + const validUser = { name: 'Alice', email: 'alice@test.com', age: 25 }; + const validResult = collection.validate(validUser); + expect(validResult.valid).toBe(true); + + // Invalid entity (missing required name) + const invalidUser = { email: 'bob@test.com', age: 30 } as never; + const invalidResult = collection.validate(invalidUser); + expect(invalidResult.valid).toBe(false); + }); + + it('should enforce strict validation on put operations', async () => { + const userSchema = CommonSchemas.User(); + const collection = new TypedCollectionImpl<{ + name: string; + email?: string; + }>('users', userSchema, schemaRegistry, { strictValidation: true }); + + collection.rhizomeConnect(node); + + // Valid put should succeed + await expect(collection.put('user1', { name: 'Alice' })).resolves.toBeDefined(); + + // Invalid put should fail + await expect(collection.put('user2', { email: 'invalid@test.com' })).rejects.toThrow(SchemaValidationError); + }); + + it('should provide validation statistics', async () => { + const userSchema = CommonSchemas.User(); + const collection = new TypedCollectionImpl<{ + name: string; + email?: string; + }>('users', userSchema, schemaRegistry); + + collection.rhizomeConnect(node); + + // Add some entities + await collection.put('user1', { name: 'Alice', email: 'alice@test.com' }); + await collection.put('user2', { name: 'Bob' }); + + // Manually create an invalid entity by bypassing validation + const invalidDelta = new Delta({ + creator: node.config.creator, + host: node.config.peerId, + pointers: [ + { localContext: 'users', target: 'user3', targetContext: 'email' }, + { localContext: 'email', target: 'invalid@test.com' } + ] + }); + node.lossless.ingestDelta(invalidDelta); + + const stats = collection.getValidationStats(); + expect(stats.totalEntities).toBe(3); + expect(stats.validEntities).toBe(2); + expect(stats.invalidEntities).toBe(1); + }); + + it('should filter valid and invalid entities', async () => { + const userSchema = CommonSchemas.User(); + const collection = new TypedCollectionImpl<{ + name: string; + email?: string; + }>('users', userSchema, schemaRegistry); + + collection.rhizomeConnect(node); + + await collection.put('user1', { name: 'Alice' }); + await collection.put('user2', { name: 'Bob' }); + + // Create invalid entity manually + const invalidDelta = new Delta({ + creator: node.config.creator, + host: node.config.peerId, + pointers: [ + { localContext: 'users', target: 'user3', targetContext: 'age' }, + { localContext: 'age', target: 'not-a-number' } + ] + }); + node.lossless.ingestDelta(invalidDelta); + + const validIds = collection.getValidEntities(); + expect(validIds).toContain('user1'); + expect(validIds).toContain('user2'); + expect(validIds).not.toContain('user3'); + + const invalidEntities = collection.getInvalidEntities(); + expect(invalidEntities).toHaveLength(1); + expect(invalidEntities[0].entityId).toBe('user3'); + }); + + it('should apply schema to lossless views', async () => { + const userSchema = CommonSchemas.User(); + const collection = new TypedCollectionImpl<{ + name: string; + age?: number; + }>('users', userSchema, schemaRegistry); + + collection.rhizomeConnect(node); + + await collection.put('user1', { name: 'Alice', age: 25 }); + + const validatedView = collection.getValidatedView('user1'); + expect(validatedView).toBeDefined(); + expect(validatedView!.schemaId).toBe('user'); + expect(validatedView!.properties.name).toBeDefined(); + expect(validatedView!.properties.age).toBeDefined(); + expect(validatedView!.metadata?.appliedAt).toBeDefined(); + }); + + it('should provide schema introspection', () => { + const userSchema = CommonSchemas.User(); + schemaRegistry.register(CommonSchemas.UserSummary()); + + const collection = new TypedCollectionImpl<{ + name: string; + }>('users', userSchema, schemaRegistry); + + const schemaInfo = collection.getSchemaInfo(); + expect(schemaInfo.schema).toEqual(userSchema); + expect(schemaInfo.dependencies).toContain('user-summary'); + expect(schemaInfo.hasCircularDependencies).toBe(false); + }); + }); + + describe('Common Schemas', () => { + it('should provide working User schema', () => { + const userSchema = CommonSchemas.User(); + expect(userSchema.id).toBe('user'); + expect(userSchema.name).toBe('User'); + expect(userSchema.properties.name).toBeDefined(); + expect(userSchema.properties.friends).toBeDefined(); + expect(userSchema.requiredProperties).toContain('name'); + }); + + it('should provide working Document schema', () => { + const docSchema = CommonSchemas.Document(); + expect(docSchema.id).toBe('document'); + expect(docSchema.properties.title).toBeDefined(); + expect(docSchema.properties.author).toBeDefined(); + expect(docSchema.requiredProperties).toContain('title'); + expect(docSchema.requiredProperties).toContain('author'); + }); + + it('should work together in a registry', () => { + schemaRegistry.register(CommonSchemas.User()); + schemaRegistry.register(CommonSchemas.UserSummary()); + schemaRegistry.register(CommonSchemas.Document()); + + expect(schemaRegistry.list()).toHaveLength(3); + expect(schemaRegistry.hasCircularDependencies()).toBe(false); // No circular deps in CommonSchemas + }); + }); +}); \ No newline at end of file diff --git a/__tests__/transactions.ts b/__tests__/transactions.ts index 61301aa..fe7c8f3 100644 --- a/__tests__/transactions.ts +++ b/__tests__/transactions.ts @@ -1,4 +1,4 @@ -import { Delta, DeltaV2 } from '../src/delta'; +import { Delta } from '../src/delta'; import { Lossless } from '../src/lossless'; import { RhizomeNode } from '../src/node'; import { DeltaFilter } from '../src/delta'; diff --git a/src/collection-typed.ts b/src/collection-typed.ts new file mode 100644 index 0000000..45341be --- /dev/null +++ b/src/collection-typed.ts @@ -0,0 +1,248 @@ +import Debug from 'debug'; +import { Collection } from './collection-abstract'; +import { LastWriteWins, ResolvedViewOne } from './last-write-wins'; +import { + ObjectSchema, + SchemaValidationResult, + SchemaAppliedView, + TypedCollection, + SchemaApplicationOptions +} from './schema'; +import { DefaultSchemaRegistry } from './schema-registry'; +import { LosslessViewOne } from './lossless'; +import { DomainEntityID, PropertyTypes } from './types'; +import { EntityProperties } from './entity'; + +const debug = Debug('rz:typed-collection'); + +export class SchemaValidationError extends Error { + constructor(message: string, public validationResult: SchemaValidationResult) { + super(message); + this.name = 'SchemaValidationError'; + } +} + +export class TypedCollectionImpl> + extends Collection + implements TypedCollection { + + schema: ObjectSchema; + private schemaRegistry: DefaultSchemaRegistry; + private applicationOptions: SchemaApplicationOptions; + + constructor( + name: string, + schema: ObjectSchema, + schemaRegistry: DefaultSchemaRegistry, + options: SchemaApplicationOptions = {} + ) { + super(name); + this.schema = schema; + this.schemaRegistry = schemaRegistry; + this.applicationOptions = { + maxDepth: 3, + includeMetadata: true, + strictValidation: false, + ...options + }; + + // Register the schema if not already registered + if (!this.schemaRegistry.get(schema.id)) { + this.schemaRegistry.register(schema); + } + + debug(`Created typed collection '${name}' with schema '${schema.id}'`); + } + + initializeView(): void { + if (!this.rhizomeNode) throw new Error('not connected to rhizome'); + this.lossy = new LastWriteWins(this.rhizomeNode.lossless); + } + + resolve(id: string): ResolvedViewOne | undefined { + if (!this.rhizomeNode) throw new Error('collection not connected to rhizome'); + if (!this.lossy) throw new Error('lossy view not initialized'); + + const res = this.lossy.resolve([id]) || {}; + return res[id]; + } + + // Validate an entity against the schema + validate(entity: T): SchemaValidationResult { + // Convert entity to a mock lossless view for validation + const mockLosslessView: LosslessViewOne = { + id: 'validation-mock', + referencedAs: [], + propertyDeltas: {} + }; + + // Create mock deltas for each property + for (const [key, value] of Object.entries(entity)) { + if (value !== undefined) { + mockLosslessView.propertyDeltas[key] = [{ + id: 'mock-delta', + timeCreated: Date.now(), + host: 'validation', + creator: 'validation', + pointers: [{ [key]: value as PropertyTypes }] + }]; + } + } + + return this.schemaRegistry.validate('validation-mock', this.schema.id, mockLosslessView); + } + + // Apply schema to a lossless view + apply(view: LosslessViewOne): SchemaAppliedView { + return this.schemaRegistry.applySchema(view, this.schema.id, this.applicationOptions); + } + + // Get a schema-validated view of an entity + getValidatedView(entityId: DomainEntityID): SchemaAppliedView | undefined { + if (!this.rhizomeNode) throw new Error('collection not connected to rhizome'); + + const losslessView = this.rhizomeNode.lossless.view([entityId])[entityId]; + if (!losslessView) return undefined; + + return this.apply(losslessView); + } + + // Get all entities in this collection with schema validation + getAllValidatedViews(): SchemaAppliedView[] { + if (!this.rhizomeNode) throw new Error('collection not connected to rhizome'); + + const entityIds = this.getIds(); + const views: SchemaAppliedView[] = []; + + for (const entityId of entityIds) { + const view = this.getValidatedView(entityId); + if (view) { + views.push(view); + } + } + + return views; + } + + // Override put to include schema validation + async put( + entityId: DomainEntityID | undefined, + properties: EntityProperties, + ): Promise { + // Validate against schema if strict validation is enabled + if (this.applicationOptions.strictValidation) { + const validationResult = this.validate(properties as T); + if (!validationResult.valid) { + throw new SchemaValidationError( + `Schema validation failed: ${validationResult.errors.map(e => e.message).join(', ')}`, + validationResult + ); + } + } + + // Call parent put method + const result = await super.put(entityId, properties); + + // Log validation warnings if any + const validationResult = this.validate(properties as T); + if (validationResult.warnings.length > 0) { + debug(`Validation warnings for entity ${entityId}:`, validationResult.warnings); + } + + return result; + } + + // Get validation statistics for the collection + getValidationStats(): { + totalEntities: number; + validEntities: number; + invalidEntities: number; + entitiesWithWarnings: number; + commonErrors: Map; + } { + const entityIds = this.getIds(); + const stats = { + totalEntities: entityIds.length, + validEntities: 0, + invalidEntities: 0, + entitiesWithWarnings: 0, + commonErrors: new Map() + }; + + for (const entityId of entityIds) { + if (!this.rhizomeNode) continue; + + const losslessView = this.rhizomeNode.lossless.view([entityId])[entityId]; + if (!losslessView) continue; + + const validationResult = this.schemaRegistry.validate(entityId, this.schema.id, losslessView); + + if (validationResult.valid) { + stats.validEntities++; + } else { + stats.invalidEntities++; + } + + if (validationResult.warnings.length > 0) { + stats.entitiesWithWarnings++; + } + + // Count common errors + for (const error of validationResult.errors) { + const count = stats.commonErrors.get(error.message) || 0; + stats.commonErrors.set(error.message, count + 1); + } + } + + return stats; + } + + // Filter entities by schema validation status + getValidEntities(): DomainEntityID[] { + if (!this.rhizomeNode) return []; + + return this.getIds().filter(entityId => { + const losslessView = this.rhizomeNode!.lossless.view([entityId])[entityId]; + if (!losslessView) return false; + + const validationResult = this.schemaRegistry.validate(entityId, this.schema.id, losslessView); + return validationResult.valid; + }); + } + + getInvalidEntities(): Array<{ entityId: DomainEntityID; errors: string[] }> { + if (!this.rhizomeNode) return []; + + const invalid: Array<{ entityId: DomainEntityID; errors: string[] }> = []; + + for (const entityId of this.getIds()) { + const losslessView = this.rhizomeNode.lossless.view([entityId])[entityId]; + if (!losslessView) continue; + + const validationResult = this.schemaRegistry.validate(entityId, this.schema.id, losslessView); + if (!validationResult.valid) { + invalid.push({ + entityId, + errors: validationResult.errors.map(e => e.message) + }); + } + } + + return invalid; + } + + // Schema introspection + getSchemaInfo(): { + schema: ObjectSchema; + dependencies: string[]; + hasCircularDependencies: boolean; + } { + const dependencies = this.schemaRegistry.getDependencyGraph().get(this.schema.id) || new Set(); + + return { + schema: this.schema, + dependencies: Array.from(dependencies), + hasCircularDependencies: this.schemaRegistry.hasCircularDependencies() + }; + } +} \ No newline at end of file diff --git a/src/delta-validation.ts b/src/delta-validation.ts index dd2fe31..c3f8038 100644 --- a/src/delta-validation.ts +++ b/src/delta-validation.ts @@ -60,8 +60,8 @@ export function validateCreatorId(creator: unknown): creator is CreatorID { } export function validatePointerTarget(target: unknown): target is PointerTarget { - if (target !== null && typeof target !== "string" && typeof target !== "number") { - throw new InvalidPointerError("Pointer target must be string, number, or null"); + if (target !== null && typeof target !== "string" && typeof target !== "number" && typeof target !== "boolean") { + throw new InvalidPointerError("Pointer target must be string, number, boolean, or null"); } return true; } @@ -120,7 +120,7 @@ export function validatePointersV2(pointers: unknown): pointers is PointersV2 { throw new InvalidPointerError("Pointer key must be a non-empty string"); } - if (value !== null && typeof value !== "string" && typeof value !== "number" && typeof value !== "object") { + if (value !== null && typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean" && typeof value !== "object") { throw new InvalidPointerError(`Invalid pointer value for key '${key}'`); } diff --git a/src/delta.ts b/src/delta.ts index 7df1727..bdaa812 100644 --- a/src/delta.ts +++ b/src/delta.ts @@ -8,7 +8,7 @@ const debug = Debug('rz:delta'); export type DeltaID = string; -export type PointerTarget = string | number | null; +export type PointerTarget = string | number | boolean | null; type PointerV1 = { localContext: string; @@ -16,7 +16,7 @@ type PointerV1 = { targetContext?: string; }; -export type Scalar = string | number | null; +export type Scalar = string | number | boolean | null; export type Reference = { [key: DomainEntityID]: PropertyID }; diff --git a/src/schema-registry.ts b/src/schema-registry.ts new file mode 100644 index 0000000..acc203b --- /dev/null +++ b/src/schema-registry.ts @@ -0,0 +1,398 @@ +import Debug from 'debug'; +import { + SchemaRegistry, + ObjectSchema, + SchemaID, + SchemaValidationResult, + SchemaValidationError, + PropertySchema, + PrimitiveSchema, + ReferenceSchema, + ArraySchema, + SchemaAppliedView, + SchemaApplicationOptions +} from './schema'; +import { LosslessViewOne } from './lossless'; +import { DomainEntityID, PropertyID } from './types'; +import { CollapsedDelta } from './lossless'; + +const debug = Debug('rz:schema-registry'); + +export class DefaultSchemaRegistry implements SchemaRegistry { + schemas = new Map(); + + register(schema: ObjectSchema): void { + this.validateSchemaStructure(schema); + this.schemas.set(schema.id, schema); + debug(`Registered schema: ${schema.id} (${schema.name})`); + } + + get(id: SchemaID): ObjectSchema | undefined { + return this.schemas.get(id); + } + + list(): ObjectSchema[] { + return Array.from(this.schemas.values()); + } + + private validateSchemaStructure(schema: ObjectSchema): void { + if (!schema.id || typeof schema.id !== 'string') { + throw new Error('Schema must have a valid string id'); + } + if (!schema.name || typeof schema.name !== 'string') { + throw new Error('Schema must have a valid string name'); + } + if (!schema.properties || typeof schema.properties !== 'object') { + throw new Error('Schema must have properties object'); + } + + // Validate property schemas + for (const [propertyId, propertySchema] of Object.entries(schema.properties)) { + this.validatePropertySchema(propertySchema, `${schema.id}.${propertyId}`); + } + + // Validate required properties exist + if (schema.requiredProperties) { + for (const required of schema.requiredProperties) { + if (!(required in schema.properties)) { + throw new Error(`Required property '${required}' not found in schema '${schema.id}'`); + } + } + } + } + + private validatePropertySchema(schema: PropertySchema, path: string): void { + switch (schema.type) { + case 'primitive': + this.validatePrimitiveSchema(schema, path); + break; + case 'reference': + this.validateReferenceSchema(schema, path); + break; + case 'array': + this.validateArraySchema(schema, path); + break; + default: + throw new Error(`Unknown schema type at ${path}`); + } + } + + private validatePrimitiveSchema(schema: PrimitiveSchema, path: string): void { + const validTypes = ['string', 'number', 'boolean', 'null']; + if (!validTypes.includes(schema.primitiveType)) { + throw new Error(`Invalid primitive type '${schema.primitiveType}' at ${path}`); + } + } + + private validateReferenceSchema(schema: ReferenceSchema, path: string): void { + if (!schema.targetSchema || typeof schema.targetSchema !== 'string') { + throw new Error(`Reference schema must have valid targetSchema at ${path}`); + } + if (schema.maxDepth !== undefined && schema.maxDepth < 0) { + throw new Error(`Reference maxDepth must be non-negative at ${path}`); + } + } + + private validateArraySchema(schema: ArraySchema, path: string): void { + this.validatePropertySchema(schema.itemSchema, `${path}[]`); + if (schema.maxItems !== undefined && schema.maxItems < 0) { + throw new Error(`Array maxItems must be non-negative at ${path}`); + } + } + + validate(entityId: DomainEntityID, schemaId: SchemaID, view: LosslessViewOne): SchemaValidationResult { + const schema = this.get(schemaId); + if (!schema) { + return { + valid: false, + errors: [{ + property: '', + message: `Schema '${schemaId}' not found` + }], + warnings: [] + }; + } + + const errors: SchemaValidationError[] = []; + const warnings: SchemaValidationError[] = []; + + // Check required properties + if (schema.requiredProperties) { + for (const required of schema.requiredProperties) { + if (!(required in view.propertyDeltas) || view.propertyDeltas[required].length === 0) { + errors.push({ + property: required, + message: `Required property '${required}' is missing or has no deltas` + }); + } + } + } + + // Validate each property in the view + for (const [propertyId, deltas] of Object.entries(view.propertyDeltas)) { + const propertySchema = schema.properties[propertyId]; + + if (!propertySchema) { + if (schema.additionalProperties === false) { + warnings.push({ + property: propertyId, + message: `Property '${propertyId}' not defined in schema and additionalProperties is false` + }); + } + continue; + } + + // Validate each delta for this property + for (const delta of deltas) { + const validationResult = this.validateDeltaAgainstPropertySchema( + delta, + propertySchema, + propertyId + ); + errors.push(...validationResult.errors); + warnings.push(...validationResult.warnings); + } + } + + // Validate properties defined in schema but missing from view + for (const [propertyId, propertySchema] of Object.entries(schema.properties)) { + if (!(propertyId in view.propertyDeltas)) { + if (propertySchema.required) { + errors.push({ + property: propertyId, + message: `Required property '${propertyId}' is missing from view` + }); + } + } + } + + return { + valid: errors.length === 0, + errors, + warnings + }; + } + + private validateDeltaAgainstPropertySchema( + delta: CollapsedDelta, + schema: PropertySchema, + propertyId: PropertyID + ): SchemaValidationResult { + const errors: SchemaValidationError[] = []; + const warnings: SchemaValidationError[] = []; + + // Extract the value from the delta + const valuePointer = delta.pointers.find(p => p[propertyId] !== undefined); + if (!valuePointer) { + errors.push({ + property: propertyId, + message: `Delta does not contain expected property '${propertyId}'` + }); + return { valid: false, errors, warnings }; + } + + const value = valuePointer[propertyId]; + + switch (schema.type) { + case 'primitive': + this.validatePrimitiveValue(value, schema, propertyId, errors); + break; + case 'reference': + this.validateReferenceValue(value, schema, propertyId, errors, warnings); + break; + case 'array': + // Arrays are complex - for now just warn that we don't fully validate them + warnings.push({ + property: propertyId, + message: `Array validation not fully implemented for property '${propertyId}'` + }); + break; + } + + return { + valid: errors.length === 0, + errors, + warnings + }; + } + + private validatePrimitiveValue( + value: unknown, + schema: PrimitiveSchema, + propertyId: PropertyID, + errors: SchemaValidationError[] + ): void { + let valid = false; + + switch (schema.primitiveType) { + case 'string': + valid = typeof value === 'string'; + break; + case 'number': + valid = typeof value === 'number'; + break; + case 'boolean': + valid = typeof value === 'boolean'; + break; + case 'null': + valid = value === null; + break; + } + + if (!valid) { + errors.push({ + property: propertyId, + message: `Expected ${schema.primitiveType} but got ${typeof value}`, + expectedType: schema.primitiveType, + actualValue: value + }); + } + } + + private validateReferenceValue( + value: unknown, + schema: ReferenceSchema, + propertyId: PropertyID, + errors: SchemaValidationError[], + warnings: SchemaValidationError[] + ): void { + if (typeof value !== 'string') { + errors.push({ + property: propertyId, + message: `Reference value must be a string (entity ID), got ${typeof value}`, + expectedType: 'string (entity ID)', + actualValue: value + }); + return; + } + + // Check if target schema exists + const targetSchema = this.get(schema.targetSchema); + if (!targetSchema) { + warnings.push({ + property: propertyId, + message: `Target schema '${schema.targetSchema}' not found for reference` + }); + } + } + + applySchema( + view: LosslessViewOne, + schemaId: SchemaID, + options: SchemaApplicationOptions = {} + ): SchemaAppliedView { + const schema = this.get(schemaId); + if (!schema) { + throw new Error(`Schema '${schemaId}' not found`); + } + + const { includeMetadata = true, strictValidation = false } = options; + + const appliedView: SchemaAppliedView = { + id: view.id, + schemaId, + properties: {} + }; + + // Apply schema to each property + for (const [propertyId, propertySchema] of Object.entries(schema.properties)) { + const deltas = view.propertyDeltas[propertyId] || []; + const validationResult = this.validate(view.id, schemaId, view); + + appliedView.properties[propertyId] = { + deltas, + schema: propertySchema, + validationResult + }; + + // If strict validation is enabled and there are errors, throw + if (strictValidation && !validationResult.valid) { + throw new Error(`Schema validation failed for property '${propertyId}': ${validationResult.errors[0]?.message}`); + } + } + + // Add metadata if requested + if (includeMetadata) { + appliedView.metadata = { + appliedAt: Date.now(), + depth: 1, // TODO: Calculate actual depth in nested references + truncated: false // TODO: Mark if we hit maxDepth limits + }; + } + + return appliedView; + } + + // Helper method to resolve circular dependencies + getDependencyGraph(): Map> { + const dependencies = new Map>(); + + for (const schema of this.schemas.values()) { + const deps = new Set(); + this.collectSchemaDependencies(schema, deps); + dependencies.set(schema.id, deps); + } + + return dependencies; + } + + private collectSchemaDependencies(schema: ObjectSchema, deps: Set): void { + for (const propertySchema of Object.values(schema.properties)) { + this.collectPropertySchemaDependencies(propertySchema, deps); + } + } + + private collectPropertySchemaDependencies(schema: PropertySchema, deps: Set): void { + switch (schema.type) { + case 'reference': + deps.add(schema.targetSchema); + break; + case 'array': + this.collectPropertySchemaDependencies(schema.itemSchema, deps); + break; + } + } + + // Check for circular dependencies + hasCircularDependencies(): boolean { + const dependencies = this.getDependencyGraph(); + const visited = new Set(); + const recursionStack = new Set(); + + for (const schemaId of dependencies.keys()) { + if (this.hasCircularDependencyDFS(schemaId, dependencies, visited, recursionStack)) { + return true; + } + } + + return false; + } + + private hasCircularDependencyDFS( + schemaId: SchemaID, + dependencies: Map>, + visited: Set, + recursionStack: Set + ): boolean { + if (recursionStack.has(schemaId)) { + return true; // Circular dependency found + } + + if (visited.has(schemaId)) { + return false; // Already processed + } + + visited.add(schemaId); + recursionStack.add(schemaId); + + const deps = dependencies.get(schemaId) || new Set(); + for (const dep of deps) { + if (this.hasCircularDependencyDFS(dep, dependencies, visited, recursionStack)) { + return true; + } + } + + recursionStack.delete(schemaId); + return false; + } +} \ No newline at end of file diff --git a/src/schema.ts b/src/schema.ts new file mode 100644 index 0000000..71133c3 --- /dev/null +++ b/src/schema.ts @@ -0,0 +1,233 @@ +import { DomainEntityID, PropertyID, PropertyTypes } from "./types"; +import { LosslessViewOne } from "./lossless"; +import { CollapsedDelta } from "./lossless"; + +// Base schema types +export type SchemaID = string; + +// Primitive schema types - these terminate the recursion +export type PrimitiveSchemaType = 'string' | 'number' | 'boolean' | 'null'; + +export interface PrimitiveSchema { + type: 'primitive'; + primitiveType: PrimitiveSchemaType; + required?: boolean; + default?: PropertyTypes; +} + +// Reference schema for linking to other objects +export interface ReferenceSchema { + type: 'reference'; + targetSchema: SchemaID; // Reference to another schema by ID + required?: boolean; + maxDepth?: number; // Prevent infinite recursion +} + +// Array schema for collections of values +export interface ArraySchema { + type: 'array'; + itemSchema: PropertySchema; + required?: boolean; + maxItems?: number; +} + +// Union type for all property schema types +export type PropertySchema = PrimitiveSchema | ReferenceSchema | ArraySchema; + +// Object schema defines the structure of an entity +export interface ObjectSchema { + id: SchemaID; + name: string; + description?: string; + properties: { + [propertyId: PropertyID]: PropertySchema; + }; + requiredProperties?: PropertyID[]; + additionalProperties?: boolean; // Allow properties not in schema +} + +// Schema registry manages all schemas +export interface SchemaRegistry { + schemas: Map; + register(schema: ObjectSchema): void; + get(id: SchemaID): ObjectSchema | undefined; + list(): ObjectSchema[]; + validate(entityId: DomainEntityID, schemaId: SchemaID, view: LosslessViewOne): SchemaValidationResult; +} + +// Validation result types +export interface SchemaValidationError { + property: PropertyID; + message: string; + expectedType?: string; + actualValue?: unknown; +} + +export interface SchemaValidationResult { + valid: boolean; + errors: SchemaValidationError[]; + warnings: SchemaValidationError[]; +} + +// Schema application options +export interface SchemaApplicationOptions { + maxDepth?: number; + includeMetadata?: boolean; + strictValidation?: boolean; +} + +// Applied schema result - a lossless view filtered through a schema +export interface SchemaAppliedView { + id: DomainEntityID; + schemaId: SchemaID; + properties: { + [propertyId: PropertyID]: { + deltas: CollapsedDelta[]; + schema: PropertySchema; + validationResult: SchemaValidationResult; + }; + }; + metadata?: { + appliedAt: number; + depth: number; + truncated: boolean; + }; +} + +// Schema-based collection interface +export interface TypedCollection { + schema: ObjectSchema; + validate(entity: T): SchemaValidationResult; + apply(view: LosslessViewOne): SchemaAppliedView; +} + +// Built-in schema helpers +export const PrimitiveSchemas = { + string: (): PrimitiveSchema => ({ type: 'primitive', primitiveType: 'string' }), + number: (): PrimitiveSchema => ({ type: 'primitive', primitiveType: 'number' }), + boolean: (): PrimitiveSchema => ({ type: 'primitive', primitiveType: 'boolean' }), + null: (): PrimitiveSchema => ({ type: 'primitive', primitiveType: 'null' }), + + requiredString: (): PrimitiveSchema => ({ type: 'primitive', primitiveType: 'string', required: true }), + requiredNumber: (): PrimitiveSchema => ({ type: 'primitive', primitiveType: 'number', required: true }), + requiredBoolean: (): PrimitiveSchema => ({ type: 'primitive', primitiveType: 'boolean', required: true }), +} as const; + +export const ReferenceSchemas = { + to: (targetSchema: SchemaID, maxDepth = 3): ReferenceSchema => ({ + type: 'reference', + targetSchema, + maxDepth + }), + + required: (targetSchema: SchemaID, maxDepth = 3): ReferenceSchema => ({ + type: 'reference', + targetSchema, + maxDepth, + required: true + }) +} as const; + +export const ArraySchemas = { + of: (itemSchema: PropertySchema, maxItems?: number): ArraySchema => ({ + type: 'array', + itemSchema, + maxItems + }), + + required: (itemSchema: PropertySchema, maxItems?: number): ArraySchema => ({ + type: 'array', + itemSchema, + maxItems, + required: true + }) +} as const; + +// Schema builder for fluent API +export class SchemaBuilder { + private schema: Partial = {}; + + static create(id: SchemaID): SchemaBuilder { + const builder = new SchemaBuilder(); + builder.schema.id = id; + builder.schema.properties = {}; + return builder; + } + + name(name: string): SchemaBuilder { + this.schema.name = name; + return this; + } + + description(description: string): SchemaBuilder { + this.schema.description = description; + return this; + } + + property(propertyId: PropertyID, schema: PropertySchema): SchemaBuilder { + if (!this.schema.properties) this.schema.properties = {}; + this.schema.properties[propertyId] = schema; + return this; + } + + required(...propertyIds: PropertyID[]): SchemaBuilder { + this.schema.requiredProperties = [ + ...(this.schema.requiredProperties || []), + ...propertyIds + ]; + return this; + } + + additionalProperties(allowed = true): SchemaBuilder { + this.schema.additionalProperties = allowed; + return this; + } + + build(): ObjectSchema { + if (!this.schema.id || !this.schema.name) { + throw new Error('Schema must have id and name'); + } + return this.schema as ObjectSchema; + } +} + +// Common schema patterns +export const CommonSchemas = { + // User schema with friends references + User: () => SchemaBuilder + .create('user') + .name('User') + .description('A user entity with profile information') + .property('name', PrimitiveSchemas.requiredString()) + .property('email', PrimitiveSchemas.string()) + .property('age', PrimitiveSchemas.number()) + .property('active', PrimitiveSchemas.boolean()) + .property('friends', ArraySchemas.of(ReferenceSchemas.to('user-summary', 1))) + .required('name') + .build(), + + // User summary schema for references to prevent infinite recursion + UserSummary: () => SchemaBuilder + .create('user-summary') + .name('User Summary') + .description('Abbreviated user information for references') + .property('name', PrimitiveSchemas.requiredString()) + .property('email', PrimitiveSchemas.string()) + .required('name') + .additionalProperties(false) + .build(), + + // Document schema + Document: () => SchemaBuilder + .create('document') + .name('Document') + .description('A document with metadata') + .property('title', PrimitiveSchemas.requiredString()) + .property('content', PrimitiveSchemas.string()) + .property('author', ReferenceSchemas.required('user-summary')) + .property('tags', ArraySchemas.of(PrimitiveSchemas.string())) + .property('created', PrimitiveSchemas.requiredNumber()) + .property('published', PrimitiveSchemas.boolean()) + .required('title', 'author', 'created') + .build() +} as const; \ No newline at end of file diff --git a/src/types.ts b/src/types.ts index 9bfe797..b49d1d1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -4,7 +4,7 @@ export type FilterExpr = JSONLogic; export type FilterGenerator = () => FilterExpr; -export type PropertyTypes = string | number | null; +export type PropertyTypes = string | number | boolean | null; export type DomainEntityID = string; export type PropertyID = string; diff --git a/todo.md b/todo.md index 20d8b2e..59dc6fe 100644 --- a/todo.md +++ b/todo.md @@ -17,12 +17,12 @@ This document tracks work needed to achieve full specification compliance, organ - [x] Test atomic transaction operations - [ ] Add transaction rollback capabilities (deferred - not critical for spec parity) -### 1.3 Schema Foundation -- [ ] Design schema type definitions based on spec -- [ ] Implement basic schema validation -- [ ] Create schema registry/storage mechanism -- [ ] Add property type enforcement -- [ ] Test schema application to collections +### 1.3 Schema Foundation ✅ +- [x] Design schema type definitions based on spec +- [x] Implement basic schema validation +- [x] Create schema registry/storage mechanism +- [x] Add property type enforcement +- [x] Test schema application to collections ## Phase 2: Core Features (Spec Compliance)