← Files EW_knowledge_forgeARCHIVED FILE
skills/mcp-knowledge-forge/scripts/knowledge_package.py
12.2 KB · Nov 21, 2026 · 06:10 UTC
#!/usr/bin/env python3
"""Validate/build EW knowledge interchange packages using Python's standard library.
The schema walker implements only keywords used by the bundled contract. It is
not a general JSON Schema implementation. Unsupported keywords fail closed.
"""
import argparse
import datetime as dt
import hashlib
import json
from pathlib import Path
import re
import sys
import zipfile
ASSETS = Path(__file__).resolve().parents[1] / 'assets'
SCHEMA = ASSETS / 'knowledge.schema.json'
def load_json(path):
def pairs(items):
result = {}
for key, value in items:
if key in result:
raise ValueError(f'duplicate JSON key: {key}')
result[key] = value
return result
return json.loads(Path(path).read_text(encoding='utf-8'), object_pairs_hook=pairs,
parse_constant=lambda x: (_ for _ in ()).throw(ValueError(f'invalid constant: {x}')))
def timestamp(value):
if not re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})', value):
raise ValueError('timezone-bearing timestamp required')
return dt.datetime.fromisoformat(value.replace('Z', '+00:00'))
def shape(value, schema, path, errors):
supported = {'$schema', 'type', 'properties', 'required', 'additionalProperties',
'items', 'minItems', 'uniqueItems', 'minLength', 'pattern', 'enum',
'const', 'format', 'minimum'}
unknown = set(schema) - supported
if unknown:
errors.append(f'{path}: unsupported schema keywords {sorted(unknown)}')
return
types = schema.get('type', [])
types = [types] if isinstance(types, str) else types
checks = {'object': lambda x: isinstance(x, dict), 'array': lambda x: isinstance(x, list),
'string': lambda x: isinstance(x, str), 'null': lambda x: x is None,
'integer': lambda x: type(x) is int}
if types and not any(checks[t](value) for t in types):
errors.append(f'{path}: expected {types}')
return
if 'const' in schema and value != schema['const']:
errors.append(f'{path}: incorrect contract value')
if 'enum' in schema and value not in schema['enum']:
errors.append(f'{path}: unsupported value')
if isinstance(value, dict):
for key in schema.get('required', []):
if key not in value:
errors.append(f'{path}.{key}: missing')
props = schema.get('properties', {})
if schema.get('additionalProperties') is False:
for key in value.keys() - props.keys():
errors.append(f'{path}.{key}: unexpected field')
for key in value.keys() & props.keys():
shape(value[key], props[key], f'{path}.{key}', errors)
if isinstance(value, list):
if len(value) < schema.get('minItems', 0):
errors.append(f'{path}: too few items')
if schema.get('uniqueItems') and len({json.dumps(x, sort_keys=True, ensure_ascii=False) for x in value}) != len(value):
errors.append(f'{path}: duplicate items')
for i, item in enumerate(value):
shape(item, schema.get('items', {}), f'{path}[{i}]', errors)
if isinstance(value, str):
if 'minLength' in schema and (len(value) < schema['minLength'] or not value.strip()):
errors.append(f'{path}: blank text')
if 'pattern' in schema and not re.search(schema['pattern'], value):
errors.append(f'{path}: invalid pattern')
try:
if schema.get('format') == 'date':
if not re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
raise ValueError('date must use YYYY-MM-DD')
dt.date.fromisoformat(value)
elif schema.get('format') == 'date-time':
timestamp(value)
except ValueError:
errors.append(f'{path}: invalid {schema["format"]}')
if type(value) is int and value < schema.get('minimum', value):
errors.append(f'{path}: below minimum')
def validate(data):
errors, warnings = [], []
shape(data, load_json(SCHEMA), '$', errors)
if errors:
return {'valid': False, 'errors': errors, 'warnings': warnings, 'record_count': 0, 'index_count': 0}
sources, records = {}, {}
for src in data['sources']:
sid = src['source_id']
if sid in sources:
errors.append(f'{sid}: duplicate source ID')
sources[sid] = src
if src['extraction_status'] != 'complete' or src['warnings']:
warnings.append(f'{sid}: extraction incomplete or source warnings present')
for rec in data['records']:
kid = rec['knowledge_id']
if kid in records:
errors.append(f'{kid}: duplicate knowledge ID')
records[kid] = rec
for kid, rec in records.items():
if timestamp(rec['updated_at']) < timestamp(rec['created_at']):
errors.append(f'{kid}: updated_at precedes created_at')
for field in ['validity', 'legal']:
period = rec[field]
if not period:
continue
if period['effective_from'] and period['effective_until'] and period['effective_until'] < period['effective_from']:
errors.append(f'{kid}.{field}: reversed validity period')
if rec['legal'] and (rec['legal']['effective_from'] is None or rec['legal']['verified_at'] is None):
warnings.append(f'{kid}: legal validity or verification date unknown')
uncertain = False
for ref in rec['source_refs']:
src = sources.get(ref['source_id'])
if src is None:
errors.append(f'{kid}: missing source {ref["source_id"]}')
elif src['extraction_status'] != 'complete' or src['warnings']:
uncertain = True
loc = ref['locator']
if loc['kind'] == 'unlocated':
uncertain = True
if loc['value'] is not None:
errors.append(f'{kid}: unlocated reference must use null value')
warnings.append(f'{kid}: source location unknown')
elif loc['value'] is None:
errors.append(f'{kid}: source location required')
access, approval = rec['access'], rec['approval']
if access['classification'] == 'unknown':
warnings.append(f'{kid}: classification unresolved')
if access['classification'] == 'P3' and access['route'] not in ['local-only', 'blocked']:
errors.append(f'{kid}: P3 requires local-only route')
if access['classification'] == 'P2' and access['route'] not in ['local-only', 'cloud-sanitized', 'blocked']:
errors.append(f'{kid}: P2 route invalid')
if approval['status'] == 'approved':
if not all(approval[k] for k in ['reviewer', 'reviewed_at', 'evidence']):
errors.append(f'{kid}: approval evidence incomplete')
elif timestamp(approval['reviewed_at']) < timestamp(rec['updated_at']):
errors.append(f'{kid}: approval predates content update')
if rec['status'] == 'approved':
if approval['status'] != 'approved':
errors.append(f'{kid}: approved record lacks approval')
if access['classification'] == 'unknown' or access['route'] == 'blocked' or not access['audiences'] or uncertain or rec['content_kind'] == 'unresolved':
errors.append(f'{kid}: approved record has unresolved source/access/content')
elif rec['status'] == 'candidate' and approval['status'] != 'pending':
errors.append(f'{kid}: candidate must have pending approval')
elif rec['status'] == 'rejected' and approval['status'] != 'rejected':
errors.append(f'{kid}: rejected record must have rejected approval')
for relation, targets in rec['relations'].items():
for target in targets:
if target == kid or target not in records:
errors.append(f'{kid}.{relation}: self or missing relation {target}')
elif relation == 'replaces' and records[target]['status'] not in ['superseded', 'withdrawn']:
errors.append(f'{kid}: replaced record {target} remains active')
# Iterative traversal avoids recursion depth failures on large chains.
for relation in ['up', 'replaces']:
finished = set()
for kid in records:
active, stack = set(), [(kid, False)]
while stack:
node, leaving = stack.pop()
if leaving:
active.discard(node)
finished.add(node)
elif node in active:
errors.append(f'{node}: cyclic {relation} relations')
break
elif node not in finished and node in records:
active.add(node)
stack.append((node, True))
stack.extend((n, False) for n in records[node]['relations'][relation])
eligible = [r for r in records.values() if r['status'] == 'approved']
return {'valid': not errors, 'errors': errors, 'warnings': warnings,
'record_count': len(data['records']), 'index_count': len(eligible) if not errors else 0}
def encode(data):
return (json.dumps(data, ensure_ascii=False, indent=2, allow_nan=False) + '\n').encode('utf-8')
def build(source, output, review_dir=None):
data = load_json(source)
report = validate(data)
if not report['valid']:
raise ValueError(json.dumps(report, ensure_ascii=False))
index = {'format': 'ew-mcp-retrieval-index', 'schema_version': '1.0.0',
'knowledge_base_id': data['knowledge_base_id'], 'records': []}
for record in data['records']:
if record['status'] == 'approved':
entry = {k: record[k] for k in ['knowledge_id', 'version', 'title', 'category', 'summary',
'keywords', 'synonyms', 'questions', 'source_refs', 'access', 'approval', 'validity', 'legal']}
entry['sha256'] = hashlib.sha256(encode(record)).hexdigest()
index['records'].append(entry)
files = {'knowledge.json': encode(data), 'knowledge.schema.json': SCHEMA.read_bytes(),
'retrieval-index.json': encode(index), 'validation-report.json': encode(report)}
if review_dir:
root = Path(review_dir)
if not root.is_dir() or root.is_symlink():
raise ValueError('review-dir must be a real directory')
for path in sorted(root.rglob('*')):
if path.is_symlink():
raise ValueError('symlinks are not allowed in review directory')
if path.is_file():
rel = path.relative_to(root)
if path.suffix.lower() != '.md' or any(x.startswith('.') for x in rel.parts):
raise ValueError('review directory permits visible Markdown files only')
files['review/' + rel.as_posix()] = path.read_bytes()
output = Path(output)
if output.resolve() == Path(source).resolve() or output.exists():
raise ValueError('output must be a new path; refusing overwrite')
output.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(output, 'x', zipfile.ZIP_DEFLATED) as archive:
for name, content in files.items():
archive.writestr(name, content)
with zipfile.ZipFile(output) as archive:
if archive.testzip() is not None:
raise ValueError('ZIP integrity failure')
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=['validate', 'build'])
parser.add_argument('input', type=Path)
parser.add_argument('--output', type=Path)
parser.add_argument('--review-dir', type=Path)
args = parser.parse_args()
try:
if args.command == 'build':
if args.output is None:
parser.error('build requires --output')
report = build(args.input, args.output, args.review_dir)
else:
report = validate(load_json(args.input))
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report['valid'] else 1
except (ValueError, OSError, zipfile.BadZipFile) as exc:
print(json.dumps({'valid': False, 'errors': [str(exc)]}, ensure_ascii=False), file=sys.stderr)
return 1
if __name__ == '__main__':
sys.exit(main())
SHA-256: 15e2f3fd7d6806ba6ffdd12a90f7b3bf616b4673f79e3a1190cad14460e32c97