add brain
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,731 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script Tester - Tests Python scripts in a skill directory
|
||||
|
||||
This script validates and tests Python scripts within a skill directory by checking
|
||||
syntax, imports, runtime execution, argparse functionality, and output formats.
|
||||
It ensures scripts meet quality standards and function correctly.
|
||||
|
||||
Usage:
|
||||
python script_tester.py <skill_path> [--timeout SECONDS] [--json] [--verbose]
|
||||
|
||||
Author: Claude Skills Engineering Team
|
||||
Version: 1.0.0
|
||||
Dependencies: Python Standard Library Only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional, Tuple, Union
|
||||
import threading
|
||||
|
||||
|
||||
class TestError(Exception):
|
||||
"""Custom exception for testing errors"""
|
||||
pass
|
||||
|
||||
|
||||
class ScriptTestResult:
|
||||
"""Container for individual script test results"""
|
||||
|
||||
def __init__(self, script_path: str):
|
||||
self.script_path = script_path
|
||||
self.script_name = Path(script_path).name
|
||||
self.timestamp = datetime.utcnow().isoformat() + "Z"
|
||||
self.tests = {}
|
||||
self.overall_status = "PENDING"
|
||||
self.execution_time = 0.0
|
||||
self.errors = []
|
||||
self.warnings = []
|
||||
|
||||
def add_test(self, test_name: str, passed: bool, message: str = "", details: Dict = None):
|
||||
"""Add a test result"""
|
||||
self.tests[test_name] = {
|
||||
"passed": passed,
|
||||
"message": message,
|
||||
"details": details or {}
|
||||
}
|
||||
|
||||
def add_error(self, error: str):
|
||||
"""Add an error message"""
|
||||
self.errors.append(error)
|
||||
|
||||
def add_warning(self, warning: str):
|
||||
"""Add a warning message"""
|
||||
self.warnings.append(warning)
|
||||
|
||||
def calculate_status(self):
|
||||
"""Calculate overall test status"""
|
||||
if not self.tests:
|
||||
self.overall_status = "NO_TESTS"
|
||||
return
|
||||
|
||||
failed_tests = [name for name, result in self.tests.items() if not result["passed"]]
|
||||
|
||||
if not failed_tests:
|
||||
self.overall_status = "PASS"
|
||||
elif len(failed_tests) <= len(self.tests) // 2:
|
||||
self.overall_status = "PARTIAL"
|
||||
else:
|
||||
self.overall_status = "FAIL"
|
||||
|
||||
|
||||
class TestSuite:
|
||||
"""Container for all test results"""
|
||||
|
||||
def __init__(self, skill_path: str):
|
||||
self.skill_path = skill_path
|
||||
self.timestamp = datetime.utcnow().isoformat() + "Z"
|
||||
self.script_results = {}
|
||||
self.summary = {}
|
||||
self.global_errors = []
|
||||
|
||||
def add_script_result(self, result: ScriptTestResult):
|
||||
"""Add a script test result"""
|
||||
self.script_results[result.script_name] = result
|
||||
|
||||
def add_global_error(self, error: str):
|
||||
"""Add a global error message"""
|
||||
self.global_errors.append(error)
|
||||
|
||||
def calculate_summary(self):
|
||||
"""Calculate summary statistics"""
|
||||
if not self.script_results:
|
||||
self.summary = {
|
||||
"total_scripts": 0,
|
||||
"passed": 0,
|
||||
"partial": 0,
|
||||
"failed": 0,
|
||||
"overall_status": "NO_SCRIPTS"
|
||||
}
|
||||
return
|
||||
|
||||
statuses = [result.overall_status for result in self.script_results.values()]
|
||||
|
||||
self.summary = {
|
||||
"total_scripts": len(self.script_results),
|
||||
"passed": statuses.count("PASS"),
|
||||
"partial": statuses.count("PARTIAL"),
|
||||
"failed": statuses.count("FAIL"),
|
||||
"no_tests": statuses.count("NO_TESTS")
|
||||
}
|
||||
|
||||
# Determine overall status
|
||||
if self.summary["failed"] == 0 and self.summary["no_tests"] == 0:
|
||||
self.summary["overall_status"] = "PASS"
|
||||
elif self.summary["passed"] > 0:
|
||||
self.summary["overall_status"] = "PARTIAL"
|
||||
else:
|
||||
self.summary["overall_status"] = "FAIL"
|
||||
|
||||
|
||||
class ScriptTester:
|
||||
"""Main script testing engine"""
|
||||
|
||||
def __init__(self, skill_path: str, timeout: int = 30, verbose: bool = False):
|
||||
self.skill_path = Path(skill_path).resolve()
|
||||
self.timeout = timeout
|
||||
self.verbose = verbose
|
||||
self.test_suite = TestSuite(str(self.skill_path))
|
||||
|
||||
def log_verbose(self, message: str):
|
||||
"""Log verbose message if verbose mode enabled"""
|
||||
if self.verbose:
|
||||
print(f"[VERBOSE] {message}", file=sys.stderr)
|
||||
|
||||
def test_all_scripts(self) -> TestSuite:
|
||||
"""Main entry point - test all scripts in the skill"""
|
||||
try:
|
||||
self.log_verbose(f"Starting script testing for {self.skill_path}")
|
||||
|
||||
# Check if skill path exists
|
||||
if not self.skill_path.exists():
|
||||
self.test_suite.add_global_error(f"Skill path does not exist: {self.skill_path}")
|
||||
return self.test_suite
|
||||
|
||||
scripts_dir = self.skill_path / "scripts"
|
||||
if not scripts_dir.exists():
|
||||
self.test_suite.add_global_error("No scripts directory found")
|
||||
return self.test_suite
|
||||
|
||||
# Find all Python scripts
|
||||
python_files = list(scripts_dir.glob("*.py"))
|
||||
if not python_files:
|
||||
self.test_suite.add_global_error("No Python scripts found in scripts directory")
|
||||
return self.test_suite
|
||||
|
||||
self.log_verbose(f"Found {len(python_files)} Python scripts to test")
|
||||
|
||||
# Test each script
|
||||
for script_path in python_files:
|
||||
try:
|
||||
result = self.test_single_script(script_path)
|
||||
self.test_suite.add_script_result(result)
|
||||
except Exception as e:
|
||||
# Create a failed result for the script
|
||||
result = ScriptTestResult(str(script_path))
|
||||
result.add_error(f"Failed to test script: {str(e)}")
|
||||
result.overall_status = "FAIL"
|
||||
self.test_suite.add_script_result(result)
|
||||
|
||||
# Calculate summary
|
||||
self.test_suite.calculate_summary()
|
||||
|
||||
except Exception as e:
|
||||
self.test_suite.add_global_error(f"Testing failed with exception: {str(e)}")
|
||||
|
||||
return self.test_suite
|
||||
|
||||
def test_single_script(self, script_path: Path) -> ScriptTestResult:
|
||||
"""Test a single Python script comprehensively"""
|
||||
result = ScriptTestResult(str(script_path))
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
self.log_verbose(f"Testing script: {script_path.name}")
|
||||
|
||||
# Read script content
|
||||
try:
|
||||
content = script_path.read_text(encoding='utf-8')
|
||||
except Exception as e:
|
||||
result.add_test("file_readable", False, f"Cannot read file: {str(e)}")
|
||||
result.add_error(f"Cannot read script file: {str(e)}")
|
||||
result.overall_status = "FAIL"
|
||||
return result
|
||||
|
||||
result.add_test("file_readable", True, "Script file is readable")
|
||||
|
||||
# Test 1: Syntax validation
|
||||
self._test_syntax(content, result)
|
||||
|
||||
# Test 2: Import validation
|
||||
self._test_imports(content, result)
|
||||
|
||||
# Test 3: Argparse validation
|
||||
self._test_argparse_implementation(content, result)
|
||||
|
||||
# Test 4: Main guard validation
|
||||
self._test_main_guard(content, result)
|
||||
|
||||
# Test 5: Runtime execution tests
|
||||
if result.tests.get("syntax_valid", {}).get("passed", False):
|
||||
self._test_script_execution(script_path, result)
|
||||
|
||||
# Test 6: Help functionality
|
||||
if result.tests.get("syntax_valid", {}).get("passed", False):
|
||||
self._test_help_functionality(script_path, result)
|
||||
|
||||
# Test 7: Sample data processing (if available)
|
||||
self._test_sample_data_processing(script_path, result)
|
||||
|
||||
# Test 8: Output format validation
|
||||
self._test_output_formats(script_path, result)
|
||||
|
||||
except Exception as e:
|
||||
result.add_error(f"Unexpected error during testing: {str(e)}")
|
||||
|
||||
finally:
|
||||
result.execution_time = time.time() - start_time
|
||||
result.calculate_status()
|
||||
|
||||
return result
|
||||
|
||||
def _test_syntax(self, content: str, result: ScriptTestResult):
|
||||
"""Test Python syntax validity"""
|
||||
self.log_verbose("Testing syntax...")
|
||||
|
||||
try:
|
||||
ast.parse(content)
|
||||
result.add_test("syntax_valid", True, "Python syntax is valid")
|
||||
except SyntaxError as e:
|
||||
result.add_test("syntax_valid", False, f"Syntax error: {str(e)}",
|
||||
{"error": str(e), "line": getattr(e, 'lineno', 'unknown')})
|
||||
result.add_error(f"Syntax error: {str(e)}")
|
||||
|
||||
def _test_imports(self, content: str, result: ScriptTestResult):
|
||||
"""Test import statements for external dependencies"""
|
||||
self.log_verbose("Testing imports...")
|
||||
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
external_imports = self._find_external_imports(tree)
|
||||
|
||||
if not external_imports:
|
||||
result.add_test("imports_valid", True, "Uses only standard library imports")
|
||||
else:
|
||||
result.add_test("imports_valid", False,
|
||||
f"Uses external imports: {', '.join(external_imports)}",
|
||||
{"external_imports": external_imports})
|
||||
result.add_error(f"External imports detected: {', '.join(external_imports)}")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("imports_valid", False, f"Error analyzing imports: {str(e)}")
|
||||
|
||||
def _find_external_imports(self, tree: ast.AST) -> List[str]:
|
||||
"""Find external (non-stdlib) imports"""
|
||||
# Comprehensive standard library module list
|
||||
stdlib_modules = {
|
||||
# Built-in modules
|
||||
'argparse', 'ast', 'json', 'os', 'sys', 'pathlib', 'datetime', 'typing',
|
||||
'collections', 're', 'math', 'random', 'itertools', 'functools', 'operator',
|
||||
'csv', 'sqlite3', 'urllib', 'http', 'html', 'xml', 'email', 'base64',
|
||||
'hashlib', 'hmac', 'secrets', 'tempfile', 'shutil', 'glob', 'fnmatch',
|
||||
'subprocess', 'threading', 'multiprocessing', 'queue', 'time', 'calendar',
|
||||
'locale', 'gettext', 'logging', 'warnings', 'unittest', 'doctest',
|
||||
'pickle', 'copy', 'pprint', 'reprlib', 'enum', 'dataclasses',
|
||||
'contextlib', 'abc', 'atexit', 'traceback', 'gc', 'weakref', 'types',
|
||||
'decimal', 'fractions', 'statistics', 'cmath', 'platform', 'errno',
|
||||
'io', 'codecs', 'unicodedata', 'stringprep', 'textwrap', 'string',
|
||||
'struct', 'difflib', 'heapq', 'bisect', 'array', 'uuid', 'mmap',
|
||||
'ctypes', 'winreg', 'msvcrt', 'winsound', 'posix', 'pwd', 'grp',
|
||||
'crypt', 'termios', 'tty', 'pty', 'fcntl', 'resource', 'nis',
|
||||
'syslog', 'signal', 'socket', 'ssl', 'select', 'selectors',
|
||||
'asyncio', 'asynchat', 'asyncore', 'netrc', 'xdrlib', 'plistlib',
|
||||
'mailbox', 'mimetypes', 'encodings', 'pkgutil', 'modulefinder',
|
||||
'runpy', 'importlib', 'imp', 'zipimport', 'zipfile', 'tarfile',
|
||||
'gzip', 'bz2', 'lzma', 'zlib', 'binascii', 'quopri', 'uu',
|
||||
'configparser', 'netrc', 'xdrlib', 'plistlib', 'token', 'tokenize',
|
||||
'keyword', 'heapq', 'bisect', 'array', 'weakref', 'types',
|
||||
'copyreg', 'shelve', 'marshal', 'dbm', 'sqlite3', 'zoneinfo'
|
||||
}
|
||||
|
||||
external_imports = []
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
module_name = alias.name.split('.')[0]
|
||||
if module_name not in stdlib_modules and not module_name.startswith('_'):
|
||||
external_imports.append(alias.name)
|
||||
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
module_name = node.module.split('.')[0]
|
||||
if module_name not in stdlib_modules and not module_name.startswith('_'):
|
||||
external_imports.append(node.module)
|
||||
|
||||
return list(set(external_imports))
|
||||
|
||||
def _test_argparse_implementation(self, content: str, result: ScriptTestResult):
|
||||
"""Test argparse implementation"""
|
||||
self.log_verbose("Testing argparse implementation...")
|
||||
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
|
||||
# Check for argparse import
|
||||
has_argparse_import = False
|
||||
has_parser_creation = False
|
||||
has_parse_args = False
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
if (isinstance(node, ast.Import) and
|
||||
any(alias.name == 'argparse' for alias in node.names)):
|
||||
has_argparse_import = True
|
||||
elif (isinstance(node, ast.ImportFrom) and
|
||||
node.module == 'argparse'):
|
||||
has_argparse_import = True
|
||||
|
||||
elif isinstance(node, ast.Call):
|
||||
# Check for ArgumentParser creation
|
||||
if (isinstance(node.func, ast.Attribute) and
|
||||
isinstance(node.func.value, ast.Name) and
|
||||
node.func.value.id == 'argparse' and
|
||||
node.func.attr == 'ArgumentParser'):
|
||||
has_parser_creation = True
|
||||
|
||||
# Check for parse_args call
|
||||
if (isinstance(node.func, ast.Attribute) and
|
||||
node.func.attr == 'parse_args'):
|
||||
has_parse_args = True
|
||||
|
||||
argparse_score = sum([has_argparse_import, has_parser_creation, has_parse_args])
|
||||
|
||||
if argparse_score == 3:
|
||||
result.add_test("argparse_implementation", True, "Complete argparse implementation found")
|
||||
elif argparse_score > 0:
|
||||
result.add_test("argparse_implementation", False,
|
||||
"Partial argparse implementation",
|
||||
{"missing_components": [
|
||||
comp for comp, present in [
|
||||
("import", has_argparse_import),
|
||||
("parser_creation", has_parser_creation),
|
||||
("parse_args", has_parse_args)
|
||||
] if not present
|
||||
]})
|
||||
result.add_warning("Incomplete argparse implementation")
|
||||
else:
|
||||
result.add_test("argparse_implementation", False, "No argparse implementation found")
|
||||
result.add_error("Script should use argparse for command-line arguments")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("argparse_implementation", False, f"Error analyzing argparse: {str(e)}")
|
||||
|
||||
def _test_main_guard(self, content: str, result: ScriptTestResult):
|
||||
"""Test for if __name__ == '__main__' guard"""
|
||||
self.log_verbose("Testing main guard...")
|
||||
|
||||
has_main_guard = 'if __name__ == "__main__"' in content or "if __name__ == '__main__'" in content
|
||||
|
||||
if has_main_guard:
|
||||
result.add_test("main_guard", True, "Has proper main guard")
|
||||
else:
|
||||
result.add_test("main_guard", False, "Missing main guard")
|
||||
result.add_error("Script should have 'if __name__ == \"__main__\"' guard")
|
||||
|
||||
def _test_script_execution(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test basic script execution"""
|
||||
self.log_verbose("Testing script execution...")
|
||||
|
||||
try:
|
||||
# Try to run the script with no arguments (should not crash immediately)
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
|
||||
# Script might exit with error code if no args provided, but shouldn't crash
|
||||
if process.returncode in (0, 1, 2): # 0=success, 1=general error, 2=misuse
|
||||
result.add_test("basic_execution", True,
|
||||
f"Script runs without crashing (exit code: {process.returncode})")
|
||||
else:
|
||||
result.add_test("basic_execution", False,
|
||||
f"Script crashed with exit code {process.returncode}",
|
||||
{"stdout": process.stdout, "stderr": process.stderr})
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
result.add_test("basic_execution", False,
|
||||
f"Script execution timed out after {self.timeout} seconds")
|
||||
result.add_error(f"Script execution timeout ({self.timeout}s)")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("basic_execution", False, f"Execution error: {str(e)}")
|
||||
result.add_error(f"Script execution failed: {str(e)}")
|
||||
|
||||
def _test_help_functionality(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test --help functionality"""
|
||||
self.log_verbose("Testing help functionality...")
|
||||
|
||||
try:
|
||||
# Test --help flag
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path), '--help'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
|
||||
if process.returncode == 0:
|
||||
help_output = process.stdout
|
||||
|
||||
# Check for reasonable help content
|
||||
help_indicators = ['usage:', 'positional arguments:', 'optional arguments:',
|
||||
'options:', 'description:', 'help']
|
||||
has_help_content = any(indicator in help_output.lower() for indicator in help_indicators)
|
||||
|
||||
if has_help_content and len(help_output.strip()) > 50:
|
||||
result.add_test("help_functionality", True, "Provides comprehensive help text")
|
||||
else:
|
||||
result.add_test("help_functionality", False,
|
||||
"Help text is too brief or missing key sections",
|
||||
{"help_output": help_output})
|
||||
result.add_warning("Help text could be more comprehensive")
|
||||
|
||||
else:
|
||||
result.add_test("help_functionality", False,
|
||||
f"Help command failed with exit code {process.returncode}",
|
||||
{"stderr": process.stderr})
|
||||
result.add_error("--help flag does not work properly")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
result.add_test("help_functionality", False, "Help command timed out")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("help_functionality", False, f"Help test error: {str(e)}")
|
||||
|
||||
def _test_sample_data_processing(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test script against sample data if available"""
|
||||
self.log_verbose("Testing sample data processing...")
|
||||
|
||||
assets_dir = self.skill_path / "assets"
|
||||
if not assets_dir.exists():
|
||||
result.add_test("sample_data_processing", True, "No sample data to test (assets dir missing)")
|
||||
return
|
||||
|
||||
# Look for sample input files
|
||||
sample_files = list(assets_dir.rglob("*sample*")) + list(assets_dir.rglob("*test*"))
|
||||
sample_files = [f for f in sample_files if f.is_file() and not f.name.startswith('.')]
|
||||
|
||||
if not sample_files:
|
||||
result.add_test("sample_data_processing", True, "No sample data files found to test")
|
||||
return
|
||||
|
||||
tested_files = 0
|
||||
successful_tests = 0
|
||||
|
||||
for sample_file in sample_files[:3]: # Test up to 3 sample files
|
||||
try:
|
||||
self.log_verbose(f"Testing with sample file: {sample_file.name}")
|
||||
|
||||
# Try to run script with the sample file as input
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path), str(sample_file)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
|
||||
tested_files += 1
|
||||
|
||||
if process.returncode == 0:
|
||||
successful_tests += 1
|
||||
else:
|
||||
self.log_verbose(f"Sample test failed for {sample_file.name}: {process.stderr}")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
tested_files += 1
|
||||
result.add_warning(f"Sample data test timed out for {sample_file.name}")
|
||||
except Exception as e:
|
||||
tested_files += 1
|
||||
self.log_verbose(f"Sample test error for {sample_file.name}: {str(e)}")
|
||||
|
||||
if tested_files == 0:
|
||||
result.add_test("sample_data_processing", True, "No testable sample data found")
|
||||
elif successful_tests == tested_files:
|
||||
result.add_test("sample_data_processing", True,
|
||||
f"Successfully processed all {tested_files} sample files")
|
||||
elif successful_tests > 0:
|
||||
result.add_test("sample_data_processing", False,
|
||||
f"Processed {successful_tests}/{tested_files} sample files",
|
||||
{"success_rate": successful_tests / tested_files})
|
||||
result.add_warning("Some sample data processing failed")
|
||||
else:
|
||||
result.add_test("sample_data_processing", False,
|
||||
"Failed to process any sample data files")
|
||||
result.add_error("Script cannot process sample data")
|
||||
|
||||
def _test_output_formats(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test output format compliance"""
|
||||
self.log_verbose("Testing output formats...")
|
||||
|
||||
# Test if script supports JSON output
|
||||
json_support = False
|
||||
human_readable_support = False
|
||||
|
||||
try:
|
||||
# Read script content to check for output format indicators
|
||||
content = script_path.read_text(encoding='utf-8')
|
||||
|
||||
# Look for JSON-related code
|
||||
if any(indicator in content.lower() for indicator in ['json.dump', 'json.load', '"json"', '--json']):
|
||||
json_support = True
|
||||
|
||||
# Look for human-readable output indicators
|
||||
if any(indicator in content for indicator in ['print(', 'format(', 'f"', "f'"]):
|
||||
human_readable_support = True
|
||||
|
||||
# Try running with --json flag if it looks like it supports it
|
||||
if '--json' in content:
|
||||
try:
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path), '--json', '--help'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
if process.returncode == 0:
|
||||
json_support = True
|
||||
except:
|
||||
pass
|
||||
|
||||
# Evaluate dual output support
|
||||
if json_support and human_readable_support:
|
||||
result.add_test("output_formats", True, "Supports both JSON and human-readable output")
|
||||
elif json_support or human_readable_support:
|
||||
format_type = "JSON" if json_support else "human-readable"
|
||||
result.add_test("output_formats", False,
|
||||
f"Supports only {format_type} output",
|
||||
{"json_support": json_support, "human_readable_support": human_readable_support})
|
||||
result.add_warning("Consider adding dual output format support")
|
||||
else:
|
||||
result.add_test("output_formats", False, "No clear output format support detected")
|
||||
result.add_warning("Output format support is unclear")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("output_formats", False, f"Error testing output formats: {str(e)}")
|
||||
|
||||
|
||||
class TestReportFormatter:
|
||||
"""Formats test reports for output"""
|
||||
|
||||
@staticmethod
|
||||
def format_json(test_suite: TestSuite) -> str:
|
||||
"""Format test suite as JSON"""
|
||||
return json.dumps({
|
||||
"skill_path": test_suite.skill_path,
|
||||
"timestamp": test_suite.timestamp,
|
||||
"summary": test_suite.summary,
|
||||
"global_errors": test_suite.global_errors,
|
||||
"script_results": {
|
||||
name: {
|
||||
"script_path": result.script_path,
|
||||
"timestamp": result.timestamp,
|
||||
"overall_status": result.overall_status,
|
||||
"execution_time": round(result.execution_time, 2),
|
||||
"tests": result.tests,
|
||||
"errors": result.errors,
|
||||
"warnings": result.warnings
|
||||
}
|
||||
for name, result in test_suite.script_results.items()
|
||||
}
|
||||
}, indent=2)
|
||||
|
||||
@staticmethod
|
||||
def format_human_readable(test_suite: TestSuite) -> str:
|
||||
"""Format test suite as human-readable text"""
|
||||
lines = []
|
||||
lines.append("=" * 60)
|
||||
lines.append("SCRIPT TESTING REPORT")
|
||||
lines.append("=" * 60)
|
||||
lines.append(f"Skill: {test_suite.skill_path}")
|
||||
lines.append(f"Timestamp: {test_suite.timestamp}")
|
||||
lines.append("")
|
||||
|
||||
# Summary
|
||||
if test_suite.summary:
|
||||
lines.append("SUMMARY:")
|
||||
lines.append(f" Total Scripts: {test_suite.summary['total_scripts']}")
|
||||
lines.append(f" Passed: {test_suite.summary['passed']}")
|
||||
lines.append(f" Partial: {test_suite.summary['partial']}")
|
||||
lines.append(f" Failed: {test_suite.summary['failed']}")
|
||||
lines.append(f" Overall Status: {test_suite.summary['overall_status']}")
|
||||
lines.append("")
|
||||
|
||||
# Global errors
|
||||
if test_suite.global_errors:
|
||||
lines.append("GLOBAL ERRORS:")
|
||||
for error in test_suite.global_errors:
|
||||
lines.append(f" • {error}")
|
||||
lines.append("")
|
||||
|
||||
# Individual script results
|
||||
for script_name, result in test_suite.script_results.items():
|
||||
lines.append(f"SCRIPT: {script_name}")
|
||||
lines.append(f" Status: {result.overall_status}")
|
||||
lines.append(f" Execution Time: {result.execution_time:.2f}s")
|
||||
lines.append("")
|
||||
|
||||
# Tests
|
||||
if result.tests:
|
||||
lines.append(" TESTS:")
|
||||
for test_name, test_result in result.tests.items():
|
||||
status = "✓ PASS" if test_result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {test_result['message']}")
|
||||
lines.append("")
|
||||
|
||||
# Errors
|
||||
if result.errors:
|
||||
lines.append(" ERRORS:")
|
||||
for error in result.errors:
|
||||
lines.append(f" • {error}")
|
||||
lines.append("")
|
||||
|
||||
# Warnings
|
||||
if result.warnings:
|
||||
lines.append(" WARNINGS:")
|
||||
for warning in result.warnings:
|
||||
lines.append(f" • {warning}")
|
||||
lines.append("")
|
||||
|
||||
lines.append("-" * 40)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Test Python scripts in a skill directory",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python script_tester.py engineering/my-skill
|
||||
python script_tester.py engineering/my-skill --timeout 60 --json
|
||||
python script_tester.py engineering/my-skill --verbose
|
||||
|
||||
Test Categories:
|
||||
- Syntax validation (AST parsing)
|
||||
- Import validation (stdlib only)
|
||||
- Argparse implementation
|
||||
- Main guard presence
|
||||
- Basic execution testing
|
||||
- Help functionality
|
||||
- Sample data processing
|
||||
- Output format compliance
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument("skill_path",
|
||||
help="Path to the skill directory containing scripts to test")
|
||||
parser.add_argument("--timeout",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Timeout for script execution tests in seconds (default: 30)")
|
||||
parser.add_argument("--json",
|
||||
action="store_true",
|
||||
help="Output results in JSON format")
|
||||
parser.add_argument("--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose logging")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Create tester and run tests
|
||||
tester = ScriptTester(args.skill_path, args.timeout, args.verbose)
|
||||
test_suite = tester.test_all_scripts()
|
||||
|
||||
# Format and output results
|
||||
if args.json:
|
||||
print(TestReportFormatter.format_json(test_suite))
|
||||
else:
|
||||
print(TestReportFormatter.format_human_readable(test_suite))
|
||||
|
||||
# Exit with appropriate code
|
||||
if test_suite.global_errors:
|
||||
sys.exit(1)
|
||||
elif test_suite.summary.get("overall_status") == "FAIL":
|
||||
sys.exit(1)
|
||||
elif test_suite.summary.get("overall_status") == "PARTIAL":
|
||||
sys.exit(2) # Partial success
|
||||
else:
|
||||
sys.exit(0) # Success
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nTesting interrupted by user", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
print(f"Testing failed: {str(e)}", file=sys.stderr)
|
||||
if args.verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,652 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Skill Validator - Validates skill directories against quality standards
|
||||
|
||||
This script validates a skill directory structure, documentation, and Python scripts
|
||||
against the claude-skills ecosystem standards. It checks for required files, proper
|
||||
formatting, and compliance with tier-specific requirements.
|
||||
|
||||
Usage:
|
||||
python skill_validator.py <skill_path> [--tier TIER] [--json] [--verbose]
|
||||
|
||||
Author: Claude Skills Engineering Team
|
||||
Version: 1.0.0
|
||||
Dependencies: Python Standard Library Only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import yaml
|
||||
import datetime as dt
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Custom exception for validation errors"""
|
||||
pass
|
||||
|
||||
|
||||
class ValidationReport:
|
||||
"""Container for validation results"""
|
||||
|
||||
def __init__(self, skill_path: str):
|
||||
self.skill_path = skill_path
|
||||
self.timestamp = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
self.checks = {}
|
||||
self.warnings = []
|
||||
self.errors = []
|
||||
self.suggestions = []
|
||||
self.overall_score = 0.0
|
||||
self.compliance_level = "FAIL"
|
||||
|
||||
def add_check(self, check_name: str, passed: bool, message: str = "", score: float = 0.0):
|
||||
"""Add a validation check result"""
|
||||
self.checks[check_name] = {
|
||||
"passed": passed,
|
||||
"message": message,
|
||||
"score": score
|
||||
}
|
||||
|
||||
def add_warning(self, message: str):
|
||||
"""Add a warning message"""
|
||||
self.warnings.append(message)
|
||||
|
||||
def add_error(self, message: str):
|
||||
"""Add an error message"""
|
||||
self.errors.append(message)
|
||||
|
||||
def add_suggestion(self, message: str):
|
||||
"""Add an improvement suggestion"""
|
||||
self.suggestions.append(message)
|
||||
|
||||
def calculate_overall_score(self):
|
||||
"""Calculate overall compliance score"""
|
||||
if not self.checks:
|
||||
self.overall_score = 0.0
|
||||
return
|
||||
|
||||
total_score = sum(check["score"] for check in self.checks.values())
|
||||
max_score = len(self.checks) * 1.0
|
||||
self.overall_score = (total_score / max_score) * 100 if max_score > 0 else 0.0
|
||||
|
||||
# Determine compliance level
|
||||
if self.overall_score >= 90:
|
||||
self.compliance_level = "EXCELLENT"
|
||||
elif self.overall_score >= 75:
|
||||
self.compliance_level = "GOOD"
|
||||
elif self.overall_score >= 60:
|
||||
self.compliance_level = "ACCEPTABLE"
|
||||
elif self.overall_score >= 40:
|
||||
self.compliance_level = "NEEDS_IMPROVEMENT"
|
||||
else:
|
||||
self.compliance_level = "POOR"
|
||||
|
||||
|
||||
class SkillValidator:
|
||||
"""Main skill validation engine"""
|
||||
|
||||
# Tier requirements
|
||||
TIER_REQUIREMENTS = {
|
||||
"BASIC": {
|
||||
"min_skill_md_lines": 100,
|
||||
"min_scripts": 1,
|
||||
"script_size_range": (100, 300),
|
||||
"required_dirs": ["scripts"],
|
||||
"optional_dirs": ["assets", "references", "expected_outputs"],
|
||||
"features_required": ["argparse", "main_guard"]
|
||||
},
|
||||
"STANDARD": {
|
||||
"min_skill_md_lines": 200,
|
||||
"min_scripts": 1,
|
||||
"script_size_range": (300, 500),
|
||||
"required_dirs": ["scripts", "assets", "references"],
|
||||
"optional_dirs": ["expected_outputs"],
|
||||
"features_required": ["argparse", "main_guard", "json_output", "help_text"]
|
||||
},
|
||||
"POWERFUL": {
|
||||
"min_skill_md_lines": 300,
|
||||
"min_scripts": 2,
|
||||
"script_size_range": (500, 800),
|
||||
"required_dirs": ["scripts", "assets", "references", "expected_outputs"],
|
||||
"optional_dirs": [],
|
||||
"features_required": ["argparse", "main_guard", "json_output", "help_text", "error_handling"]
|
||||
}
|
||||
}
|
||||
|
||||
REQUIRED_SKILL_MD_SECTIONS = [
|
||||
"Name", "Description", "Features", "Usage", "Examples"
|
||||
]
|
||||
|
||||
FRONTMATTER_REQUIRED_FIELDS = [
|
||||
"Name", "Tier", "Category", "Dependencies", "Author", "Version"
|
||||
]
|
||||
|
||||
def __init__(self, skill_path: str, target_tier: Optional[str] = None, verbose: bool = False):
|
||||
self.skill_path = Path(skill_path).resolve()
|
||||
self.target_tier = target_tier
|
||||
self.verbose = verbose
|
||||
self.report = ValidationReport(str(self.skill_path))
|
||||
|
||||
def log_verbose(self, message: str):
|
||||
"""Log verbose message if verbose mode enabled"""
|
||||
if self.verbose:
|
||||
print(f"[VERBOSE] {message}", file=sys.stderr)
|
||||
|
||||
def validate_skill_structure(self) -> ValidationReport:
|
||||
"""Main validation entry point"""
|
||||
try:
|
||||
self.log_verbose(f"Starting validation of {self.skill_path}")
|
||||
|
||||
# Check if path exists
|
||||
if not self.skill_path.exists():
|
||||
self.report.add_error(f"Skill path does not exist: {self.skill_path}")
|
||||
return self.report
|
||||
|
||||
if not self.skill_path.is_dir():
|
||||
self.report.add_error(f"Skill path is not a directory: {self.skill_path}")
|
||||
return self.report
|
||||
|
||||
# Run all validation checks
|
||||
self._validate_required_files()
|
||||
self._validate_skill_md()
|
||||
self._validate_readme()
|
||||
self._validate_directory_structure()
|
||||
self._validate_python_scripts()
|
||||
self._validate_tier_compliance()
|
||||
|
||||
# Calculate overall score
|
||||
self.report.calculate_overall_score()
|
||||
|
||||
self.log_verbose(f"Validation completed. Score: {self.report.overall_score:.1f}")
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_error(f"Validation failed with exception: {str(e)}")
|
||||
|
||||
return self.report
|
||||
|
||||
def _validate_required_files(self):
|
||||
"""Validate presence of required files"""
|
||||
self.log_verbose("Checking required files...")
|
||||
|
||||
# Check SKILL.md
|
||||
skill_md_path = self.skill_path / "SKILL.md"
|
||||
if skill_md_path.exists():
|
||||
self.report.add_check("skill_md_exists", True, "SKILL.md found", 1.0)
|
||||
else:
|
||||
self.report.add_check("skill_md_exists", False, "SKILL.md missing", 0.0)
|
||||
self.report.add_error("SKILL.md is required but missing")
|
||||
|
||||
# Check README.md
|
||||
readme_path = self.skill_path / "README.md"
|
||||
if readme_path.exists():
|
||||
self.report.add_check("readme_exists", True, "README.md found", 1.0)
|
||||
else:
|
||||
self.report.add_check("readme_exists", False, "README.md missing", 0.0)
|
||||
self.report.add_warning("README.md is recommended but missing")
|
||||
self.report.add_suggestion("Add README.md with usage instructions and examples")
|
||||
|
||||
def _validate_skill_md(self):
|
||||
"""Validate SKILL.md content and format"""
|
||||
self.log_verbose("Validating SKILL.md...")
|
||||
|
||||
skill_md_path = self.skill_path / "SKILL.md"
|
||||
if not skill_md_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
content = skill_md_path.read_text(encoding='utf-8')
|
||||
lines = content.split('\n')
|
||||
line_count = len([line for line in lines if line.strip()])
|
||||
|
||||
# Check line count
|
||||
min_lines = self._get_tier_requirement("min_skill_md_lines", 100)
|
||||
if line_count >= min_lines:
|
||||
self.report.add_check("skill_md_length", True,
|
||||
f"SKILL.md has {line_count} lines (≥{min_lines})", 1.0)
|
||||
else:
|
||||
self.report.add_check("skill_md_length", False,
|
||||
f"SKILL.md has {line_count} lines (<{min_lines})", 0.0)
|
||||
self.report.add_error(f"SKILL.md too short: {line_count} lines, minimum {min_lines}")
|
||||
|
||||
# Validate frontmatter
|
||||
self._validate_frontmatter(content)
|
||||
|
||||
# Check required sections
|
||||
self._validate_required_sections(content)
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_check("skill_md_readable", False, f"Error reading SKILL.md: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Cannot read SKILL.md: {str(e)}")
|
||||
|
||||
def _validate_frontmatter(self, content: str):
|
||||
"""Validate SKILL.md frontmatter"""
|
||||
self.log_verbose("Validating frontmatter...")
|
||||
|
||||
# Extract frontmatter
|
||||
if content.startswith('---'):
|
||||
try:
|
||||
end_marker = content.find('---', 3)
|
||||
if end_marker == -1:
|
||||
self.report.add_check("frontmatter_format", False,
|
||||
"Frontmatter closing marker not found", 0.0)
|
||||
return
|
||||
|
||||
frontmatter_text = content[3:end_marker].strip()
|
||||
frontmatter = yaml.safe_load(frontmatter_text)
|
||||
|
||||
if not isinstance(frontmatter, dict):
|
||||
self.report.add_check("frontmatter_format", False,
|
||||
"Frontmatter is not a valid dictionary", 0.0)
|
||||
return
|
||||
|
||||
# Check required fields
|
||||
missing_fields = []
|
||||
for field in self.FRONTMATTER_REQUIRED_FIELDS:
|
||||
if field not in frontmatter:
|
||||
missing_fields.append(field)
|
||||
|
||||
if not missing_fields:
|
||||
self.report.add_check("frontmatter_complete", True,
|
||||
"All required frontmatter fields present", 1.0)
|
||||
else:
|
||||
self.report.add_check("frontmatter_complete", False,
|
||||
f"Missing fields: {', '.join(missing_fields)}", 0.0)
|
||||
self.report.add_error(f"Missing frontmatter fields: {', '.join(missing_fields)}")
|
||||
|
||||
except yaml.YAMLError as e:
|
||||
self.report.add_check("frontmatter_format", False,
|
||||
f"Invalid YAML frontmatter: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Invalid YAML frontmatter: {str(e)}")
|
||||
|
||||
else:
|
||||
self.report.add_check("frontmatter_exists", False,
|
||||
"No frontmatter found", 0.0)
|
||||
self.report.add_error("SKILL.md must start with YAML frontmatter")
|
||||
|
||||
def _validate_required_sections(self, content: str):
|
||||
"""Validate required sections in SKILL.md"""
|
||||
self.log_verbose("Checking required sections...")
|
||||
|
||||
missing_sections = []
|
||||
for section in self.REQUIRED_SKILL_MD_SECTIONS:
|
||||
pattern = rf'^#+\s*{re.escape(section)}\s*$'
|
||||
if not re.search(pattern, content, re.MULTILINE | re.IGNORECASE):
|
||||
missing_sections.append(section)
|
||||
|
||||
if not missing_sections:
|
||||
self.report.add_check("required_sections", True,
|
||||
"All required sections present", 1.0)
|
||||
else:
|
||||
self.report.add_check("required_sections", False,
|
||||
f"Missing sections: {', '.join(missing_sections)}", 0.0)
|
||||
self.report.add_error(f"Missing required sections: {', '.join(missing_sections)}")
|
||||
|
||||
def _validate_readme(self):
|
||||
"""Validate README.md content"""
|
||||
self.log_verbose("Validating README.md...")
|
||||
|
||||
readme_path = self.skill_path / "README.md"
|
||||
if not readme_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
content = readme_path.read_text(encoding='utf-8')
|
||||
|
||||
# Check minimum content length
|
||||
if len(content.strip()) >= 200:
|
||||
self.report.add_check("readme_substantial", True,
|
||||
"README.md has substantial content", 1.0)
|
||||
else:
|
||||
self.report.add_check("readme_substantial", False,
|
||||
"README.md content is too brief", 0.5)
|
||||
self.report.add_suggestion("Expand README.md with more detailed usage instructions")
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_check("readme_readable", False,
|
||||
f"Error reading README.md: {str(e)}", 0.0)
|
||||
|
||||
def _validate_directory_structure(self):
|
||||
"""Validate directory structure against tier requirements"""
|
||||
self.log_verbose("Validating directory structure...")
|
||||
|
||||
required_dirs = self._get_tier_requirement("required_dirs", ["scripts"])
|
||||
optional_dirs = self._get_tier_requirement("optional_dirs", [])
|
||||
|
||||
# Check required directories
|
||||
missing_required = []
|
||||
for dir_name in required_dirs:
|
||||
dir_path = self.skill_path / dir_name
|
||||
if dir_path.exists() and dir_path.is_dir():
|
||||
self.report.add_check(f"dir_{dir_name}_exists", True,
|
||||
f"{dir_name}/ directory found", 1.0)
|
||||
else:
|
||||
missing_required.append(dir_name)
|
||||
self.report.add_check(f"dir_{dir_name}_exists", False,
|
||||
f"{dir_name}/ directory missing", 0.0)
|
||||
|
||||
if missing_required:
|
||||
self.report.add_error(f"Missing required directories: {', '.join(missing_required)}")
|
||||
|
||||
# Check optional directories and provide suggestions
|
||||
missing_optional = []
|
||||
for dir_name in optional_dirs:
|
||||
dir_path = self.skill_path / dir_name
|
||||
if not (dir_path.exists() and dir_path.is_dir()):
|
||||
missing_optional.append(dir_name)
|
||||
|
||||
if missing_optional:
|
||||
self.report.add_suggestion(f"Consider adding optional directories: {', '.join(missing_optional)}")
|
||||
|
||||
def _validate_python_scripts(self):
|
||||
"""Validate Python scripts in the scripts directory"""
|
||||
self.log_verbose("Validating Python scripts...")
|
||||
|
||||
scripts_dir = self.skill_path / "scripts"
|
||||
if not scripts_dir.exists():
|
||||
return
|
||||
|
||||
python_files = list(scripts_dir.glob("*.py"))
|
||||
min_scripts = self._get_tier_requirement("min_scripts", 1)
|
||||
|
||||
# Check minimum number of scripts
|
||||
if len(python_files) >= min_scripts:
|
||||
self.report.add_check("min_scripts_count", True,
|
||||
f"Found {len(python_files)} Python scripts (≥{min_scripts})", 1.0)
|
||||
else:
|
||||
self.report.add_check("min_scripts_count", False,
|
||||
f"Found {len(python_files)} Python scripts (<{min_scripts})", 0.0)
|
||||
self.report.add_error(f"Insufficient scripts: {len(python_files)}, minimum {min_scripts}")
|
||||
|
||||
# Validate each script
|
||||
for script_path in python_files:
|
||||
self._validate_single_script(script_path)
|
||||
|
||||
def _validate_single_script(self, script_path: Path):
|
||||
"""Validate a single Python script"""
|
||||
script_name = script_path.name
|
||||
self.log_verbose(f"Validating script: {script_name}")
|
||||
|
||||
try:
|
||||
content = script_path.read_text(encoding='utf-8')
|
||||
|
||||
# Count lines of code (excluding empty lines and comments)
|
||||
lines = content.split('\n')
|
||||
loc = len([line for line in lines if line.strip() and not line.strip().startswith('#')])
|
||||
|
||||
# Check script size against tier requirements
|
||||
size_range = self._get_tier_requirement("script_size_range", (100, 1000))
|
||||
min_size, max_size = size_range
|
||||
|
||||
if min_size <= loc <= max_size:
|
||||
self.report.add_check(f"script_size_{script_name}", True,
|
||||
f"{script_name} has {loc} LOC (within {min_size}-{max_size})", 1.0)
|
||||
else:
|
||||
self.report.add_check(f"script_size_{script_name}", False,
|
||||
f"{script_name} has {loc} LOC (outside {min_size}-{max_size})", 0.5)
|
||||
if loc < min_size:
|
||||
self.report.add_suggestion(f"Consider expanding {script_name} (currently {loc} LOC)")
|
||||
else:
|
||||
self.report.add_suggestion(f"Consider refactoring {script_name} (currently {loc} LOC)")
|
||||
|
||||
# Parse and validate Python syntax
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
self.report.add_check(f"script_syntax_{script_name}", True,
|
||||
f"{script_name} has valid Python syntax", 1.0)
|
||||
|
||||
# Check for required features
|
||||
self._validate_script_features(tree, script_name, content)
|
||||
|
||||
except SyntaxError as e:
|
||||
self.report.add_check(f"script_syntax_{script_name}", False,
|
||||
f"{script_name} has syntax error: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Syntax error in {script_name}: {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_check(f"script_readable_{script_name}", False,
|
||||
f"Cannot read {script_name}: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Cannot read {script_name}: {str(e)}")
|
||||
|
||||
def _validate_script_features(self, tree: ast.AST, script_name: str, content: str):
|
||||
"""Validate required script features"""
|
||||
required_features = self._get_tier_requirement("features_required", ["argparse", "main_guard"])
|
||||
|
||||
# Check for argparse usage
|
||||
if "argparse" in required_features:
|
||||
has_argparse = self._check_argparse_usage(tree)
|
||||
self.report.add_check(f"script_argparse_{script_name}", has_argparse,
|
||||
f"{'Uses' if has_argparse else 'Missing'} argparse in {script_name}", 1.0 if has_argparse else 0.0)
|
||||
if not has_argparse:
|
||||
self.report.add_error(f"{script_name} must use argparse for command-line arguments")
|
||||
|
||||
# Check for main guard
|
||||
if "main_guard" in required_features:
|
||||
has_main_guard = 'if __name__ == "__main__"' in content
|
||||
self.report.add_check(f"script_main_guard_{script_name}", has_main_guard,
|
||||
f"{'Has' if has_main_guard else 'Missing'} main guard in {script_name}", 1.0 if has_main_guard else 0.0)
|
||||
if not has_main_guard:
|
||||
self.report.add_error(f"{script_name} must have 'if __name__ == \"__main__\"' guard")
|
||||
|
||||
# Check for external imports (should only use stdlib)
|
||||
external_imports = self._check_external_imports(tree)
|
||||
if not external_imports:
|
||||
self.report.add_check(f"script_imports_{script_name}", True,
|
||||
f"{script_name} uses only standard library", 1.0)
|
||||
else:
|
||||
self.report.add_check(f"script_imports_{script_name}", False,
|
||||
f"{script_name} uses external imports: {', '.join(external_imports)}", 0.0)
|
||||
self.report.add_error(f"{script_name} uses external imports: {', '.join(external_imports)}")
|
||||
|
||||
def _check_argparse_usage(self, tree: ast.AST) -> bool:
|
||||
"""Check if the script uses argparse"""
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name == 'argparse':
|
||||
return True
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module == 'argparse':
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_external_imports(self, tree: ast.AST) -> List[str]:
|
||||
"""Check for external (non-stdlib) imports"""
|
||||
# Simplified check - a more comprehensive solution would use a stdlib module list
|
||||
stdlib_modules = {
|
||||
'argparse', 'ast', 'json', 'os', 'sys', 'pathlib', 'datetime', 'typing',
|
||||
'collections', 're', 'math', 'random', 'itertools', 'functools', 'operator',
|
||||
'csv', 'sqlite3', 'urllib', 'http', 'html', 'xml', 'email', 'base64',
|
||||
'hashlib', 'hmac', 'secrets', 'tempfile', 'shutil', 'glob', 'fnmatch',
|
||||
'subprocess', 'threading', 'multiprocessing', 'queue', 'time', 'calendar',
|
||||
'zoneinfo', 'locale', 'gettext', 'logging', 'warnings', 'unittest',
|
||||
'doctest', 'pickle', 'copy', 'pprint', 'reprlib', 'enum', 'dataclasses',
|
||||
'contextlib', 'abc', 'atexit', 'traceback', 'gc', 'weakref', 'types',
|
||||
'copy', 'pprint', 'reprlib', 'enum', 'decimal', 'fractions', 'statistics',
|
||||
'cmath', 'platform', 'errno', 'io', 'codecs', 'unicodedata', 'stringprep',
|
||||
'textwrap', 'string', 'struct', 'difflib', 'heapq', 'bisect', 'array',
|
||||
'weakref', 'types', 'copyreg', 'uuid', 'mmap', 'ctypes'
|
||||
}
|
||||
|
||||
external_imports = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
module_name = alias.name.split('.')[0]
|
||||
if module_name not in stdlib_modules:
|
||||
external_imports.append(alias.name)
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
module_name = node.module.split('.')[0]
|
||||
if module_name not in stdlib_modules:
|
||||
external_imports.append(node.module)
|
||||
|
||||
return list(set(external_imports))
|
||||
|
||||
def _validate_tier_compliance(self):
|
||||
"""Validate overall tier compliance"""
|
||||
if not self.target_tier:
|
||||
return
|
||||
|
||||
self.log_verbose(f"Validating {self.target_tier} tier compliance...")
|
||||
|
||||
# This is a summary check - individual checks are done in other methods
|
||||
critical_checks = ["skill_md_exists", "min_scripts_count", "skill_md_length"]
|
||||
failed_critical = [check for check in critical_checks
|
||||
if check in self.report.checks and not self.report.checks[check]["passed"]]
|
||||
|
||||
if not failed_critical:
|
||||
self.report.add_check("tier_compliance", True,
|
||||
f"Meets {self.target_tier} tier requirements", 1.0)
|
||||
else:
|
||||
self.report.add_check("tier_compliance", False,
|
||||
f"Does not meet {self.target_tier} tier requirements", 0.0)
|
||||
self.report.add_error(f"Failed critical checks for {self.target_tier} tier: {', '.join(failed_critical)}")
|
||||
|
||||
def _get_tier_requirement(self, requirement: str, default: Any) -> Any:
|
||||
"""Get tier-specific requirement value"""
|
||||
if self.target_tier and self.target_tier in self.TIER_REQUIREMENTS:
|
||||
return self.TIER_REQUIREMENTS[self.target_tier].get(requirement, default)
|
||||
return default
|
||||
|
||||
|
||||
class ReportFormatter:
|
||||
"""Formats validation reports for output"""
|
||||
|
||||
@staticmethod
|
||||
def format_json(report: ValidationReport) -> str:
|
||||
"""Format report as JSON"""
|
||||
return json.dumps({
|
||||
"skill_path": report.skill_path,
|
||||
"timestamp": report.timestamp,
|
||||
"overall_score": round(report.overall_score, 1),
|
||||
"compliance_level": report.compliance_level,
|
||||
"checks": report.checks,
|
||||
"warnings": report.warnings,
|
||||
"errors": report.errors,
|
||||
"suggestions": report.suggestions
|
||||
}, indent=2)
|
||||
|
||||
@staticmethod
|
||||
def format_human_readable(report: ValidationReport) -> str:
|
||||
"""Format report as human-readable text"""
|
||||
lines = []
|
||||
lines.append("=" * 60)
|
||||
lines.append("SKILL VALIDATION REPORT")
|
||||
lines.append("=" * 60)
|
||||
lines.append(f"Skill: {report.skill_path}")
|
||||
lines.append(f"Timestamp: {report.timestamp}")
|
||||
lines.append(f"Overall Score: {report.overall_score:.1f}/100 ({report.compliance_level})")
|
||||
lines.append("")
|
||||
|
||||
# Group checks by category
|
||||
structure_checks = {k: v for k, v in report.checks.items() if k.startswith(('skill_md', 'readme', 'dir_'))}
|
||||
script_checks = {k: v for k, v in report.checks.items() if k.startswith('script_')}
|
||||
other_checks = {k: v for k, v in report.checks.items() if k not in structure_checks and k not in script_checks}
|
||||
|
||||
if structure_checks:
|
||||
lines.append("STRUCTURE VALIDATION:")
|
||||
for check_name, result in structure_checks.items():
|
||||
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {result['message']}")
|
||||
lines.append("")
|
||||
|
||||
if script_checks:
|
||||
lines.append("SCRIPT VALIDATION:")
|
||||
for check_name, result in script_checks.items():
|
||||
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {result['message']}")
|
||||
lines.append("")
|
||||
|
||||
if other_checks:
|
||||
lines.append("OTHER CHECKS:")
|
||||
for check_name, result in other_checks.items():
|
||||
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {result['message']}")
|
||||
lines.append("")
|
||||
|
||||
if report.errors:
|
||||
lines.append("ERRORS:")
|
||||
for error in report.errors:
|
||||
lines.append(f" • {error}")
|
||||
lines.append("")
|
||||
|
||||
if report.warnings:
|
||||
lines.append("WARNINGS:")
|
||||
for warning in report.warnings:
|
||||
lines.append(f" • {warning}")
|
||||
lines.append("")
|
||||
|
||||
if report.suggestions:
|
||||
lines.append("SUGGESTIONS:")
|
||||
for suggestion in report.suggestions:
|
||||
lines.append(f" • {suggestion}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate skill directories against quality standards",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python skill_validator.py engineering/my-skill
|
||||
python skill_validator.py engineering/my-skill --tier POWERFUL --json
|
||||
python skill_validator.py engineering/my-skill --verbose
|
||||
|
||||
Tier Options:
|
||||
BASIC - Basic skill requirements (100+ lines SKILL.md, 1+ script)
|
||||
STANDARD - Standard skill requirements (200+ lines, advanced features)
|
||||
POWERFUL - Powerful skill requirements (300+ lines, comprehensive features)
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument("skill_path",
|
||||
help="Path to the skill directory to validate")
|
||||
parser.add_argument("--tier",
|
||||
choices=["BASIC", "STANDARD", "POWERFUL"],
|
||||
help="Target tier for validation (optional)")
|
||||
parser.add_argument("--json",
|
||||
action="store_true",
|
||||
help="Output results in JSON format")
|
||||
parser.add_argument("--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose logging")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Create validator and run validation
|
||||
validator = SkillValidator(args.skill_path, args.tier, args.verbose)
|
||||
report = validator.validate_skill_structure()
|
||||
|
||||
# Format and output report
|
||||
if args.json:
|
||||
print(ReportFormatter.format_json(report))
|
||||
else:
|
||||
print(ReportFormatter.format_human_readable(report))
|
||||
|
||||
# Exit with error code if validation failed
|
||||
if report.errors or report.overall_score < 60:
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nValidation interrupted by user", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
print(f"Validation failed: {str(e)}", file=sys.stderr)
|
||||
if args.verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user