add brain
This commit is contained in:
@@ -0,0 +1,488 @@
|
||||
{
|
||||
"architecture_design": {
|
||||
"pattern": "supervisor",
|
||||
"agents": [
|
||||
{
|
||||
"name": "supervisor_agent",
|
||||
"role": "Task Coordinator and Quality Controller",
|
||||
"archetype": "coordinator",
|
||||
"responsibilities": [
|
||||
"task_decomposition",
|
||||
"delegation",
|
||||
"progress_monitoring",
|
||||
"quality_assurance",
|
||||
"result_aggregation"
|
||||
],
|
||||
"capabilities": [
|
||||
"planning",
|
||||
"coordination",
|
||||
"evaluation",
|
||||
"decision_making"
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "file_manager",
|
||||
"description": "Manage files and directories",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string"
|
||||
},
|
||||
"path": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"file_operations",
|
||||
"data_management"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "low"
|
||||
},
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze and process data",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"insights": {
|
||||
"type": "array"
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"data_analysis",
|
||||
"statistics",
|
||||
"visualization"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
}
|
||||
],
|
||||
"communication_interfaces": [
|
||||
"user_interface",
|
||||
"agent_messaging"
|
||||
],
|
||||
"constraints": {
|
||||
"max_concurrent_supervisions": 5,
|
||||
"decision_timeout": "30s"
|
||||
},
|
||||
"success_criteria": [
|
||||
"successful task completion",
|
||||
"optimal resource utilization",
|
||||
"quality standards met"
|
||||
],
|
||||
"dependencies": []
|
||||
},
|
||||
{
|
||||
"name": "research_specialist",
|
||||
"role": "Research Specialist",
|
||||
"archetype": "specialist",
|
||||
"responsibilities": [
|
||||
"Conduct multi-source web research on specified topics",
|
||||
"Handle real-time information updates"
|
||||
],
|
||||
"capabilities": [
|
||||
"research_expertise",
|
||||
"specialized_tools",
|
||||
"domain_knowledge"
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"results": {
|
||||
"type": "array"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"research",
|
||||
"information_gathering"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
},
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze and process data",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"insights": {
|
||||
"type": "array"
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"data_analysis",
|
||||
"statistics",
|
||||
"visualization"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
}
|
||||
],
|
||||
"communication_interfaces": [
|
||||
"supervisor_messaging"
|
||||
],
|
||||
"constraints": {
|
||||
"domain_scope": "research",
|
||||
"task_queue_size": 10
|
||||
},
|
||||
"success_criteria": [
|
||||
"excel in research tasks",
|
||||
"maintain domain expertise",
|
||||
"provide quality output"
|
||||
],
|
||||
"dependencies": [
|
||||
"supervisor_agent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "data_specialist",
|
||||
"role": "Data Specialist",
|
||||
"archetype": "specialist",
|
||||
"responsibilities": [
|
||||
"Analyze and synthesize information from various sources",
|
||||
"Perform data processing and statistical analysis",
|
||||
"Generate visualizations and charts from data"
|
||||
],
|
||||
"capabilities": [
|
||||
"data_expertise",
|
||||
"specialized_tools",
|
||||
"domain_knowledge"
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze and process data",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"insights": {
|
||||
"type": "array"
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"data_analysis",
|
||||
"statistics",
|
||||
"visualization"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
},
|
||||
{
|
||||
"name": "file_manager",
|
||||
"description": "Manage files and directories",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string"
|
||||
},
|
||||
"path": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"file_operations",
|
||||
"data_management"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "low"
|
||||
}
|
||||
],
|
||||
"communication_interfaces": [
|
||||
"supervisor_messaging"
|
||||
],
|
||||
"constraints": {
|
||||
"domain_scope": "data",
|
||||
"task_queue_size": 10
|
||||
},
|
||||
"success_criteria": [
|
||||
"excel in data tasks",
|
||||
"maintain domain expertise",
|
||||
"provide quality output"
|
||||
],
|
||||
"dependencies": [
|
||||
"supervisor_agent"
|
||||
]
|
||||
}
|
||||
],
|
||||
"communication_topology": [
|
||||
{
|
||||
"from_agent": "supervisor_agent",
|
||||
"to_agent": "research_specialist",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_demand",
|
||||
"criticality": "high"
|
||||
},
|
||||
{
|
||||
"from_agent": "research_specialist",
|
||||
"to_agent": "supervisor_agent",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_completion",
|
||||
"criticality": "high"
|
||||
},
|
||||
{
|
||||
"from_agent": "supervisor_agent",
|
||||
"to_agent": "data_specialist",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_demand",
|
||||
"criticality": "high"
|
||||
},
|
||||
{
|
||||
"from_agent": "data_specialist",
|
||||
"to_agent": "supervisor_agent",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_completion",
|
||||
"criticality": "high"
|
||||
}
|
||||
],
|
||||
"shared_resources": [
|
||||
{
|
||||
"type": "message_queue",
|
||||
"capacity": 1000
|
||||
},
|
||||
{
|
||||
"type": "shared_memory",
|
||||
"size": "1GB"
|
||||
},
|
||||
{
|
||||
"type": "event_store",
|
||||
"retention": "30 days"
|
||||
}
|
||||
],
|
||||
"guardrails": [
|
||||
{
|
||||
"type": "input_validation",
|
||||
"rules": "strict_schema_enforcement"
|
||||
},
|
||||
{
|
||||
"type": "rate_limiting",
|
||||
"limit": "100_requests_per_minute"
|
||||
},
|
||||
{
|
||||
"type": "output_filtering",
|
||||
"rules": "content_safety_check"
|
||||
}
|
||||
],
|
||||
"scaling_strategy": {
|
||||
"horizontal_scaling": true,
|
||||
"auto_scaling_triggers": [
|
||||
"cpu > 80%",
|
||||
"queue_depth > 100"
|
||||
],
|
||||
"max_instances_per_agent": 5
|
||||
},
|
||||
"failure_handling": {
|
||||
"retry_policy": "exponential_backoff",
|
||||
"circuit_breaker": true,
|
||||
"fallback_strategies": [
|
||||
"graceful_degradation",
|
||||
"human_escalation"
|
||||
]
|
||||
}
|
||||
},
|
||||
"mermaid_diagram": "graph TD\n supervisor_agent[Task Coordinator and Quality Controller]:::coordinator\n research_specialist[Research Specialist]:::specialist\n data_specialist[Data Specialist]:::specialist\n supervisor_agent --> research_specialist\n research_specialist --> supervisor_agent\n supervisor_agent --> data_specialist\n data_specialist --> supervisor_agent\n\n classDef coordinator fill:#e1f5fe,stroke:#01579b,stroke-width:2px\n classDef specialist fill:#f3e5f5,stroke:#4a148c,stroke-width:2px\n classDef interface fill:#e8f5e8,stroke:#1b5e20,stroke-width:2px\n classDef monitor fill:#fff3e0,stroke:#e65100,stroke-width:2px\n class supervisor_agent coordinator\n class research_specialist specialist\n class data_specialist specialist",
|
||||
"implementation_roadmap": {
|
||||
"total_duration": "8-12 weeks",
|
||||
"phases": [
|
||||
{
|
||||
"phase": 1,
|
||||
"name": "Core Infrastructure",
|
||||
"duration": "2-3 weeks",
|
||||
"tasks": [
|
||||
"Set up development environment",
|
||||
"Implement basic agent framework",
|
||||
"Create communication infrastructure",
|
||||
"Set up monitoring and logging",
|
||||
"Implement basic tools"
|
||||
],
|
||||
"deliverables": [
|
||||
"Agent runtime framework",
|
||||
"Communication layer",
|
||||
"Basic monitoring dashboard"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": 2,
|
||||
"name": "Agent Implementation",
|
||||
"duration": "3-4 weeks",
|
||||
"tasks": [
|
||||
"Implement individual agent logic",
|
||||
"Create agent-specific tools",
|
||||
"Implement communication protocols",
|
||||
"Add error handling and recovery",
|
||||
"Create agent configuration system"
|
||||
],
|
||||
"deliverables": [
|
||||
"Functional agent implementations",
|
||||
"Tool integration",
|
||||
"Configuration management"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": 3,
|
||||
"name": "Integration and Testing",
|
||||
"duration": "2-3 weeks",
|
||||
"tasks": [
|
||||
"Integrate all agents",
|
||||
"End-to-end testing",
|
||||
"Performance optimization",
|
||||
"Security implementation",
|
||||
"Documentation creation"
|
||||
],
|
||||
"deliverables": [
|
||||
"Integrated system",
|
||||
"Test suite",
|
||||
"Performance benchmarks",
|
||||
"Security audit report"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": 4,
|
||||
"name": "Deployment and Monitoring",
|
||||
"duration": "1-2 weeks",
|
||||
"tasks": [
|
||||
"Production deployment",
|
||||
"Monitoring setup",
|
||||
"Alerting configuration",
|
||||
"User training",
|
||||
"Go-live support"
|
||||
],
|
||||
"deliverables": [
|
||||
"Production system",
|
||||
"Monitoring dashboard",
|
||||
"Operational runbooks",
|
||||
"Training materials"
|
||||
]
|
||||
}
|
||||
],
|
||||
"critical_path": [
|
||||
"Agent framework implementation",
|
||||
"Communication layer development",
|
||||
"Integration testing",
|
||||
"Production deployment"
|
||||
],
|
||||
"risks": [
|
||||
{
|
||||
"risk": "Communication complexity",
|
||||
"impact": "high",
|
||||
"mitigation": "Start with simple protocols, iterate"
|
||||
},
|
||||
{
|
||||
"risk": "Agent coordination failures",
|
||||
"impact": "medium",
|
||||
"mitigation": "Implement robust error handling and fallbacks"
|
||||
},
|
||||
{
|
||||
"risk": "Performance bottlenecks",
|
||||
"impact": "medium",
|
||||
"mitigation": "Early performance testing and optimization"
|
||||
}
|
||||
],
|
||||
"success_criteria": [
|
||||
"Input validation and sanitization",
|
||||
"Output content filtering",
|
||||
"Rate limiting for external APIs",
|
||||
"Error handling and graceful degradation",
|
||||
"Human oversight for critical decisions",
|
||||
"Audit logging for all operations",
|
||||
"All agents operational",
|
||||
"Communication working reliably",
|
||||
"Performance targets met",
|
||||
"Error rate below 1%"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"generated_by": "agent_planner.py",
|
||||
"requirements_file": "sample_system_requirements.json",
|
||||
"architecture_pattern": "supervisor",
|
||||
"agent_count": 3
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
{
|
||||
"summary": {
|
||||
"evaluation_period": {
|
||||
"start_time": "2024-01-15T09:00:00Z",
|
||||
"end_time": "2024-01-15T11:30:45Z",
|
||||
"total_duration_hours": 2.51
|
||||
},
|
||||
"overall_health": "good",
|
||||
"key_findings": [
|
||||
"Success rate (80.0%) below target",
|
||||
"High average latency (16.9s)",
|
||||
"2 high-impact error patterns identified"
|
||||
],
|
||||
"critical_issues": 0,
|
||||
"improvement_opportunities": 6
|
||||
},
|
||||
"system_metrics": {
|
||||
"total_tasks": 10,
|
||||
"successful_tasks": 8,
|
||||
"failed_tasks": 2,
|
||||
"partial_tasks": 1,
|
||||
"timeout_tasks": 1,
|
||||
"success_rate": 0.8,
|
||||
"failure_rate": 0.2,
|
||||
"average_duration_ms": 169800.0,
|
||||
"median_duration_ms": 152500.0,
|
||||
"percentile_95_duration_ms": 330000.0,
|
||||
"min_duration_ms": 8000,
|
||||
"max_duration_ms": 330000,
|
||||
"total_tokens_used": 53700,
|
||||
"average_tokens_per_task": 5370.0,
|
||||
"total_cost_usd": 1.074,
|
||||
"average_cost_per_task": 0.1074,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 3.98,
|
||||
"error_rate": 0.3,
|
||||
"retry_rate": 0.3
|
||||
},
|
||||
"agent_metrics": {
|
||||
"research_agent_1": {
|
||||
"total_tasks": 2,
|
||||
"successful_tasks": 2,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 1.0,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 174500.0,
|
||||
"median_duration_ms": 174500.0,
|
||||
"percentile_95_duration_ms": 195000.0,
|
||||
"min_duration_ms": 154000,
|
||||
"max_duration_ms": 195000,
|
||||
"total_tokens_used": 11050,
|
||||
"average_tokens_per_task": 5525.0,
|
||||
"total_cost_usd": 0.221,
|
||||
"average_cost_per_task": 0.1105,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 11.49,
|
||||
"error_rate": 0.0,
|
||||
"retry_rate": 0.0
|
||||
},
|
||||
"data_agent_1": {
|
||||
"total_tasks": 1,
|
||||
"successful_tasks": 1,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 1.0,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 165000.0,
|
||||
"median_duration_ms": 165000.0,
|
||||
"percentile_95_duration_ms": 165000.0,
|
||||
"min_duration_ms": 165000,
|
||||
"max_duration_ms": 165000,
|
||||
"total_tokens_used": 5000,
|
||||
"average_tokens_per_task": 5000.0,
|
||||
"total_cost_usd": 0.095,
|
||||
"average_cost_per_task": 0.095,
|
||||
"cost_per_token": 0.000019,
|
||||
"throughput_tasks_per_hour": 21.82,
|
||||
"error_rate": 0.0,
|
||||
"retry_rate": 0.0
|
||||
},
|
||||
"document_agent_1": {
|
||||
"total_tasks": 1,
|
||||
"successful_tasks": 0,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 1,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 0.0,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 140000.0,
|
||||
"median_duration_ms": 140000.0,
|
||||
"percentile_95_duration_ms": 140000.0,
|
||||
"min_duration_ms": 140000,
|
||||
"max_duration_ms": 140000,
|
||||
"total_tokens_used": 8600,
|
||||
"average_tokens_per_task": 8600.0,
|
||||
"total_cost_usd": 0.172,
|
||||
"average_cost_per_task": 0.172,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 25.71,
|
||||
"error_rate": 1.0,
|
||||
"retry_rate": 1.0
|
||||
}
|
||||
},
|
||||
"task_type_metrics": {
|
||||
"web_research": {
|
||||
"total_tasks": 3,
|
||||
"successful_tasks": 2,
|
||||
"failed_tasks": 1,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 0.667,
|
||||
"failure_rate": 0.333,
|
||||
"average_duration_ms": 226333.33,
|
||||
"median_duration_ms": 195000.0,
|
||||
"percentile_95_duration_ms": 330000.0,
|
||||
"min_duration_ms": 154000,
|
||||
"max_duration_ms": 330000,
|
||||
"total_tokens_used": 12250,
|
||||
"average_tokens_per_task": 4083.33,
|
||||
"total_cost_usd": 0.245,
|
||||
"average_cost_per_task": 0.082,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 2.65,
|
||||
"error_rate": 0.333,
|
||||
"retry_rate": 0.333
|
||||
},
|
||||
"data_analysis": {
|
||||
"total_tasks": 2,
|
||||
"successful_tasks": 1,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 1,
|
||||
"success_rate": 0.5,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 215000.0,
|
||||
"median_duration_ms": 215000.0,
|
||||
"percentile_95_duration_ms": 265000.0,
|
||||
"min_duration_ms": 165000,
|
||||
"max_duration_ms": 265000,
|
||||
"total_tokens_used": 14000,
|
||||
"average_tokens_per_task": 7000.0,
|
||||
"total_cost_usd": 0.275,
|
||||
"average_cost_per_task": 0.138,
|
||||
"cost_per_token": 0.0000196,
|
||||
"throughput_tasks_per_hour": 1.86,
|
||||
"error_rate": 0.5,
|
||||
"retry_rate": 0.0
|
||||
}
|
||||
},
|
||||
"tool_usage_analysis": {
|
||||
"web_search": {
|
||||
"usage_count": 3,
|
||||
"error_rate": 0.333,
|
||||
"avg_duration": 126666.67,
|
||||
"affected_workflows": [
|
||||
"web_research"
|
||||
],
|
||||
"retry_count": 2
|
||||
},
|
||||
"data_analyzer": {
|
||||
"usage_count": 2,
|
||||
"error_rate": 0.0,
|
||||
"avg_duration": 205000.0,
|
||||
"affected_workflows": [
|
||||
"data_analysis"
|
||||
],
|
||||
"retry_count": 0
|
||||
},
|
||||
"document_processor": {
|
||||
"usage_count": 2,
|
||||
"error_rate": 0.0,
|
||||
"avg_duration": 140000.0,
|
||||
"affected_workflows": [
|
||||
"document_processing"
|
||||
],
|
||||
"retry_count": 1
|
||||
},
|
||||
"notification_sender": {
|
||||
"usage_count": 2,
|
||||
"error_rate": 0.5,
|
||||
"avg_duration": 18750.0,
|
||||
"affected_workflows": [
|
||||
"notification"
|
||||
],
|
||||
"retry_count": 1
|
||||
},
|
||||
"task_scheduler": {
|
||||
"usage_count": 1,
|
||||
"error_rate": 0.0,
|
||||
"avg_duration": 12000.0,
|
||||
"affected_workflows": [
|
||||
"task_scheduling"
|
||||
],
|
||||
"retry_count": 0
|
||||
}
|
||||
},
|
||||
"error_analysis": [
|
||||
{
|
||||
"error_type": "timeout",
|
||||
"count": 2,
|
||||
"percentage": 20.0,
|
||||
"affected_agents": [
|
||||
"research_agent_2",
|
||||
"data_agent_2"
|
||||
],
|
||||
"affected_task_types": [
|
||||
"web_research",
|
||||
"data_analysis"
|
||||
],
|
||||
"common_patterns": [
|
||||
"timeout",
|
||||
"exceeded",
|
||||
"limit"
|
||||
],
|
||||
"suggested_fixes": [
|
||||
"Increase timeout values",
|
||||
"Optimize slow operations",
|
||||
"Add retry logic with exponential backoff",
|
||||
"Parallelize independent operations"
|
||||
],
|
||||
"impact_level": "high"
|
||||
},
|
||||
{
|
||||
"error_type": "authentication",
|
||||
"count": 1,
|
||||
"percentage": 10.0,
|
||||
"affected_agents": [
|
||||
"communication_agent_2"
|
||||
],
|
||||
"affected_task_types": [
|
||||
"notification"
|
||||
],
|
||||
"common_patterns": [
|
||||
"authentication",
|
||||
"failed",
|
||||
"invalid"
|
||||
],
|
||||
"suggested_fixes": [
|
||||
"Check credential rotation",
|
||||
"Implement token refresh logic",
|
||||
"Add authentication retry",
|
||||
"Verify permission scopes"
|
||||
],
|
||||
"impact_level": "high"
|
||||
},
|
||||
{
|
||||
"error_type": "validation",
|
||||
"count": 1,
|
||||
"percentage": 10.0,
|
||||
"affected_agents": [
|
||||
"document_agent_1"
|
||||
],
|
||||
"affected_task_types": [
|
||||
"document_processing"
|
||||
],
|
||||
"common_patterns": [
|
||||
"validation",
|
||||
"failed",
|
||||
"missing"
|
||||
],
|
||||
"suggested_fixes": [
|
||||
"Strengthen input validation",
|
||||
"Add data sanitization",
|
||||
"Improve error messages",
|
||||
"Add input examples"
|
||||
],
|
||||
"impact_level": "medium"
|
||||
}
|
||||
],
|
||||
"bottleneck_analysis": [
|
||||
{
|
||||
"bottleneck_type": "tool",
|
||||
"location": "notification_sender",
|
||||
"severity": "medium",
|
||||
"description": "Tool notification_sender has high error rate (50.0%)",
|
||||
"impact_on_performance": {
|
||||
"reliability_impact": 1.0,
|
||||
"retry_overhead": 1000
|
||||
},
|
||||
"affected_workflows": [
|
||||
"notification"
|
||||
],
|
||||
"optimization_suggestions": [
|
||||
"Review tool implementation",
|
||||
"Add better error handling for tool",
|
||||
"Implement tool fallbacks",
|
||||
"Consider alternative tools"
|
||||
],
|
||||
"estimated_improvement": {
|
||||
"error_reduction": 0.35,
|
||||
"performance_gain": 1.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"bottleneck_type": "tool",
|
||||
"location": "web_search",
|
||||
"severity": "medium",
|
||||
"description": "Tool web_search has high error rate (33.3%)",
|
||||
"impact_on_performance": {
|
||||
"reliability_impact": 1.0,
|
||||
"retry_overhead": 2000
|
||||
},
|
||||
"affected_workflows": [
|
||||
"web_research"
|
||||
],
|
||||
"optimization_suggestions": [
|
||||
"Review tool implementation",
|
||||
"Add better error handling for tool",
|
||||
"Implement tool fallbacks",
|
||||
"Consider alternative tools"
|
||||
],
|
||||
"estimated_improvement": {
|
||||
"error_reduction": 0.233,
|
||||
"performance_gain": 1.2
|
||||
}
|
||||
}
|
||||
],
|
||||
"optimization_recommendations": [
|
||||
{
|
||||
"category": "reliability",
|
||||
"priority": "high",
|
||||
"title": "Improve System Reliability",
|
||||
"description": "System success rate is 80.0%, below target of 90%",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"success_rate_improvement": 0.1,
|
||||
"cost_reduction": 0.01611
|
||||
},
|
||||
"estimated_cost_savings": 0.1074,
|
||||
"estimated_performance_gain": 1.2,
|
||||
"implementation_steps": [
|
||||
"Identify and fix top error patterns",
|
||||
"Implement better error handling and retries",
|
||||
"Add comprehensive monitoring and alerting",
|
||||
"Implement graceful degradation patterns"
|
||||
],
|
||||
"risks": [
|
||||
"Temporary increase in complexity",
|
||||
"Potential initial performance overhead"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Error analysis completion",
|
||||
"Monitoring infrastructure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "performance",
|
||||
"priority": "high",
|
||||
"title": "Reduce Task Latency",
|
||||
"description": "Average task duration (169.8s) exceeds target",
|
||||
"implementation_effort": "high",
|
||||
"expected_impact": {
|
||||
"latency_reduction": 0.49,
|
||||
"throughput_improvement": 1.5
|
||||
},
|
||||
"estimated_performance_gain": 1.4,
|
||||
"implementation_steps": [
|
||||
"Profile and optimize slow operations",
|
||||
"Implement parallel processing where possible",
|
||||
"Add caching for expensive operations",
|
||||
"Optimize API calls and reduce round trips"
|
||||
],
|
||||
"risks": [
|
||||
"Increased system complexity",
|
||||
"Potential resource usage increase"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Performance profiling tools",
|
||||
"Caching infrastructure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "cost",
|
||||
"priority": "medium",
|
||||
"title": "Optimize Token Usage and Costs",
|
||||
"description": "Average cost per task ($0.107) is above optimal range",
|
||||
"implementation_effort": "low",
|
||||
"expected_impact": {
|
||||
"cost_reduction": 0.032,
|
||||
"efficiency_improvement": 1.15
|
||||
},
|
||||
"estimated_cost_savings": 0.322,
|
||||
"estimated_performance_gain": 1.05,
|
||||
"implementation_steps": [
|
||||
"Implement prompt optimization",
|
||||
"Add response caching for repeated queries",
|
||||
"Use smaller models for simple tasks",
|
||||
"Implement token usage monitoring and alerts"
|
||||
],
|
||||
"risks": [
|
||||
"Potential quality reduction with smaller models"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Token usage analysis",
|
||||
"Caching infrastructure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "reliability",
|
||||
"priority": "high",
|
||||
"title": "Address Timeout Errors",
|
||||
"description": "Timeout errors occur in 20.0% of cases",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"error_reduction": 0.2,
|
||||
"reliability_improvement": 1.1
|
||||
},
|
||||
"estimated_cost_savings": 0.1074,
|
||||
"implementation_steps": [
|
||||
"Increase timeout values",
|
||||
"Optimize slow operations",
|
||||
"Add retry logic with exponential backoff",
|
||||
"Parallelize independent operations"
|
||||
],
|
||||
"risks": [
|
||||
"May require significant code changes"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Root cause analysis",
|
||||
"Testing framework"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "reliability",
|
||||
"priority": "high",
|
||||
"title": "Address Authentication Errors",
|
||||
"description": "Authentication errors occur in 10.0% of cases",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"error_reduction": 0.1,
|
||||
"reliability_improvement": 1.1
|
||||
},
|
||||
"estimated_cost_savings": 0.1074,
|
||||
"implementation_steps": [
|
||||
"Check credential rotation",
|
||||
"Implement token refresh logic",
|
||||
"Add authentication retry",
|
||||
"Verify permission scopes"
|
||||
],
|
||||
"risks": [
|
||||
"May require significant code changes"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Root cause analysis",
|
||||
"Testing framework"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "performance",
|
||||
"priority": "medium",
|
||||
"title": "Address Tool Bottleneck",
|
||||
"description": "Tool notification_sender has high error rate (50.0%)",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"error_reduction": 0.35,
|
||||
"performance_gain": 1.2
|
||||
},
|
||||
"estimated_performance_gain": 1.2,
|
||||
"implementation_steps": [
|
||||
"Review tool implementation",
|
||||
"Add better error handling for tool",
|
||||
"Implement tool fallbacks",
|
||||
"Consider alternative tools"
|
||||
],
|
||||
"risks": [
|
||||
"System downtime during implementation",
|
||||
"Potential cascade effects"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Impact assessment",
|
||||
"Rollback plan"
|
||||
]
|
||||
}
|
||||
],
|
||||
"trends_analysis": {
|
||||
"daily_success_rates": {
|
||||
"2024-01-15": 0.8
|
||||
},
|
||||
"daily_avg_durations": {
|
||||
"2024-01-15": 169800.0
|
||||
},
|
||||
"daily_costs": {
|
||||
"2024-01-15": 1.074
|
||||
},
|
||||
"trend_direction": {
|
||||
"success_rate": "stable",
|
||||
"duration": "stable",
|
||||
"cost": "stable"
|
||||
}
|
||||
},
|
||||
"cost_breakdown": {
|
||||
"total_cost": 1.074,
|
||||
"cost_by_agent": {
|
||||
"research_agent_1": 0.221,
|
||||
"research_agent_2": 0.024,
|
||||
"data_agent_1": 0.095,
|
||||
"data_agent_2": 0.18,
|
||||
"document_agent_1": 0.172,
|
||||
"document_agent_2": 0.174,
|
||||
"communication_agent_1": 0.007,
|
||||
"communication_agent_2": 0.004,
|
||||
"scheduler_agent_1": 0.01
|
||||
},
|
||||
"cost_by_task_type": {
|
||||
"web_research": 0.245,
|
||||
"data_analysis": 0.275,
|
||||
"document_processing": 0.346,
|
||||
"notification": 0.011,
|
||||
"task_scheduling": 0.01
|
||||
},
|
||||
"cost_per_token": 0.00002,
|
||||
"top_cost_drivers": [
|
||||
[
|
||||
"document_processing",
|
||||
0.346
|
||||
],
|
||||
[
|
||||
"data_analysis",
|
||||
0.275
|
||||
],
|
||||
[
|
||||
"web_research",
|
||||
0.245
|
||||
],
|
||||
[
|
||||
"notification",
|
||||
0.011
|
||||
],
|
||||
[
|
||||
"task_scheduling",
|
||||
0.01
|
||||
]
|
||||
]
|
||||
},
|
||||
"sla_compliance": {
|
||||
"overall_compliant": false,
|
||||
"sla_details": {
|
||||
"success_rate": {
|
||||
"target": 0.95,
|
||||
"actual": 0.8,
|
||||
"compliant": false,
|
||||
"gap": 0.15
|
||||
},
|
||||
"average_latency": {
|
||||
"target": 10000,
|
||||
"actual": 169800.0,
|
||||
"compliant": false,
|
||||
"gap": 159800.0
|
||||
},
|
||||
"error_rate": {
|
||||
"target": 0.05,
|
||||
"actual": 0.3,
|
||||
"compliant": false,
|
||||
"gap": 0.25
|
||||
}
|
||||
},
|
||||
"compliance_score": 0.0
|
||||
},
|
||||
"metadata": {
|
||||
"generated_at": "2024-01-15T12:00:00Z",
|
||||
"evaluator_version": "1.0",
|
||||
"total_logs_processed": 10,
|
||||
"agents_analyzed": 9,
|
||||
"task_types_analyzed": 5,
|
||||
"analysis_completeness": "full"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
{
|
||||
"tool_schemas": [
|
||||
{
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information on specified topics with customizable filters and result limits",
|
||||
"openai_schema": {
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information on specified topics with customizable filters and result limits",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query string to find relevant information",
|
||||
"minLength": 1,
|
||||
"maxLength": 500,
|
||||
"examples": [
|
||||
"artificial intelligence trends",
|
||||
"climate change impact",
|
||||
"python programming tutorial"
|
||||
]
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of search results to return",
|
||||
"minimum": 1,
|
||||
"maximum": 100,
|
||||
"default": 10
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"description": "Language code for search results",
|
||||
"enum": [
|
||||
"en",
|
||||
"es",
|
||||
"fr",
|
||||
"de",
|
||||
"it",
|
||||
"pt",
|
||||
"zh",
|
||||
"ja"
|
||||
],
|
||||
"default": "en"
|
||||
},
|
||||
"time_range": {
|
||||
"type": "string",
|
||||
"description": "Time range filter for search results",
|
||||
"enum": [
|
||||
"any",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"year"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"query"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"anthropic_schema": {
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information on specified topics with customizable filters and result limits",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query string to find relevant information",
|
||||
"minLength": 1,
|
||||
"maxLength": 500
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of search results to return",
|
||||
"minimum": 1,
|
||||
"maximum": 100
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"description": "Language code for search results",
|
||||
"enum": [
|
||||
"en",
|
||||
"es",
|
||||
"fr",
|
||||
"de",
|
||||
"it",
|
||||
"pt",
|
||||
"zh",
|
||||
"ja"
|
||||
]
|
||||
},
|
||||
"time_range": {
|
||||
"type": "string",
|
||||
"description": "Time range filter for search results",
|
||||
"enum": [
|
||||
"any",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"year"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"query"
|
||||
]
|
||||
}
|
||||
},
|
||||
"validation_rules": [
|
||||
{
|
||||
"parameter": "query",
|
||||
"rules": {
|
||||
"minLength": 1,
|
||||
"maxLength": 500
|
||||
}
|
||||
},
|
||||
{
|
||||
"parameter": "limit",
|
||||
"rules": {
|
||||
"minimum": 1,
|
||||
"maximum": 100
|
||||
}
|
||||
}
|
||||
],
|
||||
"error_responses": [
|
||||
{
|
||||
"error_code": "invalid_input",
|
||||
"error_message": "Invalid input parameters provided",
|
||||
"http_status": 400,
|
||||
"retry_after": null,
|
||||
"details": {
|
||||
"validation_errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"error_code": "authentication_required",
|
||||
"error_message": "Authentication required to access this tool",
|
||||
"http_status": 401,
|
||||
"retry_after": null,
|
||||
"details": null
|
||||
},
|
||||
{
|
||||
"error_code": "rate_limit_exceeded",
|
||||
"error_message": "Rate limit exceeded. Please try again later",
|
||||
"http_status": 429,
|
||||
"retry_after": 60,
|
||||
"details": null
|
||||
}
|
||||
],
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 60,
|
||||
"requests_per_hour": 1000,
|
||||
"requests_per_day": 10000,
|
||||
"burst_limit": 10,
|
||||
"cooldown_period": 60,
|
||||
"rate_limit_key": "user_id"
|
||||
},
|
||||
"examples": [
|
||||
{
|
||||
"description": "Basic web search",
|
||||
"input": {
|
||||
"query": "machine learning algorithms",
|
||||
"limit": 5
|
||||
},
|
||||
"expected_output": {
|
||||
"results": [
|
||||
{
|
||||
"title": "Introduction to Machine Learning Algorithms",
|
||||
"url": "https://example.com/ml-intro",
|
||||
"snippet": "Machine learning algorithms are computational methods...",
|
||||
"relevance_score": 0.95
|
||||
}
|
||||
],
|
||||
"total_found": 1250
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"category": "search",
|
||||
"idempotent": true,
|
||||
"side_effects": [
|
||||
"Logs search query for analytics",
|
||||
"May cache results temporarily"
|
||||
],
|
||||
"dependencies": [
|
||||
"search_api_service",
|
||||
"content_filter_service"
|
||||
],
|
||||
"security_requirements": [
|
||||
"Query sanitization",
|
||||
"Rate limiting by user",
|
||||
"Content filtering"
|
||||
],
|
||||
"generated_at": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0",
|
||||
"input_parameters": 4,
|
||||
"output_parameters": 2,
|
||||
"required_parameters": 1,
|
||||
"optional_parameters": 3
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze structured data and generate statistical insights, trends, and visualizations",
|
||||
"openai_schema": {
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze structured data and generate statistical insights, trends, and visualizations",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"description": "Structured data to analyze in JSON format",
|
||||
"properties": {
|
||||
"columns": {
|
||||
"type": "array"
|
||||
},
|
||||
"rows": {
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string",
|
||||
"description": "Type of analysis to perform",
|
||||
"enum": [
|
||||
"descriptive",
|
||||
"correlation",
|
||||
"trend",
|
||||
"distribution",
|
||||
"outlier_detection"
|
||||
]
|
||||
},
|
||||
"target_column": {
|
||||
"type": "string",
|
||||
"description": "Primary column to focus analysis on",
|
||||
"maxLength": 1000
|
||||
},
|
||||
"include_visualization": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to generate visualization data",
|
||||
"default": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"data",
|
||||
"analysis_type"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"anthropic_schema": {
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze structured data and generate statistical insights, trends, and visualizations",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"description": "Structured data to analyze in JSON format"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string",
|
||||
"description": "Type of analysis to perform",
|
||||
"enum": [
|
||||
"descriptive",
|
||||
"correlation",
|
||||
"trend",
|
||||
"distribution",
|
||||
"outlier_detection"
|
||||
]
|
||||
},
|
||||
"target_column": {
|
||||
"type": "string",
|
||||
"description": "Primary column to focus analysis on",
|
||||
"maxLength": 1000
|
||||
},
|
||||
"include_visualization": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to generate visualization data"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"data",
|
||||
"analysis_type"
|
||||
]
|
||||
}
|
||||
},
|
||||
"validation_rules": [
|
||||
{
|
||||
"parameter": "target_column",
|
||||
"rules": {
|
||||
"maxLength": 1000
|
||||
}
|
||||
}
|
||||
],
|
||||
"error_responses": [
|
||||
{
|
||||
"error_code": "invalid_input",
|
||||
"error_message": "Invalid input parameters provided",
|
||||
"http_status": 400,
|
||||
"retry_after": null,
|
||||
"details": {
|
||||
"validation_errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"error_code": "authentication_required",
|
||||
"error_message": "Authentication required to access this tool",
|
||||
"http_status": 401,
|
||||
"retry_after": null,
|
||||
"details": null
|
||||
},
|
||||
{
|
||||
"error_code": "rate_limit_exceeded",
|
||||
"error_message": "Rate limit exceeded. Please try again later",
|
||||
"http_status": 429,
|
||||
"retry_after": 60,
|
||||
"details": null
|
||||
}
|
||||
],
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 30,
|
||||
"requests_per_hour": 500,
|
||||
"requests_per_day": 5000,
|
||||
"burst_limit": 5,
|
||||
"cooldown_period": 60,
|
||||
"rate_limit_key": "user_id"
|
||||
},
|
||||
"examples": [
|
||||
{
|
||||
"description": "Basic descriptive analysis",
|
||||
"input": {
|
||||
"data": {
|
||||
"columns": [
|
||||
"age",
|
||||
"salary",
|
||||
"department"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
25,
|
||||
50000,
|
||||
"engineering"
|
||||
],
|
||||
[
|
||||
30,
|
||||
60000,
|
||||
"engineering"
|
||||
],
|
||||
[
|
||||
28,
|
||||
55000,
|
||||
"marketing"
|
||||
]
|
||||
]
|
||||
},
|
||||
"analysis_type": "descriptive",
|
||||
"target_column": "salary"
|
||||
},
|
||||
"expected_output": {
|
||||
"insights": [
|
||||
"Average salary is $55,000",
|
||||
"Salary range: $50,000 - $60,000",
|
||||
"Engineering department has higher average salary"
|
||||
],
|
||||
"statistics": {
|
||||
"mean": 55000,
|
||||
"median": 55000,
|
||||
"std_dev": 5000
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"category": "data",
|
||||
"idempotent": true,
|
||||
"side_effects": [
|
||||
"May create temporary analysis files",
|
||||
"Logs analysis parameters for optimization"
|
||||
],
|
||||
"dependencies": [
|
||||
"statistics_engine",
|
||||
"visualization_service"
|
||||
],
|
||||
"security_requirements": [
|
||||
"Data anonymization",
|
||||
"Access control validation"
|
||||
],
|
||||
"generated_at": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0",
|
||||
"input_parameters": 4,
|
||||
"output_parameters": 3,
|
||||
"required_parameters": 2,
|
||||
"optional_parameters": 2
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"generated_by": "tool_schema_generator.py",
|
||||
"input_file": "sample_tool_descriptions.json",
|
||||
"tool_count": 2,
|
||||
"generation_timestamp": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0"
|
||||
},
|
||||
"validation_summary": {
|
||||
"total_tools": 2,
|
||||
"total_parameters": 8,
|
||||
"total_validation_rules": 3,
|
||||
"total_examples": 2
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user