git:20251121.1512de0 to git:20260725.c9691fc

- ---

- name: cfn-redis-data-extraction

- description: Extract complete Redis coordination data from completed CFN Loop tasks and structure into comprehensive JSON analysis files

- version: 1.0.0

- author: CFN System

- category: coordination

- tags: [cfn, redis, data-extraction, coordination, loop-analysis]

-

- # CFN Redis Data Extraction Skill

-

- ## Purpose

-

- Extracts complete Redis coordination data from completed CFN Loop tasks and structures it into comprehensive JSON files for analysis, auditing, and performance tracking.

-

- ## When to Use

-

- - **Mandatory**: Execute after each completed CFN Loop task

- - **Timing**: After Product Owner decision, before Redis cleanup

- - **Trigger**: Main Chat coordination after loop completion

- - **Scope**: All Redis keys for a specific task ID

-

- ## Usage

-

- ```bash

- # Extract data from completed CFN Loop

- npx claude-flow-novice skill cfn-redis-data-extraction --task-id "cfn-cli-XXXXXXX-XXXXX"

-

- # Extract with custom output directory

- npx claude-flow-novice skill cfn-redis-data-extraction \

- --task-id "cfn-cli-XXXXXXX-XXXXX" \

- --output-dir "./analysis/loop-data"

-

- # Extract multiple tasks

- npx claude-flow-novice skill cfn-redis-data-extraction \

- --task-ids "cfn-cli-XXXXXXX-XXXXX,cfn-cli-YYYYYYY-YYYYY"

-

- # Extract with detailed performance metrics

- npx claude-flow-novice skill cfn-redis-data-extraction \

- --task-id "cfn-cli-XXXXXXX-XXXXX" \

- --include-performance=true

- ```

-

- ## Implementation

-

- ```bash

- #!/bin/bash

- # CFN Redis Data Extraction Script

-

- set -euo pipefail

-

- # Default values

- OUTPUT_DIR="./analysis/cfn-loop-data"

- INCLUDE_PERFORMANCE=false

- TASK_IDS=()

- VERBOSE=false

-

- # Parse arguments

- while [[ $# -gt 0 ]]; do

- case $1 in

- --task-id)

- TASK_IDS+=("$2")

- shift 2

- ;;

- --task-ids)

- IFS=',' read -ra TASK_IDS <<< "$2"

- shift 2

- ;;

- --output-dir)

- OUTPUT_DIR="$2"

- shift 2

- ;;

- --include-performance)

- INCLUDE_PERFORMANCE=true

- shift

- ;;

- --verbose)

- VERBOSE=true

- shift

- ;;

- -h|--help)

- echo "Usage: $0 --task-id <TASK_ID> [--output-dir <DIR>] [--include-performance] [--verbose]"

- exit 0

- ;;

- *)

- echo "Unknown option: $1"

- exit 1

- ;;

- esac

- done

-

- # Validate required arguments

- if [[ ${#TASK_IDS[@]} -eq 0 ]]; then

- echo "Error: --task-id or --task-ids is required"

- exit 1

- fi

-

- # Create output directory

- mkdir -p "$OUTPUT_DIR"

-

- # Redis connection check

- if ! redis-cli ping > /dev/null 2>&1; then

- echo "Error: Redis is not accessible"

- exit 1

- fi

-

- # Function to extract task data

- extract_task_data() {

- local task_id="$1"

- local output_file="$OUTPUT_DIR/cfn-loop-${task_id}-extracted.json"

-

- [[ "$VERBOSE" == true ]] && echo "Extracting data for task: $task_id"

-

- # Get all Redis keys for the task

- local redis_keys

- redis_keys=$(redis-cli keys "*${task_id}*" 2>/dev/null | sort)

-

- if [[ -z "$redis_keys" ]]; then

- echo "Warning: No Redis keys found for task: $task_id"

- return 1

- fi

-

- # Initialize JSON structure

- local json_data

- json_data=$(cat << EOF

- {

- "task_id": "$task_id",

- "extraction_timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",

- "extraction_version": "1.0.0",

- "redis_keys_analyzed": $(echo "$redis_keys" | wc -l),

- "agents": {},

- "metadata": {},

- "summary": {}

- }

- EOF

- )

-

- # Process each Redis key

- local agent_count=0

- local completion_signals=0

- local total_confidence=0

- local confidence_count=0

-

- while IFS= read -r key; do

- [[ -z "$key" ]] && continue

-

- # Skip if not a swarm key

- [[ ! "$key" =~ ^swarm: ]] && continue

-

- # Extract agent information from key

- local agent_type=""

- local agent_id=""

- local loop_number=""

- local data_type=""

-

- if [[ "$key" =~ ^swarm:([^:]+):([^:]+):([^:]+)$ ]]; then

- agent_id="${BASH_REMATCH[2]}"

- data_type="${BASH_REMATCH[3]}"

-

- # Determine agent type and loop

- if [[ "$agent_id" =~ ^cfn-v3-coordinator ]]; then

- agent_type="coordinator"

- loop_number="coordination"

- elif [[ "$agent_id" =~ -validation$ ]]; then

- agent_type="${agent_id%-validation}"

- loop_number="2"

- elif [[ "$agent_id" =~ -1$ ]] || [[ "$agent_id" =~ -2$ ]] || [[ "$agent_id" =~ -3$ ]]; then

- agent_type="${agent_id%-1}"

- agent_type="${agent_type%-2}"

- agent_type="${agent_type%-3}"

- loop_number="3"

- elif [[ "$agent_id" =~ product-owner$ ]]; then

- agent_type="product-owner"

- loop_number="4"

- else

- agent_type="$agent_id"

- loop_number="unknown"

- fi

-

- # Initialize agent in JSON if not exists

- if [[ ! "$json_data" =~ "\"$agent_id\":" ]]; then

- ((agent_count++))

- json_data=$(echo "$json_data" | jq ".agents += {\"$agent_id\": {\"agent_type\": \"$agent_type\", \"loop\": \"$loop_number\", \"data\": {}}}")

- fi

-

- # Extract data based on type

- case "$data_type" in

- "confidence")

- local confidence

- confidence=$(redis-cli get "$key" 2>/dev/null || echo "null")

- json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].confidence = $confidence")

- total_confidence=$(echo "$total_confidence + $confidence" | bc -l 2>/dev/null || echo "$total_confidence")

- ((confidence_count++))

- ;;

- "done")

- local completion_signal

- completion_signal=$(redis-cli lrange "$key" 0 -1 2>/dev/null | head -1 || echo "null")

- json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].completion_signal = \"$completion_signal\"")

- ((completion_signals++))

- ;;

- "messages")

- local messages_json="[]"

- local message_count

- message_count=$(redis-cli llen "$key" 2>/dev/null || echo "0")

-

- if [[ "$message_count" -gt 0 ]]; then

- messages_json=$(redis-cli lrange "$key" 0 -1 2>/dev/null | jq -R . | jq -s .)

- fi

-

- json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].messages = $messages_json")

- ;;

- "result")

- local result

- result=$(redis-cli get "$key" 2>/dev/null || echo "null")

- json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].result = \"$result\"")

- ;;

- esac

- fi

- done <<< "$redis_keys"

-

- # Calculate averages and summary

- local avg_confidence="0"

- if [[ "$confidence_count" -gt 0 ]]; then

- avg_confidence=$(echo "scale=3; $total_confidence / $confidence_count" | bc -l)

- fi

-

- # Extract task context if available

- local task_context

- task_context=$(redis-cli get "cfn_loop:task:$task_id:context" 2>/dev/null || echo "{}")

-

- # Update summary

- json_data=$(echo "$json_data" | jq "

- .summary += {

- \"total_agents\": $agent_count,

- \"completion_signals\": $completion_signals,

- \"average_confidence\": $avg_confidence,

- \"confidence_scores_count\": $confidence_count,

- \"extraction_status\": \"success\"

- }

- ")

-

- # Add task context

- json_data=$(echo "$json_data" | jq ".metadata.task_context = \"$task_context\"")

-

- # Add performance metrics if requested

- if [[ "$INCLUDE_PERFORMANCE" == true ]]; then

- local performance_metrics

- performance_metrics=$(cat << EOF

- {

- "redis_memory_usage": $(redis-cli info memory 2>/dev/null | grep "used_memory_human" | cut -d: -f2 | tr -d '\r' || echo "\"unknown\""),

- "redis_connected_clients": $(redis-cli info clients 2>/dev/null | grep "connected_clients" | cut -d: -f2 | tr -d '\r' || echo "0"),

- "extraction_duration_ms": 0

- }

- EOF

- )

- json_data=$(echo "$json_data" | jq ".metadata.performance = $performance_metrics")

- fi

-

- # Write to file

- echo "$json_data" | jq '.' > "$output_file"

-

- [[ "$VERBOSE" == true ]] && echo "Data extracted to: $output_file"

-

- # Generate summary report

- local summary_file="$OUTPUT_DIR/cfn-loop-${task_id}-summary.txt"

- cat > "$summary_file" << EOF

- CFN Loop Data Extraction Summary

- ================================

-

- Task ID: $task_id

- Extraction Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)

- Output File: $output_file

-

- Statistics:

- - Redis Keys Analyzed: $(echo "$redis_keys" | wc -l)

- - Total Agents: $agent_count

- - Completion Signals: $completion_signals

- - Average Confidence: $avg_confidence

- - Confidence Scores: $confidence_count

-

- Agent Types:

- $(echo "$json_data" | jq -r '.agents | to_entries[] | "- \(.key): \(.value.agent_type) (Loop \(.value.loop))"')

-

- Status: Successfully extracted

- EOF

-

- [[ "$VERBOSE" == true ]] && echo "Summary report generated: $summary_file"

-

- return 0

- }

-

- # Main execution

- echo "CFN Redis Data Extraction Started"

- echo "================================="

- echo "Output Directory: $OUTPUT_DIR"

- echo "Include Performance: $INCLUDE_PERFORMANCE"

- echo ""

-

- # Process each task

- for task_id in "${TASK_IDS[@]}"; do

- echo "Processing task: $task_id"

- if extract_task_data "$task_id"; then

- echo "✅ Successfully extracted data for: $task_id"

- else

- echo "❌ Failed to extract data for: $task_id"

- fi

- echo ""

- done

-

- echo "CFN Redis Data Extraction Completed"

- echo "==================================="

- echo "Output Directory: $OUTPUT_DIR"

- echo "Files Generated:"

- find "$OUTPUT_DIR" -name "cfn-loop-*.json" -o -name "cfn-loop-*.txt" | while read -r file; do

- echo " - $file"

- done

-

- exit 0

- ```

-

- ## Parameters

-

- | Parameter | Type | Required | Description |

- |-----------|------|----------|-------------|

- | `--task-id` | string | Yes | Single CFN Loop task ID to extract |

- | `--task-ids` | string | No | Comma-separated list of task IDs |

- | `--output-dir` | string | No | Output directory (default: ./analysis/cfn-loop-data) |

- | `--include-performance` | boolean | No | Include Redis performance metrics |

- | `--verbose` | boolean | No | Enable verbose output |

-

- ## Output Structure

-

- The skill generates two files per task:

-

- ### 1. Main JSON Data File

- `cfn-loop-{task_id}-extracted.json`

-

- ```json

- {

- "task_id": "cfn-cli-543042-13483",

- "extraction_timestamp": "2025-11-09T04:56:00Z",

- "extraction_version": "1.0.0",

- "redis_keys_analyzed": 44,

- "agents": {

- "cfn-v3-coordinator-1": {

- "agent_type": "coordinator",

- "loop": "coordination",

- "confidence": 0.85,

- "completion_signal": "complete",

- "messages": [...],

- "result": "..."

- },

- "agent-id-2": {

- "agent_type": "frontend-developer",

- "loop": "3",

- "confidence": 0.92,

- "completion_signal": "complete"

- }

- },

- "metadata": {

- "task_context": "...",

- "performance": {

- "redis_memory_usage": "2.5M",

- "redis_connected_clients": 12

- }

- },

- "summary": {

- "total_agents": 11,

- "completion_signals": 11,

- "average_confidence": 0.86,

- "extraction_status": "success"

- }

- }

- ```

-

- ### 2. Summary Text File

- `cfn-loop-{task_id}-summary.txt`

-

- Human-readable summary with key statistics and agent information.

-

- ## Integration Points

-

- ### CFN Loop Completion Workflow

- ```bash

- # After Product Owner decision, before Redis cleanup

- ./.claude/skills/cfn-redis-data-extraction/extract.sh \

- --task-id "$TASK_ID" \

- --output-dir "./analysis/loops" \

- --include-performance=true

-

- # Then proceed with Redis cleanup

- redis-cli DEL "cfn_loop:task:$TASK_ID:*" "swarm:$TASK_ID:*"

- ```

-

- ### Main Chat Automation

- ```bash

- # Automatic extraction after loop completion

- npx claude-flow-novice skill cfn-redis-data-extraction \

- --task-id "cfn-cli-XXXXXXX-XXXXX" \

- --verbose

- ```

-

- ## Error Handling

-

- - **Redis Unavailable**: Graceful exit with error message

- - **No Keys Found**: Warning message, continue processing

- - **Invalid JSON**: Fallback to basic text extraction

- - **Permission Issues**: Directory creation with appropriate permissions

-

- ## Performance Considerations

-

- - **Large Key Sets**: Processes keys in batches to avoid memory issues

- - **Concurrent Access**: Uses read-only Redis operations

- - **File Size**: JSON files are compressed and structured efficiently

- - **Extraction Time**: Typically <5 seconds per task

-

- ## Validation

-

- The skill includes built-in validation:

- - Redis connectivity check

- - JSON structure validation

- - Data completeness verification

- - Output file creation confirmation

-

- ## Maintenance

-

- - **Version Tracking**: Each extraction includes version metadata

- - **Backward Compatibility**: Supports multiple extraction formats

- - **Cleanup**: Old files can be archived or cleaned up automatically

- - **Monitoring**: Integration with system monitoring for extraction success rates

-

- ## Related Skills

-

- - `cfn-redis-coordination` - For understanding Redis coordination patterns

- - `cfn-loop-validation` - For validating loop execution

- - `cfn-agent-spawning` - For understanding agent lifecycle

-

- ## Dependencies

-

- - `redis-cli` - Redis command-line interface

- - `jq` - JSON processor for data manipulation

- - `bc` - Basic calculator for confidence averaging