{ "id": "ds-drift-13", "meta": { "instanceId": "vorlux-hub" }, "name": "Vorlux AI | Dataset Drift Detector (Daily)", "active": true, "nodes": [ { "id": "a3b4c5d6-0013-4aaa-8013-000000000001", "name": "Daily 4am", "type": "n8n-nodes-base.scheduleTrigger", "typeVersion": 1.2, "position": [220, 300], "parameters": { "rule": { "interval": [ { "field": "hours", "hoursInterval": 24 } ] } } }, { "id": "a3b4c5d6-0013-4aaa-8013-000000000002", "name": "Generate Test Workflows", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [460, 300], "notes": "Generates 5 test workflows using the workflow-agent model and validates each", "parameters": { "mode": "runOnceForAllItems", "jsCode": "const ollamaUrl = $env.OLLAMA_BASE_URL || 'http://localhost:11434';\nconst modelName = 'workflow-agent-v2';\n\nconst testPrompts = [\n 'Create an n8n workflow that monitors RSS feeds every 2 hours and stores new articles',\n 'Build a workflow with a webhook that processes incoming data and sends Discord notifications',\n 'Create a daily scheduled workflow that collects analytics and generates a summary report',\n 'Design a workflow that fetches YouTube video data and creates content projects',\n 'Build an error monitoring workflow that checks service health and alerts on failures'\n];\n\nconst results = [];\nfor (const prompt of testPrompts) {\n try {\n const res = await fetch(ollamaUrl + '/api/generate', {\n method: 'POST',\n headers: {'Content-Type':'application/json'},\n body: JSON.stringify({ model: modelName, prompt, stream: false, options: { temperature: 0.3 } }),\n signal: AbortSignal.timeout(90000)\n });\n const data = await res.json();\n const output = data.response || '';\n \n // Validate\n let score = 0;\n let issues = [];\n let parsed = null;\n \n try { parsed = JSON.parse(output); score += 25; } catch { issues.push('invalid_json'); }\n \n if (parsed) {\n if (Array.isArray(parsed.nodes) && parsed.nodes.length > 0) {\n score += 20;\n const validNodes = parsed.nodes.filter(n => n.type && n.name);\n if (validNodes.length === parsed.nodes.length) score += 15;\n else issues.push('incomplete_nodes: ' + (parsed.nodes.length - validNodes.length) + ' missing fields');\n \n const knownPrefixes = ['n8n-nodes-base.', '@n8n/'];\n const realTypes = parsed.nodes.filter(n => knownPrefixes.some(p => (n.type||'').startsWith(p)));\n if (realTypes.length >= parsed.nodes.length * 0.8) score += 15;\n else issues.push('unknown_types: ' + (parsed.nodes.length - realTypes.length));\n } else issues.push('no_nodes');\n \n if (parsed.connections && Object.keys(parsed.connections).length > 0) score += 15;\n else issues.push('no_connections');\n \n if (parsed.nodes?.some(n => n.position)) score += 10;\n else issues.push('no_positions');\n }\n \n results.push({ prompt: prompt.substring(0, 80), score, issues, nodeCount: parsed?.nodes?.length || 0 });\n } catch (err) {\n results.push({ prompt: prompt.substring(0, 80), score: 0, issues: ['generation_error: ' + String(err).substring(0, 80)] });\n }\n}\n\nconst avgScore = Math.round(results.reduce((s, r) => s + r.score, 0) / results.length);\n\nreturn [{ json: { results, avgScore, modelName, testCount: results.length } }];" } }, { "id": "a3b4c5d6-0013-4aaa-8013-000000000003", "name": "Compare with Baseline", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [700, 300], "notes": "Compares current scores against stored baseline and detects drift", "parameters": { "mode": "runOnceForAllItems", "jsCode": "const current = $input.first().json;\nconst hubUrl = $env.VORLUX_HUB_URL || 'http://localhost:3010';\n\n// Get baseline from Hub\nlet baseline = { avgScore: 70, timestamp: null };\ntry {\n const res = await fetch(hubUrl + '/api/admin/finetune/baseline?model=' + current.modelName, {\n signal: AbortSignal.timeout(10000)\n });\n const data = await res.json();\n if (data.data?.avgScore) baseline = data.data;\n} catch {}\n\nconst drift = baseline.avgScore - current.avgScore;\nconst driftPercentage = baseline.avgScore ? ((drift / baseline.avgScore) * 100).toFixed(1) : 0;\nconst driftDetected = drift > 10; // >10% drop triggers alert\n\n// Store current as new data point\ntry {\n await fetch(hubUrl + '/api/admin/finetune/drift-log', {\n method: 'POST',\n headers: {'Content-Type':'application/json'},\n body: JSON.stringify({\n model: current.modelName,\n score: current.avgScore,\n baseline: baseline.avgScore,\n drift,\n driftPercentage: parseFloat(driftPercentage),\n driftDetected,\n results: current.results,\n timestamp: new Date().toISOString()\n }),\n signal: AbortSignal.timeout(10000)\n });\n} catch {}\n\n// If this is better than baseline and no baseline exists, save as new baseline\nif (!baseline.timestamp || current.avgScore > baseline.avgScore) {\n try {\n await fetch(hubUrl + '/api/admin/finetune/baseline', {\n method: 'POST',\n headers: {'Content-Type':'application/json'},\n body: JSON.stringify({ model: current.modelName, avgScore: current.avgScore, timestamp: new Date().toISOString() }),\n signal: AbortSignal.timeout(10000)\n });\n } catch {}\n}\n\nreturn [{ json: { ...current, baseline: baseline.avgScore, drift, driftPercentage: parseFloat(driftPercentage), driftDetected, needsRetrain: driftDetected } }];" } }, { "id": "a3b4c5d6-0013-4aaa-8013-000000000004", "name": "Drift Detected?", "type": "n8n-nodes-base.if", "typeVersion": 2, "position": [940, 300], "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict" }, "conditions": [ { "leftValue": "={{ $json.driftDetected }}", "rightValue": "true", "operator": { "type": "string", "operation": "equals" } } ], "combinator": "and" } } }, { "id": "a3b4c5d6-0013-4aaa-8013-000000000005", "name": "Trigger Retrain", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [1200, 200], "parameters": { "method": "POST", "url": "={{$env.N8N_BASE_URL || 'http://localhost:5678'}}/webhook/finetune-trigger", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ force: false, reason: 'drift_detected', drift: $json.drift, baseModel: 'qwen2.5:7b', modelName: $json.modelName }) }}", "options": { "timeout": 10000 } }, "notes": "Triggers the fine-tune workflow to retrain the model" }, { "id": "a3b4c5d6-0013-4aaa-8013-000000000006", "name": "Discord Alert", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [1200, 400], "parameters": { "method": "POST", "url": "={{$env.DISCORD_OPS_WEBHOOK}}", "sendBody": true, "specifyBody": "json", "jsonBody": "={\"embeds\":[{\"title\":\"\\u26a0\\ufe0f Model Drift Detected!\",\"description\":\"**Model:** {{ $json.modelName }}\\n**Current Score:** {{ $json.avgScore }}%\\n**Baseline:** {{ $json.baseline }}%\\n**Drift:** -{{ $json.drift }}% ({{ $json.driftPercentage }}% drop)\\n\\nRetraining has been triggered.\\n\\n**Test Results:**\\n{{ $json.results.map(r => (r.score >= 60 ? '\\u2705' : '\\u274c') + ' ' + r.prompt + ' (' + r.score + '%)').join('\\\\n') }}\",\"color\":15548997,\"footer\":{\"text\":\"Daily Drift Detector\"}}]}", "options": { "timeout": 10000 } } }, { "id": "a3b4c5d6-0013-4aaa-8013-000000000007", "name": "Discord OK", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [1200, 500], "parameters": { "method": "POST", "url": "={{$env.DISCORD_OPS_WEBHOOK}}", "sendBody": true, "specifyBody": "json", "jsonBody": "={\"embeds\":[{\"title\":\"Model Drift Check: OK\",\"description\":\"**Model:** {{ $json.modelName }}\\n**Score:** {{ $json.avgScore }}% (baseline: {{ $json.baseline }}%)\\n**Drift:** {{ $json.drift > 0 ? '-' : '+' }}{{ Math.abs($json.drift) }}%\",\"color\":5763719,\"footer\":{\"text\":\"Daily Drift Detector\"}}]}", "options": { "timeout": 10000 } } } ], "connections": { "Daily 4am": { "main": [ [ { "node": "Generate Test Workflows", "type": "main", "index": 0 } ] ] }, "Generate Test Workflows": { "main": [ [ { "node": "Compare with Baseline", "type": "main", "index": 0 } ] ] }, "Compare with Baseline": { "main": [ [ { "node": "Drift Detected?", "type": "main", "index": 0 } ] ] }, "Drift Detected?": { "main": [ [ { "node": "Trigger Retrain", "type": "main", "index": 0 }, { "node": "Discord Alert", "type": "main", "index": 0 } ], [ { "node": "Discord OK", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1", "saveManualExecutions": true, "saveExecutionProgress": true }, "tags": [ { "name": "ai" }, { "name": "dataset" }, { "name": "drift" }, { "name": "monitoring" } ], "versionId": "2" }