Copied in part from https://bl.ocks.org/emeeks/21f99959d48dd0d0c746
This is a high level visualization of NEON's data flow for Barometric Pressure at Sea Level and the data products it is dependent on.
This was done in a few hours to get an idea of the scope of the processing needed to get data from sensor to the portal. It does not include data access sub-systems, asset tracking, calibration, etc.
| d3.sankey = function() { | |
| var sankey = {}, | |
| nodeWidth = 24, | |
| nodePadding = 8, | |
| size = [1, 1], | |
| nodes = [], | |
| links = []; | |
| sankey.nodeWidth = function(_) { | |
| if (!arguments.length) return nodeWidth; | |
| nodeWidth = +_; | |
| return sankey; | |
| }; | |
| sankey.nodePadding = function(_) { | |
| if (!arguments.length) return nodePadding; | |
| nodePadding = +_; | |
| return sankey; | |
| }; | |
| sankey.nodes = function(_) { | |
| if (!arguments.length) return nodes; | |
| nodes = _; | |
| return sankey; | |
| }; | |
| sankey.links = function(_) { | |
| if (!arguments.length) return links; | |
| links = _; | |
| return sankey; | |
| }; | |
| sankey.size = function(_) { | |
| if (!arguments.length) return size; | |
| size = _; | |
| return sankey; | |
| }; | |
| sankey.layout = function(iterations) { | |
| computeNodeLinks(); | |
| computeNodeValues(); | |
| computeNodeBreadths(); | |
| computeNodeDepths(iterations); | |
| computeLinkDepths(); | |
| return sankey; | |
| }; | |
| sankey.relayout = function() { | |
| computeLinkDepths(); | |
| return sankey; | |
| }; | |
| sankey.link = function() { | |
| var curvature = .5; | |
| function link(d) { | |
| var x0 = d.source.x + d.source.dx, | |
| x1 = d.target.x, | |
| xi = d3.interpolateNumber(x0, x1), | |
| x2 = xi(curvature), | |
| x3 = xi(1 - curvature), | |
| y0 = d.source.y + d.sy + d.dy / 2, | |
| y1 = d.target.y + d.ty + d.dy / 2; | |
| return "M" + x0 + "," + y0 | |
| + "C" + x2 + "," + y0 | |
| + " " + x3 + "," + y1 | |
| + " " + x1 + "," + y1; | |
| } | |
| link.curvature = function(_) { | |
| if (!arguments.length) return curvature; | |
| curvature = +_; | |
| return link; | |
| }; | |
| return link; | |
| }; | |
| // Populate the sourceLinks and targetLinks for each node. | |
| // Also, if the source and target are not objects, assume they are indices. | |
| function computeNodeLinks() { | |
| nodes.forEach(function(node) { | |
| node.sourceLinks = []; | |
| node.targetLinks = []; | |
| }); | |
| links.forEach(function(link) { | |
| var source = link.source, | |
| target = link.target; | |
| if (typeof source === "number") source = link.source = nodes[link.source]; | |
| if (typeof target === "number") target = link.target = nodes[link.target]; | |
| source.sourceLinks.push(link); | |
| target.targetLinks.push(link); | |
| }); | |
| } | |
| // Compute the value (size) of each node by summing the associated links. | |
| function computeNodeValues() { | |
| nodes.forEach(function(node) { | |
| node.value = Math.max( | |
| d3.sum(node.sourceLinks, value), | |
| d3.sum(node.targetLinks, value) | |
| ); | |
| }); | |
| } | |
| // Iteratively assign the breadth (x-position) for each node. | |
| // Nodes are assigned the maximum breadth of incoming neighbors plus one; | |
| // nodes with no incoming links are assigned breadth zero, while | |
| // nodes with no outgoing links are assigned the maximum breadth. | |
| function computeNodeBreadths() { | |
| var remainingNodes = nodes, | |
| nextNodes, | |
| x = 0; | |
| while (remainingNodes.length) { | |
| nextNodes = []; | |
| remainingNodes.forEach(function(node) { | |
| node.x = x; | |
| node.dx = nodeWidth; | |
| node.sourceLinks.forEach(function(link) { | |
| if (nextNodes.indexOf(link.target) < 0) { | |
| nextNodes.push(link.target); | |
| } | |
| }); | |
| }); | |
| remainingNodes = nextNodes; | |
| ++x; | |
| } | |
| // | |
| moveSinksRight(x); | |
| scaleNodeBreadths((size[0] - nodeWidth) / (x - 1)); | |
| } | |
| function moveSourcesRight() { | |
| nodes.forEach(function(node) { | |
| if (!node.targetLinks.length) { | |
| node.x = d3.min(node.sourceLinks, function(d) { return d.target.x; }) - 1; | |
| } | |
| }); | |
| } | |
| function moveSinksRight(x) { | |
| nodes.forEach(function(node) { | |
| if (!node.sourceLinks.length) { | |
| node.x = x - 1; | |
| } | |
| }); | |
| } | |
| function scaleNodeBreadths(kx) { | |
| nodes.forEach(function(node) { | |
| node.x *= kx; | |
| }); | |
| } | |
| function computeNodeDepths(iterations) { | |
| var nodesByBreadth = d3.nest() | |
| .key(function(d) { return d.x; }) | |
| .sortKeys(d3.ascending) | |
| .entries(nodes) | |
| .map(function(d) { return d.values; }); | |
| // | |
| initializeNodeDepth(); | |
| resolveCollisions(); | |
| for (var alpha = 1; iterations > 0; --iterations) { | |
| relaxRightToLeft(alpha *= .99); | |
| resolveCollisions(); | |
| relaxLeftToRight(alpha); | |
| resolveCollisions(); | |
| } | |
| function initializeNodeDepth() { | |
| var ky = d3.min(nodesByBreadth, function(nodes) { | |
| return (size[1] - (nodes.length - 1) * nodePadding) / d3.sum(nodes, value); | |
| }); | |
| nodesByBreadth.forEach(function(nodes) { | |
| nodes.forEach(function(node, i) { | |
| node.y = i; | |
| node.dy = node.value * ky; | |
| }); | |
| }); | |
| links.forEach(function(link) { | |
| link.dy = link.value * ky; | |
| }); | |
| } | |
| function relaxLeftToRight(alpha) { | |
| nodesByBreadth.forEach(function(nodes, breadth) { | |
| nodes.forEach(function(node) { | |
| if (node.targetLinks.length) { | |
| var y = d3.sum(node.targetLinks, weightedSource) / d3.sum(node.targetLinks, value); | |
| node.y += (y - center(node)) * alpha; | |
| } | |
| }); | |
| }); | |
| function weightedSource(link) { | |
| return center(link.source) * link.value; | |
| } | |
| } | |
| function relaxRightToLeft(alpha) { | |
| nodesByBreadth.slice().reverse().forEach(function(nodes) { | |
| nodes.forEach(function(node) { | |
| if (node.sourceLinks.length) { | |
| var y = d3.sum(node.sourceLinks, weightedTarget) / d3.sum(node.sourceLinks, value); | |
| node.y += (y - center(node)) * alpha; | |
| } | |
| }); | |
| }); | |
| function weightedTarget(link) { | |
| return center(link.target) * link.value; | |
| } | |
| } | |
| function resolveCollisions() { | |
| nodesByBreadth.forEach(function(nodes) { | |
| var node, | |
| dy, | |
| y0 = 0, | |
| n = nodes.length, | |
| i; | |
| // Push any overlapping nodes down. | |
| nodes.sort(ascendingDepth); | |
| for (i = 0; i < n; ++i) { | |
| node = nodes[i]; | |
| dy = y0 - node.y; | |
| if (dy > 0) node.y += dy; | |
| y0 = node.y + node.dy + nodePadding; | |
| } | |
| // If the bottommost node goes outside the bounds, push it back up. | |
| dy = y0 - nodePadding - size[1]; | |
| if (dy > 0) { | |
| y0 = node.y -= dy; | |
| // Push any overlapping nodes back up. | |
| for (i = n - 2; i >= 0; --i) { | |
| node = nodes[i]; | |
| dy = node.y + node.dy + nodePadding - y0; | |
| if (dy > 0) node.y -= dy; | |
| y0 = node.y; | |
| } | |
| } | |
| }); | |
| } | |
| function ascendingDepth(a, b) { | |
| return a.y - b.y; | |
| } | |
| } | |
| function computeLinkDepths() { | |
| nodes.forEach(function(node) { | |
| node.sourceLinks.sort(ascendingTargetDepth); | |
| node.targetLinks.sort(ascendingSourceDepth); | |
| }); | |
| nodes.forEach(function(node) { | |
| var sy = 0, ty = 0; | |
| node.sourceLinks.forEach(function(link) { | |
| link.sy = sy; | |
| sy += link.dy; | |
| }); | |
| node.targetLinks.forEach(function(link) { | |
| link.ty = ty; | |
| ty += link.dy; | |
| }); | |
| }); | |
| function ascendingSourceDepth(a, b) { | |
| return a.source.y - b.source.y; | |
| } | |
| function ascendingTargetDepth(a, b) { | |
| return a.target.y - b.target.y; | |
| } | |
| } | |
| function center(node) { | |
| return node.y + node.dy / 2; | |
| } | |
| function value(link) { | |
| return link.value; | |
| } | |
| return sankey; | |
| }; |
| source | target | value | text | |
|---|---|---|---|---|
| Site 1, Sensor 1 | Site 1, GRAPE 1 | 1 | Analog or digital signal | |
| Site 1, Sensor 2 | Site 1, GRAPE 1 | 1 | Analog or digital signal | |
| Site 1, Smart Sensor 1 | Location Controller 1 | 1 | Digitized measurement direct to the Location Controller via TCP-IP | |
| Site 1, Sensor 3 | Site 1, GRAPE 2 | 1 | Analog or digital signal | |
| Site 1, Sensor 4 | Site 1, GRAPE 3 | 1 | Analog or digital signal | |
| Site 1, Sensor 5 | Site 1, GRAPE 3 | 1 | Analog or digital signal | |
| Site 1, Sensor 6 | Site 1, GRAPE 4 | 1 | Analog or digital signal | |
| Site 1, Sensor 7 | Site 1, GRAPE 5 | 1 | Analog or digital signal | |
| Site 1, GRAPE 1 | Location Controller 1 | 2 | Digitizied measurement to the Location Controller via TCP-IP (RTU) | |
| Site 1, GRAPE 2 | Location Controller 1 | 1 | Digitizied measurement to the Location Controller via TCP-IP (RTU) | |
| Site 1, GRAPE 3 | Location Controller 1 | 2 | Digitizied measurement to the Location Controller via TCP-IP (RTU) | |
| Site 1, GRAPE 4 | Location Controller 1 | 1 | Digitizied measurement to the Location Controller via TCP-IP (RTU) | |
| Site 1, GRAPE 5 | Location Controller 1 | 1 | Digitizied measurement to the Location Controller via TCP-IP (RTU) | |
| Location Controller 1 | Site Queue 1 | 8 | ~900 measurement readouts from Ring Buffer to XML payload (bin2xml) | |
| Site 2, Sensor 1 | Site 2, GRAPE 1 | 1 | Analog or digital signal | |
| Site 2, GRAPE 1 | Location Controller 2 | 1 | Digitizied measurement to the Location Controller via TCP-IP (RTU) | |
| Location Controller 2 | Site Queue 2 | 1 | ~900 measurement readouts from Ring Buffer to XML payload (bin2xml) | |
| Site 3, Sensor 1 | Site 3, GRAPE 1 | 1 | Analog or digital signal | |
| Site 3, GRAPE 1 | Location Controller 3 | 1 | Digitizied measurement to the Location Controller via TCP-IP (RTU) | |
| Location Controller 3 | Site Queue 3 | 1 | ~900 measurement readouts from Ring Buffer to XML payload (bin2xml) | |
| Site Queue 1 | TIS HQ Queue | 8 | Bridged JMS queue | |
| Site Queue 2 | TIS HQ Queue | 1 | Bridged JMS queue | |
| Site Queue 3 | TIS HQ Queue | 1 | Bridged JMS queue | |
| TIS HQ Queue | L0 | 10 | Site queues aggregated to individual queue (tis.hq) | |
| L0 | Humidity L0 | 4 | TIS Message processing service (Catalog of TIS messages in PDR database) | |
| L0 | BP L0 | 3 | TIS Message processing service (Catalog of TIS messages in PDR database) | |
| Humidity L0 | Humidity L1 | 16 | TIS Transition (L0 to L1) | |
| Humidity L1 | BPSL L1 | 16 | TIS Transition (L1 to L1) | |
| Humidity L1 | Portal Database | 8 | Golden Gate Replication | |
| Portal Database | Portal File Cache | 8 | Portal Caching Process (data pull) | |
| BPSL L1 | Portal Database | 20 | Golden Gate Replication | |
| Portal Database | Portal File Cache | 10 | Portal Caching Process (data pull) | |
| BP L0 | BP L1 | 4 | TIS Transition (L0 to L1) | |
| BP L1 | BPSL L1 | 4 | TIS Transition (L1 to L1) | |
| Portal File Cache | Web Portal | 10 | Data is available at http://data/neonscience.org/ | |
| Portal File Cache | API | 10 | Data is available at http://data.neonscience.org/data-api |
| <html> | |
| <body> | |
| <script> | |
| </script> | |
| </body> | |
| </html><!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8" /> | |
| <title>Sankey Particles</title> | |
| <style> | |
| .node rect { | |
| cursor: move; | |
| fill-opacity: .9; | |
| shape-rendering: crispEdges; | |
| } | |
| .node text { | |
| pointer-events: none; | |
| text-shadow: 0 1px 0 #fff; | |
| } | |
| .link { | |
| fill: none; | |
| stroke: #000; | |
| stroke-opacity: .05; | |
| } | |
| .link:hover { | |
| stroke-opacity: .25; | |
| } | |
| svg { | |
| position: absolute; | |
| } | |
| canvas { | |
| position: absolute; | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <canvas width="2000" height="2000" ></canvas> | |
| <svg width="2000" height="2000" ></svg> | |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.16/d3.min.js" charset="utf-8" type="text/javascript"></script> | |
| <script src="d3.sankey.js" charset="utf-8" type="text/javascript"></script> | |
| <script type="text/javascript"> | |
| var margin = {top: 1, right: 1, bottom: 6, left: 1}, | |
| width = 1500 - margin.left - margin.right, | |
| height = 800 - margin.top - margin.bottom; | |
| var formatNumber = d3.format(",.0f"), | |
| format = function(d) { return formatNumber(d) + " TWh"; }, | |
| color = d3.scale.category20(); | |
| var svg = d3.select("svg") | |
| .attr("width", width + margin.left + margin.right) | |
| .attr("height", height + margin.top + margin.bottom) | |
| .append("g") | |
| .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); | |
| var sankey = d3.sankey() | |
| .nodeWidth(15) | |
| .nodePadding(50) | |
| .size([width, height]); | |
| var path = sankey.link(); | |
| var freqCounter = 1; | |
| d3.csv("flow.csv", function(error, data) { | |
| //set up graph in same style as original example but empty | |
| graph = {"nodes": [], "links": []}; | |
| data.forEach(function (d) { | |
| graph.nodes.push({"name": d.source}); | |
| graph.nodes.push({"name": d.target}); | |
| graph.links.push({ | |
| "source": d.source, | |
| "target": d.target, | |
| "value": +d.value, | |
| "text": d.text, | |
| }); | |
| }); | |
| // return only the distinct / unique nodes | |
| graph.nodes = d3.keys(d3.nest() | |
| .key(function (d) { | |
| return d.name; | |
| }) | |
| .map(graph.nodes)); | |
| // loop through each link replacing the text with its index from node | |
| graph.links.forEach(function (d, i) { | |
| graph.links[i].source = graph.nodes.indexOf(graph.links[i].source); | |
| graph.links[i].target = graph.nodes.indexOf(graph.links[i].target); | |
| }); | |
| //now loop through each nodes to make nodes an array of objects | |
| // rather than an array of strings | |
| graph.nodes.forEach(function (d, i) { | |
| graph.nodes[i] = {"name": d}; | |
| }); | |
| sankey | |
| .nodes(graph.nodes) | |
| .links(graph.links) | |
| .layout(32); | |
| var link = svg.append("g").selectAll(".link") | |
| .data(graph.links) | |
| .enter().append("path") | |
| .attr("class", "link") | |
| .attr("d", path) | |
| .style("stroke-width", function(d) { return Math.max(1, d.dy); }) | |
| .sort(function(a, b) { return b.dy - a.dy; }); | |
| link.append("title") | |
| .text(function(d) { return d.text; }); | |
| var node = svg.append("g").selectAll(".node") | |
| .data(graph.nodes) | |
| .enter().append("g") | |
| .attr("class", "node") | |
| .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; }) | |
| .call(d3.behavior.drag() | |
| .origin(function(d) { return d; }) | |
| .on("dragstart", function() { this.parentNode.appendChild(this); }) | |
| .on("drag", dragmove)); | |
| node.append("rect") | |
| .attr("height", function(d) { return d.dy; }) | |
| .attr("width", sankey.nodeWidth()) | |
| .style("fill", function(d) { return d.color = color(d.name.replace(/ .*/, "")); }) | |
| .style("stroke", "none") | |
| .append("title") | |
| .text(function(d) { return d.name + "\n" + format(d.value); }); | |
| node.append("text") | |
| .attr("x", -6) | |
| .attr("y", function(d) { return d.dy / 2; }) | |
| .attr("dy", ".35em") | |
| .attr("text-anchor", "end") | |
| .attr("transform", null) | |
| .text(function(d) { return d.name; }) | |
| .filter(function(d) { return d.x < width / 2; }) | |
| .attr("x", 6 + sankey.nodeWidth()) | |
| .attr("text-anchor", "start"); | |
| function dragmove(d) { | |
| d3.select(this).attr("transform", "translate(" + d.x + "," + (d.y = Math.max(0, Math.min(height - d.dy, d3.event.y))) + ")"); | |
| sankey.relayout(); | |
| link.attr("d", path); | |
| } | |
| var linkExtent = d3.extent(graph.links, function (d) {return d.value}); | |
| var frequencyScale = d3.scale.linear().domain(linkExtent).range([1,100]); | |
| var particleSize = d3.scale.linear().domain(linkExtent).range([1,5]); | |
| graph.links.forEach(function (link) { | |
| link.freq = frequencyScale(link.value); | |
| link.particleSize = particleSize(link.value); | |
| link.particleColor = d3.scale.linear().domain([1,1000]).range([link.source.color, link.target.color]); | |
| }) | |
| var t = d3.timer(tick, 1000); | |
| var particles = []; | |
| function tick(elapsed, time) { | |
| particles = particles.filter(function (d) {return d.time > (elapsed - 1000)}); | |
| if (freqCounter > 100) { | |
| freqCounter = 1; | |
| } | |
| d3.selectAll("path.link") | |
| .each( | |
| function (d) { | |
| if (d.freq >= freqCounter) { | |
| var offset = (Math.random() - .5) * d.dy; | |
| particles.push({link: d, time: elapsed, offset: offset, path: this}) | |
| } | |
| }); | |
| particleEdgeCanvasPath(elapsed); | |
| freqCounter++; | |
| } | |
| function particleEdgeCanvasPath(elapsed) { | |
| var context = d3.select("canvas").node().getContext("2d") | |
| context.clearRect(0,0,2000,2000); | |
| context.fillStyle = "gray"; | |
| context.lineWidth = "1px"; | |
| for (var x in particles) { | |
| var currentTime = elapsed - particles[x].time; | |
| var currentPercent = currentTime / 1000 * particles[x].path.getTotalLength(); | |
| var currentPos = particles[x].path.getPointAtLength(currentPercent) | |
| context.beginPath(); | |
| context.fillStyle = particles[x].link.particleColor(currentTime); | |
| context.arc(currentPos.x,currentPos.y + particles[x].offset,particles[x].link.particleSize,0,2*Math.PI); | |
| context.fill(); | |
| } | |
| } | |
| }); | |
| </script> | |
| </body> | |
| </html> |