Skip to content

Instantly share code, notes, and snippets.

@jzollerneon

jzollerneon/.block

Last active October 12, 2017 19:44
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jzollerneon/9700c4908bebbd5b5e546c0cd4decc4c to your computer and use it in GitHub Desktop.
Save jzollerneon/9700c4908bebbd5b5e546c0cd4decc4c to your computer and use it in GitHub Desktop.
Download ZIP
Small mammal capture heatmap
Raw
.block
license: mit
Raw
README.md

Built with blockbuilder.org

When NEON was first starting to offer small mammal box trapping data, I wanted a way to look at capture distributions. So, I put together a small D3.js visualization that showed a capture distribution heat map by counting trap locations from the capture file.

This code updates the visualization to use the NEON Data API to grab current data from the portal. The code queries the /products/DP1.10072.001 endpoint to see what data is available for small mammal capture. It uses that to populate a dropdown with site codes, and requests a list of months available for the currently selected site code. From there, it requests a list of files for each month, then pulls the capture data file for that month. For each site, it stitches the file contents together, then splits them up by plot, and creates the heatmaps.

Click the "Open" icon to the right to see this full screen!.

Raw
index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Small mammal capture heatmap</title>
<script src="https://code.jquery.com/jquery-3.1.1.min.js"
integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>
<script src="https://d3js.org/d3.v4.js"></script>
<script src="smammal.js"></script>
</head>
<body>
<select id="sites"></select>
<div id="smammal"></div>
<style>
.grid {
display: inline;
}
#desc {
padding: 10px;
}
select {
margin: 20px;
border: 1px solid #111;
background: transparent;
width: 150px;
padding: 5px;
font-size: 20px;
height: 34px;
/*-webkit-appearance: none;*/
/*-moz-appearance: none;*/
/*appearance: none;*/
}
</style>
<script>
var SERVER = "http://data.neonscience.org/api/v0/";
var SMAMMAL = "DP1.10072.001";
var availableData = {};
var nestedData = {};
var smammalGrid = smammal.smammalGrid();
//when the page loads, grab the list of all small mammal data from the /products endpoint
var productsUrl = SERVER + "products/" + SMAMMAL;
$.getJSON(productsUrl, function (result) {
result.data.siteCodes.forEach(function (d) {
availableData[d.siteCode] = [];
d.availableMonths.forEach(function (e) {
availableData[d.siteCode].push(e);
});
});
//populate the list of sites and months
var siteOptions = $("#sites");
$.each(Object.keys(availableData).sort(), function () {
siteOptions.append($("<option />").val(this).text(this));
});
//get the data for the first siteCode on the list
getData($("#sites").val());
});
//get the capture data for a given siteCode
function getData(siteCode) {
var combinedData = [];
var urlList = [];
//queue up the availability request for each month, and treat all the results in the awaitAll function
var q = d3.queue();
availableData[siteCode].forEach(function (month) {
var requestUrl = SERVER + "data/" + SMAMMAL + "/" + siteCode + "/" + month;
q.defer(d3.json, requestUrl);
});
//handle all of the available data queries
q.awaitAll(function (availableError, availableResult) {
availableResult.forEach(function (d, index, arr) {
for (var i = 0; i < d.data.files.length; i++) {
var filename = d.data.files[i].url;
if (filename.includes("pertrapnight") && !filename.includes("expanded")) {
urlList.push(d.data.files[i].url);
}
}
});
//queue up the files to be requested
var r = d3.queue();
urlList.forEach(function(e) {
r.defer(d3.csv, e);
});
//handle all of the downloaded files
r.awaitAll(function(dataError, dataResult) {
dataResult.forEach(function (fileResult) {
fileResult.forEach(function(row) {
//trapCoordinates are stored without zero or X padding, so make sure it's in there
row.trapCoordinate = pad(row.trapCoordinate);
combinedData.push(row);
});
});
//break the data down by plotID
nestedData = d3.nest().key(function(f) { return f.plotID; }).entries(combinedData);
nestedData = nestedData.sort(function(a,b) {
if (a.key > b.key) { return 1; } else { return -1;}
});
//build a grid for each plotId
nestedData.forEach(function(dataSet) {
var smammalDiv = $("#smammal");
$('<div class="grid" id="' + dataSet.key + '"></div>').appendTo(smammalDiv);
d3.select('#' + dataSet.key)
.datum(dataSet.values)
.call(smammalGrid
.width(300)
.height(300)
.title(dataSet.key));
});
});
});
}
//pad location with 0 or X as required
function pad(padstr) {
if (padstr.length == 2) {
if (padstr.substring(1, 2) !== 'X') {
padstr = padstr.substring(0, 1) + '0' + padstr.substring(1, 2);
} else {
padstr = padstr.substring(0, 1) + 'X' + padstr.substring(1, 2);
}
}
return padstr;
}
//events
$("#sites").change(function () {
$("#smammal").empty();
$("#desc").empty();
getData(this.value);
});
</script>
</body>
</html>
Raw
smammal.js
var smammal = {
smammalGrid: function smammalGrid() {
var rows = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'X'],
cols = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', 'X'],
margin = {top: 30, right: 10, bottom: 10, left: 10},
width = 600 - margin.left - margin.right,
height = 600 - margin.top - margin.bottom,
title = '',
gridCount = rows.length,
gridMargin = 2,
colors = ["#ffffd9", "#edf8b1", "#c7e9b4", "#7fcdbb", "#41b6c4", "#1d91c0", "#225ea8", "#253494", "#081d58"],
buckets = colors.length;
function grid(selection) {
//set up retained values
var closureWidth = width,
closureHeight = height,
closureTitle = title,
gridSize = Math.floor(closureWidth / gridCount),
closureGridMargin = gridMargin,
rowColsObj = {},
rowColsArr = [];
//initialize the backing data for each cell
rows.forEach(function (row, rowindex) {
cols.forEach(function (col, colindex) {
var rc = row + col;
rc = pad(rc);
rowColsObj[rc] = {};
rowColsObj[rc].row = row;
rowColsObj[rc].col = col;
rowColsObj[rc].count = 0;
rowColsObj[rc].trapCoordinate = rc;
})
});
//for each input data item
selection.each(function (data) {
//count captures
data.forEach(function (d) {
if (d.trapStatus.indexOf("5") != -1) {
rowColsObj[d.trapCoordinate].count = rowColsObj[d.trapCoordinate].count + 1;
}
});
//map everything out to an array from the object
rowColsArr = Object.keys(rowColsObj).map(function (key) {
return rowColsObj[key];
});
rowColsArr.sort(function (a, b) {
if (a.trapCoordinate < b.trapCoordinate) { return -1;} else { return 1;}
});
//TODO: replace with a constant color scale
var colorScale = d3.scaleQuantile()
.domain([0, buckets - 1, d3.max(rowColsArr, function (d) {
return d.count;
})])
.range(colors);
//build the top level svg
var svg = d3.select('#smammal').append('svg')
.attr('width', closureWidth + margin.left + margin.right)
.attr('height', closureHeight + margin.top + margin.bottom)
.append('g')
.attr('transform', 'translate(' + margin.left + ', ' + margin.top + ')');
//build the grid
var ycount = -1;
svg.selectAll('rect')
.data(rowColsArr)
.enter()
.append('rect')
.attr('x', function (d, i) {
return i % gridCount * gridSize;
})
.attr('y', function (d, i) {
if (i % gridCount === 0) ycount = ycount + 1;
return ycount * gridSize;
})
.attr('rx', 3)
.attr('ry', 3)
.attr('width', gridSize - closureGridMargin)
.attr('height', gridSize - closureGridMargin)
.style('fill', function (d) {
return colorScale(d.count);
})
.append('svg:title')
.text(function (d) {
return "Coordinate " + d.trapCoordinate + ': ' + d.count + ' captures';
});
//build the title
svg.append("text")
.attr("x", (width / 2))
.attr("y", 0 - (margin.top / 2))
.attr("text-anchor", "middle")
.style("font-size", "20px")
.text(closureTitle);
});
};
grid.width = function (value) {
if (!arguments.length) return width;
width = value;
return grid;
};
grid.height = function (value) {
if (!arguments.length) return height;
height = value;
return grid;
};
grid.title = function (value) {
if (!arguments.length) return title;
title = value;
return grid;
};
grid.gridMargin = function (value) {
if (!arguments.length) return gridMargin;
gridMargin = value;
return grid;
};
return grid;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment