Skip to content

Instantly share code, notes, and snippets.

@jyucsiro
Last active October 18, 2017 01:29
Show Gist options
  • Save jyucsiro/f4622381c06d03eccaf8edb07c5a0418 to your computer and use it in GitHub Desktop.
Save jyucsiro/f4622381c06d03eccaf8edb07c5a0418 to your computer and use it in GitHub Desktop.
AusOpenData Survey - Variation by format and counts of datasets
source format counts
www-data-vic-gov-au tabular 2666
www-data-vic-gov-au web document 7
www-data-vic-gov-au API or web service 65
www-data-vic-gov-au other 23
www-data-vic-gov-au compressed 55
www-data-vic-gov-au structured data 5
www-data-vic-gov-au gis 78
www-data-vic-gov-au media 5
data-gov-au web document 2845
data-gov-au document 14490
data-gov-au API or web service 1243
data-gov-au compressed 6700
data-gov-au gis 4759
data-gov-au tabular 4768
data-gov-au media 77
data-gov-au structured data 327
data-gov-au other 443
data-gov-au database 2
data-gov-au code 7
nsw-seed web document 51
nsw-seed structured data 50
nsw-seed document 123
nsw-seed gis 79
nsw-seed API or web service 34
nsw-seed database 1
nsw-seed tabular 12
nsw-oeh compressed 719
nsw-oeh document 3187
nsw-oeh gis 119
nsw-oeh web document 171
nsw-oeh media 1829
nsw-oeh other 1855
nsw-oeh tabular 14
nsw-oeh API or web service 27
data-qld-gov-au tabular 5269
data-qld-gov-au gis 71
data-qld-gov-au document 149
data-qld-gov-au database 37
data-qld-gov-au structured data 299
data-qld-gov-au other 19
data-qld-gov-au web document 11
data-qld-gov-au compressed 24
data-qld-gov-au media 1
data-nsw-gov-au document 216
data-nsw-gov-au tabular 231
data-nsw-gov-au web document 110
data-nsw-gov-au other 60
data-nsw-gov-au gis 24
data-nsw-gov-au structured data 39
data-nsw-gov-au API or web service 18
data-nsw-gov-au compressed 15
data-nsw-gov-au code 3
data-sa-gov-au API or web service 10
data-sa-gov-au gis 732
data-sa-gov-au tabular 321
data-sa-gov-au web document 244
data-sa-gov-au document 112
data-sa-gov-au structured data 26
data-sa-gov-au other 203
data-sa-gov-au media 23
data-sa-gov-au compressed 16
data-sa-gov-au code 1
data-wa-gov-au compressed 332
data-wa-gov-au media 291
data-wa-gov-au web document 52
data-wa-gov-au document 219
data-wa-gov-au other 19
data-wa-gov-au gis 12
data-wa-gov-au tabular 122
data-wa-gov-au API or web service 1
data-wa-gov-au structured data 10
ozflux web document 1
ozflux structured data 88
nci structured data 3253302
nci media 16241
nci other 191376
nci document 10
nci code 69
nci tabular 10008
nci database 57
nci gis 114
nci compressed 31174
aodn compressed 7828
aodn structured data 3404481
tpac structured data 303647
tds-mel-csiro-au structured data 20931
tds-mel-csiro-au other 428
tds-mel-csiro-au web document 5
csiro-dap other 2149694
csiro-dap media 409552
csiro-dap structured data 115347
csiro-dap tabular 28157
csiro-dap database 1201
csiro-dap gis 894
csiro-dap document 6995
csiro-dap compressed 39386
csiro-dap code 2934
csiro-dap web document 1904
<!DOCTYPE html>
<meta charset="utf-8">
<html>
<head>
<style>
rect.bordered {
stroke: #E6E6E6;
stroke-width:2px;
}
text.mono {
font-size: 9pt;
font-family: Consolas, courier;
fill: #aaa;
}
text.axis-workweek {
fill: #000;
}
text.axis-worktime {
fill: #000;
}
</style>
<script src="http://d3js.org/d3.v3.js"></script>
</head>
<body>
<div id="chart"></div>
<script type="text/javascript">
var margin = { top: 50, right: 0, bottom: 100, left: 300 },
width = 960 - margin.left - margin.right,
height = 630 - margin.top - margin.bottom,
gridSize = Math.floor(width / 24),
legendElementWidth = gridSize*2,
buckets = 9,
colors = ["#ffffd9","#edf8b1","#c7e9b4","#7fcdbb","#41b6c4","#1d91c0","#225ea8","#253494","#081d58"], // alternatively colorbrewer.YlGnBu[9]
source_list = ["aodn", "csiro-dap", "data-gov-au", "data-nsw-gov-au", "data-qld-gov-au", "data-sa-gov-au", "data-wa-gov-au", "nci", "nsw-oeh",
"nsw-seed", "ozflux", "tds-mel-csiro-au", "tpac", "www-data-vic-gov-au"]
formats = ["API or web service", "code", "compressed", "database", "document", "gis", "media",
"other","structured data", "tabular", "web document"];
formats_labels = ["API", "code", "z", "db", "doc", "gis", "media",
"otr","str", "csv", "web"];
datasets = ["data_all_count.tsv", "data2.tsv"];
var source_idx = {};
for(var i = 0; i < source_list.length; i++) {
source_idx[source_list[i]] = i;
}
var format_idx = {};
for(var i = 0; i < formats.length; i++) {
format_idx[formats[i]] = i;
}
var svg = d3.select("#chart").append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
var dayLabels = svg.selectAll(".dayLabel")
.data(source_list)
.enter().append("text")
.text(function (d) { return d; })
.attr("x", 0)
.attr("y", function (d, i) { return i * gridSize; })
.style("text-anchor", "end")
.attr("transform", "translate(-6," + gridSize / 1.5 + ")")
.attr("class", function (d, i) { return ((i >= 0 && i <= 4) ? "dayLabel mono axis axis-workweek" : "dayLabel mono axis"); });
var timeLabels = svg.selectAll(".timeLabel")
.data(formats_labels)
.enter().append("text")
.text(function(d) { return d; })
.attr("x", function(d, i) { return i * gridSize; })
.attr("y", 0)
.style("text-anchor", "middle")
.attr("transform", "translate(" + gridSize / 2 + ", -6)")
.attr("class", function(d, i) { return ((i >= 7 && i <= 16) ? "timeLabel mono axis axis-worktime" : "timeLabel mono axis"); });
var heatmapChart = function(tsvFile) {
d3.tsv(tsvFile,
function(d) {
console.log(d);
return { //day, hour, value
source: d.source,
format: d.format,
count: d.counts
};
},
function(error, data) {
//0, 1 mb, 100 mb, 1 gb, 50gb, 100 gb, 500gb, 1tb, 100tb
var domain = [0, 10, 100, 1000, 10000, 100000, 1000000];
var max = d3.max(data, function (d) { return d.count; });
var colorScale = d3.scale.quantile()
.domain([0, buckets - 1, max])
//.domain(domain)
.range(colors);
var cards = svg.selectAll(".hour")
.data(data, function(d) {
console.log(d);
var o = format_idx[d.format]+':'+source_idx[d.source];
return o;
});
cards.append("title");
cards.enter().append("rect")
.attr("x", function(d) { return (format_idx[d.format] ) * gridSize; })
.attr("y", function(d) { return (source_idx[d.source] ) * gridSize; })
.attr("rx", 4)
.attr("ry", 4)
.attr("class", "hour bordered")
.attr("width", gridSize)
.attr("height", gridSize)
.style("fill", colors[0]);
cards.transition().duration(1000)
.style("fill", function(d) { return colorScale(d.count); });
cards.select("title").text(function(d) { return d.count; });
cards.exit().remove();
var legend = svg.selectAll(".legend")
.data([0].concat(colorScale.quantiles()), function(d) { return d; });
legend.enter().append("g")
.attr("class", "legend");
legend.append("rect")
.attr("x", function(d, i) { return legendElementWidth * i; })
.attr("y", height)
.attr("width", legendElementWidth)
.attr("height", gridSize / 2)
.style("fill", function(d, i) { return colors[i]; });
legend.append("text")
.attr("class", "mono")
.text(function(d) { return "≥ " + Math.round(d); })
.attr("x", function(d, i) { return legendElementWidth * i; })
.attr("y", height + gridSize);
legend.exit().remove();
});
};
heatmapChart(datasets[0]);
var datasetpicker = d3.select("#dataset-picker").selectAll(".dataset-button")
.data(datasets);
datasetpicker.enter()
.append("input")
.attr("value", function(d){ return "Dataset " + d })
.attr("type", "button")
.attr("class", "dataset-button")
.on("click", function(d) {
heatmapChart(d);
});
</script>
</body>
</html>

Copyright (c) 2016, Tom May

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment