This program fetches data from the Data Canvas - Sense Your City API.
It shows how the API can be accessed using async.js for asynchronous control flow. Data is fetched for each city separately, then combined, then output as CSV.
The API has a limitation of returning at most 1000 rows.
| // This module provides an API layer above the | |
| // Data Canvas - Sense Your City API described at | |
| // http://map.datacanvas.org/#!/data | |
| define(["jquery", "lodash", "async"], function ($, _, async){ | |
| // See API documentation at http://map.datacanvas.org/#!/data | |
| var API_URL = "http://sensor-api.localdata.com/api/v1/aggregations.csv", | |
| // List of all cities with available data. | |
| cities = ["San Francisco", "Bangalore", "Boston", "Geneva", "Rio de Janeiro", "Shanghai", "Singapore"], | |
| // The default parameters to pass into the API. | |
| defaultParams = { | |
| // Use averaging as the aggregation operator. | |
| op: "mean", | |
| // Include temperature only. | |
| fields: "temperature,light,airquality_raw,sound,humidity,dust" | |
| } | |
| // Fetches the latest data for a given city. | |
| function getDataForCity(options){ | |
| return function(city, callback){ | |
| // Get data for the last 24 hours. | |
| // 1000 milliseconds/second, 60 seconds/minute, 5 minutes | |
| var params = _.extend({ | |
| "over.city": city | |
| }, options, defaultParams); | |
| // Use jQuery to fetch the data. | |
| // jQuery is used here rather than D3 because of its nice parameter syntax. | |
| $.get(API_URL, params, function(csv) { | |
| // Parse the CSV string. | |
| callback(null, d3.csv.parse(csv, function(d){ | |
| // Parse ISO date strings into Date objects. | |
| // d.date = new Date(d.timestamp); | |
| // Parse strings into Numbers for numeric fields. | |
| // d.temperature = +d.temperature; | |
| //d.light = +d.light | |
| //d.airquality_raw = +d.airquality_raw | |
| //d.sound = +d.sound | |
| //d.humidity = +d.humidity | |
| //d.dust = +d.dust | |
| return d; | |
| })); | |
| }); | |
| }; | |
| } | |
| // Fetches the current temperature across all cities. | |
| return function getData(options, callback){ | |
| async.map(cities, getDataForCity(options), function(err, results){ | |
| callback(err, _.flatten(results)); | |
| }); | |
| } | |
| }); |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <meta charset="utf-8"> | |
| <!-- Runs the main program found in main.js. --> | |
| <script data-main="main.js" src="//cdnjs.cloudflare.com/ajax/libs/require.js/2.1.14/require.js"></script> | |
| <!-- Configure RequireJS paths for third party libraries. --> | |
| <script> | |
| requirejs.config({ | |
| paths: { | |
| d3: "//d3js.org/d3.v3.min", | |
| jquery: "//code.jquery.com/jquery-2.1.1.min", | |
| lodash: "//cdnjs.cloudflare.com/ajax/libs/lodash.js/3.4.0/lodash.min", | |
| async: "//cdnjs.cloudflare.com/ajax/libs/async/0.9.0/async" | |
| } | |
| }); | |
| </script> | |
| <!-- Include CSS that styles the visualization. --> | |
| <link rel="stylesheet" href="styles.css"> | |
| <title>Fetching Data</title> | |
| </head> | |
| <body> | |
| <!-- The visualization will be injected into this div. --> | |
| <p> | |
| day.csv - past day, every 5 minutes | |
| <span class="loader" id="day-loading"><img src="ajax-loader.gif"></span> <br> | |
| <textarea id="day" rows="15"></textarea> | |
| </p> | |
| <p> | |
| week.csv - past week, every hour | |
| <span class="loader" id="week-loading"><img src="ajax-loader.gif"></span> <br> | |
| <textarea id="week" rows="15"></textarea> | |
| </p> | |
| <p> | |
| month.csv - past month, every hour | |
| <span class="loader" id="month-loading"><img src="ajax-loader.gif"></span> <br> | |
| <textarea id="month" rows="15"></textarea> | |
| </p> | |
| <p> | |
| all.csv - all available data from the API, every 5 minutes | |
| <span class="loader" id="all-loading"> | |
| <img src="ajax-loader.gif"> | |
| <span id="all-counter"></span> | |
| </span> <br> | |
| <textarea id="all" rows="15"></textarea> | |
| </p> | |
| </body> | |
| </html> |
| // This is the main program that fetches Data Canvas - Sense Your City API. | |
| // Curran Kelleher March 2015 | |
| require(["d3", "getData", "async"], function (d3, getData, async) { | |
| var now = Date.now(), | |
| pastDay = now - 1000 * 60 * 60 * 24, | |
| pastWeek = now - 1000 * 60 * 60 * 24 * 7, | |
| pastMonth = now - 1000 * 60 * 60 * 24 * 7 * 4; | |
| async.series([ | |
| function(callback){ | |
| d3.select("#day-loading").style("visibility", "visible"); | |
| getData({ | |
| resolution: "5m", | |
| from: new Date(pastDay).toISOString(), | |
| before: new Date(now).toISOString(), | |
| },function(err, data){ | |
| d3.select("#day").text(toCSV(data)); | |
| d3.select("#day-loading").style("visibility", "hidden"); | |
| d3.select("#week-loading").style("visibility", "visible"); | |
| callback(); | |
| }); | |
| }, | |
| function(callback){ | |
| getData({ | |
| resolution: "1h", | |
| from: new Date(pastWeek).toISOString(), | |
| before: new Date(now).toISOString(), | |
| },function(err, data){ | |
| d3.select("#week").text(toCSV(data)); | |
| d3.select("#week-loading").style("visibility", "hidden"); | |
| d3.select("#month-loading").style("visibility", "visible"); | |
| callback(); | |
| }); | |
| }, | |
| function(callback){ | |
| getData({ | |
| resolution: "1h", | |
| from: new Date(pastMonth).toISOString(), | |
| before: new Date(now).toISOString(), | |
| },function(err, data){ | |
| d3.select("#month").text(toCSV(data)); | |
| d3.select("#month-loading").style("visibility", "hidden"); | |
| callback(); | |
| }); | |
| }, | |
| function(callback){ | |
| // 1 day in milliseconds. | |
| var msInterval = 1000 * 60 * 60 * 24, | |
| // Start fetching data from right now. | |
| msTime = Date.now(), | |
| // Get data from back in time 100 days. | |
| intervalCountMax = 100, | |
| // The running count of days we've fetched data for. | |
| intervalCount = 0, | |
| // The cumulative data as an array of objects. | |
| data = []; | |
| d3.select("#all-loading").style("visibility", "visible"); | |
| async.whilst( | |
| function () { return intervalCount < intervalCountMax; }, | |
| function (cb) { | |
| getData({ | |
| resolution: "5m", | |
| from: new Date(msTime - msInterval * (intervalCount + 1)).toISOString(), | |
| before: new Date(msTime - msInterval * intervalCount).toISOString() | |
| },function(err, newData){ | |
| data = _.sortBy(_.union(data, newData), "timestamp"); | |
| d3.select("#all-counter").text([ | |
| "Fetched " + data.length + " records", | |
| "over " + (intervalCount + 1) + " of " + intervalCountMax + " days" | |
| ].join("\n")); | |
| intervalCount++; | |
| cb(); | |
| }); | |
| }, function(err){ | |
| d3.select("#all").text(toCSV(data)); | |
| d3.select("#all-loading").style("visibility", "hidden"); | |
| console.log("Done fetching data"); | |
| } | |
| ); | |
| callback(); | |
| } | |
| ]); | |
| function toCSV(data){ | |
| var columns = Object.keys(data[0]); | |
| return [columns.join(",")].concat(data.map(function(d){ | |
| return columns.map(function(column){ | |
| return d[column]; | |
| }).join(","); | |
| })).join("\n"); | |
| } | |
| }); |
| /* Make the visualization container fill the page. */ | |
| textarea{ | |
| width: 100%; | |
| } | |
| .loader { | |
| visibility: hidden; | |
| } |
