Skip to content

Instantly share code, notes, and snippets.

@TheMapSmith
Last active March 17, 2024 15:51
Show Gist options
  • Save TheMapSmith/33277dea01b8ae8140af85c4406ba2a1 to your computer and use it in GitHub Desktop.
Save TheMapSmith/33277dea01b8ae8140af85c4406ba2a1 to your computer and use it in GitHub Desktop.
Fetching flight info
var fs = require('fs');
var request = require('request-promise');
var moment = require('moment')
// Globals
global.timestamp = moment().unix()
global.allPlaybacks = [];
global.geojson = {};
global.geojson['type'] = 'FeatureCollection';
global.geojson['features'] = [];
global.code = "DFW"
global.folder = code + "/"
global.types = ["arrivals", "departures"]
global.FlightIds = [] // array for all IDs found
// first call
makeFolder(getFlights);
function makeFolder(callback) {
fs.stat(code, function(err, stats) {
if (err) {
// Directory doesn't exist or something.
console.log('Folder doesn\'t exist, so I made the folder ' + code);
return fs.mkdir(folder, callback);
}
if (!stats.isDirectory()) {
// This isn't a directory!
callback(new Error('temp is not a directory!'));
} else {
console.log('Folder ' + code + ' exists');
getFlights(code, types);
}
});
}
function getFlights() {
// API URL string slugs
var uriSource = [
'https://api.flightradar24.com/common/v1/airport.json?code=',
'&plugin[]=&plugin-setting[schedule][mode]=',
'&plugin-setting[schedule][timestamp]=',
'&page=-1&limit=100&token='
]
// check if we have a file
types.forEach(function(type) {
var airportFile = folder + code + "_" + type + ".json"
// if we don't have an airport file yet
if (!fs.existsSync(airportFile)) {
// build the request URL
var uri = uriSource[0] + code + uriSource[1] + type + uriSource[2] + timestamp + uriSource[3];
request({
"method": "GET",
"uri": uri,
"headers": {
"User-Agent": "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0.3 Safari/604.5.6",
"origin": "https://www.flightradar24.com",
"referer": "https://www.flightradar24.com/data/flights/ORD"
}
})
.then(function write(response, error) {
if (!error && response.statusCode == 200) {
var filename = code + "_" + type + ".json";
fs.writeFile(folder + filename, response, 'utf8', getIds)
console.log("Successfully fetched " + filename);
// TODO: Actually move on with the program from here
}
})
.catch(function(err) {
console.log("Couldn't fetch " + airportFile + ",code: " + err.statusCode)
})
} else if (type === "departures") {
// only call next function when done with the types array
getIds(airportFile, types, FlightIds, allPlaybacks);
} else if (type === "arrivals") {
console.log("arrivals")
}
})
}
// read the Airport file to extract all flightIds
function getIds(airportFile) {
types.forEach(function(type) {
var airportFile = folder + code + "_" + type + ".json"
fs.readFile(airportFile, 'utf8', function(err, data) {
if (err) throw err;
var obj = JSON.parse(data);
var schedule = obj.result.response.airport.pluginData.schedule
var data = schedule[type].data;
data.forEach(function(e) {
var id = e.flight.identification.id
if (id) {
FlightIds.push(id)
console.log(id);
}
})
console.log(type + " ids: " + FlightIds.length);
if (type === "departures") {
var i
processFlightIds(i, FlightIds);
}
})
})
};
function processFlightIds(i) {
console.log("breakpoint");
for (var i = FlightIds.length - 1; i >= 0; i--) {
// drop FlightIds[i] so that i have a call stack based on each flight id
fetchPlayback(FlightIds[i], i);
}
if (FlightIds.length === 0) {
processTracks(writeGeoJSON)
}
};
// fetch the playback files for each flightId
function fetchPlayback(flightId, i) {
var flightObj = {}
console.log("getting " + flightId);
var uriSource = [
'https://api.flightradar24.com/common/v1/flight-playback.json?flightId=',
'&timestamp=',
'&token='
]
var filename = folder + "flight-playback-" + flightId + ".json";
if (!fs.existsSync(filename)) {
var uri = uriSource[0] + flightId + uriSource[1] + timestamp + uriSource[2]
// TODO: Catch request errors
// TODO: Add delay tool - try npm bottleneck https://stackoverflow.com/questions/43715068/how-to-use-the-bottleneck-npm-module
request({
"method": "GET",
"uri": uri,
"headers": {
"User-Agent": "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0.3 Safari/604.5.6",
"origin": "https://www.flightradar24.com",
"referer": "https://www.flightradar24.com/data/flights/ORD"
}
})
.then(function write(response) {
if (!error && response.statusCode == 200) {
fs.writeFile(filename, response, 'utf8')
// make an array object out of the current flight info and push into a
// master array that will be processed all at once
flightObj.file = filename
flightObj.id = flightId
allPlaybacks.push(flightObj)
// drop a FlightId from the array to whittle down the call stack
FlightIds.pop() // drop the final one
}
})
.catch(function(err) {
console.log("Couldn't fetch " + filename + ",code: " + err.statusCode)
})
} else {
// make an array object out of the current flight info and push into a
// master array that will be processed all at once
flightObj.file = filename
flightObj.id = flightId
allPlaybacks.push(flightObj)
// drop a FlightId from the array to whittle down the call stack
FlightIds.pop() // drop the final one
}
}
// take a Flight Playback json and turn the coordinates into a GeoJSON line
function processTracks(callback) {
// read all the files
console.log("huh")
for (var i = 0; i < allPlaybacks.length; i++) {
if (i === allPlaybacks.length) {
callback(geojson)
}
var onePlayback = allPlaybacks[i];
fs.readFile(onePlayback.file, 'utf8', function(err, data) {
if (err) throw err;
var obj = JSON.parse(data);
// var flightID = obj.result.request.flightId
var track = obj.result.response.data.flight.track;
var coordinates = []
if (track) {
track.forEach(function(e) {
var pair = []
pair.push(e.longitude)
pair.push(e.latitude)
coordinates.push(pair)
})
var feature = {
"type": "Feature",
"geometry": {
"type": "LineString",
"coordinates": coordinates
},
"properties": {}
}
geojson['features'].push(feature);
}
})
}
};
function writeGeoJSON() {
var filename = code + '.geoJSON'
fs.writeFile(filename, JSON.stringify(geojson), 'utf8');
}
@rowanwins
Copy link

Hey @TheMapSmith

Looks like you got there, well done! Couple of minor comments to try and help

  • Make use of qs option when setting up your request.
    • that'll save you from having to do things like var uri = uriSource[0] + flightId + uriSource[1] + timestamp + uriSource[2]
  • Another great option for making requests is a library called axios
    • It let's you set up a base instance of a request and then modify little bits for certain requests, this would save you a bit of duplicated config.
    • Another neat thing from axios which may have helped you is how you can send a few requests off simultantiously and then wait for all their resolutions, see the bottom of this example entitled 'Performing multiple concurrent requests'

Anyway good job, well done!

@lharby
Copy link

lharby commented Mar 2, 2021

Can we have the package.json file 😃 ?

@TheMapSmith
Copy link
Author

Hi Luke, thanks for taking an interest! How did you come across this?

I'd have to dig around to find the source on my old laptop, but I think all that would be in the package.json are the libraries that you see referenced at the top in the require() section.

@lharby
Copy link

lharby commented Mar 3, 2021

Thank you. I think I just started googling flight path data api, and eventually got to an article you wrote 😄

I did try to get it to work with fetch, and so I think I have an example that is correct in terms of structure, but the api call was failing I will come back to it.

I joined a small artistic community a while back, one of the guys there was retrieving flight data from an api. I've always loved flight paths so tried to see if it might be possible to find these and then render them with svg or similar.

Check Langlands and Bell as an artist couple who made some really nice stuff with flight paths (if you don't know them already).

@TheMapSmith
Copy link
Author

Nice, thanks for sharing the artistic inspiration!

http://www.langlandsandbell.com/portfolio-item/air-routes-of-britain-night-day-2000/

The API call was failing

I eventually stopped trying to use FlightRadar24 because the code here was hitting an unpublished API, and my IP eventually go blacklisted by their CDN. Maybe that's the issue you were hitting?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment