Skip to content

Instantly share code, notes, and snippets.

@postfalk

postfalk/README.md

Last active July 25, 2017 19:02
Show Gist options
  • Save postfalk/2830343c4cfea112fed10282d2fc6962 to your computer and use it in GitHub Desktop.
Save postfalk/2830343c4cfea112fed10282d2fc6962 to your computer and use it in GitHub Desktop.
Download ZIP
Rivers Python Examples
Raw
README.md

Python examples for the Unimpaired Flows API

All Python example are written for Python 2.7.

  1. Install git from https://git-scm.com/ or copy code from below in your own project.
git clone https://gist.github.com/2830343c4cfea112fed10282d2fc6962.git pythonExamples
cd pythonExamples
  1. Install Python dependencies.
pip install -r requirements
  1. Run:
python basic_example.py

Run with jupyter

  1. Start notebook server, a web browser window will open.
jupyter notebook
  1. Open basic_example.ipynb

1. basic_example.py, basic_example.ipynb

Sends a simple request to /api/data/flat/ to demonstrate how to get data for a single stream segment.

2. pagination_example.ipynb

Demonstrates how to use pagination in code.

Display the source blob
Display the rendered blob
Raw
basic_example.ipynb
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Raw
basic_example.py
"""Example creating Pandas data frame from API return.
Dependencies (install with pip): requests, pandas"""
import os
import json
import requests
from pandas.io.json import json_normalize
# API endpoint
FLOW_URL = 'https://rivers.codefornature.org/api/data/flat/'
# You could assign your token also directly,
# please make sure NOT to commit a valid token to a GitHub repository
STATUS_CODES = {
200: 'Success',
400: 'Malformed query',
401: 'Not authorized',
404: 'Not found',
504: 'Timed out. Request smaller batch of data.'}
def get_data(comid):
"""Function retrieving data for a single stream segment
Args:
comid(int): Comid for stream segment
Returns:
dict containing data.
"""
payload = {'where': json.dumps({'stream_segment': comid})}
res = requests.get(FLOW_URL, params=payload)
print 'Requested URL:', res.url
print 'Status:', res.status_code
if res.status_code == 200:
return json.loads(res.text)['results']
try:
print STATUS_CODES[res.status_code]
return
except KeyError:
print 'Something is wrong.'
if __name__ == '__main__':
res = get_data(4438300)
if res:
df = json_normalize(res)
print df
Display the source blob
Display the rendered blob
Raw
pagination_example.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"# Example using pagination in code"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import requests"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"url = 'https://rivers.codefornature.org/api/data/flat/'\n",
"# reduce the returned amount of data somewhat and increase page_size\n",
"params = {'where': '{\"stream_order\": 7}', 'page_size': 50}"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"resp = requests.get(url, params=params)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"response = json.loads(resp.text)\n",
"resp.status_code\n",
"data = response['results']"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"79200 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=1897323&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"119844 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=1909475&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"159444 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2702592&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"200282 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2704754&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"240603 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2706572&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"280203 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2708606&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"319803 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2708746&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"359403 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2709068&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"399003 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2714312&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"438603 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2845782&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"480261 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=2852742&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"521297 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=4442469&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"562385 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=12068855&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"602777 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=12070951&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"642377 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=12074465&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"681977 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=12074715&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"722981 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=15046508&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"764384 data points loaded\n",
"https://rivers.codefornature.org/api/data/flat/?continue=948040204&where=%7B%22stream_order%22%3A+7%7D&page_size=50\n",
"765968 data points loaded\n",
"None\n"
]
}
],
"source": [
"# send new request as long as the next field does not return null and response\n",
"# is valid\n",
"while resp.status_code == 200 and response['next']:\n",
" resp = requests.get(response['next'])\n",
" if resp.status_code != 200:\n",
" print 'Data load error'\n",
" break\n",
" response = json.loads(resp.text)\n",
" data += response['results']\n",
" print '{} data points loaded'.format(len(data))\n",
" print response['next']"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"And finally, just print a random line that is not on the first page."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{u'measurement': u'mean', u'value': 6192.63311519874, u'month': 11, u'stream_segment': 2704753, u'year': 2012, u'variable': u'estimated', u'gauge': None}\n"
]
}
],
"source": [
"print data[200000]"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Raw
requirements.txt
# requirements for running Python 2.7 examples
# install with
# pip install -r requirements.txt
jupyter==1.0.0
requests==2.13.0
pandas==0.19.2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment