Skip to content

Instantly share code, notes, and snippets.

@sanand0
Last active October 5, 2020 04:22
Show Gist options
  • Save sanand0/7243974 to your computer and use it in GitHub Desktop.
Save sanand0/7243974 to your computer and use it in GitHub Desktop.
Pull currency and stock data
#! /usr/bin/env python
"""
Author: Bastin Robin
"""
import os
import logging
from urllib import urlencode
import datetime
import pandas as pd
CURRENCIES = 'AUD BRL CAD CHF CNY EUR GBP HKD ILS INR ISK JPY \
KRW MXN MYR NZD PHP PKR RUB SAR SEK SGD TWD XAG XAU XPT'.split(' ')
STOCKS = '^FTSE ^GSPC ^IXIC'.split(' ')
def get_start_date(end_date):
'''
get the start date from data.csv
'''
if os.path.exists('data.csv'):
data = pd.read_csv('data.csv')
data['Date'] = pd.to_datetime(data["Date"])
return data['Date'].max().date()
else:
return end_date - datetime.timedelta(days=90)
def get_currency(frames, start_date, end_date):
'''
get the currency updated details
'''
base = 'http://www.oanda.com/currency/historical-rates/download?'
for currency in CURRENCIES:
logging.info(currency)
currency_url = base + urlencode({
'quote_currency': currency,
'end_date': end_date.strftime('%Y-%m-%d'),
'start_date': start_date.strftime('%Y-%m-%d'),
'period': 'daily',
'display': 'absolute',
'rate': '0',
'data_range': 'c',
'price': 'mid',
'view': 'graph',
'base_currency_0': 'USD',
'download': '.csv'
})
data = pd.read_csv(currency_url,
skiprows=5, skipfooter=4, names=
['Date', 'Rate', 'x1', 'x2', 'x3', 'x4'])
data['Name'] = currency
frames.append(data[['Date', 'Rate', 'Name']])
def get_stock(frames, start_date, end_date):
'''
get the stock details
'''
for stock in STOCKS:
logging.info(stock)
stock_url = 'http://ichart.finance.yahoo.com/table.csv?' + \
urlencode({
's': stock,
'a': start_date.month - 1,
'b': start_date.day,
'c': start_date.year,
'd': end_date.month - 1,
'e': end_date.day,
'f': end_date.year,
'g': 'd',
'ignore': '.csv',
})
data = pd.read_csv(stock_url)[['Date', 'Close']].rename(columns={\
'Close': 'Rate'})
data['Name'] = stock
frames.append(data[['Date', 'Rate', 'Name']])
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
END_DATE = datetime.date.today()
START_DATE = get_start_date(END_DATE)
DATAFRAME = []
get_currency(DATAFRAME, START_DATE, END_DATE)
get_stock(DATAFRAME, START_DATE, END_DATE)
pd.concat(DATAFRAME).to_csv('data.csv', index=False)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment