Skip to content

Instantly share code, notes, and snippets.

@jsundram

jsundram/get_checkins.py

Last active February 19, 2024 00:30
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jsundram/7d8d4fcb5c5684617f4d496dc8c47347 to your computer and use it in GitHub Desktop.
Save jsundram/7d8d4fcb5c5684617f4d496dc8c47347 to your computer and use it in GitHub Desktop.
Download ZIP
Get my Swarm checkins from Foursquare
Raw
get_checkins.py
import requests
from datetime import datetime
import json
"""
Instructions:
1. go to https://swarmapp.com/history
2. open developer tools in browser, and navigate to the "network" tab.
3. filter for fetch/XHR
4. select a month in the dropdown
5. when you see a hit to the historysearch endpoint, right-click and select "copy as cURL".
6. from the curl command, extract the following parameters:
- userid (comes after /users/ in the request url)
- wsid
- oauth_token
7. save them into the config.json file below:
{
"userid": "...",
"wsid": "...",
"oauth_token": "..."
}
"""
class SwarmApi():
def __init__(self, config):
"""
config is a filename that contains the configuration file described above.
"""
with open(config) as f:
config = json.load(f)
self.wsid = config['wsid']
self.oauth_token = config['oauth_token']
self.url = 'https://api.foursquare.com/v2/users/%s/historysearch' % config['userid']
def __call__(self, start, end, limit):
params = {
'locale': 'en',
'explicit-lang': 'false',
'v': '20240216',
'offset': '0',
'limit': limit,
'm': 'swarm',
'clusters': 'false',
'afterTimestamp': start,
'beforeTimestamp': end,
'sort': 'newestfirst',
'wsid': self.wsid,
'oauth_token': self.oauth_token,
}
headers = {
'authority': 'api.foursquare.com',
'accept': 'application/json, text/javascript, */*; q=0.01',
'accept-language': 'en-US,en;q=0.9',
'dnt': '1',
'origin': 'https://swarmapp.com',
'referer': 'https://swarmapp.com/',
'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'cross-site',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
}
response = requests.get(self.url, headers=headers, params=params)
return json.loads(response.text)
def date2ts(date_str):
date = datetime.strptime(date_str, "%Y-%m-%d")
return int(date.timestamp())
def main():
# get checkins for 2023, I don't check in that much so a month at a time is fine.
dates = ["2023-M-1".replace("M", str(i)) for i in range(1, 13)] + ["2024-1-1"]
timestamps = [date2ts(d) for d in dates]
api = SwarmApi('config.json')
checkins = []
for i, (start, end) in enumerate(zip(timestamps, timestamps[1:])):
r = api(start, end, 500)
checkins.extend(r['response']['checkins']['items'])
print("Month %s: %s checkins (code %s)" % (dates[i], r['response']['checkins']['count'], r['meta']['code']))
with open('checkins_2023.json', 'w') as f:
json.dump(checkins, f, indent=4)
print("wrote %d checkins to checkins_2023.json" % len(checkins))
return checkins
if __name__ == '__main__':
main()
Raw
get_swarm_checkins.py
import requests
from datetime import datetime
import json
"""
Instructions:
1. go to https://swarmapp.com/history
2. open developer tools in browser, and navigate to the "network" tab.
3. filter for fetch/XHR
4. select a month in the dropdown
5. when you see a hit to the historysearch endpoint, right-click and select "copy as cURL".
6. from the curl command, extract the following parameters:
- userid (comes after /users/ in the request url)
- wsid
- oauth_token
7. save them in a config.json file that looks like:
{
"userid": "400000000",
"wsid": "EKD...",
"oauth_token": "K..."
}
"""
class SwarmApi():
def __init__(self, config):
"""
config is a filename that contains the configuration file described above.
"""
with open(config) as f:
config = json.load(f)
self.wsid = config['wsid']
self.oauth_token = config['oauth_token']
self.url = 'https://api.foursquare.com/v2/users/%s/historysearch' % config['userid']
def __call__(self, start, end, limit):
params = {
'locale': 'en',
'explicit-lang': 'false',
'v': '20240216',
'offset': '0',
'limit': limit,
'm': 'swarm',
'clusters': 'false',
'afterTimestamp': start,
'beforeTimestamp': end,
'sort': 'newestfirst',
'wsid': self.wsid,
'oauth_token': self.oauth_token,
}
headers = {
'authority': 'api.foursquare.com',
'accept': 'application/json, text/javascript, */*; q=0.01',
'accept-language': 'en-US,en;q=0.9',
'dnt': '1',
'origin': 'https://swarmapp.com',
'referer': 'https://swarmapp.com/',
'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'cross-site',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
}
response = requests.get(self.url, headers=headers, params=params)
return json.loads(response.text)
def date2ts(date_str):
date = datetime.strptime(date_str, "%Y-%m-%d")
return int(date.timestamp())
def main():
year = 2017 # year the account was started
limit = 500 # max number of checkins per request
outfile = 'checkins_all.json' # File to write data to.
today = datetime.now()
dates = ["%s-%s-1" % (y, m) for y in range(year, today.year) for m in range(1, 13)]
timestamps = list(filter(lambda ts: ts <= int(today.timestamp()), [date2ts(d) for d in dates]))
api = SwarmApi('config.json')
checkins = []
for i, (start, end) in enumerate(zip(timestamps, timestamps[1:])):
r = api(start, end, limit)
checkins.extend(r['response']['checkins']['items'])
print("Month %s: %s checkins (code %s)" % (dates[i], r['response']['checkins']['count'], r['meta']['code']))
if r['response']['checkins']['count'] == limit:
print("WARNING: checkin count for month %s is exactly the response limit. pagination is not implemented, so some data may be lost." % dates[i])
with open(outfile, 'w') as f:
json.dump(checkins, f, indent=4)
print("wrote %d checkins to %s" % (len(checkins), outfile))
if __name__ == '__main__':
main()
Raw
sample_config.json
{
"userid": "TODO: Change Me following instructions in get.py",
"wsid": "TODO: Change Me following instructions in get.py",
"oauth_token": "TODO: Change Me following instructions in get.py"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment