Skip to content

Instantly share code, notes, and snippets.

@jsj14

jsj14/fetch_data_upload_to_gcs

Last active February 4, 2022 03:47
Show Gist options
  • Save jsj14/8542cdb6c7e1d7d841e1a1e15f720ac2 to your computer and use it in GitHub Desktop.
Save jsj14/8542cdb6c7e1d7d841e1a1e15f720ac2 to your computer and use it in GitHub Desktop.
Download ZIP
Fetch public data and upload to a cloud storage
Raw
fetch_data_upload_to_gcs
from flask import jsonify
from geopy.geocoders import Nominatim
import requests
from google.cloud import storage
import json
storage_client = storage.Client(project='your-project-id') #replace with your projectID from Home
def weather(request):
data = {"success": False}
#https://pypi.org/project/geopy/
geolocator = Nominatim(user_agent="mlops-jsj")
# params = request.get_json()
for i in ["Orlando", "Chicago", "Texas"]:
location = geolocator.geocode(i)
# https://www.weather.gov/documentation/services-web-api
# Example query: https://api.weather.gov/points/39.7456,-97.0892
result1 = requests.get(f"https://api.weather.gov/points/{location.latitude},{location.longitude}")
# Example query: https://api.weather.gov/gridpoints/TOP/31,80
result2 = requests.get(f"{result1.json()['properties']['forecast']}")
data["response"] = result2.json()
data["success"] = True
bucket_name = 'weather_jsj_test2022'
# delete_bucket(bucket_name) - can be called if needed
# create_bucket(bucket_name) - called once for creation
local_data = json.dumps(data["response"])
file_name = 'data'+str(i)
upload_blob(bucket_name, local_data, file_name)
print('Data inside of',bucket_name,':')
return list_blobs(bucket_name)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment