This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import json | |
from subprocess import check_call | |
import click | |
def assume_role(role_arn): | |
import boto3 | |
sts_client = boto3.client('sts') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from metaflow import FlowSpec, step, retry, S3, current | |
from functools import wraps | |
import pickle | |
import random | |
PREFIX = "resumable-processing" | |
class resumable_processing: | |
def __init__(self, process="list", results="output"): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from metaflow import namespace, Metaflow, Run | |
| |
def fmt(t): | |
return t.strftime('%Y-%m-%dT%H:%M:%SZ') | |
| |
print('flow,run,step,task,created,finished,user,runtime,pod_id,pod_name') | |
namespace(None) | |
for flow in Metaflow(): | |
for run in flow: | |
if run.successful: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import random | |
from metaflow import FlowSpec, step, S3, Flow, Parameter, profile, kubernetes, conda, conda_base | |
# change columns according to your schema (or remove column list to load all) | |
COLUMNS = ['VendorID', 'tpep_pickup_datetime', 'tpep_dropoff_datetime'] | |
# group parquet files as 1GB batches | |
def shard_data(src, batch_size=1_000_000_000): | |
with S3() as s3: | |
objs = s3.list_recursive([src]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
from metaflow import S3 | |
def put_dir(local_root, s3root): | |
root = os.path.abspath(local_root) | |
objs = [] | |
for p, _, files in os.walk(root): | |
for f in files: | |
path = os.path.join(p, f) | |
key = os.path.relpath(path, start=root) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from metaflow import FlowSpec, step, IncludeFile | |
def dataset_wine(): | |
from sklearn import datasets | |
return datasets.load_wine(return_X_y=True) | |
def model_knn(train_data, train_labels): | |
from sklearn.neighbors import KNeighborsClassifier | |
model = KNeighborsClassifier() | |
model.fit(train_data, train_labels) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys | |
def baseline(k): | |
n = 0 | |
for i in range(k + 1): | |
n += str(i).count('1') | |
return n | |
def modulo(k): | |
if k == 0: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from metaflow import FlowSpec, step, Parameter, resources, conda_base, profile | |
@conda_base(python='3.8.3', libraries={'scikit-learn': '0.24.1'}) | |
class ManyKmeansFlow(FlowSpec): | |
num_docs = Parameter('num-docs', help='Number of documents', default=1000000) | |
@resources(memory=4000) | |
@step | |
def start(self): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from metaflow import FlowSpec, step | |
from functools import wraps | |
from functools import wraps | |
dir = 'mydir' | |
def magicdir(f): | |
artifact = 'magicdir' | |
@wraps(f) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from metaflow import FlowSpec, step, Parameter, IncludeFile, catch | |
import math, time, uuid, datetime, random, string, sys | |
from decimal import Decimal | |
import requests | |
class CustomClass(): | |
def __str__(self): | |
return 'a' * int(1024**2) |
NewerOlder