Skip to content

Instantly share code, notes, and snippets.

@mackuba

mackuba/bluesky_loader.rb

Last active July 3, 2023 23:01
Show Gist options
  • Save mackuba/ddcb225ae4e6cf08e0e0396b3f6a2f6d to your computer and use it in GitHub Desktop.
Save mackuba/ddcb225ae4e6cf08e0e0396b3f6a2f6d to your computer and use it in GitHub Desktop.
Download ZIP
Ruby script which loads posts from your Bluesky home feed and saves them to a local file
Raw
bluesky_loader.rb
#!/usr/bin/env ruby
# Created by Kuba Suder on 25/04/2023
# Licensed under WTFPL License
require 'json'
require 'net/http'
require 'open-uri'
require 'set'
require 'time'
require 'yaml'
# bluesky.yml:
#
# host: bsky.social
# ident: myusername.bsky.social
# pass: mypassword
class BlueskyDownloader
CONFIG_FILE = 'bluesky.yml'
FEED_FILE = 'bluesky-feed.json'
FOLLOWERS_FILE = 'bluesky-followers.json'
LIKES_FILE = 'bluesky-likes.json'
MIN_DATE = "2020-01-01"
def initialize
@config = YAML.load(File.read(CONFIG_FILE))
@base_url = "https://#{@config['host']}/xrpc"
end
def my_id
@config['ident']
end
def my_did
@config['did']
end
def access_token
@config['access_token']
end
def refresh_token
@config['refresh_token']
end
def save_config
File.write(CONFIG_FILE, YAML.dump(@config))
end
def fetch_posts_to_db
@feed = File.exist?(FEED_FILE) ? JSON.parse(File.read(FEED_FILE)) : []
latest_date = @feed[0] && date_of(@feed[0])
new_posts = fetch_main_feed(until_date: latest_date)
@feed = new_posts + @feed
File.write(FEED_FILE, JSON.pretty_generate(@feed))
end
def fetch_likes_to_db
@likes = File.exist?(LIKES_FILE) ? JSON.parse(File.read(LIKES_FILE)) : []
latest_date = @likes[0] && @likes[0]['likedAt']
new_posts = fetch_likes(until_date: latest_date)
@likes = new_posts + @likes
File.write(LIKES_FILE, JSON.pretty_generate(@likes))
end
def fetch_main_feed(until_date: nil)
until_date ||= MIN_DATE
params = { limit: 100 }
posts = fetch_all('app.bsky.feed.getTimeline', params, access_token,
field: 'feed',
break_when: ->(x) { date_of(x) <= until_date }
)
# filter out replies to people I don't follow
filtered_posts = posts.select { |x| !x['reply'] || is_reply_to_me(x) || is_post_by_me(x) }
puts " Fetched #{filtered_posts.length} (#{posts.length}) posts"
filtered_posts
end
def fetch_likes(until_date: nil)
until_date ||= MIN_DATE
params = { collection: 'app.bsky.feed.like', repo: my_id, limit: 100 }
likes = fetch_all('com.atproto.repo.listRecords', params,
field: 'records',
break_when: ->(x) { x['value']['createdAt'] <= until_date }
)
liked_posts = likes.map do |like|
print '+'
likedAt = like['value']['createdAt']
subject = like['value']['subject']['uri']
fields = subject.gsub('at://', '').split('/')
did, type, id = fields
begin
response = get_request('com.atproto.repo.getRecord', repo: did, collection: type, rkey: id)
response.merge('likedAt' => likedAt)
rescue OpenURI::HTTPError => e
if e.message.start_with?('400')
print '!'
nil
else
raise
end
end
end
puts
liked_posts.compact
end
def check_access
if !access_token || !refresh_token || !my_did
puts "Logging in..."
log_in
else
begin
get_request('com.atproto.server.getSession', nil, access_token)
rescue OpenURI::HTTPError
puts "Refreshing token..."
perform_token_refresh
end
end
end
def log_in
json = post_request('com.atproto.server.createSession', {
identifier: @config['ident'],
password: @config['pass']
})
@config['did'] = json['did']
@config['access_token'] = json['accessJwt']
@config['refresh_token'] = json['refreshJwt']
save_config
end
def perform_token_refresh
json = post_request('com.atproto.server.refreshSession', nil, refresh_token)
@config['access_token'] = json['accessJwt']
@config['refresh_token'] = json['refreshJwt']
save_config
end
def update_followers
params = { actor: my_did, limit: 100 }
@followers = File.exist?(FOLLOWERS_FILE) ? JSON.parse(File.read(FOLLOWERS_FILE)) : nil
followers = fetch_all('app.bsky.graph.getFollowers', params, access_token, field: 'followers')
puts
if @followers
old_ids = Set.new(@followers.map { |f| f['did'] })
new_ids = Set.new(followers.map { |f| f['did'] })
@followers.select { |f| !new_ids.include?(f['did']) }.each { |f| puts "- @#{f['handle']} (#{f['displayName']})" }
followers.select { |f| !old_ids.include?(f['did']) }.each { |f| puts "+ @#{f['handle']} (#{f['displayName']})" }
end
@followers = followers
File.write(FOLLOWERS_FILE, JSON.pretty_generate(@followers))
end
def fetch_all(method, params, auth = nil, field:, break_when: ->(x) { false }, progress: true)
data = []
loop do
print '.' if progress
response = get_request(method, params, auth)
records = response[field]
cursor = response['cursor']
data.concat(records)
params[:cursor] = cursor
break if cursor.nil? || records.empty? || records.any? { |x| break_when.call(x) }
end
data.reject { |x| break_when.call(x) }
end
def post_skeet(text)
post_request('com.atproto.repo.createRecord', {
repo: my_did,
collection: 'app.bsky.feed.post',
record: {
text: text,
createdAt: Time.now.iso8601
}
}, access_token)
end
def get_request(method, params = nil, auth = nil)
headers = {}
headers['Authorization'] = "Bearer #{auth}" if auth
url = "#{@base_url}/#{method}"
if params && !params.empty?
url += "?" + params.map { |k, v|
if v.is_a?(Array)
v.map { |x| "#{k}=#{x}" }.join('&')
else
"#{k}=#{v}"
end
}.join('&')
end
JSON.parse(URI.open(url, headers).read)
end
def post_request(method, params, auth = nil)
headers = { "Content-Type" => "application/json" }
headers['Authorization'] = "Bearer #{auth}" if auth
body = params ? params.to_json : ''
response = Net::HTTP.post(URI("#{@base_url}/#{method}"), body, headers)
raise "Invalid response: #{response.code} #{response.body}" if response.code.to_i / 100 != 2
JSON.parse(response.body)
end
def date_of(entry)
entry['reason'] && entry['reason']['indexedAt'] || entry['post']['indexedAt']
end
def is_post_by_me(entry)
entry['post']['author']['handle'] == my_id
end
def is_reply_to_me(entry)
entry['reply']['parent']['author']['handle'] == my_id
end
end
if $0 == __FILE__
bsky = BlueskyDownloader.new
bsky.check_access
if ARGV[0] == "post"
if ARGV[1].to_s.empty?
puts "Missing post text"
exit 1
end
bsky.post_skeet(ARGV[1])
puts "Posted ✓"
else
bsky.fetch_posts_to_db
bsky.fetch_likes_to_db
bsky.update_followers
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment