Skip to content

Instantly share code, notes, and snippets.

@mourner

mourner/LICENSE.txt

Last active March 7, 2024 16:53
Show Gist options
  • Star 4 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save mourner/73abfed774920142a4bf0395e2129b0b to your computer and use it in GitHub Desktop.
Save mourner/73abfed774920142a4bf0395e2129b0b to your computer and use it in GitHub Desktop.
Download ZIP
Scrape air raid sirens in Ukraine from Telegram
Raw
api.js
import path from 'path';
import MTProto from '@mtproto/core';
import {sleep} from '@mtproto/core/src/utils/common/index.js';
import {readFileSync} from 'fs';
const {api_id, api_hash} = JSON.parse(readFileSync('./config.json'));
class API {
constructor() {
this.mtproto = new MTProto({api_id, api_hash, storageOptions: {path: './session.json'}});
}
async call(method, params, options = {}) {
try {
const result = await this.mtproto.call(method, params, options);
return result;
} catch (error) {
console.log(`${method} error:`, error);
const {error_code, error_message} = error;
if (error_code === 420) {
const seconds = Number(error_message.split('FLOOD_WAIT_')[1]);
const ms = seconds * 1000;
await sleep(ms);
return this.call(method, params, options);
}
if (error_code === 303) {
const [type, dcIdAsString] = error_message.split('_MIGRATE_');
const dcId = Number(dcIdAsString);
// If auth.sendCode call on incorrect DC need change default DC, because
// call auth.signIn on incorrect DC return PHONE_CODE_EXPIRED error
if (type === 'PHONE') {
await this.mtproto.setDefaultDc(dcId);
} else {
Object.assign(options, {dcId});
}
return this.call(method, params, options);
}
return Promise.reject(error);
}
}
async getUser() {
try {
const user = await this.call('users.getFullUser', {
id: {
_: 'inputUserSelf',
},
});
return user;
} catch (error) {
return null;
}
}
sendCode(phone) {
return this.call('auth.sendCode', {
phone_number: phone,
settings: {
_: 'codeSettings',
},
});
}
signIn({code, phone, phone_code_hash}) {
return this.call('auth.signIn', {
phone_code: code,
phone_number: phone,
phone_code_hash: phone_code_hash,
});
}
signUp({phone, phone_code_hash}) {
return this.call('auth.signUp', {
phone_number: phone,
phone_code_hash: phone_code_hash,
first_name: 'MTProto',
last_name: 'Core',
});
}
getPassword() {
return this.call('account.getPassword');
}
checkPassword({srp_id, A, M1}) {
return this.call('auth.checkPassword', {
password: {
_: 'inputCheckPasswordSRP',
srp_id,
A,
M1,
},
});
}
}
export default new API();
Raw
download.js
import {readFileSync, writeFileSync} from 'fs';
import prompt from 'prompt';
import api from './api.js';
const user = await api.getUser();
const existingMessages = JSON.parse(readFileSync('./messages.json'));
const lastDate = existingMessages[0].date;
if (!user) {
const {phone} = await prompt.get('phone');
const {phone_code_hash} = await api.sendCode(phone);
const {code} = await prompt.get('code');
try {
const signInResult = await api.signIn({
code,
phone,
phone_code_hash,
});
if (signInResult._ === 'auth.authorizationSignUpRequired') {
const singUpResult = await api.signUp({
phone,
phone_code_hash,
});
}
const newUser = await api.getUser();
} catch (error) {
if (error.error_message !== 'SESSION_PASSWORD_NEEDED') {
console.log(`error:`, error);
}
}
}
const resolvedPeer = await api.call('contacts.resolveUsername', {
username: 'air_alert_ua',
});
const channel = resolvedPeer.chats.find((chat) => chat.id === resolvedPeer.peer.channel_id);
const inputPeer = {
_: 'inputPeerChannel',
channel_id: channel.id,
access_hash: channel.access_hash,
};
const firstHistoryResult = await api.call('messages.getHistory', {
peer: inputPeer,
limit: 1,
});
const totalCount = firstHistoryResult.count;
const allMessages = [];
const LIMIT_COUNT = 100;
for (let offset = 0; offset < totalCount; offset += LIMIT_COUNT) {
console.log(`downloading ${offset}–${offset + LIMIT_COUNT} of ${totalCount} messages`);
const history = await api.call('messages.getHistory', {
peer: inputPeer,
add_offset: offset,
limit: LIMIT_COUNT,
});
const messages = history.messages.map(({message, date}) => ({message, date}));
allMessages.push(...messages);
const oldestDownloaded = messages[messages.length - 1];
if (oldestDownloaded.date < lastDate) {
for (const message of existingMessages) {
if (message.date < oldestDownloaded.date || (message.date === oldestDownloaded.date && message.message !== oldestDownloaded.message)) {
allMessages.push(message);
}
}
break;
}
}
writeFileSync('./messages.json', '[\n' + allMessages.map(m => JSON.stringify(m)).join(',\n') + '\n]\n');
allMessages.sort((a, b) => a.date - b.date);
const ranges = {};
const lastTypes = {};
const durations = {};
const TYPES = {
'Повітряна тривога': 0,
'Відбій тривоги': 1,
'Відбій повітряної тривоги': 1
};
const re = /. \d\d:\d\d ?(?<type>Повітряна тривога|Загроза артобстрілу|Відбій( повітряної)? тривоги|Відбій загрози артобстрілу)( в (?<location>.+?)\.?|\.|!)$/m;
const dateMin = 1647295200;
const dateMax = allMessages[allMessages.length - 1].date;
for (const {message, date} of allMessages) {
if (!message) continue;
const match = message.match(re);
if (!match) {
console.log(message);
continue;
}
const tag = message.split('#')[1];
const type = TYPES[match.groups.type];
let location = match.groups.location;
if (!location) {
location = tag.replace('м_', 'м. ').replace(/_/g, ' ');
}
const locations = location.split(' та ');
for (const loc of locations) {
let lastType = lastTypes[loc];
const alerts = (ranges[loc] = ranges[loc] || []);
// skip stops that don't have starts
if (alerts.length === 0 && type === 1) continue;
if (type === lastType) continue; // skip duplicate events
lastTypes[loc] = type;
alerts.push(date);
}
}
for (const loc of Object.keys(ranges)) {
if (ranges[loc].length % 2 === 1) {
ranges[loc].push(dateMax); // extend starts that don't have a stop to the end
}
if (ranges[loc].length === 0) {
delete ranges[loc];
continue;
}
for (let i = 0; i < ranges[loc].length; i += 2) {
durations[loc] = (durations[loc] || 0) + ranges[loc][i + 1] - ranges[loc][i];
}
if (durations[loc] < 100) {
delete ranges[loc];
}
}
// sort by total siren duration
const locations = Object.keys(ranges).sort((a, b) => durations[b] - durations[a]);
const sorted = {};
for (const loc of locations) {
// console.log(durations[loc], ranges[loc].length, loc);
sorted[loc] = ranges[loc];
}
writeFileSync('./sirens.json', JSON.stringify({
locations: sorted,
updated: Math.round(Date.now() / 1000)
}));
process.exit(0);
Raw
LICENSE.txt
ISC License
Copyright (c) 2022 Volodymyr Agafonkin
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment