Scrape air raid sirens in Ukraine from Telegram

api.js

import path from 'path';
import MTProto from '@mtproto/core';
import {sleep} from '@mtproto/core/src/utils/common/index.js';
import {readFileSync} from 'fs';

const {api_id, api_hash} = JSON.parse(readFileSync('./config.json'));

class API {
    constructor() {
        this.mtproto = new MTProto({api_id, api_hash, storageOptions: {path: './session.json'}});
    }

    async call(method, params, options = {}) {
        try {
            const result = await this.mtproto.call(method, params, options);
            return result;
        } catch (error) {
            console.log(`${method} error:`, error);

            const {error_code, error_message} = error;

            if (error_code === 420) {
                const seconds = Number(error_message.split('FLOOD_WAIT_')[1]);
                const ms = seconds * 1000;

                await sleep(ms);

                return this.call(method, params, options);
            }

            if (error_code === 303) {
                const [type, dcIdAsString] = error_message.split('_MIGRATE_');

                const dcId = Number(dcIdAsString);

                // If auth.sendCode call on incorrect DC need change default DC, because
                // call auth.signIn on incorrect DC return PHONE_CODE_EXPIRED error
                if (type === 'PHONE') {
                    await this.mtproto.setDefaultDc(dcId);
                } else {
                    Object.assign(options, {dcId});
                }

                return this.call(method, params, options);
            }

            return Promise.reject(error);
        }
    }

    async getUser() {
        try {
            const user = await this.call('users.getFullUser', {
                id: {
                    _: 'inputUserSelf',
                },
            });

            return user;
        } catch (error) {
            return null;
        }
    }

    sendCode(phone) {
        return this.call('auth.sendCode', {
            phone_number: phone,
            settings: {
                _: 'codeSettings',
            },
        });
    }

    signIn({code, phone, phone_code_hash}) {
        return this.call('auth.signIn', {
            phone_code: code,
            phone_number: phone,
            phone_code_hash: phone_code_hash,
        });
    }

    signUp({phone, phone_code_hash}) {
        return this.call('auth.signUp', {
            phone_number: phone,
            phone_code_hash: phone_code_hash,
            first_name: 'MTProto',
            last_name: 'Core',
        });
    }

    getPassword() {
        return this.call('account.getPassword');
    }

    checkPassword({srp_id, A, M1}) {
        return this.call('auth.checkPassword', {
            password: {
                _: 'inputCheckPasswordSRP',
                srp_id,
                A,
                M1,
            },
        });
    }
}

export default new API();

download.js

import {readFileSync, writeFileSync} from 'fs';
import prompt from 'prompt';
import api from './api.js';

const user = await api.getUser();
const existingMessages = JSON.parse(readFileSync('./messages.json'));
const lastDate = existingMessages[0].date;

if (!user) {
    const {phone} = await prompt.get('phone');
    const {phone_code_hash} = await api.sendCode(phone);

    const {code} = await prompt.get('code');
    try {
        const signInResult = await api.signIn({
            code,
            phone,
            phone_code_hash,
        });

        if (signInResult._ === 'auth.authorizationSignUpRequired') {
            const singUpResult = await api.signUp({
                phone,
                phone_code_hash,
            });
        }

        const newUser = await api.getUser();
    } catch (error) {
        if (error.error_message !== 'SESSION_PASSWORD_NEEDED') {
            console.log(`error:`, error);
        }
    }
}

const resolvedPeer = await api.call('contacts.resolveUsername', {
    username: 'air_alert_ua',
});

const channel = resolvedPeer.chats.find((chat) => chat.id === resolvedPeer.peer.channel_id);

const inputPeer = {
    _: 'inputPeerChannel',
    channel_id: channel.id,
    access_hash: channel.access_hash,
};

const firstHistoryResult = await api.call('messages.getHistory', {
    peer: inputPeer,
    limit: 1,
});
const totalCount = firstHistoryResult.count;

const allMessages = [];
const LIMIT_COUNT = 100;

for (let offset = 0; offset < totalCount; offset += LIMIT_COUNT) {
    console.log(`downloading ${offset}–${offset + LIMIT_COUNT} of ${totalCount} messages`);

    const history = await api.call('messages.getHistory', {
        peer: inputPeer,
        add_offset: offset,
        limit: LIMIT_COUNT,
    });
    const messages = history.messages.map(({message, date}) => ({message, date}));

    allMessages.push(...messages);

    const oldestDownloaded = messages[messages.length - 1];

    if (oldestDownloaded.date < lastDate) {
        for (const message of existingMessages) {
            if (message.date < oldestDownloaded.date || (message.date === oldestDownloaded.date && message.message !== oldestDownloaded.message)) {
                allMessages.push(message);
            }
        }
        break;
    }
}

writeFileSync('./messages.json', '[\n' + allMessages.map(m => JSON.stringify(m)).join(',\n') + '\n]\n');

allMessages.sort((a, b) => a.date - b.date);

const ranges = {};
const lastTypes = {};
const durations = {};

const TYPES = {
    'Повітряна тривога': 0,
    'Відбій тривоги': 1,
    'Відбій повітряної тривоги': 1
};

const re = /. \d\d:\d\d  ?(?<type>Повітряна тривога|Загроза артобстрілу|Відбій( повітряної)? тривоги|Відбій загрози артобстрілу)( в (?<location>.+?)\.?|\.|!)$/m;

const dateMin = 1647295200;
const dateMax = allMessages[allMessages.length - 1].date;

for (const {message, date} of allMessages) {
    if (!message) continue;

    const match = message.match(re);
    if (!match) {
        console.log(message);
        continue;
    }
    const tag = message.split('#')[1];
    const type = TYPES[match.groups.type];
    let location = match.groups.location;
    if (!location) {
        location = tag.replace('м_', 'м. ').replace(/_/g, ' ');
    }
    const locations = location.split(' та ');

    for (const loc of locations) {
        let lastType = lastTypes[loc];
        const alerts = (ranges[loc] = ranges[loc] || []);

        // skip stops that don't have starts
        if (alerts.length === 0 && type === 1) continue;

        if (type === lastType) continue; // skip duplicate events

        lastTypes[loc] = type;
        alerts.push(date);
    }
}

for (const loc of Object.keys(ranges)) {
    if (ranges[loc].length % 2 === 1) {
        ranges[loc].push(dateMax); // extend starts that don't have a stop to the end
    }
    if (ranges[loc].length === 0) {
        delete ranges[loc];
        continue;
    }
    for (let i = 0; i < ranges[loc].length; i += 2) {
        durations[loc] = (durations[loc] || 0) + ranges[loc][i + 1] - ranges[loc][i];
    }
    if (durations[loc] < 100) {
        delete ranges[loc];
    }
}

// sort by total siren duration
const locations = Object.keys(ranges).sort((a, b) => durations[b] - durations[a]);

const sorted = {};
for (const loc of locations) {
    // console.log(durations[loc], ranges[loc].length, loc);
    sorted[loc] = ranges[loc];
}

writeFileSync('./sirens.json', JSON.stringify({
    locations: sorted,
    updated: Math.round(Date.now() / 1000)
}));

process.exit(0);

LICENSE.txt

ISC License

Copyright (c) 2022 Volodymyr Agafonkin

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.