u2discord_sentry.py

import asyncio
import difflib
import json
from datetime import datetime, timedelta

import aiohttp
import feedparser

##########################################################################
########################## Change this part ##############################
U2_UPDATEGISTSID = '' # Created Github U2 Gists ID
U2_GISTSFILENAME = '' # U2 Gists Filename

U2_PASSKEY = '' # U2 Torrent Passkey

GITHUBUSERNAME = '' # Github Username (email or whatever)
GITHUBPASSWORD = '' # Github Password

DISCORD_WEBHOOK = '' # Discord Webhook URL

diffThr = 20 # Difference threshold
SleepTime = 60 # In Seconds
##########################################################################
##########################################################################


##########################################################################
############################## U2 RSS Setting ###############################
##############################
## Category (use True or False) (If true, it will be added to the list)
U2Rip = False
U2RBD = False
WebDL = False
BDRip = False
DVDRip = False
HDTVRip = False
DVDISO = True
BDMV = True
LQRip = False
U2CSM = False # 外挂结构 (Custom Disc Patch)
U2DIY = False # 加流重灌 (Full DIY Disc)
Raw_Books = False
Books_CHTL = False # 港译漫画 (Books/Comics Translated to Chinese)
Books_TWTL = False # 台译漫画 (Books/Comics Translated to Taiwanese)
Lossless_Music = False
Others = False
## Setting
RSS_Rows = 10 # 10, 20, 30, 40, or 50 (Total RSS data that will be parsed)
UseSSL = True # True or False
##########################################################################
##########################################################################

class DiscordEmbed:
    def __init__(self, **kwargs):
        """
        Initiate Discord Embed format

        Shamelessly stolen from https://github.com/lovvskillz/python-discord-webhook/blob/master/discord_webhook/webhook.py#L109
        """
        self.title = kwargs.get('title')
        self.description = kwargs.get('description')
        self.url = kwargs.get('url')
        self.color = kwargs.get('color')
        self.footer = kwargs.get('footer')
        self.image = kwargs.get('image')
        self.thumbnail = kwargs.get('thumbnail')
        self.video = kwargs.get('video')
        self.provider = kwargs.get('provider')
        self.author = kwargs.get('author')
        self.fields = kwargs.get('fields', [])

    def get_json(self):
        return self.__dict__

    def set_title(self, title):
        """
        set title of embed
        :param title: title of embed
        """
        self.title = title

    def set_description(self, description):
        """
        set description of embed
        :param description: description of embed
        """
        self.description = description

    def set_url(self, url):
        """
        set url of embed
        :param url: url of embed
        """
        self.url = url

    def set_color(self, color):
        """
        set color code of the embed as int
        :param color: color code of the embed as int
        """
        self.color = color

    def set_footer(self, **kwargs):
        """
        set footer information of embed
        :keyword text: footer text
        :keyword icon_url: url of footer icon (only supports http(s) and attachments)
        :keyword proxy_icon_url: a proxied url of footer icon
        """
        self.footer = {
            'text': kwargs.get('text'),
            'icon_url': kwargs.get('icon_url'),
            'proxy_icon_url': kwargs.get('proxy_icon_url')
        }

    def set_image(self, **kwargs):
        """
        set image of embed
        :keyword url: source url of image (only supports http(s) and attachments)
        :keyword proxy_url: a proxied url of the image
        :keyword height: height of image
        :keyword width: width of image
        """
        self.image = {
            'url': kwargs.get('url'),
            'proxy_url': kwargs.get('proxy_url'),
            'height': kwargs.get('height'),
            'width': kwargs.get('width'),
        }

    def set_thumbnail(self, **kwargs):
        """
        set thumbnail of embed
        :keyword url: source url of thumbnail (only supports http(s) and attachments)
        :keyword proxy_url: a proxied thumbnail of the image
        :keyword height: height of thumbnail
        :keyword width: width of thumbnail
        """
        self.thumbnail = {
            'url': kwargs.get('url'),
            'proxy_url': kwargs.get('proxy_url'),
            'height': kwargs.get('height'),
            'width': kwargs.get('width'),
        }

    def set_video(self, **kwargs):
        """
        set video of embed
        :keyword url: source url of video
        :keyword height: height of video
        :keyword width: width of video
        """
        self.video = {
            'url': kwargs.get('url'),
            'height': kwargs.get('height'),
            'width': kwargs.get('width'),
        }

    def set_provider(self, **kwargs):
        """
        set provider of embed
        :keyword name: name of provider
        :keyword url: url of provider
        """
        self.provider = {
            'name': kwargs.get('name'),
            'url': kwargs.get('url'),
        }

    def set_author(self, **kwargs):
        """
        set author of embed
        :keyword name: name of author
        :keyword url: url of author
        :keyword icon_url: url of author icon (only supports http(s) and attachments)
        :keyword proxy_icon_url: a proxied url of author icon
        """
        self.author = {
            'name': kwargs.get('name'),
            'url': kwargs.get('url'),
            'icon_url': kwargs.get('icon_url'),
            'proxy_icon_url': kwargs.get('proxy_icon_url'),
        }

    def add_fields(self, **kwargs):
        """
        set field of embed
        :keyword name: name of the field
        :keyword value: value of the field
        :keyword inline: (optional) whether or not this field should display inline
        """
        self.fields.append({
            'name': kwargs.get('name'),
            'value': kwargs.get('value'),
            'inline': kwargs.get('inline', True)
        })


async def webhook_send_request(Embed: DiscordEmbed) -> bool:
    embed_dict = Embed.get_json()
    data = list()
    data.append(embed_dict)

    print('[@] Sending webhook requests')
    async with aiohttp.ClientSession() as sesi2:
        async with sesi2.post(DISCORD_WEBHOOK, json={'embeds': data}) as resp:
            status_code = resp.status
            text_ = await resp.text()

    if status_code not in [200, 204]:
        print('[!] Failed to send webhook requests')
        raise ConnectionError('Status code received are {}: {}'.format(status_code, text_))


async def parse_setting():
    if not U2_UPDATEGISTSID:
        raise ValueError('parse_setting: U2_UPDATEGISTSID cannot be empty')
    if not U2_GISTSFILENAME:
        raise ValueError('parse_setting: GISTSFILENAME cannot be empty')
    if not U2_PASSKEY:
        raise ValueError('parse_setting: U2_PASSKEY cannot be empty')
    if not GITHUBUSERNAME:
        raise ValueError('parse_setting: GITHUBUSERNAME cannot be empty')
    if not GITHUBPASSWORD:
        raise ValueError('parse_setting: GITHUBPASSWORD cannot be empty')
    if not DISCORD_WEBHOOK:
        raise ValueError('parse_setting: DISCORD_WEBHOOK cannot be empty')
    
    if not isinstance(U2_UPDATEGISTSID, str):
        raise ValueError('parse_setting: U2_UPDATEGISTSID is not a string')
    if not isinstance(U2_GISTSFILENAME, str):
        raise ValueError('parse_setting: U2_GISTSFILENAME is not a string')
    if not isinstance(U2_PASSKEY, str):
        raise ValueError('parse_setting: U2_PASSKEY is not a string')
    if not isinstance(GITHUBUSERNAME, str):
        raise ValueError('parse_setting: GITHUBUSERNAME is not a string')
    if not isinstance(GITHUBPASSWORD, str):
        raise ValueError('parse_setting: GITHUBPASSWORD is not a string')
    if not isinstance(DISCORD_WEBHOOK, str):
        raise ValueError('parse_setting: DISCORD_WEBHOOK is not a string')
    if not isinstance(diffThr, int):
        raise ValueError('parse_setting: diffThr is not a integer')
    u2_parsed = ''
    if U2Rip:
        u2_parsed += '&cat9=1'
    if U2RBD:
        u2_parsed += '&cat411=1'
    if WebDL:
        u2_parsed += '&cat413=1'
    if BDRip:
        u2_parsed += '&cat12=1'
    if DVDRip:
        u2_parsed += '&cat13=1'
    if HDTVRip:
        u2_parsed += '&cat14=1'
    if DVDISO:
        u2_parsed += '&cat15=1'
    if BDMV:
        u2_parsed += '&cat16=1'
    if LQRip:
        u2_parsed += '&cat17=1'
    if U2CSM:
        u2_parsed += '&cat410=1'
    if U2DIY:
        u2_parsed += '&cat412=1'
    if Raw_Books:
        u2_parsed += '&cat21=1'
    if Books_CHTL:
        u2_parsed += '&cat22=1'
    if Books_TWTL:
        u2_parsed += '&cat23=1'
    if Lossless_Music:
        u2_parsed += '&cat30=1'
    if Others:
        u2_parsed += '&cat40=1'
    if not u2_parsed:
        print("!! No category selected, using default BDMV and DVDISO only")
        u2_parsed += '&cat15=1&cat16=1'
    u2_parsed += '&icat=1&ismalldescr=1&isize=1&iuplder=1'
    if UseSSL:
        u2_parsed += '&trackerssl=1'
    if RSS_Rows in [10, 20, 30, 40, 50]:
        u2_parsed += '&rows={x}'.format(x=RSS_Rows)
    else:
        u2_parsed += '&rows=10'
    u2_parsed += '&passkey=' + U2_PASSKEY

    return u2_parsed[1:]


async def cloud_to_local():
    print('[@] Requesting to U2 gists for old data')
    async with aiohttp.ClientSession(auth=aiohttp.BasicAuth(GITHUBUSERNAME, GITHUBPASSWORD)) as sesi:
        async with sesi.get(f'https://api.github.com/gists/{U2_UPDATEGISTSID}') as resp:
            rg = await resp.json()
    u2_olddata = rg["files"][U2_GISTSFILENAME]["content"]

    print('[#] Saving old data to local disk.')
    js_d = {
        "u2": u2_olddata
    }

    with open('u2bot_sentry.local_data', 'w') as f:
        json.dump(js_d, f)


async def read_local_data():
    with open('u2bot_sentry.local_data', 'r') as f:
        js_d = json.load(f)

    return js_d['u2']


async def save_local_data(data, type_):
    with open('u2bot_sentry.local_data', 'r') as f:
        js_d = json.load(f)
    js_d[type_] = data
    with open('u2bot_sentry.local_data', 'w') as f:
        js_d = json.dump(js_d, f)


async def u2_parse(setting):
    print('[@] Fetching and Parsing U2 RSS Feed')
    u2 = feedparser.parse("https://u2.dmhy.org/torrentrss.php?{uriEx}".format(uriEx=setting))
    u2_dataset = []
    for e in u2.entries:
        try:
            temptitle = e.title
            temptor = e.author
            if temptor == '@u2.dmhy.org ()':
                temptor = temptitle[temptitle.rfind('[') + 1:-1]
                if "<i>" in temptor:
                    temptor = temptor.strip("</i>")
                author = f"{temptor}@u2.dmhy.org ({temptor})"
            else:
                author = temptor

            cuttitle = temptitle[0:temptitle.rfind('[')][temptitle.find(']') + 1:]
            tor_title = f"**{temptitle[:temptitle.find(']') + 1]}** {cuttitle[:cuttitle.rfind('[')]}"

            torrent_size = cuttitle[cuttitle.rfind('['):].lstrip('[').rstrip(']')

            u2_dataset.append([tor_title, temptitle, e.link, author, e.published, e.enclosures[0]['href'], torrent_size])
        except IndexError:
            u2_dataset.append([None] * 7)
    return u2_dataset


async def recursive_checkdiff(dataset, old_data):
    new_dataset = []
    for data in dataset:
        dlib = difflib.Differ()
        counter = 0
        if data[1]:
            diffdata  = dlib.compare(old_data, data[1])
            for d in diffdata:
                if d.startswith(('+', '-', '?')):
                    counter += 1
            if counter <= int(diffThr):
                break
            new_dataset.append(data)
    return new_dataset


async def check_dataset(u2_new):
    print('[@] Reading local data for old data')
    u2_olddata = await read_local_data()

    if u2_new != u2_olddata:
        print('[@] Start Checking U2 Dataset for different')
        u2_filtered = await recursive_checkdiff(u2_new, u2_olddata)

    if u2_filtered:
        print('[@] New U2 Torrent detected')
        u2_data = {
            "description": "U2 title checking for discord bot update/announcer",
            "files": {
                U2_GISTSFILENAME: {
                    "filename": U2_GISTSFILENAME,
                    "content": u2_filtered[0][1]
                }
            }
        }

        print('[@] Patching U2 gists with new data')
        async with aiohttp.ClientSession(auth=aiohttp.BasicAuth(GITHUBUSERNAME, GITHUBPASSWORD)) as sesi2:
            async with sesi2.patch(f'https://api.github.com/gists/{U2_UPDATEGISTSID}', json=u2_data) as resp:
                r = await resp.json()
        try:
            m = r['message']
            print('[!] Can\'t patch: {}'.format(m))
        except Exception:
            announce_u2 = True
            print('[#] Saving data locally...')
            await save_local_data(u2_filtered[0][1], 'u2')
    else:
        print('[#] No new U2 Data')

    return u2_filtered[::-1] # Reverse from oldest data one to newest one


async def Sentry():
    u2_settings = await parse_setting()
    await cloud_to_local()
    while True:
      try:
        print('[@] Checking U2 Counterpart')
        u2_dataset = await u2_parse(u2_settings)

        u2_filter = await check_dataset(u2_dataset)

        if u2_filter:
            print('[!@!] Sending U2 message')
            for u2 in u2_filter:
                embed = DiscordEmbed(color=10280703)
                embed.set_author(name="U2 Update", url="https://gist.github.com/noaione/0aa605be5261e453a7f02a5e7fa5897b", icon_url="https://cdn.discordapp.com/avatars/483138354326536202/23ad7ff79c488dc421735392fb3a0911.webp")
                embed.add_fields(name="Torrent", value=u2[0], inline=False)
                embed.add_fields(name="Size", value=u2[6], inline=False)
                embed.add_fields(name="Link", value="📥 \|| [View]({}) \|| **[Download]({})**".format(u2[2], u2[5]), inline=False)
                embed.set_footer(text="Posted: {} || Uploaded by {}".format(u2[4], u2[3]))
                await webhook_send_request(embed)
        else:
            print('[!#!] Retrying U2 Check later.')

        await asyncio.sleep(SleepTime)
      except KeyboardInterrupt:
            print('[@] CTRL+C Pressed, shutting down...')
            break

try:
    print('[@] Initiating async loop')
    loop = asyncio.get_event_loop()
    print('[!@!] Starting...')
    loop.run_until_complete(Sentry())
finally:
    print('[@] Closing loop...')
    loop.close()

u2discord_sentry.py.README.md

N4O U2 Discord Bot Announcer

This version use cloud data (from github gist) for data checking and also local data For local data only (Not relying on github), use this one: https://gist.github.com/noaione/8ee8284d182e781307a1938de04078ee

Requirements

• feedparser
• aiohttp

Setting up

1. create a new gist and create the file with any filename and extension if you want

• For example: u2_bot_rss_update_data

2. Then copy the gist id for example: https://gist.github.com/noaione/7fa5962963bffe3103f86cd18b653e08

• 7fa5962963bffe3103f86cd18b653e08 are the gists id
• Add it into U2_UPDATEGISTSID

3. Copy the filename to the U2_GISTSFILENAME
4. Add your GitHub Username to GITHUBUSERNAME and GitHub password to GITHUBPASSWORD
5. Find your U2 Passkey and add it into U2_PASSKEY

• You can find it on User CP section

6. Create a new webhook on Discord Server and copy the url, put it on DISCORD_WEBHOOK
7. Run it with python u2discord_sentry.py