From 496d5bc00cd1e78157c6b6e8afeee7d644aa8311 Mon Sep 17 00:00:00 2001 From: Myned Date: Sun, 29 Sep 2019 23:57:00 -0400 Subject: [PATCH 1/4] Change HTML sanitization to be more simplistic --- src/utils/scraper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/scraper.py b/src/utils/scraper.py index 05888c9..fa184da 100644 --- a/src/utils/scraper.py +++ b/src/utils/scraper.py @@ -43,9 +43,10 @@ from utils import utils as u async def query_kheina(url): content = await u.fetch('https://kheina.com', params={'url': url}, text=True) - content = content.replace('"', 'quot;').replace(''', 'apos;') + for e in ('"', '''): + content = content.replace(e, '') soup = BeautifulSoup(content, 'html5lib') - results = soup.find('data', id='results').string.replace('quot;', '"').replace('apos;', ''') + results = soup.find('data', id='results').string results = ast.literal_eval(results) iqdbdata = soup.find('data', id='iqdbdata').string iqdbdata = ast.literal_eval(iqdbdata) From 3ab29caee1e387163e76d67a3d72e3db7c2eafce Mon Sep 17 00:00:00 2001 From: Myned Date: Sun, 29 Sep 2019 23:57:40 -0400 Subject: [PATCH 2/4] Fix HTML entities appearing in strings --- src/utils/scraper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/scraper.py b/src/utils/scraper.py index fa184da..49edfc5 100644 --- a/src/utils/scraper.py +++ b/src/utils/scraper.py @@ -45,6 +45,8 @@ async def query_kheina(url): for e in ('"', '''): content = content.replace(e, '') + content = re.sub('', '', content) + soup = BeautifulSoup(content, 'html5lib') results = soup.find('data', id='results').string results = ast.literal_eval(results) From bf7e75b0bd79262c37dec00f62eb349190f3f2e6 Mon Sep 17 00:00:00 2001 From: Myned Date: Sun, 29 Sep 2019 23:57:56 -0400 Subject: [PATCH 3/4] Remove extraneous imports --- src/utils/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/utils/utils.py b/src/utils/utils.py index b86541e..8ec5a56 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -1,4 +1,3 @@ -import asyncio import json as jsn import os import pickle as pkl @@ -12,8 +11,6 @@ from discord import errors as err from misc import exceptions as exc -# from pync import Notifier - print('\nPID : {}\n'.format(os.getpid())) From e399050bd09d6ab1194911fccddefac5bc4822c8 Mon Sep 17 00:00:00 2001 From: Myned Date: Sun, 29 Sep 2019 23:58:35 -0400 Subject: [PATCH 4/4] Change aiohttp session to be global instead of localized --- src/utils/utils.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/utils/utils.py b/src/utils/utils.py index 8ec5a56..6298cb2 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -81,18 +81,20 @@ cogs = {} color = d.Color(0x1A1A1A) last_commands = {} +asession = aiohttp.ClientSession() + async def fetch(url, *, params={}, json=False, response=False, text=False): - async with aiohttp.ClientSession() as session: - async with session.get(url, params=params, headers={'User-Agent': 'Myned/Modufur'}, ssl=False) as r: - if json: - return await r.json() - elif response: - return r - elif text: - return await r.text() - else: - return await r.read() + async with asession.get(url, params=params, headers={ + 'User-Agent': 'Myned/Modufur (https://github.com/Myned/Modufur)'}, ssl=False) as r: + if json: + return await r.json() + elif response: + return r + elif text: + return await r.text() + else: + return await r.read() def generate_embed(ctx, *, title=d.Embed.Empty, kind='rich', description=d.Embed.Empty, url=d.Embed.Empty, timestamp=d.Embed.Empty, colour=color, footer={}, image=d.Embed.Empty, thumbnail=d.Embed.Empty, author={}, fields=[]):