diff --git a/src/utils/scraper.py b/src/utils/scraper.py index 05888c9..49edfc5 100644 --- a/src/utils/scraper.py +++ b/src/utils/scraper.py @@ -43,9 +43,12 @@ from utils import utils as u async def query_kheina(url): content = await u.fetch('https://kheina.com', params={'url': url}, text=True) - content = content.replace('"', 'quot;').replace(''', 'apos;') + for e in ('"', '''): + content = content.replace(e, '') + content = re.sub('', '', content) + soup = BeautifulSoup(content, 'html5lib') - results = soup.find('data', id='results').string.replace('quot;', '"').replace('apos;', ''') + results = soup.find('data', id='results').string results = ast.literal_eval(results) iqdbdata = soup.find('data', id='iqdbdata').string iqdbdata = ast.literal_eval(iqdbdata) diff --git a/src/utils/utils.py b/src/utils/utils.py index b86541e..6298cb2 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -1,4 +1,3 @@ -import asyncio import json as jsn import os import pickle as pkl @@ -12,8 +11,6 @@ from discord import errors as err from misc import exceptions as exc -# from pync import Notifier - print('\nPID : {}\n'.format(os.getpid())) @@ -84,18 +81,20 @@ cogs = {} color = d.Color(0x1A1A1A) last_commands = {} +asession = aiohttp.ClientSession() + async def fetch(url, *, params={}, json=False, response=False, text=False): - async with aiohttp.ClientSession() as session: - async with session.get(url, params=params, headers={'User-Agent': 'Myned/Modufur'}, ssl=False) as r: - if json: - return await r.json() - elif response: - return r - elif text: - return await r.text() - else: - return await r.read() + async with asession.get(url, params=params, headers={ + 'User-Agent': 'Myned/Modufur (https://github.com/Myned/Modufur)'}, ssl=False) as r: + if json: + return await r.json() + elif response: + return r + elif text: + return await r.text() + else: + return await r.read() def generate_embed(ctx, *, title=d.Embed.Empty, kind='rich', description=d.Embed.Empty, url=d.Embed.Empty, timestamp=d.Embed.Empty, colour=color, footer={}, image=d.Embed.Empty, thumbnail=d.Embed.Empty, author={}, fields=[]):