myned/modufur
1
0
Fork 0
mirror of https://github.com/myned/modufur.git synced 2024-12-23 22:27:27 +00:00
Code Issues Activity

Merge branch 'dev'

Browse source
This commit is contained in:
Myned 2019-09-29 23:58:44 -04:00
commit 48900ec5dd
No known key found for this signature in database
GPG key ID: BC58C09870A63E59
2 changed files with 17 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/utils/scraper.py
View file

@ -43,9 +43,12 @@ from utils import utils as u
async def query_kheina(url): async def query_kheina(url):
content = await u.fetch('https://kheina.com', params={'url': url}, text=True) content = await u.fetch('https://kheina.com', params={'url': url}, text=True)
content = content.replace('"', 'quot;').replace(''', 'apos;') for e in ('"', '''):
content = content.replace(e, '')
content = re.sub('<a href="/cdn-cgi/l/email-protection".+</a>', '', content)
soup = BeautifulSoup(content, 'html5lib') soup = BeautifulSoup(content, 'html5lib')
results = soup.find('data', id='results').string.replace('quot;', '&quot;').replace('apos;', '&apos;') results = soup.find('data', id='results').string
results = ast.literal_eval(results) results = ast.literal_eval(results)
iqdbdata = soup.find('data', id='iqdbdata').string iqdbdata = soup.find('data', id='iqdbdata').string
iqdbdata = ast.literal_eval(iqdbdata) iqdbdata = ast.literal_eval(iqdbdata)

25
src/utils/utils.py
View file

@ -1,4 +1,3 @@
import asyncio
import json as jsn import json as jsn
import os import os
import pickle as pkl import pickle as pkl
@ -12,8 +11,6 @@ from discord import errors as err
from misc import exceptions as exc from misc import exceptions as exc
# from pync import Notifier
print('\nPID : {}\n'.format(os.getpid())) print('\nPID : {}\n'.format(os.getpid()))
@ -84,18 +81,20 @@ cogs = {}
color = d.Color(0x1A1A1A) color = d.Color(0x1A1A1A)
last_commands = {} last_commands = {}
asession = aiohttp.ClientSession()
async def fetch(url, *, params={}, json=False, response=False, text=False): async def fetch(url, *, params={}, json=False, response=False, text=False):
async with aiohttp.ClientSession() as session: async with asession.get(url, params=params, headers={
async with session.get(url, params=params, headers={'User-Agent': 'Myned/Modufur'}, ssl=False) as r: 'User-Agent': 'Myned/Modufur (https://github.com/Myned/Modufur)'}, ssl=False) as r:
if json: if json:
return await r.json() return await r.json()
elif response: elif response:
return r return r
elif text: elif text:
return await r.text() return await r.text()
else: else:
return await r.read() return await r.read()
def generate_embed(ctx, *, title=d.Embed.Empty, kind='rich', description=d.Embed.Empty, url=d.Embed.Empty, timestamp=d.Embed.Empty, colour=color, footer={}, image=d.Embed.Empty, thumbnail=d.Embed.Empty, author={}, fields=[]): def generate_embed(ctx, *, title=d.Embed.Empty, kind='rich', description=d.Embed.Empty, url=d.Embed.Empty, timestamp=d.Embed.Empty, colour=color, footer={}, image=d.Embed.Empty, thumbnail=d.Embed.Empty, author={}, fields=[]):