myned/modufur
1
0
Fork 0
mirror of https://github.com/myned/modufur.git synced 2024-12-25 14:47:29 +00:00
Code Issues Activity

Merge branch 'dev'

Browse source
This commit is contained in:
Myned 2020-03-18 02:18:08 -04:00
commit 6a6c68db13
No known key found for this signature in database
GPG key ID: 2EF9C0C44229D034
2 changed files with 36 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/utils/scraper.py
View file

@ -41,37 +41,21 @@ from utils import utils as u
async def query_kheina(url): async def query_kheina(url):
try: try:
content = await u.fetch(f'https://kheina.com?url={url}', text=True) content = await u.fetch(f'https://api.kheina.com/v1/search', post={'url': url}, json=True)
for e in ('"', '''): similarity = int(content['results'][0]['similarity'])
content = content.replace(e, '')
content = re.sub('<a href="/cdn-cgi/l/email-protection".+</a>', '', content)
soup = BeautifulSoup(content, 'html5lib')
if soup.find('data', id='error'):
return None
results = soup.find('data', id='results').string
results = ast.literal_eval(results)
iqdbdata = soup.find('data', id='iqdbdata').string
iqdbdata = ast.literal_eval(iqdbdata)
similarity = int(float(iqdbdata[0]['similarity']))
if similarity < 55: if similarity < 55:
return None return None
for e in results: source = re.search('\\d+$', content['results'][0]['sources'][0]['source']).group(0)
if iqdbdata[0]['iqdbid'] in e: export = await u.fetch(f'https://faexport.spangle.org.uk/submission/{source}.json', json=True)
match = e
break
result = { result = {
'source': match[3].replace('\\', ''), 'source': content['results'][0]['sources'][0]['source'],
'artist': match[4], 'artist': content['results'][0]['sources'][0]['artist'],
'thumbnail': f'https://f002.backblazeb2.com/file/kheinacom/{match[1]}.jpg', 'thumbnail': '' if isinstance(export, int) and export != 200 else export['full'],
'similarity': str(similarity), 'similarity': str(similarity),
'database': tld.extract(match[3].replace('\\', '')).domain 'database': tld.extract(content['results'][0]['sources'][0]['source']).domain
} }
return result return result

39
src/utils/utils.py
View file

@ -84,19 +84,36 @@ last_commands = {}
asession = aiohttp.ClientSession() asession = aiohttp.ClientSession()
async def fetch(url, *, json=False, response=False, text=False): async def fetch(url, *, post={}, response=False, text=False, json=False):
if '.json' in url and ('e621' in url or 'e926' in url): if '.json' in url and ('e621' in url or 'e926' in url):
url += f'&login=BotMyned&api_key={config["e621_api"]}' url += f'&login=BotMyned&api_key={config["e621_api"]}'
async with asession.get(url, headers={
'User-Agent': 'Myned/Modufur (https://github.com/Myned/Modufur)'}, ssl=False) as r: if post:
if response: async with asession.post(url, data=post, headers={
return r 'User-Agent': 'Myned/Modufur (https://github.com/Myned/Modufur)'}, ssl=False) as r:
elif json: if r.status != 200:
return await r.json() return r.status
elif text: elif response:
return await r.text() return r
else: elif text:
return await r.read() return await r.text()
elif json:
return await r.json()
else:
return await r.read()
else:
async with asession.get(url, headers={
'User-Agent': 'Myned/Modufur (https://github.com/Myned/Modufur)'}, ssl=False) as r:
if r.status != 200:
return r.status
elif response:
return r
elif text:
return await r.text()
elif json:
return await r.json()
else:
return await r.read()
def generate_embed(ctx, *, title=d.Embed.Empty, kind='rich', description=d.Embed.Empty, url=d.Embed.Empty, timestamp=d.Embed.Empty, colour=color, footer={}, image=d.Embed.Empty, thumbnail=d.Embed.Empty, author={}, fields=[]): def generate_embed(ctx, *, title=d.Embed.Empty, kind='rich', description=d.Embed.Empty, url=d.Embed.Empty, timestamp=d.Embed.Empty, colour=color, footer={}, image=d.Embed.Empty, thumbnail=d.Embed.Empty, author={}, fields=[]):