myned/modufur
1
0
Fork 0
mirror of https://github.com/myned/modufur.git synced 2024-11-01 13:02:38 +00:00
Code Issues Activity

Remove aiohttp params argument in favor of str due to encoding issues

Browse source
This commit is contained in:
Myned 2020-03-09 11:29:56 -04:00
parent 152aafea6e
commit d6ec61b648
No known key found for this signature in database
GPG key ID: 2EF9C0C44229D034
2 changed files with 16 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/cogs/booru.py
View file

@ -94,7 +94,7 @@ class MsG(cmds.Cog):
print(f'Last updated: {self.suggested["last_update"]}') print(f'Last updated: {self.suggested["last_update"]}')
print('Updating tags...') print('Updating tags...')
content = await u.fetch('https://e621.net/tag/index.json', params={'order': 'count', 'limit': 500, 'page': page}, json=True) content = await u.fetch(f'https://e621.net/tag/index.json?order=count&limit={500}&page={page}', json=True)
while content: while content:
for tag in content: for tag in content:
self.suggested['tags'][tag['name']] = tag['count'] self.suggested['tags'][tag['name']] = tag['count']
@ -102,7 +102,7 @@ class MsG(cmds.Cog):
print(f' UPDATED : PAGE {page} / {pages}', end='\r') print(f' UPDATED : PAGE {page} / {pages}', end='\r')
page += 1 page += 1
content = await u.fetch('https://e621.net/tag/index.json', params={'order': 'count', 'limit': 500, 'page': page}, json=True) content = await u.fetch(f'https://e621.net/tag/index.json?order=count&limit={500}&page={page}', json=True)
u.dump(self.suggested, 'cogs/suggested.pkl') u.dump(self.suggested, 'cogs/suggested.pkl')
self.suggested['last_update'] = time.strftime('%d/%m/%Y/%H:%M:%S') self.suggested['last_update'] = time.strftime('%d/%m/%Y/%H:%M:%S')
@ -232,7 +232,7 @@ class MsG(cmds.Cog):
await ctx.trigger_typing() await ctx.trigger_typing()
for tag in tags: for tag in tags:
tag_request = await u.fetch('https://e621.net/tag/related.json', params={'tags': tag}, json=True) tag_request = await u.fetch(f'https://e621.net/tag/related.json?tags={tag}', json=True)
for rel in tag_request.get(tag, []): for rel in tag_request.get(tag, []):
related.append(rel[0]) related.append(rel[0])
@ -258,7 +258,7 @@ class MsG(cmds.Cog):
await ctx.trigger_typing() await ctx.trigger_typing()
for tag in tags: for tag in tags:
alias_request = await u.fetch('https://e621.net/tag_alias/index.json', params={'aliased_to': tag, 'approved': 'true'}, json=True) alias_request = await u.fetch(f'https://e621.net/tag_alias/index.json?aliased_to={tag}&approved=true', json=True)
for dic in alias_request: for dic in alias_request:
aliases.append(dic['name']) aliases.append(dic['name'])
@ -293,7 +293,7 @@ class MsG(cmds.Cog):
ident = ident if not ident.isdigit() else re.search( ident = ident if not ident.isdigit() else re.search(
'show/([0-9]+)', ident).group(1) 'show/([0-9]+)', ident).group(1)
post = await u.fetch('https://e621.net/post/show.json', params={'id': ident}, json=True) post = await u.fetch(f'https://e621.net/posts/{ident}.json', json=True)
embed = d.Embed( embed = d.Embed(
title=', '.join(post['artist']), url=f'https://e621.net/post/show/{post["id"]}', color=ctx.me.color if isinstance(ctx.channel, d.TextChannel) else u.color) title=', '.join(post['artist']), url=f'https://e621.net/post/show/{post["id"]}', color=ctx.me.color if isinstance(ctx.channel, d.TextChannel) else u.color)
@ -352,7 +352,7 @@ class MsG(cmds.Cog):
await ctx.trigger_typing() await ctx.trigger_typing()
pools = [] pools = []
pool_request = await u.fetch('https://e621.net/pool/index.json', params={'query': ' '.join(query)}, json=True) pool_request = await u.fetch(f'https://e621.net/pools.json?search[name_matches]={" ".join(query)}', json=True)
if len(pool_request) > 1: if len(pool_request) > 1:
for pool in pool_request: for pool in pool_request:
pools.append(pool['name']) pools.append(pool['name'])
@ -625,7 +625,7 @@ class MsG(cmds.Cog):
try: try:
pools = [] pools = []
pool_request = await u.fetch('https://{}.net/pool/index.json'.format(booru), params={'query': ' '.join(query)}, json=True) pool_request = await u.fetch(f'https://{booru}.net/pools.json?search[name_matches]={" ".join(query)}', json=True)
if len(pool_request) > 1: if len(pool_request) > 1:
for pool in pool_request: for pool in pool_request:
pools.append(pool['name']) pools.append(pool['name'])
@ -707,8 +707,8 @@ class MsG(cmds.Cog):
while len(posts) < limit: while len(posts) < limit:
if c == limit * 5 + (self.LIMIT / 5): if c == limit * 5 + (self.LIMIT / 5):
raise exc.Timeout raise exc.Timeout
request = await u.fetch('https://{}.net/post/index.json'.format(booru), params={'tags': ','.join([order] + tags), 'limit': int(self.LIMIT * limit)}, json=True)
if len(request) == 0: request = await u.fetch(f'https://{booru}.net/posts.json?tags={"+".join([order] + tags)}&limit={int(320)}', json=True)
raise exc.NotFound(' '.join(tags)) raise exc.NotFound(' '.join(tags))
if len(request) < limit: if len(request) < limit:
limit = len(request) limit = len(request)

17
src/utils/scraper.py
View file

@ -12,25 +12,23 @@ from utils import utils as u
# async def get_harry(url): # async def get_harry(url):
# content = await u.fetch('https://iqdb.harry.lu', params={'url': url}) # content = await u.fetch(f'https://iqdb.harry.lu?url={url}')
# soup = BeautifulSoup(content, 'html5lib') # soup = BeautifulSoup(content, 'html5lib')
# #
# if soup.find('div', id='show1').string is 'Not the right one? ': # if soup.find('div', id='show1').string is 'Not the right one? ':
# parent = soup.find('th', string='Probable match:').parent.parent # parent = soup.find('th', string='Probable match:').parent.parent
# #
# post = await u.fetch( # post = await u.fetch(
# 'https://e621.net/post/show.json', # f'https://e621.net/posts.json?id={re.search("show/([0-9]+)", parent.tr.td.a.get('href')).group(1)}',
# params={'id': re.search('show/([0-9]+)', parent.tr.td.a.get('href')).group(1)},
# json=True) # json=True)
# if (post['status'] == 'deleted'): # if (post['status'] == 'deleted'):
# post = await u.fetch( # post = await u.fetch(
# 'https://e621.net/post/show.json', # f'https://e621.net/posts.json?id={re.search("#(\\d+)", post["delreason"]).group(1)}',
# params={'id': re.search('#(\\d+)', post['delreason']).group(1)},
# json=True) # json=True)
# #
# result = { # result = {
# 'source': f'https://e621.net/post/show/{post["id"]}', # 'source': f'https://e621.net/posts/{post["id"]}',
# 'artist': ', '.join(post['artist']), # 'artist': ', '.join(post['tags']['artist']),
# 'thumbnail': parent.td.a.img.get('src'), # 'thumbnail': parent.td.a.img.get('src'),
# 'similarity': re.search('\\d+', parent.tr[4].td.string).group(0), # 'similarity': re.search('\\d+', parent.tr[4].td.string).group(0),
# 'database': 'Harry.lu' # 'database': 'Harry.lu'
@ -43,7 +41,7 @@ from utils import utils as u
async def query_kheina(url): async def query_kheina(url):
try: try:
content = await u.fetch('https://kheina.com', params={'url': url}, text=True) content = await u.fetch(f'https://kheina.com?url={url}', text=True)
for e in ('&quot;', '&apos;'): for e in ('&quot;', '&apos;'):
content = content.replace(e, '') content = content.replace(e, '')
@ -85,8 +83,7 @@ async def query_kheina(url):
async def query_saucenao(url): async def query_saucenao(url):
try: try:
content = await u.fetch( content = await u.fetch(
'https://saucenao.com/search.php', f'https://saucenao.com/search.php?url={url}&api_key={u.config["saucenao_api"]}&output_type={2}',
params={'url': url, 'api_key': u.config['saucenao_api'], 'output_type': 2},
json=True) json=True)
if content['header'].get('message', '') in ( if content['header'].get('message', '') in (