myned/modufur
1
0
Fork 0
mirror of https://github.com/myned/modufur.git synced 2024-12-25 14:47:29 +00:00
Code Issues Activity

Unchanged file update

Browse source
This commit is contained in:
Myned 2019-11-12 14:11:52 -05:00
parent 70c4b13ee9
commit 2e56f6216d
No known key found for this signature in database
GPG key ID: BC58C09870A63E59
1 changed files with 46 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
src/utils/scraper.py
View file

@ -45,35 +45,35 @@ async def query_kheina(url):
content = await u.fetch('https://kheina.com', params={'url': url}, text=True) content = await u.fetch('https://kheina.com', params={'url': url}, text=True)
for e in ('"', '''): for e in ('"', '''):
content = content.replace(e, '') content = content.replace(e, '')
content = re.sub('<a href="/cdn-cgi/l/email-protection".+</a>', '', content) content = re.sub('<a href="/cdn-cgi/l/email-protection".+</a>', '', content)
soup = BeautifulSoup(content, 'html5lib') soup = BeautifulSoup(content, 'html5lib')
if soup.find('data', id='error'): if soup.find('data', id='error'):
return False return False
results = soup.find('data', id='results').string results = soup.find('data', id='results').string
results = ast.literal_eval(results) results = ast.literal_eval(results)
iqdbdata = soup.find('data', id='iqdbdata').string iqdbdata = soup.find('data', id='iqdbdata').string
iqdbdata = ast.literal_eval(iqdbdata) iqdbdata = ast.literal_eval(iqdbdata)
similarity = int(float(iqdbdata[0]['similarity'])) similarity = int(float(iqdbdata[0]['similarity']))
if similarity < 55: if similarity < 55:
return False return False
for e in results: for e in results:
if iqdbdata[0]['iqdbid'] in e: if iqdbdata[0]['iqdbid'] in e:
match = e match = e
break break
result = { result = {
'source': match[3].replace('\\', ''), 'source': match[3].replace('\\', ''),
'artist': match[4], 'artist': match[4],
'thumbnail': f'https://f002.backblazeb2.com/file/kheinacom/{match[1]}.jpg', 'thumbnail': f'https://f002.backblazeb2.com/file/kheinacom/{match[1]}.jpg',
'similarity': str(similarity), 'similarity': str(similarity),
'database': 'Kheina' 'database': 'Kheina'
} }
return result return result
@ -86,27 +86,27 @@ async def query_saucenao(url):
content = await u.fetch( content = await u.fetch(
'https://saucenao.com/search.php', 'https://saucenao.com/search.php',
params={'url': url, 'api_key': u.config['saucenao_api'], 'output_type': 2}, params={'url': url, 'api_key': u.config['saucenao_api'], 'output_type': 2},
json=True) json=True)
if content['header'].get('message', '') in ( if content['header'].get('message', '') in (
'Access to specified file was denied... ;_;', 'Access to specified file was denied... ;_;',
'Problem with remote server...', 'Problem with remote server...',
'image dimensions too small...'): 'image dimensions too small...'):
raise exc.ImageError raise exc.ImageError
match = content['results'][0] match = content['results'][0]
similarity = int(float(match['header']['similarity'])) similarity = int(float(match['header']['similarity']))
if similarity < 55: if similarity < 55:
return False return False
source = match['data']['ext_urls'][0] source = match['data']['ext_urls'][0]
for e in match['data']['ext_urls']: for e in match['data']['ext_urls']:
if 'e621' in e: if 'e621' in e:
source = e source = e
break break
artist = 'Unknown' artist = 'Unknown'
for e in ( for e in (
'author_name', 'author_name',
'member_name', 'member_name',
@ -115,13 +115,13 @@ async def query_saucenao(url):
artist = match['data'][e] artist = match['data'][e]
break break
result = { result = {
'source': source, 'source': source,
'artist': artist, 'artist': artist,
'thumbnail': match['header']['thumbnail'], 'thumbnail': match['header']['thumbnail'],
'similarity': str(similarity), 'similarity': str(similarity),
'database': 'SauceNAO' 'database': 'SauceNAO'
} }
return result return result