myned/modufur
1
0
Fork 0
mirror of https://github.com/myned/modufur.git synced 2024-12-25 14:47:29 +00:00
Code Issues Activity

Merge branch 'dev'

Browse source
This commit is contained in:
Myned 2019-11-12 15:28:21 -05:00
commit 9a00e704fa
No known key found for this signature in database
GPG key ID: BC58C09870A63E59
1 changed files with 19 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
src/utils/scraper.py
View file

@ -51,7 +51,7 @@ async def query_kheina(url):
soup = BeautifulSoup(content, 'html5lib') soup = BeautifulSoup(content, 'html5lib')
if soup.find('data', id='error'): if soup.find('data', id='error'):
return False return None
results = soup.find('data', id='results').string results = soup.find('data', id='results').string
results = ast.literal_eval(results) results = ast.literal_eval(results)
@ -60,7 +60,7 @@ async def query_kheina(url):
similarity = int(float(iqdbdata[0]['similarity'])) similarity = int(float(iqdbdata[0]['similarity']))
if similarity < 55: if similarity < 55:
return False return None
for e in results: for e in results:
if iqdbdata[0]['iqdbid'] in e: if iqdbdata[0]['iqdbid'] in e:
@ -78,7 +78,7 @@ async def query_kheina(url):
return result return result
except Exception: except Exception:
return False return None
async def query_saucenao(url): async def query_saucenao(url):
@ -98,9 +98,13 @@ async def query_saucenao(url):
similarity = int(float(match['header']['similarity'])) similarity = int(float(match['header']['similarity']))
if similarity < 55: if similarity < 55:
return False return None
source = match['data']['ext_urls'][0] source = match['data']['ext_urls'][0]
for e in match['data']['ext_urls']:
if 'furaffinity' in e:
source = e
break
for e in match['data']['ext_urls']: for e in match['data']['ext_urls']:
if 'e621' in e: if 'e621' in e:
source = e source = e
@ -126,7 +130,7 @@ async def query_saucenao(url):
return result return result
except Exception: except Exception:
return False return None
async def get_post(url): async def get_post(url):
@ -136,9 +140,16 @@ async def get_post(url):
if filesize > 8192 * 1024: if filesize > 8192 * 1024:
raise exc.SizeError(size(filesize, system=alternative)) raise exc.SizeError(size(filesize, system=alternative))
result = await query_kheina(url) # Prioritize SauceNAO if e621/furaffinity, Kheina>SauceNAO if not
if not result: result = await query_saucenao(url)
result = await query_saucenao(url) if result:
if not any(s in result['source'] for s in ('e621', 'furaffinity')):
kheina = await query_kheina(url)
if kheina:
result = kheina
else:
result = await query_kheina(url)
if not result: if not result:
raise exc.MatchError(re.search('\\/([^\\/]+)$', url).group(1)) raise exc.MatchError(re.search('\\/([^\\/]+)$', url).group(1))