From 1941553161d4317361e5a6aec4b4705e4eb18074 Mon Sep 17 00:00:00 2001 From: Myned Date: Tue, 12 Nov 2019 15:23:11 -0500 Subject: [PATCH 1/3] Change return value for clarity --- src/utils/scraper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/utils/scraper.py b/src/utils/scraper.py index 20cb5bd..679b6bb 100644 --- a/src/utils/scraper.py +++ b/src/utils/scraper.py @@ -51,7 +51,7 @@ async def query_kheina(url): soup = BeautifulSoup(content, 'html5lib') if soup.find('data', id='error'): - return False + return None results = soup.find('data', id='results').string results = ast.literal_eval(results) @@ -60,7 +60,7 @@ async def query_kheina(url): similarity = int(float(iqdbdata[0]['similarity'])) if similarity < 55: - return False + return None for e in results: if iqdbdata[0]['iqdbid'] in e: @@ -78,7 +78,7 @@ async def query_kheina(url): return result except Exception: - return False + return None async def query_saucenao(url): @@ -98,7 +98,7 @@ async def query_saucenao(url): similarity = int(float(match['header']['similarity'])) if similarity < 55: - return False + return None source = match['data']['ext_urls'][0] for e in match['data']['ext_urls']: @@ -126,7 +126,7 @@ async def query_saucenao(url): return result except Exception: - return False + return None async def get_post(url): From b521bba22a825525e47bae9eb9a7a37600e82873 Mon Sep 17 00:00:00 2001 From: Myned Date: Tue, 12 Nov 2019 15:23:42 -0500 Subject: [PATCH 2/3] Add e621>furaffinity>other preference to SauceNAO query --- src/utils/scraper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/utils/scraper.py b/src/utils/scraper.py index 679b6bb..dada6fd 100644 --- a/src/utils/scraper.py +++ b/src/utils/scraper.py @@ -101,6 +101,10 @@ async def query_saucenao(url): return None source = match['data']['ext_urls'][0] + for e in match['data']['ext_urls']: + if 'furaffinity' in e: + source = e + break for e in match['data']['ext_urls']: if 'e621' in e: source = e From 68fc9bb54128f68e49ff55f14d60333376e83bae Mon Sep 17 00:00:00 2001 From: Myned Date: Tue, 12 Nov 2019 15:28:11 -0500 Subject: [PATCH 3/3] Change reverse image preference to SauceNAO and improve filtering --- src/utils/scraper.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/utils/scraper.py b/src/utils/scraper.py index dada6fd..17e9753 100644 --- a/src/utils/scraper.py +++ b/src/utils/scraper.py @@ -140,9 +140,16 @@ async def get_post(url): if filesize > 8192 * 1024: raise exc.SizeError(size(filesize, system=alternative)) - result = await query_kheina(url) - if not result: - result = await query_saucenao(url) + # Prioritize SauceNAO if e621/furaffinity, Kheina>SauceNAO if not + result = await query_saucenao(url) + if result: + if not any(s in result['source'] for s in ('e621', 'furaffinity')): + kheina = await query_kheina(url) + if kheina: + result = kheina + else: + result = await query_kheina(url) + if not result: raise exc.MatchError(re.search('\\/([^\\/]+)$', url).group(1))