From da6ad9ba8e860a8b630db0d571eb0b0e9b47b86d Mon Sep 17 00:00:00 2001 From: Myned Date: Thu, 12 Oct 2017 22:30:40 -0400 Subject: [PATCH] Removed redundant scraper methods, autopep8 --- src/main/utils/scraper.py | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/src/main/utils/scraper.py b/src/main/utils/scraper.py index 6e3bb2e..818e41c 100644 --- a/src/main/utils/scraper.py +++ b/src/main/utils/scraper.py @@ -1,33 +1,17 @@ -import requests +import aiohttp as aio from bs4 import BeautifulSoup from lxml import html -from misc import exceptions as exc -def check_match(url): - r = requests.get(url) - soup = BeautifulSoup(r.content, 'html.parser') +from misc import exceptions as exc +from utils import utils as u + + +async def check_match(url): + r = await u.fetch('http://iqdb.harry.lu/?url={}'.format(url)) + soup = BeautifulSoup(await r.read(), 'html.parser') value = soup.find_all('a')[1].get('href') + if value != '#': return value else: raise exc.MatchError(value) - -def find_pool(url): - r = requests.get(url) - tree = html.fromstring(r.content) - post = tree.xpath('/html/body/div[@id="content"]/div[@id="pool-show"]/div[@style="margin-top: 2em;"]/span/a/@href') - print(post) - if post: - return post - else: - raise exc.PostError(post) - -def find_image_url(url): - r = requests.get(url) - tree = html.fromstring(r.content) - image_url = tree.xpath('/html/body/div[@id="content"]/div[@id="post-view"]/div[@class="content"]/div/img/@src') - print(image_url) - if image_url: - return image_url - else: - raise exc.ImageError(image_url)