myned/modufur
1
0
Fork 0
mirror of https://github.com/myned/modufur.git synced 2024-11-01 21:02:38 +00:00
Code Issues Activity

Removed redundant scraper methods, autopep8

Browse source
This commit is contained in:
Myned 2017-10-12 22:30:40 -04:00
parent af9ed570a2
commit da6ad9ba8e
1 changed files with 9 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/main/utils/scraper.py
View file

@ -1,33 +1,17 @@
import requests import aiohttp as aio
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from lxml import html from lxml import html
from misc import exceptions as exc
def check_match(url): from misc import exceptions as exc
r = requests.get(url) from utils import utils as u
soup = BeautifulSoup(r.content, 'html.parser')
async def check_match(url):
r = await u.fetch('http://iqdb.harry.lu/?url={}'.format(url))
soup = BeautifulSoup(await r.read(), 'html.parser')
value = soup.find_all('a')[1].get('href') value = soup.find_all('a')[1].get('href')
if value != '#': if value != '#':
return value return value
else: else:
raise exc.MatchError(value) raise exc.MatchError(value)
def find_pool(url):
r = requests.get(url)
tree = html.fromstring(r.content)
post = tree.xpath('/html/body/div[@id="content"]/div[@id="pool-show"]/div[@style="margin-top: 2em;"]/span/a/@href')
print(post)
if post:
return post
else:
raise exc.PostError(post)
def find_image_url(url):
r = requests.get(url)
tree = html.fromstring(r.content)
image_url = tree.xpath('/html/body/div[@id="content"]/div[@id="post-view"]/div[@class="content"]/div/img/@src')
print(image_url)
if image_url:
return image_url
else:
raise exc.ImageError(image_url)