myned/modufur
1
0
Fork 0
mirror of https://github.com/myned/modufur.git synced 2024-12-25 14:47:29 +00:00
Code Issues Activity

Fix www appearing as domain

Browse source
This commit is contained in:
Myned 2019-11-30 22:39:36 -05:00
parent ffee58656c
commit 861259ec01
No known key found for this signature in database
GPG key ID: BC58C09870A63E59
2 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Pipfile
View file

@ -21,6 +21,7 @@ beautifulsoup4 = "*"
"hurry.filesize" = "*" "hurry.filesize" = "*"
requests = "*" requests = "*"
html5lib = "*" html5lib = "*"
tldextract = "*"
[dev-packages] [dev-packages]
lxml = "*" lxml = "*"

6
src/utils/scraper.py
View file

@ -1,11 +1,11 @@
import aiohttp import aiohttp
import ast import ast
import re import re
from urllib.parse import urlparse
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import lxml import lxml
from hurry.filesize import size, alternative from hurry.filesize import size, alternative
import tldextract as tld
from misc import exceptions as exc from misc import exceptions as exc
from utils import utils as u from utils import utils as u
@ -73,7 +73,7 @@ async def query_kheina(url):
'artist': match[4], 'artist': match[4],
'thumbnail': f'https://f002.backblazeb2.com/file/kheinacom/{match[1]}.jpg', 'thumbnail': f'https://f002.backblazeb2.com/file/kheinacom/{match[1]}.jpg',
'similarity': str(similarity), 'similarity': str(similarity),
'database': urlparse(match[3].replace('\\', '')).hostname.split('.')[0] 'database': tld.extract(match[3].replace('\\', '')).domain
} }
return result return result
@ -125,7 +125,7 @@ async def query_saucenao(url):
'artist': artist, 'artist': artist,
'thumbnail': match['header']['thumbnail'], 'thumbnail': match['header']['thumbnail'],
'similarity': str(similarity), 'similarity': str(similarity),
'database': urlparse(source).hostname.split('.')[0] 'database': tld.extract(source).domain
} }
return result return result