user01/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix for load metadata from amazon

Browse Source
This commit is contained in:
Ozzie Isaacs
2024-09-14 10:23:11 +02:00
parent 481237dc77
commit 88cb58c285
2 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cps/metadata_provider/amazon.py
+13 -11
View File
@@ -38,14 +38,16 @@ class Amazon(Metadata):
__name__ = "Amazon"
__id__ = "amazon"
headers = {'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'sec-gpc': '1',
'sec-fetch-site': 'none',
'sec-fetch-mode': 'navigate',
'sec-fetch-user': '?1',
'sec-fetch-dest': 'document',
'accept-encoding': 'gzip, deflate, br',
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Upgrade-Insecure-Requests': '1',
'Alt-Used' : 'www.amazon.com',
'Priority' : 'u=0, i',
'accept-encoding': 'gzip, deflate, br, zstd',
'accept-language': 'en-US,en;q=0.9'}
session = requests.Session()
session.headers=headers
@@ -62,7 +64,7 @@ class Amazon(Metadata):
log.warning(ex)
return []
long_soup = BS(r.text, "lxml") #~4sec :/
soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-ppd_csm_instrumentation_wrapper"})
if soup2 is None:
return []
try:
@@ -106,7 +108,7 @@ class Amazon(Metadata):
except (AttributeError, ValueError):
match.rating = 0
try:
match.cover = soup2.find("img", attrs={"class": "a-dynamic-image frontImage"})["src"]
match.cover = soup2.find("img", attrs={"class": "a-dynamic-image"})["src"]
except (AttributeError, TypeError):
match.cover = ""
return match, index
@@ -132,7 +134,7 @@ class Amazon(Metadata):
links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
soup.findAll("div", attrs={"data-component-type": "s-search-result"})]
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])}
fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:3])}
val = list(map(lambda x : x.result(), concurrent.futures.as_completed(fut)))
result = list(filter(lambda x: x, val))
return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance