diff --git a/cps/metadata_provider/amazon.py b/cps/metadata_provider/amazon.py index ee3de1ceb..5b07819cb 100644 --- a/cps/metadata_provider/amazon.py +++ b/cps/metadata_provider/amazon.py @@ -38,14 +38,16 @@ class Amazon(Metadata): __name__ = "Amazon" __id__ = "amazon" headers = {'upgrade-insecure-requests': '1', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36', - 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'sec-gpc': '1', - 'sec-fetch-site': 'none', - 'sec-fetch-mode': 'navigate', - 'sec-fetch-user': '?1', - 'sec-fetch-dest': 'document', - 'accept-encoding': 'gzip, deflate, br', + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0', + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8', + 'Sec-Fetch-Site': 'same-origin', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-User': '?1', + 'Sec-Fetch-Dest': 'document', + 'Upgrade-Insecure-Requests': '1', + 'Alt-Used' : 'www.amazon.com', + 'Priority' : 'u=0, i', + 'accept-encoding': 'gzip, deflate, br, zstd', 'accept-language': 'en-US,en;q=0.9'} session = requests.Session() session.headers=headers @@ -62,7 +64,7 @@ class Amazon(Metadata): log.warning(ex) return [] long_soup = BS(r.text, "lxml") #~4sec :/ - soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"}) + soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-ppd_csm_instrumentation_wrapper"}) if soup2 is None: return [] try: @@ -106,7 +108,7 @@ class Amazon(Metadata): except (AttributeError, ValueError): match.rating = 0 try: - match.cover = soup2.find("img", attrs={"class": "a-dynamic-image frontImage"})["src"] + match.cover = soup2.find("img", attrs={"class": "a-dynamic-image"})["src"] except (AttributeError, TypeError): match.cover = "" return match, index @@ -132,7 +134,7 @@ class Amazon(Metadata): links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in soup.findAll("div", attrs={"data-component-type": "s-search-result"})] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])} + fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:3])} val = list(map(lambda x : x.result(), concurrent.futures.as_completed(fut))) result = list(filter(lambda x: x, val)) return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance diff --git a/cps/search_metadata.py b/cps/search_metadata.py index b818a87c0..719331f0f 100644 --- a/cps/search_metadata.py +++ b/cps/search_metadata.py @@ -33,7 +33,6 @@ from cps.services.Metadata import Metadata from . import constants, logger, ub, web_server from .usermanagement import user_login_required -# current_milli_time = lambda: int(round(time() * 1000)) meta = Blueprint("metadata", __name__) @@ -130,7 +129,7 @@ def metadata_search(): locale = get_locale() if query: static_cover = url_for("static", filename="generic_cover.jpg") - # start = current_milli_time() + # ret = cl[0].search(query, static_cover, locale) with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: meta = { executor.submit(c.search, query, static_cover, locale): c @@ -139,5 +138,4 @@ def metadata_search(): } for future in concurrent.futures.as_completed(meta): data.extend([asdict(x) for x in future.result() if x]) - # log.info({'Time elapsed {}'.format(current_milli_time()-start)}) return Response(json.dumps(data), mimetype="application/json")