Skip to content

Commit eecd466

Browse files
grayespinozagrayespinoza
committed
perf: scrape in parallel and reuse browser contexts
1 parent 2b0c1f3 commit eecd466

10 files changed

Lines changed: 282 additions & 277 deletions

File tree

‎graphicnovelpricescraper/data.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
@dataclass
55
class GraphicNovel:
6-
isbn: str
6+
isbn: int
77
retailer: str
88
title: str | None = None
99
price: float | None = None

‎graphicnovelpricescraper/retailers/amazon.py‎

Lines changed: 52 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -3,58 +3,56 @@
33

44

55
class Amazon(Scraper):
6-
def __init__(self, browser):
7-
self.browser = browser
8-
9-
async def scrape(self, isbn: str, title: str) -> GraphicNovel:
10-
context = await self.browser.new_context(
11-
user_agent="Mozilla/5.0 ... Chrome/137",
12-
viewport={"width": 1280, "height": 800},
13-
locale="en-US",
14-
)
15-
16-
page = await context.new_page()
17-
page.set_default_timeout(8000)
18-
19-
title = ""
20-
price = None
21-
url = None
22-
23-
try:
24-
await page.goto(
25-
f"https://www.amazon.com/s?k={isbn}&i=stripbooks", timeout=12000
6+
async def scrape(self, isbn: int, title: str) -> GraphicNovel:
7+
async with self.semaphore:
8+
if self.context is None:
9+
raise RuntimeError("Context not initialized!")
10+
11+
page = await self.context.new_page()
12+
13+
title = ""
14+
price = None
15+
url = None
16+
17+
try:
18+
await page.goto(
19+
f"https://www.amazon.com/s?k={isbn}&i=stripbooks",
20+
timeout=12000,
21+
wait_until="domcontentloaded",
22+
)
23+
24+
title = await page.locator(
25+
"div[data-cy='title-recipe'] h2 span"
26+
).first.text_content()
27+
if title:
28+
title = title.strip()
29+
30+
price_whole = await page.locator(
31+
"span.a-price-whole"
32+
).first.text_content()
33+
price_fraction = await page.locator(
34+
"span.a-price-fraction"
35+
).first.text_content()
36+
if price_whole and price_fraction:
37+
price_whole = price_whole.strip()
38+
price_fraction = price_fraction.strip()
39+
price = float(f"{price_whole}{price_fraction}")
40+
41+
href = await page.locator(
42+
"div[data-cy='title-recipe'] a"
43+
).first.get_attribute("href")
44+
if href:
45+
href = href.strip().split("?", 1)[0]
46+
url = f"https://www.amazon.com{href}"
47+
except Exception as e:
48+
print(f"Unable to scrape {isbn} from Amazon: {e}")
49+
finally:
50+
await page.close()
51+
52+
return GraphicNovel(
53+
isbn=isbn,
54+
retailer="Amazon",
55+
title=title,
56+
price=price,
57+
url=url,
2658
)
27-
28-
title = await page.locator(
29-
"div[data-cy='title-recipe'] h2 span"
30-
).first.text_content()
31-
if title:
32-
title = title.strip()
33-
34-
price_whole = await page.locator("span.a-price-whole").first.text_content()
35-
price_fraction = await page.locator(
36-
"span.a-price-fraction"
37-
).first.text_content()
38-
if price_whole and price_fraction:
39-
price_whole = price_whole.strip()
40-
price_fraction = price_fraction.strip()
41-
price = float(f"{price_whole}{price_fraction}")
42-
43-
href = await page.locator(
44-
"div[data-cy='title-recipe'] a"
45-
).first.get_attribute("href")
46-
if href:
47-
href = href.strip().split("?", 1)[0]
48-
url = f"https://www.amazon.com{href}"
49-
except Exception as e:
50-
print(f"Unable to scrape {isbn} from Amazon: {e}")
51-
finally:
52-
await context.close()
53-
54-
return GraphicNovel(
55-
isbn=isbn,
56-
retailer="Amazon",
57-
title=title,
58-
price=price,
59-
url=url,
60-
)

‎graphicnovelpricescraper/retailers/barnesandnoble.py‎

Lines changed: 50 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -3,56 +3,54 @@
33

44

55
class BarnesAndNoble(Scraper):
6-
def __init__(self, browser):
7-
self.browser = browser
8-
9-
async def scrape(self, isbn: str, title: str) -> GraphicNovel:
10-
context = await self.browser.new_context(
11-
user_agent="Mozilla/5.0 ... Chrome/137",
12-
viewport={"width": 1280, "height": 800},
13-
locale="en-US",
14-
)
15-
16-
page = await context.new_page()
17-
page.set_default_timeout(8000)
18-
19-
title = ""
20-
price = None
21-
url = None
22-
23-
try:
24-
await page.goto(
25-
f"https://www.barnesandnoble.com/search?q={isbn}", timeout=12000
6+
async def scrape(self, isbn: int, title: str) -> GraphicNovel:
7+
async with self.semaphore:
8+
if self.context is None:
9+
raise RuntimeError("Context not initialized!")
10+
11+
page = await self.context.new_page()
12+
13+
title = ""
14+
price = None
15+
url = None
16+
17+
try:
18+
await page.goto(
19+
f"https://www.barnesandnoble.com/search?q={isbn}",
20+
timeout=12000,
21+
wait_until="domcontentloaded",
22+
)
23+
24+
title = await page.locator(
25+
".product-item-card__title"
26+
).first.text_content()
27+
if title:
28+
title = title.strip()
29+
30+
price_text = await page.locator(
31+
".product-item-card__current-price"
32+
).first.text_content()
33+
if price_text:
34+
price_text = price_text.strip()
35+
price = float(price_text.replace("$", ""))
36+
37+
href = (
38+
await page.locator(".product-item-card__title")
39+
.first.locator("xpath=ancestor::a")
40+
.get_attribute("href")
41+
)
42+
if href:
43+
href = href.strip().split("?", 1)[0]
44+
url = f"https://www.barnesandnoble.com{href}"
45+
except Exception as e:
46+
print(f"Unable to scrape {isbn} from Barnes & Noble: {e}")
47+
finally:
48+
await page.close()
49+
50+
return GraphicNovel(
51+
isbn=isbn,
52+
retailer="Barnes & Noble",
53+
title=title,
54+
price=price,
55+
url=url,
2656
)
27-
28-
title = await page.locator(".product-item-card__title").first.text_content()
29-
if title:
30-
title = title.strip()
31-
32-
price_text = await page.locator(
33-
".product-item-card__current-price"
34-
).first.text_content()
35-
if price_text:
36-
price_text = price_text.strip()
37-
price = float(price_text.replace("$", ""))
38-
39-
href = (
40-
await page.locator(".product-item-card__title")
41-
.first.locator("xpath=ancestor::a")
42-
.get_attribute("href")
43-
)
44-
if href:
45-
href = href.strip().split("?", 1)[0]
46-
url = f"https://www.barnesandnoble.com{href}"
47-
except Exception as e:
48-
print(f"Unable to scrape {isbn} from Barnes & Noble: {e}")
49-
finally:
50-
await context.close()
51-
52-
return GraphicNovel(
53-
isbn=isbn,
54-
retailer="Barnes & Noble",
55-
title=title,
56-
price=price,
57-
url=url,
58-
)

‎graphicnovelpricescraper/retailers/cheapgraphicnovels.py‎

Lines changed: 44 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,55 +3,48 @@
33

44

55
class CheapGraphicNovels(Scraper):
6-
def __init__(self, browser):
7-
self.browser = browser
8-
9-
async def scrape(self, isbn: str, title: str) -> GraphicNovel:
10-
context = await self.browser.new_context(
11-
user_agent="Mozilla/5.0 ... Chrome/137",
12-
viewport={"width": 1280, "height": 800},
13-
locale="en-US",
14-
)
15-
16-
page = await context.new_page()
17-
page.set_default_timeout(8000)
18-
19-
title = ""
20-
price = None
21-
url = None
22-
23-
# TODO: filter out (NICK AND DENT)
24-
# Using ".fn.url:not(:has-text('Nick and Dent'))" works, but we would need a different solution for price.
25-
# Tried f"https://cheapgraphicnovels.com/?target=search&mode=search&substring={isbn}&sortOrder=asc", but that has no effect.
26-
27-
try:
28-
await page.goto(
29-
f"https://cheapgraphicnovels.com/?target=search&mode=search&substring={isbn}",
30-
timeout=12000,
6+
async def scrape(self, isbn: int, title: str) -> GraphicNovel:
7+
async with self.semaphore:
8+
if self.context is None:
9+
raise RuntimeError("Context not initialized!")
10+
11+
page = await self.context.new_page()
12+
13+
title = ""
14+
price = None
15+
url = None
16+
17+
try:
18+
await page.goto(
19+
f"https://cheapgraphicnovels.com/?target=search&mode=search&substring={isbn}",
20+
timeout=12000,
21+
wait_until="domcontentloaded",
22+
)
23+
24+
title = await page.locator(".fn.url").first.text_content()
25+
if title:
26+
title = title.strip()
27+
28+
price_text = await page.locator(
29+
".price.product-price"
30+
).first.text_content()
31+
if price_text:
32+
price_text = price_text.strip()
33+
price = float(price_text.replace("$", ""))
34+
35+
href = await page.locator(".fn.url").first.get_attribute("href")
36+
if href:
37+
href = href.strip().split("?", 1)[0]
38+
url = f"https://cheapgraphicnovels.com/{href}"
39+
except Exception as e:
40+
print(f"Unable to scrape {isbn} from Cheap Graphic Novels: {e}")
41+
finally:
42+
await page.close()
43+
44+
return GraphicNovel(
45+
isbn=isbn,
46+
retailer="Cheap Graphic Novels",
47+
title=title,
48+
price=price,
49+
url=url,
3150
)
32-
33-
title = await page.locator(".fn.url").first.text_content()
34-
if title:
35-
title = title.strip()
36-
37-
price_text = await page.locator(".price.product-price").first.text_content()
38-
if price_text:
39-
price_text = price_text.strip()
40-
price = float(price_text.replace("$", ""))
41-
42-
href = await page.locator(".fn.url").first.get_attribute("href")
43-
if href:
44-
href = href.strip().split("?", 1)[0]
45-
url = f"https://cheapgraphicnovels.com/{href}"
46-
except Exception as e:
47-
print(f"Unable to scrape {isbn} from Cheap Graphic Novels: {e}")
48-
finally:
49-
await context.close()
50-
51-
return GraphicNovel(
52-
isbn=isbn,
53-
retailer="Cheap Graphic Novels",
54-
title=title,
55-
price=price,
56-
url=url,
57-
)

0 commit comments

Comments
 (0)