feat: initial commit
This commit is contained in:
31
service/scrapers/dynamic.py
Normal file
31
service/scrapers/dynamic.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from scrapling import PlayWrightFetcher
|
||||
|
||||
from ..models.request import ScrapeRequest
|
||||
from ..models.response import ScrapeResponse
|
||||
from .base import BaseScraper
|
||||
|
||||
|
||||
class DynamicScraper(BaseScraper):
|
||||
"""Wraps Scrapling's PlayWrightFetcher — full browser via Playwright."""
|
||||
|
||||
async def scrape(self, req: ScrapeRequest) -> ScrapeResponse:
|
||||
start = time.perf_counter()
|
||||
|
||||
kwargs: dict = {
|
||||
"url": req.url,
|
||||
"headless": req.headless,
|
||||
"timeout": req.timeout,
|
||||
"network_idle": req.network_idle,
|
||||
}
|
||||
if req.wait_selector:
|
||||
kwargs["wait_selector"] = req.wait_selector
|
||||
if req.proxy:
|
||||
kwargs["proxy"] = req.proxy
|
||||
|
||||
fetcher = PlayWrightFetcher(auto_match=False)
|
||||
page = await fetcher.async_fetch(**kwargs)
|
||||
return self._build_response(req, page, "dynamic", start)
|
||||
Reference in New Issue
Block a user