How to use the ruia.response.Response function in ruia

To help you get started, we’ve selected a few ruia examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github howie6879 / ruia / ruia / request.py View on Github external
async def fetch(self) -> Response:
        """Fetch all the information by using aiohttp"""
        if self.request_config.get("DELAY", 0) > 0:
            await asyncio.sleep(self.request_config["DELAY"])

        timeout = self.request_config.get("TIMEOUT", 10)
        try:
            async with async_timeout.timeout(timeout):
                resp = await self._make_request()
            try:
                resp_data = await resp.text(encoding=self.encoding)
            except UnicodeDecodeError:
                resp_data = await resp.read()

            response = Response(
                url=self.url,
                method=self.method,
                encoding=resp.get_encoding(),
                html=resp_data,
                metadata=self.metadata,
                cookies=resp.cookies,
                headers=resp.headers,
                history=resp.history,
                status=resp.status,
                aws_json=resp.json,
                aws_text=resp.text,
                aws_read=resp.read,
            )
            # Retry middleware
            aws_valid_response = self.request_config.get("VALID")
            if aws_valid_response and iscoroutinefunction(aws_valid_response):
github python-ruia / ruia-pyppeteer / ruia_pyppeteer / response.py View on Github external
#!/usr/bin/env python
from typing import Callable

from ruia.response import Response


class PyppeteerResponse(Response):
    def __init__(
        self,
        url: str,
        method: str,
        *,
        encoding: str = "",
        html: str = "",
        page,
        browser,
        metadata: dict,
        cookies,
        history,
        headers: dict = None,
        status: int = -1,
        aws_json: Callable = None,
        aws_read: Callable = None,
github python-ruia / ruia-pyppeteer / ruia_pyppeteer / request.py View on Github external
page=page,
                    browser=self.browser,
                    metadata=self.metadata,
                    cookies=await page.cookies(),
                    headers=resp.headers,
                    history=(),
                    status=resp.status,
                    aws_json=resp.json,
                    aws_text=resp.text,
                    aws_read=resp.buffer,
                )
            else:
                async with async_timeout.timeout(timeout):
                    resp = await self._make_request()
                resp_data = await resp.text(encoding=self.encoding)
                response = Response(
                    url=self.url,
                    method=self.method,
                    encoding=resp.get_encoding(),
                    html=resp_data,
                    metadata=self.metadata,
                    cookies=resp.cookies,
                    headers=resp.headers,
                    history=resp.history,
                    status=resp.status,
                    aws_json=resp.json,
                    aws_text=resp.text,
                    aws_read=resp.read,
                )
            if not response.ok:
                return await self._retry(
                    error_msg=f"Request url failed with status {response.status}!"
github howie6879 / ruia / ruia / request.py View on Github external
async def _retry(self, error_msg):
        """Manage request"""
        if self.retry_times > 0:
            retry_times = self.request_config.get("RETRIES", 3) - self.retry_times + 1
            self.logger.error(
                f", Retry times: {retry_times}, Retry message: {error_msg}>"
            )
            self.retry_times -= 1
            retry_func = self.request_config.get("RETRY_FUNC")
            if retry_func and iscoroutinefunction(retry_func):
                request_ins = await retry_func(weakref.proxy(self))
                if isinstance(request_ins, Request):
                    return await request_ins.fetch()
            return await self.fetch()
        else:
            response = Response(
                url=self.url,
                method=self.method,
                metadata=self.metadata,
                cookies={},
                history=(),
                headers=None,
            )

            return response
github howie6879 / ruia / ruia / spider.py View on Github external
    async def handle_request(self, request: Request) -> typing.Tuple[AsyncGeneratorType, Response]:
        """
        Wrap request with middleware.
        :param request:
        :return:
        """
        callback_result, response = None, None

        await self._run_request_middleware(request)
        try:
            callback_result, response = await request.fetch_callback(self.sem)
        except NotImplementedParseError as e:
            self.logger.error(e)
        except NothingMatchedError as e:
            self.logger.error(f'')
        except Exception as e:
            self.logger.error(f'
github howie6879 / ruia / ruia / spider.py View on Github external
async def handle_request(
        self, request: Request
    ) -> typing.Tuple[AsyncGeneratorType, Response]:
        """
        Wrap request with middleware.
        :param request:
        :return:
        """
        callback_result, response = None, None

        await self._run_request_middleware(request)

        try:
            callback_result, response = await request.fetch_callback(self.sem)
        except NotImplementedParseError as e:
            self.logger.error(e)
        except NothingMatchedError as e:
            self.logger.error(f"")
        except Exception as e:
github howie6879 / ruia / ruia / request.py View on Github external
async def fetch_callback(
        self, sem: Semaphore
    ) -> Tuple[AsyncGeneratorType, Response]:
        """
        Request the target url and then call the callback function
        :param sem: Semaphore
        :return: Tuple[AsyncGeneratorType, Response]
        """
        try:
            async with sem:
                response = await self.fetch()
        except Exception as e:
            response = None
            self.logger.error(f"")

        if self.callback is not None:
            if iscoroutinefunction(self.callback):
                callback_result = await self.callback(response)
                response.callback_result = callback_result