How to use @hint/utils-connector-tools - 10 common examples

public fetchContent(target: URL | string, customHeaders?: object): Promise {
        /*
         * TODO: This should create a new tab, navigate to the
         * resource and control what is received somehow via an event.
         */
        const assigns = compact([this && this._headers, customHeaders]);
        const headers = Object.assign({}, ...assigns);
        const href = typeof target === 'string' ? target : target.href;
        const options = {
            headers,
            // we sync the ignore SSL error options with `request`. This is neeeded for local https tests
            rejectUnauthorized: !this._options.ignoreHTTPSErrors,
            strictSSL: !this._options.ignoreHTTPSErrors
        };

        const request = new Requester(options);

        return request.get(href);
    }

private fetchContent(href: string, headers: HttpHeaders) {
        const options = {
            headers,
            // we sync the ignore SSL error options with `request`. This is neeeded for local https tests
            rejectUnauthorized: !this._ignoreHTTPSErrors,
            strictSSL: !this._ignoreHTTPSErrors
        };

        const request: Requester = new Requester(options);

        return request.get(href);
    }

private _fetchUrl(target: URL, customHeaders?: object): Promise {
        const uri: string = url.format(target);

        /* istanbul ignore else */
        if (!customHeaders) {
            return this.request.get(uri);
        }

        const r: Requester = new Requester({
            headers: customHeaders,
            rejectUnauthorized: !this._options.ignoreHTTPSErrors,
            strictSSL: !this._options.ignoreHTTPSErrors
        });

        return r.get(uri);
    }

public constructor(server: Engine, config?: any) {
        this._options = { ...defaultOptions, ...config };

        const requesterOptions = {
            rejectUnauthorized: !this._options.ignoreHTTPSErrors,
            strictSSL: !this._options.ignoreHTTPSErrors,
            ...this._options.requestOptions || {}
        };

        this.request = new Requester(requesterOptions);
        this.server = server;
        this._timeout = server.timeout;
        this._subprocesses = new Set();

        (server as Engine).on('parse::end::html', (event) => {
            /* istanbul ignore if */
            if (!this._originalDocument) {
                this._originalDocument = event.document;
            }
        });
    }

public fetchContent(target: URL | string, customHeaders?: object): Promise {
        /*
         * TODO: This should create a new tab, navigate to the
         * resource and control what is received somehow via an event.
         */
        const href: string = typeof target === 'string' ? target : target.href;
        const options = {
            headers: customHeaders || {},
            // we sync the ignore SSL error options with `request`. This is neeeded for local https tests
            rejectUnauthorized: !this._options.ignoreHTTPSErrors,
            strictSSL: !this._options.ignoreHTTPSErrors
        };

        const request: Requester = new Requester(options);

        return request.get(href);
    }

private async onRequest(request: puppeteer.Request) {
        /* istanbul ignore next */
        if (this.isIgnoredMethod(request.method())) {
            return;
        }

        if (request.isNavigationRequest()) {
            this._headers = normalizeHeaders(request.headers())!;
        }

        const { name, payload } = onRequestHandler(request);

        await this._engine.emitAsync(name, payload);
    }

debug(`Error finding element for request ${requestResponse.requestId}. element will be null`);
            }
        }

        const response: Response = requestResponse.getResponse(element);

        // Doing a check so TypeScript is happy during `normalizeHeaders` later on
        if (!requestResponse.responseReceived) {

            const message = `Trying to emit "fetch::end" but no responseReceived for ${requestResponse.requestId} found`;

            throw new Error(message);
        }

        const request: Request = {
            headers: normalizeHeaders(requestResponse.responseReceived.response.requestHeaders) as HttpHeaders,
            url: originalUrl
        };

        const data: FetchEnd = {
            element,
            request,
            resource: resourceUrl,
            response
        };

        if (isTarget) {
            this._targetNetworkData = {
                request,
                response
            };

return async function (this: any) {

        const that = this; // eslint-disable-line

        if (that._rawResponse) {
            return that._rawResponse;
        }

        const rawContent = await response.buffer();
        const responseHeaders = normalizeHeaders(response.headers())!;

        if (rawContent && rawContent.length.toString() === responseHeaders['content-length']) {
            // Response wasn't compressed so both buffers are the same
            return rawContent;
        }

        const requestHeaders = response.request().headers();
        const responseUrl = response.url();

        /*
         * Real browser connectors automatically request using HTTP2. This spec has
         * [`Pseudo-Header Fields`](https://tools.ietf.org/html/rfc7540#section-8.1.2.3):
         * `:authority`, `:method`, `:path` and `:scheme`.
         *
         * An example of request with those `Pseudo-Header Fields` to google.com:
         *

public getResponse(element: HTMLElement | null): Response {
        if (!this._response) {

            const { headers, status } = this.responseReceived!.response;
            const normalizedHeaders = normalizeHeaders(headers);
            const that = this;

            let rawContent = Buffer.alloc(0);
            let rBody = {
                content: '',
                rawContent,
                rawResponse: () => {
                    return Promise.resolve(Buffer.alloc(0));
                }
            };

            if (this._responseBody) {
                const { body, base64Encoded } = this._responseBody;
                const encoding = base64Encoded ? 'base64' : 'utf-8';
                const content = base64Encoded ? atob(body) : body; // There are some JS responses that are base64 encoded for some reason

public async fetchContent(target: URL | string, customHeaders?: object): Promise {
        /*
         * TODO: This should create a new tab, navigate to the
         * resource and control what is received somehow via an event.
         */
        const assigns = compact([this && this._headers, customHeaders]);
        const headers = Object.assign({}, ...assigns);
        const href: string = typeof target === 'string' ? target : target.href;
        const options = {
            headers,
            // we sync the ignore SSL error options with `request`. This is neeeded for local https tests
            rejectUnauthorized: !this._options.overrideInvalidCert,
            strictSSL: !this._options.overrideInvalidCert
        };

        const request: Requester = new Requester(options);
        const response: NetworkData = await request.get(href);

        return response;
    }