Refactor the Bitbucket Discovery processor

Signed-off-by: Mathias Åhsberg <mathias.ahsberg@resurs.se>

docs/integrations/bitbucket/discovery.md

@@ -1,24 +1,22 @@
 ---
 id: discovery
-title: Bitbucket Server Discovery
+title: Bitbucket Discovery
 sidebar_label: Discovery
 description:
-  Automatically discovering catalog entities from repositories in a Bitbucket
-  Server instance
+  Automatically discovering catalog entities from repositories in Bitbucket
 ---
 
 The Bitbucket integration has a special discovery processor for discovering
-catalog entities within a Bitbucket Server instance. The processor will crawl
-the Bitbucket Server instance and register entities matching the configured
-path. This can be useful as an alternative to static locations or manually
-adding things to the catalog.
+catalog entities located in Bitbucket. The processor will crawl your Bitbucket
+account and register entities matching the configured path. This can be useful
+as an alternative to static locations or manually adding things to the catalog.
 
 > Note: The Bitbucket Discovery Processor currently only supports a self-hosted
 > Bitbucket Server, and not the hosted Bitbucket Cloud product.
 
 To use the discovery processor, you'll need a Bitbucket integration
-[set up](locations.md) with a `BITBUCKET_TOKEN`. Then you can add a location
-target to the catalog configuration:
+[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`.
+Then you can add a location target to the catalog configuration:
 
 ```yaml
 catalog:
@@ -36,8 +34,8 @@ The target is composed of four parts:
   `*` to scan repositories from all projects. This example only scans for
   repositories in the `my-project` project.
 - The repository blob to scan, which accepts \* wildcard tokens. This can simply
-  be `*` to scan all repositories in the organization. This example only looks
-  for repositories prefixed with `service-`.
+  be `*` to scan all repositories in the project. This example only looks for
+  repositories prefixed with `service-`.
 - The path within each repository to find the catalog YAML file. This will
   usually be `/catalog-info.yaml` or a similar variation for catalog files
   stored in the root directory of each repository.

docs/integrations/bitbucket/locations.md

@@ -1,12 +1,12 @@
 ---
 id: locations
-title: BitBucket Locations
+title: Bitbucket Locations
 sidebar_label: Locations
 description:
-  Integrating source code stored in BitBucket into the Backstage catalog
+  Integrating source code stored in Bitbucket into the Backstage catalog
 ---
 
-The BitBucket integration supports loading catalog entities from bitbucket.com
+The Bitbucket integration supports loading catalog entities from bitbucket.org
 or a self-hosted BitBucket. Entities can be added to
 [static catalog configuration](../../features/software-catalog/configuration.md),
 or registered with the
@@ -21,21 +21,21 @@ integrations:
       token: ${BITBUCKET_TOKEN}
 ```
 
-> Note: A public BitBucket provider is added automatically at startup for
+> Note: A public Bitbucket provider is added automatically at startup for
 > convenience, so you only need to list it if you want to supply a
 > [token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html).
 
 Directly under the `bitbucket` key is a list of provider configurations, where
-you can list the BitBucket providers you want to fetch data from. Each entry is
+you can list the Bitbucket providers you want to fetch data from. Each entry is
 a structure with up to four elements:
 
-- `host`: The host of the BitBucket instance, e.g. `bitbucket.company.com`.
-- `token` (optional): An personal access token as expected by BitBucket. Either
+- `host`: The host of the Bitbucket instance, e.g. `bitbucket.company.com`.
+- `token` (optional): An personal access token as expected by Bitbucket. Either
   an access token **or** a username + appPassword may be supplied.
-- `username`: The BitBucket username to use in API requests. If neither a
+- `username`: The Bitbucket username to use in API requests. If neither a
   username nor token are supplied, anonymous access will be used.
-- `appPassword` (optional): The password for the BitBucket user. Only needed
+- `appPassword` (optional): The password for the Bitbucket user. Only needed
   when using `username` instead of `token`.
-- `apiBaseUrl` (optional): The URL of the GitLab API. For self-hosted
-  installations, it is commonly at `https://<host>/api/v4`. For gitlab.com, this
-  configuration is not needed as it can be inferred.
+- `apiBaseUrl` (optional): The URL of the Bitbucket API. For self-hosted
+  installations, it is commonly at `https://<host>/rest/api/1.0`. For
+  bitbucket.org, this configuration is not needed as it can be inferred.

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts

@@ -120,12 +120,13 @@ describe('BitbucketDiscoveryProcessor', () => {
       );
 
       const actual = await readBitbucketOrg(client, target);
-      expect(actual).toContainEqual({
+      expect(actual.scanned).toBe(2);
+      expect(actual.matches).toContainEqual({
         type: 'url',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
       });
-      expect(actual).toContainEqual({
+      expect(actual.matches).toContainEqual({
         type: 'url',
         target:
           'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
@@ -168,13 +169,13 @@ describe('BitbucketDiscoveryProcessor', () => {
       );
 
       const actual = await readBitbucketOrg(client, target);
-
-      expect(actual).toContainEqual({
+      expect(actual.scanned).toBe(3);
+      expect(actual.matches).toContainEqual({
         type: 'url',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
       });
-      expect(actual).toContainEqual({
+      expect(actual.matches).toContainEqual({
         type: 'url',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
@@ -206,8 +207,8 @@ describe('BitbucketDiscoveryProcessor', () => {
       );
 
       const actual = await readBitbucketOrg(client, target);
-
-      expect(actual).toContainEqual({
+      expect(actual.scanned).toBe(3);
+      expect(actual.matches).toContainEqual({
         type: 'url',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts

@@ -16,14 +16,17 @@
 import { Logger } from 'winston';
 import { Config } from '@backstage/config';
 
-import { ScmIntegrations } from '@backstage/integration';
+import {
+  ScmIntegrationRegistry,
+  ScmIntegrations,
+} from '@backstage/integration';
 import { LocationSpec } from '@backstage/catalog-model';
-import { BitbucketClient, pageIterator } from './bitbucket';
+import { BitbucketClient, paginated } from './bitbucket';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
 import { results } from './index';
 
 export class BitbucketDiscoveryProcessor implements CatalogProcessor {
-  private readonly integrations: ScmIntegrations;
+  private readonly integrations: ScmIntegrationRegistry;
   private readonly logger: Logger;
 
   static fromConfig(config: Config, options: { logger: Logger }) {
@@ -35,7 +38,10 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
     });
   }
 
-  constructor(options: { integrations: ScmIntegrations; logger: Logger }) {
+  constructor(options: {
+    integrations: ScmIntegrationRegistry;
+    logger: Logger;
+  }) {
     this.integrations = options.integrations;
     this.logger = options.logger;
   }
@@ -67,9 +73,9 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
     const startTimestamp = Date.now();
     this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
 
-    const repositories = await readBitbucketOrg(client, location.target);
+    const result = await readBitbucketOrg(client, location.target);
 
-    for (const repository of repositories) {
+    for (const repository of result.matches) {
       emit(
         results.location(
           repository,
@@ -82,8 +88,8 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
     }
 
     const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
-    this.logger.info(
-      `Read ${repositories.length} Bitbucket repositories in ${duration} seconds`,
+    this.logger.debug(
+      `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`,
     );
 
     return true;
@@ -93,30 +99,29 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
 export async function readBitbucketOrg(
   client: BitbucketClient,
   target: string,
-): Promise<LocationSpec[]> {
+): Promise<Result> {
   const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
-  const projectIterator = pageIterator(options => client.listProjects(options));
-  let result: LocationSpec[] = [];
-
-  for await (const page of projectIterator) {
-    for (const project of page.values) {
-      if (!projectSearchPath.test(project.key)) {
-        continue;
-      }
-      const repoIterator = pageIterator(options =>
-        client.listRepositories(project.key, options),
-      );
-      for await (const repoPage of repoIterator) {
-        result = result.concat(
-          repoPage.values
-            .filter(v => repoSearchPath.test(v.slug))
-            .map(repo => {
-              return {
-                type: 'url',
-                target: `${repo.links.self[0].href}${catalogPath}`,
-              };
-            }),
-        );
+  const projects = paginated(options => client.listProjects(options));
+  const result: Result = {
+    scanned: 0,
+    matches: [],
+  };
+
+  for await (const project of projects) {
+    if (!projectSearchPath.test(project.key)) {
+      continue;
+    }
+    const repositories = paginated(options =>
+      client.listRepositories(project.key, options),
+    );
+    for await (const repository of repositories) {
+      result.scanned++;
+
+      if (repoSearchPath.test(repository.slug)) {
+        result.matches.push({
+          type: 'url',
+          target: `${repository.links.self[0].href}${catalogPath}`,
+        });
       }
     }
   }
@@ -144,3 +149,8 @@ function parseUrl(
 function escapeRegExp(str: string): RegExp {
   return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
 }
+
+type Result = {
+  scanned: number;
+  matches: LocationSpec[];
+};

plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts

@@ -46,14 +46,12 @@ export class BitbucketClient {
     options?: ListOptions,
   ): Promise<PagedResponse<any>> {
     const request = new URL(endpoint);
-    if (options) {
-      (Object.keys(options) as Array<keyof typeof options>).forEach(key => {
-        const value: any = options[key] as any;
-        if (value) {
-          request.searchParams.append(key, value);
-        }
-      });
+    for (const key in options) {
+      if (options[key]) {
+        request.searchParams.append(key, options[key]!.toString());
+      }
     }
+
     const response = await fetch(
       request.toString(),
       getBitbucketRequestOptions(this.config),
@@ -72,6 +70,7 @@ export class BitbucketClient {
 }
 
 export type ListOptions = {
+  [key: string]: number | undefined;
   limit?: number | undefined;
   start?: number | undefined;
 };
@@ -85,42 +84,17 @@ export type PagedResponse<T> = {
   nextPageStart: number;
 };
 
-export function pageIterator(
-  pagedRequest: (options: ListOptions) => Promise<PagedResponse<any>>,
+export async function* paginated(
+  request: (options: ListOptions) => Promise<PagedResponse<any>>,
   options?: ListOptions,
-): AsyncIterable<PagedResponse<any>> {
-  return {
-    [Symbol.asyncIterator]: () => {
-      const opts = options || { start: 0 };
-      let finished = false;
-      return {
-        async next() {
-          if (!finished) {
-            try {
-              const response = await pagedRequest(opts);
-              finished = response.isLastPage;
-              opts.start = response.nextPageStart;
-              return Promise.resolve({
-                value: response,
-                done: false,
-              });
-            } catch (error) {
-              return Promise.reject({
-                value: undefined,
-                done: true,
-                error: error,
-              });
-            }
-          } else {
-            opts.start = 0;
-            finished = false;
-            return Promise.resolve({
-              value: undefined,
-              done: true,
-            });
-          }
-        },
-      };
-    },
-  };
+) {
+  const opts = options || { start: 0 };
+  let res;
+  do {
+    res = await request(opts);
+    opts.start = res.nextPageStart;
+    for (const item of res.values) {
+      yield item;
+    }
+  } while (!res.isLastPage);
 }

plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts

@@ -13,5 +13,5 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export { BitbucketClient, pageIterator } from './client';
+export { BitbucketClient, paginated } from './client';
 export type { PagedResponse } from './client';