WIP Sourcegraph Entity Provider

app-config.yaml

@@ -79,8 +79,6 @@ scaffolder:
 
 catalog:
   locations:
-    - type: url
-      target: https://github.com/thefrontside/backstage/blob/main/catalog-info.yaml
     - type: url
       target: https://github.com/thefrontside/backstage/blob/main/templates/standard-microservice/template.yaml
       rules:
@@ -89,4 +87,8 @@ catalog:
 humanitec:
   orgId: the-frontside-software-inc
   registryUrl: "northamerica-northeast1-docker.pkg.dev/frontside-backstage/frontside-artifacts"
-  token: ${HUMANITEC_TOKEN}
+  token: ${HUMANITEC_TOKEN}
+
+sourcegraph:
+  orgId: frontside
+  token: ${SOURCEGRAPH_TOKEN}

package.json

@@ -61,6 +61,7 @@
     ]
   },
   "volta": {
-    "node": "14.20.0"
+    "node": "14.20.0",
+    "yarn": "1.22.19"
   }
 }

packages/backend/package.json

@@ -34,18 +34,18 @@
     "@backstage/plugin-proxy-backend": "^0.2.25",
     "@backstage/plugin-scaffolder-backend": "^1.1.0",
     "@backstage/plugin-search-backend": "^1.0.2",
-    "@backstage/plugin-search-backend-node": "^1.0.2",
     "@backstage/plugin-search-backend-module-pg": "^0.4.0",
+    "@backstage/plugin-search-backend-node": "^1.0.2",
     "@backstage/plugin-techdocs-backend": "^1.1.0",
-    "@frontside/backstage-plugin-effection-inspector-backend": "0.1.2",
     "@frontside/backstage-plugin-batch-loader": "0.2.2",
-    "@frontside/backstage-plugin-humanitec-backend": "^0.3.1",
+    "@frontside/backstage-plugin-effection-inspector-backend": "0.1.2",
     "@frontside/backstage-plugin-graphql": "^0.4.1",
+    "@frontside/backstage-plugin-humanitec-backend": "^0.3.1",
     "@frontside/backstage-plugin-incremental-ingestion-backend": "*",
     "@frontside/backstage-plugin-incremental-ingestion-github": "*",
-    "graphql-modules": "^2.1.0",
     "@gitbeaker/node": "^34.6.0",
     "@internal/plugin-healthcheck": "0.1.1",
+    "@frontside/backstage-plugin-sourcegraph-entity-provider": "^0.1.0",
     "@octokit/rest": "^18.5.3",
     "app": "*",
     "async-wait-until": "^2.0.12",
@@ -54,6 +54,7 @@
     "effection": "^2.0.4",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
+    "graphql-modules": "^2.1.0",
     "knex": "^2.0.0",
     "luxon": "^2.3.1",
     "pg": "^8.3.0",
@@ -65,8 +66,8 @@
     "@octokit/types": "^6.34.0",
     "@types/dockerode": "^3.3.0",
     "@types/express": "^4.17.6",
-    "@types/luxon": "^2.0.4",
     "@types/express-serve-static-core": "^4.17.5",
+    "@types/luxon": "^2.0.4",
     "better-sqlite3": "^7.5.0"
   },
   "files": [

packages/backend/src/plugins/catalog.ts

@@ -1,12 +1,14 @@
 import {
-  CatalogBuilder
+  CatalogBuilder,
+  EntityProvider
 } from '@backstage/plugin-catalog-backend';
 import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
-import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend';
-import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github';
+// import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend';
+// import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github';
 import { Router } from 'express';
-import { Duration } from 'luxon';
+// import { Duration } from 'luxon';
 import { PluginEnvironment } from '../types';
+import { SourcegraphEntityProvider } from "@frontside/backstage-plugin-sourcegraph-entity-provider";
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -15,30 +17,44 @@ export default async function createPlugin(
   const builder = CatalogBuilder.create(env);
   // incremental builder receives builder because it'll register
   // incremental entity providers with the builder 
-  const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder);
+  // const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder);
 
-  const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ 
-    host: 'github.com', 
-    searchQuery: "created:>1970-01-01 user:thefrontside", 
-    config: env.config 
-  })
-
-  incrementalBuilder.addIncrementalEntityProvider(
-    githubRepositoryProvider,
-    {
-      burstInterval: Duration.fromObject({ seconds: 3 }),
-      burstLength: Duration.fromObject({ seconds: 3 }),
-      restLength: Duration.fromObject({ day: 1 })
-    }
-  )
+  // const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ 
+  //   host: 'github.com', 
+  //   searchQuery: "created:>1970-01-01 user:thefrontside", 
+  //   config: env.config 
+  // })
+
+  // incrementalBuilder.addIncrementalEntityProvider(
+  //   githubRepositoryProvider,
+  //   {
+  //     burstInterval: Duration.fromObject({ seconds: 3 }),
+  //     burstLength: Duration.fromObject({ seconds: 3 }),
+  //     restLength: Duration.fromObject({ day: 1 })
+  //   }
+  // )
 
   builder.addProcessor(new ScaffolderEntitiesProcessor());
 
+  const sourcegraphProvider = SourcegraphEntityProvider.create(env.config);
+  builder.addEntityProvider(sourcegraphProvider as EntityProvider);
+
   const { processingEngine, router } = await builder.build();
 
   // this has to run after `await builder.build()` so ensure that catalog migrations are completed 
   // before incremental builder migrations are executed 
-  await incrementalBuilder.build();
+  // await incrementalBuilder.build();
+
+  await env.scheduler.scheduleTask({
+    id: "test-task-scheduler",
+    frequency: { cron: '0 0 * * 7' },
+    timeout: { minutes: 10 },
+    fn: async () => await sourcegraphProvider.full(),
+  });
+
+  router.post("/sourcegraph/webhook", async (req, _res) => {
+    await sourcegraphProvider.delta(req.body);
+  });
 
   await processingEngine.start();
 

plugins/sourcegraph-entity-provider-backend/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/sourcegraph-entity-provider-backend/.gitignore

@@ -0,0 +1,2 @@
+src/providers/GitHub*
+src/lib

plugins/sourcegraph-entity-provider-backend/README.md

@@ -0,0 +1,14 @@
+# sourcegraph-entity-provider-backend
+
+Welcome to the sourcegraph-entity-provider-backend backend plugin!
+
+_This plugin was created through the Backstage CLI_
+
+## Getting started
+
+Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn
+start` in the root directory, and then navigating to [/sourcegraph-entity-provider-backend](http://localhost:3000/sourcegraph-entity-provider-backend).
+
+You can also serve the plugin in isolation by running `yarn start` in the plugin directory.
+This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads.
+It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory.

plugins/sourcegraph-entity-provider-backend/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@frontside/backstage-plugin-sourcegraph-entity-provider",
+  "version": "0.1.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin"
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.15.1",
+    "@backstage/backend-tasks": "^0.3.6-next.2",
+    "@backstage/config": "^1.0.2",
+    "@types/express": "*",
+    "express": "^4.17.1",
+    "express-promise-router": "^4.1.0",
+    "graphql-request": "^5.0.0",
+    "node-fetch": "^2.6.7",
+    "winston": "^3.2.1",
+    "yn": "^4.0.0"
+  },
+  "devDependencies": {
+    "@backstage/cli": "^0.19.0",
+    "@types/supertest": "^2.0.8",
+    "msw": "^0.46.0",
+    "supertest": "^4.0.2"
+  },
+  "files": [
+    "dist"
+  ]
+}

plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts

@@ -0,0 +1,147 @@
+import {
+  ANNOTATION_LOCATION,
+  ANNOTATION_ORIGIN_LOCATION,
+} from '@backstage/catalog-model';
+import {
+  CatalogProcessorEntityResult,
+  EntityProvider,
+  EntityProviderConnection,
+  DeferredEntity,
+  parseEntityYaml
+} from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { GraphQLClient, gql } from "graphql-request";
+import { SourcegraphSearch, SourcegraphWebhookPayload } from "./types";
+
+const sourcegraphFileMatchQuery = gql`
+  query ($search: String!) {
+    search(query: $search) {
+      results {
+        matchCount
+        results {
+          __typename
+          ... on FileMatch {
+            repository {
+              name
+            }
+            file {
+              url
+              content
+            }
+          }
+        }
+      }
+    }
+  }
+`;
+
+const parseSourcegraphSearch = (data: SourcegraphSearch, providerName: string) => {
+  const parseResults: DeferredEntity[] = [];
+  data.search.results.results.forEach((result) => {
+    const location = {
+      type: "url",
+      target: `${result.repository.name}/catalog-info.yaml`,
+    };
+
+    const catalogInfoYamlContent = Buffer.from(result.file.content, "utf8");
+    for (const parseResult of parseEntityYaml(catalogInfoYamlContent, location)) {
+      const parsed = parseResult as CatalogProcessorEntityResult;
+      const annotated: DeferredEntity = {
+        entity: {
+          ...parsed.entity,
+          metadata: {
+            ...parsed.entity.metadata,
+            annotations: {
+              ...parsed.entity.metadata.annotations,
+              [ANNOTATION_LOCATION]: `url:${parsed.location.target}`,
+              [ANNOTATION_ORIGIN_LOCATION]: providerName,
+            }
+          }
+        },
+        locationKey: parsed.location.target
+      };
+      parseResults.push(annotated);
+    }
+  });
+  return parseResults;
+}
+
+export class SourcegraphEntityProvider implements EntityProvider {
+  private readonly config: Config;
+  private connection?: EntityProviderConnection;
+  private graphQLClient?: GraphQLClient;
+
+  static create(config: Config) {
+    return new SourcegraphEntityProvider(config)
+  }
+
+  private constructor(config: Config) {
+    this.config = config;
+  }
+
+  getProviderName(): string {
+    return `sourcegraph-provider:${this.config.getString("sourcegraph.orgId")}`;
+  }
+
+  async connect(connection: EntityProviderConnection): Promise<void> {
+    this.connection = connection;
+    const endpoint = `https://${this.config.getString("sourcegraph.orgId")}.sourcegraph.com/.api/graphql`
+    this.graphQLClient = new GraphQLClient(endpoint, {
+      headers: {
+        authorization: `token ${this.config.getString("sourcegraph.token")}`
+      }
+    });
+    await this.full();
+  }
+
+  async full() {
+    if (!this.connection) throw new Error('Not initialized');
+    if (!this.graphQLClient) throw new Error('GraphQL client not initialized')
+
+    const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, {
+      search: "file:^catalog-info.yaml$"
+    });
+    const parsed = parseSourcegraphSearch(data, this.getProviderName());
+    await this.connection.applyMutation({
+      type: 'full',
+      entities: parsed.map(entity => ({
+        entity: entity.entity,
+        locationKey: entity.locationKey,
+      })),
+    });
+  }
+
+  async delta(payload: SourcegraphWebhookPayload) {
+    if (!this.connection) throw new Error('Not initialized');
+
+    let toAdd: DeferredEntity[] = [];
+    let toRemove: DeferredEntity[] = [];
+    payload.results.forEach(async result => {
+      if (!this.graphQLClient) throw new Error('GraphQL client not initialized');
+      const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, {
+        search: `file:^catalog-info.yaml$ repo:${result.repository}$`
+      });
+      const parsed = parseSourcegraphSearch(data, this.getProviderName());
+      if (parsed.length) {
+        toAdd.push(parsed[0]);
+      } else {
+        toRemove.push({
+          "entity": {
+            "apiVersion": "backstage.io/v1alpha1",
+            "kind": "Component",
+            "metadata": {
+              "name": /[^\/]*$/.exec(result.repository)![0]
+            },
+          },
+          "locationKey": `${result.repository}/catalog-info.yaml`
+        }); 
+      }
+    });
+
+    await this.connection.applyMutation({
+      type: 'delta',
+      added: [...toAdd],
+      removed: [...toRemove],
+    });
+  }
+}

plugins/sourcegraph-entity-provider-backend/src/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './SourcegraphEntityProvider';