From 4fe981574c7cc18832e84039de790669ae5a1c76 Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:45:14 -0400 Subject: [PATCH 01/13] Add sourcegraph org and token to app-config --- app-config.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index 90190a4587..5b81bd35f5 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -89,4 +89,8 @@ catalog: humanitec: orgId: the-frontside-software-inc registryUrl: "northamerica-northeast1-docker.pkg.dev/frontside-backstage/frontside-artifacts" - token: ${HUMANITEC_TOKEN} \ No newline at end of file + token: ${HUMANITEC_TOKEN} + +sourcegraph: + orgId: frontside + token: ${SOURCEGRAPH_TOKEN} From 8aeaa91fbab5c49af62d6d07a47ceed169d517ef Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:45:45 -0400 Subject: [PATCH 02/13] Remove backstage component url location --- app-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 5b81bd35f5..89f6a6ac3a 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -79,8 +79,6 @@ scaffolder: catalog: locations: - - type: url - target: https://github.com/thefrontside/backstage/blob/main/catalog-info.yaml - type: url target: https://github.com/thefrontside/backstage/blob/main/templates/standard-microservice/template.yaml rules: From c9f1980f4946e40d6ff4cff87c5782601c80c1bd Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:49:36 -0400 Subject: [PATCH 03/13] Create sourcegraph entity provider with full mutation --- package.json | 3 +- .../.eslintrc.js | 1 + .../.gitignore | 2 + .../README.md | 14 ++ .../package.json | 46 +++++ .../src/SourcegraphEntityProvider.ts | 132 ++++++++++++++ .../src/index.ts | 17 ++ .../src/setupTests.ts | 17 ++ yarn.lock | 167 +++++++++++++++++- 9 files changed, 396 insertions(+), 3 deletions(-) create mode 100644 plugins/sourcegraph-entity-provider-backend/.eslintrc.js create mode 100644 plugins/sourcegraph-entity-provider-backend/.gitignore create mode 100644 plugins/sourcegraph-entity-provider-backend/README.md create mode 100644 plugins/sourcegraph-entity-provider-backend/package.json create mode 100644 plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts create mode 100644 plugins/sourcegraph-entity-provider-backend/src/index.ts create mode 100644 plugins/sourcegraph-entity-provider-backend/src/setupTests.ts diff --git a/package.json b/package.json index 2a79e92b1e..45a4d4acd2 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ ] }, "volta": { - "node": "14.20.0" + "node": "14.20.0", + "yarn": "1.22.19" } } diff --git a/plugins/sourcegraph-entity-provider-backend/.eslintrc.js b/plugins/sourcegraph-entity-provider-backend/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/sourcegraph-entity-provider-backend/.gitignore b/plugins/sourcegraph-entity-provider-backend/.gitignore new file mode 100644 index 0000000000..333570558d --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/.gitignore @@ -0,0 +1,2 @@ +src/providers/GitHub* +src/lib \ No newline at end of file diff --git a/plugins/sourcegraph-entity-provider-backend/README.md b/plugins/sourcegraph-entity-provider-backend/README.md new file mode 100644 index 0000000000..b1c1a49f64 --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/README.md @@ -0,0 +1,14 @@ +# sourcegraph-entity-provider-backend + +Welcome to the sourcegraph-entity-provider-backend backend plugin! + +_This plugin was created through the Backstage CLI_ + +## Getting started + +Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn +start` in the root directory, and then navigating to [/sourcegraph-entity-provider-backend](http://localhost:3000/sourcegraph-entity-provider-backend). + +You can also serve the plugin in isolation by running `yarn start` in the plugin directory. +This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. +It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory. diff --git a/plugins/sourcegraph-entity-provider-backend/package.json b/plugins/sourcegraph-entity-provider-backend/package.json new file mode 100644 index 0000000000..4322a7e6d0 --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/package.json @@ -0,0 +1,46 @@ +{ + "name": "@frontside/backstage-plugin-sourcegraph-entity-provider", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/backend-common": "^0.15.1", + "@backstage/backend-tasks": "^0.3.6-next.2", + "@backstage/config": "^1.0.2", + "@types/express": "*", + "express": "^4.17.1", + "express-promise-router": "^4.1.0", + "graphql-request": "^5.0.0", + "node-fetch": "^2.6.7", + "winston": "^3.2.1", + "yn": "^4.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.19.0", + "@types/supertest": "^2.0.8", + "msw": "^0.46.0", + "supertest": "^4.0.2" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts new file mode 100644 index 0000000000..53e35d9a61 --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts @@ -0,0 +1,132 @@ +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, +} from '@backstage/catalog-model'; +import { + CatalogProcessorEntityResult, + EntityProvider, + EntityProviderConnection, + DeferredEntity, + parseEntityYaml +} from '@backstage/plugin-catalog-backend'; +import { Config } from '@backstage/config'; +import { GraphQLClient, gql } from "graphql-request"; + +interface SourcegraphSearch { + search: { + results: { + results: { + repository: { + name: string; + }; + file: { + url: string; + content: string; + } + }[]; + } + } +} + +const sourcegraphFileMatchQuery = gql` + query ($search: String!) { + search(query: $search) { + results { + matchCount + results { + __typename + ... on FileMatch { + repository { + name + } + file { + url + content + } + } + } + } + } + } +`; + +const parseSourcegraphSearch = (data: SourcegraphSearch, providerName: string) => { + const parseResults: DeferredEntity[] = []; + data.search.results.results.forEach((result) => { + const location = { + type: "url", + target: `${result.repository.name}/catalog-info.yaml`, + }; + + const catalogInfoYamlContent = Buffer.from(result.file.content, "utf8"); + for (const parseResult of parseEntityYaml(catalogInfoYamlContent, location)) { + const parsed = parseResult as CatalogProcessorEntityResult; + const annotated: DeferredEntity = { + entity: { + ...parsed.entity, + metadata: { + ...parsed.entity.metadata, + annotations: { + ...parsed.entity.metadata.annotations, + [ANNOTATION_LOCATION]: `url:${parsed.location.target}`, + [ANNOTATION_ORIGIN_LOCATION]: providerName, + } + } + }, + locationKey: parsed.location.target + }; + parseResults.push(annotated); + } + }); + return parseResults; +} + +export class SourcegraphEntityProvider implements EntityProvider { + private readonly config: Config; + private connection?: EntityProviderConnection; + private graphQLClient: GraphQLClient; + + static create(config: Config) { + return new SourcegraphEntityProvider(config) + } + + private constructor( + config: Config + ) { + this.config = config; + this.graphQLClient = new GraphQLClient(""); + } + + getProviderName(): string { + return `sourcegraph-provider:${this.config.getString("sourcegraph.orgId")}`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + const endpoint = `https://${this.config.getString("sourcegraph.orgId")}.sourcegraph.com/.api/graphql` + this.graphQLClient = new GraphQLClient(endpoint, { + headers: { + authorization: `token ${this.config.getString("sourcegraph.token")}` + } + }); + await this.full(); + } + + async full() { + if (!this.connection) { + throw new Error('Not initialized'); + } + const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, { + search: "file:^catalog-info.yaml" + }); + const parsed = parseSourcegraphSearch(data, this.getProviderName()); + console.log(JSON.stringify(parsed, null, 2)) + await this.connection.applyMutation({ + type: 'full', + entities: parsed.map(entity => ({ + entity: entity.entity, + locationKey: entity.locationKey, + })), + }); + } +} diff --git a/plugins/sourcegraph-entity-provider-backend/src/index.ts b/plugins/sourcegraph-entity-provider-backend/src/index.ts new file mode 100644 index 0000000000..0975a6489e --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './SourcegraphEntityProvider'; diff --git a/plugins/sourcegraph-entity-provider-backend/src/setupTests.ts b/plugins/sourcegraph-entity-provider-backend/src/setupTests.ts new file mode 100644 index 0000000000..d3232290a7 --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/yarn.lock b/yarn.lock index 069517df09..514b3b0036 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1690,6 +1690,62 @@ yauzl "^2.10.0" yn "^4.0.0" +"@backstage/backend-common@^0.15.2-next.2": + version "0.15.2-next.2" + resolved "https://registry.yarnpkg.com/@backstage/backend-common/-/backend-common-0.15.2-next.2.tgz#288450672d4b3a68a81875731f3f100c53aa6b91" + integrity sha512-5sdrZs+pg6RrYHnDR1Y/znf5rhjhIKZqmnAZA36ERGCI427MZBx6aWqZ5HnSHfOswxes8jMau5FsZO8dSDHleQ== + dependencies: + "@backstage/cli-common" "^0.1.10" + "@backstage/config" "^1.0.3-next.2" + "@backstage/config-loader" "^1.1.5-next.2" + "@backstage/errors" "^1.1.2-next.2" + "@backstage/integration" "^1.3.2-next.2" + "@backstage/types" "^1.0.0" + "@google-cloud/storage" "^6.0.0" + "@keyv/redis" "^2.2.3" + "@kubernetes/client-node" "^0.17.0" + "@manypkg/get-packages" "^1.1.3" + "@octokit/rest" "^19.0.3" + "@types/cors" "^2.8.6" + "@types/dockerode" "^3.3.0" + "@types/express" "^4.17.6" + "@types/luxon" "^3.0.0" + "@types/webpack-env" "^1.15.2" + archiver "^5.0.2" + aws-sdk "^2.840.0" + base64-stream "^1.0.0" + compression "^1.7.4" + concat-stream "^2.0.0" + cors "^2.8.5" + dockerode "^3.3.1" + express "^4.17.1" + express-promise-router "^4.1.0" + fs-extra "10.1.0" + git-url-parse "^13.0.0" + helmet "^6.0.0" + isomorphic-git "^1.8.0" + jose "^4.6.0" + keyv "^4.0.3" + keyv-memcache "^1.2.5" + knex "^2.0.0" + lodash "^4.17.21" + logform "^2.3.2" + luxon "^3.0.0" + minimatch "^5.0.0" + minimist "^1.2.5" + morgan "^1.10.0" + node-abort-controller "^3.0.1" + node-fetch "^2.6.7" + raw-body "^2.4.1" + request "^2.88.2" + selfsigned "^2.0.0" + stoppable "^1.1.0" + tar "^6.1.2" + uuid "^8.3.2" + winston "^3.2.1" + yauzl "^2.10.0" + yn "^4.0.0" + "@backstage/backend-plugin-api@^0.1.2": version "0.1.2" resolved "https://registry.yarnpkg.com/@backstage/backend-plugin-api/-/backend-plugin-api-0.1.2.tgz#54a885d600981c218f373c129027681289791e43" @@ -1723,6 +1779,25 @@ winston "^3.2.1" zod "^3.9.5" +"@backstage/backend-tasks@^0.3.6-next.2": + version "0.3.6-next.2" + resolved "https://registry.yarnpkg.com/@backstage/backend-tasks/-/backend-tasks-0.3.6-next.2.tgz#192204dc209559f18723afd633a99676a0b79d1f" + integrity sha512-zGETLBT8TDdxo90l0pKjlZlPjvgMOe6k6pw2rL+zC1RLssugcX14X9LdJCgKW0oaZUm/GkOQpET9e6Zfiop7uw== + dependencies: + "@backstage/backend-common" "^0.15.2-next.2" + "@backstage/config" "^1.0.3-next.2" + "@backstage/errors" "^1.1.2-next.2" + "@backstage/types" "^1.0.0" + "@types/luxon" "^3.0.0" + cron "^2.0.0" + knex "^2.0.0" + lodash "^4.17.21" + luxon "^3.0.0" + node-abort-controller "^3.0.1" + uuid "^8.0.0" + winston "^3.2.1" + zod "^3.9.5" + "@backstage/catalog-client@^1.0.1", "@backstage/catalog-client@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@backstage/catalog-client/-/catalog-client-1.1.0.tgz#66948ef9beb45d3211fbb5b2e5139d1869bb4f57" @@ -1874,6 +1949,27 @@ yaml "^2.0.0" yup "^0.32.9" +"@backstage/config-loader@^1.1.5-next.2": + version "1.1.5-next.2" + resolved "https://registry.yarnpkg.com/@backstage/config-loader/-/config-loader-1.1.5-next.2.tgz#1ca14268df35b716f25ddfc550402fba1dcd09f7" + integrity sha512-cLj2yryMW46b4XPe3GylZ435Yq/lickImvxihfpgA7M1eEw4l+cC/mjwcOCt8+rySejQvwgu/rutLPMoDYkM+w== + dependencies: + "@backstage/cli-common" "^0.1.10" + "@backstage/config" "^1.0.3-next.2" + "@backstage/errors" "^1.1.2-next.2" + "@backstage/types" "^1.0.0" + "@types/json-schema" "^7.0.6" + ajv "^8.10.0" + chokidar "^3.5.2" + fs-extra "10.1.0" + json-schema "^0.4.0" + json-schema-merge-allof "^0.8.1" + json-schema-traverse "^1.0.0" + node-fetch "^2.6.7" + typescript-json-schema "^0.54.0" + yaml "^2.0.0" + yup "^0.32.9" + "@backstage/config@^1.0.0", "@backstage/config@^1.0.1", "@backstage/config@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@backstage/config/-/config-1.0.2.tgz#539be20a83ba7e7b40637fc962e92d23c80c88c8" @@ -1882,6 +1978,14 @@ "@backstage/types" "^1.0.0" lodash "^4.17.21" +"@backstage/config@^1.0.3-next.2": + version "1.0.3-next.2" + resolved "https://registry.yarnpkg.com/@backstage/config/-/config-1.0.3-next.2.tgz#e6330334724f25b5adc785d4d3f9a879003e4a79" + integrity sha512-A66mPp2/z8tawOPcfk7VyxJ1NdkIpo9X6xvhTyugnPR9xmxSfDZxuqlQFswQENY0R4W7Jgn/3BNoqIub/fTJTQ== + dependencies: + "@backstage/types" "^1.0.0" + lodash "^4.17.21" + "@backstage/core-app-api@^1.0.1", "@backstage/core-app-api@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@backstage/core-app-api/-/core-app-api-1.1.0.tgz#7ad4046591c5478acb4cc0ae8ad2788448af57ba" @@ -1983,6 +2087,15 @@ cross-fetch "^3.1.5" serialize-error "^8.0.1" +"@backstage/errors@^1.1.2-next.2": + version "1.1.2-next.2" + resolved "https://registry.yarnpkg.com/@backstage/errors/-/errors-1.1.2-next.2.tgz#cb7a9e6e63e07db7245d9e8bd126d86d84ffc25d" + integrity sha512-JcdesBLF7Fhs7fps0jzhlYrjljwXk726xtpOgtXirm2JwgB2Rd4K09bdRT5mYd5F5UU+t/OTn+j+r9338jkLGg== + dependencies: + "@backstage/types" "^1.0.0" + cross-fetch "^3.1.5" + serialize-error "^8.0.1" + "@backstage/integration-react@^1.0.1", "@backstage/integration-react@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@backstage/integration-react/-/integration-react-1.1.4.tgz#7ab915a92bd4f62df69cf0654ad934564b8915f3" @@ -2012,6 +2125,20 @@ lodash "^4.17.21" luxon "^3.0.0" +"@backstage/integration@^1.3.2-next.2": + version "1.3.2-next.2" + resolved "https://registry.yarnpkg.com/@backstage/integration/-/integration-1.3.2-next.2.tgz#847ab9e053f9cdfb7916caca52d3d64af4971531" + integrity sha512-desONZR9C7W6Y6TzQWx/BcQkoznfnURfWPYC3/SgHzlCj8zvh1CIdT0vjHykm3sjIMuxn+viwTXK+TwX3qOH/Q== + dependencies: + "@backstage/config" "^1.0.3-next.2" + "@backstage/errors" "^1.1.2-next.2" + "@octokit/auth-app" "^4.0.0" + "@octokit/rest" "^19.0.3" + cross-fetch "^3.1.5" + git-url-parse "^13.0.0" + lodash "^4.17.21" + luxon "^3.0.0" + "@backstage/plugin-api-docs@^0.8.4": version "0.8.9" resolved "https://registry.yarnpkg.com/@backstage/plugin-api-docs/-/plugin-api-docs-0.8.9.tgz#e1b27e623b01ce84c7ebc3976ad9a3a14d0ec092" @@ -4187,7 +4314,7 @@ tslib "^2.4.0" value-or-promise "1.0.11" -"@graphql-typed-document-node/core@^3.1.0": +"@graphql-typed-document-node/core@^3.1.0", "@graphql-typed-document-node/core@^3.1.1": version "3.1.1" resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.1.1.tgz#076d78ce99822258cf813ecc1e7fa460fa74d052" integrity sha512-NQ17ii0rK1b34VZonlmT2QMJFI70m0TRwbknO/ihlbatXyaktDhN/98vBiUU6kNBPljqGqyIrl2T4nY2RpFANg== @@ -5430,7 +5557,7 @@ outvariant "^1.2.1" strict-event-emitter "^0.2.4" -"@mswjs/interceptors@^0.17.5": +"@mswjs/interceptors@^0.17.2", "@mswjs/interceptors@^0.17.5": version "0.17.6" resolved "https://registry.yarnpkg.com/@mswjs/interceptors/-/interceptors-0.17.6.tgz#7f7900f4cd26f70d9f698685e4485b2f4101d26a" integrity sha512-201pBIWehTURb6q8Gheu4Zhvd3Ox1U4BJq5KiOQsYzkWyfiOG4pwcz5hPZIEryztgrf8/sdwABpvY757xMmfrQ== @@ -12920,6 +13047,16 @@ graphql-request@^4.0.0: extract-files "^9.0.0" form-data "^3.0.0" +graphql-request@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/graphql-request/-/graphql-request-5.0.0.tgz#7504a807d0e11be11a3c448e900f0cc316aa18ef" + integrity sha512-SpVEnIo2J5k2+Zf76cUkdvIRaq5FMZvGQYnA4lUWYbc99m+fHh4CZYRRO/Ff4tCLQ613fzCm3SiDT64ubW5Gyw== + dependencies: + "@graphql-typed-document-node/core" "^3.1.1" + cross-fetch "^3.1.5" + extract-files "^9.0.0" + form-data "^3.0.0" + graphql-tag@^2.10.3, graphql-tag@^2.11.0: version "2.12.6" resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.6.tgz#d441a569c1d2537ef10ca3d1633b48725329b5f1" @@ -16783,6 +16920,32 @@ msw@^0.42.0: type-fest "^1.2.2" yargs "^17.3.1" +msw@^0.46.0: + version "0.46.1" + resolved "https://registry.yarnpkg.com/msw/-/msw-0.46.1.tgz#4da4a679ada1ab25fc91dc11a210517204ce078d" + integrity sha512-yEKJcHjUbee6oeD/RDakFdr+RcdMtiREH4U4m/8eJnKpsH2kIA5DSU2wvpR1VKOBVVPyvmAIaU/OrPke8jNdTQ== + dependencies: + "@mswjs/cookies" "^0.2.2" + "@mswjs/interceptors" "^0.17.2" + "@open-draft/until" "^1.0.3" + "@types/cookie" "^0.4.1" + "@types/js-levenshtein" "^1.1.1" + chalk "4.1.1" + chokidar "^3.4.2" + cookie "^0.4.2" + graphql "^15.0.0 || ^16.0.0" + headers-polyfill "^3.0.4" + inquirer "^8.2.0" + is-node-process "^1.0.1" + js-levenshtein "^1.1.6" + node-fetch "^2.6.7" + outvariant "^1.3.0" + path-to-regexp "^6.2.0" + statuses "^2.0.0" + strict-event-emitter "^0.2.0" + type-fest "^2.19.0" + yargs "^17.3.1" + msw@^0.47.0: version "0.47.4" resolved "https://registry.yarnpkg.com/msw/-/msw-0.47.4.tgz#5551011609890c6b62a2047055f475a9afae2ad4" From 7feda752273993c02597f24f73b773d59b9019c9 Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:50:52 -0400 Subject: [PATCH 04/13] Add delta mutation for code monitoring webhooks --- .../src/SourcegraphEntityProvider.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts index 53e35d9a61..b3fd45e795 100644 --- a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts +++ b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts @@ -12,6 +12,18 @@ import { import { Config } from '@backstage/config'; import { GraphQLClient, gql } from "graphql-request"; +interface SourcegraphWebhookPayload { + monitorDescription: string; + monitorUrl: string; + query: string; + results: { + repository: string; + commit: string; + diff: string; + matchedDiffRanges: number[][]; + }[]; +} + interface SourcegraphSearch { search: { results: { @@ -129,4 +141,38 @@ export class SourcegraphEntityProvider implements EntityProvider { })), }); } + + async delta(payload: SourcegraphWebhookPayload) { + if (!this.connection) { + throw new Error('Not initialized'); + } + let toAdd: DeferredEntity[] = []; + let toRemove: DeferredEntity[] = []; + payload.results.forEach(async result => { + const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, { + search: `file:^catalog-info.yaml repo:${result.repository}$` + }); + const parsed = parseSourcegraphSearch(data, this.getProviderName()); + if (parsed.length) { + toAdd.push(parsed[0]); + } else { + toRemove.push({ + "entity": { + "apiVersion": "backstage.io/v1alpha1", + "kind": "Component", + "metadata": { + "name": /[^\/]*$/.exec(result.repository)![0] + }, + }, + "locationKey": `${result.repository}/catalog-info.yaml` + }); + } + }); + + await this.connection.applyMutation({ + type: 'delta', + added: [...toAdd], + removed: [...toRemove], + }); + } } From 68d6694f622b86e0f2bd2782fb7f1035d410f2ad Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:51:59 -0400 Subject: [PATCH 05/13] Extract types to its own file --- .../src/SourcegraphEntityProvider.ts | 29 +------------------ .../src/types.ts | 27 +++++++++++++++++ 2 files changed, 28 insertions(+), 28 deletions(-) create mode 100644 plugins/sourcegraph-entity-provider-backend/src/types.ts diff --git a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts index b3fd45e795..79c3495045 100644 --- a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts +++ b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts @@ -11,34 +11,7 @@ import { } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; import { GraphQLClient, gql } from "graphql-request"; - -interface SourcegraphWebhookPayload { - monitorDescription: string; - monitorUrl: string; - query: string; - results: { - repository: string; - commit: string; - diff: string; - matchedDiffRanges: number[][]; - }[]; -} - -interface SourcegraphSearch { - search: { - results: { - results: { - repository: { - name: string; - }; - file: { - url: string; - content: string; - } - }[]; - } - } -} +import { SourcegraphSearch, SourcegraphWebhookPayload } from "./types"; const sourcegraphFileMatchQuery = gql` query ($search: String!) { diff --git a/plugins/sourcegraph-entity-provider-backend/src/types.ts b/plugins/sourcegraph-entity-provider-backend/src/types.ts new file mode 100644 index 0000000000..98134f6597 --- /dev/null +++ b/plugins/sourcegraph-entity-provider-backend/src/types.ts @@ -0,0 +1,27 @@ +export interface SourcegraphWebhookPayload { + monitorDescription: string; + monitorUrl: string; + query: string; + results: { + repository: string; + commit: string; + diff: string; + matchedDiffRanges: number[][]; + }[]; +} + +export interface SourcegraphSearch { + search: { + results: { + results: { + repository: { + name: string; + }; + file: { + url: string; + content: string; + } + }[]; + } + } +} From 84064e5fa8128b0c27b319b879fe88246ad6f1dd Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:53:32 -0400 Subject: [PATCH 06/13] Disable Frontside ingestion from catalog --- packages/backend/src/plugins/catalog.ts | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 61f287cba4..f2300349c8 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -2,10 +2,10 @@ import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; -import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; -import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; +// import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; +// import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; import { Router } from 'express'; -import { Duration } from 'luxon'; +// import { Duration } from 'luxon'; import { PluginEnvironment } from '../types'; export default async function createPlugin( @@ -15,22 +15,22 @@ export default async function createPlugin( const builder = CatalogBuilder.create(env); // incremental builder receives builder because it'll register // incremental entity providers with the builder - const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder); + // const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder); - const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ - host: 'github.com', - searchQuery: "created:>1970-01-01 user:thefrontside", - config: env.config - }) + // const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ + // host: 'github.com', + // searchQuery: "created:>1970-01-01 user:thefrontside", + // config: env.config + // }) - incrementalBuilder.addIncrementalEntityProvider( - githubRepositoryProvider, - { - burstInterval: Duration.fromObject({ seconds: 3 }), - burstLength: Duration.fromObject({ seconds: 3 }), - restLength: Duration.fromObject({ day: 1 }) - } - ) + // incrementalBuilder.addIncrementalEntityProvider( + // githubRepositoryProvider, + // { + // burstInterval: Duration.fromObject({ seconds: 3 }), + // burstLength: Duration.fromObject({ seconds: 3 }), + // restLength: Duration.fromObject({ day: 1 }) + // } + // ) builder.addProcessor(new ScaffolderEntitiesProcessor()); @@ -38,7 +38,7 @@ export default async function createPlugin( // this has to run after `await builder.build()` so ensure that catalog migrations are completed // before incremental builder migrations are executed - await incrementalBuilder.build(); + // await incrementalBuilder.build(); await processingEngine.start(); From 7da515d57192379939a8a90564d8ef1034b32e4d Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:54:52 -0400 Subject: [PATCH 07/13] Install sourcegraph entity provider in backend --- packages/backend/package.json | 11 ++++++----- packages/backend/src/plugins/catalog.ts | 7 ++++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index a593ba08f4..5342ca23fe 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,18 +34,18 @@ "@backstage/plugin-proxy-backend": "^0.2.25", "@backstage/plugin-scaffolder-backend": "^1.1.0", "@backstage/plugin-search-backend": "^1.0.2", - "@backstage/plugin-search-backend-node": "^1.0.2", "@backstage/plugin-search-backend-module-pg": "^0.4.0", + "@backstage/plugin-search-backend-node": "^1.0.2", "@backstage/plugin-techdocs-backend": "^1.1.0", - "@frontside/backstage-plugin-effection-inspector-backend": "0.1.2", "@frontside/backstage-plugin-batch-loader": "0.2.2", - "@frontside/backstage-plugin-humanitec-backend": "^0.3.1", + "@frontside/backstage-plugin-effection-inspector-backend": "0.1.2", "@frontside/backstage-plugin-graphql": "^0.4.1", + "@frontside/backstage-plugin-humanitec-backend": "^0.3.1", "@frontside/backstage-plugin-incremental-ingestion-backend": "*", "@frontside/backstage-plugin-incremental-ingestion-github": "*", - "graphql-modules": "^2.1.0", "@gitbeaker/node": "^34.6.0", "@internal/plugin-healthcheck": "0.1.1", + "@frontside/backstage-plugin-sourcegraph-entity-provider": "^0.1.0", "@octokit/rest": "^18.5.3", "app": "*", "async-wait-until": "^2.0.12", @@ -54,6 +54,7 @@ "effection": "^2.0.4", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "graphql-modules": "^2.1.0", "knex": "^2.0.0", "luxon": "^2.3.1", "pg": "^8.3.0", @@ -65,8 +66,8 @@ "@octokit/types": "^6.34.0", "@types/dockerode": "^3.3.0", "@types/express": "^4.17.6", - "@types/luxon": "^2.0.4", "@types/express-serve-static-core": "^4.17.5", + "@types/luxon": "^2.0.4", "better-sqlite3": "^7.5.0" }, "files": [ diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index f2300349c8..1bebc9bf16 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -1,5 +1,6 @@ import { - CatalogBuilder + CatalogBuilder, + EntityProvider } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; // import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; @@ -7,6 +8,7 @@ import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backen import { Router } from 'express'; // import { Duration } from 'luxon'; import { PluginEnvironment } from '../types'; +import { SourcegraphEntityProvider } from "@frontside/backstage-plugin-sourcegraph-entity-provider"; export default async function createPlugin( env: PluginEnvironment, @@ -34,6 +36,9 @@ export default async function createPlugin( builder.addProcessor(new ScaffolderEntitiesProcessor()); + const sourcegraphProvider = SourcegraphEntityProvider.create(env.config); + builder.addEntityProvider(sourcegraphProvider as EntityProvider); + const { processingEngine, router } = await builder.build(); // this has to run after `await builder.build()` so ensure that catalog migrations are completed From a669d47b73b5e5a8cf565c4d8fb85945bde346b3 Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:55:51 -0400 Subject: [PATCH 08/13] Add task scheduler to run full mutation weekly --- packages/backend/src/plugins/catalog.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 1bebc9bf16..8c3a193885 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -45,6 +45,13 @@ export default async function createPlugin( // before incremental builder migrations are executed // await incrementalBuilder.build(); + await env.scheduler.scheduleTask({ + id: "test-task-scheduler", + frequency: { cron: '0 0 * * 7' }, + timeout: { minutes: 10 }, + fn: async () => await sourcegraphProvider.full(), + }); + await processingEngine.start(); return router; From 334452f4481d745f28c308d359c8b81ed105dc4e Mon Sep 17 00:00:00 2001 From: minkimcello Date: Tue, 18 Oct 2022 22:56:37 -0400 Subject: [PATCH 09/13] Add post to receive webhook events and trigger delta --- packages/backend/src/plugins/catalog.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 8c3a193885..a196391e5e 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -52,6 +52,10 @@ export default async function createPlugin( fn: async () => await sourcegraphProvider.full(), }); + router.post("/sourcegraph/webhook", async (req, _res) => { + await sourcegraphProvider.delta(req.body); + }); + await processingEngine.start(); return router; From 541241300c580a785851c33ee2b32a71287dc635 Mon Sep 17 00:00:00 2001 From: minkimcello Date: Wed, 19 Oct 2022 11:49:59 -0400 Subject: [PATCH 10/13] Remove console log --- .../src/SourcegraphEntityProvider.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts index 79c3495045..e4ecde4a05 100644 --- a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts +++ b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts @@ -105,7 +105,6 @@ export class SourcegraphEntityProvider implements EntityProvider { search: "file:^catalog-info.yaml" }); const parsed = parseSourcegraphSearch(data, this.getProviderName()); - console.log(JSON.stringify(parsed, null, 2)) await this.connection.applyMutation({ type: 'full', entities: parsed.map(entity => ({ From 27835e935157c36951493c45396aa6961cb4c202 Mon Sep 17 00:00:00 2001 From: minkimcello Date: Wed, 19 Oct 2022 12:26:14 -0400 Subject: [PATCH 11/13] Update query to search exact file match --- .../src/SourcegraphEntityProvider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts index e4ecde4a05..27f95cc0eb 100644 --- a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts +++ b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts @@ -102,7 +102,7 @@ export class SourcegraphEntityProvider implements EntityProvider { throw new Error('Not initialized'); } const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, { - search: "file:^catalog-info.yaml" + search: "file:^catalog-info.yaml$" }); const parsed = parseSourcegraphSearch(data, this.getProviderName()); await this.connection.applyMutation({ @@ -122,7 +122,7 @@ export class SourcegraphEntityProvider implements EntityProvider { let toRemove: DeferredEntity[] = []; payload.results.forEach(async result => { const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, { - search: `file:^catalog-info.yaml repo:${result.repository}$` + search: `file:^catalog-info.yaml$ repo:${result.repository}$` }); const parsed = parseSourcegraphSearch(data, this.getProviderName()); if (parsed.length) { From 07da31120a8177e4ee1f9bdf4e1704acf8dc6991 Mon Sep 17 00:00:00 2001 From: minkimcello Date: Wed, 19 Oct 2022 15:55:13 -0400 Subject: [PATCH 12/13] No need to initialize graphql client in constructor --- .../src/SourcegraphEntityProvider.ts | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts index 27f95cc0eb..c878ef61ee 100644 --- a/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts +++ b/plugins/sourcegraph-entity-provider-backend/src/SourcegraphEntityProvider.ts @@ -69,17 +69,14 @@ const parseSourcegraphSearch = (data: SourcegraphSearch, providerName: string) = export class SourcegraphEntityProvider implements EntityProvider { private readonly config: Config; private connection?: EntityProviderConnection; - private graphQLClient: GraphQLClient; + private graphQLClient?: GraphQLClient; static create(config: Config) { return new SourcegraphEntityProvider(config) } - private constructor( - config: Config - ) { + private constructor(config: Config) { this.config = config; - this.graphQLClient = new GraphQLClient(""); } getProviderName(): string { @@ -98,9 +95,9 @@ export class SourcegraphEntityProvider implements EntityProvider { } async full() { - if (!this.connection) { - throw new Error('Not initialized'); - } + if (!this.connection) throw new Error('Not initialized'); + if (!this.graphQLClient) throw new Error('GraphQL client not initialized') + const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, { search: "file:^catalog-info.yaml$" }); @@ -115,12 +112,12 @@ export class SourcegraphEntityProvider implements EntityProvider { } async delta(payload: SourcegraphWebhookPayload) { - if (!this.connection) { - throw new Error('Not initialized'); - } + if (!this.connection) throw new Error('Not initialized'); + let toAdd: DeferredEntity[] = []; let toRemove: DeferredEntity[] = []; payload.results.forEach(async result => { + if (!this.graphQLClient) throw new Error('GraphQL client not initialized'); const data: SourcegraphSearch = await this.graphQLClient.request(sourcegraphFileMatchQuery, { search: `file:^catalog-info.yaml$ repo:${result.repository}$` }); From 8ed7a10eea21b6966712eb979ec0a772298b7ad8 Mon Sep 17 00:00:00 2001 From: minkimcello Date: Wed, 19 Oct 2022 16:01:19 -0400 Subject: [PATCH 13/13] Enable publishing --- plugins/sourcegraph-entity-provider-backend/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/sourcegraph-entity-provider-backend/package.json b/plugins/sourcegraph-entity-provider-backend/package.json index 4322a7e6d0..81fce09fc0 100644 --- a/plugins/sourcegraph-entity-provider-backend/package.json +++ b/plugins/sourcegraph-entity-provider-backend/package.json @@ -4,7 +4,6 @@ "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", - "private": true, "publishConfig": { "access": "public", "main": "dist/index.cjs.js",