WIP Sourcegraph Entity Provider

app-config.yaml

@@ -89,4 +89,8 @@ catalog:
 humanitec:
   orgId: the-frontside-software-inc
   registryUrl: "northamerica-northeast1-docker.pkg.dev/frontside-backstage/frontside-artifacts"
-  token: ${HUMANITEC_TOKEN}
+  token: ${HUMANITEC_TOKEN}
+
+sourcegraph:
+  orgId: frontside
+  token: ${SOURCEGRAPH_TOKEN}

package.json

@@ -61,6 +61,7 @@
     ]
   },
   "volta": {
-    "node": "14.20.0"
+    "node": "14.20.0",
+    "yarn": "1.22.19"
   }
 }

packages/backend/package.json

@@ -34,18 +34,18 @@
     "@backstage/plugin-proxy-backend": "^0.2.25",
     "@backstage/plugin-scaffolder-backend": "^1.1.0",
     "@backstage/plugin-search-backend": "^1.0.2",
-    "@backstage/plugin-search-backend-node": "^1.0.2",
     "@backstage/plugin-search-backend-module-pg": "^0.4.0",
+    "@backstage/plugin-search-backend-node": "^1.0.2",
     "@backstage/plugin-techdocs-backend": "^1.1.0",
-    "@frontside/backstage-plugin-effection-inspector-backend": "0.1.2",
     "@frontside/backstage-plugin-batch-loader": "0.2.2",
-    "@frontside/backstage-plugin-humanitec-backend": "^0.3.1",
+    "@frontside/backstage-plugin-effection-inspector-backend": "0.1.2",
     "@frontside/backstage-plugin-graphql": "^0.4.1",
+    "@frontside/backstage-plugin-humanitec-backend": "^0.3.1",
     "@frontside/backstage-plugin-incremental-ingestion-backend": "*",
     "@frontside/backstage-plugin-incremental-ingestion-github": "*",
-    "graphql-modules": "^2.1.0",
     "@gitbeaker/node": "^34.6.0",
     "@internal/plugin-healthcheck": "0.1.1",
+    "@frontside/backstage-plugin-sourcegraph-entity-provider": "^0.1.0",
     "@octokit/rest": "^18.5.3",
     "app": "*",
     "async-wait-until": "^2.0.12",
@@ -54,6 +54,7 @@
     "effection": "^2.0.4",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
+    "graphql-modules": "^2.1.0",
     "knex": "^2.0.0",
     "luxon": "^2.3.1",
     "pg": "^8.3.0",
@@ -65,8 +66,8 @@
     "@octokit/types": "^6.34.0",
     "@types/dockerode": "^3.3.0",
     "@types/express": "^4.17.6",
-    "@types/luxon": "^2.0.4",
     "@types/express-serve-static-core": "^4.17.5",
+    "@types/luxon": "^2.0.4",
     "better-sqlite3": "^7.5.0"
   },
   "files": [

packages/backend/src/plugins/catalog.ts

@@ -1,12 +1,14 @@
 import {
-  CatalogBuilder
+  CatalogBuilder,
+  EntityProvider
 } from '@backstage/plugin-catalog-backend';
 import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
 import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend';
 import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github';
 import { Router } from 'express';
 import { Duration } from 'luxon';
 import { PluginEnvironment } from '../types';
+import { SourcegraphEntityProvider } from "@frontside/backstage-plugin-sourcegraph-entity-provider";
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -34,6 +36,13 @@ export default async function createPlugin(
 
   builder.addProcessor(new ScaffolderEntitiesProcessor());
 
+  const sourcegraphProvider = SourcegraphEntityProvider.create({
+    config: env.config,
+    logger: env.logger,
+  });
+
+  builder.addEntityProvider(sourcegraphProvider as EntityProvider);
+
   const { processingEngine, router } = await builder.build();
 
   // this has to run after `await builder.build()` so ensure that catalog migrations are completed 
@@ -42,5 +51,14 @@ export default async function createPlugin(
 
   await processingEngine.start();
 
+  // const logg = env.logger;
+  // const scheduler = TaskScheduler.fromConfig(env.config);
+  // await env.scheduler.scheduleTask({
+  //   id: "test-task-scheduler",
+  //   frequency: {seconds: 3},
+  //   timeout: {minutes: 10},
+  //   fn: async () => { await sourcegraphProvider.test() }
+  // })
+
   return router;
 }

plugins/sourcegraph-entity-provider-backend/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/sourcegraph-entity-provider-backend/.gitignore

@@ -0,0 +1,2 @@
+src/providers/GitHub*
+src/lib

plugins/sourcegraph-entity-provider-backend/README.md

@@ -0,0 +1,14 @@
+# sourcegraph-entity-provider-backend
+
+Welcome to the sourcegraph-entity-provider-backend backend plugin!
+
+_This plugin was created through the Backstage CLI_
+
+## Getting started
+
+Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn
+start` in the root directory, and then navigating to [/sourcegraph-entity-provider-backend](http://localhost:3000/sourcegraph-entity-provider-backend).
+
+You can also serve the plugin in isolation by running `yarn start` in the plugin directory.
+This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads.
+It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory.

plugins/sourcegraph-entity-provider-backend/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@frontside/backstage-plugin-sourcegraph-entity-provider",
+  "version": "0.1.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": true,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin"
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.15.1",
+    "@backstage/backend-tasks": "^0.3.6-next.2",
+    "@backstage/config": "^1.0.2",
+    "@types/express": "*",
+    "express": "^4.17.1",
+    "express-promise-router": "^4.1.0",
+    "graphql-request": "^5.0.0",
+    "node-fetch": "^2.6.7",
+    "winston": "^3.2.1",
+    "yn": "^4.0.0"
+  },
+  "devDependencies": {
+    "@backstage/cli": "^0.19.0",
+    "@types/supertest": "^2.0.8",
+    "msw": "^0.46.0",
+    "supertest": "^4.0.2"
+  },
+  "files": [
+    "dist"
+  ]
+}

plugins/sourcegraph-entity-provider-backend/src/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './providers/SourcegraphEntityProvider';

plugins/sourcegraph-entity-provider-backend/src/providers/SourcegraphEntityProvider.ts

@@ -0,0 +1,128 @@
+import {
+  EntityProvider,
+  EntityProviderConnection,
+  parseEntityYaml
+} from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+import { GraphQLClient, gql } from "graphql-request";
+
+interface SourcegraphSearch {
+  search: {
+    results: {
+      results: [{
+        repository: {
+          name: string;
+        };
+        file: {
+          url: string;
+          content: string;
+        }
+      }]
+    }
+  }
+}
+
+export class SourcegraphEntityProvider implements EntityProvider {
+  private readonly config: Config;
+  private connection?: EntityProviderConnection;
+  private readonly logger: Logger;
+  private graphQLClient: GraphQLClient;
+
+  static create({ config, logger }: { config: Config, logger: Logger }) {
+    return new SourcegraphEntityProvider({ config, logger })
+  }
+
+  private constructor(options: {
+    config: Config,
+    logger: Logger,
+  }) {
+    this.config = options.config;
+    this.logger = options.logger.child({
+      target: this.getProviderName(),
+    });
+    this.graphQLClient = new GraphQLClient("");
+  }
+
+  getProviderName(): string {
+    return `sourcegraph-provider:${this.config.getString("sourcegraph.orgId")}`;
+  }
+
+  async connect(connection: EntityProviderConnection): Promise<void> {
+    this.connection = connection;
+    const endpoint = `https://${this.config.getString("sourcegraph.orgId")}.sourcegraph.com/.api/graphql`
+    this.graphQLClient = new GraphQLClient(endpoint, {
+      headers: {
+        authorization: `token ${this.config.getString("sourcegraph.token")}`
+      }
+    });
+    await this.test();
+  }
+
+  async test_old() {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+    const query = gql`
+      {
+        currentUser {
+          username
+        }
+      }
+    `;
+    const data = await this.graphQLClient.request(query);
+    this.logger.info(`🚨 HELLO ${JSON.stringify(data, null, 2)}`)
+  }
+  async test() {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+    const query = gql`
+      query ($query: String!) {
+        search(query: $query) {
+          results {
+            matchCount
+            results {
+              __typename
+              ... on FileMatch {
+                repository {
+                  name
+                }
+                file {
+                  url
+                  content
+                }
+              }
+            }
+          }
+        }
+      }
+    `;
+    const data: SourcegraphSearch = await this.graphQLClient.request(query, {
+      query: "file:^catalog-info.yaml"
+    });
+    data.search.results.results.forEach((result) => {
+      const location = {
+        type: "url",
+        target: result.file.url
+      };
+      this.logger.info(`✅ ${result.repository.name}`)
+
+      // const content = Buffer.from(data, "utf8");
+      // for (const parseResult of parseEntityYaml(content, location)) {
+      //   parseResults.push(parseResult)
+      // }
+    })
+  }
+
+  async full(logger: Logger) {
+    logger.info(`🚨 FULL}`)
+    // full mutation on schedule
+  }
+
+  async delta(logger: Logger) {
+    logger.info(`🚨 DELTA}`)
+    // delta mutation from webhooks
+  }
+
+}

plugins/sourcegraph-entity-provider-backend/src/setupTests.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export {};