feat(orchestrator): verify availability and cache workflow definition IDs (janus-idp#1309)

* feat(orchestrator): verify availability and cache workflow definition IDs

* Remove unnecessary line

* Minor fix after conflict resolution

* Minor improvements

* Add tests for OrchestratorService

* Cached data should be considerd for listing queries

* Remove definition from cache that does not exist on Data Index anymore

* Update the API sha

* Fix for empty cache corner case

Author: caponetto

plugins/orchestrator-backend/dev/index.ts

@@ -1,4 +1,5 @@
 import { createServiceBuilder, UrlReader } from '@backstage/backend-common';
+import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { CatalogApi } from '@backstage/catalog-client';
 import { Config } from '@backstage/config';
 import { DiscoveryApi } from '@backstage/core-plugin-api';
@@ -19,6 +20,7 @@ export interface ServerOptions {
   discovery: DiscoveryApi;
   catalogApi: CatalogApi;
   urlReader: UrlReader;
+  scheduler: PluginTaskScheduler;
 }
 
 export async function startStandaloneServer(
@@ -32,6 +34,7 @@ export async function startStandaloneServer(
     discovery: options.discovery,
     catalogApi: options.catalogApi,
     urlReader: options.urlReader,
+    scheduler: options.scheduler,
   });
 
   let service = createServiceBuilder(module)

plugins/orchestrator-backend/src/OrchestratorPlugin.ts

@@ -17,6 +17,7 @@ export const orchestratorPlugin = createBackendPlugin({
         discovery: coreServices.discovery,
         httpRouter: coreServices.httpRouter,
         urlReader: coreServices.urlReader,
+        scheduler: coreServices.scheduler,
         catalogApi: catalogServiceRef,
       },
       async init({
@@ -26,6 +27,7 @@ export const orchestratorPlugin = createBackendPlugin({
         httpRouter,
         catalogApi,
         urlReader,
+        scheduler,
       }) {
         const log = loggerToWinstonLogger(logger);
         const router = await createRouter({
@@ -34,6 +36,7 @@ export const orchestratorPlugin = createBackendPlugin({
           discovery: discovery,
           catalogApi: catalogApi,
           urlReader: urlReader,
+          scheduler: scheduler,
         });
         httpRouter.use(router);
       },

plugins/orchestrator-backend/src/routerWrapper/index.ts

@@ -1,54 +1,46 @@
 import { UrlReader } from '@backstage/backend-common';
+import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { CatalogApi } from '@backstage/catalog-client';
 import { Config } from '@backstage/config';
 import { DiscoveryApi } from '@backstage/core-plugin-api';
 
 import express from 'express';
 import { Logger } from 'winston';
 
-import { DataIndexService } from '../service/DataIndexService';
+import { DevModeService } from '../service/DevModeService';
 import { createBackendRouter } from '../service/router';
-import { SonataFlowService } from '../service/SonataFlowService';
 
 export interface RouterArgs {
   config: Config;
   logger: Logger;
   discovery: DiscoveryApi;
   catalogApi: CatalogApi;
   urlReader: UrlReader;
+  scheduler: PluginTaskScheduler;
 }
 
 export async function createRouter(args: RouterArgs): Promise<express.Router> {
-  const dataIndexService = initDataIndexService(args.logger, args.config);
-  const sonataFlowService = new SonataFlowService(
-    args.config,
-    dataIndexService,
-    args.logger,
-  );
-
-  const router = await createBackendRouter({
+  const autoStartDevMode =
+    args.config.getOptionalBoolean(
+      'orchestrator.sonataFlowService.autoStart',
+    ) ?? false;
+
+  if (autoStartDevMode) {
+    const devModeService = new DevModeService(args.config, args.logger);
+
+    const isSonataFlowUp = await devModeService.launchDevMode();
+
+    if (!isSonataFlowUp) {
+      args.logger.error('SonataFlow is not up. Check your configuration.');
+    }
+  }
+
+  return await createBackendRouter({
     config: args.config,
     logger: args.logger,
     discovery: args.discovery,
     catalogApi: args.catalogApi,
     urlReader: args.urlReader,
-    sonataFlowService,
-    dataIndexService,
+    scheduler: args.scheduler,
   });
-
-  const isSonataFlowUp = await sonataFlowService.connect();
-
-  if (!isSonataFlowUp) {
-    args.logger.error('SonataFlow is not up. Check your configuration.');
-  }
-
-  return router;
-}
-
-function initDataIndexService(
-  logger: Logger,
-  config: Config,
-): DataIndexService {
-  const dataIndexUrl = config.getString('orchestrator.dataIndexService.url');
-  return new DataIndexService(dataIndexUrl, logger);
 }

plugins/orchestrator-backend/src/run.ts

@@ -4,6 +4,7 @@ import {
   loadBackendConfig,
   UrlReaders,
 } from '@backstage/backend-common';
+import { TaskScheduler } from '@backstage/backend-tasks';
 import { CatalogClient } from '@backstage/catalog-client';
 import { DefaultEventBroker } from '@backstage/plugin-events-backend';
 
@@ -17,6 +18,7 @@ const logger = getRootLogger();
 const config = await loadBackendConfig({ logger, argv: process.argv });
 const eventBroker = new DefaultEventBroker(logger);
 const discovery = HostDiscovery.fromConfig(config);
+const scheduler = TaskScheduler.fromConfig(config).forPlugin('orchestrator');
 const catalogApi = new CatalogClient({
   discoveryApi: HostDiscovery.fromConfig(config),
 });
@@ -31,6 +33,7 @@ startStandaloneServer({
   discovery,
   catalogApi,
   urlReader,
+  scheduler,
 }).catch(err => {
   logger.error(err);
   process.exit(1);