source methods and tests

Author: cgardens

data/config/STANDARD_WORKSPACE_CONFIGURATION/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json renamed to data/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json

@@ -222,7 +222,7 @@ paths:
       tags:
       - source_implementation
       summary: Get source implementations for workspace
-      operationId: getSourceImplementationsForWorkspace
+      operationId: listSourceImplementationsForWorkspace
       requestBody:
         content:
           application/json:

dataline-api/src/main/openapi/config.yaml

@@ -1,4 +1,6 @@
 dependencies {
+    implementation group: 'commons-io', name: 'commons-io', version: '2.7'
+
     implementation group: "com.fasterxml.jackson.core", name: "jackson-databind", version: "2.9.8"
     implementation group: "com.networknt", name: "json-schema-validator", version: "1.0.42"
 

dataline-config-persistence/build.gradle

@@ -12,18 +12,32 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
 
 // we force all interaction with disk storage to be effectively single threaded.
 public class ConfigPersistenceImpl implements ConfigPersistence {
   private static final Object lock = new Object();
-  private static final String CONFIG_STORAGE_ROOT = "data/config/";
-  private static final String CONFIG_SCHEMA_ROOT = "dataline-config/src/main/resources/json/";
+  private static final String CONFIG_STORAGE_ROOT = "../data/config/";
+  private static final String TEST_STORAGE_ROOT = "/tmp/data/config/";
+  private static final String CONFIG_SCHEMA_ROOT = "../dataline-config/src/main/resources/json/";
 
   private final ObjectMapper objectMapper;
-  final JsonSchemaValidation jsonSchemaValidation;
+  private final JsonSchemaValidation jsonSchemaValidation;
+  private final String storageRoot;
+  private final boolean testEnv;
 
-  public ConfigPersistenceImpl() {
-    jsonSchemaValidation = JsonSchemaValidation.getInstance();
+  public static ConfigPersistenceImpl get() {
+    return new ConfigPersistenceImpl(CONFIG_STORAGE_ROOT, false);
+  }
+
+  public static ConfigPersistenceImpl getTest() {
+    return new ConfigPersistenceImpl(TEST_STORAGE_ROOT, true);
+  }
+
+  private ConfigPersistenceImpl(String storageRoot, boolean testEnv) {
+    this.storageRoot = storageRoot;
+    this.testEnv = testEnv;
+    jsonSchemaValidation = JsonSchemaValidation.getSingletonInstance();
     objectMapper = new ObjectMapper();
   }
 
@@ -70,14 +84,28 @@ public <T> Set<T> getConfigs(PersistenceConfigType persistenceConfigType, Class<
   public <T> void writeConfig(
       PersistenceConfigType persistenceConfigType, String configId, T config) {
     synchronized (lock) {
+      final String configPath = getConfigPath(persistenceConfigType, configId);
+      ensureDirectory(getConfigDirectory(persistenceConfigType));
       try {
-        objectMapper.writeValue(new File(getConfigPath(persistenceConfigType, configId)), config);
+        objectMapper.writeValue(new File(configPath), config);
       } catch (IOException e) {
         throw new RuntimeException(e);
       }
     }
   }
 
+  public void deleteAll() {
+    if (!testEnv) {
+      throw new RuntimeException("deleteAll operation is only allowed in test environment");
+    }
+
+    try {
+      FileUtils.forceDelete(new File(storageRoot));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   private JsonNode getSchema(PersistenceConfigType persistenceConfigType) {
     String configSchemaFilename =
         standardConfigTypeToConfigSchema(persistenceConfigType).getSchemaFilename();
@@ -99,7 +127,7 @@ private Set<Path> getFiles(PersistenceConfigType persistenceConfigType) {
   }
 
   private String getConfigDirectory(PersistenceConfigType persistenceConfigType) {
-    return String.format("%s/%s", CONFIG_STORAGE_ROOT, persistenceConfigType.toString());
+    return String.format("%s/%s", storageRoot, persistenceConfigType.toString());
   }
 
   private String getConfigPath(PersistenceConfigType persistenceConfigType, String configId) {
@@ -113,8 +141,8 @@ private Set<String> getConfigIds(PersistenceConfigType persistenceConfigType) {
   }
 
   private Optional<Path> getFile(PersistenceConfigType persistenceConfigType, String id) {
-    String configPath = getConfigPath(persistenceConfigType, id);
-    final Path path = Paths.get(configPath);
+    ensureDirectory(getConfigDirectory(persistenceConfigType));
+    final Path path = Paths.get(getConfigPath(persistenceConfigType, id));
     if (Files.exists(path)) {
       return Optional.of(path);
     } else {
@@ -184,7 +212,10 @@ private File getFileOrThrow(PersistenceConfigType persistenceConfigType, String
             () ->
                 new ConfigNotFoundException(
                     String.format(
-                        "config type: %s id: %s not found", persistenceConfigType, configId)));
+                        "config type: %s id: %s not found in path %s",
+                        persistenceConfigType,
+                        configId,
+                        getConfigPath(persistenceConfigType, configId))));
   }
 
   private <T> T fileToPojo(File file, Class<T> clazz) {
@@ -194,4 +225,12 @@ private <T> T fileToPojo(File file, Class<T> clazz) {
       throw new RuntimeException(e);
     }
   }
+
+  private void ensureDirectory(String path) {
+    try {
+      FileUtils.forceMkdir(new File(path));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
 }

dataline-config-persistence/src/main/java/io/dataline/config/persistence/ConfigPersistenceImpl.java

@@ -11,7 +11,7 @@ public class JsonSchemaValidation {
 
   private static final JsonSchemaValidation INSTANCE = new JsonSchemaValidation();
 
-  public static JsonSchemaValidation getInstance() {
+  public static JsonSchemaValidation getSingletonInstance() {
     return INSTANCE;
   }
 

dataline-config-persistence/src/main/java/io/dataline/config/persistence/JsonSchemaValidation.java

@@ -3,13 +3,17 @@
   "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/SourceConnectionConfiguration.json",
   "title": "SourceConnectionConfiguration",
   "description": "information required for connection to a destination.",  "type": "object",
-  "required": ["sourceSpecificationId", "sourceImplementationId", "configuration"],
+  "required": ["sourceSpecificationId", "sourceImplementationId", "workspaceId","configuration"],
   "additionalProperties": false,
   "properties": {
     "sourceSpecificationId": {
       "type": "string",
       "format": "uuid"
     },
+    "workspaceId": {
+      "type": "string",
+      "format": "uuid"
+    },
     "sourceImplementationId": {
       "type": "string",
       "format": "uuid"

dataline-config/src/main/resources/json/SourceConnectionImplementation.json

@@ -14,6 +14,7 @@ dependencies {
     implementation group: "com.fasterxml.jackson.core", name: "jackson-databind", version: "2.9.8"
     implementation group: "com.networknt", name: "json-schema-validator", version: "1.0.42"
 
+
     implementation project(':dataline-api')
     implementation project(':dataline-commons')
     implementation project(':dataline-config')

dataline-server/build.gradle

@@ -18,7 +18,7 @@ public class ConfigurationApi implements io.dataline.api.V1Api {
   private final SourceImplementationsHandler sourceImplementationsHandler;
 
   public ConfigurationApi() {
-    ConfigPersistence configPersistence = new ConfigPersistenceImpl();
+    ConfigPersistence configPersistence = ConfigPersistenceImpl.get();
     workspacesHandler = new WorkspacesHandler(configPersistence);
     sourcesHandler = new SourcesHandler(configPersistence);
     sourceSpecificationsHandler = new SourceSpecificationsHandler(configPersistence);
@@ -69,22 +69,17 @@ public SourceImplementationRead createSourceImplementation(
     return sourceImplementationsHandler.createSourceImplementation(sourceImplementationCreate);
   }
 
-  @Override
-  public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(
-      @Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
-    return null;
-  }
-
   @Override
   public SourceImplementationRead getSourceImplementation(
       @Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
-    return null;
+    return sourceImplementationsHandler.getSourceImplementation(sourceImplementationIdRequestBody);
   }
 
   @Override
-  public SourceImplementationReadList getSourceImplementationsForWorkspace(
+  public SourceImplementationReadList listSourceImplementationsForWorkspace(
       @Valid WorkspaceIdRequestBody workspaceIdRequestBody) {
-    return null;
+    return sourceImplementationsHandler.listSourceImplementationsForWorkspace(
+        workspaceIdRequestBody);
   }
 
   @Override
@@ -96,6 +91,12 @@ public SourceImplementationTestConnectionRead testConnectionToSourceImplementati
   @Override
   public SourceImplementationRead updateSourceImplementation(
       @Valid SourceImplementationUpdate sourceImplementationUpdate) {
+    return sourceImplementationsHandler.updateSourceImplementation(sourceImplementationUpdate);
+  }
+
+  @Override
+  public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(
+      @Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
     return null;
   }
 

dataline-server/src/main/java/io/dataline/server/apis/ConfigurationApi.java

@@ -4,12 +4,16 @@
 import javax.ws.rs.core.Response;
 import javax.ws.rs.ext.ExceptionMapper;
 import javax.ws.rs.ext.Provider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Provider
 public class CatchAllExceptionMapper implements ExceptionMapper<Throwable> {
+  private static final Logger LOGGER = LoggerFactory.getLogger(CatchAllExceptionMapper.class);
 
   @Override
   public Response toResponse(Throwable e) {
+    LOGGER.debug("catch all exception", e);
     return Response.status(500)
         .entity(
             new ObjectMapper()

dataline-server/src/main/java/io/dataline/server/errors/CatchAllExceptionMapper.java

@@ -4,14 +4,18 @@
 import javax.ws.rs.core.Response;
 import javax.ws.rs.ext.ExceptionMapper;
 import javax.ws.rs.ext.Provider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Provider
 public class KnownExceptionMapper implements ExceptionMapper<KnownException> {
+  private static final Logger LOGGER = LoggerFactory.getLogger(KnownExceptionMapper.class);
 
   @Override
   public Response toResponse(KnownException e) {
+    LOGGER.debug("known exception", e);
     return Response.status(e.getHttpCode())
-        .entity(new ObjectMapper().createObjectNode().put("message: ", e.getMessage()))
+        .entity(new ObjectMapper().createObjectNode().put("message", e.getMessage()))
         .type("application/json")
         .build();
   }