Skip to content

Apply pmd to airbyte-config #13003

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
May 24, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@
* This config persistence contains all seed definitions according to the yaml files. It is
* read-only.
*/
public class YamlSeedConfigPersistence implements ConfigPersistence {
final public class YamlSeedConfigPersistence implements ConfigPersistence {

private static final String PERSISTENCE_READ_ONLY_ERROR_MSG = "The seed config persistence is read only.";
public static final Class<?> DEFAULT_SEED_DEFINITION_RESOURCE_CLASS = SeedType.class;

private static final Map<AirbyteConfig, SeedType> CONFIG_SCHEMA_MAP = Map.of(
Expand Down Expand Up @@ -178,22 +179,22 @@ public <T> List<ConfigWithMetadata<T>> listConfigsWithMetadata(final AirbyteConf

@Override
public <T> void writeConfig(final AirbyteConfig configType, final String configId, final T config) {
throw new UnsupportedOperationException("The seed config persistence is read only.");
throw new UnsupportedOperationException(PERSISTENCE_READ_ONLY_ERROR_MSG);
}

@Override
public <T> void writeConfigs(final AirbyteConfig configType, final Map<String, T> configs) {
throw new UnsupportedOperationException("The seed config persistence is read only.");
throw new UnsupportedOperationException(PERSISTENCE_READ_ONLY_ERROR_MSG);
}

@Override
public void deleteConfig(final AirbyteConfig configType, final String configId) {
throw new UnsupportedOperationException("The seed config persistence is read only.");
throw new UnsupportedOperationException(PERSISTENCE_READ_ONLY_ERROR_MSG);
}

@Override
public void replaceAllConfigs(final Map<AirbyteConfig, Stream<?>> configs, final boolean dryRun) {
throw new UnsupportedOperationException("The seed config persistence is read only.");
throw new UnsupportedOperationException(PERSISTENCE_READ_ONLY_ERROR_MSG);
}

@Override
Expand All @@ -205,7 +206,7 @@ public Map<String, Stream<JsonNode>> dumpConfigs() {

@Override
public void loadData(final ConfigPersistence seedPersistence) throws IOException {
throw new UnsupportedOperationException("The seed config persistence is read only.");
throw new UnsupportedOperationException(PERSISTENCE_READ_ONLY_ERROR_MSG);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.junit.jupiter.api.Test;

@Slf4j
public class SpecFormatTest {
class SpecFormatTest {

@Test
void testOnAllExistingConfig() throws IOException, JsonValidationException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,20 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

public class YamlSeedConfigPersistenceTest {
class YamlSeedConfigPersistenceTest {

private static YamlSeedConfigPersistence PERSISTENCE;
private static YamlSeedConfigPersistence persistence;

@BeforeAll
static void setup() throws IOException {
PERSISTENCE = YamlSeedConfigPersistence.getDefault();
persistence = YamlSeedConfigPersistence.getDefault();
}

@Test
public void testGetConfig() throws Exception {
void testGetConfig() throws Exception {
// source
final String mySqlSourceId = "435bb9a5-7887-4809-aa58-28c27df0d7ad";
final StandardSourceDefinition mysqlSource = PERSISTENCE
final StandardSourceDefinition mysqlSource = persistence
.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, mySqlSourceId, StandardSourceDefinition.class);
assertEquals(mySqlSourceId, mysqlSource.getSourceDefinitionId().toString());
assertEquals("MySQL", mysqlSource.getName());
Expand All @@ -49,7 +49,7 @@ public void testGetConfig() throws Exception {

// destination
final String s3DestinationId = "4816b78f-1489-44c1-9060-4b19d5fa9362";
final StandardDestinationDefinition s3Destination = PERSISTENCE
final StandardDestinationDefinition s3Destination = persistence
.getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, s3DestinationId, StandardDestinationDefinition.class);
assertEquals(s3DestinationId, s3Destination.getDestinationDefinitionId().toString());
assertEquals("S3", s3Destination.getName());
Expand All @@ -61,27 +61,27 @@ public void testGetConfig() throws Exception {
}

@Test
public void testGetInvalidConfig() {
void testGetInvalidConfig() {
assertThrows(
UnsupportedOperationException.class,
() -> PERSISTENCE.getConfig(ConfigSchema.STANDARD_SYNC, "invalid_id", StandardSync.class));
() -> persistence.getConfig(ConfigSchema.STANDARD_SYNC, "invalid_id", StandardSync.class));
assertThrows(
ConfigNotFoundException.class,
() -> PERSISTENCE.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, "invalid_id", StandardWorkspace.class));
() -> persistence.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, "invalid_id", StandardWorkspace.class));
}

@Test
public void testDumpConfigs() {
final Map<String, Stream<JsonNode>> allSeedConfigs = PERSISTENCE.dumpConfigs();
void testDumpConfigs() {
final Map<String, Stream<JsonNode>> allSeedConfigs = persistence.dumpConfigs();
assertEquals(2, allSeedConfigs.size());
assertTrue(allSeedConfigs.get(ConfigSchema.STANDARD_SOURCE_DEFINITION.name()).findAny().isPresent());
assertTrue(allSeedConfigs.get(ConfigSchema.STANDARD_DESTINATION_DEFINITION.name()).findAny().isPresent());
}

@Test
public void testWriteMethods() {
assertThrows(UnsupportedOperationException.class, () -> PERSISTENCE.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, "id", new Object()));
assertThrows(UnsupportedOperationException.class, () -> PERSISTENCE.replaceAllConfigs(Collections.emptyMap(), false));
void testWriteMethods() {
assertThrows(UnsupportedOperationException.class, () -> persistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, "id", new Object()));
assertThrows(UnsupportedOperationException.class, () -> persistence.replaceAllConfigs(Collections.emptyMap(), false));
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

public class AirbyteConfigValidator extends AbstractSchemaValidator<ConfigSchema> {

public static AirbyteConfigValidator AIRBYTE_CONFIG_VALIDATOR = new AirbyteConfigValidator();
final public static AirbyteConfigValidator AIRBYTE_CONFIG_VALIDATOR = new AirbyteConfigValidator();

@Override
public Path getSchemaPath(final ConfigSchema configType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.nio.file.Path;
import java.util.function.Function;

@SuppressWarnings({"PMD.AvoidThrowingRawExceptionTypes", "PMD.NullAssignment"})
public enum ConfigSchema implements AirbyteConfig {

// workspace
Expand Down Expand Up @@ -108,6 +109,7 @@ public File getConfigSchemaFile() {
return KNOWN_SCHEMAS_ROOT.resolve(schemaFilename).toFile();
}

@Override
public <T> Class<T> getClassName() {
return (Class<T>) className;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.time.Instant;
import java.util.Objects;

@SuppressWarnings("PMD.ShortVariable")
public class ConfigWithMetadata<T> {

private final String configId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
* <p>
* 2. 'Alpha support' if a var does not have proper support and should be used with care.
*/

@SuppressWarnings("PMD.BooleanGetMethodName")
public interface Configs {

// CORE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@SuppressWarnings({"PMD.LongVariable", "PMD.CyclomaticComplexity", "PMD.AvoidReassigningParameters"})
public class EnvConfigs implements Configs {

private static final Logger LOGGER = LoggerFactory.getLogger(EnvConfigs.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
/**
* Represents a minimal io.fabric8.kubernetes.api.model.Toleration
*/
@SuppressWarnings("PMD.ShortVariable")
public class TolerationPOJO {

private final String key;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
* that path 2) log files names start with timestamps, making it possible extract the time the file
* was written from it's name.
*/
@SuppressWarnings("PMD.AvoidThrowingRawExceptionTypes")
public interface CloudLogs {

Logger LOGGER = LoggerFactory.getLogger(CloudLogs.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@SuppressWarnings({"PMD.AvoidFileStream", "PMD.ShortVariable", "PMD.CloseResource", "PMD.AvoidInstantiatingObjectsInLoops"})
public class GcsLogs implements CloudLogs {

private static final Logger LOGGER = LoggerFactory.getLogger(GcsLogs.class);

private static Storage GCS;
private static Storage gcs;
private final Supplier<Storage> gcsClientFactory;

public GcsLogs(final Supplier<Storage> gcsClientFactory) {
Expand Down Expand Up @@ -120,10 +121,10 @@ public void deleteLogs(final LogConfigs configs, final String logPath) {
}

private Storage getOrCreateGcsClient() {
if (GCS == null) {
GCS = gcsClientFactory.get();
if (gcs == null) {
gcs = gcsClientFactory.get();
}
return GCS;
return gcs;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
* {@link LogConfigs} within this class. Beyond this class, all configuration consumption is via the
* {@link LogConfigs} interface via the {@link CloudLogs} interface.
*/
@SuppressWarnings({"PMD.AvoidThrowingRawExceptionTypes", "PMD.AvoidSynchronizedAtMethodLevel"})
public class LogClientSingleton {

private static final Logger LOGGER = LoggerFactory.getLogger(LogClientSingleton.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
*/
public class LogConfigs {

public static LogConfigs EMPTY = new LogConfigs(null);
public final static LogConfigs EMPTY = new LogConfigs(null);

private final CloudStorageConfigs storageConfigs;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;

@SuppressWarnings({"PMD.ShortVariable", "PMD.CloseResource", "PMD.AvoidFileStream"})
public class S3Logs implements CloudLogs {

private static final Logger LOGGER = LoggerFactory.getLogger(S3Logs.class);

private static S3Client S3;
private static S3Client s3;

private final Supplier<S3Client> s3ClientFactory;

Expand Down Expand Up @@ -100,7 +101,7 @@ public List<String> tailCloudLog(final LogConfigs configs, final String logPath,

final var s3Bucket = getBucketName(configs.getStorageConfigs());
LOGGER.debug("Start making S3 list request.");
final ArrayList<String> ascendingTimestampKeys = getAscendingObjectKeys(s3Client, logPath, s3Bucket);
final List<String> ascendingTimestampKeys = getAscendingObjectKeys(s3Client, logPath, s3Bucket);
final var descendingTimestampKeys = Lists.reverse(ascendingTimestampKeys);

final var lines = new ArrayList<String>();
Expand Down Expand Up @@ -145,13 +146,13 @@ public void deleteLogs(final LogConfigs configs, final String logPath) {
}

private S3Client getOrCreateS3Client() {
if (S3 == null) {
S3 = s3ClientFactory.get();
if (s3 == null) {
s3 = s3ClientFactory.get();
}
return S3;
return s3;
}

private static ArrayList<String> getAscendingObjectKeys(final S3Client s3Client, final String logPath, final String s3Bucket) {
private static List<String> getAscendingObjectKeys(final S3Client s3Client, final String logPath, final String s3Bucket) {
final var listObjReq = ListObjectsV2Request.builder().bucket(s3Bucket).prefix(logPath).build();
final var ascendingTimestampObjs = new ArrayList<String>();

Expand All @@ -164,7 +165,7 @@ private static ArrayList<String> getAscendingObjectKeys(final S3Client s3Client,
return ascendingTimestampObjs;
}

private static ArrayList<String> getCurrFile(final S3Client s3Client, final String s3Bucket, final String poppedKey) throws IOException {
private static List<String> getCurrFile(final S3Client s3Client, final String s3Bucket, final String poppedKey) throws IOException {
final var getObjReq = GetObjectRequest.builder()
.key(poppedKey)
.bucket(s3Bucket)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.airbyte.config.Schedule;
import java.util.concurrent.TimeUnit;

@SuppressWarnings("PMD.AvoidThrowingRawExceptionTypes")
public class ScheduleHelpers {

public static Long getSecondsInUnit(final Schedule.TimeUnit timeUnitEnum) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,42 +31,43 @@
*
* Methods in these class throw Runtime exceptions upon validation failure.
*/
@SuppressWarnings("PMD.ShortVariable")
public class YamlListToStandardDefinitions {

private static final Map<String, String> classNameToIdName = Map.ofEntries(
private static final Map<String, String> CLASS_NAME_TO_ID_NAME = Map.ofEntries(
new SimpleImmutableEntry<>(StandardDestinationDefinition.class.getCanonicalName(), "destinationDefinitionId"),
new SimpleImmutableEntry<>(StandardSourceDefinition.class.getCanonicalName(), "sourceDefinitionId"));

public static List<StandardSourceDefinition> toStandardSourceDefinitions(final String yamlStr) throws RuntimeException {
public static List<StandardSourceDefinition> toStandardSourceDefinitions(final String yamlStr) {
return verifyAndConvertToModelList(StandardSourceDefinition.class, yamlStr);
}

public static List<StandardDestinationDefinition> toStandardDestinationDefinitions(final String yamlStr) throws RuntimeException {
public static List<StandardDestinationDefinition> toStandardDestinationDefinitions(final String yamlStr) {
return verifyAndConvertToModelList(StandardDestinationDefinition.class, yamlStr);
}

public static JsonNode verifyAndConvertToJsonNode(final String idName, final String yamlStr) throws RuntimeException {
public static JsonNode verifyAndConvertToJsonNode(final String idName, final String yamlStr) {
final var jsonNode = Yamls.deserialize(yamlStr);
checkYamlIsPresentWithNoDuplicates(jsonNode, idName);
return jsonNode;
}

@VisibleForTesting
static <T> List<T> verifyAndConvertToModelList(final Class<T> klass, final String yamlStr) throws RuntimeException {
static <T> List<T> verifyAndConvertToModelList(final Class<T> klass, final String yamlStr) {
final var jsonNode = Yamls.deserialize(yamlStr);
final var idName = classNameToIdName.get(klass.getCanonicalName());
final var idName = CLASS_NAME_TO_ID_NAME.get(klass.getCanonicalName());
checkYamlIsPresentWithNoDuplicates(jsonNode, idName);
return toStandardXDefinitions(jsonNode.elements(), klass);
}

private static void checkYamlIsPresentWithNoDuplicates(final JsonNode deserialize, final String idName) throws RuntimeException {
private static void checkYamlIsPresentWithNoDuplicates(final JsonNode deserialize, final String idName) {
final var presentDestList = !deserialize.elements().equals(ClassUtil.emptyIterator());
Preconditions.checkState(presentDestList, "Definition list is empty");
checkNoDuplicateNames(deserialize.elements());
checkNoDuplicateIds(deserialize.elements(), idName);
}

private static void checkNoDuplicateNames(final Iterator<JsonNode> iter) throws IllegalArgumentException {
private static void checkNoDuplicateNames(final Iterator<JsonNode> iter) {
final var names = new HashSet<String>();
while (iter.hasNext()) {
final var element = Jsons.clone(iter.next());
Expand All @@ -78,7 +79,7 @@ private static void checkNoDuplicateNames(final Iterator<JsonNode> iter) throws
}
}

private static void checkNoDuplicateIds(final Iterator<JsonNode> fileIterator, final String idName) throws IllegalArgumentException {
private static void checkNoDuplicateIds(final Iterator<JsonNode> fileIterator, final String idName) {
final var ids = new HashSet<String>();
while (fileIterator.hasNext()) {
final var element = Jsons.clone(fileIterator.next());
Expand All @@ -90,7 +91,7 @@ private static void checkNoDuplicateIds(final Iterator<JsonNode> fileIterator, f
}
}

private static <T> List<T> toStandardXDefinitions(final Iterator<JsonNode> iter, final Class<T> c) throws RuntimeException {
private static <T> List<T> toStandardXDefinitions(final Iterator<JsonNode> iter, final Class<T> c) {
final Iterable<JsonNode> iterable = () -> iter;
final var defList = new ArrayList<T>();
for (final JsonNode n : iterable) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
* wherever that cloud storage is used and then, based on the configuration, spin up the correct
* client. This configuration object allows us to do that.
*/
@SuppressWarnings("PMD.ShortMethodName")
public class CloudStorageConfigs {

public enum WorkerStorageType {
Expand Down Expand Up @@ -88,7 +89,7 @@ public GcsConfig getGcsConfig() {
return gcsConfig;
}

public static abstract class S3ApiWorkerStorageConfig {
public static class S3ApiWorkerStorageConfig {

private final String bucketName;
private final String awsAccessKey;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
* Takes in the constructor our standard format for gcs configuration and provides a factory that
* uses that configuration to create a GCS client (Storage).
*/
@SuppressWarnings("PMD.AvoidThrowingRawExceptionTypes")
public class DefaultGcsClientFactory implements Supplier<Storage> {

private final GcsConfig config;
Expand All @@ -38,7 +39,7 @@ public Storage get() {
final var credentialsByteStream = new ByteArrayInputStream(Files.readAllBytes(Path.of(config.getGoogleApplicationCredentials())));
final var credentials = ServiceAccountCredentials.fromStream(credentialsByteStream);
return StorageOptions.newBuilder().setCredentials(credentials).build().getService();
} catch (Exception e) {
} catch (final Exception e) {
throw new RuntimeException(e);
}
}
Expand Down
Loading