Skip to content

Commit b154acf

Browse files
akidambisrinivasanakidambisrinivasan
authored
Address KCLv3 issues from github (#1398)
* Address KCLv3 issues reported on github 1. Fix transitive dependencies and add a maven plugin to catch these at build time 2. Remove the redundant shutdown of the leaseCoordinatorThreadPool 3. Fix typo THROUGHOUT_PUT_KBPS 4. Fix shutdown sequence - make sure scheduler shutdown without invoking run works 5. Fix backward compatibility check - Avoid flagging methods as deleted if it is marked synchronized. Also mark interfaces introduced in KCLv3 as internal.
1 parent 004f6d3 commit b154acf

11 files changed

+139
-13
lines changed

‎.github/scripts/backwards_compatibility_check.sh

+9-5
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,18 @@ is_non_public_class() {
5858
return $?
5959
}
6060

61-
# Ignore methods that change from abstract to non-abstract (and vice versa) if the class is an interface.
62-
ignore_abstract_changes_in_interfaces() {
61+
# Ignore methods that change from abstract to non-abstract (and vice-versa) if the class is an interface.\
62+
# Ignore methods that change from synchronized to non-synchronized (and vice-versa)
63+
ignore_non_breaking_changes() {
6364
local current_class="$1"
6465
local class_definition=$(javap -classpath "$LATEST_JAR" "$current_class" | head -2 | tail -1)
6566
if [[ $class_definition == *"interface"* ]]
6667
then
67-
LATEST_METHODS=${LATEST_METHODS// abstract / }
68-
CURRENT_METHODS=${CURRENT_METHODS// abstract / }
68+
LATEST_METHODS=${LATEST_METHODS//abstract /}
69+
CURRENT_METHODS=${CURRENT_METHODS//abstract /}
70+
else
71+
LATEST_METHODS=${LATEST_METHODS//synchronized /}
72+
CURRENT_METHODS=${CURRENT_METHODS//synchronized /}
6973
fi
7074
}
7175

@@ -103,7 +107,7 @@ find_removed_methods() {
103107

104108
LATEST_METHODS=$(javap -classpath "$LATEST_JAR" "$class")
105109

106-
ignore_abstract_changes_in_interfaces "$class"
110+
ignore_non_breaking_changes "$class"
107111

108112
local removed_methods=$(diff <(echo "$LATEST_METHODS") <(echo "$CURRENT_METHODS") | grep '^<')
109113

‎amazon-kinesis-client/pom.xml

+109
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,42 @@
9090
<artifactId>netty-nio-client</artifactId>
9191
<version>${awssdk.version}</version>
9292
</dependency>
93+
<dependency>
94+
<groupId>software.amazon.awssdk</groupId>
95+
<artifactId>sdk-core</artifactId>
96+
<version>${awssdk.version}</version>
97+
</dependency>
98+
<dependency>
99+
<groupId>software.amazon.awssdk</groupId>
100+
<artifactId>aws-core</artifactId>
101+
<version>${awssdk.version}</version>
102+
</dependency>
103+
<dependency>
104+
<groupId>software.amazon.awssdk</groupId>
105+
<artifactId>arns</artifactId>
106+
<version>${awssdk.version}</version>
107+
</dependency>
108+
<dependency>
109+
<groupId>software.amazon.awssdk</groupId>
110+
<artifactId>regions</artifactId>
111+
<version>${awssdk.version}</version>
112+
</dependency>
113+
<dependency>
114+
<groupId>software.amazon.awssdk</groupId>
115+
<artifactId>utils</artifactId>
116+
<version>${awssdk.version}</version>
117+
</dependency>
118+
<dependency>
119+
<groupId>software.amazon.awssdk</groupId>
120+
<artifactId>http-client-spi</artifactId>
121+
<version>${awssdk.version}</version>
122+
</dependency>
123+
<dependency>
124+
<groupId>software.amazon.awssdk</groupId>
125+
<artifactId>dynamodb-enhanced</artifactId>
126+
<version>${awssdk.version}</version>
127+
</dependency>
128+
93129
<dependency>
94130
<groupId>software.amazon.glue</groupId>
95131
<artifactId>schema-registry-serde</artifactId>
@@ -127,6 +163,36 @@
127163
<artifactId>commons-collections</artifactId>
128164
<version>3.2.2</version>
129165
</dependency>
166+
<dependency>
167+
<groupId>org.apache.commons</groupId>
168+
<artifactId>commons-collections4</artifactId>
169+
<version>4.4</version>
170+
</dependency>
171+
<dependency>
172+
<groupId>io.netty</groupId>
173+
<artifactId>netty-handler</artifactId>
174+
<version>4.1.108.Final</version>
175+
</dependency>
176+
<dependency>
177+
<groupId>com.google.code.findbugs</groupId>
178+
<artifactId>jsr305</artifactId>
179+
<version>3.0.2</version>
180+
</dependency>
181+
<dependency>
182+
<groupId>com.fasterxml.jackson.core</groupId>
183+
<artifactId>jackson-databind</artifactId>
184+
<version>2.10.1</version>
185+
</dependency>
186+
<dependency>
187+
<groupId>org.reactivestreams</groupId>
188+
<artifactId>reactive-streams</artifactId>
189+
<version>1.0.4</version>
190+
</dependency>
191+
<dependency>
192+
<groupId>software.amazon.awssdk</groupId>
193+
<artifactId>annotations</artifactId>
194+
<version>2.25.64</version>
195+
</dependency>
130196
<dependency>
131197
<groupId>org.slf4j</groupId>
132198
<artifactId>slf4j-api</artifactId>
@@ -153,6 +219,18 @@
153219
</dependency>
154220

155221
<!-- Test -->
222+
<dependency>
223+
<groupId>software.amazon.awssdk</groupId>
224+
<artifactId>sts</artifactId>
225+
<version>${awssdk.version}</version>
226+
<scope>test</scope>
227+
</dependency>
228+
<dependency>
229+
<groupId>software.amazon.awssdk</groupId>
230+
<artifactId>auth</artifactId>
231+
<version>${awssdk.version}</version>
232+
<scope>test</scope>
233+
</dependency>
156234
<!-- TODO: Migrate all tests to Junit5 -->
157235
<dependency>
158236
<groupId>org.junit.jupiter</groupId>
@@ -180,12 +258,24 @@
180258
<version>3.12.4</version>
181259
<scope>test</scope>
182260
</dependency>
261+
<dependency>
262+
<groupId>org.mockito</groupId>
263+
<artifactId>mockito-core</artifactId>
264+
<version>3.12.4</version>
265+
<scope>test</scope>
266+
</dependency>
183267
<dependency>
184268
<groupId>org.hamcrest</groupId>
185269
<artifactId>hamcrest-all</artifactId>
186270
<version>1.3</version>
187271
<scope>test</scope>
188272
</dependency>
273+
<dependency>
274+
<groupId>org.hamcrest</groupId>
275+
<artifactId>hamcrest-core</artifactId>
276+
<version>1.3</version>
277+
<scope>test</scope>
278+
</dependency>
189279
<!-- Using older version to be compatible with Java 8 -->
190280
<!-- https://mvnrepository.com/artifact/com.amazonaws/DynamoDBLocal -->
191281
<dependency>
@@ -464,6 +554,25 @@
464554
</execution>
465555
</executions>
466556
</plugin>
557+
<plugin>
558+
<groupId>org.apache.maven.plugins</groupId>
559+
<artifactId>maven-dependency-plugin</artifactId>
560+
<version>3.1.2</version>
561+
<executions>
562+
<execution>
563+
<id>analyze-dependencies</id>
564+
<phase>verify</phase>
565+
<goals>
566+
<goal>analyze-only</goal>
567+
</goals>
568+
<configuration>
569+
<failOnWarning>true</failOnWarning>
570+
<!-- Ignore Runtime/Provided/Test/System scopes for unused dependency analysis. -->
571+
<ignoreNonCompile>true</ignoreNonCompile>
572+
</configuration>
573+
</execution>
574+
</executions>
575+
</plugin>
467576
</plugins>
468577

469578
</build>

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/DynamicMigrationComponentsInitializer.java

+6
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,9 @@ void shutdown() {
223223
workerMetricsThreadPool.shutdown();
224224
try {
225225
if (!lamThreadPool.awaitTermination(SCHEDULER_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
226+
log.info(
227+
"LamThreadPool did not shutdown in {}s, forcefully shutting down",
228+
SCHEDULER_SHUTDOWN_TIMEOUT_SECONDS);
226229
lamThreadPool.shutdownNow();
227230
}
228231
} catch (final InterruptedException e) {
@@ -232,6 +235,9 @@ void shutdown() {
232235

233236
try {
234237
if (!workerMetricsThreadPool.awaitTermination(SCHEDULER_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
238+
log.info(
239+
"WorkerMetricsThreadPool did not shutdown in {}s, forcefully shutting down",
240+
SCHEDULER_SHUTDOWN_TIMEOUT_SECONDS);
235241
workerMetricsThreadPool.shutdownNow();
236242
}
237243
} catch (final InterruptedException e) {

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/assignment/LeaseAssignmentDecider.java

+2
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717

1818
import java.util.List;
1919

20+
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
2021
import software.amazon.kinesis.leases.Lease;
2122

23+
@KinesisClientInternalApi
2224
public interface LeaseAssignmentDecider {
2325

2426
/**

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/migration/MigrationClientVersion3xWithRollbackState.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public synchronized void enter(final ClientVersion fromClientVersion) throws Dep
8787
}
8888

8989
@Override
90-
public void leave() {
90+
public synchronized void leave() {
9191
if (entered && !left) {
9292
log.info("Leaving {}", this);
9393
cancelRollbackMonitor();

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/migration/MigrationClientVersionState.java

+2
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@
1414
*/
1515
package software.amazon.kinesis.coordinator.migration;
1616

17+
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
1718
import software.amazon.kinesis.leases.exceptions.DependencyException;
1819

1920
/**
2021
* Interface of a state implementation for the MigrationStateMachine
2122
*/
23+
@KinesisClientInternalApi
2224
public interface MigrationClientVersionState {
2325

2426
/**

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/migration/MigrationStateMachine.java

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
*/
1515
package software.amazon.kinesis.coordinator.migration;
1616

17+
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
1718
import software.amazon.kinesis.leases.exceptions.DependencyException;
1819
import software.amazon.kinesis.leases.exceptions.InvalidStateException;
1920

@@ -28,6 +29,7 @@
2829
* 3. Instant roll-forwards - Once any issue has been mitigated, rollfowards are supported instantly
2930
* with KCL Migration tool.
3031
*/
32+
@KinesisClientInternalApi
3133
public interface MigrationStateMachine {
3234
/**
3335
* Initialize the state machine by identifying the initial state when the KCL worker comes up for the first time.

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/migration/MigrationStateMachineImpl.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ public void shutdown() {
126126
if (!stateMachineThreadPool.isShutdown()) {
127127
stateMachineThreadPool.shutdown();
128128
try {
129-
if (stateMachineThreadPool.awaitTermination(THREAD_POOL_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
129+
if (!stateMachineThreadPool.awaitTermination(THREAD_POOL_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
130130
log.info(
131131
"StateMachineThreadPool did not shutdown within {} seconds, forcefully shutting down",
132132
THREAD_POOL_SHUTDOWN_TIMEOUT_SECONDS);

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/LeaseDiscoverer.java

+2
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@
1717

1818
import java.util.List;
1919

20+
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
2021
import software.amazon.kinesis.leases.exceptions.DependencyException;
2122
import software.amazon.kinesis.leases.exceptions.InvalidStateException;
2223
import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
2324

25+
@KinesisClientInternalApi
2426
public interface LeaseDiscoverer {
2527
/**
2628
* Identifies the leases that are assigned to the current worker but are not being tracked and processed by the

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseCoordinator.java

-1
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,6 @@ public void stop() {
383383
}
384384

385385
leaseRenewalThreadpool.shutdownNow();
386-
leaseCoordinatorThreadPool.shutdownNow();
387386
leaseGracefulShutdownHandler.stop();
388387
synchronized (shutdownLock) {
389388
leaseRenewer.clearCurrentlyHeldLeases();

‎amazon-kinesis-client/src/main/java/software/amazon/kinesis/leases/dynamodb/DynamoDBLeaseSerializer.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public class DynamoDBLeaseSerializer implements LeaseSerializer {
5454
private static final String CHILD_SHARD_IDS_KEY = "childShardIds";
5555
private static final String STARTING_HASH_KEY = "startingHashKey";
5656
private static final String ENDING_HASH_KEY = "endingHashKey";
57-
private static final String THROUGHOUT_PUT_KBPS = "throughputKBps";
57+
private static final String THROUGHPUT_KBPS = "throughputKBps";
5858
private static final String CHECKPOINT_SEQUENCE_NUMBER_KEY = "checkpoint";
5959
static final String CHECKPOINT_OWNER = "checkpointOwner";
6060
static final String LEASE_OWNER_KEY = "leaseOwner";
@@ -113,7 +113,7 @@ public Map<String, AttributeValue> toDynamoRecord(final Lease lease) {
113113
}
114114

115115
if (lease.throughputKBps() != null) {
116-
result.put(THROUGHOUT_PUT_KBPS, DynamoUtils.createAttributeValue(lease.throughputKBps()));
116+
result.put(THROUGHPUT_KBPS, DynamoUtils.createAttributeValue(lease.throughputKBps()));
117117
}
118118

119119
if (lease.checkpointOwner() != null) {
@@ -155,8 +155,8 @@ public Lease fromDynamoRecord(Map<String, AttributeValue> dynamoRecord, Lease le
155155
leaseToUpdate.hashKeyRange(HashKeyRangeForLease.deserialize(startingHashKey, endingHashKey));
156156
}
157157

158-
if (DynamoUtils.safeGetDouble(dynamoRecord, THROUGHOUT_PUT_KBPS) != null) {
159-
leaseToUpdate.throughputKBps(DynamoUtils.safeGetDouble(dynamoRecord, THROUGHOUT_PUT_KBPS));
158+
if (DynamoUtils.safeGetDouble(dynamoRecord, THROUGHPUT_KBPS) != null) {
159+
leaseToUpdate.throughputKBps(DynamoUtils.safeGetDouble(dynamoRecord, THROUGHPUT_KBPS));
160160
}
161161

162162
if (DynamoUtils.safeGetString(dynamoRecord, CHECKPOINT_OWNER) != null) {
@@ -466,7 +466,7 @@ public Map<String, AttributeValueUpdate> getDynamoLeaseThroughputKbpsUpdate(Leas
466466
.value(DynamoUtils.createAttributeValue(lease.throughputKBps()))
467467
.action(AttributeAction.PUT)
468468
.build();
469-
result.put(THROUGHOUT_PUT_KBPS, avu);
469+
result.put(THROUGHPUT_KBPS, avu);
470470
return result;
471471
}
472472

0 commit comments

Comments
 (0)