Skip to content

Commit 62d0dc6

Browse files
Neeharika-Sompallitinker-michaelj
Neeharika-Sompalli
and
tinker-michaelj
authored
fix: (cherry-pick) Fix issues found during replaying events and dumping state (#12531) (#12560)
Signed-off-by: Neeharika-Sompalli <[email protected]> Signed-off-by: Michael Tinker <[email protected]> Co-authored-by: Michael Tinker <[email protected]>
1 parent 970250e commit 62d0dc6

File tree

29 files changed

+580
-273
lines changed

29 files changed

+580
-273
lines changed

‎hedera-node/hedera-app/src/main/java/com/hedera/node/app/statedumpers/StateDumper.java

+16-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import static com.hedera.node.app.ids.EntityIdService.ENTITY_ID_STATE_KEY;
2020
import static com.hedera.node.app.records.BlockRecordService.BLOCK_INFO_STATE_KEY;
2121
import static com.hedera.node.app.records.BlockRecordService.RUNNING_HASHES_STATE_KEY;
22+
import static com.hedera.node.app.service.schedule.impl.ScheduleServiceImpl.SCHEDULES_BY_EQUALITY_KEY;
23+
import static com.hedera.node.app.service.schedule.impl.ScheduleServiceImpl.SCHEDULES_BY_EXPIRY_SEC_KEY;
2224
import static com.hedera.node.app.service.schedule.impl.ScheduleServiceImpl.SCHEDULES_BY_ID_KEY;
2325
import static com.hedera.node.app.service.token.impl.TokenServiceImpl.ACCOUNTS_KEY;
2426
import static com.hedera.node.app.service.token.impl.TokenServiceImpl.NFTS_KEY;
@@ -58,8 +60,11 @@
5860
import com.hedera.hapi.node.state.congestion.CongestionLevelStarts;
5961
import com.hedera.hapi.node.state.consensus.Topic;
6062
import com.hedera.hapi.node.state.contract.Bytecode;
63+
import com.hedera.hapi.node.state.primitives.ProtoBytes;
64+
import com.hedera.hapi.node.state.primitives.ProtoLong;
6165
import com.hedera.hapi.node.state.recordcache.TransactionRecordEntry;
6266
import com.hedera.hapi.node.state.schedule.Schedule;
67+
import com.hedera.hapi.node.state.schedule.ScheduleList;
6368
import com.hedera.hapi.node.state.throttles.ThrottleUsageSnapshots;
6469
import com.hedera.hapi.node.state.token.Account;
6570
import com.hedera.hapi.node.state.token.NetworkStakingRewards;
@@ -157,10 +162,19 @@ public static void dumpModChildrenFrom(
157162
state.getChild(state.findNodeIndex(ConsensusService.NAME, ConsensusServiceImpl.TOPICS_KEY)));
158163
dumpModTopics(Paths.get(dumpLoc, SEMANTIC_TOPICS), topics, checkpoint);
159164

160-
final VirtualMap<OnDiskKey<ScheduleID>, OnDiskValue<Schedule>> scheduledTransactions =
165+
final VirtualMap<OnDiskKey<ScheduleID>, OnDiskValue<Schedule>> scheduledTransactionsByKey =
161166
requireNonNull(state.getChild(state.findNodeIndex(ScheduleService.NAME, SCHEDULES_BY_ID_KEY)));
167+
final VirtualMap<OnDiskKey<ProtoBytes>, OnDiskValue<com.hedera.hapi.node.state.schedule.ScheduleList>>
168+
scheduledTransactionsByEquality = requireNonNull(
169+
state.getChild(state.findNodeIndex(ScheduleService.NAME, SCHEDULES_BY_EQUALITY_KEY)));
170+
final VirtualMap<OnDiskKey<ProtoLong>, OnDiskValue<ScheduleList>> scheduledTransactionsByExpiry =
171+
requireNonNull(state.getChild(state.findNodeIndex(ScheduleService.NAME, SCHEDULES_BY_EXPIRY_SEC_KEY)));
162172
dumpModScheduledTransactions(
163-
Paths.get(dumpLoc, SEMANTIC_SCHEDULED_TRANSACTIONS), scheduledTransactions, checkpoint);
173+
Paths.get(dumpLoc, SEMANTIC_SCHEDULED_TRANSACTIONS),
174+
scheduledTransactionsByKey,
175+
scheduledTransactionsByEquality,
176+
scheduledTransactionsByExpiry,
177+
checkpoint);
164178

165179
final VirtualMap<OnDiskKey<TokenID>, OnDiskValue<Token>> tokenTypes =
166180
requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, TOKENS_KEY)));

‎hedera-node/hedera-app/src/main/java/com/hedera/node/app/statedumpers/scheduledtransactions/ScheduledTransactionsDumpUtils.java

+102-18
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,25 @@
1717
package com.hedera.node.app.statedumpers.scheduledtransactions;
1818

1919
import static com.hedera.node.app.service.mono.statedumpers.associations.BBMTokenAssociation.entityIdFrom;
20+
import static com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.ScheduledTransactionsDumpUtils.reportOnScheduledTransactionsByEquality;
21+
import static com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.ScheduledTransactionsDumpUtils.reportOnScheduledTransactionsByExpiry;
2022
import static com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.ScheduledTransactionsDumpUtils.reportOnScheduledTransactionsById;
2123
import static com.swirlds.common.threading.manager.AdHocThreadManager.getStaticThreadManager;
2224

2325
import com.hedera.hapi.node.base.ScheduleID;
26+
import com.hedera.hapi.node.state.primitives.ProtoBytes;
27+
import com.hedera.hapi.node.state.primitives.ProtoLong;
2428
import com.hedera.hapi.node.state.schedule.Schedule;
29+
import com.hedera.hapi.node.state.schedule.ScheduleList;
2530
import com.hedera.node.app.service.mono.legacy.core.jproto.JKey;
2631
import com.hedera.node.app.service.mono.pbj.PbjConverter;
2732
import com.hedera.node.app.service.mono.state.adapters.VirtualMapLike;
2833
import com.hedera.node.app.service.mono.state.submerkle.RichInstant;
2934
import com.hedera.node.app.service.mono.statedumpers.DumpCheckpoint;
35+
import com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.BBMScheduledEqualityValue;
36+
import com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.BBMScheduledId;
37+
import com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.BBMScheduledSecondValue;
3038
import com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.BBMScheduledTransaction;
31-
import com.hedera.node.app.service.mono.statedumpers.scheduledtransactions.BBMScheduledTransactionId;
3239
import com.hedera.node.app.service.mono.statedumpers.utils.Writer;
3340
import com.hedera.node.app.state.merkle.disk.OnDiskKey;
3441
import com.hedera.node.app.state.merkle.disk.OnDiskValue;
@@ -38,45 +45,97 @@
3845
import java.nio.file.Path;
3946
import java.security.InvalidKeyException;
4047
import java.time.Instant;
48+
import java.util.ArrayList;
4149
import java.util.HashMap;
50+
import java.util.List;
4251
import java.util.Map;
4352
import java.util.Optional;
53+
import java.util.TreeMap;
4454
import java.util.concurrent.ConcurrentLinkedQueue;
55+
import java.util.concurrent.CopyOnWriteArrayList;
4556

4657
public class ScheduledTransactionsDumpUtils {
4758

4859
public static void dumpModScheduledTransactions(
4960
@NonNull final Path path,
5061
@NonNull final VirtualMap<OnDiskKey<ScheduleID>, OnDiskValue<Schedule>> scheduledTransactions,
62+
@NonNull final VirtualMap<OnDiskKey<ProtoBytes>, OnDiskValue<ScheduleList>> byEquality,
63+
@NonNull final VirtualMap<OnDiskKey<ProtoLong>, OnDiskValue<ScheduleList>> byExpiry,
5164
@NonNull final DumpCheckpoint checkpoint) {
5265
try (@NonNull final var writer = new Writer(path)) {
53-
final var dumpableScheduledTransactions = gatherModScheduledTransactions(scheduledTransactions);
54-
reportOnScheduledTransactionsById(writer, dumpableScheduledTransactions);
66+
final var dumpableScheduledTransactionsById = gatherModScheduledTransactionsById(scheduledTransactions);
67+
reportOnScheduledTransactionsById(writer, dumpableScheduledTransactionsById);
5568
System.out.printf(
5669
"=== mod scheduled transactions report is %d bytes at checkpoint %s%n",
5770
writer.getSize(), checkpoint.name());
71+
final var byEqualityDump = gatherModScheduledTransactionsByEquality(byEquality);
72+
reportOnScheduledTransactionsByEquality(writer, byEqualityDump);
73+
// Not sure how to compare Equality Virtual map in mono and mod
74+
final var byExpiryDump = gatherModScheduledTransactionsByExpiry(byExpiry);
75+
reportOnScheduledTransactionsByExpiry(writer, byExpiryDump);
76+
System.out.printf(
77+
"=== mod scheduled transactions by expiry report is %d bytes at checkpoint %s%n",
78+
writer.getSize(), checkpoint.name());
79+
}
80+
}
81+
82+
private static List<BBMScheduledEqualityValue> gatherModScheduledTransactionsByEquality(
83+
final VirtualMap<OnDiskKey<ProtoBytes>, OnDiskValue<ScheduleList>> source) {
84+
final List<BBMScheduledEqualityValue> r = new ArrayList<>();
85+
final var scheduledTransactions = new CopyOnWriteArrayList<BBMScheduledEqualityValue>();
86+
87+
try {
88+
VirtualMapLike.from(source)
89+
.extractVirtualMapDataC(
90+
getStaticThreadManager(),
91+
p -> scheduledTransactions.add(
92+
fromMod(p.key().getKey(), p.value().getValue())),
93+
8);
94+
} catch (final InterruptedException ex) {
95+
System.err.println("*** Traversal of scheduledTransactions by equality virtual map interrupted!");
96+
Thread.currentThread().interrupt();
97+
}
98+
return r;
99+
}
100+
101+
private static Map<BBMScheduledId, BBMScheduledSecondValue> gatherModScheduledTransactionsByExpiry(
102+
final VirtualMap<OnDiskKey<ProtoLong>, OnDiskValue<ScheduleList>> source) {
103+
final var r = new HashMap<BBMScheduledId, BBMScheduledSecondValue>();
104+
final var scheduledTransactions = new ConcurrentLinkedQueue<Pair<BBMScheduledId, BBMScheduledSecondValue>>();
105+
106+
try {
107+
VirtualMapLike.from(source)
108+
.extractVirtualMapDataC(
109+
getStaticThreadManager(),
110+
p -> scheduledTransactions.add(Pair.of(
111+
new BBMScheduledId(p.key().getKey().value()),
112+
fromMod(p.key().getKey(), p.value().getValue()))),
113+
8);
114+
} catch (final InterruptedException ex) {
115+
System.err.println("*** Traversal of scheduledTransactions virtual map interrupted!");
116+
Thread.currentThread().interrupt();
58117
}
118+
while (!scheduledTransactions.isEmpty()) {
119+
final var mapping = scheduledTransactions.poll();
120+
r.put(mapping.key(), mapping.value());
121+
}
122+
return r;
59123
}
60124

61125
@NonNull
62-
private static Map<BBMScheduledTransactionId, BBMScheduledTransaction> gatherModScheduledTransactions(
126+
private static Map<BBMScheduledId, BBMScheduledTransaction> gatherModScheduledTransactionsById(
63127
VirtualMap<OnDiskKey<ScheduleID>, OnDiskValue<Schedule>> source) {
64-
final var r = new HashMap<BBMScheduledTransactionId, BBMScheduledTransaction>();
65-
final var scheduledTransactions =
66-
new ConcurrentLinkedQueue<Pair<BBMScheduledTransactionId, BBMScheduledTransaction>>();
128+
final var r = new HashMap<BBMScheduledId, BBMScheduledTransaction>();
129+
final var scheduledTransactions = new ConcurrentLinkedQueue<Pair<BBMScheduledId, BBMScheduledTransaction>>();
67130

68131
try {
69132
VirtualMapLike.from(source)
70133
.extractVirtualMapDataC(
71134
getStaticThreadManager(),
72135
p -> {
73-
try {
74-
scheduledTransactions.add(Pair.of(
75-
fromMod(p.key().getKey()),
76-
fromMod(p.value().getValue())));
77-
} catch (InvalidKeyException e) {
78-
throw new RuntimeException(e);
79-
}
136+
scheduledTransactions.add(Pair.of(
137+
fromMod(p.key().getKey()),
138+
fromMod(p.value().getValue())));
80139
},
81140
8);
82141
} catch (final InterruptedException ex) {
@@ -90,10 +149,35 @@ private static Map<BBMScheduledTransactionId, BBMScheduledTransaction> gatherMod
90149
return r;
91150
}
92151

93-
static BBMScheduledTransaction fromMod(@NonNull final Schedule value) throws InvalidKeyException {
152+
static BBMScheduledSecondValue fromMod(final ProtoLong expiry, @NonNull final ScheduleList value) {
153+
final var newMap = new TreeMap<Instant, List<Long>>();
154+
final var longsList = value.schedules().stream()
155+
.map(a -> a.scheduleId().scheduleNum())
156+
.toList();
157+
newMap.put(Instant.ofEpochSecond(expiry.value()), longsList);
158+
return new BBMScheduledSecondValue(newMap);
159+
}
160+
161+
static BBMScheduledEqualityValue fromMod(final ProtoBytes hash, @NonNull final ScheduleList value) {
162+
final var newMap = new TreeMap<String, Long>();
163+
final var longsList = value.schedules().stream()
164+
.map(a -> a.scheduleId().scheduleNum())
165+
.toList();
166+
newMap.put(hash.value().asUtf8String(), longsList.get(0));
167+
return new BBMScheduledEqualityValue(newMap);
168+
}
169+
170+
static BBMScheduledTransaction fromMod(@NonNull final Schedule value) {
171+
Optional<JKey> adminKey;
172+
try {
173+
adminKey = value.adminKey() != null ? Optional.of(JKey.mapKey(value.adminKey())) : Optional.empty();
174+
} catch (InvalidKeyException e) {
175+
adminKey = Optional.empty();
176+
}
177+
94178
return new BBMScheduledTransaction(
95179
value.scheduleId().scheduleNum(),
96-
value.adminKey() != null ? Optional.of(JKey.mapKey(value.adminKey())) : Optional.empty(),
180+
adminKey,
97181
value.memo(),
98182
value.deleted(),
99183
value.executed(),
@@ -116,8 +200,8 @@ static BBMScheduledTransaction fromMod(@NonNull final Schedule value) throws Inv
116200
value.signatories().stream().map(k -> toPrimitiveKey(k)).toList());
117201
}
118202

119-
static BBMScheduledTransactionId fromMod(@NonNull final ScheduleID scheduleID) {
120-
return new BBMScheduledTransactionId(scheduleID.scheduleNum());
203+
static BBMScheduledId fromMod(@NonNull final ScheduleID scheduleID) {
204+
return new BBMScheduledId(scheduleID.scheduleNum());
121205
}
122206

123207
static byte[] toPrimitiveKey(com.hedera.hapi.node.base.Key key) {

‎hedera-node/hedera-app/src/main/java/com/hedera/node/app/statedumpers/singleton/PayerRecordsDumpUtils.java

+2
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ public static void dumpModTxnRecordQueue(
3838
@NonNull final QueueNode<TransactionRecordEntry> queue,
3939
@NonNull final DumpCheckpoint checkpoint) {
4040
var transactionRecords = gatherTxnRecordsFromMod(queue);
41+
System.out.println("=== Dumping payer records ===");
42+
System.out.println(transactionRecords.size() + " records found");
4143
int reportSize;
4244
try (@NonNull final var writer = new Writer(path)) {
4345
reportOnTxnRecords(writer, transactionRecords);

‎hedera-node/hedera-app/src/main/java/com/hedera/node/app/statedumpers/topics/TopicDumpUtils.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ static BBMTopic fromMod(@NonNull final com.hedera.hapi.node.state.consensus.Topi
9898
topic.runningHash().toByteArray(),
9999
topic.sequenceNumber(),
100100
topic.autoRenewPeriod(),
101-
EntityId.fromNum(topic.autoRenewAccountId().accountNum()));
101+
topic.hasAutoRenewAccountId()
102+
? EntityId.fromNum(topic.autoRenewAccountId().accountNum())
103+
: null);
102104
}
103105
}

‎hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/state/migration/AccountStateTranslator.java

+22-15
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,16 @@ public static Account accountFromMerkle(
108108
.stakedAccountId(stakedAccountId)
109109
.stakedNodeId(stakedNodeId)
110110
.firstContractStorageKey(firstContractStorageKey)
111-
.headNftId(NftID.newBuilder()
112-
.tokenId(TokenID.newBuilder()
113-
.tokenNum(account.getHeadNftTokenNum())
114-
.realmNum(StaticProperties.getRealm())
115-
.shardNum(StaticProperties.getShard()))
116-
.serialNumber(account.getHeadNftSerialNum()))
111+
.headNftId(
112+
(account.getHeadNftTokenNum() != 0 && account.getHeadNftSerialNum() != 0)
113+
? (NftID.newBuilder()
114+
.tokenId(TokenID.newBuilder()
115+
.tokenNum(account.getHeadNftTokenNum())
116+
.realmNum(StaticProperties.getRealm())
117+
.shardNum(StaticProperties.getShard()))
118+
.serialNumber(account.getHeadNftSerialNum())
119+
.build())
120+
: null)
117121
.autoRenewAccountId(AccountID.newBuilder()
118122
.accountNum(Optional.ofNullable(account.getAutoRenewAccount())
119123
.map(EntityId::num)
@@ -173,12 +177,16 @@ public static Account accountFromOnDiskAccount(@NonNull final OnDiskAccount acco
173177
.stakedToMe(account.getStakedToMe())
174178
.stakePeriodStart(account.getStakePeriodStart())
175179
.firstContractStorageKey(firstContractStorageKey)
176-
.headNftId(NftID.newBuilder()
177-
.tokenId(TokenID.newBuilder()
178-
.tokenNum(account.getHeadNftTokenNum())
179-
.realmNum(StaticProperties.getRealm())
180-
.shardNum(StaticProperties.getShard()))
181-
.serialNumber(account.getHeadNftSerialNum()))
180+
.headNftId(
181+
(account.getHeadNftTokenNum() != 0 && account.getHeadNftSerialNum() != 0)
182+
? NftID.newBuilder()
183+
.tokenId(TokenID.newBuilder()
184+
.tokenNum(account.getHeadNftTokenNum())
185+
.realmNum(StaticProperties.getRealm())
186+
.shardNum(StaticProperties.getShard()))
187+
.serialNumber(account.getHeadNftSerialNum())
188+
.build()
189+
: null)
182190
.autoRenewAccountId(
183191
account.hasAutoRenewAccount()
184192
? AccountID.newBuilder()
@@ -345,9 +353,8 @@ public static com.hedera.node.app.service.mono.state.merkle.MerkleAccount merkle
345353
merkleAccount.setAutoRenewAccount(new EntityId(
346354
0, 0, account.autoRenewAccountIdOrElse(AccountID.DEFAULT).accountNum()));
347355
merkleAccount.setExpiredAndPendingRemoval(account.expiredAndPendingRemoval());
348-
merkleAccount.setHeadNftId(account.headNftIdOrElse(NftID.DEFAULT)
349-
.tokenIdOrElse(TokenID.DEFAULT)
350-
.tokenNum());
356+
merkleAccount.setHeadNftId(
357+
account.headNftIdOrElse(null).tokenIdOrElse(TokenID.DEFAULT).tokenNum());
351358
merkleAccount.setHeadNftSerialNum(account.headNftSerialNumber());
352359
if (account.firstContractStorageKey() != null
353360
&& account.firstContractStorageKey().length() > 0)

‎hedera-node/hedera-mono-service/src/main/java/com/hedera/node/app/service/mono/statedumpers/scheduledtransactions/BBMScheduledEqualityValue.java

+26-4
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,35 @@
1818

1919
import com.hedera.node.app.service.mono.state.virtual.schedule.ScheduleEqualityVirtualValue;
2020
import edu.umd.cs.findbugs.annotations.NonNull;
21+
import java.util.Map;
2122
import java.util.SortedMap;
2223

23-
@SuppressWarnings("java:S6218") // "Equals/hashcode methods should be overridden in records containing array fields"
24-
record BBMScheduledEqualityValue(long number, SortedMap<String, Long> ids) {
24+
@SuppressWarnings("java:S6218")
25+
public // "Equals/hashcode methods should be overridden in records containing array fields"
26+
record BBMScheduledEqualityValue(SortedMap<String, Long> ids) {
2527

2628
static BBMScheduledEqualityValue fromMono(@NonNull final ScheduleEqualityVirtualValue scheduleVirtualValue) {
27-
return new BBMScheduledEqualityValue(
28-
scheduleVirtualValue.getKey().getKeyAsLong(), scheduleVirtualValue.getIds());
29+
return new BBMScheduledEqualityValue(scheduleVirtualValue.getIds());
30+
}
31+
32+
@Override
33+
public String toString() {
34+
return "BBMScheduledEqualityValue{" + "ids=" + idsToString(ids) + '}';
35+
}
36+
37+
public static String idsToString(SortedMap<String, Long> ids) {
38+
StringBuilder sb = new StringBuilder();
39+
sb.append("{");
40+
for (Map.Entry<String, Long> entry : ids.entrySet()) {
41+
final var key = entry.getKey();
42+
final var value = entry.getValue();
43+
sb.append(key).append(": ").append(value).append(", ");
44+
}
45+
if (!ids.isEmpty()) {
46+
// Remove the trailing comma and space
47+
sb.setLength(sb.length() - 2);
48+
}
49+
sb.append("}");
50+
return sb.toString();
2951
}
3052
}

0 commit comments

Comments
 (0)