Skip to content

Commit 6f9c522

Browse files
ltaragiltaragi
committed
Separate out changes to add primary store size in status
Signed-off-by: Lakshya Taragi <[email protected]>
1 parent 440e1e4 commit 6f9c522

24 files changed

+50
-201
lines changed

‎server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ private void assertShallowV2SnapshotStatus(SnapshotStatus snapshotStatus, boolea
269269
if (hasIndexFilter) {
270270
assertEquals(0, snapshotStatus.getStats().getTotalSize());
271271
} else {
272-
assertTrue(snapshotStatus.getStats().getTotalSize() > 0);
272+
// TODO: after adding primary store size at the snapshot level, total size here should be > 0
273273
}
274274
// assert that total and incremental values of file count and size_in_bytes are 0 at index and shard levels
275275
assertEquals(0, snapshotStatus.getStats().getTotalFileCount());

‎server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java

Lines changed: 7 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232

3333
package org.opensearch.action.admin.cluster.snapshots.status;
3434

35-
import org.opensearch.Version;
3635
import org.opensearch.cluster.SnapshotsInProgress;
3736
import org.opensearch.cluster.SnapshotsInProgress.State;
3837
import org.opensearch.common.Nullable;
@@ -87,8 +86,6 @@ public class SnapshotStatus implements ToXContentObject, Writeable {
8786

8887
private SnapshotStats stats;
8988

90-
private final long initialTotalSizeInBytes;
91-
9289
@Nullable
9390
private final Boolean includeGlobalState;
9491

@@ -99,12 +96,7 @@ public class SnapshotStatus implements ToXContentObject, Writeable {
9996
includeGlobalState = in.readOptionalBoolean();
10097
final long startTime = in.readLong();
10198
final long time = in.readLong();
102-
if (in.getVersion().onOrAfter(Version.V_2_17_0)) {
103-
initialTotalSizeInBytes = in.readOptionalLong();
104-
} else {
105-
initialTotalSizeInBytes = 0L;
106-
}
107-
updateShardStats(startTime, time, initialTotalSizeInBytes);
99+
updateShardStats(startTime, time);
108100
}
109101

110102
SnapshotStatus(
@@ -113,18 +105,15 @@ public class SnapshotStatus implements ToXContentObject, Writeable {
113105
List<SnapshotIndexShardStatus> shards,
114106
Boolean includeGlobalState,
115107
long startTime,
116-
long time,
117-
long initialTotalSizeInBytes
108+
long time
118109
) {
119110
this.snapshot = Objects.requireNonNull(snapshot);
120111
this.state = Objects.requireNonNull(state);
121112
this.shards = Objects.requireNonNull(shards);
122113
this.includeGlobalState = includeGlobalState;
123114
shardsStats = new SnapshotShardsStats(shards);
124115
assert time >= 0 : "time must be >= 0 but received [" + time + "]";
125-
this.initialTotalSizeInBytes = initialTotalSizeInBytes;
126-
assert initialTotalSizeInBytes >= 0 : "initialTotalSizeInBytes must be >= 0 but received [" + initialTotalSizeInBytes + "]";
127-
updateShardStats(startTime, time, initialTotalSizeInBytes);
116+
updateShardStats(startTime, time);
128117
}
129118

130119
private SnapshotStatus(
@@ -134,8 +123,7 @@ private SnapshotStatus(
134123
Map<String, SnapshotIndexStatus> indicesStatus,
135124
SnapshotShardsStats shardsStats,
136125
SnapshotStats stats,
137-
Boolean includeGlobalState,
138-
long initialTotalSizeInBytes
126+
Boolean includeGlobalState
139127
) {
140128
this.snapshot = snapshot;
141129
this.state = state;
@@ -144,7 +132,6 @@ private SnapshotStatus(
144132
this.shardsStats = shardsStats;
145133
this.stats = stats;
146134
this.includeGlobalState = includeGlobalState;
147-
this.initialTotalSizeInBytes = initialTotalSizeInBytes;
148135
}
149136

150137
/**
@@ -217,9 +204,6 @@ public void writeTo(StreamOutput out) throws IOException {
217204
out.writeOptionalBoolean(includeGlobalState);
218205
out.writeLong(stats.getStartTime());
219206
out.writeLong(stats.getTime());
220-
if (out.getVersion().onOrAfter(Version.V_2_17_0)) {
221-
out.writeOptionalLong(initialTotalSizeInBytes);
222-
}
223207
}
224208

225209
@Override
@@ -240,7 +224,6 @@ public SnapshotStats getStats() {
240224
private static final String STATE = "state";
241225
private static final String INDICES = "indices";
242226
private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
243-
private static final String INITIAL_TOTAL_SIZE_IN_BYTES = "initial_total_size_in_bytes";
244227

245228
@Override
246229
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
@@ -252,9 +235,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
252235
if (includeGlobalState != null) {
253236
builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState);
254237
}
255-
if (initialTotalSizeInBytes != 0) {
256-
builder.field(INITIAL_TOTAL_SIZE_IN_BYTES, initialTotalSizeInBytes);
257-
}
258238
builder.field(SnapshotShardsStats.Fields.SHARDS_STATS, shardsStats, params);
259239
builder.field(SnapshotStats.Fields.STATS, stats, params);
260240
builder.startObject(INDICES);
@@ -276,7 +256,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
276256
String uuid = (String) parsedObjects[i++];
277257
String rawState = (String) parsedObjects[i++];
278258
Boolean includeGlobalState = (Boolean) parsedObjects[i++];
279-
Long initialTotalSizeInBytes = (Long) parsedObjects[i++];
280259
SnapshotStats stats = ((SnapshotStats) parsedObjects[i++]);
281260
SnapshotShardsStats shardsStats = ((SnapshotShardsStats) parsedObjects[i++]);
282261
@SuppressWarnings("unchecked")
@@ -297,16 +276,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
297276
shards.addAll(index.getShards().values());
298277
}
299278
}
300-
return new SnapshotStatus(
301-
snapshot,
302-
state,
303-
shards,
304-
indicesStatus,
305-
shardsStats,
306-
stats,
307-
includeGlobalState,
308-
initialTotalSizeInBytes
309-
);
279+
return new SnapshotStatus(snapshot, state, shards, indicesStatus, shardsStats, stats, includeGlobalState);
310280
}
311281
);
312282
static {
@@ -315,7 +285,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
315285
PARSER.declareString(constructorArg(), new ParseField(UUID));
316286
PARSER.declareString(constructorArg(), new ParseField(STATE));
317287
PARSER.declareBoolean(optionalConstructorArg(), new ParseField(INCLUDE_GLOBAL_STATE));
318-
PARSER.declareLong(optionalConstructorArg(), new ParseField(INITIAL_TOTAL_SIZE_IN_BYTES));
319288
PARSER.declareField(
320289
constructorArg(),
321290
SnapshotStats::fromXContent,
@@ -330,8 +299,8 @@ public static SnapshotStatus fromXContent(XContentParser parser) throws IOExcept
330299
return PARSER.parse(parser, null);
331300
}
332301

333-
private void updateShardStats(long startTime, long time, long initialTotalSizeInBytes) {
334-
stats = new SnapshotStats(startTime, time, 0, 0, 0, 0, initialTotalSizeInBytes, 0);
302+
private void updateShardStats(long startTime, long time) {
303+
stats = new SnapshotStats(startTime, time, 0, 0, 0, 0, 0, 0);
335304
shardsStats = new SnapshotShardsStats(shards);
336305
for (SnapshotIndexShardStatus shard : shards) {
337306
// BWC: only update timestamps when we did not get a start time from an old node

‎server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -297,8 +297,7 @@ private void buildResponse(
297297
Collections.unmodifiableList(shardStatusBuilder),
298298
entry.includeGlobalState(),
299299
entry.startTime(),
300-
Math.max(threadPool.absoluteTimeInMillis() - entry.startTime(), 0L),
301-
0L
300+
Math.max(threadPool.absoluteTimeInMillis() - entry.startTime(), 0L)
302301
)
303302
);
304303
}
@@ -345,7 +344,7 @@ private void loadRepositoryData(
345344
boolean isShallowV2Snapshot = snapshotInfo.getPinnedTimestamp() > 0;
346345
long initialSnapshotTotalSize = 0;
347346
if (isShallowV2Snapshot && request.indices().length == 0) {
348-
initialSnapshotTotalSize = snapshotInfo.getSnapshotSizeInBytes();
347+
// TODO: add primary store size in bytes at the snapshot level
349348
}
350349

351350
for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardStatus : shardStatuses.entrySet()) {
@@ -378,8 +377,7 @@ private void loadRepositoryData(
378377
snapshotInfo.includeGlobalState(),
379378
startTime,
380379
// Use current time to calculate overall runtime for in-progress snapshots that have endTime == 0
381-
(endTime == 0 ? threadPool.absoluteTimeInMillis() : endTime) - startTime,
382-
initialSnapshotTotalSize
380+
(endTime == 0 ? threadPool.absoluteTimeInMillis() : endTime) - startTime
383381
)
384382
);
385383
}

‎server/src/main/java/org/opensearch/cluster/ClusterInfo.java

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,11 @@ public class ClusterInfo implements ToXContentFragment, Writeable {
6969
final Map<ShardRouting, String> routingToDataPath;
7070
final Map<NodeAndPath, ReservedSpace> reservedSpace;
7171
final Map<String, FileCacheStats> nodeFileCacheStats;
72-
final long primaryStoreSize;
73-
7472
private long avgTotalBytes;
7573
private long avgFreeByte;
7674

7775
protected ClusterInfo() {
78-
this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), 0L);
76+
this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
7977
}
8078

8179
/**
@@ -86,7 +84,6 @@ protected ClusterInfo() {
8684
* @param shardSizes a shardkey to size in bytes mapping per shard.
8785
* @param routingToDataPath the shard routing to datapath mapping
8886
* @param reservedSpace reserved space per shard broken down by node and data path
89-
* @param primaryStoreSize total size in bytes for all the primary shards
9087
* @see #shardIdentifierFromRouting
9188
*/
9289
public ClusterInfo(
@@ -95,16 +92,14 @@ public ClusterInfo(
9592
final Map<String, Long> shardSizes,
9693
final Map<ShardRouting, String> routingToDataPath,
9794
final Map<NodeAndPath, ReservedSpace> reservedSpace,
98-
final Map<String, FileCacheStats> nodeFileCacheStats,
99-
final long primaryStoreSize
95+
final Map<String, FileCacheStats> nodeFileCacheStats
10096
) {
10197
this.leastAvailableSpaceUsage = leastAvailableSpaceUsage;
10298
this.shardSizes = shardSizes;
10399
this.mostAvailableSpaceUsage = mostAvailableSpaceUsage;
104100
this.routingToDataPath = routingToDataPath;
105101
this.reservedSpace = reservedSpace;
106102
this.nodeFileCacheStats = nodeFileCacheStats;
107-
this.primaryStoreSize = primaryStoreSize;
108103
calculateAvgFreeAndTotalBytes(mostAvailableSpaceUsage);
109104
}
110105

@@ -115,6 +110,7 @@ public ClusterInfo(StreamInput in) throws IOException {
115110
Map<ShardRouting, String> routingMap = in.readMap(ShardRouting::new, StreamInput::readString);
116111
Map<NodeAndPath, ReservedSpace> reservedSpaceMap;
117112
reservedSpaceMap = in.readMap(NodeAndPath::new, ReservedSpace::new);
113+
118114
this.leastAvailableSpaceUsage = Collections.unmodifiableMap(leastMap);
119115
this.mostAvailableSpaceUsage = Collections.unmodifiableMap(mostMap);
120116
this.shardSizes = Collections.unmodifiableMap(sizeMap);
@@ -125,11 +121,6 @@ public ClusterInfo(StreamInput in) throws IOException {
125121
} else {
126122
this.nodeFileCacheStats = Map.of();
127123
}
128-
if (in.getVersion().onOrAfter(Version.V_2_17_0)) {
129-
this.primaryStoreSize = in.readOptionalLong();
130-
} else {
131-
this.primaryStoreSize = 0L;
132-
}
133124

134125
calculateAvgFreeAndTotalBytes(mostAvailableSpaceUsage);
135126
}
@@ -175,9 +166,6 @@ public void writeTo(StreamOutput out) throws IOException {
175166
if (out.getVersion().onOrAfter(Version.V_2_10_0)) {
176167
out.writeMap(this.nodeFileCacheStats, StreamOutput::writeString, (o, v) -> v.writeTo(o));
177168
}
178-
if (out.getVersion().onOrAfter(Version.V_2_17_0)) {
179-
out.writeOptionalLong(this.primaryStoreSize);
180-
}
181169
}
182170

183171
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
@@ -232,7 +220,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
232220
}
233221
}
234222
builder.endArray(); // end "reserved_sizes"
235-
builder.field("primary_store_size", this.primaryStoreSize);
236223
return builder;
237224
}
238225

@@ -259,13 +246,6 @@ public Map<String, FileCacheStats> getNodeFileCacheStats() {
259246
return Collections.unmodifiableMap(this.nodeFileCacheStats);
260247
}
261248

262-
/**
263-
* Returns the total size in bytes for all the primary shards
264-
*/
265-
public long getPrimaryStoreSize() {
266-
return primaryStoreSize;
267-
}
268-
269249
/**
270250
* Returns the shard size for the given shard routing or <code>null</code> it that metric is not available.
271251
*/

‎server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,6 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt
115115
private volatile Map<String, FileCacheStats> nodeFileCacheStats;
116116
private volatile IndicesStatsSummary indicesStatsSummary;
117117
// null if this node is not currently the cluster-manager
118-
119-
private volatile long primaryStoreSize;
120-
121118
private final AtomicReference<RefreshAndRescheduleRunnable> refreshAndRescheduleRunnable = new AtomicReference<>();
122119
private volatile boolean enabled;
123120
private volatile TimeValue fetchTimeout;
@@ -130,7 +127,6 @@ public InternalClusterInfoService(Settings settings, ClusterService clusterServi
130127
this.mostAvailableSpaceUsages = Map.of();
131128
this.nodeFileCacheStats = Map.of();
132129
this.indicesStatsSummary = IndicesStatsSummary.EMPTY;
133-
this.primaryStoreSize = 0L;
134130
this.threadPool = threadPool;
135131
this.client = client;
136132
this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings);
@@ -217,8 +213,7 @@ public ClusterInfo getClusterInfo() {
217213
indicesStatsSummary.shardSizes,
218214
indicesStatsSummary.shardRoutingToDataPath,
219215
indicesStatsSummary.reservedSpace,
220-
nodeFileCacheStats,
221-
primaryStoreSize
216+
nodeFileCacheStats
222217
);
223218
}
224219

@@ -310,13 +305,8 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) {
310305
final Map<String, Long> shardSizeByIdentifierBuilder = new HashMap<>();
311306
final Map<ShardRouting, String> dataPathByShardRoutingBuilder = new HashMap<>();
312307
final Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace.Builder> reservedSpaceBuilders = new HashMap<>();
313-
primaryStoreSize = buildShardLevelInfo(
314-
logger,
315-
stats,
316-
shardSizeByIdentifierBuilder,
317-
dataPathByShardRoutingBuilder,
318-
reservedSpaceBuilders
319-
);
308+
buildShardLevelInfo(logger, stats, shardSizeByIdentifierBuilder, dataPathByShardRoutingBuilder, reservedSpaceBuilders);
309+
320310
final Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> rsrvdSpace = new HashMap<>();
321311
reservedSpaceBuilders.forEach((nodeAndPath, builder) -> rsrvdSpace.put(nodeAndPath, builder.build()));
322312

@@ -376,14 +366,13 @@ public void addListener(Consumer<ClusterInfo> clusterInfoConsumer) {
376366
listeners.add(clusterInfoConsumer);
377367
}
378368

379-
static long buildShardLevelInfo(
369+
static void buildShardLevelInfo(
380370
Logger logger,
381371
ShardStats[] stats,
382372
final Map<String, Long> shardSizes,
383373
final Map<ShardRouting, String> newShardRoutingToDataPath,
384374
final Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace.Builder> reservedSpaceByShard
385375
) {
386-
long currentPrimaryStoreSize = 0L;
387376
for (ShardStats s : stats) {
388377
final ShardRouting shardRouting = s.getShardRouting();
389378
newShardRoutingToDataPath.put(shardRouting, s.getDataPath());
@@ -393,9 +382,6 @@ static long buildShardLevelInfo(
393382
continue;
394383
}
395384
final long size = storeStats.sizeInBytes();
396-
if (shardRouting.primary()) {
397-
currentPrimaryStoreSize += size;
398-
}
399385
final long reserved = storeStats.getReservedSize().getBytes();
400386

401387
final String shardIdentifier = ClusterInfo.shardIdentifierFromRouting(shardRouting);
@@ -410,7 +396,6 @@ static long buildShardLevelInfo(
410396
reservedSpaceBuilder.add(shardRouting.shardId(), reserved);
411397
}
412398
}
413-
return currentPrimaryStoreSize;
414399
}
415400

416401
static void fillDiskUsagePerNode(

‎server/src/main/java/org/opensearch/node/Node.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1187,7 +1187,6 @@ protected Node(
11871187
clusterModule.getIndexNameExpressionResolver(),
11881188
repositoryService,
11891189
transportService,
1190-
clusterInfoService,
11911190
actionModule.getActionFilters(),
11921191
remoteStorePinnedTimestampService
11931192
);

0 commit comments

Comments
 (0)