Skip to content

[Enhancement] modify histogram_statistic cache expire-update logical #57238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 24, 2025
Merged

[Enhancement] modify histogram_statistic cache expire-update logical #57238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CachedStatisticStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,36 @@ public void refreshColumnStatistics(Table table, List<String> columns, boolean i
}
}

@Override
public void refreshHistogramStatistics(Table table, List<String> columns, boolean isSync) {
Preconditions.checkState(table != null);

if (StatisticUtils.statisticTableBlackListCheck(table.getId()) ||
!StatisticUtils.checkStatisticTableStateNormal()) {
return;
}

List<ColumnStatsCacheKey> cacheKeys = new ArrayList<>();
long tableId = table.getId();
for (String column : columns) {
cacheKeys.add(new ColumnStatsCacheKey(tableId, column));
}

try {
ColumnHistogramStatsCacheLoader loader = new ColumnHistogramStatsCacheLoader();
CompletableFuture<Map<ColumnStatsCacheKey, Optional<Histogram>>> future =
loader.asyncLoadAll(cacheKeys, statsCacheRefresherExecutor);
if (isSync) {
Map<ColumnStatsCacheKey, Optional<Histogram>> result = future.get();
histogramCache.synchronous().putAll(result);
} else {
future.whenComplete((res, e) -> histogramCache.synchronous().putAll(res));
}
} catch (Exception e) {
LOG.warn("Failed to refresh histogram", e);
}
}

@Override
public List<ConnectorTableColumnStats> getConnectorTableStatistics(Table table, List<String> columns) {
Preconditions.checkState(table != null);
Expand Down
9 changes: 4 additions & 5 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ default void refreshColumnStatistics(Table table, List<String> columns, boolean
default void refreshMultiColumnStatistics(Long tableId) {
}

default void refreshHistogramStatistics(Table table, List<String> columns, boolean isSync) {
}

/**
* Overwrite the statistics of `targetPartition` with `sourcePartition`
*/
Expand Down Expand Up @@ -74,11 +77,7 @@ default List<ConnectorTableColumnStats> getConnectorTableStatisticsSync(Table ta
default Map<String, Histogram> getHistogramStatistics(Table table, List<String> columns) {
return Maps.newHashMap();
}

default Map<String, Histogram> getHistogramStatisticsSync(Table table, List<String> columns) {
return getHistogramStatistics(table, columns);
}


default Map<String, Histogram> getConnectorHistogramStatistics(Table table, List<String> columns) {
return Maps.newHashMap();
}
Expand Down
7 changes: 1 addition & 6 deletions fe/fe-core/src/main/java/com/starrocks/statistic/AnalyzeMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,12 +381,7 @@ public void refreshHistogramStatisticsCache(Long dbId, Long tableId, List<String
return;
}

GlobalStateMgr.getCurrentState().getStatisticStorage().expireHistogramStatistics(table.getId(), columns);
if (async) {
GlobalStateMgr.getCurrentState().getStatisticStorage().getHistogramStatistics(table, columns);
} else {
GlobalStateMgr.getCurrentState().getStatisticStorage().getHistogramStatisticsSync(table, columns);
}
GlobalStateMgr.getCurrentState().getStatisticStorage().refreshHistogramStatistics(table, columns, !async);
}

public void replayRemoveHistogramStatsMeta(HistogramStatsMeta histogramStatsMeta) {
Expand Down
Loading