Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Refactor] refactor PartitionInfo property access (backport #51855) #51875

Merged
merged 2 commits into from
Oct 14, 2024
+53 −79
Merged

[Refactor] refactor PartitionInfo property access (backport #51855) #51875

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f35b5e9
[Refactor] refactor PartitionInfo property access (#51855)
kevincai Oct 14, 2024
49a852b
fix conflict
kevincai Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 15 additions & 25 deletions fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,15 +308,15 @@ private String singleListPartitionSql(OlapTable table, List<Long> partitionIds,
StringBuilder sb = new StringBuilder();
this.idToLiteralExprValues.forEach((partitionId, values) -> {
if (partitionIds.contains(partitionId)) {
Short partitionReplicaNum = table.getPartitionInfo().idToReplicationNum.get(partitionId);
short partitionReplicaNum = table.getPartitionInfo().getReplicationNum(partitionId);
Optional.ofNullable(table.getPartition(partitionId)).ifPresent(partition -> {
String partitionName = partition.getName();
sb.append(" PARTITION ")
.append(partitionName)
.append(" VALUES IN ")
.append(this.valuesToString(values));

if (partitionReplicaNum != null && partitionReplicaNum != tableReplicationNum) {
if (partitionReplicaNum != tableReplicationNum) {
sb.append(" (").append("\"" + PROPERTIES_REPLICATION_NUM + "\" = \"").append(partitionReplicaNum)
.append("\")");
}
Expand All @@ -336,15 +336,15 @@ private String multiListPartitionSql(OlapTable table, List<Long> partitionIds, s
StringBuilder sb = new StringBuilder();
this.idToMultiLiteralExprValues.forEach((partitionId, multiValues) -> {
if (partitionIds.contains(partitionId)) {
Short partitionReplicaNum = table.getPartitionInfo().idToReplicationNum.get(partitionId);
short partitionReplicaNum = table.getPartitionInfo().getReplicationNum(partitionId);
Optional.ofNullable(table.getPartition(partitionId)).ifPresent(partition -> {
String partitionName = partition.getName();
sb.append(" PARTITION ")
.append(partitionName)
.append(" VALUES IN ")
.append(this.multiValuesToString(multiValues));

if (partitionReplicaNum != null && partitionReplicaNum != tableReplicationNum) {
if (partitionReplicaNum != tableReplicationNum) {
sb.append(" (").append("\"" + PROPERTIES_REPLICATION_NUM + "\" = \"").append(partitionReplicaNum)
.append("\")");
}
Expand Down Expand Up @@ -396,9 +396,6 @@ public void handleNewListPartitionDescs(List<Pair<Partition, PartitionDesc>> par
PartitionDesc partitionDesc = entry.second;
Preconditions.checkArgument(partitionDesc instanceof SinglePartitionDesc);
Preconditions.checkArgument(((SinglePartitionDesc) partitionDesc).isAnalyzed());
this.idToDataProperty.put(partitionId, partitionDesc.getPartitionDataProperty());
this.idToReplicationNum.put(partitionId, partitionDesc.getReplicationNum());
this.idToInMemory.put(partitionId, partitionDesc.isInMemory());
if (partitionDesc instanceof MultiItemListPartitionDesc) {
MultiItemListPartitionDesc multiItemListPartitionDesc =
(MultiItemListPartitionDesc) partitionDesc;
Expand All @@ -414,7 +411,9 @@ public void handleNewListPartitionDescs(List<Pair<Partition, PartitionDesc>> par
"add list partition only support single item or multi item list partition now");
}
this.idToIsTempPartition.put(partitionId, isTempPartition);
this.idToStorageCacheInfo.put(partitionId, partitionDesc.getDataCacheInfo());
super.addPartition(partitionId, partitionDesc.getPartitionDataProperty(),
partitionDesc.getReplicationNum(), partitionDesc.isInMemory(),
partitionDesc.getDataCacheInfo());
}
}
} catch (Exception e) {
Expand All @@ -426,11 +425,10 @@ public void unprotectHandleNewPartitionDesc(ListPartitionPersistInfo partitionPe
throws AnalysisException {
Partition partition = partitionPersistInfo.getPartition();
long partitionId = partition.getId();
this.idToDataProperty.put(partitionId, partitionPersistInfo.getDataProperty());
this.idToReplicationNum.put(partitionId, partitionPersistInfo.getReplicationNum());
this.idToInMemory.put(partitionId, partitionPersistInfo.isInMemory());
this.idToIsTempPartition.put(partitionId, partitionPersistInfo.isTempPartition());
this.idToStorageCacheInfo.put(partitionId, partitionPersistInfo.getDataCacheInfo());
super.addPartition(partitionId, partitionPersistInfo.getDataProperty(),
partitionPersistInfo.getReplicationNum(), partitionPersistInfo.isInMemory(),
partitionPersistInfo.getDataCacheInfo());

List<List<String>> multiValues = partitionPersistInfo.getMultiValues();
if (multiValues != null && multiValues.size() > 0) {
Expand Down Expand Up @@ -465,15 +463,14 @@ public void addPartition(long partitionId, DataProperty dataProperty, short repl
DataCacheInfo dataCacheInfo, List<String> values,
List<List<String>> multiValues) throws AnalysisException {
super.addPartition(partitionId, dataProperty, replicationNum, isInMemory, dataCacheInfo);
if (multiValues != null && multiValues.size() > 0) {
if (multiValues != null && !multiValues.isEmpty()) {
this.idToMultiValues.put(partitionId, multiValues);
this.setMultiLiteralExprValues(partitionId, multiValues);
}
if (values != null && values.size() > 0) {
if (values != null && !values.isEmpty()) {
this.idToValues.put(partitionId, values);
this.setLiteralExprValues(partitionId, values);
}
this.idToStorageCacheInfo.put(partitionId, dataCacheInfo);
idToIsTempPartition.put(partitionId, false);
}

Expand All @@ -486,11 +483,8 @@ public void createAutomaticShadowPartition(long partitionId, String replicateNum
idToValues.put(partitionId, Collections.emptyList());
idToLiteralExprValues.put(partitionId, Collections.emptyList());
}

idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD));
idToReplicationNum.put(partitionId, Short.valueOf(replicateNum));
idToInMemory.put(partitionId, false);
idToStorageCacheInfo.put(partitionId, new DataCacheInfo(true, false));
super.addPartition(partitionId, new DataProperty(TStorageMedium.HDD), Short.valueOf(replicateNum), false,
new DataCacheInfo(true, false));
}

public static int compareByValue(List<List<String>> left, List<List<String>> right) {
Expand All @@ -507,10 +501,6 @@ public static int compareByValue(List<List<String>> left, List<List<String>> rig
return 0;
}

public void setStorageCacheInfo(long partitionId, DataCacheInfo dataCacheInfo) {
idToStorageCacheInfo.put(partitionId, dataCacheInfo);
}

@Override
public Object clone() {
ListPartitionInfo info = (ListPartitionInfo) super.clone();
Expand All @@ -523,4 +513,4 @@ public Object clone() {
info.automaticPartition = this.automaticPartition;
return info;
}
}
}
31 changes: 17 additions & 14 deletions fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -755,16 +755,16 @@ public Status resetIdsForRestore(GlobalStateMgr globalStateMgr, Database db, int
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo;
for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
long newPartId = globalStateMgr.getNextId();
rangePartitionInfo.idToDataProperty.put(newPartId,
rangePartitionInfo.idToDataProperty.remove(entry.getValue()));
rangePartitionInfo.idToReplicationNum.remove(entry.getValue());
rangePartitionInfo.idToReplicationNum.put(newPartId,
(short) restoreReplicationNum);
rangePartitionInfo.getIdToRange(false).put(newPartId,
rangePartitionInfo.getIdToRange(false).remove(entry.getValue()));

rangePartitionInfo.idToInMemory
.put(newPartId, rangePartitionInfo.idToInMemory.remove(entry.getValue()));
// preserve existing info
DataProperty dataProperty = rangePartitionInfo.getDataProperty(entry.getValue());
boolean inMemory = rangePartitionInfo.getIsInMemory(entry.getValue());
DataCacheInfo dataCacheInfo = rangePartitionInfo.getDataCacheInfo(entry.getValue());
Range<PartitionKey> range = rangePartitionInfo.getIdToRange(false).get(entry.getValue());
// replace with new info
rangePartitionInfo.dropPartition(entry.getValue());
rangePartitionInfo.addPartition(newPartId, false, range, dataProperty, (short) restoreReplicationNum,
inMemory, dataCacheInfo);

idToPartition.get(entry.getValue()).getSubPartitions().forEach(physicalPartition -> {
physicalPartitionIdToPartitionId.remove(physicalPartition.getId());
});
Expand All @@ -785,10 +785,13 @@ public Status resetIdsForRestore(GlobalStateMgr globalStateMgr, Database db, int
// Single partitioned
long newPartId = globalStateMgr.getNextId();
for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
partitionInfo.idToDataProperty.put(newPartId, partitionInfo.idToDataProperty.remove(entry.getValue()));
partitionInfo.idToReplicationNum.remove(entry.getValue());
partitionInfo.idToReplicationNum.put(newPartId, (short) restoreReplicationNum);
partitionInfo.idToInMemory.put(newPartId, partitionInfo.idToInMemory.remove(entry.getValue()));
DataProperty dataProperty = partitionInfo.getDataProperty(entry.getValue());
boolean inMemory = partitionInfo.getIsInMemory(entry.getValue());
DataCacheInfo dataCacheInfo = partitionInfo.getDataCacheInfo(entry.getValue());
partitionInfo.dropPartition(entry.getValue());
partitionInfo.addPartition(newPartId, dataProperty, (short) restoreReplicationNum, inMemory,
dataCacheInfo);

idToPartition.get(entry.getValue()).getSubPartitions().forEach(physicalPartition -> {
physicalPartitionIdToPartitionId.remove(physicalPartition.getId());
});
Expand Down
6 changes: 4 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/catalog/PartitionInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,13 @@ public int getQuorumNum(long partitionId, TWriteQuorumType writeQuorum) {
}

public short getReplicationNum(long partitionId) {
if (!idToReplicationNum.containsKey(partitionId)) {
// Perform the op under no lock, the formal containsKey() call can't guarantee the later get() op success.
Short replicationNum = idToReplicationNum.get(partitionId);
if (replicationNum == null) {
LOG.debug("failed to get replica num for partition: {}", partitionId);
return (short) -1;
}
return idToReplicationNum.get(partitionId);
return replicationNum;
}

public short getMinReplicationNum() {
Expand Down
33 changes: 10 additions & 23 deletions fe/fe-core/src/main/java/com/starrocks/catalog/RangePartitionInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,10 +224,8 @@ public Range<PartitionKey> handleNewSinglePartitionDesc(SingleRangePartitionDesc
// Range.closedOpen may throw this if (lower > upper)
throw new DdlException("Invalid key range: " + e.getMessage());
}
idToDataProperty.put(partitionId, desc.getPartitionDataProperty());
idToReplicationNum.put(partitionId, desc.getReplicationNum());
idToInMemory.put(partitionId, desc.isInMemory());
idToStorageCacheInfo.put(partitionId, desc.getDataCacheInfo());
super.addPartition(partitionId, desc.getPartitionDataProperty(), desc.getReplicationNum(), desc.isInMemory(),
desc.getDataCacheInfo());
return range;
}

Expand All @@ -244,10 +242,8 @@ public void createAutomaticShadowPartition(long partitionId, String replicateNum
} catch (AnalysisException e) {
throw new DdlException("Invalid key range: " + e.getMessage());
}
idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD));
idToReplicationNum.put(partitionId, Short.valueOf(replicateNum));
idToInMemory.put(partitionId, false);
idToStorageCacheInfo.put(partitionId, new DataCacheInfo(true, false));
super.addPartition(partitionId, new DataProperty(TStorageMedium.HDD), Short.valueOf(replicateNum), false,
new DataCacheInfo(true, false));
}

public void handleNewRangePartitionDescs(List<Pair<Partition, PartitionDesc>> partitionList,
Expand All @@ -268,21 +264,16 @@ public void handleNewRangePartitionDescs(List<Pair<Partition, PartitionDesc>> pa
// Range.closedOpen may throw this if (lower > upper)
throw new DdlException("Invalid key range: " + e.getMessage());
}
idToDataProperty.put(partitionId, desc.getPartitionDataProperty());
idToReplicationNum.put(partitionId, desc.getReplicationNum());
idToInMemory.put(partitionId, desc.isInMemory());
idToStorageCacheInfo.put(partitionId, desc.getDataCacheInfo());
super.addPartition(partitionId, desc.getPartitionDataProperty(), desc.getReplicationNum(),
desc.isInMemory(), desc.getDataCacheInfo());
}
}
} catch (Exception e) {
// cleanup
partitionList.forEach(entry -> {
long partitionId = entry.first.getId();
removeRangeInternal(partitionId, isTemp);
idToDataProperty.remove(partitionId);
idToReplicationNum.remove(partitionId);
idToInMemory.remove(partitionId);
idToStorageCacheInfo.remove(partitionId);
super.dropPartition(partitionId);
});
throw e;
}
Expand All @@ -292,9 +283,7 @@ public void unprotectHandleNewSinglePartitionDesc(long partitionId, boolean isTe
DataProperty dataProperty, short replicationNum,
boolean isInMemory) {
setRangeInternal(partitionId, isTemp, range);
idToDataProperty.put(partitionId, dataProperty);
idToReplicationNum.put(partitionId, replicationNum);
idToInMemory.put(partitionId, isInMemory);
super.addPartition(partitionId, dataProperty, replicationNum, isInMemory);
}

/**
Expand All @@ -305,10 +294,8 @@ public void unprotectHandleNewSinglePartitionDesc(RangePartitionPersistInfo info
Partition partition = info.getPartition();
long partitionId = partition.getId();
setRangeInternal(partitionId, info.isTempPartition(), info.getRange());
idToDataProperty.put(partitionId, info.getDataProperty());
idToReplicationNum.put(partitionId, info.getReplicationNum());
idToInMemory.put(partitionId, info.isInMemory());
idToStorageCacheInfo.put(partitionId, info.getDataCacheInfo());
super.addPartition(partitionId, info.getDataProperty(), info.getReplicationNum(), info.isInMemory(),
info.getDataCacheInfo());
}

public void setRange(long partitionId, boolean isTemp, Range<PartitionKey> range) {
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/sql/ast/ListPartitionDesc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ public PartitionInfo toPartitionInfo(List<Column> columns, Map<String, Long> par
listPartitionInfo.setValues(partitionId, desc.getValues());
listPartitionInfo.setLiteralExprValues(partitionId, desc.getValues());
listPartitionInfo.setIdToIsTempPartition(partitionId, isTemp);
listPartitionInfo.setStorageCacheInfo(partitionId, desc.getDataCacheInfo());
listPartitionInfo.setDataCacheInfo(partitionId, desc.getDataCacheInfo());
}
for (MultiItemListPartitionDesc desc : this.multiListPartitionDescs) {
long partitionId = partitionNameToId.get(desc.getPartitionName());
Expand All @@ -301,7 +301,7 @@ public PartitionInfo toPartitionInfo(List<Column> columns, Map<String, Long> par
listPartitionInfo.setMultiValues(partitionId, desc.getMultiValues());
listPartitionInfo.setMultiLiteralExprValues(partitionId, desc.getMultiValues());
listPartitionInfo.setIdToIsTempPartition(partitionId, isTemp);
listPartitionInfo.setStorageCacheInfo(partitionId, desc.getDataCacheInfo());
listPartitionInfo.setDataCacheInfo(partitionId, desc.getDataCacheInfo());
}
listPartitionInfo.setAutomaticPartition(isAutoPartitionTable);
return listPartitionInfo;
Expand Down
Loading
Loading