diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java index b6037a76e1e32..6b8a09f2a26f2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java @@ -308,7 +308,7 @@ private String singleListPartitionSql(OlapTable table, List partitionIds, StringBuilder sb = new StringBuilder(); this.idToLiteralExprValues.forEach((partitionId, values) -> { if (partitionIds.contains(partitionId)) { - Short partitionReplicaNum = table.getPartitionInfo().idToReplicationNum.get(partitionId); + short partitionReplicaNum = table.getPartitionInfo().getReplicationNum(partitionId); Optional.ofNullable(table.getPartition(partitionId)).ifPresent(partition -> { String partitionName = partition.getName(); sb.append(" PARTITION ") @@ -316,7 +316,7 @@ private String singleListPartitionSql(OlapTable table, List partitionIds, .append(" VALUES IN ") .append(this.valuesToString(values)); - if (partitionReplicaNum != null && partitionReplicaNum != tableReplicationNum) { + if (partitionReplicaNum != tableReplicationNum) { sb.append(" (").append("\"" + PROPERTIES_REPLICATION_NUM + "\" = \"").append(partitionReplicaNum) .append("\")"); } @@ -336,7 +336,7 @@ private String multiListPartitionSql(OlapTable table, List partitionIds, s StringBuilder sb = new StringBuilder(); this.idToMultiLiteralExprValues.forEach((partitionId, multiValues) -> { if (partitionIds.contains(partitionId)) { - Short partitionReplicaNum = table.getPartitionInfo().idToReplicationNum.get(partitionId); + short partitionReplicaNum = table.getPartitionInfo().getReplicationNum(partitionId); Optional.ofNullable(table.getPartition(partitionId)).ifPresent(partition -> { String partitionName = partition.getName(); sb.append(" PARTITION ") @@ -344,7 +344,7 @@ private String multiListPartitionSql(OlapTable table, List partitionIds, s .append(" VALUES IN ") .append(this.multiValuesToString(multiValues)); - if (partitionReplicaNum != null && partitionReplicaNum != tableReplicationNum) { + if (partitionReplicaNum != tableReplicationNum) { sb.append(" (").append("\"" + PROPERTIES_REPLICATION_NUM + "\" = \"").append(partitionReplicaNum) .append("\")"); } @@ -396,9 +396,6 @@ public void handleNewListPartitionDescs(List> par PartitionDesc partitionDesc = entry.second; Preconditions.checkArgument(partitionDesc instanceof SinglePartitionDesc); Preconditions.checkArgument(((SinglePartitionDesc) partitionDesc).isAnalyzed()); - this.idToDataProperty.put(partitionId, partitionDesc.getPartitionDataProperty()); - this.idToReplicationNum.put(partitionId, partitionDesc.getReplicationNum()); - this.idToInMemory.put(partitionId, partitionDesc.isInMemory()); if (partitionDesc instanceof MultiItemListPartitionDesc) { MultiItemListPartitionDesc multiItemListPartitionDesc = (MultiItemListPartitionDesc) partitionDesc; @@ -414,7 +411,9 @@ public void handleNewListPartitionDescs(List> par "add list partition only support single item or multi item list partition now"); } this.idToIsTempPartition.put(partitionId, isTempPartition); - this.idToStorageCacheInfo.put(partitionId, partitionDesc.getDataCacheInfo()); + super.addPartition(partitionId, partitionDesc.getPartitionDataProperty(), + partitionDesc.getReplicationNum(), partitionDesc.isInMemory(), + partitionDesc.getDataCacheInfo()); } } } catch (Exception e) { @@ -426,11 +425,10 @@ public void unprotectHandleNewPartitionDesc(ListPartitionPersistInfo partitionPe throws AnalysisException { Partition partition = partitionPersistInfo.getPartition(); long partitionId = partition.getId(); - this.idToDataProperty.put(partitionId, partitionPersistInfo.getDataProperty()); - this.idToReplicationNum.put(partitionId, partitionPersistInfo.getReplicationNum()); - this.idToInMemory.put(partitionId, partitionPersistInfo.isInMemory()); this.idToIsTempPartition.put(partitionId, partitionPersistInfo.isTempPartition()); - this.idToStorageCacheInfo.put(partitionId, partitionPersistInfo.getDataCacheInfo()); + super.addPartition(partitionId, partitionPersistInfo.getDataProperty(), + partitionPersistInfo.getReplicationNum(), partitionPersistInfo.isInMemory(), + partitionPersistInfo.getDataCacheInfo()); List> multiValues = partitionPersistInfo.getMultiValues(); if (multiValues != null && multiValues.size() > 0) { @@ -465,15 +463,14 @@ public void addPartition(long partitionId, DataProperty dataProperty, short repl DataCacheInfo dataCacheInfo, List values, List> multiValues) throws AnalysisException { super.addPartition(partitionId, dataProperty, replicationNum, isInMemory, dataCacheInfo); - if (multiValues != null && multiValues.size() > 0) { + if (multiValues != null && !multiValues.isEmpty()) { this.idToMultiValues.put(partitionId, multiValues); this.setMultiLiteralExprValues(partitionId, multiValues); } - if (values != null && values.size() > 0) { + if (values != null && !values.isEmpty()) { this.idToValues.put(partitionId, values); this.setLiteralExprValues(partitionId, values); } - this.idToStorageCacheInfo.put(partitionId, dataCacheInfo); idToIsTempPartition.put(partitionId, false); } @@ -486,11 +483,8 @@ public void createAutomaticShadowPartition(long partitionId, String replicateNum idToValues.put(partitionId, Collections.emptyList()); idToLiteralExprValues.put(partitionId, Collections.emptyList()); } - - idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD)); - idToReplicationNum.put(partitionId, Short.valueOf(replicateNum)); - idToInMemory.put(partitionId, false); - idToStorageCacheInfo.put(partitionId, new DataCacheInfo(true, false)); + super.addPartition(partitionId, new DataProperty(TStorageMedium.HDD), Short.valueOf(replicateNum), false, + new DataCacheInfo(true, false)); } public static int compareByValue(List> left, List> right) { @@ -507,10 +501,6 @@ public static int compareByValue(List> left, List> rig return 0; } - public void setStorageCacheInfo(long partitionId, DataCacheInfo dataCacheInfo) { - idToStorageCacheInfo.put(partitionId, dataCacheInfo); - } - @Override public Object clone() { ListPartitionInfo info = (ListPartitionInfo) super.clone(); @@ -523,4 +513,4 @@ public Object clone() { info.automaticPartition = this.automaticPartition; return info; } -} \ No newline at end of file +} diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index 9a909bedccbd3..44847fbd3fe56 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -755,16 +755,16 @@ public Status resetIdsForRestore(GlobalStateMgr globalStateMgr, Database db, int RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; for (Map.Entry entry : origPartNameToId.entrySet()) { long newPartId = globalStateMgr.getNextId(); - rangePartitionInfo.idToDataProperty.put(newPartId, - rangePartitionInfo.idToDataProperty.remove(entry.getValue())); - rangePartitionInfo.idToReplicationNum.remove(entry.getValue()); - rangePartitionInfo.idToReplicationNum.put(newPartId, - (short) restoreReplicationNum); - rangePartitionInfo.getIdToRange(false).put(newPartId, - rangePartitionInfo.getIdToRange(false).remove(entry.getValue())); - - rangePartitionInfo.idToInMemory - .put(newPartId, rangePartitionInfo.idToInMemory.remove(entry.getValue())); + // preserve existing info + DataProperty dataProperty = rangePartitionInfo.getDataProperty(entry.getValue()); + boolean inMemory = rangePartitionInfo.getIsInMemory(entry.getValue()); + DataCacheInfo dataCacheInfo = rangePartitionInfo.getDataCacheInfo(entry.getValue()); + Range range = rangePartitionInfo.getIdToRange(false).get(entry.getValue()); + // replace with new info + rangePartitionInfo.dropPartition(entry.getValue()); + rangePartitionInfo.addPartition(newPartId, false, range, dataProperty, (short) restoreReplicationNum, + inMemory, dataCacheInfo); + idToPartition.get(entry.getValue()).getSubPartitions().forEach(physicalPartition -> { physicalPartitionIdToPartitionId.remove(physicalPartition.getId()); }); @@ -785,10 +785,13 @@ public Status resetIdsForRestore(GlobalStateMgr globalStateMgr, Database db, int // Single partitioned long newPartId = globalStateMgr.getNextId(); for (Map.Entry entry : origPartNameToId.entrySet()) { - partitionInfo.idToDataProperty.put(newPartId, partitionInfo.idToDataProperty.remove(entry.getValue())); - partitionInfo.idToReplicationNum.remove(entry.getValue()); - partitionInfo.idToReplicationNum.put(newPartId, (short) restoreReplicationNum); - partitionInfo.idToInMemory.put(newPartId, partitionInfo.idToInMemory.remove(entry.getValue())); + DataProperty dataProperty = partitionInfo.getDataProperty(entry.getValue()); + boolean inMemory = partitionInfo.getIsInMemory(entry.getValue()); + DataCacheInfo dataCacheInfo = partitionInfo.getDataCacheInfo(entry.getValue()); + partitionInfo.dropPartition(entry.getValue()); + partitionInfo.addPartition(newPartId, dataProperty, (short) restoreReplicationNum, inMemory, + dataCacheInfo); + idToPartition.get(entry.getValue()).getSubPartitions().forEach(physicalPartition -> { physicalPartitionIdToPartitionId.remove(physicalPartition.getId()); }); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionInfo.java index 1e8466b482acd..33c7e0220d78d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionInfo.java @@ -142,11 +142,13 @@ public int getQuorumNum(long partitionId, TWriteQuorumType writeQuorum) { } public short getReplicationNum(long partitionId) { - if (!idToReplicationNum.containsKey(partitionId)) { + // Perform the op under no lock, the formal containsKey() call can't guarantee the later get() op success. + Short replicationNum = idToReplicationNum.get(partitionId); + if (replicationNum == null) { LOG.debug("failed to get replica num for partition: {}", partitionId); return (short) -1; } - return idToReplicationNum.get(partitionId); + return replicationNum; } public short getMinReplicationNum() { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/RangePartitionInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/RangePartitionInfo.java index 79e5540114933..8391d44f5a1c2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/RangePartitionInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/RangePartitionInfo.java @@ -224,10 +224,8 @@ public Range handleNewSinglePartitionDesc(SingleRangePartitionDesc // Range.closedOpen may throw this if (lower > upper) throw new DdlException("Invalid key range: " + e.getMessage()); } - idToDataProperty.put(partitionId, desc.getPartitionDataProperty()); - idToReplicationNum.put(partitionId, desc.getReplicationNum()); - idToInMemory.put(partitionId, desc.isInMemory()); - idToStorageCacheInfo.put(partitionId, desc.getDataCacheInfo()); + super.addPartition(partitionId, desc.getPartitionDataProperty(), desc.getReplicationNum(), desc.isInMemory(), + desc.getDataCacheInfo()); return range; } @@ -244,10 +242,8 @@ public void createAutomaticShadowPartition(long partitionId, String replicateNum } catch (AnalysisException e) { throw new DdlException("Invalid key range: " + e.getMessage()); } - idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD)); - idToReplicationNum.put(partitionId, Short.valueOf(replicateNum)); - idToInMemory.put(partitionId, false); - idToStorageCacheInfo.put(partitionId, new DataCacheInfo(true, false)); + super.addPartition(partitionId, new DataProperty(TStorageMedium.HDD), Short.valueOf(replicateNum), false, + new DataCacheInfo(true, false)); } public void handleNewRangePartitionDescs(List> partitionList, @@ -268,10 +264,8 @@ public void handleNewRangePartitionDescs(List> pa // Range.closedOpen may throw this if (lower > upper) throw new DdlException("Invalid key range: " + e.getMessage()); } - idToDataProperty.put(partitionId, desc.getPartitionDataProperty()); - idToReplicationNum.put(partitionId, desc.getReplicationNum()); - idToInMemory.put(partitionId, desc.isInMemory()); - idToStorageCacheInfo.put(partitionId, desc.getDataCacheInfo()); + super.addPartition(partitionId, desc.getPartitionDataProperty(), desc.getReplicationNum(), + desc.isInMemory(), desc.getDataCacheInfo()); } } } catch (Exception e) { @@ -279,10 +273,7 @@ public void handleNewRangePartitionDescs(List> pa partitionList.forEach(entry -> { long partitionId = entry.first.getId(); removeRangeInternal(partitionId, isTemp); - idToDataProperty.remove(partitionId); - idToReplicationNum.remove(partitionId); - idToInMemory.remove(partitionId); - idToStorageCacheInfo.remove(partitionId); + super.dropPartition(partitionId); }); throw e; } @@ -292,9 +283,7 @@ public void unprotectHandleNewSinglePartitionDesc(long partitionId, boolean isTe DataProperty dataProperty, short replicationNum, boolean isInMemory) { setRangeInternal(partitionId, isTemp, range); - idToDataProperty.put(partitionId, dataProperty); - idToReplicationNum.put(partitionId, replicationNum); - idToInMemory.put(partitionId, isInMemory); + super.addPartition(partitionId, dataProperty, replicationNum, isInMemory); } /** @@ -305,10 +294,8 @@ public void unprotectHandleNewSinglePartitionDesc(RangePartitionPersistInfo info Partition partition = info.getPartition(); long partitionId = partition.getId(); setRangeInternal(partitionId, info.isTempPartition(), info.getRange()); - idToDataProperty.put(partitionId, info.getDataProperty()); - idToReplicationNum.put(partitionId, info.getReplicationNum()); - idToInMemory.put(partitionId, info.isInMemory()); - idToStorageCacheInfo.put(partitionId, info.getDataCacheInfo()); + super.addPartition(partitionId, info.getDataProperty(), info.getReplicationNum(), info.isInMemory(), + info.getDataCacheInfo()); } public void setRange(long partitionId, boolean isTemp, Range range) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ListPartitionDesc.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ListPartitionDesc.java index 45825446a84cd..687458eb32554 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ListPartitionDesc.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ListPartitionDesc.java @@ -290,7 +290,7 @@ public PartitionInfo toPartitionInfo(List columns, Map par listPartitionInfo.setValues(partitionId, desc.getValues()); listPartitionInfo.setLiteralExprValues(partitionId, desc.getValues()); listPartitionInfo.setIdToIsTempPartition(partitionId, isTemp); - listPartitionInfo.setStorageCacheInfo(partitionId, desc.getDataCacheInfo()); + listPartitionInfo.setDataCacheInfo(partitionId, desc.getDataCacheInfo()); } for (MultiItemListPartitionDesc desc : this.multiListPartitionDescs) { long partitionId = partitionNameToId.get(desc.getPartitionName()); @@ -301,7 +301,7 @@ public PartitionInfo toPartitionInfo(List columns, Map par listPartitionInfo.setMultiValues(partitionId, desc.getMultiValues()); listPartitionInfo.setMultiLiteralExprValues(partitionId, desc.getMultiValues()); listPartitionInfo.setIdToIsTempPartition(partitionId, isTemp); - listPartitionInfo.setStorageCacheInfo(partitionId, desc.getDataCacheInfo()); + listPartitionInfo.setDataCacheInfo(partitionId, desc.getDataCacheInfo()); } listPartitionInfo.setAutomaticPartition(isAutoPartitionTable); return listPartitionInfo; diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java index 98fee3a7dc3e9..a946666961a0e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java @@ -326,15 +326,16 @@ public static Database mockDb() throws AnalysisException { Partition partition2 = new Partition(TEST_PARTITION2_ID, TEST_PARTITION2_NAME, baseIndexP2, distributionInfo2); RangePartitionInfo rangePartitionInfo = new RangePartitionInfo(Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0))); - + DataProperty dataPropertyP1 = new DataProperty(TStorageMedium.HDD); PartitionKey rangeP1Lower = PartitionKey.createInfinityPartitionKey(Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0)), false); PartitionKey rangeP1Upper = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")), Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0))); Range rangeP1 = Range.closedOpen(rangeP1Lower, rangeP1Upper); - rangePartitionInfo.setRange(TEST_PARTITION1_ID, false, rangeP1); + rangePartitionInfo.addPartition(TEST_PARTITION1_ID, false, rangeP1, dataPropertyP1, (short) 3, false); + DataProperty dataPropertyP2 = new DataProperty(TStorageMedium.HDD); PartitionKey rangeP2Lower = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")), Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0))); @@ -342,14 +343,7 @@ public static Database mockDb() throws AnalysisException { PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("20")), Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0))); Range rangeP2 = Range.closedOpen(rangeP2Lower, rangeP2Upper); - rangePartitionInfo.setRange(TEST_PARTITION2_ID, false, rangeP2); - - rangePartitionInfo.setReplicationNum(TEST_PARTITION1_ID, (short) 3); - rangePartitionInfo.setReplicationNum(TEST_PARTITION2_ID, (short) 3); - DataProperty dataPropertyP1 = new DataProperty(TStorageMedium.HDD); - DataProperty dataPropertyP2 = new DataProperty(TStorageMedium.HDD); - rangePartitionInfo.setDataProperty(TEST_PARTITION1_ID, dataPropertyP1); - rangePartitionInfo.setDataProperty(TEST_PARTITION2_ID, dataPropertyP2); + rangePartitionInfo.addPartition(TEST_PARTITION2_ID, false, rangeP2, dataPropertyP2, (short) 3, false); OlapTable olapTable2 = new OlapTable(TEST_TBL2_ID, TEST_TBL2_NAME, TEST_TBL_BASE_SCHEMA, KeysType.AGG_KEYS, rangePartitionInfo, distributionInfo2); @@ -503,9 +497,7 @@ public static Database mockDb() throws AnalysisException { partition1.addSubPartition(physicalPartition2); rangePartitionInfo = new RangePartitionInfo(Lists.newArrayList(TEST_TBL_BASE_SCHEMA.get(0))); - rangePartitionInfo.setRange(TEST_PARTITION1_ID, false, rangeP1); - rangePartitionInfo.setReplicationNum(TEST_PARTITION1_ID, (short) 3); - rangePartitionInfo.setDataProperty(TEST_PARTITION1_ID, dataPropertyP1); + rangePartitionInfo.addPartition(TEST_PARTITION1_ID, false, rangeP1, dataPropertyP1, (short) 3, false); baseTabletP1 = new LocalTablet(TEST_BASE_TABLET_P1_ID); tabletMetaBaseTabletP1 = new TabletMeta(TEST_DB_ID, TEST_TBL4_ID, TEST_PARTITION1_ID,