Search in sources :

Example 1 with DeltaInfoPair

use of com.facebook.presto.raptor.metadata.DeltaInfoPair in project presto by prestodb.

the class RaptorMetadata method finishDelete.

@Override
public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<Slice> fragments) {
    RaptorTableHandle table = (RaptorTableHandle) tableHandle;
    long transactionId = table.getTransactionId().getAsLong();
    long tableId = table.getTableId();
    List<ColumnInfo> columns = getColumnHandles(session, tableHandle).values().stream().map(RaptorColumnHandle.class::cast).map(ColumnInfo::fromHandle).collect(toList());
    if (table.isTableSupportsDeltaDelete()) {
        ImmutableMap.Builder<UUID, DeltaInfoPair> shardMapBuilder = ImmutableMap.builder();
        fragments.stream().map(fragment -> SHARD_DELETE_DELTA_CODEC.fromJson(fragment.getBytes())).forEach(delta -> shardMapBuilder.put(delta.getOldShardUuid(), delta.getDeltaInfoPair()));
        OptionalLong updateTime = OptionalLong.of(session.getStartTime());
        log.info("Finishing delete for tableId %s (affected shardUuid: %s)", tableId, shardMapBuilder.build().size());
        shardManager.replaceDeltaUuids(transactionId, tableId, columns, shardMapBuilder.build(), updateTime);
    } else {
        ImmutableSet.Builder<UUID> oldShardUuidsBuilder = ImmutableSet.builder();
        ImmutableList.Builder<ShardInfo> newShardsBuilder = ImmutableList.builder();
        fragments.stream().map(fragment -> SHARD_DELTA_CODEC.fromJson(fragment.getBytes())).forEach(delta -> {
            oldShardUuidsBuilder.addAll(delta.getOldShardUuids());
            newShardsBuilder.addAll(delta.getNewShards());
        });
        Set<UUID> oldShardUuids = oldShardUuidsBuilder.build();
        List<ShardInfo> newShards = newShardsBuilder.build();
        OptionalLong updateTime = OptionalLong.of(session.getStartTime());
        log.info("Finishing delete for tableId %s (removed: %s, rewritten: %s)", tableId, oldShardUuids.size() - newShards.size(), newShards.size());
        shardManager.replaceShardUuids(transactionId, tableId, columns, oldShardUuids, newShards, updateTime);
    }
    clearRollback();
}
Also used : ShardDeleteDelta(com.facebook.presto.raptor.metadata.ShardDeleteDelta) SHARD_UUID_COLUMN_TYPE(com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE) NOT_FOUND(com.facebook.presto.spi.StandardErrorCode.NOT_FOUND) ComputedStatistics(com.facebook.presto.spi.statistics.ComputedStatistics) DatabaseUtil.runIgnoringConstraintViolation(com.facebook.presto.raptor.util.DatabaseUtil.runIgnoringConstraintViolation) Map(java.util.Map) ShardManager(com.facebook.presto.raptor.metadata.ShardManager) SystemTable(com.facebook.presto.spi.SystemTable) ColumnRangesSystemTable.getSourceTable(com.facebook.presto.raptor.systemtables.ColumnRangesSystemTable.getSourceTable) INVALID_TABLE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) RaptorTableProperties.getSortColumns(com.facebook.presto.raptor.RaptorTableProperties.getSortColumns) ColumnInfo(com.facebook.presto.raptor.metadata.ColumnInfo) ViewResult(com.facebook.presto.raptor.metadata.ViewResult) Set(java.util.Set) JsonCodec.jsonCodec(com.facebook.airlift.json.JsonCodec.jsonCodec) ConnectorSession(com.facebook.presto.spi.ConnectorSession) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) RaptorColumnHandle.shardUuidColumnHandle(com.facebook.presto.raptor.RaptorColumnHandle.shardUuidColumnHandle) ConnectorPartitioningHandle(com.facebook.presto.spi.connector.ConnectorPartitioningHandle) Slice(io.airlift.slice.Slice) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) ColumnRangesSystemTable(com.facebook.presto.raptor.systemtables.ColumnRangesSystemTable) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) DATE(com.facebook.presto.common.type.DateType.DATE) Multimaps(com.google.common.collect.Multimaps) ArrayList(java.util.ArrayList) OptionalLong(java.util.OptionalLong) BUCKET_NUMBER_COLUMN_NAME(com.facebook.presto.raptor.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME) RaptorSessionProperties.getExternalBatchId(com.facebook.presto.raptor.RaptorSessionProperties.getExternalBatchId) DatabaseUtil.daoTransaction(com.facebook.presto.raptor.util.DatabaseUtil.daoTransaction) Nullable(javax.annotation.Nullable) RaptorTableProperties.isOrganized(com.facebook.presto.raptor.RaptorTableProperties.isOrganized) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) Distribution(com.facebook.presto.raptor.metadata.Distribution) BUCKETED_ON_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.BUCKETED_ON_PROPERTY) ORDERING_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.ORDERING_PROPERTY) RaptorSessionProperties.getOneSplitPerBucketThreshold(com.facebook.presto.raptor.RaptorSessionProperties.getOneSplitPerBucketThreshold) Constraint(com.facebook.presto.spi.Constraint) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) RaptorTableProperties.getBucketColumns(com.facebook.presto.raptor.RaptorTableProperties.getBucketColumns) MetadataDao(com.facebook.presto.raptor.metadata.MetadataDao) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) RAPTOR_ERROR(com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) TreeMap(java.util.TreeMap) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) BUCKET_COUNT_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.BUCKET_COUNT_PROPERTY) DeltaInfoPair(com.facebook.presto.raptor.metadata.DeltaInfoPair) RaptorTableProperties.isTableSupportsDeltaDelete(com.facebook.presto.raptor.RaptorTableProperties.isTableSupportsDeltaDelete) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) JsonCodec(com.facebook.airlift.json.JsonCodec) ShardDelta(com.facebook.presto.raptor.metadata.ShardDelta) IDBI(org.skife.jdbi.v2.IDBI) RaptorBucketFunction.validateBucketType(com.facebook.presto.raptor.RaptorBucketFunction.validateBucketType) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ConnectorTablePartitioning(com.facebook.presto.spi.ConnectorTablePartitioning) TableColumn(com.facebook.presto.raptor.metadata.TableColumn) DatabaseUtil.runTransaction(com.facebook.presto.raptor.util.DatabaseUtil.runTransaction) ImmutableSet(com.google.common.collect.ImmutableSet) RaptorTableProperties.getDistributionName(com.facebook.presto.raptor.RaptorTableProperties.getDistributionName) ImmutableMap(com.google.common.collect.ImmutableMap) DISTRIBUTION_NAME_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.DISTRIBUTION_NAME_PROPERTY) Collections.nCopies(java.util.Collections.nCopies) Collection(java.util.Collection) RaptorTableProperties.getTemporalColumn(com.facebook.presto.raptor.RaptorTableProperties.getTemporalColumn) UUID(java.util.UUID) RaptorColumnHandle.isHiddenColumn(com.facebook.presto.raptor.RaptorColumnHandle.isHiddenColumn) Collectors(java.util.stream.Collectors) RaptorColumnHandle.bucketNumberColumnHandle(com.facebook.presto.raptor.RaptorColumnHandle.bucketNumberColumnHandle) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) RaptorTableProperties.getBucketCount(com.facebook.presto.raptor.RaptorTableProperties.getBucketCount) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) TEMPORAL_COLUMN_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY) Optional(java.util.Optional) StorageTypeConverter(com.facebook.presto.raptor.storage.StorageTypeConverter) SortedMap(java.util.SortedMap) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Logger(com.facebook.airlift.log.Logger) ORGANIZED_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.ORGANIZED_PROPERTY) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) TABLE_SUPPORTS_DELTA_DELETE(com.facebook.presto.raptor.RaptorTableProperties.TABLE_SUPPORTS_DELTA_DELETE) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) AtomicReference(java.util.concurrent.atomic.AtomicReference) Collectors.toCollection(java.util.stream.Collectors.toCollection) ImmutableList(com.google.common.collect.ImmutableList) ALREADY_EXISTS(com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS) TypeManager(com.facebook.presto.common.type.TypeManager) SHARD_UUID_COLUMN_NAME(com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_NAME) Objects.requireNonNull(java.util.Objects.requireNonNull) DatabaseUtil.onDemandDao(com.facebook.presto.raptor.util.DatabaseUtil.onDemandDao) Type(com.facebook.presto.common.type.Type) Table(com.facebook.presto.raptor.metadata.Table) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorOutputMetadata(com.facebook.presto.spi.connector.ConnectorOutputMetadata) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) ASC_NULLS_FIRST(com.facebook.presto.common.block.SortOrder.ASC_NULLS_FIRST) Maps(com.google.common.collect.Maps) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) ShardInfo(com.facebook.presto.raptor.metadata.ShardInfo) LongConsumer(java.util.function.LongConsumer) Collectors.toList(java.util.stream.Collectors.toList) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Handle(org.skife.jdbi.v2.Handle) RaptorColumnHandle.shardRowIdHandle(com.facebook.presto.raptor.RaptorColumnHandle.shardRowIdHandle) ImmutableList(com.google.common.collect.ImmutableList) DeltaInfoPair(com.facebook.presto.raptor.metadata.DeltaInfoPair) ColumnInfo(com.facebook.presto.raptor.metadata.ColumnInfo) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableSet(com.google.common.collect.ImmutableSet) OptionalLong(java.util.OptionalLong) UUID(java.util.UUID) ShardInfo(com.facebook.presto.raptor.metadata.ShardInfo)

Aggregations

JsonCodec (com.facebook.airlift.json.JsonCodec)1 JsonCodec.jsonCodec (com.facebook.airlift.json.JsonCodec.jsonCodec)1 Logger (com.facebook.airlift.log.Logger)1 ASC_NULLS_FIRST (com.facebook.presto.common.block.SortOrder.ASC_NULLS_FIRST)1 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)1 DATE (com.facebook.presto.common.type.DateType.DATE)1 INTEGER (com.facebook.presto.common.type.IntegerType.INTEGER)1 TIMESTAMP (com.facebook.presto.common.type.TimestampType.TIMESTAMP)1 Type (com.facebook.presto.common.type.Type)1 TypeManager (com.facebook.presto.common.type.TypeManager)1 RaptorBucketFunction.validateBucketType (com.facebook.presto.raptor.RaptorBucketFunction.validateBucketType)1 BUCKET_NUMBER_COLUMN_NAME (com.facebook.presto.raptor.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME)1 SHARD_UUID_COLUMN_NAME (com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_NAME)1 SHARD_UUID_COLUMN_TYPE (com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE)1 RaptorColumnHandle.bucketNumberColumnHandle (com.facebook.presto.raptor.RaptorColumnHandle.bucketNumberColumnHandle)1 RaptorColumnHandle.isHiddenColumn (com.facebook.presto.raptor.RaptorColumnHandle.isHiddenColumn)1 RaptorColumnHandle.shardRowIdHandle (com.facebook.presto.raptor.RaptorColumnHandle.shardRowIdHandle)1 RaptorColumnHandle.shardUuidColumnHandle (com.facebook.presto.raptor.RaptorColumnHandle.shardUuidColumnHandle)1 RAPTOR_ERROR (com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR)1 RaptorSessionProperties.getExternalBatchId (com.facebook.presto.raptor.RaptorSessionProperties.getExternalBatchId)1