Search in sources :

Example 11 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class IcebergAbstractMetadata method finishInsert.

@Override
public Optional<ConnectorOutputMetadata> finishInsert(ConnectorSession session, ConnectorInsertTableHandle insertHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics) {
    if (fragments.isEmpty()) {
        transaction.commitTransaction();
        return Optional.empty();
    }
    IcebergWritableTableHandle table = (IcebergWritableTableHandle) insertHandle;
    org.apache.iceberg.Table icebergTable = transaction.table();
    List<CommitTaskData> commitTasks = fragments.stream().map(slice -> commitTaskCodec.fromJson(slice.getBytes())).collect(toImmutableList());
    Type[] partitionColumnTypes = icebergTable.spec().fields().stream().map(field -> field.transform().getResultType(icebergTable.schema().findType(field.sourceId()))).toArray(Type[]::new);
    AppendFiles appendFiles = transaction.newFastAppend();
    for (CommitTaskData task : commitTasks) {
        DataFiles.Builder builder = DataFiles.builder(icebergTable.spec()).withPath(task.getPath()).withFileSizeInBytes(task.getFileSizeInBytes()).withFormat(table.getFileFormat()).withMetrics(task.getMetrics().metrics());
        if (!icebergTable.spec().fields().isEmpty()) {
            String partitionDataJson = task.getPartitionDataJson().orElseThrow(() -> new VerifyException("No partition data for partitioned table"));
            builder.withPartition(PartitionData.fromJson(partitionDataJson, partitionColumnTypes));
        }
        appendFiles.appendFile(builder.build());
    }
    appendFiles.commit();
    transaction.commitTransaction();
    return Optional.of(new HiveWrittenPartitions(commitTasks.stream().map(CommitTaskData::getPath).collect(toImmutableList())));
}
Also used : JsonCodec(com.facebook.airlift.json.JsonCodec) TypeUtil(org.apache.iceberg.types.TypeUtil) Types(org.apache.iceberg.types.Types) ComputedStatistics(com.facebook.presto.spi.statistics.ComputedStatistics) AppendFiles(org.apache.iceberg.AppendFiles) Collections.singletonList(java.util.Collections.singletonList) SchemaTableName(com.facebook.presto.spi.SchemaTableName) DataFiles(org.apache.iceberg.DataFiles) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) PartitionFields.toPartitionFields(com.facebook.presto.iceberg.PartitionFields.toPartitionFields) FILE_FORMAT_PROPERTY(com.facebook.presto.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY) SystemTable(com.facebook.presto.spi.SystemTable) TypeConverter.toPrestoType(com.facebook.presto.iceberg.TypeConverter.toPrestoType) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) PARTITIONING_PROPERTY(com.facebook.presto.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY) Set(java.util.Set) Schema(org.apache.iceberg.Schema) PartitionSpecParser(org.apache.iceberg.PartitionSpecParser) SchemaParser(org.apache.iceberg.SchemaParser) ConnectorSession(com.facebook.presto.spi.ConnectorSession) Type(org.apache.iceberg.types.Type) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) TypeConverter.toIcebergType(com.facebook.presto.iceberg.TypeConverter.toIcebergType) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Optional(java.util.Optional) HiveWrittenPartitions(com.facebook.presto.hive.HiveWrittenPartitions) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Logger(com.facebook.airlift.log.Logger) Slice(io.airlift.slice.Slice) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) ArrayList(java.util.ArrayList) IcebergUtil.getFileFormat(com.facebook.presto.iceberg.IcebergUtil.getFileFormat) IcebergUtil.resolveSnapshotIdByName(com.facebook.presto.iceberg.IcebergUtil.resolveSnapshotIdByName) ImmutableList(com.google.common.collect.ImmutableList) TypeManager(com.facebook.presto.common.type.TypeManager) Objects.requireNonNull(java.util.Objects.requireNonNull) IcebergUtil.getColumns(com.facebook.presto.iceberg.IcebergUtil.getColumns) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) VerifyException(com.google.common.base.VerifyException) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) ConnectorOutputMetadata(com.facebook.presto.spi.connector.ConnectorOutputMetadata) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Constraint(com.facebook.presto.spi.Constraint) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Transaction(org.apache.iceberg.Transaction) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout) AppendFiles(org.apache.iceberg.AppendFiles) DataFiles(org.apache.iceberg.DataFiles) HiveWrittenPartitions(com.facebook.presto.hive.HiveWrittenPartitions) TypeConverter.toPrestoType(com.facebook.presto.iceberg.TypeConverter.toPrestoType) Type(org.apache.iceberg.types.Type) TypeConverter.toIcebergType(com.facebook.presto.iceberg.TypeConverter.toIcebergType) VerifyException(com.google.common.base.VerifyException)

Example 12 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class MetadataManager method beginRefreshMaterializedView.

@Override
public InsertTableHandle beginRefreshMaterializedView(Session session, TableHandle tableHandle) {
    ConnectorId connectorId = tableHandle.getConnectorId();
    CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId);
    ConnectorMetadata metadata = catalogMetadata.getMetadata();
    ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId);
    ConnectorInsertTableHandle handle = metadata.beginRefreshMaterializedView(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle());
    return new InsertTableHandle(tableHandle.getConnectorId(), transactionHandle, handle);
}
Also used : ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorId(com.facebook.presto.spi.ConnectorId)

Example 13 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method doTestMismatchSchemaTable.

protected void doTestMismatchSchemaTable(SchemaTableName schemaTableName, HiveStorageFormat storageFormat, List<ColumnMetadata> tableBefore, MaterializedResult dataBefore, List<ColumnMetadata> tableAfter, MaterializedResult dataAfter, List<RowExpression> afterFilters, List<Predicate<MaterializedRow>> afterResultPredicates) throws Exception {
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    doCreateEmptyTable(schemaTableName, storageFormat, tableBefore);
    // insert the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle, TEST_HIVE_PAGE_SINK_CONTEXT);
        sink.appendPage(dataBefore.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of());
        transaction.commit();
    }
    // load the table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataBefore.getMaterializedRows());
        transaction.commit();
    }
    // alter the table schema
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), false, DEFAULT_COLUMN_CONVERTER_PROVIDER);
        PrincipalPrivileges principalPrivileges = testingPrincipalPrivilege(session);
        Table oldTable = transaction.getMetastore().getTable(metastoreContext, schemaName, tableName).get();
        HiveTypeTranslator hiveTypeTranslator = new HiveTypeTranslator();
        List<Column> dataColumns = tableAfter.stream().filter(columnMetadata -> !columnMetadata.getName().equals("ds")).map(columnMetadata -> new Column(columnMetadata.getName(), toHiveType(hiveTypeTranslator, columnMetadata.getType()), Optional.empty(), Optional.empty())).collect(toList());
        Table.Builder newTable = Table.builder(oldTable).setDataColumns(dataColumns);
        transaction.getMetastore().replaceView(metastoreContext, schemaName, tableName, newTable.build(), principalPrivileges);
        transaction.commit();
    }
    // load the altered table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataAfter.getMaterializedRows());
        int filterCount = afterFilters.size();
        for (int i = 0; i < filterCount; i++) {
            RowExpression predicate = afterFilters.get(i);
            ConnectorTableLayoutHandle layoutHandle = pushdownFilter(session, metadata, transaction.getMetastore(), ROW_EXPRESSION_SERVICE, FUNCTION_RESOLUTION, hivePartitionManager, METADATA.getFunctionAndTypeManager(), tableHandle, predicate, Optional.empty()).getLayout().getHandle();
            // Read all columns with a filter
            MaterializedResult filteredResult = readTable(transaction, tableHandle, layoutHandle, columnHandles, session, OptionalInt.empty(), Optional.empty());
            Predicate<MaterializedRow> rowPredicate = afterResultPredicates.get(i);
            List<MaterializedRow> expectedRows = dataAfter.getMaterializedRows().stream().filter(rowPredicate::apply).collect(toList());
            assertEqualsIgnoreOrder(filteredResult.getMaterializedRows(), expectedRows);
            // Read all columns except the ones used in the filter
            Set<String> filterColumnNames = extractUnique(predicate).stream().map(VariableReferenceExpression::getName).collect(toImmutableSet());
            List<ColumnHandle> nonFilterColumns = columnHandles.stream().filter(column -> !filterColumnNames.contains(((HiveColumnHandle) column).getName())).collect(toList());
            int resultCount = readTable(transaction, tableHandle, layoutHandle, nonFilterColumns, session, OptionalInt.empty(), Optional.empty()).getRowCount();
            assertEquals(resultCount, expectedRows.size());
        }
        transaction.commit();
    }
    // insertions to the partitions with type mismatches should fail
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle, TEST_HIVE_PAGE_SINK_CONTEXT);
        sink.appendPage(dataAfter.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of());
        transaction.commit();
        fail("expected exception");
    } catch (PrestoException e) {
        // expected
        assertEquals(e.getErrorCode(), HIVE_PARTITION_SCHEMA_MISMATCH.toErrorCode());
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) SkipException(org.testng.SkipException) WarningCollector(com.facebook.presto.spi.WarningCollector) CharType.createCharType(com.facebook.presto.common.type.CharType.createCharType) DateTimeZone(org.joda.time.DateTimeZone) SORTED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) FileSystem(org.apache.hadoop.fs.FileSystem) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) CounterStat(com.facebook.airlift.stats.CounterStat) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) FileStatus(org.apache.hadoop.fs.FileStatus) MoreFutures.getFutureValue(com.facebook.airlift.concurrent.MoreFutures.getFutureValue) MAX_PARTITION_KEY_COLUMN_INDEX(com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX) NOT_PARTITIONED(com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) Slices(io.airlift.slice.Slices) Map(java.util.Map) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) NullableValue(com.facebook.presto.common.predicate.NullableValue) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) SqlFunctionProperties(com.facebook.presto.common.function.SqlFunctionProperties) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) TABLE(com.facebook.presto.hive.CacheQuotaScope.TABLE) ZoneId(java.time.ZoneId) HiveColumnHandle.bucketColumnHandle(com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) SqlTimestamp(com.facebook.presto.common.type.SqlTimestamp) Predicate(com.google.common.base.Predicate) HiveColumnStatistics.createBinaryColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) StandardTypes(com.facebook.presto.common.type.StandardTypes) Table(com.facebook.presto.hive.metastore.Table) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveTestUtils.arrayType(com.facebook.presto.hive.HiveTestUtils.arrayType) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) REAL(com.facebook.presto.common.type.RealType.REAL) PAGEFILE(com.facebook.presto.hive.HiveStorageFormat.PAGEFILE) OptionalLong(java.util.OptionalLong) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) GIGABYTE(io.airlift.units.DataSize.Unit.GIGABYTE) HIVE_BYTE(com.facebook.presto.hive.HiveType.HIVE_BYTE) SqlVarbinary(com.facebook.presto.common.type.SqlVarbinary) Assertions.assertInstanceOf(com.facebook.airlift.testing.Assertions.assertInstanceOf) CSV(com.facebook.presto.hive.HiveStorageFormat.CSV) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) MetastoreCacheScope(com.facebook.presto.hive.metastore.CachingHiveMetastore.MetastoreCacheScope) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) AfterClass(org.testng.annotations.AfterClass) FileUtils.makePartName(org.apache.hadoop.hive.common.FileUtils.makePartName) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) HostAndPort(com.google.common.net.HostAndPort) Domain(com.facebook.presto.common.predicate.Domain) FUNCTION_AND_TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.FUNCTION_AND_TYPE_MANAGER) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) SqlFunctionId(com.facebook.presto.spi.function.SqlFunctionId) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) SplitSchedulingContext(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingContext) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) HiveColumnStatistics.createDecimalColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) RowType(com.facebook.presto.common.type.RowType) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveFilterPushdown.pushdownFilter(com.facebook.presto.hive.rule.HiveFilterPushdown.pushdownFilter) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) HIVE_FLOAT(com.facebook.presto.hive.HiveType.HIVE_FLOAT) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) TypeProvider(com.facebook.presto.sql.planner.TypeProvider) Locale(java.util.Locale) UNGROUPED_SCHEDULING(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) Thread.sleep(java.lang.Thread.sleep) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) PageFilePageSource(com.facebook.presto.hive.pagefile.PageFilePageSource) ImmutableSet(com.google.common.collect.ImmutableSet) OFFLINE_DATA_DEBUG_MODE_ENABLED(com.facebook.presto.hive.HiveSessionProperties.OFFLINE_DATA_DEBUG_MODE_ENABLED) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) Range(com.facebook.presto.common.predicate.Range) Objects(java.util.Objects) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) MetastoreUtil.toPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues) HiveColumnStatistics.createBooleanColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics) METADATA(com.facebook.presto.hive.HiveTestUtils.METADATA) HiveTestUtils.getDefaultHiveBatchPageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories) IntStream(java.util.stream.IntStream) MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) MapType(com.facebook.presto.common.type.MapType) FILTER_STATS_CALCULATOR_SERVICE(com.facebook.presto.hive.HiveTestUtils.FILTER_STATS_CALCULATOR_SERVICE) Assert.assertNull(org.testng.Assert.assertNull) HiveColumnStatistics.createIntegerColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) OptionalDouble(java.util.OptionalDouble) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) OptionalInt(java.util.OptionalInt) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) HIVE_DOUBLE(com.facebook.presto.hive.HiveType.HIVE_DOUBLE) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) HiveTestUtils.mapType(com.facebook.presto.hive.HiveTestUtils.mapType) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) Type(com.facebook.presto.common.type.Type) ExecutorService(java.util.concurrent.ExecutorService) NamedTypeSignature(com.facebook.presto.common.type.NamedTypeSignature) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) DEFAULT_COLUMN_CONVERTER_PROVIDER(com.facebook.presto.hive.HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER) USER(com.facebook.presto.spi.security.PrincipalType.USER) HiveTestUtils.getDefaultHiveSelectivePageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories) HiveColumnStatistics.createStringColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createStringColumnStatistics) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) PageSinkContext(com.facebook.presto.spi.PageSinkContext) OutputStreamDataSinkFactory(com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory) MetastoreUtil.createDirectory(com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory) TRANSACTION_CONFLICT(com.facebook.presto.spi.StandardErrorCode.TRANSACTION_CONFLICT) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) Hashing.sha256(com.google.common.hash.Hashing.sha256) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPartitioningMetadata(com.facebook.presto.spi.connector.ConnectorPartitioningMetadata) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Assert.assertTrue(org.testng.Assert.assertTrue) PlanBuilder.expression(com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) DecimalType.createDecimalType(com.facebook.presto.common.type.DecimalType.createDecimalType) HIVE_COMPATIBLE(com.facebook.presto.hive.BucketFunctionType.HIVE_COMPATIBLE) Page(com.facebook.presto.common.Page) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ColumnStatistics(com.facebook.presto.spi.statistics.ColumnStatistics) HIVE_SHORT(com.facebook.presto.hive.HiveType.HIVE_SHORT) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) SqlInvokedFunction(com.facebook.presto.spi.function.SqlInvokedFunction) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) HiveColumnStatistics.createDoubleColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics) BigDecimal(java.math.BigDecimal) TupleDomain.withColumnDomains(com.facebook.presto.common.predicate.TupleDomain.withColumnDomains) Sets.difference(com.google.common.collect.Sets.difference) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) PRESTO_VERSION_NAME(com.facebook.presto.hive.HiveMetadata.PRESTO_VERSION_NAME) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TestingRowExpressionTranslator(com.facebook.presto.sql.TestingRowExpressionTranslator) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Executors(java.util.concurrent.Executors) ConnectorSession(com.facebook.presto.spi.ConnectorSession) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) Assertions.assertGreaterThan(com.facebook.airlift.testing.Assertions.assertGreaterThan) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) OrcSelectivePageSource(com.facebook.presto.hive.orc.OrcSelectivePageSource) STAGE_AND_MOVE_TO_TARGET_DIRECTORY(com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY) VariablesExtractor.extractUnique(com.facebook.presto.sql.planner.VariablesExtractor.extractUnique) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Slice(io.airlift.slice.Slice) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) TINYINT(com.facebook.presto.common.type.TinyintType.TINYINT) Assertions.assertGreaterThanOrEqual(com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) WriteInfo(com.facebook.presto.hive.LocationService.WriteInfo) HYPER_LOG_LOG(com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG) HiveBasicStatistics.createEmptyStatistics(com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics) DATE(com.facebook.presto.common.type.DateType.DATE) FUNCTION_RESOLUTION(com.facebook.presto.hive.HiveTestUtils.FUNCTION_RESOLUTION) ArrayList(java.util.ArrayList) ROW_EXPRESSION_SERVICE(com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) SqlDate(com.facebook.presto.common.type.SqlDate) NON_CACHEABLE(com.facebook.presto.spi.SplitContext.NON_CACHEABLE) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) TableHandle(com.facebook.presto.spi.TableHandle) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) HiveTestUtils.rowType(com.facebook.presto.hive.HiveTestUtils.rowType) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) LongStream(java.util.stream.LongStream) Executor(java.util.concurrent.Executor) Constraint(com.facebook.presto.spi.Constraint) UTC_KEY(com.facebook.presto.common.type.TimeZoneKey.UTC_KEY) UTC(org.joda.time.DateTimeZone.UTC) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SMALLINT(com.facebook.presto.common.type.SmallintType.SMALLINT) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Assertions.assertLessThanOrEqual(com.facebook.airlift.testing.Assertions.assertLessThanOrEqual) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) HIVE_BOOLEAN(com.facebook.presto.hive.HiveType.HIVE_BOOLEAN) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) HiveColumnStatistics.createDateColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics) ValueSet(com.facebook.presto.common.predicate.ValueSet) SORTED_WRITE_TO_TEMP_PATH_ENABLED(com.facebook.presto.hive.HiveSessionProperties.SORTED_WRITE_TO_TEMP_PATH_ENABLED) MetadataManager(com.facebook.presto.metadata.MetadataManager) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) TableStatistics(com.facebook.presto.spi.statistics.TableStatistics) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) CacheConfig(com.facebook.presto.cache.CacheConfig) HiveTestUtils.getDefaultOrcFileWriterFactory(com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) MoreCollectors.onlyElement(com.google.common.collect.MoreCollectors.onlyElement) OrcBatchPageSource(com.facebook.presto.hive.orc.OrcBatchPageSource) Iterables.concat(com.google.common.collect.Iterables.concat) MANAGED_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) Path(org.apache.hadoop.fs.Path) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) METASTORE_CONTEXT(com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableMap(com.google.common.collect.ImmutableMap) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) Threads.daemonThreadsNamed(com.facebook.airlift.concurrent.Threads.daemonThreadsNamed) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) PAGE_SORTER(com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER) RecordCursor(com.facebook.presto.spi.RecordCursor) DataSize(io.airlift.units.DataSize) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) Logger(com.facebook.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) DateTimeTestingUtils.sqlTimestampOf(com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) AtomicReference(java.util.concurrent.atomic.AtomicReference) VarcharType.createVarcharType(com.facebook.presto.common.type.VarcharType.createVarcharType) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Verify.verify(com.google.common.base.Verify.verify) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Objects.requireNonNull(java.util.Objects.requireNonNull) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) RowExpression(com.facebook.presto.spi.relation.RowExpression) Assertions.assertEqualsIgnoreOrder(com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) ConnectorIdentity(com.facebook.presto.spi.security.ConnectorIdentity) HiveBasicStatistics.createZeroStatistics(com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) RowFieldName(com.facebook.presto.common.type.RowFieldName) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Block(com.facebook.presto.common.block.Block) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) SECONDS(java.util.concurrent.TimeUnit.SECONDS) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) Column(com.facebook.presto.hive.metastore.Column) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) HiveColumnHandle.bucketColumnHandle(com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) Table(com.facebook.presto.hive.metastore.Table) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) RowExpression(com.facebook.presto.spi.relation.RowExpression) Constraint(com.facebook.presto.spi.Constraint) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) Slice(io.airlift.slice.Slice) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) MaterializedResult(com.facebook.presto.testing.MaterializedResult) MaterializedRow(com.facebook.presto.testing.MaterializedRow)

Aggregations

ConnectorInsertTableHandle (com.facebook.presto.spi.ConnectorInsertTableHandle)13 ConnectorTableHandle (com.facebook.presto.spi.ConnectorTableHandle)11 ConnectorSession (com.facebook.presto.spi.ConnectorSession)10 ConnectorMetadata (com.facebook.presto.spi.connector.ConnectorMetadata)10 ColumnHandle (com.facebook.presto.spi.ColumnHandle)9 ConnectorTableMetadata (com.facebook.presto.spi.ConnectorTableMetadata)8 Constraint (com.facebook.presto.spi.Constraint)8 PrestoException (com.facebook.presto.spi.PrestoException)8 ColumnMetadata (com.facebook.presto.spi.ColumnMetadata)7 ConnectorPageSink (com.facebook.presto.spi.ConnectorPageSink)7 SchemaTableName (com.facebook.presto.spi.SchemaTableName)7 ConnectorTransactionHandle (com.facebook.presto.spi.connector.ConnectorTransactionHandle)7 TestingConnectorSession (com.facebook.presto.testing.TestingConnectorSession)7 Slice (io.airlift.slice.Slice)7 Optional (java.util.Optional)7 ConnectorOutputTableHandle (com.facebook.presto.spi.ConnectorOutputTableHandle)6 ConnectorTableLayout (com.facebook.presto.spi.ConnectorTableLayout)6 ConnectorTableLayoutHandle (com.facebook.presto.spi.ConnectorTableLayoutHandle)6 ConnectorTableLayoutResult (com.facebook.presto.spi.ConnectorTableLayoutResult)6 SchemaTablePrefix (com.facebook.presto.spi.SchemaTablePrefix)6