Search in sources :

Example 26 with ConnectorMetadata

use of com.facebook.presto.spi.connector.ConnectorMetadata in project presto by prestodb.

the class AbstractTestHiveClient method testGetPartitionSplitsBatch.

@Test
public void testGetPartitionSplitsBatch() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat);
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty());
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle());
        assertEquals(getSplitCount(splitSource), partitionCount);
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) TypeManager(com.facebook.presto.spi.type.TypeManager) Assertions.assertInstanceOf(io.airlift.testing.Assertions.assertInstanceOf) FileSystem(org.apache.hadoop.fs.FileSystem) TypeRegistry(com.facebook.presto.type.TypeRegistry) SqlDate(com.facebook.presto.spi.type.SqlDate) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) FileStatus(org.apache.hadoop.fs.FileStatus) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) Sets.difference(com.google.common.collect.Sets.difference) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) Map(java.util.Map) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) StandardTypes(com.facebook.presto.spi.type.StandardTypes) HiveWriteUtils.createDirectory(com.facebook.presto.hive.HiveWriteUtils.createDirectory) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) TINYINT(com.facebook.presto.spi.type.TinyintType.TINYINT) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ROW(com.facebook.presto.spi.type.StandardTypes.ROW) Domain(com.facebook.presto.spi.predicate.Domain) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) BridgingHiveMetastore(com.facebook.presto.hive.metastore.BridgingHiveMetastore) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) DOUBLE(com.facebook.presto.spi.type.DoubleType.DOUBLE) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) UTC_KEY(com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY) MapType(com.facebook.presto.type.MapType) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) ARRAY(com.facebook.presto.spi.type.StandardTypes.ARRAY) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) Type(com.facebook.presto.spi.type.Type) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) TIMESTAMP(com.facebook.presto.spi.type.TimestampType.TIMESTAMP) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) TestException(org.testng.TestException) AfterClass(org.testng.annotations.AfterClass) HYPER_LOG_LOG(com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Range(com.facebook.presto.spi.predicate.Range) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) UTC(org.joda.time.DateTimeZone.UTC) HostAndPort(com.google.common.net.HostAndPort) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) TableType(org.apache.hadoop.hive.metastore.TableType) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) TypeSignature.parseTypeSignature(com.facebook.presto.spi.type.TypeSignature.parseTypeSignature) ThriftHiveMetastore(com.facebook.presto.hive.metastore.ThriftHiveMetastore) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) ParquetHiveRecordCursor(com.facebook.presto.hive.parquet.ParquetHiveRecordCursor) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HIVE_METASTORE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR) Iterables.concat(com.google.common.collect.Iterables.concat) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) TypeSignatureParameter(com.facebook.presto.spi.type.TypeSignatureParameter) Path(org.apache.hadoop.fs.Path) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) NullableValue(com.facebook.presto.spi.predicate.NullableValue) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) RecordCursor(com.facebook.presto.spi.RecordCursor) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Varchars.isVarcharType(com.facebook.presto.spi.type.Varchars.isVarcharType) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) JsonCodec(io.airlift.json.JsonCodec) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Assert.assertNull(org.testng.Assert.assertNull) Logger(io.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) ArrayType(com.facebook.presto.type.ArrayType) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HadoopFileStatus(com.facebook.presto.hadoop.HadoopFileStatus) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) MAP(com.facebook.presto.spi.type.StandardTypes.MAP) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) OrcPageSource(com.facebook.presto.hive.orc.OrcPageSource) ImmutableList(com.google.common.collect.ImmutableList) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) ValueSet(com.facebook.presto.spi.predicate.ValueSet) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) NamedTypeSignature(com.facebook.presto.spi.type.NamedTypeSignature) Objects.requireNonNull(java.util.Objects.requireNonNull) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) VARBINARY(com.facebook.presto.spi.type.VarbinaryType.VARBINARY) ExecutorService(java.util.concurrent.ExecutorService) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) SMALLINT(com.facebook.presto.spi.type.SmallintType.SMALLINT) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) DATE(com.facebook.presto.spi.type.DateType.DATE) REAL(com.facebook.presto.spi.type.RealType.REAL) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) Assert.assertTrue(org.testng.Assert.assertTrue) ImmutableCollectors(com.facebook.presto.util.ImmutableCollectors) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Chars.isCharType(com.facebook.presto.spi.type.Chars.isCharType) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Test(org.testng.annotations.Test)

Example 27 with ConnectorMetadata

use of com.facebook.presto.spi.connector.ConnectorMetadata in project presto by prestodb.

the class AbstractTestHiveClient method testGetAllTableColumns.

@Test
public void testGetAllTableColumns() {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        Map<SchemaTableName, List<ColumnMetadata>> allColumns = metadata.listTableColumns(newSession(), new SchemaTablePrefix());
        assertTrue(allColumns.containsKey(tablePartitionFormat));
        assertTrue(allColumns.containsKey(tableUnpartitioned));
    }
}
Also used : SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) SchemaTableName(com.facebook.presto.spi.SchemaTableName) Test(org.testng.annotations.Test)

Example 28 with ConnectorMetadata

use of com.facebook.presto.spi.connector.ConnectorMetadata in project presto by prestodb.

the class AbstractTestHiveClient method testGetTableNames.

@Test
public void testGetTableNames() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        List<SchemaTableName> tables = metadata.listTables(newSession(), database);
        assertTrue(tables.contains(tablePartitionFormat));
        assertTrue(tables.contains(tableUnpartitioned));
    }
}
Also used : ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) SchemaTableName(com.facebook.presto.spi.SchemaTableName) Test(org.testng.annotations.Test)

Example 29 with ConnectorMetadata

use of com.facebook.presto.spi.connector.ConnectorMetadata in project presto by prestodb.

the class AbstractTestHiveClient method testGetPartitionSplitsBatchUnpartitioned.

@Test
public void testGetPartitionSplitsBatchUnpartitioned() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned);
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty());
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle());
        assertEquals(getSplitCount(splitSource), 1);
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) TypeManager(com.facebook.presto.spi.type.TypeManager) Assertions.assertInstanceOf(io.airlift.testing.Assertions.assertInstanceOf) FileSystem(org.apache.hadoop.fs.FileSystem) TypeRegistry(com.facebook.presto.type.TypeRegistry) SqlDate(com.facebook.presto.spi.type.SqlDate) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) FileStatus(org.apache.hadoop.fs.FileStatus) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) Sets.difference(com.google.common.collect.Sets.difference) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) Map(java.util.Map) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) StandardTypes(com.facebook.presto.spi.type.StandardTypes) HiveWriteUtils.createDirectory(com.facebook.presto.hive.HiveWriteUtils.createDirectory) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) TINYINT(com.facebook.presto.spi.type.TinyintType.TINYINT) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ROW(com.facebook.presto.spi.type.StandardTypes.ROW) Domain(com.facebook.presto.spi.predicate.Domain) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) BridgingHiveMetastore(com.facebook.presto.hive.metastore.BridgingHiveMetastore) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) DOUBLE(com.facebook.presto.spi.type.DoubleType.DOUBLE) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) UTC_KEY(com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY) MapType(com.facebook.presto.type.MapType) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) ARRAY(com.facebook.presto.spi.type.StandardTypes.ARRAY) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) Type(com.facebook.presto.spi.type.Type) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) TIMESTAMP(com.facebook.presto.spi.type.TimestampType.TIMESTAMP) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) TestException(org.testng.TestException) AfterClass(org.testng.annotations.AfterClass) HYPER_LOG_LOG(com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Range(com.facebook.presto.spi.predicate.Range) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) UTC(org.joda.time.DateTimeZone.UTC) HostAndPort(com.google.common.net.HostAndPort) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) TableType(org.apache.hadoop.hive.metastore.TableType) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) TypeSignature.parseTypeSignature(com.facebook.presto.spi.type.TypeSignature.parseTypeSignature) ThriftHiveMetastore(com.facebook.presto.hive.metastore.ThriftHiveMetastore) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) ParquetHiveRecordCursor(com.facebook.presto.hive.parquet.ParquetHiveRecordCursor) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HIVE_METASTORE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR) Iterables.concat(com.google.common.collect.Iterables.concat) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) TypeSignatureParameter(com.facebook.presto.spi.type.TypeSignatureParameter) Path(org.apache.hadoop.fs.Path) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) NullableValue(com.facebook.presto.spi.predicate.NullableValue) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) RecordCursor(com.facebook.presto.spi.RecordCursor) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Varchars.isVarcharType(com.facebook.presto.spi.type.Varchars.isVarcharType) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) JsonCodec(io.airlift.json.JsonCodec) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Assert.assertNull(org.testng.Assert.assertNull) Logger(io.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) ArrayType(com.facebook.presto.type.ArrayType) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HadoopFileStatus(com.facebook.presto.hadoop.HadoopFileStatus) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) MAP(com.facebook.presto.spi.type.StandardTypes.MAP) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) OrcPageSource(com.facebook.presto.hive.orc.OrcPageSource) ImmutableList(com.google.common.collect.ImmutableList) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) ValueSet(com.facebook.presto.spi.predicate.ValueSet) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) NamedTypeSignature(com.facebook.presto.spi.type.NamedTypeSignature) Objects.requireNonNull(java.util.Objects.requireNonNull) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) VARBINARY(com.facebook.presto.spi.type.VarbinaryType.VARBINARY) ExecutorService(java.util.concurrent.ExecutorService) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) SMALLINT(com.facebook.presto.spi.type.SmallintType.SMALLINT) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) DATE(com.facebook.presto.spi.type.DateType.DATE) REAL(com.facebook.presto.spi.type.RealType.REAL) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) Assert.assertTrue(org.testng.Assert.assertTrue) ImmutableCollectors(com.facebook.presto.util.ImmutableCollectors) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Chars.isCharType(com.facebook.presto.spi.type.Chars.isCharType) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Test(org.testng.annotations.Test)

Example 30 with ConnectorMetadata

use of com.facebook.presto.spi.connector.ConnectorMetadata in project presto by prestodb.

the class AbstractTestHiveClient method testTableCreationRollback.

@Test
public void testTableCreationRollback() throws Exception {
    try {
        Path stagingPathRoot;
        try (Transaction transaction = newTransaction()) {
            ConnectorSession session = newSession();
            ConnectorMetadata metadata = transaction.getMetadata();
            // begin creating the table
            ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(temporaryCreateRollbackTable, CREATE_TABLE_COLUMNS, createTableProperties(RCBINARY));
            ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty());
            // write the data
            ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle);
            sink.appendPage(CREATE_TABLE_DATA.toPage());
            getFutureValue(sink.finish());
            // verify we have data files
            stagingPathRoot = getStagingPathRoot(outputHandle);
            assertFalse(listAllDataFiles(stagingPathRoot).isEmpty());
            // rollback the table
            transaction.rollback();
        }
        // verify all files have been deleted
        assertTrue(listAllDataFiles(stagingPathRoot).isEmpty());
        // verify table is not in the metastore
        try (Transaction transaction = newTransaction()) {
            ConnectorSession session = newSession();
            ConnectorMetadata metadata = transaction.getMetadata();
            assertNull(metadata.getTableHandle(session, temporaryCreateRollbackTable));
        }
    } finally {
        dropTable(temporaryCreateRollbackTable);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) Test(org.testng.annotations.Test)

Aggregations

ConnectorMetadata (com.facebook.presto.spi.connector.ConnectorMetadata)100 ConnectorSession (com.facebook.presto.spi.ConnectorSession)49 Test (org.testng.annotations.Test)48 ConnectorTableHandle (com.facebook.presto.spi.ConnectorTableHandle)42 TestingConnectorSession (com.facebook.presto.testing.TestingConnectorSession)39 ConnectorId (com.facebook.presto.connector.ConnectorId)38 ColumnHandle (com.facebook.presto.spi.ColumnHandle)33 ConnectorTableMetadata (com.facebook.presto.spi.ConnectorTableMetadata)33 SchemaTableName (com.facebook.presto.spi.SchemaTableName)31 MaterializedResult (com.facebook.presto.testing.MaterializedResult)28 ColumnMetadata (com.facebook.presto.spi.ColumnMetadata)26 ConnectorOutputTableHandle (com.facebook.presto.spi.ConnectorOutputTableHandle)25 ConnectorInsertTableHandle (com.facebook.presto.spi.ConnectorInsertTableHandle)24 ConnectorPageSink (com.facebook.presto.spi.ConnectorPageSink)24 ConnectorPageSource (com.facebook.presto.spi.ConnectorPageSource)24 PrestoException (com.facebook.presto.spi.PrestoException)24 Path (org.apache.hadoop.fs.Path)24 Constraint (com.facebook.presto.spi.Constraint)23 SchemaTablePrefix (com.facebook.presto.spi.SchemaTablePrefix)23 MaterializedRow (com.facebook.presto.testing.MaterializedRow)23