Search in sources :

Example 11 with ConnectorTableLayoutHandle

use of com.facebook.presto.spi.ConnectorTableLayoutHandle in project presto by prestodb.

the class AbstractTestHiveClient method getAllSplits.

private List<ConnectorSplit> getAllSplits(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) throws InterruptedException {
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain, bindings -> true), Optional.empty());
        ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle();
        return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle));
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) TypeManager(com.facebook.presto.spi.type.TypeManager) Assertions.assertInstanceOf(io.airlift.testing.Assertions.assertInstanceOf) FileSystem(org.apache.hadoop.fs.FileSystem) TypeRegistry(com.facebook.presto.type.TypeRegistry) SqlDate(com.facebook.presto.spi.type.SqlDate) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) FileStatus(org.apache.hadoop.fs.FileStatus) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) Sets.difference(com.google.common.collect.Sets.difference) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) Map(java.util.Map) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) StandardTypes(com.facebook.presto.spi.type.StandardTypes) HiveWriteUtils.createDirectory(com.facebook.presto.hive.HiveWriteUtils.createDirectory) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) TINYINT(com.facebook.presto.spi.type.TinyintType.TINYINT) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ROW(com.facebook.presto.spi.type.StandardTypes.ROW) Domain(com.facebook.presto.spi.predicate.Domain) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) BridgingHiveMetastore(com.facebook.presto.hive.metastore.BridgingHiveMetastore) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) DOUBLE(com.facebook.presto.spi.type.DoubleType.DOUBLE) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) UTC_KEY(com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY) MapType(com.facebook.presto.type.MapType) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) ARRAY(com.facebook.presto.spi.type.StandardTypes.ARRAY) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) Type(com.facebook.presto.spi.type.Type) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) TIMESTAMP(com.facebook.presto.spi.type.TimestampType.TIMESTAMP) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) TestException(org.testng.TestException) AfterClass(org.testng.annotations.AfterClass) HYPER_LOG_LOG(com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Range(com.facebook.presto.spi.predicate.Range) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) UTC(org.joda.time.DateTimeZone.UTC) HostAndPort(com.google.common.net.HostAndPort) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) TableType(org.apache.hadoop.hive.metastore.TableType) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) TypeSignature.parseTypeSignature(com.facebook.presto.spi.type.TypeSignature.parseTypeSignature) ThriftHiveMetastore(com.facebook.presto.hive.metastore.ThriftHiveMetastore) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) ParquetHiveRecordCursor(com.facebook.presto.hive.parquet.ParquetHiveRecordCursor) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HIVE_METASTORE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR) Iterables.concat(com.google.common.collect.Iterables.concat) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) TypeSignatureParameter(com.facebook.presto.spi.type.TypeSignatureParameter) Path(org.apache.hadoop.fs.Path) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) NullableValue(com.facebook.presto.spi.predicate.NullableValue) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) RecordCursor(com.facebook.presto.spi.RecordCursor) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Varchars.isVarcharType(com.facebook.presto.spi.type.Varchars.isVarcharType) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) JsonCodec(io.airlift.json.JsonCodec) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Assert.assertNull(org.testng.Assert.assertNull) Logger(io.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) ArrayType(com.facebook.presto.type.ArrayType) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HadoopFileStatus(com.facebook.presto.hadoop.HadoopFileStatus) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) MAP(com.facebook.presto.spi.type.StandardTypes.MAP) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) OrcPageSource(com.facebook.presto.hive.orc.OrcPageSource) ImmutableList(com.google.common.collect.ImmutableList) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) ValueSet(com.facebook.presto.spi.predicate.ValueSet) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) NamedTypeSignature(com.facebook.presto.spi.type.NamedTypeSignature) Objects.requireNonNull(java.util.Objects.requireNonNull) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) VARBINARY(com.facebook.presto.spi.type.VarbinaryType.VARBINARY) ExecutorService(java.util.concurrent.ExecutorService) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) SMALLINT(com.facebook.presto.spi.type.SmallintType.SMALLINT) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) DATE(com.facebook.presto.spi.type.DateType.DATE) REAL(com.facebook.presto.spi.type.RealType.REAL) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) Assert.assertTrue(org.testng.Assert.assertTrue) ImmutableCollectors(com.facebook.presto.util.ImmutableCollectors) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Chars.isCharType(com.facebook.presto.spi.type.Chars.isCharType) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata)

Aggregations

ConnectorSplit (com.facebook.presto.spi.ConnectorSplit)11 ConnectorTableLayoutHandle (com.facebook.presto.spi.ConnectorTableLayoutHandle)11 ColumnHandle (com.facebook.presto.spi.ColumnHandle)10 ConnectorSplitSource (com.facebook.presto.spi.ConnectorSplitSource)10 ConnectorTransactionHandle (com.facebook.presto.spi.connector.ConnectorTransactionHandle)8 Test (org.testng.annotations.Test)8 ConnectorSession (com.facebook.presto.spi.ConnectorSession)7 ConnectorSplitManager (com.facebook.presto.spi.connector.ConnectorSplitManager)7 NullableValue (com.facebook.presto.spi.predicate.NullableValue)7 ImmutableList (com.google.common.collect.ImmutableList)7 ImmutableMap (com.google.common.collect.ImmutableMap)7 List (java.util.List)7 Objects.requireNonNull (java.util.Objects.requireNonNull)7 GroupByHashPageIndexerFactory (com.facebook.presto.GroupByHashPageIndexerFactory)5 HadoopFileStatus (com.facebook.presto.hadoop.HadoopFileStatus)5 COMMIT (com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT)5 ROLLBACK_AFTER_APPEND_PAGE (com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE)5 ROLLBACK_AFTER_BEGIN_INSERT (com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT)5 ROLLBACK_AFTER_DELETE (com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE)5 ROLLBACK_AFTER_FINISH_INSERT (com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT)5