Search in sources :

Example 11 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class AbstractTestHiveClient method doTestMismatchSchemaTable.

protected void doTestMismatchSchemaTable(SchemaTableName schemaTableName, HiveStorageFormat storageFormat, List<ColumnMetadata> tableBefore, MaterializedResult dataBefore, List<ColumnMetadata> tableAfter, MaterializedResult dataAfter) throws Exception {
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    doCreateEmptyTable(schemaTableName, storageFormat, tableBefore);
    // insert the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
        sink.appendPage(dataBefore.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments);
        transaction.commit();
    }
    // load the table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataBefore.getMaterializedRows());
        transaction.commit();
    }
    // alter the table schema
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        PrincipalPrivileges principalPrivileges = new PrincipalPrivileges(ImmutableMultimap.<String, HivePrivilegeInfo>builder().put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.SELECT, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.INSERT, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.UPDATE, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.DELETE, true)).build(), ImmutableMultimap.of());
        Table oldTable = transaction.getMetastore(schemaName).getTable(schemaName, tableName).get();
        HiveTypeTranslator hiveTypeTranslator = new HiveTypeTranslator();
        List<Column> dataColumns = tableAfter.stream().filter(columnMetadata -> !columnMetadata.getName().equals("ds")).map(columnMetadata -> new Column(columnMetadata.getName(), toHiveType(hiveTypeTranslator, columnMetadata.getType()), Optional.empty())).collect(toList());
        Table.Builder newTable = Table.builder(oldTable).setDataColumns(dataColumns);
        transaction.getMetastore(schemaName).replaceView(schemaName, tableName, newTable.build(), principalPrivileges);
        transaction.commit();
    }
    // load the altered table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataAfter.getMaterializedRows());
        transaction.commit();
    }
    // insertions to the partitions with type mismatches should fail
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
        sink.appendPage(dataAfter.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments);
        transaction.commit();
        fail("expected exception");
    } catch (PrestoException e) {
        // expected
        assertEquals(e.getErrorCode(), HIVE_PARTITION_SCHEMA_MISMATCH.toErrorCode());
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) TypeManager(com.facebook.presto.spi.type.TypeManager) Assertions.assertInstanceOf(io.airlift.testing.Assertions.assertInstanceOf) FileSystem(org.apache.hadoop.fs.FileSystem) TypeRegistry(com.facebook.presto.type.TypeRegistry) SqlDate(com.facebook.presto.spi.type.SqlDate) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) FileStatus(org.apache.hadoop.fs.FileStatus) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) Sets.difference(com.google.common.collect.Sets.difference) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) Map(java.util.Map) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) StandardTypes(com.facebook.presto.spi.type.StandardTypes) HiveWriteUtils.createDirectory(com.facebook.presto.hive.HiveWriteUtils.createDirectory) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) TINYINT(com.facebook.presto.spi.type.TinyintType.TINYINT) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ROW(com.facebook.presto.spi.type.StandardTypes.ROW) Domain(com.facebook.presto.spi.predicate.Domain) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) BridgingHiveMetastore(com.facebook.presto.hive.metastore.BridgingHiveMetastore) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) DOUBLE(com.facebook.presto.spi.type.DoubleType.DOUBLE) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) UTC_KEY(com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY) MapType(com.facebook.presto.type.MapType) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) ARRAY(com.facebook.presto.spi.type.StandardTypes.ARRAY) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) Type(com.facebook.presto.spi.type.Type) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) TIMESTAMP(com.facebook.presto.spi.type.TimestampType.TIMESTAMP) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) TestException(org.testng.TestException) AfterClass(org.testng.annotations.AfterClass) HYPER_LOG_LOG(com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Range(com.facebook.presto.spi.predicate.Range) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) UTC(org.joda.time.DateTimeZone.UTC) HostAndPort(com.google.common.net.HostAndPort) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) TableType(org.apache.hadoop.hive.metastore.TableType) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) TypeSignature.parseTypeSignature(com.facebook.presto.spi.type.TypeSignature.parseTypeSignature) ThriftHiveMetastore(com.facebook.presto.hive.metastore.ThriftHiveMetastore) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) ParquetHiveRecordCursor(com.facebook.presto.hive.parquet.ParquetHiveRecordCursor) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HIVE_METASTORE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR) Iterables.concat(com.google.common.collect.Iterables.concat) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) TypeSignatureParameter(com.facebook.presto.spi.type.TypeSignatureParameter) Path(org.apache.hadoop.fs.Path) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) NullableValue(com.facebook.presto.spi.predicate.NullableValue) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) RecordCursor(com.facebook.presto.spi.RecordCursor) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Varchars.isVarcharType(com.facebook.presto.spi.type.Varchars.isVarcharType) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) JsonCodec(io.airlift.json.JsonCodec) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Assert.assertNull(org.testng.Assert.assertNull) Logger(io.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) ArrayType(com.facebook.presto.type.ArrayType) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HadoopFileStatus(com.facebook.presto.hadoop.HadoopFileStatus) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) MAP(com.facebook.presto.spi.type.StandardTypes.MAP) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) OrcPageSource(com.facebook.presto.hive.orc.OrcPageSource) ImmutableList(com.google.common.collect.ImmutableList) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) ValueSet(com.facebook.presto.spi.predicate.ValueSet) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) NamedTypeSignature(com.facebook.presto.spi.type.NamedTypeSignature) Objects.requireNonNull(java.util.Objects.requireNonNull) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) VARBINARY(com.facebook.presto.spi.type.VarbinaryType.VARBINARY) ExecutorService(java.util.concurrent.ExecutorService) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) SMALLINT(com.facebook.presto.spi.type.SmallintType.SMALLINT) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) DATE(com.facebook.presto.spi.type.DateType.DATE) REAL(com.facebook.presto.spi.type.RealType.REAL) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) Assert.assertTrue(org.testng.Assert.assertTrue) ImmutableCollectors(com.facebook.presto.util.ImmutableCollectors) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Chars.isCharType(com.facebook.presto.spi.type.Chars.isCharType) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ColumnHandle(com.facebook.presto.spi.ColumnHandle) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) Table(com.facebook.presto.hive.metastore.Table) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Column(com.facebook.presto.hive.metastore.Column) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) Slice(io.airlift.slice.Slice) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) MaterializedResult(com.facebook.presto.testing.MaterializedResult)

Example 12 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class AbstractTestHiveClientS3 method testGetRecordsS3.

@Test
public void testGetRecordsS3() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle table = getTableHandle(metadata, tableS3);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, table).values());
        Map<String, Integer> columnIndex = indexColumns(columnHandles);
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, table, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty());
        HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle();
        assertEquals(layoutHandle.getPartitions().get().size(), 1);
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle);
        long sum = 0;
        for (ConnectorSplit split : getAllSplits(splitSource)) {
            try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) {
                MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
                for (MaterializedRow row : result) {
                    sum += (Long) row.getField(columnIndex.get("t_bigint"));
                }
            }
        }
        assertEquals(sum, 78300);
    }
}
Also used : HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) FileSystem(org.apache.hadoop.fs.FileSystem) TypeRegistry(com.facebook.presto.type.TypeRegistry) Test(org.testng.annotations.Test) AbstractTestHiveClient.getAllSplits(com.facebook.presto.hive.AbstractTestHiveClient.getAllSplits) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) Map(java.util.Map) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Path(org.apache.hadoop.fs.Path) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) AbstractTestHiveClient.filterNonHiddenColumnHandles(com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnHandles) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableMap(com.google.common.collect.ImmutableMap) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) UUID(java.util.UUID) String.format(java.lang.String.format) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) BridgingHiveMetastore(com.facebook.presto.hive.metastore.BridgingHiveMetastore) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) JsonCodec(io.airlift.json.JsonCodec) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) HiveTransaction(com.facebook.presto.hive.AbstractTestHiveClient.HiveTransaction) Database(com.facebook.presto.hive.metastore.Database) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ImmutableList(com.google.common.collect.ImmutableList) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) AbstractTestHiveClient.createTableProperties(com.facebook.presto.hive.AbstractTestHiveClient.createTableProperties) ExecutorService(java.util.concurrent.ExecutorService) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) AfterClass(org.testng.annotations.AfterClass) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) Transaction(com.facebook.presto.hive.AbstractTestHiveClient.Transaction) Throwables(com.google.common.base.Throwables) Constraint(com.facebook.presto.spi.Constraint) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) HostAndPort(com.google.common.net.HostAndPort) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) AbstractTestHiveClient.filterNonHiddenColumnMetadata(com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnMetadata) Assert.assertTrue(org.testng.Assert.assertTrue) HadoopFileStatus.isDirectory(com.facebook.presto.hadoop.HadoopFileStatus.isDirectory) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) ThriftHiveMetastore(com.facebook.presto.hive.metastore.ThriftHiveMetastore) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) HiveTransaction(com.facebook.presto.hive.AbstractTestHiveClient.HiveTransaction) Transaction(com.facebook.presto.hive.AbstractTestHiveClient.Transaction) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) MaterializedResult(com.facebook.presto.testing.MaterializedResult) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) MaterializedRow(com.facebook.presto.testing.MaterializedRow) Test(org.testng.annotations.Test)

Example 13 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class AbstractTestHiveClientS3 method indexColumns.

private static ImmutableMap<String, Integer> indexColumns(List<ColumnHandle> columnHandles) {
    ImmutableMap.Builder<String, Integer> index = ImmutableMap.builder();
    int i = 0;
    for (ColumnHandle columnHandle : columnHandles) {
        HiveColumnHandle hiveColumnHandle = (HiveColumnHandle) columnHandle;
        index.put(hiveColumnHandle.getName(), i);
        i++;
    }
    return index.build();
}
Also used : ColumnHandle(com.facebook.presto.spi.ColumnHandle) ImmutableMap(com.google.common.collect.ImmutableMap) Constraint(com.facebook.presto.spi.Constraint)

Example 14 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class LookupSymbolResolver method getValue.

@Override
public Object getValue(Symbol symbol) {
    ColumnHandle column = assignments.get(symbol);
    checkArgument(column != null, "Missing column assignment for %s", symbol);
    if (!bindings.containsKey(column)) {
        return symbol.toSymbolReference();
    }
    return bindings.get(column).getValue();
}
Also used : ColumnHandle(com.facebook.presto.spi.ColumnHandle)

Example 15 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class LocalQueryRunner method createTableScanOperator.

public OperatorFactory createTableScanOperator(Session session, int operatorId, PlanNodeId planNodeId, String tableName, String... columnNames) {
    checkArgument(session.getCatalog().isPresent(), "catalog not set");
    checkArgument(session.getSchema().isPresent(), "schema not set");
    // look up the table
    QualifiedObjectName qualifiedTableName = new QualifiedObjectName(session.getCatalog().get(), session.getSchema().get(), tableName);
    TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).orElse(null);
    checkArgument(tableHandle != null, "Table %s does not exist", qualifiedTableName);
    // lookup the columns
    Map<String, ColumnHandle> allColumnHandles = metadata.getColumnHandles(session, tableHandle);
    ImmutableList.Builder<ColumnHandle> columnHandlesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Type> columnTypesBuilder = ImmutableList.builder();
    for (String columnName : columnNames) {
        ColumnHandle columnHandle = allColumnHandles.get(columnName);
        checkArgument(columnHandle != null, "Table %s does not have a column %s", tableName, columnName);
        columnHandlesBuilder.add(columnHandle);
        ColumnMetadata columnMetadata = metadata.getColumnMetadata(session, tableHandle, columnHandle);
        columnTypesBuilder.add(columnMetadata.getType());
    }
    List<ColumnHandle> columnHandles = columnHandlesBuilder.build();
    List<Type> columnTypes = columnTypesBuilder.build();
    // get the split for this table
    List<TableLayoutResult> layouts = metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty());
    Split split = getLocalQuerySplit(session, layouts.get(0).getLayout().getHandle());
    return new OperatorFactory() {

        @Override
        public List<Type> getTypes() {
            return columnTypes;
        }

        @Override
        public Operator createOperator(DriverContext driverContext) {
            OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, "BenchmarkSource");
            ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, columnHandles);
            return new PageSourceOperator(pageSource, columnTypes, operatorContext);
        }

        @Override
        public void close() {
        }

        @Override
        public OperatorFactory duplicate() {
            throw new UnsupportedOperationException();
        }
    };
}
Also used : ColumnHandle(com.facebook.presto.spi.ColumnHandle) DriverContext(com.facebook.presto.operator.DriverContext) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) ImmutableList(com.google.common.collect.ImmutableList) TableLayoutResult(com.facebook.presto.metadata.TableLayoutResult) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) QualifiedObjectName(com.facebook.presto.metadata.QualifiedObjectName) PageSourceOperator(com.facebook.presto.operator.PageSourceOperator) Type(com.facebook.presto.spi.type.Type) OperatorFactory(com.facebook.presto.operator.OperatorFactory) OperatorContext(com.facebook.presto.operator.OperatorContext) TableHandle(com.facebook.presto.metadata.TableHandle) ScheduledSplit(com.facebook.presto.ScheduledSplit) Split(com.facebook.presto.metadata.Split)

Aggregations

ColumnHandle (com.facebook.presto.spi.ColumnHandle)243 ImmutableList (com.google.common.collect.ImmutableList)90 ImmutableMap (com.google.common.collect.ImmutableMap)81 Test (org.testng.annotations.Test)81 ConnectorSession (com.facebook.presto.spi.ConnectorSession)80 ConnectorTableHandle (com.facebook.presto.spi.ConnectorTableHandle)70 Constraint (com.facebook.presto.spi.Constraint)66 Map (java.util.Map)65 SchemaTableName (com.facebook.presto.spi.SchemaTableName)60 Optional (java.util.Optional)57 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)54 List (java.util.List)54 Objects.requireNonNull (java.util.Objects.requireNonNull)53 ColumnMetadata (com.facebook.presto.spi.ColumnMetadata)47 ConnectorMetadata (com.facebook.presto.spi.connector.ConnectorMetadata)47 ConnectorTableLayoutHandle (com.facebook.presto.spi.ConnectorTableLayoutHandle)46 ImmutableSet (com.google.common.collect.ImmutableSet)46 TestingConnectorSession (com.facebook.presto.testing.TestingConnectorSession)43 ConnectorTableMetadata (com.facebook.presto.spi.ConnectorTableMetadata)42 PrestoException (com.facebook.presto.spi.PrestoException)42