Search in sources :

Example 1 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method doInsertIntoNewPartition.

private void doInsertIntoNewPartition(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception {
    // creating the table
    doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED);
    // insert the data
    String queryId = insertData(tableName, CREATE_TABLE_PARTITIONED_DATA);
    Set<String> existingFiles;
    try (Transaction transaction = newTransaction()) {
        // verify partitions were created
        List<String> partitionNames = transaction.getMetastore(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available"));
        assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream().map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)).collect(toList()));
        // verify the node versions in partitions
        Map<String, Optional<Partition>> partitions = getMetastoreClient(tableName.getSchemaName()).getPartitionsByNames(tableName.getSchemaName(), tableName.getTableName(), partitionNames);
        assertEquals(partitions.size(), partitionNames.size());
        for (String partitionName : partitionNames) {
            Partition partition = partitions.get(partitionName).get();
            assertEquals(partition.getParameters().get(HiveMetadata.PRESTO_VERSION_NAME), TEST_SERVER_VERSION);
            assertEquals(partition.getParameters().get(HiveMetadata.PRESTO_QUERY_ID_NAME), queryId);
        }
        // load the new table
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        // verify the data
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
        assertEqualsIgnoreOrder(result.getMaterializedRows(), CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows());
        // test rollback
        existingFiles = listAllDataFiles(transaction, tableName.getSchemaName(), tableName.getTableName());
        assertFalse(existingFiles.isEmpty());
    }
    Path stagingPathRoot;
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        // "stage" insert data
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        stagingPathRoot = getStagingPathRoot(insertTableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
        sink.appendPage(CREATE_TABLE_PARTITIONED_DATA_2ND.toPage());
        getFutureValue(sink.finish());
        // verify we did not modify the table directory
        assertEquals(listAllDataFiles(transaction, tableName.getSchemaName(), tableName.getTableName()), existingFiles);
        // verify all temp files start with the unique prefix
        Set<String> tempFiles = listAllDataFiles(getStagingPathRoot(insertTableHandle));
        assertTrue(!tempFiles.isEmpty());
        for (String filePath : tempFiles) {
            assertTrue(new Path(filePath).getName().startsWith(getFilePrefix(insertTableHandle)));
        }
        // rollback insert
        transaction.rollback();
    }
    // verify the data is unchanged
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, newSession(), TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows());
        // verify we did not modify the table directory
        assertEquals(listAllDataFiles(transaction, tableName.getSchemaName(), tableName.getTableName()), existingFiles);
        // verify temp directory is empty
        assertTrue(listAllDataFiles(stagingPathRoot).isEmpty());
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Partition(com.facebook.presto.hive.metastore.Partition) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Optional(java.util.Optional) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) MaterializedResult(com.facebook.presto.testing.MaterializedResult) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink)

Example 2 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method doTestMismatchSchemaTable.

protected void doTestMismatchSchemaTable(SchemaTableName schemaTableName, HiveStorageFormat storageFormat, List<ColumnMetadata> tableBefore, MaterializedResult dataBefore, List<ColumnMetadata> tableAfter, MaterializedResult dataAfter) throws Exception {
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    doCreateEmptyTable(schemaTableName, storageFormat, tableBefore);
    // insert the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
        sink.appendPage(dataBefore.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments);
        transaction.commit();
    }
    // load the table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataBefore.getMaterializedRows());
        transaction.commit();
    }
    // alter the table schema
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        PrincipalPrivileges principalPrivileges = new PrincipalPrivileges(ImmutableMultimap.<String, HivePrivilegeInfo>builder().put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.SELECT, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.INSERT, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.UPDATE, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.DELETE, true)).build(), ImmutableMultimap.of());
        Table oldTable = transaction.getMetastore(schemaName).getTable(schemaName, tableName).get();
        HiveTypeTranslator hiveTypeTranslator = new HiveTypeTranslator();
        List<Column> dataColumns = tableAfter.stream().filter(columnMetadata -> !columnMetadata.getName().equals("ds")).map(columnMetadata -> new Column(columnMetadata.getName(), toHiveType(hiveTypeTranslator, columnMetadata.getType()), Optional.empty())).collect(toList());
        Table.Builder newTable = Table.builder(oldTable).setDataColumns(dataColumns);
        transaction.getMetastore(schemaName).replaceView(schemaName, tableName, newTable.build(), principalPrivileges);
        transaction.commit();
    }
    // load the altered table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataAfter.getMaterializedRows());
        transaction.commit();
    }
    // insertions to the partitions with type mismatches should fail
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
        sink.appendPage(dataAfter.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments);
        transaction.commit();
        fail("expected exception");
    } catch (PrestoException e) {
        // expected
        assertEquals(e.getErrorCode(), HIVE_PARTITION_SCHEMA_MISMATCH.toErrorCode());
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) TypeManager(com.facebook.presto.spi.type.TypeManager) Assertions.assertInstanceOf(io.airlift.testing.Assertions.assertInstanceOf) FileSystem(org.apache.hadoop.fs.FileSystem) TypeRegistry(com.facebook.presto.type.TypeRegistry) SqlDate(com.facebook.presto.spi.type.SqlDate) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) FileStatus(org.apache.hadoop.fs.FileStatus) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) Sets.difference(com.google.common.collect.Sets.difference) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) Map(java.util.Map) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) StandardTypes(com.facebook.presto.spi.type.StandardTypes) HiveWriteUtils.createDirectory(com.facebook.presto.hive.HiveWriteUtils.createDirectory) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) TINYINT(com.facebook.presto.spi.type.TinyintType.TINYINT) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ROW(com.facebook.presto.spi.type.StandardTypes.ROW) Domain(com.facebook.presto.spi.predicate.Domain) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) BridgingHiveMetastore(com.facebook.presto.hive.metastore.BridgingHiveMetastore) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) DOUBLE(com.facebook.presto.spi.type.DoubleType.DOUBLE) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) UTC_KEY(com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY) MapType(com.facebook.presto.type.MapType) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) ARRAY(com.facebook.presto.spi.type.StandardTypes.ARRAY) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) Type(com.facebook.presto.spi.type.Type) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) TIMESTAMP(com.facebook.presto.spi.type.TimestampType.TIMESTAMP) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) TestException(org.testng.TestException) AfterClass(org.testng.annotations.AfterClass) HYPER_LOG_LOG(com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Range(com.facebook.presto.spi.predicate.Range) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) UTC(org.joda.time.DateTimeZone.UTC) HostAndPort(com.google.common.net.HostAndPort) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) TableType(org.apache.hadoop.hive.metastore.TableType) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) TypeSignature.parseTypeSignature(com.facebook.presto.spi.type.TypeSignature.parseTypeSignature) ThriftHiveMetastore(com.facebook.presto.hive.metastore.ThriftHiveMetastore) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) ParquetHiveRecordCursor(com.facebook.presto.hive.parquet.ParquetHiveRecordCursor) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HIVE_METASTORE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR) Iterables.concat(com.google.common.collect.Iterables.concat) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) TypeSignatureParameter(com.facebook.presto.spi.type.TypeSignatureParameter) Path(org.apache.hadoop.fs.Path) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) NullableValue(com.facebook.presto.spi.predicate.NullableValue) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) RecordCursor(com.facebook.presto.spi.RecordCursor) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Varchars.isVarcharType(com.facebook.presto.spi.type.Varchars.isVarcharType) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) JsonCodec(io.airlift.json.JsonCodec) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Assert.assertNull(org.testng.Assert.assertNull) Logger(io.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) ArrayType(com.facebook.presto.type.ArrayType) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HadoopFileStatus(com.facebook.presto.hadoop.HadoopFileStatus) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) MAP(com.facebook.presto.spi.type.StandardTypes.MAP) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) OrcPageSource(com.facebook.presto.hive.orc.OrcPageSource) ImmutableList(com.google.common.collect.ImmutableList) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) ValueSet(com.facebook.presto.spi.predicate.ValueSet) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) NamedTypeSignature(com.facebook.presto.spi.type.NamedTypeSignature) Objects.requireNonNull(java.util.Objects.requireNonNull) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) VARBINARY(com.facebook.presto.spi.type.VarbinaryType.VARBINARY) ExecutorService(java.util.concurrent.ExecutorService) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) SMALLINT(com.facebook.presto.spi.type.SmallintType.SMALLINT) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) DATE(com.facebook.presto.spi.type.DateType.DATE) REAL(com.facebook.presto.spi.type.RealType.REAL) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) Assert.assertTrue(org.testng.Assert.assertTrue) ImmutableCollectors(com.facebook.presto.util.ImmutableCollectors) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Chars.isCharType(com.facebook.presto.spi.type.Chars.isCharType) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ColumnHandle(com.facebook.presto.spi.ColumnHandle) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) Table(com.facebook.presto.hive.metastore.Table) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Column(com.facebook.presto.hive.metastore.Column) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) Slice(io.airlift.slice.Slice) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) MaterializedResult(com.facebook.presto.testing.MaterializedResult)

Example 3 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class TestRaptorConnector method assertSplitShard.

private void assertSplitShard(Type temporalType, String min, String max, String userTimeZone, int expectedSplits) throws Exception {
    ConnectorSession session = new TestingConnectorSession("user", Optional.of("test"), Optional.empty(), getTimeZoneKey(userTimeZone), ENGLISH, System.currentTimeMillis(), new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties(), ImmutableMap.of(), true, Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of());
    ConnectorTransactionHandle transaction = connector.beginTransaction(READ_COMMITTED, false);
    connector.getMetadata(transaction).createTable(SESSION, new ConnectorTableMetadata(new SchemaTableName("test", "test"), ImmutableList.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("time", temporalType)), ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "time", TABLE_SUPPORTS_DELTA_DELETE, false)), false);
    connector.commit(transaction);
    ConnectorTransactionHandle txn1 = connector.beginTransaction(READ_COMMITTED, false);
    ConnectorTableHandle handle1 = getTableHandle(connector.getMetadata(txn1), "test");
    ConnectorInsertTableHandle insertTableHandle = connector.getMetadata(txn1).beginInsert(session, handle1);
    ConnectorPageSink raptorPageSink = connector.getPageSinkProvider().createPageSink(txn1, session, insertTableHandle, PageSinkContext.defaultContext());
    Object timestamp1 = null;
    Object timestamp2 = null;
    if (temporalType.equals(TIMESTAMP)) {
        timestamp1 = new SqlTimestamp(parseTimestampLiteral(getTimeZoneKey(userTimeZone), min), getTimeZoneKey(userTimeZone));
        timestamp2 = new SqlTimestamp(parseTimestampLiteral(getTimeZoneKey(userTimeZone), max), getTimeZoneKey(userTimeZone));
    } else if (temporalType.equals(DATE)) {
        timestamp1 = new SqlDate(parseDate(min));
        timestamp2 = new SqlDate(parseDate(max));
    }
    Page inputPage = MaterializedResult.resultBuilder(session, ImmutableList.of(BIGINT, temporalType)).row(1L, timestamp1).row(2L, timestamp2).build().toPage();
    raptorPageSink.appendPage(inputPage);
    Collection<Slice> shards = raptorPageSink.finish().get();
    assertEquals(shards.size(), expectedSplits);
    connector.getMetadata(txn1).dropTable(session, handle1);
    connector.commit(txn1);
}
Also used : ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) SqlTimestamp(com.facebook.presto.common.type.SqlTimestamp) Page(com.facebook.presto.common.Page) SchemaTableName(com.facebook.presto.spi.SchemaTableName) StorageManagerConfig(com.facebook.presto.raptor.storage.StorageManagerConfig) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Slice(io.airlift.slice.Slice) SqlDate(com.facebook.presto.common.type.SqlDate) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata)

Example 4 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class TestRaptorMetadata method testTransactionInsert.

@Test
public void testTransactionInsert() {
    // creating a table allocates a transaction
    long transactionId = 1;
    metadata.createTable(SESSION, getOrdersTable(), false);
    assertTrue(transactionSuccessful(transactionId));
    // start insert
    transactionId++;
    ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS);
    ConnectorInsertTableHandle insertHandle = metadata.beginInsert(SESSION, tableHandle);
    // transaction is in progress
    assertTrue(transactionExists(transactionId));
    assertNull(transactionSuccessful(transactionId));
    // commit insert
    metadata.finishInsert(SESSION, insertHandle, ImmutableList.of(), ImmutableList.of());
    assertTrue(transactionExists(transactionId));
    assertTrue(transactionSuccessful(transactionId));
}
Also used : ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Test(org.testng.annotations.Test)

Example 5 with ConnectorInsertTableHandle

use of com.facebook.presto.spi.ConnectorInsertTableHandle in project presto by prestodb.

the class RaptorMetadata method beginInsert.

@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) {
    RaptorTableHandle handle = (RaptorTableHandle) tableHandle;
    long tableId = handle.getTableId();
    ImmutableList.Builder<RaptorColumnHandle> columnHandles = ImmutableList.builder();
    ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
    for (TableColumn column : dao.listTableColumns(tableId)) {
        columnHandles.add(new RaptorColumnHandle(connectorId, column.getColumnName(), column.getColumnId(), column.getDataType()));
        columnTypes.add(column.getDataType());
    }
    long transactionId = shardManager.beginTransaction();
    setTransactionId(transactionId);
    Optional<String> externalBatchId = getExternalBatchId(session);
    List<RaptorColumnHandle> sortColumnHandles = getSortColumnHandles(tableId);
    List<RaptorColumnHandle> bucketColumnHandles = getBucketColumnHandles(tableId);
    Optional<RaptorColumnHandle> temporalColumnHandle = Optional.ofNullable(dao.getTemporalColumnId(tableId)).map(temporalColumnId -> getOnlyElement(columnHandles.build().stream().filter(columnHandle -> columnHandle.getColumnId() == temporalColumnId).collect(toList())));
    return new RaptorInsertTableHandle(connectorId, transactionId, tableId, columnHandles.build(), columnTypes.build(), externalBatchId, sortColumnHandles, nCopies(sortColumnHandles.size(), ASC_NULLS_FIRST), handle.getBucketCount(), bucketColumnHandles, temporalColumnHandle);
}
Also used : ShardDeleteDelta(com.facebook.presto.raptor.metadata.ShardDeleteDelta) SHARD_UUID_COLUMN_TYPE(com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE) NOT_FOUND(com.facebook.presto.spi.StandardErrorCode.NOT_FOUND) ComputedStatistics(com.facebook.presto.spi.statistics.ComputedStatistics) DatabaseUtil.runIgnoringConstraintViolation(com.facebook.presto.raptor.util.DatabaseUtil.runIgnoringConstraintViolation) Map(java.util.Map) ShardManager(com.facebook.presto.raptor.metadata.ShardManager) SystemTable(com.facebook.presto.spi.SystemTable) ColumnRangesSystemTable.getSourceTable(com.facebook.presto.raptor.systemtables.ColumnRangesSystemTable.getSourceTable) INVALID_TABLE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) RaptorTableProperties.getSortColumns(com.facebook.presto.raptor.RaptorTableProperties.getSortColumns) ColumnInfo(com.facebook.presto.raptor.metadata.ColumnInfo) ViewResult(com.facebook.presto.raptor.metadata.ViewResult) Set(java.util.Set) JsonCodec.jsonCodec(com.facebook.airlift.json.JsonCodec.jsonCodec) ConnectorSession(com.facebook.presto.spi.ConnectorSession) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) RaptorColumnHandle.shardUuidColumnHandle(com.facebook.presto.raptor.RaptorColumnHandle.shardUuidColumnHandle) ConnectorPartitioningHandle(com.facebook.presto.spi.connector.ConnectorPartitioningHandle) Slice(io.airlift.slice.Slice) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) ColumnRangesSystemTable(com.facebook.presto.raptor.systemtables.ColumnRangesSystemTable) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) DATE(com.facebook.presto.common.type.DateType.DATE) Multimaps(com.google.common.collect.Multimaps) ArrayList(java.util.ArrayList) OptionalLong(java.util.OptionalLong) BUCKET_NUMBER_COLUMN_NAME(com.facebook.presto.raptor.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME) RaptorSessionProperties.getExternalBatchId(com.facebook.presto.raptor.RaptorSessionProperties.getExternalBatchId) DatabaseUtil.daoTransaction(com.facebook.presto.raptor.util.DatabaseUtil.daoTransaction) Nullable(javax.annotation.Nullable) RaptorTableProperties.isOrganized(com.facebook.presto.raptor.RaptorTableProperties.isOrganized) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) Distribution(com.facebook.presto.raptor.metadata.Distribution) BUCKETED_ON_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.BUCKETED_ON_PROPERTY) ORDERING_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.ORDERING_PROPERTY) RaptorSessionProperties.getOneSplitPerBucketThreshold(com.facebook.presto.raptor.RaptorSessionProperties.getOneSplitPerBucketThreshold) Constraint(com.facebook.presto.spi.Constraint) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) RaptorTableProperties.getBucketColumns(com.facebook.presto.raptor.RaptorTableProperties.getBucketColumns) MetadataDao(com.facebook.presto.raptor.metadata.MetadataDao) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) RAPTOR_ERROR(com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) TreeMap(java.util.TreeMap) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) BUCKET_COUNT_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.BUCKET_COUNT_PROPERTY) DeltaInfoPair(com.facebook.presto.raptor.metadata.DeltaInfoPair) RaptorTableProperties.isTableSupportsDeltaDelete(com.facebook.presto.raptor.RaptorTableProperties.isTableSupportsDeltaDelete) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) JsonCodec(com.facebook.airlift.json.JsonCodec) ShardDelta(com.facebook.presto.raptor.metadata.ShardDelta) IDBI(org.skife.jdbi.v2.IDBI) RaptorBucketFunction.validateBucketType(com.facebook.presto.raptor.RaptorBucketFunction.validateBucketType) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ConnectorTablePartitioning(com.facebook.presto.spi.ConnectorTablePartitioning) TableColumn(com.facebook.presto.raptor.metadata.TableColumn) DatabaseUtil.runTransaction(com.facebook.presto.raptor.util.DatabaseUtil.runTransaction) ImmutableSet(com.google.common.collect.ImmutableSet) RaptorTableProperties.getDistributionName(com.facebook.presto.raptor.RaptorTableProperties.getDistributionName) ImmutableMap(com.google.common.collect.ImmutableMap) DISTRIBUTION_NAME_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.DISTRIBUTION_NAME_PROPERTY) Collections.nCopies(java.util.Collections.nCopies) Collection(java.util.Collection) RaptorTableProperties.getTemporalColumn(com.facebook.presto.raptor.RaptorTableProperties.getTemporalColumn) UUID(java.util.UUID) RaptorColumnHandle.isHiddenColumn(com.facebook.presto.raptor.RaptorColumnHandle.isHiddenColumn) Collectors(java.util.stream.Collectors) RaptorColumnHandle.bucketNumberColumnHandle(com.facebook.presto.raptor.RaptorColumnHandle.bucketNumberColumnHandle) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) RaptorTableProperties.getBucketCount(com.facebook.presto.raptor.RaptorTableProperties.getBucketCount) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) TEMPORAL_COLUMN_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY) Optional(java.util.Optional) StorageTypeConverter(com.facebook.presto.raptor.storage.StorageTypeConverter) SortedMap(java.util.SortedMap) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Logger(com.facebook.airlift.log.Logger) ORGANIZED_PROPERTY(com.facebook.presto.raptor.RaptorTableProperties.ORGANIZED_PROPERTY) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) TABLE_SUPPORTS_DELTA_DELETE(com.facebook.presto.raptor.RaptorTableProperties.TABLE_SUPPORTS_DELTA_DELETE) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) AtomicReference(java.util.concurrent.atomic.AtomicReference) Collectors.toCollection(java.util.stream.Collectors.toCollection) ImmutableList(com.google.common.collect.ImmutableList) ALREADY_EXISTS(com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS) TypeManager(com.facebook.presto.common.type.TypeManager) SHARD_UUID_COLUMN_NAME(com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_NAME) Objects.requireNonNull(java.util.Objects.requireNonNull) DatabaseUtil.onDemandDao(com.facebook.presto.raptor.util.DatabaseUtil.onDemandDao) Type(com.facebook.presto.common.type.Type) Table(com.facebook.presto.raptor.metadata.Table) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorOutputMetadata(com.facebook.presto.spi.connector.ConnectorOutputMetadata) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) ASC_NULLS_FIRST(com.facebook.presto.common.block.SortOrder.ASC_NULLS_FIRST) Maps(com.google.common.collect.Maps) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) ShardInfo(com.facebook.presto.raptor.metadata.ShardInfo) LongConsumer(java.util.function.LongConsumer) Collectors.toList(java.util.stream.Collectors.toList) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Handle(org.skife.jdbi.v2.Handle) RaptorColumnHandle.shardRowIdHandle(com.facebook.presto.raptor.RaptorColumnHandle.shardRowIdHandle) ImmutableList(com.google.common.collect.ImmutableList) TableColumn(com.facebook.presto.raptor.metadata.TableColumn) RaptorBucketFunction.validateBucketType(com.facebook.presto.raptor.RaptorBucketFunction.validateBucketType) Type(com.facebook.presto.common.type.Type)

Aggregations

ConnectorInsertTableHandle (com.facebook.presto.spi.ConnectorInsertTableHandle)13 ConnectorTableHandle (com.facebook.presto.spi.ConnectorTableHandle)11 ConnectorSession (com.facebook.presto.spi.ConnectorSession)10 ConnectorMetadata (com.facebook.presto.spi.connector.ConnectorMetadata)10 ColumnHandle (com.facebook.presto.spi.ColumnHandle)9 ConnectorTableMetadata (com.facebook.presto.spi.ConnectorTableMetadata)8 Constraint (com.facebook.presto.spi.Constraint)8 PrestoException (com.facebook.presto.spi.PrestoException)8 ColumnMetadata (com.facebook.presto.spi.ColumnMetadata)7 ConnectorPageSink (com.facebook.presto.spi.ConnectorPageSink)7 SchemaTableName (com.facebook.presto.spi.SchemaTableName)7 ConnectorTransactionHandle (com.facebook.presto.spi.connector.ConnectorTransactionHandle)7 TestingConnectorSession (com.facebook.presto.testing.TestingConnectorSession)7 Slice (io.airlift.slice.Slice)7 Optional (java.util.Optional)7 ConnectorOutputTableHandle (com.facebook.presto.spi.ConnectorOutputTableHandle)6 ConnectorTableLayout (com.facebook.presto.spi.ConnectorTableLayout)6 ConnectorTableLayoutHandle (com.facebook.presto.spi.ConnectorTableLayoutHandle)6 ConnectorTableLayoutResult (com.facebook.presto.spi.ConnectorTableLayoutResult)6 SchemaTablePrefix (com.facebook.presto.spi.SchemaTablePrefix)6