Search in sources :

Example 1 with ConnectorOutputTableHandle

use of io.prestosql.spi.connector.ConnectorOutputTableHandle in project hetu-core by openlookeng.

the class TestMemoryMetadata method testRenameTable.

@Test
public void testRenameTable() {
    SchemaTableName tableName = new SchemaTableName("test_schema", "test_table_to_be_renamed");
    metadata.createSchema(SESSION, "test_schema", ImmutableMap.of());
    ConnectorOutputTableHandle table = metadata.beginCreateTable(SESSION, new ConnectorTableMetadata(tableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty());
    metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of());
    // rename table to schema which does not exist
    SchemaTableName invalidSchemaTableName = new SchemaTableName("test_schema_not_exist", "test_table_renamed");
    ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, tableName);
    Throwable throwable = expectThrows(SchemaNotFoundException.class, () -> metadata.renameTable(SESSION, tableHandle, invalidSchemaTableName));
    assertTrue(throwable.getMessage().equals("Schema test_schema_not_exist not found"));
    // rename table to same schema
    SchemaTableName sameSchemaTableName = new SchemaTableName("test_schema", "test_renamed");
    metadata.renameTable(SESSION, metadata.getTableHandle(SESSION, tableName), sameSchemaTableName);
    assertEquals(metadata.listTables(SESSION, Optional.of("test_schema")), ImmutableList.of(sameSchemaTableName));
}
Also used : ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) Test(org.testng.annotations.Test)

Example 2 with ConnectorOutputTableHandle

use of io.prestosql.spi.connector.ConnectorOutputTableHandle in project hetu-core by openlookeng.

the class TestMemoryMetadata method tableIsCreatedAfterCommits.

@Test
public void tableIsCreatedAfterCommits() {
    assertNoTables();
    SchemaTableName schemaTableName = new SchemaTableName("default", "temp_table");
    ConnectorOutputTableHandle table = metadata.beginCreateTable(SESSION, new ConnectorTableMetadata(schemaTableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty());
    metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of());
    List<SchemaTableName> tables = metadata.listTables(SESSION, Optional.empty());
    assertEquals(tables.size(), 1, "Expected only one table");
    assertEquals(tables.get(0).getTableName(), "temp_table", "Expected table with name 'temp_table'");
}
Also used : ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Test(org.testng.annotations.Test)

Example 3 with ConnectorOutputTableHandle

use of io.prestosql.spi.connector.ConnectorOutputTableHandle in project hetu-core by openlookeng.

the class MemoryMetadata method createTable.

@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    ConnectorOutputTableHandle outputTableHandle = beginCreateTable(session, tableMetadata, Optional.empty());
    finishCreateTable(session, outputTableHandle, ImmutableList.of(), ImmutableList.of());
}
Also used : ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle)

Example 4 with ConnectorOutputTableHandle

use of io.prestosql.spi.connector.ConnectorOutputTableHandle in project hetu-core by openlookeng.

the class AbstractTestHiveFileSystem method createTable.

private void createTable(SchemaTableName tableName, HiveStorageFormat storageFormat) throws Exception {
    List<ColumnMetadata> columns = ImmutableList.<ColumnMetadata>builder().add(new ColumnMetadata("id", BIGINT)).build();
    MaterializedResult data = MaterializedResult.resultBuilder(newSession(), BIGINT).row(1L).row(3L).row(2L).build();
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // begin creating the table
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, columns, createTableProperties(storageFormat));
        ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty());
        // write the records
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle);
        sink.appendPage(data.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        // commit the table
        metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of());
        transaction.commit();
        // Hack to work around the metastore not being configured for S3 or other FS.
        // The metastore tries to validate the location when creating the
        // table, which fails without explicit configuration for file system.
        // We work around that by using a dummy location when creating the
        // table and update it here to the correct location.
        metastoreClient.updateTableLocation(database, tableName.getTableName(), locationService.getTableWriteInfo(((HiveOutputTableHandle) outputHandle).getLocationHandle(), false).getTargetPath().toString());
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // load the new table
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        // verify the metadata
        ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, getTableHandle(metadata, tableName));
        assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), columns);
        // verify the data
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING);
        ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
        try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle, columnHandles)) {
            MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
            assertEqualsIgnoreOrder(result.getMaterializedRows(), data.getMaterializedRows());
        }
    }
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) AbstractTestHive.filterNonHiddenColumnMetadata(io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnMetadata) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) ConnectorSplitSource(io.prestosql.spi.connector.ConnectorSplitSource) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) HiveTransaction(io.prestosql.plugin.hive.AbstractTestHive.HiveTransaction) Transaction(io.prestosql.plugin.hive.AbstractTestHive.Transaction) Slice(io.airlift.slice.Slice) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) MaterializedResult(io.prestosql.testing.MaterializedResult) ConnectorPageSink(io.prestosql.spi.connector.ConnectorPageSink) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata)

Example 5 with ConnectorOutputTableHandle

use of io.prestosql.spi.connector.ConnectorOutputTableHandle in project hetu-core by openlookeng.

the class AbstractTestHive method doTestBucketSortedTables.

private void doTestBucketSortedTables(SchemaTableName table) throws IOException {
    int bucketCount = 3;
    int expectedRowCount = 0;
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        // begin creating the table
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(table, ImmutableList.<ColumnMetadata>builder().add(new ColumnMetadata("id", VARCHAR)).add(new ColumnMetadata("value_asc", VARCHAR)).add(new ColumnMetadata("value_desc", BIGINT)).add(new ColumnMetadata("ds", VARCHAR)).build(), ImmutableMap.<String, Object>builder().put(STORAGE_FORMAT_PROPERTY, RCBINARY).put(PARTITIONED_BY_PROPERTY, ImmutableList.of("ds")).put(BUCKETED_BY_PROPERTY, ImmutableList.of("id")).put(BUCKET_COUNT_PROPERTY, bucketCount).put(SORTED_BY_PROPERTY, ImmutableList.builder().add(new SortingColumn("value_asc", SortingColumn.Order.ASCENDING)).add(new SortingColumn("value_desc", SortingColumn.Order.DESCENDING)).build()).build());
        ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty());
        // write the data
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle);
        List<Type> types = tableMetadata.getColumns().stream().map(ColumnMetadata::getType).collect(toList());
        ThreadLocalRandom random = ThreadLocalRandom.current();
        for (int i = 0; i < 50; i++) {
            MaterializedResult.Builder builder = MaterializedResult.resultBuilder(session, types);
            for (int j = 0; j < 1000; j++) {
                builder.row(sha256().hashLong(random.nextLong()).toString(), "test" + random.nextInt(100), random.nextLong(100_000), "2018-04-01");
                expectedRowCount++;
            }
            sink.appendPage(builder.build().toPage());
        }
        // verify we have enough temporary files per bucket to require multiple passes
        Path stagingPathRoot = getStagingPathRoot(outputHandle);
        HdfsContext context = new HdfsContext(session, table.getSchemaName(), table.getTableName());
        assertThat(listAllDataFiles(context, stagingPathRoot)).filteredOn(file -> file.contains(".tmp-sort.")).size().isGreaterThan(bucketCount * getHiveConfig().getMaxOpenSortFiles() * 2);
        // finish the write
        Collection<Slice> fragments = getFutureValue(sink.finish());
        // verify there are no temporary files
        for (String file : listAllDataFiles(context, stagingPathRoot)) {
            assertThat(file).doesNotContain(".tmp-sort.");
        }
        // finish creating table
        metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of());
        transaction.commit();
    }
    // verify that bucket files are sorted
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        metadata.beginQuery(session);
        ConnectorTableHandle tableHandle = getTableHandle(metadata, table);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
        List<ConnectorSplit> splits = getAllSplits(tableHandle, transaction, session);
        assertThat(splits).hasSize(bucketCount);
        int actualRowCount = 0;
        for (ConnectorSplit split : splits) {
            try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle, columnHandles)) {
                String lastValueAsc = null;
                long lastValueDesc = -1;
                while (!pageSource.isFinished()) {
                    Page page = pageSource.getNextPage();
                    if (page == null) {
                        continue;
                    }
                    for (int i = 0; i < page.getPositionCount(); i++) {
                        Block blockAsc = page.getBlock(1);
                        Block blockDesc = page.getBlock(2);
                        assertFalse(blockAsc.isNull(i));
                        assertFalse(blockDesc.isNull(i));
                        String valueAsc = VARCHAR.getSlice(blockAsc, i).toStringUtf8();
                        if (lastValueAsc != null) {
                            assertGreaterThanOrEqual(valueAsc, lastValueAsc);
                            if (valueAsc.equals(lastValueAsc)) {
                                long valueDesc = BIGINT.getLong(blockDesc, i);
                                if (lastValueDesc != -1) {
                                    assertLessThanOrEqual(valueDesc, lastValueDesc);
                                }
                                lastValueDesc = valueDesc;
                            } else {
                                lastValueDesc = -1;
                            }
                        }
                        lastValueAsc = valueAsc;
                        actualRowCount++;
                    }
                }
            }
        }
        assertThat(actualRowCount).isEqualTo(expectedRowCount);
    }
}
Also used : ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) Page(io.prestosql.spi.Page) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) HdfsContext(io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Path(org.apache.hadoop.fs.Path) HiveColumnHandle.bucketColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle.bucketColumnHandle) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SortingColumn(io.prestosql.plugin.hive.metastore.SortingColumn) Constraint(io.prestosql.spi.connector.Constraint) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) HiveType.toHiveType(io.prestosql.plugin.hive.HiveType.toHiveType) RowType(io.prestosql.spi.type.RowType) Chars.isCharType(io.prestosql.spi.type.Chars.isCharType) HiveTestUtils.mapType(io.prestosql.plugin.hive.HiveTestUtils.mapType) Varchars.isVarcharType(io.prestosql.spi.type.Varchars.isVarcharType) Type(io.prestosql.spi.type.Type) DecimalType.createDecimalType(io.prestosql.spi.type.DecimalType.createDecimalType) VarcharType.createVarcharType(io.prestosql.spi.type.VarcharType.createVarcharType) MapType(io.prestosql.spi.type.MapType) HiveTestUtils.rowType(io.prestosql.plugin.hive.HiveTestUtils.rowType) TableType(org.apache.hadoop.hive.metastore.TableType) ArrayType(io.prestosql.spi.type.ArrayType) CharType.createCharType(io.prestosql.spi.type.CharType.createCharType) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) HiveTestUtils.arrayType(io.prestosql.plugin.hive.HiveTestUtils.arrayType) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) Slice(io.airlift.slice.Slice) Block(io.prestosql.spi.block.Block) ConnectorPageSink(io.prestosql.spi.connector.ConnectorPageSink) MaterializedResult(io.prestosql.testing.MaterializedResult) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit)

Aggregations

ConnectorOutputTableHandle (io.prestosql.spi.connector.ConnectorOutputTableHandle)11 ConnectorTableMetadata (io.prestosql.spi.connector.ConnectorTableMetadata)9 ConnectorMetadata (io.prestosql.spi.connector.ConnectorMetadata)7 ConnectorSession (io.prestosql.spi.connector.ConnectorSession)7 ConnectorTableHandle (io.prestosql.spi.connector.ConnectorTableHandle)5 SchemaTableName (io.prestosql.spi.connector.SchemaTableName)5 TestingConnectorSession (io.prestosql.testing.TestingConnectorSession)5 Slice (io.airlift.slice.Slice)4 HdfsContext (io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext)4 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)4 ColumnMetadata (io.prestosql.spi.connector.ColumnMetadata)4 ConnectorPageSink (io.prestosql.spi.connector.ConnectorPageSink)4 Test (org.testng.annotations.Test)4 Path (org.apache.hadoop.fs.Path)3 HiveIdentity (io.prestosql.plugin.hive.authentication.HiveIdentity)2 Table (io.prestosql.plugin.hive.metastore.Table)2 ConnectorInsertTableHandle (io.prestosql.spi.connector.ConnectorInsertTableHandle)2 ConnectorPageSource (io.prestosql.spi.connector.ConnectorPageSource)2 ConnectorSplit (io.prestosql.spi.connector.ConnectorSplit)2 ConnectorTransactionHandle (io.prestosql.spi.connector.ConnectorTransactionHandle)2