Examples with ConnectorTableHandle com.facebook.presto.spi.ConnectorTableHandle used on opensource projects

Example 36 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method testGetRecordsInvalidColumn.

@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*" + INVALID_COLUMN + ".*")
public void testGetRecordsInvalidColumn() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata connectorMetadata = transaction.getMetadata();
        ConnectorTableHandle table = getTableHandle(connectorMetadata, tableUnpartitioned);
        readTable(transaction, table, ImmutableList.of(invalidColumnHandle), newSession(), TupleDomain.all(), OptionalInt.empty(), Optional.empty());
    }
}

Example 37 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method doCreateEmptyTable.

protected void doCreateEmptyTable(SchemaTableName tableName, HiveStorageFormat storageFormat, List<ColumnMetadata> createTableColumns) throws Exception {
    List<String> partitionedBy = createTableColumns.stream().filter(column -> column.getName().equals("ds")).map(ColumnMetadata::getName).collect(toList());
    String queryId;
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        queryId = session.getQueryId();
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, createTableColumns, createTableProperties(storageFormat, partitionedBy));
        metadata.createTable(session, tableMetadata, false);
        transaction.commit();
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), false, DEFAULT_COLUMN_CONVERTER_PROVIDER);
        // load the new table
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        // verify the metadata
        ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, getTableHandle(metadata, tableName));
        List<ColumnMetadata> expectedColumns = createTableColumns.stream().map(column -> new ColumnMetadata(column.getName(), column.getType(), column.getComment(), columnExtraInfo(partitionedBy.contains(column.getName())), false)).collect(toList());
        assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), expectedColumns);
        // verify table format
        Table table = transaction.getMetastore().getTable(metastoreContext, tableName.getSchemaName(), tableName.getTableName()).get();
        assertEquals(table.getStorage().getStorageFormat().getInputFormat(), storageFormat.getInputFormat());
        // verify the node version and query ID
        assertEquals(table.getParameters().get(PRESTO_VERSION_NAME), TEST_SERVER_VERSION);
        assertEquals(table.getParameters().get(PRESTO_QUERY_ID_NAME), queryId);
        // verify the table is empty
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
        assertEquals(result.getRowCount(), 0);
        // verify basic statistics
        if (partitionedBy.isEmpty()) {
            assertEmptyTableStatistics(metastoreContext, tableName, transaction);
        }
    }
}

Example 38 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method doTestBucketedTableEvolutionWithDifferentReadCount.

private void doTestBucketedTableEvolutionWithDifferentReadCount(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception {
    int rowCount = 100;
    int bucketCount = 16;
    // Produce a table with a partition with bucket count different but compatible with the table bucket count
    createEmptyTable(tableName, storageFormat, ImmutableList.of(new Column("id", HIVE_LONG, Optional.empty(), Optional.empty()), new Column("name", HIVE_STRING, Optional.empty(), Optional.empty())), ImmutableList.of(new Column("pk", HIVE_STRING, Optional.empty(), Optional.empty())), Optional.of(new HiveBucketProperty(ImmutableList.of("id"), 4, ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty())));
    // write a 4-bucket partition
    MaterializedResult.Builder bucket8Builder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> bucket8Builder.row((long) i, String.valueOf(i), "four"));
    insertData(tableName, bucket8Builder.build());
    // Alter the bucket count to 16
    alterBucketProperty(tableName, Optional.of(new HiveBucketProperty(ImmutableList.of("id"), bucketCount, ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty())));
    MaterializedResult result;
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tableName);
        // read entire table
        List<ColumnHandle> columnHandles = ImmutableList.<ColumnHandle>builder().addAll(metadata.getColumnHandles(session, hiveTableHandle).values()).build();
        HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getTableLayout(session, transaction.getMetadata(), hiveTableHandle, Constraint.alwaysTrue(), transaction).getHandle();
        HiveBucketHandle bucketHandle = layoutHandle.getBucketHandle().get();
        HiveTableLayoutHandle modifiedReadBucketCountLayoutHandle = new HiveTableLayoutHandle(layoutHandle.getSchemaTableName(), layoutHandle.getTablePath(), layoutHandle.getPartitionColumns(), layoutHandle.getDataColumns(), layoutHandle.getTableParameters(), layoutHandle.getPartitions().get(), layoutHandle.getDomainPredicate(), layoutHandle.getRemainingPredicate(), layoutHandle.getPredicateColumns(), layoutHandle.getPartitionColumnPredicate(), Optional.of(new HiveBucketHandle(bucketHandle.getColumns(), bucketHandle.getTableBucketCount(), 2)), layoutHandle.getBucketFilter(), false, "layout", Optional.empty(), false);
        List<ConnectorSplit> splits = getAllSplits(session, transaction, modifiedReadBucketCountLayoutHandle);
        assertEquals(splits.size(), 16);
        TableHandle tableHandle = toTableHandle(transaction, hiveTableHandle, modifiedReadBucketCountLayoutHandle);
        ImmutableList.Builder<MaterializedRow> allRows = ImmutableList.builder();
        for (ConnectorSplit split : splits) {
            try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle.getLayout().get(), columnHandles, NON_CACHEABLE)) {
                MaterializedResult intermediateResult = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
                allRows.addAll(intermediateResult.getMaterializedRows());
            }
        }
        result = new MaterializedResult(allRows.build(), getTypes(columnHandles));
        assertEquals(result.getRowCount(), rowCount);
        Map<String, Integer> columnIndex = indexColumns(columnHandles);
        int nameColumnIndex = columnIndex.get("name");
        int bucketColumnIndex = columnIndex.get(BUCKET_COLUMN_NAME);
        for (MaterializedRow row : result.getMaterializedRows()) {
            String name = (String) row.getField(nameColumnIndex);
            int bucket = (int) row.getField(bucketColumnIndex);
            assertEquals(bucket, Integer.parseInt(name) % bucketCount);
        }
    }
}

Example 39 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method testGetPartialRecords.

@Test
public void testGetPartialRecords() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tablePartitionFormat);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, hiveTableHandle).values());
        Map<String, Integer> columnIndex = indexColumns(columnHandles);
        ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, hiveTableHandle, TupleDomain.all());
        List<ConnectorSplit> splits = getAllSplits(session, transaction, layoutHandle);
        assertEquals(splits.size(), partitionCount);
        for (ConnectorSplit split : splits) {
            HiveSplit hiveSplit = (HiveSplit) split;
            List<HivePartitionKey> partitionKeys = hiveSplit.getPartitionKeys();
            String ds = partitionKeys.get(0).getValue().orElse(null);
            String fileFormat = partitionKeys.get(1).getValue().orElse(null);
            HiveStorageFormat fileType = HiveStorageFormat.valueOf(fileFormat.toUpperCase());
            int dummyPartition = Integer.parseInt(partitionKeys.get(2).getValue().orElse(null));
            long rowNumber = 0;
            try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, layoutHandle, columnHandles, NON_CACHEABLE)) {
                assertPageSourceType(pageSource, fileType);
                MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
                for (MaterializedRow row : result) {
                    rowNumber++;
                    assertEquals(row.getField(columnIndex.get("t_double")), 6.2 + rowNumber);
                    assertEquals(row.getField(columnIndex.get("ds")), ds);
                    assertEquals(row.getField(columnIndex.get("file_format")), fileFormat);
                    assertEquals(row.getField(columnIndex.get("dummy")), dummyPartition);
                }
            }
            assertEquals(rowNumber, 100);
        }
    }
}

Example 40 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method testGetPartitionNames.

@Test
public void testGetPartitionNames() {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat);
        assertExpectedTableLayout(getTableLayout(newSession(), metadata, tableHandle, Constraint.alwaysTrue(), transaction), tableLayout);
    }
}