Examples with ConnectorId com.facebook.presto.spi.ConnectorId used on opensource projects

Example 46 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestValidateAggregationsWithDefaultValues method setup.

@BeforeClass
public void setup() {
    metadata = getQueryRunner().getMetadata();
    builder = new PlanBuilder(TEST_SESSION, new PlanNodeIdAllocator(), metadata);
    ConnectorId connectorId = getCurrentConnectorId();
    TpchTableHandle nationTpchTableHandle = new TpchTableHandle("nation", 1.0);
    TableHandle nationTableHandle = new TableHandle(connectorId, nationTpchTableHandle, TestingTransactionHandle.create(), Optional.of(new TpchTableLayoutHandle(nationTpchTableHandle, TupleDomain.all())));
    TpchColumnHandle nationkeyColumnHandle = new TpchColumnHandle("nationkey", BIGINT);
    variable = builder.variable("nationkey");
    tableScanNode = builder.tableScan(nationTableHandle, ImmutableList.of(variable), ImmutableMap.of(variable, nationkeyColumnHandle));
}

Example 47 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class AbstractTestHiveFileSystem method testGetRecords.

@Test
public void testGetRecords() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle table = getTableHandle(metadata, this.table);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, table).values());
        Map<String, Integer> columnIndex = indexColumns(columnHandles);
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, table, Constraint.alwaysTrue(), Optional.empty());
        HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle();
        assertEquals(layoutHandle.getPartitions().get().size(), 1);
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, SPLIT_SCHEDULING_CONTEXT);
        TableHandle tableHandle = new TableHandle(new ConnectorId(database), table, transaction.getTransactionHandle(), Optional.of(layoutHandle));
        long sum = 0;
        for (ConnectorSplit split : getAllSplits(splitSource)) {
            try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle.getLayout().get(), columnHandles, NON_CACHEABLE)) {
                MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
                for (MaterializedRow row : result) {
                    sum += (Long) row.getField(columnIndex.get("t_bigint"));
                }
            }
        }
        // The test table is made up of multiple S3 objects with same data and different compression codec
        // formats: uncompressed | .gz | .lz4 | .bz2
        assertEquals(sum, 78300 * 4);
    }
}

Example 48 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class AbstractTestHiveFileSystem method createTable.

private void createTable(MetastoreContext metastoreContext, SchemaTableName tableName, HiveStorageFormat storageFormat) throws Exception {
    List<ColumnMetadata> columns = ImmutableList.<ColumnMetadata>builder().add(new ColumnMetadata("id", BIGINT)).build();
    MaterializedResult data = MaterializedResult.resultBuilder(newSession(), BIGINT).row(1L).row(3L).row(2L).build();
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // begin creating the table
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, columns, createTableProperties(storageFormat));
        ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty());
        // write the records
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle, TEST_HIVE_PAGE_SINK_CONTEXT);
        sink.appendPage(data.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        // commit the table
        metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of());
        transaction.commit();
        // Hack to work around the metastore not being configured for S3 or other FS.
        // The metastore tries to validate the location when creating the
        // table, which fails without explicit configuration for file system.
        // We work around that by using a dummy location when creating the
        // table and update it here to the correct location.
        metastoreClient.updateTableLocation(metastoreContext, database, tableName.getTableName(), locationService.getTableWriteInfo(((HiveOutputTableHandle) outputHandle).getLocationHandle()).getTargetPath().toString());
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // load the new table
        ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, hiveTableHandle).values());
        // verify the metadata
        ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, getTableHandle(metadata, tableName));
        assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), columns);
        // verify the data
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, hiveTableHandle, Constraint.alwaysTrue(), Optional.empty());
        HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle();
        assertEquals(layoutHandle.getPartitions().get().size(), 1);
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, SPLIT_SCHEDULING_CONTEXT);
        ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
        TableHandle tableHandle = new TableHandle(new ConnectorId("hive"), hiveTableHandle, transaction.getTransactionHandle(), Optional.of(layoutHandle));
        try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle.getLayout().get(), columnHandles, NON_CACHEABLE)) {
            MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
            assertEqualsIgnoreOrder(result.getMaterializedRows(), data.getMaterializedRows());
        }
    }
}

Example 49 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestDynamicPruning method createTestingPageSource.

private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle transaction, HiveClientConfig config, SplitContext splitContext, MetastoreClientConfig metastoreClientConfig, File outputFile) {
    ImmutableList<HivePartitionKey> partitionKeys = ImmutableList.of(new HivePartitionKey(PARTITION_COLUMN.getName(), Optional.of("2020-09-09")));
    Map<Integer, Column> partitionSchemaDifference = ImmutableMap.of(1, new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty()));
    HiveSplit split = new HiveSplit(SCHEMA_NAME, TABLE_NAME, "", "file:///" + outputFile.getAbsolutePath(), 0, outputFile.length(), outputFile.length(), Instant.now().toEpochMilli(), new Storage(StorageFormat.create(config.getHiveStorageFormat().getSerDe(), config.getHiveStorageFormat().getInputFormat(), config.getHiveStorageFormat().getOutputFormat()), "location", Optional.of(new HiveBucketProperty(ImmutableList.of("l_orderkey"), 10, ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty())), false, ImmutableMap.of(), ImmutableMap.of()), partitionKeys, ImmutableList.of(), OptionalInt.of(1), OptionalInt.of(1), NO_PREFERENCE, getColumnHandles().size(), TableToPartitionMapping.mapColumnsByIndex(partitionSchemaDifference), Optional.empty(), false, Optional.empty(), NO_CACHE_REQUIREMENT, Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), SplitWeight.standard());
    TableHandle tableHandle = new TableHandle(new ConnectorId(HIVE_CATALOG), new HiveTableHandle(SCHEMA_NAME, TABLE_NAME), transaction, Optional.of(new HiveTableLayoutHandle(new SchemaTableName(SCHEMA_NAME, TABLE_NAME), "path", ImmutableList.of(PARTITION_HIVE_COLUMN_HANDLE), getColumnHandles().stream().map(column -> new Column(column.getName(), column.getHiveType(), Optional.empty(), Optional.empty())).collect(toImmutableList()), ImmutableMap.of(), TupleDomain.all(), TRUE_CONSTANT, ImmutableMap.of(), TupleDomain.all(), Optional.empty(), Optional.empty(), false, "layout", Optional.empty(), false)));
    HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config, metastoreClientConfig), getDefaultHiveRecordCursorProvider(config, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(config, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(config, metastoreClientConfig), FUNCTION_AND_TYPE_MANAGER, ROW_EXPRESSION_SERVICE);
    return provider.createPageSource(transaction, getSession(config), split, tableHandle.getLayout().get(), ImmutableList.copyOf(getColumnHandles()), splitContext);
}

Example 50 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestPhasedExecutionSchedule method createBroadcastJoinPlanFragment.

private static PlanFragment createBroadcastJoinPlanFragment(String name, PlanFragment buildFragment) {
    VariableReferenceExpression variable = new VariableReferenceExpression(Optional.empty(), "column", BIGINT);
    PlanNode tableScan = new TableScanNode(Optional.empty(), new PlanNodeId(name), new TableHandle(new ConnectorId("test"), new TestingTableHandle(), TestingTransactionHandle.create(), Optional.empty()), ImmutableList.of(variable), ImmutableMap.of(variable, new TestingColumnHandle("column")), TupleDomain.all(), TupleDomain.all());
    RemoteSourceNode remote = new RemoteSourceNode(Optional.empty(), new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPLICATE);
    PlanNode join = new JoinNode(Optional.empty(), new PlanNodeId(name + "_id"), INNER, tableScan, remote, ImmutableList.of(), ImmutableList.<VariableReferenceExpression>builder().addAll(tableScan.getOutputVariables()).addAll(remote.getOutputVariables()).build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(REPLICATED), ImmutableMap.of());
    return createFragment(join);
}