Examples with ConnectorPageSource com.facebook.presto.spi.ConnectorPageSource used on opensource projects

Example 41 with ConnectorPageSource

use of com.facebook.presto.spi.ConnectorPageSource in project presto by prestodb.

the class HivePageSourceProvider method createHivePageSource.

public static Optional<ConnectorPageSource> createHivePageSource(Set<HiveRecordCursorProvider> cursorProviders, Set<HivePageSourceFactory> pageSourceFactories, String clientId, Configuration configuration, ConnectorSession session, Path path, OptionalInt bucketNumber, long start, long length, Properties schema, TupleDomain<HiveColumnHandle> effectivePredicate, List<HiveColumnHandle> hiveColumns, List<HivePartitionKey> partitionKeys, DateTimeZone hiveStorageTimeZone, TypeManager typeManager, Map<Integer, HiveType> columnCoercions) {
    List<ColumnMapping> columnMappings = ColumnMapping.buildColumnMappings(partitionKeys, hiveColumns, columnCoercions, path, bucketNumber);
    List<ColumnMapping> regularColumnMappings = ColumnMapping.extractRegularColumnMappings(columnMappings);
    for (HivePageSourceFactory pageSourceFactory : pageSourceFactories) {
        Optional<? extends ConnectorPageSource> pageSource = pageSourceFactory.createPageSource(configuration, session, path, start, length, schema, extractRegularColumnHandles(regularColumnMappings, true), effectivePredicate, hiveStorageTimeZone);
        if (pageSource.isPresent()) {
            return Optional.of(new HivePageSource(columnMappings, hiveStorageTimeZone, typeManager, pageSource.get()));
        }
    }
    for (HiveRecordCursorProvider provider : cursorProviders) {
        // GenericHiveRecordCursor will automatically do the coercion without HiveCoercionRecordCursor
        boolean doCoercion = !(provider instanceof GenericHiveRecordCursorProvider);
        Optional<RecordCursor> cursor = provider.createRecordCursor(clientId, configuration, session, path, start, length, schema, extractRegularColumnHandles(regularColumnMappings, doCoercion), effectivePredicate, hiveStorageTimeZone, typeManager);
        if (cursor.isPresent()) {
            RecordCursor delegate = cursor.get();
            // Need to wrap RcText and RcBinary into a wrapper, which will do the coercion for mismatch columns
            if (doCoercion) {
                delegate = new HiveCoercionRecordCursor(regularColumnMappings, typeManager, delegate);
            }
            HiveRecordCursor hiveRecordCursor = new HiveRecordCursor(columnMappings, hiveStorageTimeZone, typeManager, delegate);
            List<Type> columnTypes = hiveColumns.stream().map(input -> typeManager.getType(input.getTypeSignature())).collect(toList());
            return Optional.of(new RecordPageSource(columnTypes, hiveRecordCursor));
        }
    }
    return Optional.empty();
}

Example 42 with ConnectorPageSource

use of com.facebook.presto.spi.ConnectorPageSource in project presto by prestodb.

the class ScanFilterAndProjectOperator method getOutput.

@Override
public Page getOutput() {
    if (split == null) {
        return null;
    }
    if (!finishing) {
        if ((pageSource == null) && (cursor == null)) {
            ConnectorPageSource source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns);
            if (source instanceof RecordPageSource) {
                cursor = ((RecordPageSource) source).getCursor();
            } else {
                pageSource = source;
            }
        }
        if (cursor != null) {
            int rowsProcessed = cursorProcessor.process(operatorContext.getSession().toConnectorSession(), cursor, ROWS_PER_PAGE, pageBuilder);
            pageSourceMemoryContext.setBytes(cursor.getSystemMemoryUsage());
            long bytesProcessed = cursor.getCompletedBytes() - completedBytes;
            long elapsedNanos = cursor.getReadTimeNanos() - readTimeNanos;
            operatorContext.recordGeneratedInput(bytesProcessed, rowsProcessed, elapsedNanos);
            completedBytes = cursor.getCompletedBytes();
            readTimeNanos = cursor.getReadTimeNanos();
            if (rowsProcessed == 0) {
                finishing = true;
            }
        } else {
            if (currentPage == null) {
                currentPage = pageSource.getNextPage();
                if (currentPage != null) {
                    // update operator stats
                    long endCompletedBytes = pageSource.getCompletedBytes();
                    long endReadTimeNanos = pageSource.getReadTimeNanos();
                    operatorContext.recordGeneratedInput(endCompletedBytes - completedBytes, currentPage.getPositionCount(), endReadTimeNanos - readTimeNanos);
                    completedBytes = endCompletedBytes;
                    readTimeNanos = endReadTimeNanos;
                }
                currentPosition = 0;
            }
            if (currentPage != null) {
                switch(processingOptimization) {
                    case COLUMNAR:
                        {
                            Page page = pageProcessor.processColumnar(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
                            currentPage = null;
                            currentPosition = 0;
                            return page;
                        }
                    case COLUMNAR_DICTIONARY:
                        {
                            Page page = pageProcessor.processColumnarDictionary(operatorContext.getSession().toConnectorSession(), currentPage, getTypes());
                            currentPage = null;
                            currentPosition = 0;
                            return page;
                        }
                    case DISABLED:
                        {
                            currentPosition = pageProcessor.process(operatorContext.getSession().toConnectorSession(), currentPage, currentPosition, currentPage.getPositionCount(), pageBuilder);
                            if (currentPosition == currentPage.getPositionCount()) {
                                currentPage = null;
                                currentPosition = 0;
                            }
                            break;
                        }
                    default:
                        throw new IllegalStateException(String.format("Found unexpected value %s for processingOptimization", processingOptimization));
                }
            }
            pageSourceMemoryContext.setBytes(pageSource.getSystemMemoryUsage());
        }
    }
    // only return a full page if buffer is full or we are finishing
    if (pageBuilder.isEmpty() || (!finishing && !pageBuilder.isFull())) {
        pageBuilderMemoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
        return null;
    }
    Page page = pageBuilder.build();
    pageBuilder.reset();
    pageBuilderMemoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
    return page;
}

Example 43 with ConnectorPageSource

use of com.facebook.presto.spi.ConnectorPageSource in project presto by prestodb.

the class AbstractTestHiveClientS3 method doCreateTable.

private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageFormat) throws Exception {
    List<ColumnMetadata> columns = ImmutableList.<ColumnMetadata>builder().add(new ColumnMetadata("id", BIGINT)).build();
    MaterializedResult data = MaterializedResult.resultBuilder(newSession(), BIGINT).row(1L).row(3L).row(2L).build();
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // begin creating the table
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, columns, createTableProperties(storageFormat));
        ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty());
        // write the records
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle);
        sink.appendPage(data.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        // commit the table
        metadata.finishCreateTable(session, outputHandle, fragments);
        transaction.commit();
        // Hack to work around the metastore not being configured for S3.
        // The metastore tries to validate the location when creating the
        // table, which fails without explicit configuration for S3.
        // We work around that by using a dummy location when creating the
        // table and update it here to the correct S3 location.
        metastoreClient.updateTableLocation(database, tableName.getTableName(), locationService.writePathRoot(((HiveOutputTableHandle) outputHandle).getLocationHandle()).get().toString());
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // load the new table
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        // verify the metadata
        ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, getTableHandle(metadata, tableName));
        assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), columns);
        // verify the data
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty());
        HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle();
        assertEquals(layoutHandle.getPartitions().get().size(), 1);
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle);
        ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
        try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) {
            MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
            assertEqualsIgnoreOrder(result.getMaterializedRows(), data.getMaterializedRows());
        }
    }
}

Example 44 with ConnectorPageSource

use of com.facebook.presto.spi.ConnectorPageSource in project presto by prestodb.

the class AbstractTestHiveClient method testTypesRcBinaryRecordCursor.

@Test
public void testTypesRcBinaryRecordCursor() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        if (metadata.getTableHandle(session, new SchemaTableName(database, "presto_test_types_rcbinary")) == null) {
            return;
        }
        ConnectorTableHandle tableHandle = getTableHandle(metadata, new SchemaTableName(database, "presto_test_types_rcbinary"));
        ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
        HiveSplit hiveSplit = getHiveSplit(tableHandle);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
        ConnectorPageSourceProvider pageSourceProvider = new HivePageSourceProvider(new HiveClientConfig().setTimeZone(timeZone.getID()), hdfsEnvironment, ImmutableSet.of(new ColumnarBinaryHiveRecordCursorProvider(hdfsEnvironment)), ImmutableSet.of(), TYPE_MANAGER);
        ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, columnHandles);
        assertGetRecords(RCBINARY, tableMetadata, hiveSplit, pageSource, columnHandles);
    }
}

Example 45 with ConnectorPageSource

use of com.facebook.presto.spi.ConnectorPageSource in project presto by prestodb.

the class AbstractOperatorBenchmark method createTableScanOperator.

protected final OperatorFactory createTableScanOperator(int operatorId, PlanNodeId planNodeId, String tableName, String... columnNames) {
    checkArgument(session.getCatalog().isPresent(), "catalog not set");
    checkArgument(session.getSchema().isPresent(), "schema not set");
    // look up the table
    Metadata metadata = localQueryRunner.getMetadata();
    QualifiedObjectName qualifiedTableName = new QualifiedObjectName(session.getCatalog().get(), session.getSchema().get(), tableName);
    TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).orElse(null);
    checkArgument(tableHandle != null, "Table %s does not exist", qualifiedTableName);
    // lookup the columns
    Map<String, ColumnHandle> allColumnHandles = metadata.getColumnHandles(session, tableHandle);
    ImmutableList.Builder<ColumnHandle> columnHandlesBuilder = ImmutableList.builder();
    for (String columnName : columnNames) {
        ColumnHandle columnHandle = allColumnHandles.get(columnName);
        checkArgument(columnHandle != null, "Table %s does not have a column %s", tableName, columnName);
        columnHandlesBuilder.add(columnHandle);
    }
    List<ColumnHandle> columnHandles = columnHandlesBuilder.build();
    Split split = getLocalQuerySplit(session, tableHandle);
    return new OperatorFactory() {

        @Override
        public Operator createOperator(DriverContext driverContext) {
            OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, "BenchmarkSource");
            ConnectorPageSource pageSource = localQueryRunner.getPageSourceManager().createPageSource(session, split, tableHandle.withDynamicFilter(TupleDomain::all), columnHandles);
            return new PageSourceOperator(pageSource, operatorContext);
        }

        @Override
        public void noMoreOperators() {
        }

        @Override
        public OperatorFactory duplicate() {
            throw new UnsupportedOperationException();
        }
    };
}