Examples with ConnectorPageSource io.prestosql.spi.connector.ConnectorPageSource used on opensource projects

Example 1 with ConnectorPageSource

use of io.prestosql.spi.connector.ConnectorPageSource in project hetu-core by openlookeng.

the class ThriftTpchService method getRowsSync.

private PrestoThriftPageResult getRowsSync(PrestoThriftId splitId, List<String> outputColumns, long maxBytes, PrestoThriftNullableToken nextToken) {
    SplitInfo splitInfo = SPLIT_INFO_CODEC.fromJson(splitId.getId());
    checkArgument(maxBytes >= DEFAULT_MAX_PAGE_SIZE_IN_BYTES, "requested maxBytes is too small");
    ConnectorPageSource pageSource;
    if (!splitInfo.isIndexSplit()) {
        // normal scan
        pageSource = createPageSource(splitInfo, outputColumns);
    } else {
        // index lookup
        pageSource = createLookupPageSource(splitInfo, outputColumns);
    }
    return getRowsInternal(pageSource, splitInfo.getTableName(), outputColumns, nextToken.getToken());
}

Example 2 with ConnectorPageSource

use of io.prestosql.spi.connector.ConnectorPageSource in project hetu-core by openlookeng.

the class CarbondataPageSink method vacuum.

@Override
public VacuumResult vacuum(ConnectorPageSourceProvider pageSourceProvider, ConnectorTransactionHandle transactionHandle, ConnectorTableHandle connectorTableHandle, List<ConnectorSplit> splits) throws PrestoException {
    // TODO: Convert this to page based calls
    List<HiveSplit> hiveSplits = new ArrayList<>();
    // This will get only 1 split in case of Carbondata as we merged all the same task id ones into 1 CarbonLocalMultiBlockSplit in getSplits
    splits.forEach(split -> {
        hiveSplits.addAll(((HiveSplitWrapper) split).getSplits());
    });
    List<ColumnHandle> inputColumns = new ArrayList<>(super.inputColumns);
    ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transactionHandle, session, splits.get(0), connectorTableHandle, inputColumns);
    try {
        performCompaction((CarbondataPageSource) pageSource, hiveSplits.get(0));
    } catch (PrestoException e) {
        LOGGER.error("Error in worker for performing carbon vacuum");
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e.getMessage());
    }
    return new VacuumResult(null, true);
}

Example 3 with ConnectorPageSource

use of io.prestosql.spi.connector.ConnectorPageSource in project hetu-core by openlookeng.

the class CarbondataPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns) {
    HiveSplit carbonSplit = Types.checkType(((HiveSplitWrapper) (split)).getSplits().get(0), HiveSplit.class, "split is not class HiveSplit");
    this.queryId = carbonSplit.getSchema().getProperty("queryId");
    if (this.queryId == null) {
        // Fall back to hive pagesource.
        return super.createPageSource(transactionHandle, session, split, table, columns);
    }
    try {
        hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, carbonSplit.getDatabase()), new Path(carbonSplit.getSchema().getProperty("tablePath")));
    } catch (IOException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed to get file system: " + e.getMessage());
    }
    return hdfsEnvironment.doAs(session.getUser(), () -> {
        Configuration configuration = this.hdfsEnvironment.getConfiguration(new HdfsEnvironment.HdfsContext(session, carbonSplit.getDatabase(), carbonSplit.getTable()), new Path(carbonSplit.getSchema().getProperty("tablePath")));
        CarbonTable carbonTable = getCarbonTable(carbonSplit, configuration);
        /* So that CarbonTLS can access it */
        ThreadLocalSessionInfo.setConfigurationToCurrentThread(configuration);
        boolean isFullACID = isFullAcidTable(Maps.fromProperties(carbonSplit.getSchema()));
        boolean isDirectVectorFill = (carbonTableReader.config.getPushRowFilter() == null) || carbonTableReader.config.getPushRowFilter().equalsIgnoreCase("false") || columns.stream().anyMatch(c -> c.getColumnName().equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID));
        return new CarbondataPageSource(carbonTable, queryId, carbonSplit, columns, table, configuration, isDirectVectorFill, isFullACID, session.getUser(), hdfsEnvironment);
    });
}

Example 4 with ConnectorPageSource

use of io.prestosql.spi.connector.ConnectorPageSource in project hetu-core by openlookeng.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveConfig config, HiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    HiveWriterStats stats = new HiveWriterStats();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastore, new Path("file:///" + outputPath), stats);
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(HiveTestUtils.TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            pages.add(nextPage.getLoadedPage());
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    assertEquals(round(stats.getInputPageSizeInBytes().getAllTime().getMax()), page.getRetainedSizeInBytes());
    return length;
}

Example 5 with ConnectorPageSource

use of io.prestosql.spi.connector.ConnectorPageSource in project hetu-core by openlookeng.

the class TestHivePageSink method createPageSource.

private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveConfig config, File outputFile) {
    Properties splitProperties = new Properties();
    splitProperties.setProperty(FILE_INPUT_FORMAT, config.getHiveStorageFormat().getInputFormat());
    splitProperties.setProperty(SERIALIZATION_LIB, config.getHiveStorageFormat().getSerDe());
    splitProperties.setProperty("columns", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getName).collect(toList())));
    splitProperties.setProperty("columns.types", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getHiveType).map(hiveType -> hiveType.getHiveTypeName().toString()).collect(toList())));
    HiveSplitWrapper split = null;
    try {
        split = HiveSplitWrapper.wrap(new HiveSplit(SCHEMA_NAME, TABLE_NAME, "", "file:///" + outputFile.getCanonicalPath(), 0, outputFile.length(), outputFile.length(), 0, splitProperties, ImmutableList.of(), ImmutableList.of(), OptionalInt.empty(), false, ImmutableMap.of(), Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of()));
    } catch (IOException e) {
        System.out.println(e.getMessage());
    }
    ConnectorTableHandle table = new HiveTableHandle(SCHEMA_NAME, TABLE_NAME, ImmutableMap.of(), ImmutableList.of(), Optional.empty());
    HivePageSourceProvider provider = new HivePageSourceProvider(config, HiveTestUtils.createTestHdfsEnvironment(config), HiveTestUtils.getDefaultHiveRecordCursorProvider(config), HiveTestUtils.getDefaultHiveDataStreamFactories(config), HiveTestUtils.TYPE_MANAGER, HiveTestUtils.getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
    return provider.createPageSource(transaction, getSession(config), split, table, ImmutableList.copyOf(getColumnHandles()));
}