Examples with ResultSetLoaderImpl org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl used on opensource projects

Example 1 with ResultSetLoaderImpl

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl in project drill by apache.

the class ReaderLifecycle method buildLoader.

public ResultSetLoader buildLoader() {
    Preconditions.checkState(state == State.START);
    ScanLifecycleBuilder scanOptions = scanOptions();
    ResultSetOptionBuilder options = new ResultSetOptionBuilder().rowCountLimit(Math.min(schemaNegotiator.batchSize, scanOptions.scanBatchRecordLimit())).vectorCache(scanLifecycle.vectorCache()).batchSizeLimit(scanOptions.scanBatchByteLimit()).errorContext(errorContext()).projectionFilter(schemaTracker().projectionFilter(errorContext())).readerSchema(schemaNegotiator.readerSchema).limit(limit);
    // Resolve the scan schema if possible.
    applyEarlySchema();
    // Create the table loader
    tableLoader = new ResultSetLoaderImpl(scanLifecycle.allocator(), options.build());
    state = State.DATA;
    return tableLoader;
}

Example 2 with ResultSetLoaderImpl

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl in project drill by apache.

the class MetadataHandlerBatch method getResultSetLoaderForMetadata.

private ResultSetLoader getResultSetLoaderForMetadata(BaseMetadata baseMetadata) {
    SchemaBuilder schemaBuilder = new SchemaBuilder().addNullable(MetastoreAnalyzeConstants.LOCATION_FIELD, MinorType.VARCHAR);
    for (String segmentColumn : popConfig.getContext().segmentColumns()) {
        schemaBuilder.addNullable(segmentColumn, MinorType.VARCHAR);
    }
    baseMetadata.getColumnsStatistics().entrySet().stream().sorted(Comparator.comparing(e -> e.getKey().getRootSegmentPath())).forEach(entry -> {
        for (StatisticsKind<?> statisticsKind : AnalyzeColumnUtils.COLUMN_STATISTICS_FUNCTIONS.keySet()) {
            MinorType type = AnalyzeColumnUtils.COLUMN_STATISTICS_TYPES.get(statisticsKind);
            type = type != null ? type : entry.getValue().getComparatorType();
            schemaBuilder.addNullable(AnalyzeColumnUtils.getColumnStatisticsFieldName(entry.getKey().getRootSegmentPath(), statisticsKind), type);
        }
    });
    for (StatisticsKind<?> statisticsKind : AnalyzeColumnUtils.META_STATISTICS_FUNCTIONS.keySet()) {
        schemaBuilder.addNullable(AnalyzeColumnUtils.getMetadataStatisticsFieldName(statisticsKind), AnalyzeColumnUtils.COLUMN_STATISTICS_TYPES.get(statisticsKind));
    }
    schemaBuilder.addMapArray(MetastoreAnalyzeConstants.COLLECTED_MAP_FIELD).resumeSchema();
    if (metadataType == MetadataType.SEGMENT) {
        schemaBuilder.addArray(MetastoreAnalyzeConstants.LOCATIONS_FIELD, MinorType.VARCHAR);
    }
    if (metadataType == MetadataType.ROW_GROUP) {
        schemaBuilder.addNullable(columnNamesOptions.rowGroupIndex(), MinorType.VARCHAR);
        schemaBuilder.addNullable(columnNamesOptions.rowGroupStart(), MinorType.VARCHAR);
        schemaBuilder.addNullable(columnNamesOptions.rowGroupLength(), MinorType.VARCHAR);
    }
    schemaBuilder.addNullable(MetastoreAnalyzeConstants.SCHEMA_FIELD, MinorType.VARCHAR).addNullable(columnNamesOptions.lastModifiedTime(), MinorType.VARCHAR).add(MetastoreAnalyzeConstants.METADATA_TYPE, MinorType.VARCHAR);
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().readerSchema(schemaBuilder.buildSchema()).build();
    return new ResultSetLoaderImpl(container.getAllocator(), options);
}

Example 3 with ResultSetLoaderImpl

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl in project drill by apache.

the class TestLoad method testLoadValueVectorEmptyVarCharArray.

@Test
public void testLoadValueVectorEmptyVarCharArray() throws Exception {
    try (BufferAllocator allocator = RootAllocatorFactory.newRoot(drillConfig)) {
        TupleMetadata schema = new SchemaBuilder().addArray("chars", MinorType.VARCHAR).build();
        ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().readerSchema(schema).build();
        ResultSetLoader resultSetLoader = new ResultSetLoaderImpl(allocator, options);
        resultSetLoader.startBatch();
        RowSetLoader rowWriter = resultSetLoader.writer();
        rowWriter.addRow(new Object[] { null });
        VectorContainer harvest = resultSetLoader.harvest();
        // Create vectors
        List<ValueVector> vectors = StreamSupport.stream(harvest.spliterator(), false).map(VectorWrapper::getValueVector).collect(Collectors.toList());
        // Writeable batch now owns vector buffers
        WritableBatch writableBatch = WritableBatch.getBatchNoHV(1, vectors, false);
        // Serialize the vectors
        DrillBuf byteBuf = serializeBatch(allocator, writableBatch);
        // Batch loader does NOT take ownership of the serialized buffer
        RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
        batchLoader.load(writableBatch.getDef(), byteBuf);
        // Release the serialized buffer.
        byteBuf.release();
        assertEquals(1, batchLoader.getRecordCount());
        // Free the original vectors
        writableBatch.clear();
        // Free the deserialized vectors
        batchLoader.clear();
    }
}

Example 4 with ResultSetLoaderImpl

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl in project drill by apache.

the class MetadataHandlerBatch method getResultSetLoaderWithBatchSchema.

private ResultSetLoader getResultSetLoaderWithBatchSchema() {
    SchemaBuilder schemaBuilder = new SchemaBuilder();
    // adds fields to the schema preserving their order to avoid issues in outcoming batches
    for (VectorWrapper<?> vectorWrapper : container) {
        MaterializedField field = vectorWrapper.getField();
        String fieldName = field.getName();
        if (fieldName.equals(MetastoreAnalyzeConstants.LOCATION_FIELD) || fieldName.equals(MetastoreAnalyzeConstants.SCHEMA_FIELD) || fieldName.equals(columnNamesOptions.lastModifiedTime()) || fieldName.equals(columnNamesOptions.rowGroupIndex()) || fieldName.equals(columnNamesOptions.rowGroupStart()) || fieldName.equals(columnNamesOptions.rowGroupLength()) || fieldName.equals(MetastoreAnalyzeConstants.METADATA_TYPE) || popConfig.getContext().segmentColumns().contains(fieldName)) {
            schemaBuilder.add(fieldName, field.getType().getMinorType(), field.getDataMode());
        } else if (AnalyzeColumnUtils.isColumnStatisticsField(fieldName) || AnalyzeColumnUtils.isMetadataStatisticsField(fieldName)) {
            schemaBuilder.add(fieldName, field.getType().getMinorType(), field.getType().getMode());
        } else if (fieldName.equals(MetastoreAnalyzeConstants.COLLECTED_MAP_FIELD)) {
            schemaBuilder.addMapArray(fieldName).resumeSchema();
        } else if (fieldName.equals(MetastoreAnalyzeConstants.LOCATIONS_FIELD)) {
            schemaBuilder.addArray(fieldName, MinorType.VARCHAR);
        } else {
            throw new UnsupportedOperationException(String.format("Found unexpected field [%s] in incoming batch.", field));
        }
    }
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().readerSchema(schemaBuilder.buildSchema()).build();
    return new ResultSetLoaderImpl(container.getAllocator(), options);
}

Example 5 with ResultSetLoaderImpl

use of org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl in project drill by apache.

the class ReaderSchemaOrchestrator method makeTableLoader.

public ResultSetLoader makeTableLoader(CustomErrorContext errorContext, TupleMetadata readerSchema, long localLimit) {
    ResultSetOptionBuilder options = new ResultSetOptionBuilder();
    options.rowCountLimit(Math.min(readerBatchSize, scanOrchestrator.options.scanBatchRecordLimit));
    options.vectorCache(scanOrchestrator.vectorCache);
    options.batchSizeLimit(scanOrchestrator.options.scanBatchByteLimit);
    options.errorContext(errorContext);
    // Set up a selection list if available and is a subset of
    // table columns. (Only needed for non-wildcard queries.)
    // The projection list includes all candidate table columns
    // whether or not they exist in the up-front schema. Handles
    // the odd case where the reader claims a fixed schema, but
    // adds a column later.
    options.projectionFilter(scanOrchestrator.scanProj.readerProjection);
    options.readerSchema(readerSchema);
    if (limit < 0) {
        limit = localLimit;
    } else if (localLimit >= 0) {
        limit = Math.min(localLimit, limit);
    }
    options.limit(limit);
    // Create the table loader
    tableLoader = new ResultSetLoaderImpl(scanOrchestrator.allocator, options.build());
    return tableLoader;
}