Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 81 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class SliceDictionaryBatchStreamReader method readBlock.

@Override
public Block readBlock() throws IOException {
    if (!rowGroupOpen) {
        openRowGroup();
    }
    if (readOffset > 0) {
        if (presentStream != null) {
            // skip ahead the present bit reader, but count the set bits
            // and use this as the skip size for the length reader
            readOffset = presentStream.countBitsSet(readOffset);
        }
        if (readOffset > 0) {
            if (dataStream == null) {
                throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
            }
            if (inDictionaryStream != null) {
                inDictionaryStream.skip(readOffset);
            }
            dataStream.skip(readOffset);
        }
    }
    int[] idsVector = new int[nextBatchSize];
    if (presentStream == null) {
        // Data doesn't have nulls
        if (dataStream == null) {
            throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
        }
        if (inDictionaryStream == null) {
            dataStream.next(idsVector, nextBatchSize);
        } else {
            for (int i = 0; i < nextBatchSize; i++) {
                idsVector[i] = toIntExact(dataStream.next());
                if (!inDictionaryStream.nextBit()) {
                    // row group dictionary elements are after the main dictionary
                    idsVector[i] += stripeDictionarySize;
                }
            }
        }
    } else {
        // Data has nulls
        if (dataStream == null) {
            // The only valid case for dataStream is null when data has nulls is that all values are nulls.
            // In that case the only element in the dictionaryBlock is null and the ids in idsVector should
            // be all 0's, so we don't need to update idVector again.
            int nullValues = presentStream.getUnsetBits(nextBatchSize);
            if (nullValues != nextBatchSize) {
                throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
            }
        } else {
            for (int i = 0; i < nextBatchSize; i++) {
                if (!presentStream.nextBit()) {
                    // null is the last entry in the slice dictionary
                    idsVector[i] = dictionaryBlock.getPositionCount() - 1;
                } else {
                    idsVector[i] = toIntExact(dataStream.next());
                    if (inDictionaryStream != null && !inDictionaryStream.nextBit()) {
                        // row group dictionary elements are after the main dictionary
                        idsVector[i] += stripeDictionarySize;
                    }
                }
            }
        }
    }
    Block block = new DictionaryBlock(nextBatchSize, dictionaryBlock, idsVector);
    readOffset = 0;
    nextBatchSize = 0;
    return block;
}

Example 82 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class SliceDictionarySelectiveReader method getBlockView.

@Override
public BlockLease getBlockView(int[] positions, int positionCount) {
    checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
    checkState(outputRequired, "This stream reader doesn't produce output");
    checkState(positionCount <= outputPositionCount, "Not enough values");
    checkState(!valuesInUse, "BlockLease hasn't been closed yet");
    if (allNulls) {
        return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount));
    }
    if (positionCount < outputPositionCount) {
        compactValues(positions, positionCount);
    }
    wrapDictionaryIfNecessary();
    return newLease(new DictionaryBlock(positionCount, dictionary, values));
}

Example 83 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class TestPageProcessorCompiler method testSanityColumnarDictionary.

@Test
public void testSanityColumnarDictionary() {
    PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(field(0, VARCHAR)), false, MAX_BATCH_SIZE).get();
    Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10);
}

Example 84 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class TestPageProcessorCompiler method testSanityFilterOnDictionary.

@Test
public void testSanityFilterOnDictionary() {
    FunctionAndTypeManager functionAndTypeManager = createTestMetadataManager().getFunctionAndTypeManager();
    CallExpression lengthVarchar = new CallExpression("length", functionAndTypeManager.lookupFunction("length", fromTypes(VARCHAR)), BIGINT, ImmutableList.of(field(0, VARCHAR)));
    FunctionHandle lessThan = functionAndTypeManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT));
    CallExpression filter = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(lengthVarchar, constant(10L, BIGINT)));
    PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(0, VARCHAR)), false, MAX_BATCH_SIZE).get();
    Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10);
    // test filter caching
    Page outputPage2 = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage2.getPositionCount(), 100);
    assertTrue(outputPage2.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock2 = (DictionaryBlock) outputPage2.getBlock(0);
    // both output pages must have the same dictionary
    assertEquals(dictionaryBlock2.getDictionary(), dictionaryBlock.getDictionary());
}

Example 85 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class FilterFunction method filterWithDictionary.

private int filterWithDictionary(Page page, int[] positions, int positionCount, RuntimeException[] errors) {
    int outputCount = 0;
    DictionaryBlock block = (DictionaryBlock) page.getBlock(0);
    Block dictionary = block.getDictionary();
    if (dictionary != previousDictionary) {
        previousDictionary = dictionary;
        int numEntries = dictionary.getPositionCount();
        dictionaryPage = new Page(numEntries, dictionary);
        dictionaryResults = ensureCapacity(dictionaryResults, numEntries);
        fill(dictionaryResults, 0, numEntries, FILTER_NOT_EVALUATED);
    }
    for (int i = 0; i < positionCount; i++) {
        int position = positions[i];
        int dictionaryPosition = block.getId(position);
        byte result = dictionaryResults[dictionaryPosition];
        switch(result) {
            case FILTER_FAILED:
                continue;
            case FILTER_PASSED:
                positions[outputCount] = position;
                errors[outputCount] = errors[i];
                outputCount++;
                continue;
            case FILTER_NOT_EVALUATED:
                try {
                    if (predicate.evaluate(properties, dictionaryPage, dictionaryPosition)) {
                        positions[outputCount] = position;
                        errors[outputCount] = errors[i];
                        outputCount++;
                        dictionaryResults[dictionaryPosition] = FILTER_PASSED;
                    } else {
                        dictionaryResults[dictionaryPosition] = FILTER_FAILED;
                    }
                } catch (RuntimeException e) {
                    // We do not record errors in the dictionary results.
                    positions[outputCount] = position;
                    // keep last error
                    errors[outputCount] = e;
                    outputCount++;
                }
                break;
            default:
                verify(false, "Unexpected filter result: " + result);
        }
    }
    return outputCount;
}