Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 51 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class MapFlatBatchStreamReader method readBlock.

@Override
public Block readBlock() throws IOException {
    if (!rowGroupOpen) {
        openRowGroup();
    }
    if (readOffset > 0) {
        if (presentStream != null) {
            // skip ahead the present bit reader, but count the set bits
            // and use this as the skip size for the data reader
            readOffset = presentStream.countBitsSet(readOffset);
        }
        if (readOffset > 0) {
            for (int i = 0; i < valueStreamReaders.size(); i++) {
                int valueReadOffset = inMapStreams.get(i).countBitsSet(readOffset);
                valueStreamReaders.get(i).prepareNextRead(valueReadOffset);
            }
        }
    }
    boolean[][] inMapVectors = new boolean[inMapStreamSources.size()][];
    boolean[] nullVector = null;
    int totalMapEntries = 0;
    if (presentStream == null) {
        for (int keyIndex = 0; keyIndex < inMapStreams.size(); keyIndex++) {
            inMapVectors[keyIndex] = new boolean[nextBatchSize];
            totalMapEntries += inMapStreams.get(keyIndex).getSetBits(nextBatchSize, inMapVectors[keyIndex]);
        }
    } else {
        nullVector = new boolean[nextBatchSize];
        int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector);
        if (nullValues != nextBatchSize) {
            for (int i = 0; i < inMapStreams.size(); i++) {
                inMapVectors[i] = new boolean[nextBatchSize];
                totalMapEntries += inMapStreams.get(i).getSetBits(nextBatchSize, inMapVectors[i], nullVector);
            }
        }
    }
    MapType mapType = (MapType) type;
    Type valueType = mapType.getValueType();
    Block[] valueBlocks = new Block[valueStreamReaders.size()];
    if (totalMapEntries > 0) {
        for (int keyIndex = 0; keyIndex < valueStreamReaders.size(); keyIndex++) {
            int mapsContainingKey = 0;
            for (int mapIndex = 0; mapIndex < nextBatchSize; mapIndex++) {
                if (inMapVectors[keyIndex][mapIndex]) {
                    mapsContainingKey++;
                }
            }
            if (mapsContainingKey > 0) {
                BatchStreamReader streamReader = valueStreamReaders.get(keyIndex);
                streamReader.prepareNextRead(mapsContainingKey);
                valueBlocks[keyIndex] = streamReader.readBlock();
            } else {
                valueBlocks[keyIndex] = valueType.createBlockBuilder(null, 0).build();
            }
        }
    }
    int[] valueBlockPositions = new int[inMapVectors.length];
    BlockBuilder valueBlockBuilder = valueType.createBlockBuilder(null, totalMapEntries);
    int[] keyIds = new int[totalMapEntries];
    int keyIdsIndex = 0;
    int[] mapOffsets = new int[nextBatchSize + 1];
    mapOffsets[0] = 0;
    for (int mapIndex = 0; mapIndex < nextBatchSize; mapIndex++) {
        int mapLength = 0;
        if (totalMapEntries > 0) {
            for (int keyIndex = 0; keyIndex < inMapVectors.length; keyIndex++) {
                if (inMapVectors[keyIndex][mapIndex]) {
                    mapLength++;
                    valueType.appendTo(valueBlocks[keyIndex], valueBlockPositions[keyIndex], valueBlockBuilder);
                    keyIds[keyIdsIndex++] = keyIndex;
                    valueBlockPositions[keyIndex]++;
                }
            }
        }
        mapOffsets[mapIndex + 1] = mapOffsets[mapIndex] + mapLength;
    }
    Block block = mapType.createBlockFromKeyValue(nextBatchSize, Optional.ofNullable(nullVector), mapOffsets, new DictionaryBlock(keyBlockTemplate, keyIds), valueBlockBuilder);
    readOffset = 0;
    nextBatchSize = 0;
    return block;
}

Example 52 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class MapFlatSelectiveStreamReader method assembleMapBlock.

private Block assembleMapBlock(boolean includeNulls) {
    offsets = ensureCapacity(offsets, outputPositionCount + 1);
    offsets[0] = 0;
    int offset = 0;
    int inMapIndex = 0;
    for (int i = 0; i < outputPositionCount; i++) {
        if (!includeNulls || !nulls[i]) {
            offset += nestedLengths[inMapIndex];
            inMapIndex++;
        }
        offsets[i + 1] = offset;
    }
    BlockLease[] valueBlockLeases = new BlockLease[keyCount];
    Block[] valueBlocks = new Block[keyCount];
    for (int i = 0; i < keyCount; i++) {
        if (nestedPositionCounts[i] > 0) {
            valueBlockLeases[i] = valueStreamReaders.get(i).getBlockView(nestedOutputPositions[i], nestedPositionCounts[i]);
            valueBlocks[i] = valueBlockLeases[i].get();
        } else {
            valueBlocks[i] = outputType.getKeyType().createBlockBuilder(null, 0).build();
        }
    }
    int[] keyIds = new int[offset];
    int count = 0;
    Type valueType = outputType.getValueType();
    BlockBuilder valueBlockBuilder;
    if (valueType instanceof FixedWidthType) {
        valueBlockBuilder = ((FixedWidthType) valueType).createFixedSizeBlockBuilder(offset);
    } else {
        valueBlockBuilder = valueType.createBlockBuilder(null, offset);
    }
    int[] valueBlockPositions = new int[keyCount];
    inMapIndex = 0;
    for (int i = 0; i < outputPositionCount; i++) {
        if (includeNulls && nulls[i]) {
            continue;
        }
        for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) {
            if (inMap[keyIndex][inMapIndex]) {
                valueType.appendTo(valueBlocks[keyIndex], valueBlockPositions[keyIndex], valueBlockBuilder);
                valueBlockPositions[keyIndex]++;
                keyIds[count++] = keyIndex;
            }
        }
        inMapIndex++;
    }
    for (int i = 0; i < keyCount; i++) {
        if (valueBlockLeases[i] != null) {
            valueBlockLeases[i].close();
        }
    }
    return outputType.createBlockFromKeyValue(outputPositionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, new DictionaryBlock(keyBlock, keyIds), valueBlockBuilder.build());
}

Example 53 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class TestPage method testCompactDictionaryBlocks.

@Test
public void testCompactDictionaryBlocks() {
    int positionCount = 100;
    // Create 2 dictionary blocks with the same source id
    DictionaryId commonSourceId = randomDictionaryId();
    int commonDictionaryUsedPositions = 20;
    int[] commonDictionaryIds = getDictionaryIds(positionCount, commonDictionaryUsedPositions);
    // first dictionary contains "varbinary" values
    Slice[] dictionaryValues1 = createExpectedValues(50);
    Block dictionary1 = createSlicesBlock(dictionaryValues1);
    DictionaryBlock commonSourceIdBlock1 = new DictionaryBlock(positionCount, dictionary1, commonDictionaryIds, commonSourceId);
    // second dictionary block is "length(firstColumn)"
    BlockBuilder dictionary2 = BIGINT.createBlockBuilder(null, dictionary1.getPositionCount());
    for (Slice expectedValue : dictionaryValues1) {
        BIGINT.writeLong(dictionary2, expectedValue.length());
    }
    DictionaryBlock commonSourceIdBlock2 = new DictionaryBlock(positionCount, dictionary2.build(), commonDictionaryIds, commonSourceId);
    // Create block with a different source id, dictionary size, used
    int otherDictionaryUsedPositions = 30;
    int[] otherDictionaryIds = getDictionaryIds(positionCount, otherDictionaryUsedPositions);
    Block dictionary3 = createSlicesBlock(createExpectedValues(70));
    DictionaryBlock randomSourceIdBlock = new DictionaryBlock(dictionary3, otherDictionaryIds);
    Page page = new Page(commonSourceIdBlock1, randomSourceIdBlock, commonSourceIdBlock2);
    page.compact();
    // dictionary blocks should all be compact
    assertTrue(((DictionaryBlock) page.getBlock(0)).isCompact());
    assertTrue(((DictionaryBlock) page.getBlock(1)).isCompact());
    assertTrue(((DictionaryBlock) page.getBlock(2)).isCompact());
    assertEquals(((DictionaryBlock) page.getBlock(0)).getDictionary().getPositionCount(), commonDictionaryUsedPositions);
    assertEquals(((DictionaryBlock) page.getBlock(1)).getDictionary().getPositionCount(), otherDictionaryUsedPositions);
    assertEquals(((DictionaryBlock) page.getBlock(2)).getDictionary().getPositionCount(), commonDictionaryUsedPositions);
    // Blocks that had the same source id before compacting page should have the same source id after compacting page
    assertNotEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(1)).getDictionarySourceId());
    assertEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(2)).getDictionarySourceId());
}

Example 54 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class FilterFunction method filterWithDictionary.

private int filterWithDictionary(Page page, int[] positions, int positionCount, RuntimeException[] errors) {
    int outputCount = 0;
    DictionaryBlock block = (DictionaryBlock) page.getBlock(0);
    Block dictionary = block.getDictionary();
    if (dictionary != previousDictionary) {
        previousDictionary = dictionary;
        int numEntries = dictionary.getPositionCount();
        dictionaryPage = new Page(numEntries, dictionary);
        dictionaryResults = ensureCapacity(dictionaryResults, numEntries);
        fill(dictionaryResults, 0, numEntries, FILTER_NOT_EVALUATED);
    }
    for (int i = 0; i < positionCount; i++) {
        int position = positions[i];
        int dictionaryPosition = block.getId(position);
        byte result = dictionaryResults[dictionaryPosition];
        switch(result) {
            case FILTER_FAILED:
                continue;
            case FILTER_PASSED:
                positions[outputCount] = position;
                errors[outputCount] = errors[i];
                outputCount++;
                continue;
            case FILTER_NOT_EVALUATED:
                try {
                    if (predicate.evaluate(properties, dictionaryPage, dictionaryPosition)) {
                        positions[outputCount] = position;
                        errors[outputCount] = errors[i];
                        outputCount++;
                        dictionaryResults[dictionaryPosition] = FILTER_PASSED;
                    } else {
                        dictionaryResults[dictionaryPosition] = FILTER_FAILED;
                    }
                } catch (RuntimeException e) {
                    // We do not record errors in the dictionary results.
                    positions[outputCount] = position;
                    // keep last error
                    errors[outputCount] = e;
                    outputCount++;
                }
                break;
            default:
                verify(false, "Unexpected filter result: " + result);
        }
    }
    return outputCount;
}

Example 55 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class Page method compact.

public Page compact() {
    if (getRetainedSizeInBytes() <= getSizeInBytes()) {
        return this;
    }
    for (int i = 0; i < blocks.length; i++) {
        Block block = blocks[i];
        if (block instanceof DictionaryBlock) {
            continue;
        }
        // Compact the block
        blocks[i] = block.copyRegion(0, block.getPositionCount());
    }
    Map<DictionaryId, DictionaryBlockIndexes> dictionaryBlocks = getRelatedDictionaryBlocks();
    for (DictionaryBlockIndexes blockIndexes : dictionaryBlocks.values()) {
        List<DictionaryBlock> compactBlocks = compactRelatedBlocks(blockIndexes.getBlocks());
        List<Integer> indexes = blockIndexes.getIndexes();
        for (int i = 0; i < compactBlocks.size(); i++) {
            blocks[indexes.get(i)] = compactBlocks.get(i);
        }
    }
    updateRetainedSize();
    return this;
}