Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 36 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class OptimizedTypedSet method getBlock.

/**
 * Build and return the block representing this set
 */
public Block getBlock() {
    if (size == 0) {
        return elementType.createBlockBuilder(null, 0).build();
    }
    if (currentBlockIndex == 0) {
        // Just one block. Return a DictionaryBlock
        Block block = blocks[currentBlockIndex];
        SelectedPositions selectedPositions = getPositionsForBlocks().get(currentBlockIndex);
        return new DictionaryBlock(selectedPositions.getOffset(), selectedPositions.size(), block, selectedPositions.getPositions(), false, DictionaryId.randomDictionaryId());
    }
    Block firstBlock = blocks[0];
    BlockBuilder blockBuilder = elementType.createBlockBuilder(null, size, toIntExact(firstBlock.getApproximateRegionLogicalSizeInBytes(0, firstBlock.getPositionCount()) / max(1, toIntExact(firstBlock.getPositionCount()))));
    for (int i = 0; i <= currentBlockIndex; i++) {
        Block block = blocks[i];
        SelectedPositions selectedPositions = getPositionsForBlocks().get(i);
        int positionCount = selectedPositions.size();
        if (!selectedPositions.isList()) {
            if (positionCount == block.getPositionCount()) {
                return block;
            } else {
                return block.getRegion(selectedPositions.getOffset(), positionCount);
            }
        }
        int[] positions = selectedPositions.getPositions();
        for (int j = 0; j < positionCount; j++) {
            // offset is always 0
            int position = positions[j];
            if (block.isNull(position)) {
                blockBuilder.appendNull();
            } else {
                elementType.appendTo(block, position, blockBuilder);
            }
        }
    }
    return blockBuilder.build();
}

Example 37 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class Page method getRelatedDictionaryBlocks.

private Map<DictionaryId, DictionaryBlockIndexes> getRelatedDictionaryBlocks() {
    Map<DictionaryId, DictionaryBlockIndexes> relatedDictionaryBlocks = new HashMap<>();
    for (int i = 0; i < blocks.length; i++) {
        Block block = blocks[i];
        if (block instanceof DictionaryBlock) {
            DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
            relatedDictionaryBlocks.computeIfAbsent(dictionaryBlock.getDictionarySourceId(), id -> new DictionaryBlockIndexes()).addBlock(dictionaryBlock, i);
        }
    }
    return relatedDictionaryBlocks;
}

Example 38 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class TestGroupByHash method testMemoryReservationYieldWithDictionary.

@Test
public void testMemoryReservationYieldWithDictionary() {
    // Create a page with positionCount >> expected size of groupByHash
    int dictionaryLength = 1_000;
    int length = 2_000_000;
    int[] ids = IntStream.range(0, dictionaryLength).toArray();
    DictionaryId dictionaryId = randomDictionaryId();
    Block valuesBlock = new DictionaryBlock(dictionaryLength, createStringSequenceBlock(0, length), ids, dictionaryId);
    Block hashBlock = new DictionaryBlock(dictionaryLength, getHashBlock(ImmutableList.of(VARCHAR), valuesBlock), ids, dictionaryId);
    Page page = new Page(valuesBlock, hashBlock);
    AtomicInteger currentQuota = new AtomicInteger(0);
    AtomicInteger allowedQuota = new AtomicInteger(3);
    UpdateMemory updateMemory = () -> {
        if (currentQuota.get() < allowedQuota.get()) {
            currentQuota.getAndIncrement();
            return true;
        }
        return false;
    };
    int yields = 0;
    // test addPage
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    boolean finish = false;
    Work<?> addPageWork = groupByHash.addPage(page);
    while (!finish) {
        finish = addPageWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(addPageWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
    // test getGroupIds
    currentQuota.set(0);
    allowedQuota.set(3);
    yields = 0;
    groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    finish = false;
    Work<GroupByIdBlock> getGroupIdsWork = groupByHash.getGroupIds(page);
    while (!finish) {
        finish = getGroupIdsWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(getGroupIdsWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    assertEquals(dictionaryLength, getGroupIdsWork.getResult().getPositionCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log2(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
}

Example 39 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class TestFilterFunction method testFilter.

@Test
public void testFilter() {
    ConnectorSession session = new TestingConnectorSession(ImmutableList.of());
    FilterFunction filter = new FilterFunction(session.getSqlFunctionProperties(), true, new IsOddPredicate());
    Block numbers = makeNumbers(0, 1000);
    int[] allPositions = makePositions(0, 1000, 1);
    assertFilter(filter, numbers, allPositions, allPositions.length);
    Block dictionaryNumbers = new DictionaryBlock(numbers, allPositions);
    // Sparse coverage of the dictionary values
    int[] sparsePositions = makePositions(1, 300, 3);
    assertFilter(filter, dictionaryNumbers, sparsePositions, sparsePositions.length);
    // Full coverage of the dictionary values
    assertFilter(filter, dictionaryNumbers, allPositions, allPositions.length);
    // Test with a different DictionaryBlock over the same numbers. Results are reused. The DictionaryBlock covers the
    // values sparsely. TheDictionaryBlock itself is accessed sparsely.
    DictionaryBlock otherDictionary = new DictionaryBlock(numbers, makePositions(1, 332, 3));
    int[] otherDictionaryPositions = makePositions(0, 150, 2);
    assertFilter(filter, otherDictionary, otherDictionaryPositions, otherDictionaryPositions.length);
    // Repeat test on a DictionaryBlock over different content to make sure that cached results are not reused.
    assertFilter(filter, new DictionaryBlock(makeNumbers(1, 1001), allPositions), allPositions, allPositions.length);
}

Example 40 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class CommonSubExpressionBenchmark method createDictionaryStringJsonPage.

private static Page createDictionaryStringJsonPage() {
    int dictionarySize = POSITIONS / 5;
    BlockBuilder builder = VARCHAR.createBlockBuilder(null, dictionarySize);
    for (int i = 0; i < dictionarySize; i++) {
        VARCHAR.writeString(builder, "{\"a\": 1, \"b\": 2}");
    }
    int[] ids = new int[POSITIONS];
    for (int i = 0; i < POSITIONS; i++) {
        ids[i] = i % dictionarySize;
    }
    return new Page(new DictionaryBlock(builder.build(), ids));
}