Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 1 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class Page method compact.

public Page compact() {
    if (getRetainedSizeInBytes() <= getSizeInBytes()) {
        return this;
    }
    for (int i = 0; i < blocks.length; i++) {
        Block block = blocks[i];
        if (block instanceof DictionaryBlock) {
            continue;
        }
        // Compact the block
        blocks[i] = block.copyRegion(0, block.getPositionCount());
    }
    Map<DictionaryId, DictionaryBlockIndexes> dictionaryBlocks = getRelatedDictionaryBlocks();
    for (DictionaryBlockIndexes blockIndexes : dictionaryBlocks.values()) {
        List<DictionaryBlock> compactBlocks = compactRelatedBlocks(blockIndexes.getBlocks());
        List<Integer> indexes = blockIndexes.getIndexes();
        for (int i = 0; i < compactBlocks.size(); i++) {
            blocks[indexes.get(i)] = compactBlocks.get(i);
        }
    }
    updateRetainedSize();
    return this;
}

Example 2 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class Page method compactRelatedBlocks.

private static List<DictionaryBlock> compactRelatedBlocks(List<DictionaryBlock> blocks) {
    DictionaryBlock firstDictionaryBlock = blocks.get(0);
    Block dictionary = firstDictionaryBlock.getDictionary();
    int positionCount = firstDictionaryBlock.getPositionCount();
    int dictionarySize = dictionary.getPositionCount();
    // determine which dictionary entries are referenced and build a reindex for them
    int[] dictionaryPositionsToCopy = new int[min(dictionarySize, positionCount)];
    int[] remapIndex = new int[dictionarySize];
    Arrays.fill(remapIndex, -1);
    int numberOfIndexes = 0;
    for (int i = 0; i < positionCount; i++) {
        int position = firstDictionaryBlock.getId(i);
        if (remapIndex[position] == -1) {
            dictionaryPositionsToCopy[numberOfIndexes] = position;
            remapIndex[position] = numberOfIndexes;
            numberOfIndexes++;
        }
    }
    // entire dictionary is referenced
    if (numberOfIndexes == dictionarySize) {
        return blocks;
    }
    // compact the dictionaries
    int[] newIds = getNewIds(positionCount, firstDictionaryBlock, remapIndex);
    List<DictionaryBlock> outputDictionaryBlocks = new ArrayList<>(blocks.size());
    DictionaryId newDictionaryId = randomDictionaryId();
    for (DictionaryBlock dictionaryBlock : blocks) {
        if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) {
            throw new IllegalArgumentException("dictionarySourceIds must be the same");
        }
        try {
            Block compactDictionary = dictionaryBlock.getDictionary().copyPositions(dictionaryPositionsToCopy, 0, numberOfIndexes);
            outputDictionaryBlocks.add(new DictionaryBlock(positionCount, compactDictionary, newIds, true, newDictionaryId));
        } catch (UnsupportedOperationException e) {
            // ignore if copy positions is not supported for the dictionary
            outputDictionaryBlocks.add(dictionaryBlock);
        }
    }
    return outputDictionaryBlocks;
}

Example 3 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestGroupByHash method testMemoryReservationYieldWithDictionary.

@Test
public void testMemoryReservationYieldWithDictionary() {
    // Create a page with positionCount >> expected size of groupByHash
    int dictionaryLength = 1_000;
    int length = 2_000_000;
    int[] ids = IntStream.range(0, dictionaryLength).toArray();
    DictionaryId dictionaryId = randomDictionaryId();
    Block valuesBlock = new DictionaryBlock(dictionaryLength, createStringSequenceBlock(0, length), ids, dictionaryId);
    Block hashBlock = new DictionaryBlock(dictionaryLength, getHashBlock(ImmutableList.of(VARCHAR), valuesBlock), ids, dictionaryId);
    Page page = new Page(valuesBlock, hashBlock);
    AtomicInteger currentQuota = new AtomicInteger(0);
    AtomicInteger allowedQuota = new AtomicInteger(3);
    UpdateMemory updateMemory = () -> {
        if (currentQuota.get() < allowedQuota.get()) {
            currentQuota.getAndIncrement();
            return true;
        }
        return false;
    };
    int yields = 0;
    // test addPage
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    boolean finish = false;
    Work<?> addPageWork = groupByHash.addPage(page);
    while (!finish) {
        finish = addPageWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(addPageWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
    // test getGroupIds
    currentQuota.set(0);
    allowedQuota.set(3);
    yields = 0;
    groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    finish = false;
    Work<GroupByIdBlock> getGroupIdsWork = groupByHash.getGroupIds(page);
    while (!finish) {
        finish = getGroupIdsWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(getGroupIdsWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    assertEquals(dictionaryLength, getGroupIdsWork.getResult().getPositionCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log2(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
}

Example 4 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class SliceDictionarySelectiveReader method getBlockView.

@Override
public BlockLease getBlockView(int[] positions, int positionCount) {
    checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
    checkState(outputRequired, "This stream reader doesn't produce output");
    checkState(positionCount <= outputPositionCount, "Not enough values");
    checkState(!valuesInUse, "BlockLease hasn't been closed yet");
    if (allNulls) {
        return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount));
    }
    if (positionCount < outputPositionCount) {
        compactValues(positions, positionCount);
    }
    wrapDictionaryIfNecessary();
    return newLease(new DictionaryBlock(positionCount, dictionary, values));
}

Example 5 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestLookupJoinPageBuilder method testDifferentPositions.

@Test
public void testDifferentPositions() {
    int entries = 100;
    BlockBuilder blockBuilder = BIGINT.createFixedSizeBlockBuilder(entries);
    for (int i = 0; i < entries; i++) {
        BIGINT.writeLong(blockBuilder, i);
    }
    Block block = blockBuilder.build();
    Page page = new Page(block);
    JoinProbeFactory joinProbeFactory = new JoinProbeFactory(new int[] { 0 }, ImmutableList.of(0), OptionalInt.empty());
    LookupSource lookupSource = new TestLookupSource(ImmutableList.of(BIGINT), page);
    LookupJoinPageBuilder lookupJoinPageBuilder = new LookupJoinPageBuilder(ImmutableList.of(BIGINT));
    // empty
    JoinProbe probe = joinProbeFactory.createJoinProbe(page);
    Page output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertTrue(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), 0);
    lookupJoinPageBuilder.reset();
    // the probe covers non-sequential positions
    probe = joinProbeFactory.createJoinProbe(page);
    for (int joinPosition = 0; probe.advanceNextPosition(); joinPosition++) {
        if (joinPosition % 2 == 1) {
            continue;
        }
        lookupJoinPageBuilder.appendRow(probe, lookupSource, joinPosition);
    }
    output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertTrue(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), entries / 2);
    for (int i = 0; i < entries / 2; i++) {
        assertEquals(output.getBlock(0).getLong(i), i * 2);
        assertEquals(output.getBlock(1).getLong(i), i * 2);
    }
    lookupJoinPageBuilder.reset();
    // the probe covers everything
    probe = joinProbeFactory.createJoinProbe(page);
    for (int joinPosition = 0; probe.advanceNextPosition(); joinPosition++) {
        lookupJoinPageBuilder.appendRow(probe, lookupSource, joinPosition);
    }
    output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertFalse(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), entries);
    for (int i = 0; i < entries; i++) {
        assertEquals(output.getBlock(0).getLong(i), i);
        assertEquals(output.getBlock(1).getLong(i), i);
    }
    lookupJoinPageBuilder.reset();
    // the probe covers some sequential positions
    probe = joinProbeFactory.createJoinProbe(page);
    for (int joinPosition = 0; probe.advanceNextPosition(); joinPosition++) {
        if (joinPosition < 10 || joinPosition >= 50) {
            continue;
        }
        lookupJoinPageBuilder.appendRow(probe, lookupSource, joinPosition);
    }
    output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertFalse(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), 40);
    for (int i = 10; i < 50; i++) {
        assertEquals(output.getBlock(0).getLong(i - 10), i);
        assertEquals(output.getBlock(1).getLong(i - 10), i);
    }
}