Search in sources :

Example 1 with DictionaryId

use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.

the class TestDictionaryBlock method testBasicGetPositions.

@Test
public void testBasicGetPositions() {
    Slice[] expectedValues = createExpectedValues(10);
    Block dictionaryBlock = new DictionaryBlock(createSlicesBlock(expectedValues), new int[] { 0, 1, 2, 3, 4, 5 });
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[0], expectedValues[1], expectedValues[2], expectedValues[3], expectedValues[4], expectedValues[5] });
    DictionaryId dictionaryId = ((DictionaryBlock) dictionaryBlock).getDictionarySourceId();
    // first getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 8, 1, 2, 4, 5, 7, 9 }, 2, 4);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[4], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // second getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 1, 3, 0, 0 }, 0, 3);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // third getPositions; we do not validate if -1 is an invalid position
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { -1, -1, 0, 1, 2 }, 2, 3);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // mixed getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 2, 2 }, 0, 3);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[5], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // duplicated getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 1, 1, 1, 1, 1 }, 0, 5);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // out of range
    for (int position : ImmutableList.of(-1, 6)) {
        try {
            dictionaryBlock.getPositions(new int[] { position }, 0, 1);
            fail("Expected to fail");
        } catch (IllegalArgumentException e) {
            assertTrue(e.getMessage().startsWith("Invalid position"));
        }
    }
    for (int offset : ImmutableList.of(-1, 6)) {
        try {
            dictionaryBlock.getPositions(new int[] { 0 }, offset, 1);
            fail("Expected to fail");
        } catch (IndexOutOfBoundsException e) {
            assertTrue(e.getMessage().startsWith("Invalid offset"));
        }
    }
    for (int length : ImmutableList.of(-1, 6)) {
        try {
            dictionaryBlock.getPositions(new int[] { 0 }, 0, length);
            fail("Expected to fail");
        } catch (IndexOutOfBoundsException e) {
            assertTrue(e.getMessage().startsWith("Invalid offset"));
        }
    }
}
Also used : Slice(io.airlift.slice.Slice) DictionaryId(io.prestosql.spi.block.DictionaryId) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) BlockAssertions.createSlicesBlock(io.prestosql.block.BlockAssertions.createSlicesBlock) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) Test(org.testng.annotations.Test)

Example 2 with DictionaryId

use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.

the class Page method compact.

public void compact() {
    if (getRetainedSizeInBytes() <= getSizeInBytes()) {
        return;
    }
    for (int i = 0; i < blocks.length; i++) {
        Block block = blocks[i];
        if (block instanceof DictionaryBlock) {
            continue;
        }
        // Compact the block
        blocks[i] = block.copyRegion(0, block.getPositionCount());
    }
    Map<DictionaryId, DictionaryBlockIndexes> dictionaryBlocks = getRelatedDictionaryBlocks();
    for (DictionaryBlockIndexes blockIndexes : dictionaryBlocks.values()) {
        List<DictionaryBlock> compactBlocks = compactRelatedBlocks(blockIndexes.getBlocks());
        List<Integer> indexes = blockIndexes.getIndexes();
        for (int i = 0; i < compactBlocks.size(); i++) {
            blocks[indexes.get(i)] = compactBlocks.get(i);
        }
    }
    updateRetainedSize();
}
Also used : DictionaryId.randomDictionaryId(io.prestosql.spi.block.DictionaryId.randomDictionaryId) DictionaryId(io.prestosql.spi.block.DictionaryId) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block)

Example 3 with DictionaryId

use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.

the class Page method getRelatedDictionaryBlocks.

private Map<DictionaryId, DictionaryBlockIndexes> getRelatedDictionaryBlocks() {
    Map<DictionaryId, DictionaryBlockIndexes> relatedDictionaryBlocks = new HashMap<>();
    for (int i = 0; i < blocks.length; i++) {
        Block block = blocks[i];
        if (block instanceof DictionaryBlock) {
            DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
            relatedDictionaryBlocks.computeIfAbsent(dictionaryBlock.getDictionarySourceId(), id -> new DictionaryBlockIndexes()).addBlock(dictionaryBlock, i);
        }
    }
    return relatedDictionaryBlocks;
}
Also used : Arrays(java.util.Arrays) Properties(java.util.Properties) DictionaryId.randomDictionaryId(io.prestosql.spi.block.DictionaryId.randomDictionaryId) HashMap(java.util.HashMap) Math.min(java.lang.Math.min) String.format(java.lang.String.format) DictionaryId(io.prestosql.spi.block.DictionaryId) ArrayList(java.util.ArrayList) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) ClassLayout(org.openjdk.jol.info.ClassLayout) SizeOf.sizeOf(io.airlift.slice.SizeOf.sizeOf) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) HashMap(java.util.HashMap) DictionaryId.randomDictionaryId(io.prestosql.spi.block.DictionaryId.randomDictionaryId) DictionaryId(io.prestosql.spi.block.DictionaryId) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block)

Example 4 with DictionaryId

use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.

the class Page method compactRelatedBlocks.

private static List<DictionaryBlock> compactRelatedBlocks(List<DictionaryBlock> blocks) {
    DictionaryBlock firstDictionaryBlock = blocks.get(0);
    Block dictionary = firstDictionaryBlock.getDictionary();
    int count = firstDictionaryBlock.getPositionCount();
    int dictionarySize = dictionary.getPositionCount();
    // determine which dictionary entries are referenced and build a reindex for them
    int[] dictionaryPositionsToCopy = new int[min(dictionarySize, count)];
    int[] remapIndex = new int[dictionarySize];
    Arrays.fill(remapIndex, -1);
    int numberOfIndexes = 0;
    for (int i = 0; i < count; i++) {
        int position = firstDictionaryBlock.getId(i);
        if (remapIndex[position] == -1) {
            dictionaryPositionsToCopy[numberOfIndexes] = position;
            remapIndex[position] = numberOfIndexes;
            numberOfIndexes++;
        }
    }
    // entire dictionary is referenced
    if (numberOfIndexes == dictionarySize) {
        return blocks;
    }
    // compact the dictionaries
    int[] newIds = getNewIds(count, firstDictionaryBlock, remapIndex);
    List<DictionaryBlock> outputDictionaryBlocks = new ArrayList<>(blocks.size());
    DictionaryId newDictionaryId = randomDictionaryId();
    for (DictionaryBlock dictionaryBlock : blocks) {
        if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) {
            throw new IllegalArgumentException("dictionarySourceIds must be the same");
        }
        try {
            Block compactDictionary = dictionaryBlock.getDictionary().copyPositions(dictionaryPositionsToCopy, 0, numberOfIndexes);
            outputDictionaryBlocks.add(new DictionaryBlock(count, compactDictionary, newIds, true, newDictionaryId));
        } catch (UnsupportedOperationException e) {
            // ignore if copy positions is not supported for the dictionary
            outputDictionaryBlocks.add(dictionaryBlock);
        }
    }
    return outputDictionaryBlocks;
}
Also used : DictionaryId.randomDictionaryId(io.prestosql.spi.block.DictionaryId.randomDictionaryId) DictionaryId(io.prestosql.spi.block.DictionaryId) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) ArrayList(java.util.ArrayList) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block)

Example 5 with DictionaryId

use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.

the class TestGroupByHash method testMemoryReservationYieldWithDictionary.

@Test
public void testMemoryReservationYieldWithDictionary() {
    // Create a page with positionCount >> expected size of groupByHash
    int dictionaryLength = 1_000;
    int length = 2_000_000;
    int[] ids = IntStream.range(0, dictionaryLength).toArray();
    DictionaryId dictionaryId = randomDictionaryId();
    Block valuesBlock = new DictionaryBlock(dictionaryLength, createStringSequenceBlock(0, length), ids, dictionaryId);
    Block hashBlock = new DictionaryBlock(dictionaryLength, getHashBlock(ImmutableList.of(VARCHAR), valuesBlock), ids, dictionaryId);
    Page page = new Page(valuesBlock, hashBlock);
    AtomicInteger currentQuota = new AtomicInteger(0);
    AtomicInteger allowedQuota = new AtomicInteger(3);
    UpdateMemory updateMemory = () -> {
        if (currentQuota.get() < allowedQuota.get()) {
            currentQuota.getAndIncrement();
            return true;
        }
        return false;
    };
    int yields = 0;
    // test addPage
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    boolean finish = false;
    Work<?> addPageWork = groupByHash.addPage(page);
    while (!finish) {
        finish = addPageWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(addPageWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
    // test getGroupIds
    currentQuota.set(0);
    allowedQuota.set(3);
    yields = 0;
    groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    finish = false;
    Work<GroupByIdBlock> getGroupIdsWork = groupByHash.getGroupIds(page);
    while (!finish) {
        finish = getGroupIdsWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(getGroupIdsWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    assertEquals(dictionaryLength, getGroupIdsWork.getResult().getPositionCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log2(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
}
Also used : DictionaryId(io.prestosql.spi.block.DictionaryId) DictionaryId.randomDictionaryId(io.prestosql.spi.block.DictionaryId.randomDictionaryId) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Page(io.prestosql.spi.Page) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) GroupByHash.createGroupByHash(io.prestosql.operator.GroupByHash.createGroupByHash) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) BlockAssertions.createLongsBlock(io.prestosql.block.BlockAssertions.createLongsBlock) TypeUtils.getHashBlock(io.prestosql.type.TypeUtils.getHashBlock) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) BlockAssertions.createStringSequenceBlock(io.prestosql.block.BlockAssertions.createStringSequenceBlock) Test(org.testng.annotations.Test)

Aggregations

Block (io.prestosql.spi.block.Block)6 DictionaryBlock (io.prestosql.spi.block.DictionaryBlock)6 DictionaryId (io.prestosql.spi.block.DictionaryId)6 DictionaryId.randomDictionaryId (io.prestosql.spi.block.DictionaryId.randomDictionaryId)5 Test (org.testng.annotations.Test)3 Slice (io.airlift.slice.Slice)2 ArrayList (java.util.ArrayList)2 SizeOf.sizeOf (io.airlift.slice.SizeOf.sizeOf)1 BlockAssertions.createLongSequenceBlock (io.prestosql.block.BlockAssertions.createLongSequenceBlock)1 BlockAssertions.createLongsBlock (io.prestosql.block.BlockAssertions.createLongsBlock)1 BlockAssertions.createSlicesBlock (io.prestosql.block.BlockAssertions.createSlicesBlock)1 BlockAssertions.createStringSequenceBlock (io.prestosql.block.BlockAssertions.createStringSequenceBlock)1 GroupByHash.createGroupByHash (io.prestosql.operator.GroupByHash.createGroupByHash)1 Page (io.prestosql.spi.Page)1 BlockBuilder (io.prestosql.spi.block.BlockBuilder)1 TypeUtils.getHashBlock (io.prestosql.type.TypeUtils.getHashBlock)1 Math.min (java.lang.Math.min)1 String.format (java.lang.String.format)1 Arrays (java.util.Arrays)1 HashMap (java.util.HashMap)1