Search in sources :

Example 6 with TypeUtils.getHashBlock

use of io.prestosql.type.TypeUtils.getHashBlock in project hetu-core by openlookeng.

the class TestGroupByHash method testAppendTo.

@Test
public void testAppendTo() {
    Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100);
    Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock);
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER);
    Work<GroupByIdBlock> work = groupByHash.getGroupIds(new Page(valuesBlock, hashBlock));
    work.process();
    GroupByIdBlock groupIds = work.getResult();
    for (int i = 0; i < groupIds.getPositionCount(); i++) {
        assertEquals(groupIds.getGroupId(i), i);
    }
    assertEquals(groupByHash.getGroupCount(), 100);
    PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
    for (int i = 0; i < groupByHash.getGroupCount(); i++) {
        pageBuilder.declarePosition();
        groupByHash.appendValuesTo(i, pageBuilder, 0);
    }
    Page page = pageBuilder.build();
    // Ensure that all blocks have the same positionCount
    for (int i = 0; i < groupByHash.getTypes().size(); i++) {
        assertEquals(page.getBlock(i).getPositionCount(), 100);
    }
    assertEquals(page.getPositionCount(), 100);
    BlockAssertions.assertBlockEquals(VARCHAR, page.getBlock(0), valuesBlock);
    BlockAssertions.assertBlockEquals(BIGINT, page.getBlock(1), hashBlock);
}
Also used : GroupByHash.createGroupByHash(io.prestosql.operator.GroupByHash.createGroupByHash) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) BlockAssertions.createLongsBlock(io.prestosql.block.BlockAssertions.createLongsBlock) TypeUtils.getHashBlock(io.prestosql.type.TypeUtils.getHashBlock) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) BlockAssertions.createStringSequenceBlock(io.prestosql.block.BlockAssertions.createStringSequenceBlock) Page(io.prestosql.spi.Page) PageBuilder(io.prestosql.spi.PageBuilder) Test(org.testng.annotations.Test)

Example 7 with TypeUtils.getHashBlock

use of io.prestosql.type.TypeUtils.getHashBlock in project hetu-core by openlookeng.

the class TestGroupByHash method testGetGroupNeedRehash.

@Test
public void testGetGroupNeedRehash() {
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER);
    // GroupByHash: the default maxFill=192, so we add a null into block when position is 192.
    Block block = createLongSequenceBlockWithNull(1, 200, 192);
    Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(BIGINT), block);
    Page page = new Page(block, hashBlock);
    Work<GroupByIdBlock> work = groupByHash.getGroupIds(page);
    boolean flag = work.process();
    if (!flag) {
        throw new RuntimeException("the block not be all process.");
    }
}
Also used : GroupByHash.createGroupByHash(io.prestosql.operator.GroupByHash.createGroupByHash) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) BlockAssertions.createLongsBlock(io.prestosql.block.BlockAssertions.createLongsBlock) TypeUtils.getHashBlock(io.prestosql.type.TypeUtils.getHashBlock) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) BlockAssertions.createStringSequenceBlock(io.prestosql.block.BlockAssertions.createStringSequenceBlock) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 8 with TypeUtils.getHashBlock

use of io.prestosql.type.TypeUtils.getHashBlock in project hetu-core by openlookeng.

the class TestGroupByHash method testForceRehash.

@Test
public void testForceRehash() {
    // Create a page with positionCount >> expected size of groupByHash
    Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100);
    Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock);
    // Create group by hash with extremely small size
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 4, JOIN_COMPILER);
    groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)).process();
    // Ensure that all groups are present in group by hash
    for (int i = 0; i < valuesBlock.getPositionCount(); i++) {
        assertTrue(groupByHash.contains(i, new Page(valuesBlock, hashBlock), CONTAINS_CHANNELS));
    }
}
Also used : GroupByHash.createGroupByHash(io.prestosql.operator.GroupByHash.createGroupByHash) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) BlockAssertions.createLongsBlock(io.prestosql.block.BlockAssertions.createLongsBlock) TypeUtils.getHashBlock(io.prestosql.type.TypeUtils.getHashBlock) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) BlockAssertions.createStringSequenceBlock(io.prestosql.block.BlockAssertions.createStringSequenceBlock) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 9 with TypeUtils.getHashBlock

use of io.prestosql.type.TypeUtils.getHashBlock in project hetu-core by openlookeng.

the class TestGroupByHash method testAddPage.

@Test
public void testAddPage() {
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER);
    for (int tries = 0; tries < 2; tries++) {
        for (int value = 0; value < MAX_GROUP_ID; value++) {
            Block block = BlockAssertions.createLongsBlock(value);
            Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(BIGINT), block);
            Page page = new Page(block, hashBlock);
            for (int addValuesTries = 0; addValuesTries < 10; addValuesTries++) {
                groupByHash.addPage(page).process();
                assertEquals(groupByHash.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID);
                // add the page again using get group ids and make sure the group count didn't change
                Work<GroupByIdBlock> work = groupByHash.getGroupIds(page);
                work.process();
                GroupByIdBlock groupIds = work.getResult();
                assertEquals(groupByHash.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID);
                assertEquals(groupIds.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID);
                // verify the first position
                assertEquals(groupIds.getPositionCount(), 1);
                long groupId = groupIds.getGroupId(0);
                assertEquals(groupId, value);
            }
        }
    }
}
Also used : GroupByHash.createGroupByHash(io.prestosql.operator.GroupByHash.createGroupByHash) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) BlockAssertions.createLongsBlock(io.prestosql.block.BlockAssertions.createLongsBlock) TypeUtils.getHashBlock(io.prestosql.type.TypeUtils.getHashBlock) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) BlockAssertions.createStringSequenceBlock(io.prestosql.block.BlockAssertions.createStringSequenceBlock) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 10 with TypeUtils.getHashBlock

use of io.prestosql.type.TypeUtils.getHashBlock in project hetu-core by openlookeng.

the class TestGroupByHash method testContains.

@Test
public void testContains() {
    Block valuesBlock = BlockAssertions.createDoubleSequenceBlock(0, 10);
    Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(DOUBLE), valuesBlock);
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(DOUBLE), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER);
    groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)).process();
    Block testBlock = BlockAssertions.createDoublesBlock((double) 3);
    Block testHashBlock = TypeUtils.getHashBlock(ImmutableList.of(DOUBLE), testBlock);
    assertTrue(groupByHash.contains(0, new Page(testBlock, testHashBlock), CONTAINS_CHANNELS));
    testBlock = BlockAssertions.createDoublesBlock(11.0);
    testHashBlock = TypeUtils.getHashBlock(ImmutableList.of(DOUBLE), testBlock);
    assertFalse(groupByHash.contains(0, new Page(testBlock, testHashBlock), CONTAINS_CHANNELS));
}
Also used : GroupByHash.createGroupByHash(io.prestosql.operator.GroupByHash.createGroupByHash) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) BlockAssertions.createLongsBlock(io.prestosql.block.BlockAssertions.createLongsBlock) TypeUtils.getHashBlock(io.prestosql.type.TypeUtils.getHashBlock) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) BlockAssertions.createStringSequenceBlock(io.prestosql.block.BlockAssertions.createStringSequenceBlock) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Aggregations

BlockAssertions.createLongSequenceBlock (io.prestosql.block.BlockAssertions.createLongSequenceBlock)10 BlockAssertions.createLongsBlock (io.prestosql.block.BlockAssertions.createLongsBlock)10 BlockAssertions.createStringSequenceBlock (io.prestosql.block.BlockAssertions.createStringSequenceBlock)10 GroupByHash.createGroupByHash (io.prestosql.operator.GroupByHash.createGroupByHash)10 Page (io.prestosql.spi.Page)10 Block (io.prestosql.spi.block.Block)10 DictionaryBlock (io.prestosql.spi.block.DictionaryBlock)10 TypeUtils.getHashBlock (io.prestosql.type.TypeUtils.getHashBlock)10 Test (org.testng.annotations.Test)10 PageBuilder (io.prestosql.spi.PageBuilder)3 ArrayList (java.util.ArrayList)1