use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.
the class TestDictionaryBlock method testBasicGetPositions.
@Test
public void testBasicGetPositions() {
Slice[] expectedValues = createExpectedValues(10);
Block dictionaryBlock = new DictionaryBlock(createSlicesBlock(expectedValues), new int[] { 0, 1, 2, 3, 4, 5 });
assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[0], expectedValues[1], expectedValues[2], expectedValues[3], expectedValues[4], expectedValues[5] });
DictionaryId dictionaryId = ((DictionaryBlock) dictionaryBlock).getDictionarySourceId();
// first getPositions
dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 8, 1, 2, 4, 5, 7, 9 }, 2, 4);
assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[4], expectedValues[5] });
assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
// second getPositions
dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 1, 3, 0, 0 }, 0, 3);
assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[5] });
assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
// third getPositions; we do not validate if -1 is an invalid position
dictionaryBlock = dictionaryBlock.getPositions(new int[] { -1, -1, 0, 1, 2 }, 2, 3);
assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[5] });
assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
// mixed getPositions
dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 2, 2 }, 0, 3);
assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[5], expectedValues[5] });
assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
// duplicated getPositions
dictionaryBlock = dictionaryBlock.getPositions(new int[] { 1, 1, 1, 1, 1 }, 0, 5);
assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5] });
assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
// out of range
for (int position : ImmutableList.of(-1, 6)) {
try {
dictionaryBlock.getPositions(new int[] { position }, 0, 1);
fail("Expected to fail");
} catch (IllegalArgumentException e) {
assertTrue(e.getMessage().startsWith("Invalid position"));
}
}
for (int offset : ImmutableList.of(-1, 6)) {
try {
dictionaryBlock.getPositions(new int[] { 0 }, offset, 1);
fail("Expected to fail");
} catch (IndexOutOfBoundsException e) {
assertTrue(e.getMessage().startsWith("Invalid offset"));
}
}
for (int length : ImmutableList.of(-1, 6)) {
try {
dictionaryBlock.getPositions(new int[] { 0 }, 0, length);
fail("Expected to fail");
} catch (IndexOutOfBoundsException e) {
assertTrue(e.getMessage().startsWith("Invalid offset"));
}
}
}
use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.
the class Page method compact.
public void compact() {
if (getRetainedSizeInBytes() <= getSizeInBytes()) {
return;
}
for (int i = 0; i < blocks.length; i++) {
Block block = blocks[i];
if (block instanceof DictionaryBlock) {
continue;
}
// Compact the block
blocks[i] = block.copyRegion(0, block.getPositionCount());
}
Map<DictionaryId, DictionaryBlockIndexes> dictionaryBlocks = getRelatedDictionaryBlocks();
for (DictionaryBlockIndexes blockIndexes : dictionaryBlocks.values()) {
List<DictionaryBlock> compactBlocks = compactRelatedBlocks(blockIndexes.getBlocks());
List<Integer> indexes = blockIndexes.getIndexes();
for (int i = 0; i < compactBlocks.size(); i++) {
blocks[indexes.get(i)] = compactBlocks.get(i);
}
}
updateRetainedSize();
}
use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.
the class Page method getRelatedDictionaryBlocks.
private Map<DictionaryId, DictionaryBlockIndexes> getRelatedDictionaryBlocks() {
Map<DictionaryId, DictionaryBlockIndexes> relatedDictionaryBlocks = new HashMap<>();
for (int i = 0; i < blocks.length; i++) {
Block block = blocks[i];
if (block instanceof DictionaryBlock) {
DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
relatedDictionaryBlocks.computeIfAbsent(dictionaryBlock.getDictionarySourceId(), id -> new DictionaryBlockIndexes()).addBlock(dictionaryBlock, i);
}
}
return relatedDictionaryBlocks;
}
use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.
the class Page method compactRelatedBlocks.
private static List<DictionaryBlock> compactRelatedBlocks(List<DictionaryBlock> blocks) {
DictionaryBlock firstDictionaryBlock = blocks.get(0);
Block dictionary = firstDictionaryBlock.getDictionary();
int count = firstDictionaryBlock.getPositionCount();
int dictionarySize = dictionary.getPositionCount();
// determine which dictionary entries are referenced and build a reindex for them
int[] dictionaryPositionsToCopy = new int[min(dictionarySize, count)];
int[] remapIndex = new int[dictionarySize];
Arrays.fill(remapIndex, -1);
int numberOfIndexes = 0;
for (int i = 0; i < count; i++) {
int position = firstDictionaryBlock.getId(i);
if (remapIndex[position] == -1) {
dictionaryPositionsToCopy[numberOfIndexes] = position;
remapIndex[position] = numberOfIndexes;
numberOfIndexes++;
}
}
// entire dictionary is referenced
if (numberOfIndexes == dictionarySize) {
return blocks;
}
// compact the dictionaries
int[] newIds = getNewIds(count, firstDictionaryBlock, remapIndex);
List<DictionaryBlock> outputDictionaryBlocks = new ArrayList<>(blocks.size());
DictionaryId newDictionaryId = randomDictionaryId();
for (DictionaryBlock dictionaryBlock : blocks) {
if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) {
throw new IllegalArgumentException("dictionarySourceIds must be the same");
}
try {
Block compactDictionary = dictionaryBlock.getDictionary().copyPositions(dictionaryPositionsToCopy, 0, numberOfIndexes);
outputDictionaryBlocks.add(new DictionaryBlock(count, compactDictionary, newIds, true, newDictionaryId));
} catch (UnsupportedOperationException e) {
// ignore if copy positions is not supported for the dictionary
outputDictionaryBlocks.add(dictionaryBlock);
}
}
return outputDictionaryBlocks;
}
use of io.prestosql.spi.block.DictionaryId in project hetu-core by openlookeng.
the class TestGroupByHash method testMemoryReservationYieldWithDictionary.
@Test
public void testMemoryReservationYieldWithDictionary() {
// Create a page with positionCount >> expected size of groupByHash
int dictionaryLength = 1_000;
int length = 2_000_000;
int[] ids = IntStream.range(0, dictionaryLength).toArray();
DictionaryId dictionaryId = randomDictionaryId();
Block valuesBlock = new DictionaryBlock(dictionaryLength, createStringSequenceBlock(0, length), ids, dictionaryId);
Block hashBlock = new DictionaryBlock(dictionaryLength, getHashBlock(ImmutableList.of(VARCHAR), valuesBlock), ids, dictionaryId);
Page page = new Page(valuesBlock, hashBlock);
AtomicInteger currentQuota = new AtomicInteger(0);
AtomicInteger allowedQuota = new AtomicInteger(3);
UpdateMemory updateMemory = () -> {
if (currentQuota.get() < allowedQuota.get()) {
currentQuota.getAndIncrement();
return true;
}
return false;
};
int yields = 0;
// test addPage
GroupByHash groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
boolean finish = false;
Work<?> addPageWork = groupByHash.addPage(page);
while (!finish) {
finish = addPageWork.process();
if (!finish) {
assertEquals(currentQuota.get(), allowedQuota.get());
// assert if we are blocked, we are going to be blocked again without changing allowedQuota
assertFalse(addPageWork.process());
assertEquals(currentQuota.get(), allowedQuota.get());
yields++;
allowedQuota.getAndAdd(3);
}
}
// assert there is not anything missing
assertEquals(dictionaryLength, groupByHash.getGroupCount());
// assert we yield for every 3 rehashes
// currentQuota is essentially the count we have successfully rehashed
// the rehash count is 10 = log(1_000 / 0.75)
assertEquals(currentQuota.get(), 10);
assertEquals(currentQuota.get() / 3, yields);
// test getGroupIds
currentQuota.set(0);
allowedQuota.set(3);
yields = 0;
groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
finish = false;
Work<GroupByIdBlock> getGroupIdsWork = groupByHash.getGroupIds(page);
while (!finish) {
finish = getGroupIdsWork.process();
if (!finish) {
assertEquals(currentQuota.get(), allowedQuota.get());
// assert if we are blocked, we are going to be blocked again without changing allowedQuota
assertFalse(getGroupIdsWork.process());
assertEquals(currentQuota.get(), allowedQuota.get());
yields++;
allowedQuota.getAndAdd(3);
}
}
// assert there is not anything missing
assertEquals(dictionaryLength, groupByHash.getGroupCount());
assertEquals(dictionaryLength, getGroupIdsWork.getResult().getPositionCount());
// assert we yield for every 3 rehashes
// currentQuota is essentially the count we have successfully rehashed
// the rehash count is 10 = log2(1_000 / 0.75)
assertEquals(currentQuota.get(), 10);
assertEquals(currentQuota.get() / 3, yields);
}
Aggregations