Search in sources :

Example 16 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessor method testYieldProjection.

@Test
public void testYieldProjection() {
    // each projection can finish without yield
    // while between two projections, there is a yield
    int rows = 128;
    int columns = 20;
    DriverYieldSignal yieldSignal = new DriverYieldSignal();
    PageProcessor pageProcessor = new PageProcessor(Optional.empty(), Collections.nCopies(columns, new YieldPageProjection(new InputPageProjection(0, VARCHAR))), OptionalInt.of(MAX_BATCH_SIZE));
    Slice[] slices = new Slice[rows];
    Arrays.fill(slices, Slices.allocate(rows));
    Page inputPage = new Page(createSlicesBlock(slices));
    Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, yieldSignal, inputPage);
    // Also, we would like to reset the yield signal before starting to process the next column in order NOT to yield per position inside the column.
    for (int i = 0; i < columns - 1; i++) {
        assertTrue(output.hasNext());
        assertNull(output.next().orElse(null));
        assertTrue(yieldSignal.isSet());
        yieldSignal.reset();
    }
    assertTrue(output.hasNext());
    Page actualPage = output.next().orElse(null);
    assertNotNull(actualPage);
    assertTrue(yieldSignal.isSet());
    yieldSignal.reset();
    Block[] blocks = new Block[columns];
    Arrays.fill(blocks, createSlicesBlock(Arrays.copyOfRange(slices, 0, rows)));
    Page expectedPage = new Page(blocks);
    assertPageEquals(Collections.nCopies(columns, VARCHAR), actualPage, expectedPage);
    assertFalse(output.hasNext());
}
Also used : Optional(java.util.Optional) Slice(io.airlift.slice.Slice) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) BlockAssertions.createSlicesBlock(io.prestosql.block.BlockAssertions.createSlicesBlock) BlockAssertions.createStringsBlock(io.prestosql.block.BlockAssertions.createStringsBlock) LazyBlock(io.prestosql.spi.block.LazyBlock) VariableWidthBlock(io.prestosql.spi.block.VariableWidthBlock) Block(io.prestosql.spi.block.Block) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 17 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessor method testProjectLazyLoad.

@Test
public void testProjectLazyLoad() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection()), OptionalInt.of(MAX_BATCH_SIZE));
    // if channel 1 is loaded, test will fail
    Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 1);
    assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100)));
}
Also used : TestingTicker(io.airlift.testing.TestingTicker) ADD(io.prestosql.spi.function.OperatorType.ADD) Arrays(java.util.Arrays) MIN_PAGE_SIZE_IN_BYTES(io.prestosql.operator.project.PageProcessor.MIN_PAGE_SIZE_IN_BYTES) BuiltInFunctionHandle(io.prestosql.spi.function.BuiltInFunctionHandle) Test(org.testng.annotations.Test) Duration(io.airlift.units.Duration) CallExpression(io.prestosql.spi.relation.CallExpression) MAX_BATCH_SIZE(io.prestosql.operator.project.PageProcessor.MAX_BATCH_SIZE) Executors.newSingleThreadScheduledExecutor(java.util.concurrent.Executors.newSingleThreadScheduledExecutor) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Expressions.call(io.prestosql.sql.relational.Expressions.call) SelectedPositions.positionsRange(io.prestosql.operator.project.SelectedPositions.positionsRange) Signature.internalOperator(io.prestosql.spi.function.Signature.internalOperator) Slices(io.airlift.slice.Slices) BlockAssertions.createSlicesBlock(io.prestosql.block.BlockAssertions.createSlicesBlock) AggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Type(io.prestosql.spi.type.Type) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Assert.assertFalse(org.testng.Assert.assertFalse) BlockAssertions.createStringsBlock(io.prestosql.block.BlockAssertions.createStringsBlock) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) Collections.nCopies(java.util.Collections.nCopies) LazyBlock(io.prestosql.spi.block.LazyBlock) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Assert.assertNotNull(org.testng.Assert.assertNotNull) List(java.util.List) ClassLayout(org.openjdk.jol.info.ClassLayout) SPLIT_RUN_QUANTA(io.prestosql.execution.executor.PrioritizedSplitRunner.SPLIT_RUN_QUANTA) Optional(java.util.Optional) Work(io.prestosql.operator.Work) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) Slice(io.airlift.slice.Slice) Assert.assertNull(org.testng.Assert.assertNull) SESSION(io.prestosql.testing.TestingConnectorSession.SESSION) VariableWidthBlock(io.prestosql.spi.block.VariableWidthBlock) NANOSECONDS(java.util.concurrent.TimeUnit.NANOSECONDS) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Assert.assertEquals(org.testng.Assert.assertEquals) CompletedWork(io.prestosql.operator.CompletedWork) OptionalInt(java.util.OptionalInt) Supplier(java.util.function.Supplier) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) String.join(java.lang.String.join) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Block(io.prestosql.spi.block.Block) AfterClass(org.testng.annotations.AfterClass) MAX_PAGE_SIZE_IN_BYTES(io.prestosql.operator.project.PageProcessor.MAX_PAGE_SIZE_IN_BYTES) LocalMemoryContext(io.prestosql.memory.context.LocalMemoryContext) Iterator(java.util.Iterator) Expressions.constant(io.prestosql.sql.relational.Expressions.constant) Page(io.prestosql.spi.Page) ExpressionProfiler(io.prestosql.sql.gen.ExpressionProfiler) Expressions.field(io.prestosql.sql.relational.Expressions.field) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) Assert.assertTrue(org.testng.Assert.assertTrue) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) PageAssertions.assertPageEquals(io.prestosql.operator.PageAssertions.assertPageEquals) LocalMemoryContext(io.prestosql.memory.context.LocalMemoryContext) LazyBlock(io.prestosql.spi.block.LazyBlock) Optional(java.util.Optional) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 18 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestDictionaryAwarePageProjection method testProjectFastReturnIgnoreYield.

private static void testProjectFastReturnIgnoreYield(Block block, DictionaryAwarePageProjection projection) {
    DriverYieldSignal yieldSignal = new DriverYieldSignal();
    Work<Block> work = projection.project(null, yieldSignal, new Page(block), SelectedPositions.positionsRange(5, 10));
    yieldSignal.setWithDelay(1, executor);
    yieldSignal.forceYieldForTesting();
    // yield signal is ignored given the block has already been loaded
    assertTrue(work.process());
    Block result = work.getResult();
    yieldSignal.reset();
    assertBlockEquals(BIGINT, result, block.getRegion(5, 10));
    assertInstanceOf(result, DictionaryBlock.class);
}
Also used : DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) RunLengthEncodedBlock(io.prestosql.spi.block.RunLengthEncodedBlock) LongArrayBlock(io.prestosql.spi.block.LongArrayBlock) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) Block(io.prestosql.spi.block.Block) LazyBlock(io.prestosql.spi.block.LazyBlock) BlockAssertions.createLongSequenceBlock(io.prestosql.block.BlockAssertions.createLongSequenceBlock) Page(io.prestosql.spi.Page)

Example 19 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessorCompiler method testSanityRLE.

@Test
public void testSanityRLE() {
    PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get();
    Slice varcharValue = Slices.utf8Slice("hello");
    Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock);
    assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock);
    RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0);
    assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L);
    RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1);
    assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue);
}
Also used : PageProcessor(io.prestosql.operator.project.PageProcessor) Slice(io.airlift.slice.Slice) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Page(io.prestosql.spi.Page) RunLengthEncodedBlock(io.prestosql.spi.block.RunLengthEncodedBlock) Test(org.testng.annotations.Test)

Example 20 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessorCompiler method testSanityColumnarDictionary.

@Test
public void testSanityColumnarDictionary() {
    PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get();
    Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10);
}
Also used : PageProcessor(io.prestosql.operator.project.PageProcessor) DictionaryBlock(io.prestosql.spi.block.DictionaryBlock) BlockAssertions.createLongDictionaryBlock(io.prestosql.block.BlockAssertions.createLongDictionaryBlock) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Aggregations

DriverYieldSignal (io.prestosql.operator.DriverYieldSignal)26 Page (io.prestosql.spi.Page)22 Test (org.testng.annotations.Test)18 Block (io.prestosql.spi.block.Block)9 Optional (java.util.Optional)9 BlockAssertions.createLongSequenceBlock (io.prestosql.block.BlockAssertions.createLongSequenceBlock)7 PageProcessor (io.prestosql.operator.project.PageProcessor)7 LazyBlock (io.prestosql.spi.block.LazyBlock)7 DictionaryBlock (io.prestosql.spi.block.DictionaryBlock)6 BuiltInFunctionHandle (io.prestosql.spi.function.BuiltInFunctionHandle)6 CallExpression (io.prestosql.spi.relation.CallExpression)6 Slice (io.airlift.slice.Slice)5 LocalMemoryContext (io.prestosql.memory.context.LocalMemoryContext)5 RunLengthEncodedBlock (io.prestosql.spi.block.RunLengthEncodedBlock)5 VariableWidthBlock (io.prestosql.spi.block.VariableWidthBlock)5 Type (io.prestosql.spi.type.Type)5 BlockAssertions.createSlicesBlock (io.prestosql.block.BlockAssertions.createSlicesBlock)4 BlockAssertions.createStringsBlock (io.prestosql.block.BlockAssertions.createStringsBlock)4 AggregatedMemoryContext (io.prestosql.memory.context.AggregatedMemoryContext)4 AggregatedMemoryContext.newSimpleAggregatedMemoryContext (io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext)4