use of io.prestosql.spi.plan.PlanNodeId in project hetu-core by openlookeng.
the class TestHashSemiJoinOperator method testHashSemiJoinSnapshot.
@Test
public void testHashSemiJoinSnapshot() {
DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
// build
OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
RowPagesBuilder rowPagesBuilder = rowPagesBuilder(true, Ints.asList(0), BIGINT);
Operator buildOperator = createValuesOperator(operatorContext, rowPagesBuilder.row(10L).row(30L).row(30L).row(35L).row(36L).row(37L).row(50L).build());
SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(1, new PlanNodeId("test"), rowPagesBuilder.getTypes().get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager()));
Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
while (!driver.isFinished()) {
driver.process();
}
Object snapshot = setBuilderOperator.capture(setBuilderOperator.getOperatorContext().getDriverContext().getSerde());
assertEquals(SnapshotTestUtil.toSimpleSnapshotMapping(snapshot), createSetBuilderOperatorExpectedMapping());
setBuilderOperator.restore(snapshot, operatorContext.getDriverContext().getSerde());
// probe
List<Type> probeTypes = ImmutableList.of(BIGINT, BIGINT);
RowPagesBuilder rowPagesBuilderProbe = rowPagesBuilder(true, Ints.asList(0), BIGINT, BIGINT);
List<Page> probeInput = rowPagesBuilderProbe.addSequencePage(10, 30, 0).build();
Optional<Integer> probeHashChannel = true ? Optional.of(probeTypes.size()) : Optional.empty();
HashSemiJoinOperatorFactory joinOperatorFactory = new HashSemiJoinOperatorFactory(2, new PlanNodeId("test"), setBuilderOperatorFactory.getSetProvider(), rowPagesBuilderProbe.getTypes(), 0, probeHashChannel);
// expected
MaterializedResult expected = resultBuilder(driverContext.getSession(), concat(probeTypes, ImmutableList.of(BOOLEAN))).row(30L, 0L, true).row(31L, 1L, false).row(32L, 2L, false).row(33L, 3L, false).row(34L, 4L, false).row(35L, 5L, true).row(36L, 6L, true).row(37L, 7L, true).row(38L, 8L, false).row(39L, 9L, false).build();
OperatorAssertion.assertOperatorEqualsWithSimpleStateComparison(joinOperatorFactory, driverContext, probeInput, expected, true, ImmutableList.of(probeTypes.size()), createProbeExpectedMapping());
}
use of io.prestosql.spi.plan.PlanNodeId in project hetu-core by openlookeng.
the class TestHashSemiJoinOperator method testSemiJoin.
@Test(dataProvider = "hashEnabledValues")
public void testSemiJoin(boolean hashEnabled) {
DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
// build
OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
Operator buildOperator = createValuesOperator(operatorContext, rowPagesBuilder.row(10L).row(30L).row(30L).row(35L).row(36L).row(37L).row(50L).build());
SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(1, new PlanNodeId("test"), rowPagesBuilder.getTypes().get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager()));
Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
while (!driver.isFinished()) {
driver.process();
}
// probe
List<Type> probeTypes = ImmutableList.of(BIGINT, BIGINT);
RowPagesBuilder rowPagesBuilderProbe = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT, BIGINT);
List<Page> probeInput = rowPagesBuilderProbe.addSequencePage(10, 30, 0).build();
Optional<Integer> probeHashChannel = hashEnabled ? Optional.of(probeTypes.size()) : Optional.empty();
HashSemiJoinOperatorFactory joinOperatorFactory = new HashSemiJoinOperatorFactory(2, new PlanNodeId("test"), setBuilderOperatorFactory.getSetProvider(), rowPagesBuilderProbe.getTypes(), 0, probeHashChannel);
// expected
MaterializedResult expected = resultBuilder(driverContext.getSession(), concat(probeTypes, ImmutableList.of(BOOLEAN))).row(30L, 0L, true).row(31L, 1L, false).row(32L, 2L, false).row(33L, 3L, false).row(34L, 4L, false).row(35L, 5L, true).row(36L, 6L, true).row(37L, 7L, true).row(38L, 8L, false).row(39L, 9L, false).build();
OperatorAssertion.assertOperatorEquals(joinOperatorFactory, driverContext, probeInput, expected, hashEnabled, ImmutableList.of(probeTypes.size()));
}
use of io.prestosql.spi.plan.PlanNodeId in project hetu-core by openlookeng.
the class TestHashSemiJoinOperator method testProbeAndBuildNulls.
@Test(dataProvider = "hashEnabledValues")
public void testProbeAndBuildNulls(boolean hashEnabled) {
DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
// build
OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
List<Type> buildTypes = ImmutableList.of(BIGINT);
RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes);
Operator buildOperator = createValuesOperator(operatorContext, rowPagesBuilder.row(0L).row(1L).row((Object) null).row(3L).build());
SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(1, new PlanNodeId("test"), buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager()));
Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
while (!driver.isFinished()) {
driver.process();
}
// probe
List<Type> probeTypes = ImmutableList.of(BIGINT);
RowPagesBuilder rowPagesBuilderProbe = rowPagesBuilder(hashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = rowPagesBuilderProbe.row(0L).row((Object) null).row(1L).row(2L).build();
Optional<Integer> probeHashChannel = hashEnabled ? Optional.of(probeTypes.size()) : Optional.empty();
HashSemiJoinOperatorFactory joinOperatorFactory = new HashSemiJoinOperatorFactory(2, new PlanNodeId("test"), setBuilderOperatorFactory.getSetProvider(), rowPagesBuilderProbe.getTypes(), 0, probeHashChannel);
// expected
MaterializedResult expected = resultBuilder(driverContext.getSession(), concat(probeTypes, ImmutableList.of(BOOLEAN))).row(0L, true).row(null, null).row(1L, true).row(2L, null).build();
OperatorAssertion.assertOperatorEquals(joinOperatorFactory, driverContext, probeInput, expected, hashEnabled, ImmutableList.of(probeTypes.size()));
}
use of io.prestosql.spi.plan.PlanNodeId in project hetu-core by openlookeng.
the class TestScanFilterAndProjectOperator method testRecordCursorYield.
@Test
public void testRecordCursorYield() {
// create a generic long function that yields for projection on every row
// verify we will yield #row times totally
// create a table with 15 rows
int length = 15;
Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), length, 0);
DriverContext driverContext = newDriverContext();
// set up generic long function with a callback to force yield
Metadata localMetadata = functionAssertions.getMetadata();
localMetadata.getFunctionAndTypeManager().registerBuiltInFunctions(ImmutableList.of(new GenericLongFunction("record_cursor", value -> {
driverContext.getYieldSignal().forceYieldForTesting();
return value;
})));
ExpressionCompiler compiler = new ExpressionCompiler(localMetadata, new PageFunctionCompiler(localMetadata, 0));
List<RowExpression> projections = ImmutableList.of(call(QualifiedObjectName.valueOfDefaultFunction("generic_long_record_cursor").toString(), new BuiltInFunctionHandle(internalScalarFunction(QualifiedObjectName.valueOfDefaultFunction("generic_long_record_cursor"), BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature()))), BIGINT, field(0, BIGINT)));
Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(Optional.empty(), projections, "key");
Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections);
ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(BIGINT), new DataSize(0, BYTE), 0, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
SourceOperator operator = factory.createOperator(driverContext);
operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
operator.noMoreSplits();
// start driver; get null value due to yield for the first 15 times
for (int i = 0; i < length; i++) {
driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
assertNull(operator.getOutput());
driverContext.getYieldSignal().reset();
}
// the 16th yield is not going to prevent the operator from producing a page
driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
Page output = operator.getOutput();
driverContext.getYieldSignal().reset();
assertNotNull(output);
assertEquals(toValues(BIGINT, output.getBlock(0)), toValues(BIGINT, input.getBlock(0)));
}
use of io.prestosql.spi.plan.PlanNodeId in project hetu-core by openlookeng.
the class TestStreamingAggregationOperator method setUp.
@BeforeMethod
public void setUp() {
executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s"));
scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
operatorFactory = new StreamingAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BOOLEAN, VARCHAR, BIGINT), ImmutableList.of(VARCHAR), ImmutableList.of(1), AggregationNode.Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(2), Optional.empty())), new JoinCompiler(createTestMetadataManager()));
}
Aggregations