use of io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory in project hetu-core by openlookeng.
the class IndexBuildDriverFactoryProvider method createStreaming.
public DriverFactory createStreaming(PageBuffer pageBuffer, Page indexKeyTuple) {
ImmutableList.Builder<OperatorFactory> operatorFactories = ImmutableList.<OperatorFactory>builder().addAll(coreOperatorFactories);
if (dynamicTupleFilterFactory.isPresent()) {
// Bind in a dynamic tuple filter if necessary
operatorFactories.add(dynamicTupleFilterFactory.get().filterWithTuple(indexKeyTuple));
}
operatorFactories.add(new PageBufferOperatorFactory(outputOperatorId, planNodeId, pageBuffer));
return new DriverFactory(pipelineId, inputDriver, false, operatorFactories.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
}
use of io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory in project hetu-core by openlookeng.
the class TestHashJoinOperator method innerJoinWithSpill.
private void innerJoinWithSpill(boolean probeHashEnabled, List<WhenSpill> whenSpill, SingleStreamSpillerFactory buildSpillerFactory, PartitioningSpillerFactory joinSpillerFactory) throws Exception {
TaskStateMachine taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0), executor);
TaskContext taskContext = TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, taskStateMachine);
DriverContext joinDriverContext = taskContext.addPipelineContext(2, true, true, false).addDriverContext();
// force a yield for every match in LookupJoinOperator, set called to true after first
AtomicBoolean called = new AtomicBoolean(false);
InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction((leftPosition, leftPage, rightPosition, rightPage) -> {
called.set(true);
joinDriverContext.getYieldSignal().forceYieldForTesting();
return true;
});
// build factory
RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(VARCHAR, BIGINT)).addSequencePage(4, 20, 200).addSequencePage(4, 20, 200).addSequencePage(4, 30, 300).addSequencePage(4, 40, 400);
BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), true, buildSpillerFactory);
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
// probe factory
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT)).row("20", 123_000L).row("20", 123_000L).pageBreak().addSequencePage(20, 0, 123_000).addSequencePage(10, 30, 123_000);
OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactoryManager, probePages, joinSpillerFactory);
// build drivers and operators
instantiateBuildDrivers(buildSideSetup, taskContext);
List<Driver> buildDrivers = buildSideSetup.getBuildDrivers();
int buildOperatorCount = buildDrivers.size();
checkState(buildOperatorCount == whenSpill.size());
LookupSourceFactory lookupSourceFactory = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide());
boolean closed = false;
try (Operator joinOperator = joinOperatorFactory.createOperator(joinDriverContext)) {
// build lookup source
ListenableFuture<LookupSourceProvider> lookupSourceProvider = lookupSourceFactory.createLookupSourceProvider();
List<Boolean> revoked = new ArrayList<>(nCopies(buildOperatorCount, false));
while (!lookupSourceProvider.isDone()) {
for (int i = 0; i < buildOperatorCount; i++) {
checkErrors(taskStateMachine);
buildDrivers.get(i).process();
HashBuilderOperator buildOperator = buildSideSetup.getBuildOperators().get(i);
if (whenSpill.get(i) == WhenSpill.DURING_BUILD && buildOperator.getOperatorContext().getReservedRevocableBytes() > 0) {
checkState(!lookupSourceProvider.isDone(), "Too late, LookupSource already done");
revokeMemory(buildOperator);
revoked.set(i, true);
}
}
}
getFutureValue(lookupSourceProvider).close();
assertEquals(revoked, whenSpill.stream().map(WhenSpill.DURING_BUILD::equals).collect(toImmutableList()), "Some operators not spilled before LookupSource built");
for (int i = 0; i < buildOperatorCount; i++) {
if (whenSpill.get(i) == WhenSpill.AFTER_BUILD) {
revokeMemory(buildSideSetup.getBuildOperators().get(i));
}
}
for (Driver buildDriver : buildDrivers) {
runDriverInThread(executor, buildDriver);
}
joinOperatorFactory.noMoreOperators(joinDriverContext.getLifespan());
joinOperatorFactory.noMoreOperators();
closed = true;
ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(17, new PlanNodeId("values"), probePages.build());
PageBuffer pageBuffer = new PageBuffer(10);
PageBufferOperatorFactory pageBufferOperatorFactory = new PageBufferOperatorFactory(18, new PlanNodeId("pageBuffer"), pageBuffer);
Driver joinDriver = Driver.createDriver(joinDriverContext, valuesOperatorFactory.createOperator(joinDriverContext), joinOperator, pageBufferOperatorFactory.createOperator(joinDriverContext));
while (!called.get()) {
// process first row of first page of LookupJoinOperator
processRow(joinDriver, taskStateMachine);
}
for (int i = 0; i < buildOperatorCount; i++) {
if (whenSpill.get(i) == WhenSpill.DURING_USAGE) {
triggerMemoryRevokingAndWait(buildSideSetup.getBuildOperators().get(i), taskStateMachine);
}
}
// process remaining LookupJoinOperator pages
while (!joinDriver.isFinished()) {
checkErrors(taskStateMachine);
processRow(joinDriver, taskStateMachine);
}
checkErrors(taskStateMachine);
List<Page> actualPages = getPages(pageBuffer);
MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probePages.getTypesWithoutHash(), buildPages.getTypesWithoutHash())).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("30", 123_000L, "30", 300L).row("31", 123_001L, "31", 301L).row("32", 123_002L, "32", 302L).row("33", 123_003L, "33", 303L).build();
assertEqualsIgnoreOrder(getProperColumns(joinOperator, concat(probePages.getTypes(), buildPages.getTypes()), probePages, actualPages).getMaterializedRows(), expected.getMaterializedRows());
} finally {
if (!closed) {
joinOperatorFactory.noMoreOperators(joinDriverContext.getLifespan());
joinOperatorFactory.noMoreOperators();
}
}
}
Aggregations