use of io.prestosql.operator.PipelineContext in project hetu-core by openlookeng.
the class TestMemoryRevokingScheduler method testScheduleMemoryRevoking.
@Test
public void testScheduleMemoryRevoking() throws Exception {
SqlTask sqlTask1 = newSqlTask();
SqlTask sqlTask2 = newSqlTask();
TaskContext taskContext1 = sqlTask1.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q1", 1, 1), executor), session, false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false);
DriverContext driverContext111 = pipelineContext11.addDriverContext();
OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na");
OperatorContext operatorContext2 = driverContext111.addOperatorContext(2, new PlanNodeId("na"), "na");
DriverContext driverContext112 = pipelineContext11.addDriverContext();
OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na");
TaskContext taskContext2 = sqlTask2.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q2", 1, 1), executor), session, false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false);
DriverContext driverContext211 = pipelineContext21.addDriverContext();
OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na");
OperatorContext operatorContext5 = driverContext211.addOperatorContext(5, new PlanNodeId("na"), "na");
Collection<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, executor, 1.0, 1.0, false, new DataSize(512, MEGABYTE).toBytes());
allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2, operatorContext3, operatorContext4, operatorContext5);
assertMemoryRevokingNotRequested();
requestMemoryRevoking(scheduler);
assertEquals(10, memoryPool.getFreeBytes());
assertMemoryRevokingNotRequested();
LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext();
LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext();
LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext();
LocalMemoryContext revocableMemory5 = operatorContext5.localRevocableMemoryContext();
revocableMemory1.setBytes(3);
revocableMemory3.setBytes(6);
assertEquals(1, memoryPool.getFreeBytes());
requestMemoryRevoking(scheduler);
// we are still good - no revoking needed
assertMemoryRevokingNotRequested();
revocableMemory4.setBytes(7);
assertEquals(-6, memoryPool.getFreeBytes());
requestMemoryRevoking(scheduler);
// we need to revoke 3 and 6
assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
// yet another revoking request should not change anything
requestMemoryRevoking(scheduler);
assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
// lets revoke some bytes
revocableMemory1.setBytes(0);
operatorContext1.resetMemoryRevokingRequested();
requestMemoryRevoking(scheduler);
assertMemoryRevokingRequestedFor(operatorContext3);
assertEquals(-3, memoryPool.getFreeBytes());
// and allocate some more
revocableMemory5.setBytes(3);
assertEquals(-6, memoryPool.getFreeBytes());
requestMemoryRevoking(scheduler);
// we are still good with just OC3 in process of revoking
assertMemoryRevokingRequestedFor(operatorContext3);
// and allocate some more
revocableMemory5.setBytes(4);
assertEquals(-7, memoryPool.getFreeBytes());
requestMemoryRevoking(scheduler);
// no we have to trigger revoking for OC4
assertMemoryRevokingRequestedFor(operatorContext3, operatorContext4);
}
use of io.prestosql.operator.PipelineContext in project hetu-core by openlookeng.
the class TestMemoryRevokingScheduler method createContexts.
private OperatorContext createContexts(SqlTask sqlTask) {
TaskContext taskContext = sqlTask.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q", 1, 1), executor), session, false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false);
DriverContext driverContext = pipelineContext.addDriverContext();
OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("na"), "na");
return operatorContext;
}
use of io.prestosql.operator.PipelineContext in project hetu-core by openlookeng.
the class TestSpatialJoinOperator method testDuplicateProbeFactory.
@Test(dataProvider = "testDuplicateProbeFactoryDataProvider")
public void testDuplicateProbeFactory(boolean createSecondaryOperators) throws Exception {
TaskContext taskContext = createTaskContext();
PipelineContext pipelineContext = taskContext.addPipelineContext(0, true, true, false);
DriverContext probeDriver = pipelineContext.addDriverContext();
DriverContext buildDriver = pipelineContext.addDriverContext();
RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, DOUBLE)).row(stPoint(0, 0), "0_0", 1.5);
PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(buildDriver, (build, probe, r) -> build.distance(probe) <= r.getAsDouble(), Optional.of(2), Optional.empty(), buildPages);
RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)).row(stPoint(0, 1), "0_1");
OperatorFactory firstFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
for (int i = 0; i < 3; i++) {
DriverContext secondDriver = pipelineContext.addDriverContext();
OperatorFactory secondFactory = firstFactory.duplicate();
if (createSecondaryOperators) {
try (Operator secondOperator = secondFactory.createOperator(secondDriver)) {
assertEquals(toPages(secondOperator, emptyIterator()), ImmutableList.of());
}
}
secondFactory.noMoreOperators();
}
MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).row("0_1", "0_0").build();
assertOperatorEquals(firstFactory, probeDriver, probePages.build(), expected);
}
use of io.prestosql.operator.PipelineContext in project hetu-core by openlookeng.
the class SqlTask method createTaskStatus.
private TaskStatus createTaskStatus(TaskHolder taskHolder) {
// Always return a new TaskInfo with a larger version number;
// otherwise a client will not accept the update
long versionNumber = nextTaskInfoVersion.getAndIncrement();
TaskState state = taskStateMachine.getState();
List<ExecutionFailureInfo> failures = ImmutableList.of();
if (state == FAILED) {
failures = toFailures(taskStateMachine.getFailureCauses());
}
int queuedPartitionedDrivers = 0;
int runningPartitionedDrivers = 0;
DataSize physicalWrittenDataSize = new DataSize(0, BYTE);
DataSize userMemoryReservation = new DataSize(0, BYTE);
DataSize systemMemoryReservation = new DataSize(0, BYTE);
DataSize revocableMemoryReservation = new DataSize(0, BYTE);
// TODO: add a mechanism to avoid sending the whole completedDriverGroups set over the wire for every task status reply
Set<Lifespan> completedDriverGroups = ImmutableSet.of();
long fullGcCount = 0;
Duration fullGcTime = new Duration(0, MILLISECONDS);
Map<Long, SnapshotInfo> snapshotCaptureResult = ImmutableMap.of();
Optional<RestoreResult> snapshotRestoreResult = Optional.empty();
TaskInfo finalTaskInfo = taskHolder.getFinalTaskInfo();
if (finalTaskInfo != null) {
TaskStats taskStats = finalTaskInfo.getStats();
queuedPartitionedDrivers = taskStats.getQueuedPartitionedDrivers();
runningPartitionedDrivers = taskStats.getRunningPartitionedDrivers();
physicalWrittenDataSize = taskStats.getPhysicalWrittenDataSize();
userMemoryReservation = taskStats.getUserMemoryReservation();
systemMemoryReservation = taskStats.getSystemMemoryReservation();
revocableMemoryReservation = taskStats.getRevocableMemoryReservation();
fullGcCount = taskStats.getFullGcCount();
fullGcTime = taskStats.getFullGcTime();
if (isSnapshotEnabled) {
// Add snapshot result
snapshotCaptureResult = finalTaskInfo.getTaskStatus().getSnapshotCaptureResult();
snapshotRestoreResult = finalTaskInfo.getTaskStatus().getSnapshotRestoreResult();
}
} else if (taskHolder.getTaskExecution() != null) {
long physicalWrittenBytes = 0;
TaskContext taskContext = taskHolder.getTaskExecution().getTaskContext();
for (PipelineContext pipelineContext : taskContext.getPipelineContexts()) {
PipelineStatus pipelineStatus = pipelineContext.getPipelineStatus();
queuedPartitionedDrivers += pipelineStatus.getQueuedPartitionedDrivers();
runningPartitionedDrivers += pipelineStatus.getRunningPartitionedDrivers();
physicalWrittenBytes += pipelineContext.getPhysicalWrittenDataSize();
}
physicalWrittenDataSize = succinctBytes(physicalWrittenBytes);
userMemoryReservation = taskContext.getMemoryReservation();
systemMemoryReservation = taskContext.getSystemMemoryReservation();
revocableMemoryReservation = taskContext.getRevocableMemoryReservation();
completedDriverGroups = taskContext.getCompletedDriverGroups();
fullGcCount = taskContext.getFullGcCount();
fullGcTime = taskContext.getFullGcTime();
if (isSnapshotEnabled) {
// Add snapshot result
TaskSnapshotManager snapshotManager = taskHolder.taskExecution.getTaskContext().getSnapshotManager();
snapshotCaptureResult = snapshotManager.getSnapshotCaptureResult();
snapshotRestoreResult = Optional.ofNullable(snapshotManager.getSnapshotRestoreResult());
}
}
return new TaskStatus(taskStateMachine.getTaskId(), confirmationInstanceId, versionNumber, state, location, nodeId, completedDriverGroups, failures, queuedPartitionedDrivers, runningPartitionedDrivers, isOutputBufferOverutilized(), physicalWrittenDataSize, userMemoryReservation, systemMemoryReservation, revocableMemoryReservation, fullGcCount, fullGcTime, snapshotCaptureResult, snapshotRestoreResult);
}
Aggregations