use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.
the class TestSystemMemoryBlocking method testTableScanSystemMemoryBlocking.
@Test
public void testTableScanSystemMemoryBlocking() {
PlanNodeId sourceId = new PlanNodeId("source");
final List<Type> types = ImmutableList.of(VARCHAR);
TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).build()), TEST_TABLE_HANDLE, ImmutableList.of(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), types, false, Optional.empty(), 0, 0);
PageConsumerOperator sink = createSinkOperator(types);
Driver driver = Driver.createDriver(driverContext, source, sink);
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
Split testSplit = new Split(new CatalogName("test"), new TestSplit(), Lifespan.taskWide());
driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
ListenableFuture<?> blocked = driver.processFor(new Duration(1, NANOSECONDS));
// the driver shouldn't block in the first call as it will be able to move a page between source and the sink operator
// but the operator should be blocked
assertTrue(blocked.isDone());
assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
// and they should stay blocked until more memory becomes available
for (int i = 0; i < 10; i++) {
blocked = driver.processFor(new Duration(1, NANOSECONDS));
assertFalse(blocked.isDone());
assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
}
// free up some memory
memoryPool.free(QUERY_ID, "test", memoryPool.getReservedBytes());
// the operator should be unblocked
assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
// the driver shouldn't be blocked
blocked = driver.processFor(new Duration(1, NANOSECONDS));
assertTrue(blocked.isDone());
}
use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.
the class HttpRemoteTask method processTaskUpdate.
private synchronized void processTaskUpdate(TaskInfo newValue, List<TaskSource> sources) {
updateTaskInfo(newValue);
// remove acknowledged splits, which frees memory
for (TaskSource source : sources) {
PlanNodeId planNodeId = source.getPlanNodeId();
int removed = 0;
for (ScheduledSplit split : source.getSplits()) {
if (pendingSplits.remove(planNodeId, split)) {
removed++;
}
}
if (source.isNoMoreSplits()) {
noMoreSplits.put(planNodeId, false);
}
for (Lifespan lifespan : source.getNoMoreSplitsForLifespan()) {
pendingNoMoreSplitsForLifespan.remove(planNodeId, lifespan);
}
if (planFragment.isPartitionedSources(planNodeId)) {
pendingSourceSplitCount -= removed;
}
}
updateSplitQueueSpace();
partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount());
}
use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.
the class HttpRemoteTask method sendUpdate.
private synchronized void sendUpdate() {
if (abandoned.get()) {
// Snapshot: Corresponding task has been canceled to resume. Stop any communication with it.
return;
}
TaskStatus taskStatus = getTaskStatus();
// don't update if the task hasn't been started yet or if it is already finished
if (!needsUpdate.get() || taskStatus.getState().isDone()) {
return;
}
// if there is a request already running, wait for it to complete
if (this.currentRequest != null && !this.currentRequest.isDone()) {
return;
}
// if throttled due to error, asynchronously wait for timeout and try again
ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
if (!errorRateLimit.isDone()) {
errorRateLimit.addListener(this::sendUpdate, executor);
return;
}
List<TaskSource> sources = getSources();
Optional<PlanFragment> fragment = sendPlan.get() ? Optional.of(planFragment) : Optional.empty();
TaskUpdateRequest updateRequest = new TaskUpdateRequest(// so receiver can verify if the instance id matches
instanceId, session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), totalPartitions, parent);
byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest);
if (fragment.isPresent()) {
stats.updateWithPlanBytes(taskUpdateRequestJson.length);
}
HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
Request request = setContentTypeHeaders(isBinaryEncoding, preparePost()).setUri(uriBuilder.build()).setBodyGenerator(StaticBodyGenerator.createStaticBodyGenerator(taskUpdateRequestJson)).build();
ResponseHandler responseHandler;
if (isBinaryEncoding) {
responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
} else {
responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(taskInfoCodec));
}
updateErrorTracker.startRequest();
ListenableFuture<BaseResponse<TaskInfo>> future = httpClient.executeAsync(request, responseHandler);
currentRequest = future;
currentRequestStartNanos = System.nanoTime();
// The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
// and does so without grabbing the instance lock.
needsUpdate.set(false);
Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri(), stats), executor);
}
use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.
the class HttpRemoteTask method getSource.
private synchronized TaskSource getSource(PlanNodeId planNodeId) {
Set<ScheduledSplit> splits = ImmutableSet.copyOf(pendingSplits.get(planNodeId));
boolean pendingNoMoreSplits = Boolean.TRUE.equals(this.noMoreSplits.get(planNodeId));
boolean tmpNoMoreSplits = this.noMoreSplits.containsKey(planNodeId);
Set<Lifespan> noMoreSplitsForLifespan = pendingNoMoreSplitsForLifespan.get(planNodeId);
TaskSource element = null;
if (!splits.isEmpty() || !noMoreSplitsForLifespan.isEmpty() || pendingNoMoreSplits) {
element = new TaskSource(planNodeId, splits, noMoreSplitsForLifespan, tmpNoMoreSplits);
}
return element;
}
use of io.prestosql.execution.TaskSource in project hetu-core by openlookeng.
the class TestDriver method testBrokenOperatorAddSource.
@Test
public void testBrokenOperatorAddSource() throws Exception {
PlanNodeId sourceId = new PlanNodeId("source");
final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
// create a table scan operator that does not block, which will cause the driver loop to busy wait
TableScanOperator source = new NotBlockedTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
final Driver driver = Driver.createDriver(driverContext, source, brokenOperator);
// block thread in operator processing
Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() {
@Override
public Boolean call() {
return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
}
});
brokenOperator.waitForLocked();
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
// processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
assertFalse(driver.isFinished());
driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
assertFalse(driver.isFinished());
// processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
assertFalse(driver.isFinished());
driver.close();
assertTrue(driver.isFinished());
try {
driverProcessFor.get(1, TimeUnit.SECONDS);
fail("Expected InterruptedException");
} catch (ExecutionException e) {
assertDriverInterrupted(e.getCause());
}
}
Aggregations