use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class HttpRemoteTask method sendUpdate.
private synchronized void sendUpdate() {
if (abandoned.get()) {
// Snapshot: Corresponding task has been canceled to resume. Stop any communication with it.
return;
}
TaskStatus taskStatus = getTaskStatus();
// don't update if the task hasn't been started yet or if it is already finished
if (!needsUpdate.get() || taskStatus.getState().isDone()) {
return;
}
// if there is a request already running, wait for it to complete
if (this.currentRequest != null && !this.currentRequest.isDone()) {
return;
}
// if throttled due to error, asynchronously wait for timeout and try again
ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
if (!errorRateLimit.isDone()) {
errorRateLimit.addListener(this::sendUpdate, executor);
return;
}
List<TaskSource> sources = getSources();
Optional<PlanFragment> fragment = sendPlan.get() ? Optional.of(planFragment) : Optional.empty();
TaskUpdateRequest updateRequest = new TaskUpdateRequest(// so receiver can verify if the instance id matches
instanceId, session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), totalPartitions, parent);
byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest);
if (fragment.isPresent()) {
stats.updateWithPlanBytes(taskUpdateRequestJson.length);
}
HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
Request request = setContentTypeHeaders(isBinaryEncoding, preparePost()).setUri(uriBuilder.build()).setBodyGenerator(StaticBodyGenerator.createStaticBodyGenerator(taskUpdateRequestJson)).build();
ResponseHandler responseHandler;
if (isBinaryEncoding) {
responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
} else {
responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(taskInfoCodec));
}
updateErrorTracker.startRequest();
ListenableFuture<BaseResponse<TaskInfo>> future = httpClient.executeAsync(request, responseHandler);
currentRequest = future;
currentRequestStartNanos = System.nanoTime();
// The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
// and does so without grabbing the instance lock.
needsUpdate.set(false);
Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri(), stats), executor);
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class TestPhasedExecutionSchedule method testJoin.
@Test
public void testJoin() {
PlanFragment buildFragment = createTableScanPlanFragment("build");
PlanFragment probeFragment = createTableScanPlanFragment("probe");
PlanFragment joinFragment = createJoinPlanFragment(INNER, "join", buildFragment, probeFragment);
List<Set<PlanFragmentId>> phases = PhasedExecutionSchedule.extractPhases(ImmutableList.of(joinFragment, buildFragment, probeFragment));
assertEquals(phases, ImmutableList.of(ImmutableSet.of(joinFragment.getId()), ImmutableSet.of(buildFragment.getId()), ImmutableSet.of(probeFragment.getId())));
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class TestPhasedExecutionSchedule method testRightJoin.
@Test
public void testRightJoin() {
PlanFragment buildFragment = createTableScanPlanFragment("build");
PlanFragment probeFragment = createTableScanPlanFragment("probe");
PlanFragment joinFragment = createJoinPlanFragment(RIGHT, "join", buildFragment, probeFragment);
List<Set<PlanFragmentId>> phases = PhasedExecutionSchedule.extractPhases(ImmutableList.of(joinFragment, buildFragment, probeFragment));
assertEquals(phases, ImmutableList.of(ImmutableSet.of(joinFragment.getId()), ImmutableSet.of(buildFragment.getId()), ImmutableSet.of(probeFragment.getId())));
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class SplitFiltering method getFilterNode.
private static List<PlanNode> getFilterNode(SqlStageExecution stage) {
PlanFragment fragment = stage.getFragment();
PlanNode root = fragment.getRoot();
List<PlanNode> result = new LinkedList<>();
Queue<PlanNode> queue = new LinkedList<>();
queue.add(root);
while (!queue.isEmpty()) {
PlanNode node = queue.poll();
if (node instanceof FilterNode || node instanceof TableScanNode) {
result.add(node);
}
queue.addAll(node.getSources());
}
return result;
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class AllAtOnceExecutionSchedule method getPreferredScheduleOrder.
@VisibleForTesting
static List<PlanFragmentId> getPreferredScheduleOrder(Collection<PlanFragment> fragments) {
// determine output fragment
Set<PlanFragmentId> remoteSources = fragments.stream().map(PlanFragment::getRemoteSourceNodes).flatMap(Collection::stream).map(RemoteSourceNode::getSourceFragmentIds).flatMap(Collection::stream).collect(toImmutableSet());
Set<PlanFragment> rootFragments = fragments.stream().filter(fragment -> !remoteSources.contains(fragment.getId())).collect(toImmutableSet());
checkArgument(rootFragments.size() == 1, "Expected one root fragment, but found: " + rootFragments);
Visitor visitor = new Visitor(fragments);
visitor.processFragment(getOnlyElement(rootFragments).getId());
return visitor.getSchedulerOrder();
}
Aggregations