Search in sources :

Example 6 with PlanFragment

use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.

the class HttpRemoteTask method sendUpdate.

private synchronized void sendUpdate() {
    if (abandoned.get()) {
        // Snapshot: Corresponding task has been canceled to resume. Stop any communication with it.
        return;
    }
    TaskStatus taskStatus = getTaskStatus();
    // don't update if the task hasn't been started yet or if it is already finished
    if (!needsUpdate.get() || taskStatus.getState().isDone()) {
        return;
    }
    // if there is a request already running, wait for it to complete
    if (this.currentRequest != null && !this.currentRequest.isDone()) {
        return;
    }
    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::sendUpdate, executor);
        return;
    }
    List<TaskSource> sources = getSources();
    Optional<PlanFragment> fragment = sendPlan.get() ? Optional.of(planFragment) : Optional.empty();
    TaskUpdateRequest updateRequest = new TaskUpdateRequest(// so receiver can verify if the instance id matches
    instanceId, session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), totalPartitions, parent);
    byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest);
    if (fragment.isPresent()) {
        stats.updateWithPlanBytes(taskUpdateRequestJson.length);
    }
    HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
    Request request = setContentTypeHeaders(isBinaryEncoding, preparePost()).setUri(uriBuilder.build()).setBodyGenerator(StaticBodyGenerator.createStaticBodyGenerator(taskUpdateRequestJson)).build();
    ResponseHandler responseHandler;
    if (isBinaryEncoding) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(taskInfoCodec));
    }
    updateErrorTracker.startRequest();
    ListenableFuture<BaseResponse<TaskInfo>> future = httpClient.executeAsync(request, responseHandler);
    currentRequest = future;
    currentRequestStartNanos = System.nanoTime();
    // The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
    // and does so without grabbing the instance lock.
    needsUpdate.set(false);
    Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri(), stats), executor);
}
Also used : SmileCodec(io.prestosql.protocol.SmileCodec) AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler(io.prestosql.protocol.AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler) ResponseHandler(io.airlift.http.client.ResponseHandler) FullSmileResponseHandler.createFullSmileResponseHandler(io.prestosql.protocol.FullSmileResponseHandler.createFullSmileResponseHandler) TaskUpdateRequest(io.prestosql.server.TaskUpdateRequest) Request(io.airlift.http.client.Request) TaskUpdateRequest(io.prestosql.server.TaskUpdateRequest) TaskStatus(io.prestosql.execution.TaskStatus) PlanFragment(io.prestosql.sql.planner.PlanFragment) BaseResponse(io.prestosql.protocol.BaseResponse) HttpUriBuilder(io.airlift.http.client.HttpUriBuilder) TaskSource(io.prestosql.execution.TaskSource)

Example 7 with PlanFragment

use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.

the class TestPhasedExecutionSchedule method testJoin.

@Test
public void testJoin() {
    PlanFragment buildFragment = createTableScanPlanFragment("build");
    PlanFragment probeFragment = createTableScanPlanFragment("probe");
    PlanFragment joinFragment = createJoinPlanFragment(INNER, "join", buildFragment, probeFragment);
    List<Set<PlanFragmentId>> phases = PhasedExecutionSchedule.extractPhases(ImmutableList.of(joinFragment, buildFragment, probeFragment));
    assertEquals(phases, ImmutableList.of(ImmutableSet.of(joinFragment.getId()), ImmutableSet.of(buildFragment.getId()), ImmutableSet.of(probeFragment.getId())));
}
Also used : ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) PlanFragment(io.prestosql.sql.planner.PlanFragment) Test(org.testng.annotations.Test)

Example 8 with PlanFragment

use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.

the class TestPhasedExecutionSchedule method testRightJoin.

@Test
public void testRightJoin() {
    PlanFragment buildFragment = createTableScanPlanFragment("build");
    PlanFragment probeFragment = createTableScanPlanFragment("probe");
    PlanFragment joinFragment = createJoinPlanFragment(RIGHT, "join", buildFragment, probeFragment);
    List<Set<PlanFragmentId>> phases = PhasedExecutionSchedule.extractPhases(ImmutableList.of(joinFragment, buildFragment, probeFragment));
    assertEquals(phases, ImmutableList.of(ImmutableSet.of(joinFragment.getId()), ImmutableSet.of(buildFragment.getId()), ImmutableSet.of(probeFragment.getId())));
}
Also used : ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) PlanFragment(io.prestosql.sql.planner.PlanFragment) Test(org.testng.annotations.Test)

Example 9 with PlanFragment

use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.

the class SplitFiltering method getFilterNode.

private static List<PlanNode> getFilterNode(SqlStageExecution stage) {
    PlanFragment fragment = stage.getFragment();
    PlanNode root = fragment.getRoot();
    List<PlanNode> result = new LinkedList<>();
    Queue<PlanNode> queue = new LinkedList<>();
    queue.add(root);
    while (!queue.isEmpty()) {
        PlanNode node = queue.poll();
        if (node instanceof FilterNode || node instanceof TableScanNode) {
            result.add(node);
        }
        queue.addAll(node.getSources());
    }
    return result;
}
Also used : PlanNode(io.prestosql.spi.plan.PlanNode) TableScanNode(io.prestosql.spi.plan.TableScanNode) FilterNode(io.prestosql.spi.plan.FilterNode) PlanFragment(io.prestosql.sql.planner.PlanFragment) LinkedList(java.util.LinkedList)

Example 10 with PlanFragment

use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.

the class AllAtOnceExecutionSchedule method getPreferredScheduleOrder.

@VisibleForTesting
static List<PlanFragmentId> getPreferredScheduleOrder(Collection<PlanFragment> fragments) {
    // determine output fragment
    Set<PlanFragmentId> remoteSources = fragments.stream().map(PlanFragment::getRemoteSourceNodes).flatMap(Collection::stream).map(RemoteSourceNode::getSourceFragmentIds).flatMap(Collection::stream).collect(toImmutableSet());
    Set<PlanFragment> rootFragments = fragments.stream().filter(fragment -> !remoteSources.contains(fragment.getId())).collect(toImmutableSet());
    checkArgument(rootFragments.size() == 1, "Expected one root fragment, but found: " + rootFragments);
    Visitor visitor = new Visitor(fragments);
    visitor.processFragment(getOnlyElement(rootFragments).getId());
    return visitor.getSchedulerOrder();
}
Also used : RUNNING(io.prestosql.execution.StageState.RUNNING) InternalPlanVisitor(io.prestosql.sql.planner.plan.InternalPlanVisitor) SCHEDULED(io.prestosql.execution.StageState.SCHEDULED) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) LinkedHashSet(java.util.LinkedHashSet) JoinNode(io.prestosql.spi.plan.JoinNode) PlanFragment(io.prestosql.sql.planner.PlanFragment) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) PlanNode(io.prestosql.spi.plan.PlanNode) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) SpatialJoinNode(io.prestosql.sql.planner.plan.SpatialJoinNode) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) StageState(io.prestosql.execution.StageState) Ordering(com.google.common.collect.Ordering) Function.identity(java.util.function.Function.identity) UnionNode(io.prestosql.spi.plan.UnionNode) VisibleForTesting(com.google.common.annotations.VisibleForTesting) SqlStageExecution(io.prestosql.execution.SqlStageExecution) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) InternalPlanVisitor(io.prestosql.sql.planner.plan.InternalPlanVisitor) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) PlanFragment(io.prestosql.sql.planner.PlanFragment) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

PlanFragment (io.prestosql.sql.planner.PlanFragment)29 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)13 Test (org.testng.annotations.Test)12 ImmutableSet (com.google.common.collect.ImmutableSet)11 Set (java.util.Set)11 PlanFragmentId (io.prestosql.sql.planner.plan.PlanFragmentId)10 TableInfo (io.prestosql.execution.TableInfo)9 UUID (java.util.UUID)9 Symbol (io.prestosql.spi.plan.Symbol)8 SqlStageExecution (io.prestosql.execution.SqlStageExecution)7 PlanNode (io.prestosql.spi.plan.PlanNode)7 PartitioningScheme (io.prestosql.sql.planner.PartitioningScheme)7 Split (io.prestosql.metadata.Split)5 QuerySnapshotManager (io.prestosql.snapshot.QuerySnapshotManager)5 QualifiedObjectName (io.prestosql.spi.connector.QualifiedObjectName)5 ConnectorAwareSplitSource (io.prestosql.split.ConnectorAwareSplitSource)5 StageExecutionPlan (io.prestosql.sql.planner.StageExecutionPlan)5 DynamicFilterService (io.prestosql.dynamicfilter.DynamicFilterService)4 MockRemoteTaskFactory (io.prestosql.execution.MockRemoteTaskFactory)4 NodeTaskMap (io.prestosql.execution.NodeTaskMap)4