Examples with TableWriteInfo com.facebook.presto.execution.scheduler.TableWriteInfo used on opensource projects

Example 1 with TableWriteInfo

use of com.facebook.presto.execution.scheduler.TableWriteInfo in project presto by prestodb.

the class TestLocalExecutionPlanner method getLocalExecutionPlan.

private LocalExecutionPlan getLocalExecutionPlan(Session session) {
    SubPlan subPlan = runner.inTransaction(session, transactionSession -> {
        Plan plan = runner.createPlan(transactionSession, "SELECT avg(totalprice) FROM orders", OPTIMIZED_AND_VALIDATED, false, WarningCollector.NOOP);
        return runner.createSubPlans(transactionSession, plan, false);
    });
    // Expect only one child sub plan doing partial aggregation.
    assertEquals(subPlan.getChildren().size(), 1);
    PlanFragment leafFragment = subPlan.getChildren().get(0).getFragment();
    return createTestingPlanner().plan(createTaskContext(EXECUTOR, SCHEDULED_EXECUTOR, session), leafFragment.getRoot(), leafFragment.getPartitioningScheme(), leafFragment.getStageExecutionDescriptor(), leafFragment.getTableScanSchedulingOrder(), new TestingOutputBuffer(), new TestingRemoteSourceFactory(), new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()));
}

Example 2 with TableWriteInfo

use of com.facebook.presto.execution.scheduler.TableWriteInfo in project presto by prestodb.

the class SqlStageExecution method scheduleTask.

private synchronized RemoteTask scheduleTask(InternalNode node, TaskId taskId, Multimap<PlanNodeId, Split> sourceSplits) {
    checkArgument(!allTasks.contains(taskId), "A task with id %s already exists", taskId);
    ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder();
    initialSplits.putAll(sourceSplits);
    sourceTasks.forEach((planNodeId, task) -> {
        TaskStatus status = task.getTaskStatus();
        if (status.getState() != TaskState.FINISHED) {
            initialSplits.put(planNodeId, createRemoteSplitFor(taskId, task.getRemoteTaskLocation(), task.getTaskId()));
        }
    });
    OutputBuffers outputBuffers = this.outputBuffers.get();
    checkState(outputBuffers != null, "Initial output buffers must be set before a task can be scheduled");
    RemoteTask task = remoteTaskFactory.createRemoteTask(session, taskId, node, planFragment, initialSplits.build(), outputBuffers, nodeTaskMap.createTaskStatsTracker(node, taskId), summarizeTaskInfo, tableWriteInfo);
    completeSources.forEach(task::noMoreSplits);
    allTasks.add(taskId);
    tasks.computeIfAbsent(node, key -> newConcurrentHashSet()).add(task);
    nodeTaskMap.addTask(node, task);
    task.addStateChangeListener(new StageTaskListener(taskId));
    task.addFinalTaskInfoListener(this::updateFinalTaskInfo);
    if (!stateMachine.getState().isDone()) {
        task.start();
    } else {
        // stage finished while we were scheduling this task
        task.abort();
    }
    return task;
}

Example 3 with TableWriteInfo

use of com.facebook.presto.execution.scheduler.TableWriteInfo in project presto by prestodb.

the class HttpRemoteTask method sendUpdate.

private synchronized void sendUpdate() {
    TaskStatus taskStatus = getTaskStatus();
    // don't update if the task hasn't been started yet or if it is already finished
    if (!started.get() || !needsUpdate.get() || taskStatus.getState().isDone()) {
        return;
    }
    // if there is a request already running, wait for it to complete
    if (this.currentRequest != null && !this.currentRequest.isDone()) {
        return;
    }
    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::sendUpdate, executor);
        return;
    }
    List<TaskSource> sources = getSources();
    Optional<byte[]> fragment = sendPlan.get() ? Optional.of(planFragment.toBytes(planFragmentCodec)) : Optional.empty();
    Optional<TableWriteInfo> writeInfo = sendPlan.get() ? Optional.of(tableWriteInfo) : Optional.empty();
    TaskUpdateRequest updateRequest = new TaskUpdateRequest(session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), writeInfo);
    byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest);
    taskUpdateRequestSize.add(taskUpdateRequestJson.length);
    if (taskUpdateRequestJson.length > maxTaskUpdateSizeInBytes) {
        failTask(new PrestoException(EXCEEDED_TASK_UPDATE_SIZE_LIMIT, format("TaskUpdate size of %d Bytes has exceeded the limit of %d Bytes", taskUpdateRequestJson.length, maxTaskUpdateSizeInBytes)));
    }
    if (fragment.isPresent()) {
        stats.updateWithPlanSize(taskUpdateRequestJson.length);
    } else {
        if (ThreadLocalRandom.current().nextDouble() < UPDATE_WITHOUT_PLAN_STATS_SAMPLE_RATE) {
            // This is to keep track of the task update size even when the plan fragment is NOT present
            stats.updateWithoutPlanSize(taskUpdateRequestJson.length);
        }
    }
    HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
    Request request = setContentTypeHeaders(binaryTransportEnabled, preparePost()).setUri(uriBuilder.build()).setBodyGenerator(createStaticBodyGenerator(taskUpdateRequestJson)).build();
    ResponseHandler responseHandler;
    if (binaryTransportEnabled) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler((JsonCodec<TaskInfo>) taskInfoCodec);
    }
    updateErrorTracker.startRequest();
    ListenableFuture<BaseResponse<TaskInfo>> future = httpClient.executeAsync(request, responseHandler);
    currentRequest = future;
    currentRequestStartNanos = System.nanoTime();
    // The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
    // and does so without grabbing the instance lock.
    needsUpdate.set(false);
    Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri(), stats.getHttpResponseStats(), REMOTE_TASK_ERROR), executor);
}

Example 4 with TableWriteInfo

use of com.facebook.presto.execution.scheduler.TableWriteInfo in project presto by prestodb.

the class TestSqlTask method testEmptyQuery.

@Test
public void testEmptyQuery() {
    SqlTask sqlTask = createInitialTask();
    TaskInfo taskInfo = sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), ImmutableList.of(), createInitialEmptyOutputBuffers(PARTITIONED).withNoMoreBufferIds(), Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())));
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING);
    taskInfo = sqlTask.getTaskInfo();
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING);
    taskInfo = sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(), true)), createInitialEmptyOutputBuffers(PARTITIONED).withNoMoreBufferIds(), Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())));
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED);
    taskInfo = sqlTask.getTaskInfo();
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED);
}

Example 5 with TableWriteInfo

use of com.facebook.presto.execution.scheduler.TableWriteInfo in project presto by prestodb.

the class TestSqlTask method testCancel.

@Test
public void testCancel() {
    SqlTask sqlTask = createInitialTask();
    TaskInfo taskInfo = sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), ImmutableList.of(), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())));
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING);
    assertNull(taskInfo.getStats().getEndTime());
    taskInfo = sqlTask.getTaskInfo();
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING);
    assertNull(taskInfo.getStats().getEndTime());
    taskInfo = sqlTask.cancel();
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED);
    assertNotNull(taskInfo.getStats().getEndTime());
    taskInfo = sqlTask.getTaskInfo();
    assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED);
    assertNotNull(taskInfo.getStats().getEndTime());
}