Examples with SmileCodec io.prestosql.protocol.SmileCodec used on opensource projects

Example 1 with SmileCodec

use of io.prestosql.protocol.SmileCodec in project hetu-core by openlookeng.

the class HttpRemoteTask method sendUpdate.

private synchronized void sendUpdate() {
    if (abandoned.get()) {
        // Snapshot: Corresponding task has been canceled to resume. Stop any communication with it.
        return;
    }
    TaskStatus taskStatus = getTaskStatus();
    // don't update if the task hasn't been started yet or if it is already finished
    if (!needsUpdate.get() || taskStatus.getState().isDone()) {
        return;
    }
    // if there is a request already running, wait for it to complete
    if (this.currentRequest != null && !this.currentRequest.isDone()) {
        return;
    }
    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::sendUpdate, executor);
        return;
    }
    List<TaskSource> sources = getSources();
    Optional<PlanFragment> fragment = sendPlan.get() ? Optional.of(planFragment) : Optional.empty();
    TaskUpdateRequest updateRequest = new TaskUpdateRequest(// so receiver can verify if the instance id matches
    instanceId, session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), totalPartitions, parent);
    byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest);
    if (fragment.isPresent()) {
        stats.updateWithPlanBytes(taskUpdateRequestJson.length);
    }
    HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
    Request request = setContentTypeHeaders(isBinaryEncoding, preparePost()).setUri(uriBuilder.build()).setBodyGenerator(StaticBodyGenerator.createStaticBodyGenerator(taskUpdateRequestJson)).build();
    ResponseHandler responseHandler;
    if (isBinaryEncoding) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(taskInfoCodec));
    }
    updateErrorTracker.startRequest();
    ListenableFuture<BaseResponse<TaskInfo>> future = httpClient.executeAsync(request, responseHandler);
    currentRequest = future;
    currentRequestStartNanos = System.nanoTime();
    // The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
    // and does so without grabbing the instance lock.
    needsUpdate.set(false);
    Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri(), stats), executor);
}

Example 2 with SmileCodec

use of io.prestosql.protocol.SmileCodec in project hetu-core by openlookeng.

the class HttpRemoteTask method doScheduleAsyncCleanupRequest.

private void doScheduleAsyncCleanupRequest(Backoff cleanupBackoff, Request request, String action) {
    ResponseHandler responseHandler;
    if (isBinaryEncoding) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(taskInfoCodec));
    }
    Futures.addCallback(httpClient.executeAsync(request, responseHandler), new FutureCallback<BaseResponse<TaskInfo>>() {

        @Override
        public void onSuccess(BaseResponse<TaskInfo> result) {
            try {
                updateTaskInfo(result.getValue());
            } finally {
                if (!getTaskInfo().getTaskStatus().getState().isDone()) {
                    cleanUpLocally();
                }
            }
        }

        @Override
        public void onFailure(Throwable t) {
            if (cancelledToResume.get()) {
                // Remote worker is probably unreachable. Don't make additional attempts.
                cleanUpLocally();
                return;
            }
            if (t instanceof RejectedExecutionException && httpClient.isClosed()) {
                logError(t, "Unable to %s task at %s. HTTP client is closed.", action, request.getUri());
                cleanUpLocally();
                return;
            }
            // record failure
            if (cleanupBackoff.failure()) {
                logError(t, "Unable to %s task at %s. Back off depleted.", action, request.getUri());
                cleanUpLocally();
                return;
            }
            // reschedule
            long delayNanos = cleanupBackoff.getBackoffDelayNanos();
            if (delayNanos == 0) {
                doScheduleAsyncCleanupRequest(cleanupBackoff, request, action);
            } else {
                errorScheduledExecutor.schedule(() -> doScheduleAsyncCleanupRequest(cleanupBackoff, request, action), delayNanos, NANOSECONDS);
            }
        }

        private void cleanUpLocally() {
            // Update the taskInfo with the new taskStatus.
            // Generally, we send a cleanup request to the worker, and update the TaskInfo on
            // the coordinator based on what we fetched from the worker. If we somehow cannot
            // get the cleanup request to the worker, the TaskInfo that we fetch for the worker
            // likely will not say the task is done however many times we try. In this case,
            // we have to set the local query info directly so that we stop trying to fetch
            // updated TaskInfo from the worker. This way, the task on the worker eventually
            // expires due to lack of activity.
            // This is required because the query state machine depends on TaskInfo (instead of task status)
            // to transition its own state.
            // TODO: Update the query state machine and stage state machine to depend on TaskStatus instead
            // Since this TaskInfo is updated in the client the "complete" flag will not be set,
            // indicating that the stats may not reflect the final stats on the worker.
            TaskStatus taskStatus = getTaskStatus();
            if (cancelledToResume.get()) {
                // When the task is cancelled to resume, then make sure it gets the new state, so query can start resuming.
                // Check for task state is in QueryInfo#areAllStagesDone.
                taskStatus = TaskStatus.failWith(taskStatus, CANCELED_TO_RESUME, ImmutableList.of());
            }
            updateTaskInfo(getTaskInfo().withTaskStatus(taskStatus));
        }
    }, executor);
}

Example 3 with SmileCodec

use of io.prestosql.protocol.SmileCodec in project hetu-core by openlookeng.

the class RemoteNodeMemory method asyncRefresh.

public void asyncRefresh(MemoryPoolAssignmentsRequest assignments) {
    Duration sinceUpdate = nanosSince(lastUpdateNanos.get());
    if (nanosSince(lastWarningLogged.get()).toMillis() > 1_000 && sinceUpdate.toMillis() > 10_000 && future.get() != null) {
        log.warn("Memory info update request to %s has not returned in %s", memoryInfoUri, sinceUpdate.toString(SECONDS));
        lastWarningLogged.set(System.nanoTime());
    }
    if (sinceUpdate.toMillis() > 1_000 && future.get() == null) {
        Request request = setContentTypeHeaders(isBinaryEncoding, preparePost()).setUri(memoryInfoUri).setBodyGenerator(createBodyGenerator(assignments)).build();
        ResponseHandler responseHandler;
        if (isBinaryEncoding) {
            responseHandler = createFullSmileResponseHandler((SmileCodec<MemoryInfo>) memoryInfoCodec);
        } else {
            responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(memoryInfoCodec));
        }
        HttpResponseFuture<BaseResponse<MemoryInfo>> responseFuture = httpClient.executeAsync(request, responseHandler);
        future.compareAndSet(null, responseFuture);
        Futures.addCallback(responseFuture, new FutureCallback<BaseResponse<MemoryInfo>>() {

            @Override
            public void onSuccess(@Nullable BaseResponse<MemoryInfo> result) {
                lastUpdateNanos.set(System.nanoTime());
                future.compareAndSet(responseFuture, null);
                long version = currentAssignmentVersion.get();
                if (result != null) {
                    if (result.hasValue()) {
                        memoryInfo.set(Optional.ofNullable(result.getValue()));
                    }
                    if (result.getStatusCode() != OK.code()) {
                        log.warn("Error fetching memory info from %s returned status %d: %s", memoryInfoUri, result.getStatusCode(), result.getStatusCode());
                        return;
                    }
                }
                currentAssignmentVersion.compareAndSet(version, assignments.getVersion());
            }

            @Override
            public void onFailure(Throwable t) {
                log.warn("Error fetching memory info from %s: %s", memoryInfoUri, t.getMessage());
                lastUpdateNanos.set(System.nanoTime());
                future.compareAndSet(responseFuture, null);
            }
        }, directExecutor());
    }
}