Examples with PagesSerde io.trino.execution.buffer.PagesSerde used on opensource projects

Example 1 with PagesSerde

use of io.trino.execution.buffer.PagesSerde in project trino by trinodb.

the class HttpPageBufferClient method sendGetResults.

private synchronized void sendGetResults() {
    URI uri = HttpUriBuilder.uriBuilderFrom(location).appendPath(String.valueOf(token)).build();
    HttpResponseFuture<PagesResponse> resultFuture = httpClient.executeAsync(prepareGet().setHeader(TRINO_MAX_SIZE, maxResponseSize.toString()).setUri(uri).build(), new PageResponseHandler(dataIntegrityVerification != DataIntegrityVerification.NONE));
    future = resultFuture;
    Futures.addCallback(resultFuture, new FutureCallback<>() {

        @Override
        public void onSuccess(PagesResponse result) {
            assertNotHoldsLock(this);
            backoff.success();
            List<Slice> pages;
            boolean pagesAccepted;
            try {
                if (result.isTaskFailed()) {
                    throw new TrinoException(REMOTE_TASK_FAILED, format("Remote task failed: %s", remoteTaskId));
                }
                boolean shouldAcknowledge = false;
                synchronized (HttpPageBufferClient.this) {
                    if (taskInstanceId == null) {
                        taskInstanceId = result.getTaskInstanceId();
                    }
                    if (!isNullOrEmpty(taskInstanceId) && !result.getTaskInstanceId().equals(taskInstanceId)) {
                        throw new TrinoException(REMOTE_TASK_MISMATCH, format("%s (%s). Expected taskInstanceId: %s, received taskInstanceId: %s", REMOTE_TASK_MISMATCH_ERROR, fromUri(uri), taskInstanceId, result.getTaskInstanceId()));
                    }
                    if (result.getToken() == token) {
                        pages = result.getPages();
                        token = result.getNextToken();
                        shouldAcknowledge = pages.size() > 0;
                    } else {
                        pages = ImmutableList.of();
                    }
                }
                if (shouldAcknowledge && acknowledgePages) {
                    // Acknowledge token without handling the response.
                    // The next request will also make sure the token is acknowledged.
                    // This is to fast release the pages on the buffer side.
                    URI uri = HttpUriBuilder.uriBuilderFrom(location).appendPath(String.valueOf(result.getNextToken())).appendPath("acknowledge").build();
                    httpClient.executeAsync(prepareGet().setUri(uri).build(), new ResponseHandler<Void, RuntimeException>() {

                        @Override
                        public Void handleException(Request request, Exception exception) {
                            log.debug(exception, "Acknowledge request failed: %s", uri);
                            return null;
                        }

                        @Override
                        public Void handle(Request request, Response response) {
                            if (familyForStatusCode(response.getStatusCode()) != HttpStatus.Family.SUCCESSFUL) {
                                log.debug("Unexpected acknowledge response code: %s", response.getStatusCode());
                            }
                            return null;
                        }
                    });
                }
                // add pages:
                // addPages must be called regardless of whether pages is an empty list because
                // clientCallback can keep stats of requests and responses. For example, it may
                // keep track of how often a client returns empty response and adjust request
                // frequency or buffer size.
                pagesAccepted = clientCallback.addPages(HttpPageBufferClient.this, pages);
            } catch (TrinoException e) {
                handleFailure(e, resultFuture);
                return;
            }
            // update client stats
            if (!pages.isEmpty()) {
                int pageCount = pages.size();
                long rowCount = pages.stream().mapToLong(PagesSerde::getSerializedPagePositionCount).sum();
                if (pagesAccepted) {
                    pagesReceived.addAndGet(pageCount);
                    rowsReceived.addAndGet(rowCount);
                } else {
                    pagesRejected.addAndGet(pageCount);
                    rowsRejected.addAndGet(rowCount);
                }
            }
            requestsCompleted.incrementAndGet();
            synchronized (HttpPageBufferClient.this) {
                // client is complete, acknowledge it by sending it a delete in the next request
                if (result.isClientComplete()) {
                    completed = true;
                }
                if (future == resultFuture) {
                    future = null;
                }
                lastUpdate = DateTime.now();
            }
            clientCallback.requestComplete(HttpPageBufferClient.this);
        }

        @Override
        public void onFailure(Throwable t) {
            log.debug("Request to %s failed %s", uri, t);
            assertNotHoldsLock(this);
            if (t instanceof ChecksumVerificationException) {
                switch(dataIntegrityVerification) {
                    case NONE:
                    // In case of NONE, failure is possible in case of inconsistent cluster configuration, so we should not retry.
                    case ABORT:
                        // TrinoException will not be retried
                        t = new TrinoException(GENERIC_INTERNAL_ERROR, format("Checksum verification failure on %s when reading from %s: %s", selfAddress, uri, t.getMessage()), t);
                        break;
                    case RETRY:
                        log.warn("Checksum verification failure on %s when reading from %s, may be retried: %s", selfAddress, uri, t.getMessage());
                        break;
                    default:
                        throw new AssertionError("Unsupported option: " + dataIntegrityVerification);
                }
            }
            t = rewriteException(t);
            if (!(t instanceof TrinoException) && backoff.failure()) {
                String message = format("%s (%s - %s failures, failure duration %s, total failed request time %s)", WORKER_NODE_ERROR, uri, backoff.getFailureCount(), backoff.getFailureDuration().convertTo(SECONDS), backoff.getFailureRequestTimeTotal().convertTo(SECONDS));
                t = new PageTransportTimeoutException(fromUri(uri), message, t);
            }
            handleFailure(t, resultFuture);
        }
    }, pageBufferClientCallbackExecutor);
}

Example 2 with PagesSerde

use of io.trino.execution.buffer.PagesSerde in project trino by trinodb.

the class FileSingleStreamSpillerFactory method create.

@Override
public SingleStreamSpiller create(List<Type> types, SpillContext spillContext, LocalMemoryContext memoryContext) {
    Optional<SpillCipher> spillCipher = Optional.empty();
    if (spillEncryptionEnabled) {
        spillCipher = Optional.of(new AesSpillCipher());
    }
    PagesSerde serde = serdeFactory.createPagesSerdeForSpill(spillCipher);
    return new FileSingleStreamSpiller(serde, executor, getNextSpillPath(), spillerStats, spillContext, memoryContext, spillCipher, spillPathHealthCache::invalidateAll);
}

Example 3 with PagesSerde

use of io.trino.execution.buffer.PagesSerde in project trino by trinodb.

the class TestSpillCipherPagesSerde method test.

@Test
public void test() {
    SpillCipher cipher = new AesSpillCipher();
    PagesSerde serde = new TestingPagesSerdeFactory().createPagesSerdeForSpill(Optional.of(cipher));
    List<Type> types = ImmutableList.of(VARCHAR);
    Page emptyPage = new Page(VARCHAR.createBlockBuilder(null, 0).build());
    try (PagesSerde.PagesSerdeContext context = serde.newContext()) {
        assertPageEquals(types, serde.deserialize(serde.serialize(context, emptyPage)), emptyPage);
        BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 2);
        VARCHAR.writeString(blockBuilder, "hello");
        VARCHAR.writeString(blockBuilder, "world");
        Page helloWorldPage = new Page(blockBuilder.build());
        Slice serialized = serde.serialize(context, helloWorldPage);
        assertPageEquals(types, serde.deserialize(serialized), helloWorldPage);
        assertTrue(isSerializedPageEncrypted(serialized), "page should be encrypted");
        cipher.close();
        assertFailure(() -> serde.serialize(context, helloWorldPage), "Spill cipher already closed");
        assertFailure(() -> serde.deserialize(context, serialized), "Spill cipher already closed");
    }
}