Examples with SerializedPage io.hetu.core.transport.execution.buffer.SerializedPage used on opensource projects

Example 26 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class ExchangeClient method pollPageImpl.

private synchronized Pair<SerializedPage, String> pollPageImpl(String target) {
    SerializedPage page = pageBuffer.poll();
    if (page == null || page == NO_MORE_PAGES) {
        // We handle it separately because a null Optional object is problematic
        return Pair.of(page, null);
    }
    Set<String> targets = targetBuffer.poll();
    Optional<String> origin = originBuffer.poll();
    if (page != null && page.isMarkerPage()) {
        if (targets.contains(target)) {
            targets.remove(target);
            if (!targets.isEmpty()) {
                // Put unfinished marker back at top of queue for other targets to retrieve.
                pageBuffer.addFirst(page);
                targetBuffer.addFirst(targets);
                originBuffer.addFirst(origin);
            }
        } else {
            SerializedPage marker = page;
            Optional<String> markerOrigin = origin;
            // Already sent marker to this target. Poll other pages.
            Pair<SerializedPage, String> pair = pollPageImpl(target);
            page = pair.getLeft();
            origin = Optional.ofNullable(pair.getRight());
            if (page == NO_MORE_PAGES) {
                // Can't grab the no-more-pages marker when there are pending marker pages
                pageBuffer.addFirst(NO_MORE_PAGES);
                page = null;
            }
            // Put unfinished marker back at top of queue for other targets to retrieve.
            pageBuffer.addFirst(marker);
            targetBuffer.addFirst(targets);
            originBuffer.addFirst(markerOrigin);
        }
    }
    return Pair.of(page, origin.orElse(null));
}

Example 27 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class ExchangeClient method addPages.

@VisibleForTesting
synchronized boolean addPages(List<SerializedPage> pages, String location) {
    if (isClosed() || isFailed()) {
        return false;
    }
    long sizeAdjustment = 0;
    if (!pages.isEmpty()) {
        if (!snapshotEnabled) {
            pageBuffer.addAll(pages);
        } else {
            for (SerializedPage page : pages) {
                if (snapshotState != null) {
                    // Only for MergeOperator
                    SerializedPage processedPage;
                    synchronized (snapshotState) {
                        // This may look suspicious, in that if there are "pending pages" in the snapshot state, then
                        // a) those pages were associated with specific input channels (exchange source/sink) when the state
                        // was captured, but now they would be returned to any channel asking for the next page, and
                        // b) when the pending page is returned, the current page (in pageReference) is discarded and lost.
                        // But the above never happens because "merge" operators are always preceded by OrderByOperators,
                        // which only send data pages at the end, *after* all markers. That means when snapshot is taken,
                        // no data page has been received, so when the snapshot is restored, there won't be any pending pages.
                        processedPage = snapshotState.processSerializedPage(() -> Pair.of(page, location)).orElse(null);
                    }
                    if (processedPage == null || processedPage.isMarkerPage()) {
                        // Instead, markers are stored in and returned by the snapshot state.
                        continue;
                    }
                }
                pageBuffer.add(page);
                originBuffer.add(Optional.ofNullable(location));
                if (page.isMarkerPage()) {
                    if (!noMoreTargets) {
                        pendingMarkers.add(page);
                        pendingOrigins.add(Optional.ofNullable(location));
                    }
                    targetBuffer.add(new HashSet<>(allTargets));
                    // This page will be sent out multiple times. Adjust total size.
                    sizeAdjustment += page.getRetainedSizeInBytes() * (allTargets.size() - 1);
                } else {
                    targetBuffer.add(Collections.emptySet());
                }
            }
        }
        // notify all blocked callers
        notifyBlockedCallers();
    }
    long pagesRetainedSizeInBytes = pages.stream().mapToLong(SerializedPage::getRetainedSizeInBytes).sum();
    bufferRetainedSizeInBytes += pagesRetainedSizeInBytes + sizeAdjustment;
    maxBufferRetainedSizeInBytes = Math.max(maxBufferRetainedSizeInBytes, bufferRetainedSizeInBytes);
    systemMemoryContext.setBytes(bufferRetainedSizeInBytes);
    successfulRequests++;
    long responseSize = pages.stream().mapToLong(SerializedPage::getSizeInBytes).sum();
    // AVG_n = AVG_(n-1) * (n-1)/n + VALUE_n / n
    averageBytesPerRequest = (long) (1.0 * averageBytesPerRequest * (successfulRequests - 1) / successfulRequests + responseSize / successfulRequests);
    return true;
}

Example 28 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class MergeOperator method addSplit.

@Override
public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) {
    requireNonNull(split, "split is null");
    checkArgument(split.getConnectorSplit() instanceof RemoteSplit, "split is not a remote split");
    checkState(!blockedOnSplits.isDone(), "noMoreSplits has been called already");
    URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation();
    String instanceId = ((RemoteSplit) split.getConnectorSplit()).getInstanceId();
    ExchangeClient exchangeClient = closer.register(exchangeClientSupplier.get(operatorContext.localSystemMemoryContext()));
    if (operatorContext.isSnapshotEnabled()) {
        exchangeClient.setSnapshotEnabled(operatorContext.getDriverContext().getPipelineContext().getTaskContext().getSnapshotManager().getQuerySnapshotManager());
        exchangeClient.setSnapshotState(snapshotState);
    }
    exchangeClient.addTarget(id);
    exchangeClient.noMoreTargets();
    exchangeClient.addLocation(new TaskLocation(location, instanceId));
    exchangeClient.noMoreLocations();
    clients.add(exchangeClient);
    pageProducers.add(exchangeClient.pages(id).map(new WorkProcessor.RestorableFunction<SerializedPage, Page>() {

        @Override
        public Page apply(SerializedPage serializedPage) {
            operatorContext.recordNetworkInput(serializedPage.getSizeInBytes(), serializedPage.getPositionCount());
            return operatorContext.getDriverContext().getSerde().deserialize(serializedPage);
        }

        @Override
        public Object captureResult(Page result, BlockEncodingSerdeProvider serdeProvider) {
            if (result != null) {
                return ((PagesSerde) serdeProvider).serialize(result).capture(serdeProvider);
            }
            return null;
        }

        @Override
        public Page restoreResult(Object resultState, BlockEncodingSerdeProvider serdeProvider) {
            if (resultState != null) {
                SerializedPage serializedPage = SerializedPage.restoreSerializedPage(resultState);
                return ((PagesSerde) serdeProvider).deserialize(serializedPage);
            }
            return null;
        }

        @Override
        public Object captureInput(SerializedPage input, BlockEncodingSerdeProvider serdeProvider) {
            if (input != null) {
                return input.capture(serdeProvider);
            }
            return null;
        }

        @Override
        public SerializedPage restoreInput(Object inputState, BlockEncodingSerdeProvider serdeProvider) {
            if (inputState != null) {
                return SerializedPage.restoreSerializedPage(inputState);
            }
            return null;
        }

        @Override
        public Object capture(BlockEncodingSerdeProvider serdeProvider) {
            return 0;
        }

        @Override
        public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) {
        }
    }));
    if (snapshotState != null) {
        // When inputChannels is not empty, then we should have received all locations
        checkState(!inputChannels.isPresent());
    }
    return Optional::empty;
}

Example 29 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class NestedLoopJoinOperator method restore.

@Override
public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) {
    NestedLoopJoinOperatorState myState = (NestedLoopJoinOperatorState) state;
    this.operatorContext.restore(myState.operatorContext, serdeProvider);
    if (myState.buildPages != null && this.buildPages == null) {
        PagesSerde serde = (PagesSerde) serdeProvider;
        ImmutableList.Builder<Page> builder = ImmutableList.builder();
        for (Object obj : myState.buildPages) {
            SerializedPage sp = SerializedPage.restoreSerializedPage(obj);
            builder.add(serde.deserialize(sp));
        }
        this.buildPages = builder.build();
    }
    this.finishing = myState.finishing;
    this.closed = myState.closed;
}

Example 30 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class NestedLoopJoinOperator method capture.

@Override
public Object capture(BlockEncodingSerdeProvider serdeProvider) {
    NestedLoopJoinOperatorState myState = new NestedLoopJoinOperatorState();
    myState.operatorContext = operatorContext.capture(serdeProvider);
    if (buildPages != null) {
        myState.buildPages = new Object[buildPages.size()];
        PagesSerde serde = (PagesSerde) serdeProvider;
        for (int i = 0; i < buildPages.size(); i++) {
            SerializedPage sp = serde.serialize(buildPages.get(i));
            myState.buildPages[i] = sp.capture(serdeProvider);
        }
    }
    myState.finishing = finishing;
    myState.closed = closed;
    return myState;
}