Examples with MarkerPage io.prestosql.spi.snapshot.MarkerPage used on opensource projects

Example 21 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class PageAssertions method assertPageEquals.

public static void assertPageEquals(List<? extends Type> types, Page actualPage, Page expectedPage) {
    if (expectedPage instanceof MarkerPage) {
        assertTrue(expectedPage instanceof MarkerPage);
        MarkerPage actual = (MarkerPage) actualPage;
        MarkerPage expected = (MarkerPage) expectedPage;
        assertEquals(actual.getSnapshotId(), expected.getSnapshotId());
        assertEquals(actual.isResuming(), expected.isResuming());
        return;
    }
    assertEquals(types.size(), actualPage.getChannelCount());
    assertEquals(actualPage.getChannelCount(), expectedPage.getChannelCount());
    assertEquals(actualPage.getPositionCount(), expectedPage.getPositionCount());
    for (int i = 0; i < actualPage.getChannelCount(); i++) {
        assertBlockEquals(types.get(i), actualPage.getBlock(i), expectedPage.getBlock(i));
    }
}

Example 22 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class UpdateIndexOperator method addInput.

@Override
public void addInput(Page page) {
    checkState(needsInput(), "Operator is already finishing");
    requireNonNull(page, "page is null");
    // TODO-cp-I38S9O: Operator currently not supported for Snapshot
    if (page instanceof MarkerPage) {
        throw new UnsupportedOperationException("Operator doesn't support snapshotting.");
    }
    // if operator is still receiving input, it's not finished
    finished.putIfAbsent(this, false);
    if (page.getPositionCount() == 0) {
        return;
    }
    IndexRecord indexRecord;
    try {
        indexRecord = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(createIndexMetadata.getIndexName());
    } catch (IOException e) {
        throw new UncheckedIOException("Error reading index records, ", e);
    }
    if (createIndexMetadata.getCreateLevel() == CreateIndexMetadata.Level.UNDEFINED) {
        boolean tableIsPartitioned = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), indexRecord.qualifiedTable) != null;
        createIndexMetadata.decideIndexLevel(tableIsPartitioned);
    }
    Map<String, List<Object>> values = new HashMap<>();
    for (int blockId = 0; blockId < page.getChannelCount(); blockId++) {
        Block block = page.getBlock(blockId);
        Pair<String, Type> entry = createIndexMetadata.getIndexColumns().get(blockId);
        String indexColumn = entry.getFirst();
        Type type = entry.getSecond();
        for (int position = 0; position < block.getPositionCount(); ++position) {
            Object value = getNativeValue(type, block, position);
            value = getActualValue(type, value);
            values.computeIfAbsent(indexColumn, k -> new ArrayList<>()).add(value);
        }
    }
    Properties connectorMetadata = new Properties();
    connectorMetadata.put(HetuConstant.DATASOURCE_CATALOG, createIndexMetadata.getTableName().split("\\.")[0]);
    connectorMetadata.putAll(page.getPageMetadata());
    try {
        switch(createIndexMetadata.getCreateLevel()) {
            case STRIPE:
                {
                    String filePath = page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH);
                    // The orc file this page resides in wasn't modified from when the index was created/last updated
                    if (pathToModifiedTime.containsKey(filePath) && pathToModifiedTime.get(filePath).equals(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION))) {
                        return;
                    }
                    levelWriter.computeIfAbsent(filePath, k -> heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(filePath), this);
                    levelWriter.get(filePath).addData(values, connectorMetadata);
                    break;
                }
            case PARTITION:
                {
                    String partition = getPartitionName(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_PATH), createIndexMetadata.getTableName());
                    indexLevelToMaxModifiedTime.compute(partition, (k, v) -> {
                        if (v != null && v >= (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)))) {
                            return v;
                        }
                        return (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)));
                    });
                    levelWriter.putIfAbsent(partition, heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(partition), this);
                    levelWriter.get(partition).addData(values, connectorMetadata);
                    break;
                }
            case TABLE:
                {
                    indexLevelToMaxModifiedTime.compute(createIndexMetadata.getTableName(), (k, v) -> {
                        if (v != null && v >= (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)))) {
                            return v;
                        }
                        return (Long.parseLong(page.getPageMetadata().getProperty(HetuConstant.DATASOURCE_FILE_MODIFICATION)));
                    });
                    levelWriter.putIfAbsent(createIndexMetadata.getTableName(), heuristicIndexerManager.getIndexWriter(createIndexMetadata, connectorMetadata));
                    persistBy.putIfAbsent(levelWriter.get(createIndexMetadata.getTableName()), this);
                    levelWriter.get(createIndexMetadata.getTableName()).addData(values, connectorMetadata);
                    break;
                }
            default:
                throw new IllegalArgumentException("Create level not supported");
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 23 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestPartitionedOutputBuffer method testMarkers.

@Test
public void testMarkers() {
    int firstPartition = 0;
    int secondPartition = 1;
    PartitionedOutputBuffer buffer = createPartitionedBuffer(createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(FIRST, firstPartition).withBuffer(SECOND, secondPartition).withNoMoreBufferIds(), sizeOfPages(6));
    ScheduledExecutorService scheduler = newScheduledThreadPool(4, daemonThreadsNamed("test-%s"));
    ScheduledExecutorService scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
    TaskContext taskContext = createTaskContext(scheduler, scheduledExecutor, TEST_SNAPSHOT_SESSION);
    buffer.setTaskContext(taskContext);
    buffer.addInputChannel("id");
    buffer.setNoMoreInputChannels();
    MarkerPage marker1 = MarkerPage.snapshotPage(1);
    MarkerPage marker2 = MarkerPage.snapshotPage(2);
    // add 1 item to each buffer
    addPage(buffer, createPage(0), firstPartition);
    addPage(buffer, createPage(0), secondPartition);
    // broadcast 2 pages
    addPage(buffer, marker1, true);
    addPage(buffer, marker2, true);
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(3, 3));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(0L, 0L));
    // try to add one more page, which should block
    ListenableFuture<?> future = enqueuePage(buffer, createPage(3), firstPartition);
    assertFalse(future.isDone());
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(4, 3));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(0L, 0L));
    // get most elements
    assertBufferResultEquals(TYPES, getBufferResult(buffer, FIRST, 0, sizeOfPages(3), NO_WAIT), bufferResult(0, createPage(0), marker1, marker2));
    assertBufferResultEquals(TYPES, getBufferResult(buffer, SECOND, 0, sizeOfPages(3), NO_WAIT), bufferResult(0, createPage(0), marker1, marker2));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(4, 3));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(0L, 0L));
    // we should still be blocked
    assertFalse(future.isDone());
    // acknowledge all pages
    buffer.get(FIRST, 3, sizeOfPages(3)).cancel(true);
    buffer.get(SECOND, 3, sizeOfPages(3)).cancel(true);
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(1, 0));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(3L, 3L));
    // finish the buffer
    assertFalse(buffer.isFinished());
    buffer.setNoMorePages();
    assertBufferResultEquals(TYPES, getBufferResult(buffer, FIRST, 3, sizeOfPages(1), NO_WAIT), bufferResult(3, createPage(3)));
    buffer.get(FIRST, 4, sizeOfPages(1)).cancel(true);
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(0, 0));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(4L, 3L));
    assertFutureIsDone(future);
    buffer.abort(FIRST);
    buffer.abort(SECOND);
    assertQueueClosed(buffer, FIRST, 4);
    assertQueueClosed(buffer, SECOND, 3);
    assertFinished(buffer);
}

Example 24 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class LocalExchangeSource method addPage.

void addPage(PageReference pageReference, String origin) {
    checkNotHoldsLock();
    boolean added = false;
    SettableFuture<?> notEmptySettableFuture;
    synchronized (lock) {
        // ignore pages after finish
        if (!finishing) {
            if (snapshotState != null) {
                // For local-merge
                Page page;
                synchronized (snapshotState) {
                    // This may look suspicious, in that if there are "pending pages" in the snapshot state, then
                    // a) those pages were associated with specific input channels (exchange source/sink) when the state
                    // was captured, but now they would be returned to any channel asking for the next page, and
                    // b) when the pending page is returned, the current page (in pageReference) is discarded and lost.
                    // But the above never happens because "merge" operators are always preceded by OrderByOperators,
                    // which only send data pages at the end, *after* all markers. That means when snapshot is taken,
                    // no data page has been received, so when the snapshot is restored, there won't be any pending pages.
                    page = snapshotState.processPage(() -> Pair.of(pageReference.peekPage(), origin)).orElse(null);
                }
                // if new input page is marker, we don't add it to buffer, it will be obtained through MultiInputSnapshotState's getPendingMarker()
                if (page instanceof MarkerPage || page == null) {
                    pageReference.removePage();
                    // if it is, this local exchange source will unblock in order for next operator to ask for output to pass down the marker.
                    if (!this.notEmptyFuture.isDone()) {
                        notEmptySettableFuture = this.notEmptyFuture;
                        this.notEmptyFuture = NOT_EMPTY;
                        notEmptySettableFuture.set(null);
                    }
                    return;
                }
            }
            // buffered bytes must be updated before adding to the buffer to assure
            // the count does not go negative
            bufferedBytes.addAndGet(pageReference.getRetainedSizeInBytes());
            buffer.add(pageReference);
            originBuffer.add(Optional.ofNullable(origin));
            added = true;
        }
        // we just added a page (or we are finishing) so we are not empty
        notEmptySettableFuture = this.notEmptyFuture;
        this.notEmptyFuture = NOT_EMPTY;
    }
    if (!added) {
        // dereference the page outside of lock
        pageReference.removePage();
    }
    // notify readers outside of lock since this may result in a callback
    notEmptySettableFuture.set(null);
}

Example 25 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestArbitraryOutputBuffer method testSnapshotState.

@Test(dataProvider = "bufferTypes")
public void testSnapshotState(OutputBuffers.BufferType bufferType) {
    OutputBuffer buffer = createOutputBuffer(bufferType);
    TaskContext taskContext = BufferTestUtils.newTestingTaskContext();
    buffer.setTaskContext(taskContext);
    buffer.addInputChannel("1");
    buffer.addInputChannel("2");
    buffer.setNoMoreInputChannels();
    MarkerPage markerPage = MarkerPage.snapshotPage(1);
    SerializedPage marker = SerializedPage.forMarker(markerPage);
    buffer.enqueue(Collections.singletonList(marker), "1");
    List<SerializedPage> pages = getBufferResult(buffer, FIRST, 0, sizeOfPages(3), NO_WAIT).getSerializedPages();
    assertEquals(pages.size(), 1);
    assertTrue(pages.get(0).isMarkerPage());
    // Acknowledge
    buffer.get(FIRST, 1, sizeOfPages(1)).cancel(true);
    buffer.enqueue(Collections.singletonList(marker), "2");
    pages = getBufferResult(buffer, FIRST, 0, sizeOfPages(3), NO_WAIT).getSerializedPages();
    assertEquals(pages.size(), 0);
}