Examples with MarkerPage io.prestosql.spi.snapshot.MarkerPage used on opensource projects

Example 16 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestHashJoinOperator method testOuterJoinSnapshot.

@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinSnapshot(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws ExecutionException, InterruptedException {
    TaskContext taskContext = createSnapshotTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)).addSequencePage(10, 20, 30, 40);
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY, true);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.addSequencePage(10, 15, 1020, 2020).build();
    // Add markers between pages
    List<Page> withMarkers = new ArrayList<>();
    long snapshotId = 1;
    for (Page page : probeInput) {
        withMarkers.add(page);
        withMarkers.add(MarkerPage.snapshotPage(snapshotId++));
    }
    probeInput = withMarkers;
    OperatorFactory joinOperatorFactory = new LookupJoinOperators().fullOuterJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // Construct lookup-outer operator
    PipelineContext buildPipeline = taskContext.addPipelineContext(2, false, true, false);
    DriverContext outerDriverContext = buildPipeline.addDriverContext(Lifespan.taskWide(), 0);
    Operator lookupOuter = ((LookupJoinOperatorFactory) joinOperatorFactory).createOuterOperatorFactory().get().getOuterOperatorFactory().createOperator(outerDriverContext);
    assertFalse(lookupOuter.isBlocked().isDone());
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("15", 1020L, 2020L, null, null, null).row("16", 1021L, 2021L, null, null, null).row("17", 1022L, 2022L, null, null, null).row("18", 1023L, 2023L, null, null, null).row("19", 1024L, 2024L, null, null, null).row("20", 1025L, 2025L, "20", 30L, 40L).row("21", 1026L, 2026L, "21", 31L, 41L).row("22", 1027L, 2027L, "22", 32L, 42L).row("23", 1028L, 2028L, "23", 33L, 43L).row("24", 1029L, 2029L, "24", 34L, 44L).build();
    List<Integer> hashChannels = getHashChannels(probePages, buildPages);
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, hashChannels);
    assertTrue(lookupOuter.isBlocked().isDone());
    assertTrue(lookupOuter.getOutput() instanceof MarkerPage);
    Object state = lookupOuter.capture(null);
    lookupOuter.isBlocked().get();
    Page page = lookupOuter.getOutput();
    page = dropChannel(ImmutableList.of(page), hashChannels).get(0);
    MaterializedResult outerExpected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row(null, null, null, "25", 35L, 45L).row(null, null, null, "26", 36L, 46L).row(null, null, null, "27", 37L, 47L).row(null, null, null, "28", 38L, 48L).row(null, null, null, "29", 39L, 49L).build();
    List<MaterializedRow> rows = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).page(page).build().getMaterializedRows();
    rows = new ArrayList<>(rows);
    rows.sort(Comparator.comparing(a -> (String) a.getField(3)));
    MaterializedResult outerResult = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).rows(rows).build();
    assertEquals(outerResult, outerExpected);
    long matched;
    if (state instanceof boolean[]) {
        boolean[] positions = (boolean[]) state;
        matched = Booleans.asList(positions).stream().filter(e -> e).count();
    } else {
        ByteBuffer bb = ByteBuffer.wrap((byte[]) state);
        List<RoaringBitmap> visitedPositions = new ArrayList<>();
        for (int i = 0; i < (parallelBuild ? PARTITION_COUNT : 1); i++) {
            ImmutableRoaringBitmap bm = new ImmutableRoaringBitmap(bb);
            visitedPositions.add(new RoaringBitmap(bm));
            bb.position(bb.position() + visitedPositions.get(i).serializedSizeInBytes());
        }
        matched = visitedPositions.stream().mapToLong(rr -> rr.getCardinality()).sum();
    }
    assertEquals(matched, 5);
}

Example 17 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestExchangeClient method testMarkers.

@Test
public void testMarkers() {
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    String instanceId = "testing instance id";
    processor.addPage(location, createPage(1));
    MarkerPage marker = MarkerPage.snapshotPage(1);
    processor.addPage(location, marker);
    processor.addPage(location, createPage(2));
    processor.setComplete(location);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), true, new TestingHttpClient(processor, scheduler), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, new NoOpFailureDetector());
    exchangeClient.setSnapshotEnabled(NOOP_SNAPSHOT_UTILS.getQuerySnapshotManager(new QueryId("query")));
    final String target1 = "target1";
    final String target2 = "target2";
    exchangeClient.addLocation(new TaskLocation(location, instanceId));
    exchangeClient.noMoreLocations();
    exchangeClient.addTarget(target1);
    exchangeClient.addTarget(target2);
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target1), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target2), marker);
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target2), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    // Target2 won't get any page, but because there are pending markers, it won't cause the client to close either
    assertTrue(exchangeClient.isBlocked().isDone());
    assertNull(getNextPage(exchangeClient, target2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target1), marker);
    // New target receives pending marker
    assertEquals(exchangeClient.isClosed(), false);
    final String target3 = "target3";
    exchangeClient.addTarget(target3);
    assertPageEquals(getNextPage(exchangeClient, target3), marker);
    assertFalse(exchangeClient.isBlocked().isDone());
    assertEquals(exchangeClient.isClosed(), false);
    exchangeClient.noMoreTargets();
    assertNull(getNextPage(exchangeClient));
    assertEquals(exchangeClient.isClosed(), true);
    ExchangeClientStatus status = exchangeClient.getStatus();
    assertEquals(status.getBufferedPages(), 0);
    assertEquals(status.getBufferedBytes(), 0);
    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    assertStatus(status.getPageBufferClientStatuses().get(0), location, "closed", 3, 3, 3, "not scheduled");
}

Example 18 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestExchangeClient method testAddTarget.

@Test
public void testAddTarget() {
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), new DataSize(10, Unit.MEGABYTE), 1, new Duration(1, TimeUnit.MINUTES), true, mock(HttpClient.class), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, new NoOpFailureDetector());
    exchangeClient.setSnapshotEnabled(NOOP_SNAPSHOT_UTILS.getQuerySnapshotManager(new QueryId("query")));
    String origin1 = "location1";
    String origin2 = "location2";
    final String target1 = "target1";
    final String target2 = "target2";
    PagesSerde serde = testingPagesSerde();
    exchangeClient.addTarget(target1);
    exchangeClient.addPages(ImmutableList.of(serde.serialize(createPage(1))), origin1);
    MarkerPage marker = MarkerPage.snapshotPage(1);
    exchangeClient.addPages(ImmutableList.of(SerializedPage.forMarker(marker)), origin1);
    exchangeClient.addPages(ImmutableList.of(serde.serialize(createPage(2))), origin2);
    assertPageEquals(getNextPage(exchangeClient, target1), createPage(1));
    assertPageEquals(getNextPage(exchangeClient, target1), marker);
    exchangeClient.addTarget(target2);
    assertPageEquals(getNextPage(exchangeClient, target2, origin1), marker);
    assertPageEquals(getNextPage(exchangeClient, target1, origin2), createPage(2));
}

Example 19 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestExchangeOperator method waitForPages.

private static List<Page> waitForPages(Operator operator, int expectedPageCount) throws InterruptedException {
    // read expected pages or until 10 seconds has passed
    long endTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
    List<Page> outputPages = new ArrayList<>();
    boolean greaterThanZero = false;
    while (System.nanoTime() - endTime < 0) {
        if (operator.isFinished()) {
            break;
        }
        if (operator.getOperatorContext().getDriverContext().getPipelineContext().getPipelineStats().getSystemMemoryReservation().toBytes() > 0) {
            greaterThanZero = true;
            break;
        } else {
            Thread.sleep(10);
        }
    }
    assertTrue(greaterThanZero);
    while (outputPages.size() < expectedPageCount && System.nanoTime() < endTime) {
        assertEquals(operator.needsInput(), false);
        if (operator.isFinished()) {
            break;
        }
        Page outputPage = operator.getOutput();
        if (outputPage != null) {
            outputPages.add(outputPage);
        } else {
            Thread.sleep(10);
        }
    }
    // sleep for a bit to make sure that there aren't extra pages on the way
    Thread.sleep(10);
    // verify state
    assertEquals(operator.needsInput(), false);
    assertNull(operator.getOutput());
    // verify pages
    assertEquals(outputPages.size(), expectedPageCount);
    for (Page page : outputPages) {
        if (!(page instanceof MarkerPage)) {
            assertPageEquals(TYPES, page, PAGE);
        }
    }
    assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0);
    return outputPages;
}

Example 20 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class OperatorAssertion method dropChannel.

static List<Page> dropChannel(List<Page> pages, List<Integer> channels) {
    List<Page> actualPages = new ArrayList<>();
    for (Page page : pages) {
        if (page instanceof MarkerPage) {
            continue;
        }
        int channel = 0;
        Block[] blocks = new Block[page.getChannelCount() - channels.size()];
        for (int i = 0; i < page.getChannelCount(); i++) {
            if (channels.contains(i)) {
                continue;
            }
            blocks[channel++] = page.getBlock(i);
        }
        actualPages.add(new Page(blocks));
    }
    return actualPages;
}