Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 41 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestExchangeClient method testInitialRequestLimit.

@Test
public void testInitialRequestLimit() {
    DataSize bufferCapacity = new DataSize(16, MEGABYTE);
    DataSize maxResponseSize = new DataSize(DEFAULT_MAX_PAGE_SIZE_IN_BYTES, BYTE);
    CountDownLatch countDownLatch = new CountDownLatch(1);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize) {

        @Override
        public Response handle(Request request) {
            if (!awaitUninterruptibly(countDownLatch, 10, SECONDS)) {
                throw new UncheckedTimeoutException();
            }
            return super.handle(request);
        }
    };
    List<URI> locations = new ArrayList<>();
    int numLocations = 16;
    List<DataSize> expectedMaxSizes = new ArrayList<>();
    // add pages
    for (int i = 0; i < numLocations; i++) {
        URI location = URI.create("http://localhost:" + (8080 + i));
        locations.add(location);
        processor.addPage(location, createPage(DEFAULT_MAX_PAGE_SIZE_IN_BYTES));
        processor.addPage(location, createPage(DEFAULT_MAX_PAGE_SIZE_IN_BYTES));
        processor.addPage(location, createPage(DEFAULT_MAX_PAGE_SIZE_IN_BYTES));
        processor.setComplete(location);
        expectedMaxSizes.add(maxResponseSize);
    }
    try (ExchangeClient exchangeClient = createExchangeClient(processor, bufferCapacity, maxResponseSize)) {
        for (int i = 0; i < numLocations; i++) {
            exchangeClient.addLocation(locations.get(i), TaskId.valueOf("taskid.0.0." + i));
        }
        exchangeClient.noMoreLocations();
        assertFalse(exchangeClient.isClosed());
        long start = System.nanoTime();
        countDownLatch.countDown();
        // wait for a page to be fetched
        do {
            // there is no thread coordination here, so sleep is the best we can do
            assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
            sleepUninterruptibly(100, MILLISECONDS);
        } while (exchangeClient.getStatus().getBufferedPages() < 16);
        // Client should have sent 16 requests for a single page (0) and gotten them back
        // The memory limit should be hit immediately and then it doesn't fetch the third page from each
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 16);
        assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
        List<PageBufferClientStatus> pageBufferClientStatuses = exchangeClient.getStatus().getPageBufferClientStatuses();
        assertEquals(16, pageBufferClientStatuses.stream().filter(status -> status.getPagesReceived() == 1).mapToInt(PageBufferClientStatus::getPagesReceived).sum());
        assertEquals(processor.getRequestMaxSizes(), expectedMaxSizes);
        for (int i = 0; i < numLocations * 3; i++) {
            assertNotNull(getNextPage(exchangeClient));
        }
        do {
            // there is no thread coordination here, so sleep is the best we can do
            assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
            sleepUninterruptibly(100, MILLISECONDS);
        } while (processor.getRequestMaxSizes().size() < 64);
        for (int i = 0; i < 48; i++) {
            expectedMaxSizes.add(maxResponseSize);
        }
        assertEquals(processor.getRequestMaxSizes(), expectedMaxSizes);
    }
}

Example 42 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testOuterJoinWithNullOnBothSidesAndFilterFunction.

@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinWithNullOnBothSidesAndFilterFunction(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> ImmutableSet.of("a", "c").contains(VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii())));
    // build factory
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR)).row("a").row((String) null).row((String) null).row("a").row("b");
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build();
    OperatorFactory joinOperatorFactory = probeOuterJoinOperatorFactory(lookupSourceFactory, probePages);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildPages.getTypesWithoutHash())).row("a", "a").row("a", "a").row("b", null).row(null, null).row("c", null).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}

Example 43 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testOuterJoinWithNullBuildAndFilterFunction.

@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinWithNullBuildAndFilterFunction(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> ImmutableSet.of("a", "c").contains(VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii())));
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR)).row("a").row((String) null).row((String) null).row("a").row("b");
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row("b").row("c").build();
    OperatorFactory joinOperatorFactory = probeOuterJoinOperatorFactory(lookupSourceFactory, probePages);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("a", "a").row("a", "a").row("b", null).row("c", null).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}

Example 44 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testOuterJoinWithNullProbeAndFilterFunction.

@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinWithNullProbeAndFilterFunction(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii().equals("a")));
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes).row("a").row("b").row("c");
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row((String) null).row((String) null).row("a").row("b").build();
    OperatorFactory joinOperatorFactory = probeOuterJoinOperatorFactory(lookupSourceFactory, probePages);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("a", "a").row(null, null).row(null, null).row("a", "a").row("b", null).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}

Example 45 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testInnerJoinWithNonEmptyLookupSourceAndEmptyProbe.

@Test(dataProvider = "hashJoinTestValues")
public void testInnerJoinWithNonEmptyLookupSourceAndEmptyProbe(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes).row("a").row("b").row((String) null).row("c");
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.build();
    OperatorFactory joinOperatorFactory = new LookupJoinOperators().innerJoin(0, new PlanNodeId("test"), lookupSourceFactoryManager, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}