Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 26 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class PageFunctionCompiler method definePageProjectWorkClass.

private ClassDefinition definePageProjectWorkClass(SqlFunctionProperties sqlFunctionProperties, Map<SqlFunctionId, SqlInvokedFunction> sessionFunctions, List<RowExpression> projections, CallSiteBinder callSiteBinder, boolean isOptimizeCommonSubExpression, Optional<String> classNameSuffix) {
    ClassDefinition classDefinition = new ClassDefinition(a(PUBLIC, FINAL), generateProjectionWorkClassName(classNameSuffix), type(Object.class), type(Work.class));
    FieldDefinition blockBuilderFields = classDefinition.declareField(a(PRIVATE), "blockBuilders", type(List.class, BlockBuilder.class));
    FieldDefinition propertiesField = classDefinition.declareField(a(PRIVATE), "properties", SqlFunctionProperties.class);
    FieldDefinition pageField = classDefinition.declareField(a(PRIVATE), "page", Page.class);
    FieldDefinition selectedPositionsField = classDefinition.declareField(a(PRIVATE), "selectedPositions", SelectedPositions.class);
    FieldDefinition nextIndexOrPositionField = classDefinition.declareField(a(PRIVATE), "nextIndexOrPosition", int.class);
    FieldDefinition resultField = classDefinition.declareField(a(PRIVATE), "result", List.class);
    CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder);
    // process
    generateProcessMethod(classDefinition, blockBuilderFields, projections.size(), propertiesField, pageField, selectedPositionsField, nextIndexOrPositionField, resultField);
    // getResult
    MethodDefinition method = classDefinition.declareMethod(a(PUBLIC), "getResult", type(Object.class), ImmutableList.of());
    method.getBody().append(method.getThis().getField(resultField)).ret(Object.class);
    Map<LambdaDefinitionExpression, CompiledLambda> compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, projections, metadata, sqlFunctionProperties, sessionFunctions, "");
    // cse
    Map<VariableReferenceExpression, CommonSubExpressionFields> cseFields = ImmutableMap.of();
    RowExpressionCompiler compiler = new RowExpressionCompiler(classDefinition, callSiteBinder, cachedInstanceBinder, new FieldAndVariableReferenceCompiler(callSiteBinder, cseFields), metadata, sqlFunctionProperties, sessionFunctions, compiledLambdaMap);
    if (isOptimizeCommonSubExpression) {
        Map<Integer, Map<RowExpression, VariableReferenceExpression>> commonSubExpressionsByLevel = collectCSEByLevel(projections);
        if (!commonSubExpressionsByLevel.isEmpty()) {
            cseFields = declareCommonSubExpressionFields(classDefinition, commonSubExpressionsByLevel);
            compiler = new RowExpressionCompiler(classDefinition, callSiteBinder, cachedInstanceBinder, new FieldAndVariableReferenceCompiler(callSiteBinder, cseFields), metadata, sqlFunctionProperties, sessionFunctions, compiledLambdaMap);
            generateCommonSubExpressionMethods(classDefinition, compiler, commonSubExpressionsByLevel, cseFields);
            Map<RowExpression, VariableReferenceExpression> commonSubExpressions = commonSubExpressionsByLevel.values().stream().flatMap(m -> m.entrySet().stream()).collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
            projections = projections.stream().map(projection -> rewriteExpressionWithCSE(projection, commonSubExpressions)).collect(toImmutableList());
            if (log.isDebugEnabled()) {
                log.debug("Extracted %d common sub-expressions", commonSubExpressions.size());
                commonSubExpressions.entrySet().forEach(entry -> log.debug("\t%s = %s", entry.getValue(), entry.getKey()));
                log.debug("Rewrote %d projections: %s", projections.size(), Joiner.on(", ").join(projections));
            }
        }
    }
    // evaluate
    generateEvaluateMethod(classDefinition, compiler, projections, blockBuilderFields, cseFields);
    // constructor
    Parameter blockBuilders = arg("blockBuilders", type(List.class, BlockBuilder.class));
    Parameter properties = arg("properties", SqlFunctionProperties.class);
    Parameter page = arg("page", Page.class);
    Parameter selectedPositions = arg("selectedPositions", SelectedPositions.class);
    MethodDefinition constructorDefinition = classDefinition.declareConstructor(a(PUBLIC), blockBuilders, properties, page, selectedPositions);
    BytecodeBlock body = constructorDefinition.getBody();
    Variable thisVariable = constructorDefinition.getThis();
    body.comment("super();").append(thisVariable).invokeConstructor(Object.class).append(thisVariable.setField(blockBuilderFields, invokeStatic(ImmutableList.class, "copyOf", ImmutableList.class, blockBuilders.cast(Collection.class)))).append(thisVariable.setField(propertiesField, properties)).append(thisVariable.setField(pageField, page)).append(thisVariable.setField(selectedPositionsField, selectedPositions)).append(thisVariable.setField(nextIndexOrPositionField, selectedPositions.invoke("getOffset", int.class))).append(thisVariable.setField(resultField, constantNull(Block.class)));
    initializeCommonSubExpressionFields(cseFields.values(), thisVariable, body);
    cachedInstanceBinder.generateInitializations(thisVariable, body);
    body.ret();
    return classDefinition;
}

Example 27 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestGroupByHash method testMemoryReservationYieldWithDictionary.

@Test
public void testMemoryReservationYieldWithDictionary() {
    // Create a page with positionCount >> expected size of groupByHash
    int dictionaryLength = 1_000;
    int length = 2_000_000;
    int[] ids = IntStream.range(0, dictionaryLength).toArray();
    DictionaryId dictionaryId = randomDictionaryId();
    Block valuesBlock = new DictionaryBlock(dictionaryLength, createStringSequenceBlock(0, length), ids, dictionaryId);
    Block hashBlock = new DictionaryBlock(dictionaryLength, getHashBlock(ImmutableList.of(VARCHAR), valuesBlock), ids, dictionaryId);
    Page page = new Page(valuesBlock, hashBlock);
    AtomicInteger currentQuota = new AtomicInteger(0);
    AtomicInteger allowedQuota = new AtomicInteger(3);
    UpdateMemory updateMemory = () -> {
        if (currentQuota.get() < allowedQuota.get()) {
            currentQuota.getAndIncrement();
            return true;
        }
        return false;
    };
    int yields = 0;
    // test addPage
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    boolean finish = false;
    Work<?> addPageWork = groupByHash.addPage(page);
    while (!finish) {
        finish = addPageWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(addPageWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
    // test getGroupIds
    currentQuota.set(0);
    allowedQuota.set(3);
    yields = 0;
    groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    finish = false;
    Work<GroupByIdBlock> getGroupIdsWork = groupByHash.getGroupIds(page);
    while (!finish) {
        finish = getGroupIdsWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(getGroupIdsWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    assertEquals(dictionaryLength, getGroupIdsWork.getResult().getPositionCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log2(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
}

Example 28 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestGroupedTopNBuilder method testLargePagesMemoryTracking.

@Test(dataProvider = "pageRowCounts")
public void testLargePagesMemoryTracking(int pageCount, int rowCount) {
    // Memory tracking has been tested in other tests for various cases (e.g., compaction, row removal, etc).
    // The purpose of this test is to verify:
    // (1) the sizes of containers (e.g., big array, heap, etc) are properly tracked when they grow and
    // (2) when we flush, the memory usage decreases accordingly.
    List<Type> types = ImmutableList.of(BIGINT, DOUBLE);
    // Create pageCount pages each page is with rowCount positions:
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(types);
    for (int i = 0; i < pageCount; i++) {
        rowPagesBuilder.addSequencePage(rowCount, 0, rowCount * i);
    }
    List<Page> input = rowPagesBuilder.build();
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(types.get(0)), ImmutableList.of(0), NOOP);
    GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder(types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST)), pageCount * rowCount, false, groupByHash);
    // Assert memory usage gradually goes up
    for (int i = 0; i < pageCount; i++) {
        assertTrue(groupedTopNBuilder.processPage(input.get(i)).process());
        assertBuilderSize(groupByHash, types, Collections.nCopies(i + 1, rowCount), Collections.nCopies(rowCount, i + 1), groupedTopNBuilder.getEstimatedSizeInBytes());
    }
    // Assert memory usage gradually goes down (i.e., proportional to the number of rows/pages we have produced)
    int outputPageCount = 0;
    int remainingRows = pageCount * rowCount;
    Iterator<Page> output = groupedTopNBuilder.buildResult();
    while (output.hasNext()) {
        remainingRows -= output.next().getPositionCount();
        assertBuilderSize(groupByHash, types, remainingRows == 0 ? Collections.nCopies(pageCount, 0) : Collections.nCopies(pageCount, rowCount), new ImmutableList.Builder<Integer>().addAll(Collections.nCopies((remainingRows + pageCount - 1) / pageCount, pageCount)).addAll(Collections.nCopies(rowCount - (remainingRows + pageCount - 1) / pageCount, 0)).build(), groupedTopNBuilder.getEstimatedSizeInBytes());
        outputPageCount++;
    }
    assertEquals(remainingRows, 0);
    assertGreaterThan(outputPageCount, 3);
    assertBuilderSize(groupByHash, types, Collections.nCopies(pageCount, 0), Collections.nCopies(rowCount, 0), groupedTopNBuilder.getEstimatedSizeInBytes());
}

Example 29 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestGroupedTopNBuilder method testAutoCompact.

@Test
public void testAutoCompact() {
    List<Type> types = ImmutableList.of(BIGINT, DOUBLE);
    List<Page> input = rowPagesBuilder(types).row(1L, 0.8).row(2L, 0.7).row(3L, 0.9).row(3L, 0.2).row(3L, 0.2).row(3L, 0.2).row(3L, 0.2).pageBreak().row(3L, 0.8).pageBreak().row(2L, 0.6).row(3L, 0.1).pageBreak().row(1L, 0.7).pageBreak().row(1L, 0.6).build();
    GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder(types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST)), 1, false, createGroupByHash(ImmutableList.of(types.get(0)), ImmutableList.of(0), NOOP));
    // page 1:
    // the first page will be compacted
    assertTrue(groupedTopNBuilder.processPage(input.get(0)).process());
    assertEquals(groupedTopNBuilder.getBufferedPages().size(), 1);
    Page firstCompactPage = groupedTopNBuilder.getBufferedPages().get(0);
    Page expected = rowPagesBuilder(types).row(1L, 0.8).row(2L, 0.7).row(3L, 0.2).build().get(0);
    assertPageEquals(types, firstCompactPage, expected);
    // page 2:
    // the second page will be removed
    assertTrue(groupedTopNBuilder.processPage(input.get(1)).process());
    assertEquals(groupedTopNBuilder.getBufferedPages().size(), 1);
    // assert the first page is not affected
    assertEquals(firstCompactPage, groupedTopNBuilder.getBufferedPages().get(0));
    // page 3:
    // the third page will trigger another compaction of the first page
    assertTrue(groupedTopNBuilder.processPage(input.get(2)).process());
    List<Page> bufferedPages = groupedTopNBuilder.getBufferedPages();
    assertEquals(bufferedPages.size(), 2);
    // assert the previously compacted first page no longer exists
    assertNotEquals(firstCompactPage, bufferedPages.get(0));
    assertNotEquals(firstCompactPage, bufferedPages.get(1));
    List<Page> expectedPages = rowPagesBuilder(types).row(1L, 0.8).pageBreak().row(2L, 0.6).row(3L, 0.1).build();
    assertPageEquals(types, bufferedPages.get(0), expectedPages.get(0));
    assertPageEquals(types, bufferedPages.get(1), expectedPages.get(1));
    // page 4:
    // the fourth page will remove the first page; also it leaves it with an empty slot
    assertTrue(groupedTopNBuilder.processPage(input.get(3)).process());
    bufferedPages = groupedTopNBuilder.getBufferedPages();
    assertEquals(bufferedPages.size(), 2);
    expectedPages = rowPagesBuilder(types).row(2L, 0.6).row(3L, 0.1).pageBreak().row(1L, 0.7).build();
    assertPageEquals(types, bufferedPages.get(0), expectedPages.get(0));
    assertPageEquals(types, bufferedPages.get(1), expectedPages.get(1));
    // page 5:
    // the fifth page will remove the fourth page and it will take the empty slot from the first page
    assertTrue(groupedTopNBuilder.processPage(input.get(4)).process());
    bufferedPages = groupedTopNBuilder.getBufferedPages();
    assertEquals(bufferedPages.size(), 2);
    // assert the fifth page indeed takes the first empty slot
    expectedPages = rowPagesBuilder(types).row(1L, 0.6).pageBreak().row(2L, 0.6).row(3L, 0.1).build();
    assertPageEquals(types, bufferedPages.get(0), expectedPages.get(0));
    assertPageEquals(types, bufferedPages.get(1), expectedPages.get(1));
}

Example 30 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testInnerJoinWithNullBuild.

@Test(dataProvider = "hashJoinTestValues")
public void testInnerJoinWithNullBuild(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes).row("a").row((String) null).row((String) null).row("a").row("b");
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row("b").row("c").build();
    OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactory, probePages, PARTITIONING_SPILLER_FACTORY);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("a", "a").row("a", "a").row("b", "b").build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}