Examples with Metadata io.prestosql.metadata.Metadata used on opensource projects

Example 76 with Metadata

use of io.prestosql.metadata.Metadata in project hetu-core by openlookeng.

the class TestHashAggregationOperator method testSpillerFailure.

@Test
public void testSpillerFailure() {
    Metadata localMetadata = createTestMetadataManager();
    InternalAggregationFunction maxVarcharColumn = localMetadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(new Signature(QualifiedObjectName.valueOfDefaultFunction("max"), AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.VARCHAR)));
    List<Integer> hashChannels = Ints.asList(1);
    ImmutableList<Type> types = ImmutableList.of(VARCHAR, BIGINT, VARCHAR, BIGINT);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(false, hashChannels, types);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100, 0, 100, 0).addSequencePage(10, 100, 0, 200, 0).addSequencePage(10, 100, 0, 300, 0).build();
    DriverContext driverContext = TestingTaskContext.builder(executor, scheduledExecutor, TEST_SESSION).setQueryMaxMemory(DataSize.valueOf("7MB")).setMemoryPoolSize(DataSize.valueOf("1GB")).build().addPipelineContext(0, true, true, false).addDriverContext();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.SINGLE, false, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(3), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(3), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(new DataSize(16, MEGABYTE)), true, succinctBytes(8), succinctBytes(Integer.MAX_VALUE), new FailingSpillerFactory(), joinCompiler, false);
    try {
        toPages(operatorFactory, driverContext, input);
        fail("An exception was expected");
    } catch (RuntimeException expected) {
        if (!nullToEmpty(expected.getMessage()).matches(".* Failed to spill")) {
            fail("Exception other than expected was thrown", expected);
        }
    }
}

Example 77 with Metadata

use of io.prestosql.metadata.Metadata in project hetu-core by openlookeng.

the class TestHashAggregationOperator method testHashAggregationSnapshot.

@Test
public void testHashAggregationSnapshot() {
    // make operator produce multiple pages during finish phase
    long memoryLimitForMerge = 0;
    long memoryLimitForMergeWithMemory = 0;
    int numberOfRows = 40_000;
    Metadata localMetadata = createTestMetadataManager();
    InternalAggregationFunction countVarcharColumn = localMetadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(new Signature(QualifiedObjectName.valueOfDefaultFunction("count"), AGGREGATE, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.VARCHAR)));
    InternalAggregationFunction countBooleanColumn = localMetadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(new Signature(QualifiedObjectName.valueOfDefaultFunction("count"), AGGREGATE, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BOOLEAN)));
    InternalAggregationFunction maxVarcharColumn = localMetadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(new Signature(QualifiedObjectName.valueOfDefaultFunction("max"), AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.VARCHAR)));
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(false, hashChannels, VARCHAR, VARCHAR, VARCHAR, BIGINT, BOOLEAN);
    List<Page> input = rowPagesBuilder.addSequencePage(numberOfRows, 100, 0, 100_000, 0, 500).addSequencePage(numberOfRows, 100, 0, 200_000, 0, 500).addSequencePage(numberOfRows, 100, 0, 300_000, 0, 500).build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR), hashChannels, ImmutableList.of(), Step.SINGLE, false, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(3), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(3), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty()), countVarcharColumn.bind(ImmutableList.of(0), Optional.empty()), countBooleanColumn.bind(ImmutableList.of(4), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(new DataSize(16, MEGABYTE)), false, succinctBytes(memoryLimitForMerge), succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, false);
    DriverContext driverContext = createDriverContext(memoryLimitForMerge);
    MaterializedResult.Builder expectedBuilder = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT);
    for (int i = 0; i < numberOfRows; ++i) {
        expectedBuilder.row(Integer.toString(i), 3L, 3L * i, (double) i, Integer.toString(300_000 + i), 3L, 3L);
    }
    MaterializedResult expected = expectedBuilder.build();
    List<Page> pages = toPagesCompareStateSimple(operatorFactory, driverContext, input, false, createExpectedMapping());
    assertGreaterThan(pages.size(), 1, "Expected more than one output page");
    assertPagesEqualIgnoreOrder(driverContext, pages, expected, false, Optional.of(hashChannels.size()));
    assertFalse(spillerFactory.getSpillsCount() > 0, format("Spill state mismatch. Expected spill: %s, spill count: %s", false, spillerFactory.getSpillsCount()));
}

Example 78 with Metadata

use of io.prestosql.metadata.Metadata in project hetu-core by openlookeng.

the class TestHashAggregationOperator method testHashAggregationWithGlobals.

@Test(dataProvider = "hashEnabledAndMemoryLimitForMergeValues")
public void testHashAggregationWithGlobals(boolean hashEnabled, boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimitForMerge, long memoryLimitForMergeWithMemory) {
    Metadata localMetadata = createTestMetadataManager();
    InternalAggregationFunction countVarcharColumn = localMetadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(new Signature(QualifiedObjectName.valueOfDefaultFunction("count"), AGGREGATE, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.VARCHAR)));
    InternalAggregationFunction countBooleanColumn = localMetadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(new Signature(QualifiedObjectName.valueOfDefaultFunction("count"), AGGREGATE, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BOOLEAN)));
    InternalAggregationFunction maxVarcharColumn = localMetadata.getFunctionAndTypeManager().getAggregateFunctionImplementation(new Signature(QualifiedObjectName.valueOfDefaultFunction("max"), AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.VARCHAR)));
    Optional<Integer> groupIdChannel = Optional.of(1);
    List<Integer> groupByChannels = Ints.asList(1, 2);
    List<Integer> globalAggregationGroupIds = Ints.asList(42, 49);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, groupByChannels, VARCHAR, VARCHAR, VARCHAR, BIGINT, BIGINT, BOOLEAN);
    List<Page> input = rowPagesBuilder.build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), groupByChannels, globalAggregationGroupIds, Step.SINGLE, true, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(4), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(4), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty()), countVarcharColumn.bind(ImmutableList.of(0), Optional.empty()), countBooleanColumn.bind(ImmutableList.of(5), Optional.empty())), rowPagesBuilder.getHashChannel(), groupIdChannel, 100_000, Optional.of(new DataSize(16, MEGABYTE)), spillEnabled, succinctBytes(memoryLimitForMerge), succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, false);
    DriverContext driverContext = createDriverContext(memoryLimitForMerge);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT).row(null, 42L, 0L, null, null, null, 0L, 0L).row(null, 49L, 0L, null, null, null, 0L, 0L).build();
    assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, expected, hashEnabled, Optional.of(groupByChannels.size()), revokeMemoryWhenAddingPages);
}

Example 79 with Metadata

use of io.prestosql.metadata.Metadata in project hetu-core by openlookeng.

the class TestFilterAndProjectOperator method testSnapshot.

@Test
public void testSnapshot() {
    List<Page> input = rowPagesBuilder(VARCHAR, BIGINT).addSequencePage(100, 0, 0).addSequencePage(100, 0, 0).addSequencePage(100, 0, 0).build();
    RowExpression filter = call(BETWEEN.getFunctionName().toString(), new BuiltInFunctionHandle(Signature.internalOperator(BETWEEN, BOOLEAN.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature(), BIGINT.getTypeSignature()))), BOOLEAN, field(1, BIGINT), constant(10L, BIGINT), constant(19L, BIGINT));
    RowExpression field0 = field(0, VARCHAR);
    RowExpression add5 = call(ADD.getFunctionName().toString(), new BuiltInFunctionHandle(Signature.internalOperator(ADD, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature()))), BIGINT, field(1, BIGINT), constant(5L, BIGINT));
    Metadata metadata = createTestMetadataManager();
    ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    Supplier<PageProcessor> processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field0, add5));
    OperatorFactory operatorFactory = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(0, new PlanNodeId("test"), processor, ImmutableList.of(VARCHAR, BIGINT), new DataSize(0, BYTE), 0);
    MaterializedResult expected = MaterializedResult.resultBuilder(driverContext.getSession(), VARCHAR, BIGINT).row("10", 15L).row("11", 16L).row("12", 17L).row("13", 18L).row("14", 19L).row("15", 20L).row("16", 21L).row("17", 22L).row("18", 23L).row("19", 24L).row("10", 15L).row("11", 16L).row("12", 17L).row("13", 18L).row("14", 19L).row("15", 20L).row("16", 21L).row("17", 22L).row("18", 23L).row("19", 24L).row("10", 15L).row("11", 16L).row("12", 17L).row("13", 18L).row("14", 19L).row("15", 20L).row("16", 21L).row("17", 22L).row("18", 23L).row("19", 24L).build();
    assertOperatorEqualsWithStateComparison(operatorFactory, driverContext, input, expected, false, ImmutableList.of(), true, createExpectedMapping());
}

Example 80 with Metadata

use of io.prestosql.metadata.Metadata in project hetu-core by openlookeng.

the class TestFilterAndProjectOperator method test.

@Test
public void test() {
    List<Page> input = rowPagesBuilder(VARCHAR, BIGINT).addSequencePage(100, 0, 0).build();
    RowExpression filter = call(BETWEEN.getFunctionName().toString(), new BuiltInFunctionHandle(Signature.internalOperator(BETWEEN, BOOLEAN.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature(), BIGINT.getTypeSignature()))), BOOLEAN, field(1, BIGINT), constant(10L, BIGINT), constant(19L, BIGINT));
    RowExpression field0 = field(0, VARCHAR);
    RowExpression add5 = call(ADD.getFunctionName().toString(), new BuiltInFunctionHandle(Signature.internalOperator(ADD, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature()))), BIGINT, field(1, BIGINT), constant(5L, BIGINT));
    Metadata metadata = createTestMetadataManager();
    ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    Supplier<PageProcessor> processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field0, add5));
    OperatorFactory operatorFactory = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(0, new PlanNodeId("test"), processor, ImmutableList.of(VARCHAR, BIGINT), new DataSize(0, BYTE), 0);
    MaterializedResult expected = MaterializedResult.resultBuilder(driverContext.getSession(), VARCHAR, BIGINT).row("10", 15L).row("11", 16L).row("12", 17L).row("13", 18L).row("14", 19L).row("15", 20L).row("16", 21L).row("17", 22L).row("18", 23L).row("19", 24L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected);
}