Examples with FunctionAndTypeManager com.facebook.presto.metadata.FunctionAndTypeManager used on opensource projects

Example 46 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestPlanRemoteProjections method testRemoteFunctionDisabled.

@Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = ".*Remote functions are not enabled")
public void testRemoteFunctionDisabled() {
    RuleTester tester = new RuleTester(ImmutableList.of());
    FunctionAndTypeManager functionAndTypeManager = tester.getMetadata().getFunctionAndTypeManager();
    functionAndTypeManager.addFunctionNamespace("unittest", new InMemoryFunctionNamespaceManager("unittest", new SqlFunctionExecutors(ImmutableMap.of(SQL, FunctionImplementationType.SQL, JAVA, THRIFT), new NoopSqlFunctionExecutor()), new SqlInvokedFunctionNamespaceManagerConfig().setSupportedFunctionLanguages("sql,java")));
    functionAndTypeManager.createFunction(FUNCTION_REMOTE_FOO_1, true);
    tester.assertThat(new PlanRemotePojections(functionAndTypeManager)).on(p -> {
        p.variable("x", INTEGER);
        return p.project(Assignments.builder().put(p.variable("a"), p.rowExpression("unittest.memory.remote_foo(x)")).build(), p.values(p.variable("x", INTEGER)));
    }).matches(anyTree());
}

Example 47 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testPageYield.

@Test
public void testPageYield() {
    int totalRows = 1000;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), totalRows, 1);
    DriverContext driverContext = newDriverContext();
    // 20 columns; each column is associated with a function that will force yield per projection
    int totalColumns = 20;
    ImmutableList.Builder<SqlScalarFunction> functions = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        functions.add(new GenericLongFunction("page_col" + i, value -> {
            driverContext.getYieldSignal().forceYieldForTesting();
            return value;
        }));
    }
    Metadata metadata = functionAssertions.getMetadata();
    FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager();
    functionAndTypeManager.registerBuiltInFunctions(functions.build());
    // match each column with a projection
    ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    ImmutableList.Builder<RowExpression> projections = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        projections.add(call("generic_long_page_col", functionAndTypeManager.lookupFunction("generic_long_page_col" + i, fromTypes(BIGINT)), BIGINT, field(0, BIGINT)));
    }
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections.build(), "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections.build(), false, MAX_BATCH_SIZE);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    // exactly 20 blocks (one for each column) and the PageProcessor will be able to create a Page out of it.
    for (int i = 1; i <= totalRows * totalColumns; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        Page page = operator.getOutput();
        if (i == totalColumns) {
            assertNotNull(page);
            assertEquals(page.getPositionCount(), totalRows);
            assertEquals(page.getChannelCount(), totalColumns);
            for (int j = 0; j < totalColumns; j++) {
                assertEquals(toValues(BIGINT, page.getBlock(j)), toValues(BIGINT, input.getBlock(0)));
            }
        } else {
            assertNull(page);
        }
        driverContext.getYieldSignal().reset();
    }
}

Example 48 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testRecordCursorYield.

@Test
public void testRecordCursorYield() {
    // create a generic long function that yields for projection on every row
    // verify we will yield #row times totally
    // create a table with 15 rows
    int length = 15;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), length, 0);
    DriverContext driverContext = newDriverContext();
    // set up generic long function with a callback to force yield
    Metadata metadata = functionAssertions.getMetadata();
    FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager();
    functionAndTypeManager.registerBuiltInFunctions(ImmutableList.of(new GenericLongFunction("record_cursor", value -> {
        driverContext.getYieldSignal().forceYieldForTesting();
        return value;
    })));
    ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    List<RowExpression> projections = ImmutableList.of(call("generic_long_record_cursor", functionAndTypeManager.lookupFunction("generic_long_record_cursor", fromTypes(BIGINT)), BIGINT, field(0, BIGINT)));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), cursorProcessor, pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    // start driver; get null value due to yield for the first 15 times
    for (int i = 0; i < length; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        assertNull(operator.getOutput());
        driverContext.getYieldSignal().reset();
    }
    // the 16th yield is not going to prevent the operator from producing a page
    driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
    Page output = operator.getOutput();
    driverContext.getYieldSignal().reset();
    assertNotNull(output);
    assertEquals(toValues(BIGINT, output.getBlock(0)), toValues(BIGINT, input.getBlock(0)));
}

Example 49 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestTableWriterOperator method testStatisticsAggregation.

@Test
public void testStatisticsAggregation() throws Exception {
    PageSinkManager pageSinkManager = new PageSinkManager();
    pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
    ImmutableList<Type> outputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    TaskContext taskContext = createTaskContext(executor, scheduledExecutor, session);
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    TaskMetadataContext taskMetadataContext = taskContext.getTaskMetadataContext();
    FunctionAndTypeManager functionAndTypeManager = createTestMetadataManager().getFunctionAndTypeManager();
    InternalAggregationFunction longMaxFunction = functionAndTypeManager.getAggregateFunctionImplementation(functionAndTypeManager.lookupFunction("max", fromTypes(BIGINT)));
    TableWriterOperator operator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(longMaxFunction.bind(ImmutableList.of(0), Optional.empty())), true), outputTypes, session, taskMetadataContext, driverContext);
    operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0));
    operator.addInput(rowPagesBuilder(BIGINT).row(43).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    operator.finish();
    assertFalse(operator.isFinished());
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(null, null, getTableCommitContext(false), 43).build().get(0));
    BlockBuilder rowsBuilder = BIGINT.createBlockBuilder(null, 1);
    BlockBuilder fragmentsBuilder = VARBINARY.createBlockBuilder(null, 1);
    rowsBuilder.writeLong(1);
    fragmentsBuilder.appendNull();
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(2, null, getTableCommitContext(true), null).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertMemoryIsReleased(operator);
    TableWriterInfo info = operator.getInfo();
    assertThat(info.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(info.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
}

Example 50 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestHttpRemoteTask method createHttpRemoteTaskFactory.

private static HttpRemoteTaskFactory createHttpRemoteTaskFactory(TestingTaskResource testingTaskResource, boolean useThriftEncoding) throws Exception {
    Bootstrap app = new Bootstrap(new JsonModule(), new SmileModule(), new ThriftCodecModule(), new HandleJsonModule(), new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bind(JsonMapper.class);
            binder.bind(ThriftMapper.class);
            configBinder(binder).bindConfig(FeaturesConfig.class);
            FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
            binder.bind(TypeManager.class).toInstance(functionAndTypeManager);
            jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
            newSetBinder(binder, Type.class);
            smileCodecBinder(binder).bindSmileCodec(TaskStatus.class);
            smileCodecBinder(binder).bindSmileCodec(TaskInfo.class);
            smileCodecBinder(binder).bindSmileCodec(TaskUpdateRequest.class);
            smileCodecBinder(binder).bindSmileCodec(PlanFragment.class);
            smileCodecBinder(binder).bindSmileCodec(MetadataUpdates.class);
            jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class);
            jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class);
            jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class);
            jsonCodecBinder(binder).bindJsonCodec(PlanFragment.class);
            jsonCodecBinder(binder).bindJsonCodec(MetadataUpdates.class);
            jsonBinder(binder).addKeySerializerBinding(VariableReferenceExpression.class).to(Serialization.VariableReferenceExpressionSerializer.class);
            jsonBinder(binder).addKeyDeserializerBinding(VariableReferenceExpression.class).to(Serialization.VariableReferenceExpressionDeserializer.class);
            thriftCodecBinder(binder).bindThriftCodec(TaskStatus.class);
        }

        @Provides
        private HttpRemoteTaskFactory createHttpRemoteTaskFactory(JsonMapper jsonMapper, ThriftMapper thriftMapper, JsonCodec<TaskStatus> taskStatusJsonCodec, SmileCodec<TaskStatus> taskStatusSmileCodec, ThriftCodec<TaskStatus> taskStatusThriftCodec, JsonCodec<TaskInfo> taskInfoJsonCodec, SmileCodec<TaskInfo> taskInfoSmileCodec, JsonCodec<TaskUpdateRequest> taskUpdateRequestJsonCodec, SmileCodec<TaskUpdateRequest> taskUpdateRequestSmileCodec, JsonCodec<PlanFragment> planFragmentJsonCodec, SmileCodec<PlanFragment> planFragmentSmileCodec, JsonCodec<MetadataUpdates> metadataUpdatesJsonCodec, SmileCodec<MetadataUpdates> metadataUpdatesSmileCodec) {
            JaxrsTestingHttpProcessor jaxrsTestingHttpProcessor = new JaxrsTestingHttpProcessor(URI.create("http://fake.invalid/"), testingTaskResource, jsonMapper, thriftMapper);
            TestingHttpClient testingHttpClient = new TestingHttpClient(jaxrsTestingHttpProcessor.setTrace(TRACE_HTTP));
            testingTaskResource.setHttpClient(testingHttpClient);
            return new HttpRemoteTaskFactory(new QueryManagerConfig(), TASK_MANAGER_CONFIG, testingHttpClient, new TestSqlTaskManager.MockLocationFactory(), taskStatusJsonCodec, taskStatusSmileCodec, taskStatusThriftCodec, taskInfoJsonCodec, taskInfoSmileCodec, taskUpdateRequestJsonCodec, taskUpdateRequestSmileCodec, planFragmentJsonCodec, planFragmentSmileCodec, metadataUpdatesJsonCodec, metadataUpdatesSmileCodec, new RemoteTaskStats(), new InternalCommunicationConfig().setThriftTransportEnabled(useThriftEncoding), createTestMetadataManager(), new TestQueryManager());
        }
    });
    Injector injector = app.doNotInitializeLogging().quiet().initialize();
    HandleResolver handleResolver = injector.getInstance(HandleResolver.class);
    handleResolver.addConnectorName("test", new TestingHandleResolver());
    return injector.getInstance(HttpRemoteTaskFactory.class);
}