Examples with UnmodifiableConfiguration org.apache.flink.configuration.UnmodifiableConfiguration used on opensource projects

Example 1 with UnmodifiableConfiguration

use of org.apache.flink.configuration.UnmodifiableConfiguration in project flink by apache.

the class BoltWrapperTest method testOpen.

@SuppressWarnings("unchecked")
@Test
public void testOpen() throws Exception {
    // utility mocks
    final StormConfig stormConfig = new StormConfig();
    final Configuration flinkConfig = new Configuration();
    final ExecutionConfig taskConfig = mock(ExecutionConfig.class);
    when(taskConfig.getGlobalJobParameters()).thenReturn(null).thenReturn(stormConfig).thenReturn(flinkConfig);
    final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class);
    when(taskContext.getExecutionConfig()).thenReturn(taskConfig);
    when(taskContext.getTaskName()).thenReturn("name");
    when(taskContext.getMetricGroup()).thenReturn(new UnregisteredMetricsGroup());
    final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer();
    declarer.declare(new Fields("dummy"));
    PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
    // (1) open with no configuration
    {
        ExecutionConfig execConfig = mock(ExecutionConfig.class);
        when(execConfig.getGlobalJobParameters()).thenReturn(null);
        final IRichBolt bolt = mock(IRichBolt.class);
        BoltWrapper<Object, Object> wrapper = new BoltWrapper<Object, Object>(bolt);
        wrapper.setup(createMockStreamTask(execConfig), new StreamConfig(new Configuration()), mock(Output.class));
        wrapper.open();
        verify(bolt).prepare(any(Map.class), any(TopologyContext.class), any(OutputCollector.class));
    }
    // (2) open with a storm specific configuration
    {
        ExecutionConfig execConfig = mock(ExecutionConfig.class);
        when(execConfig.getGlobalJobParameters()).thenReturn(stormConfig);
        final IRichBolt bolt = mock(IRichBolt.class);
        BoltWrapper<Object, Object> wrapper = new BoltWrapper<Object, Object>(bolt);
        wrapper.setup(createMockStreamTask(execConfig), new StreamConfig(new Configuration()), mock(Output.class));
        wrapper.open();
        verify(bolt).prepare(same(stormConfig), any(TopologyContext.class), any(OutputCollector.class));
    }
    // (3) open with a flink config
    {
        final Configuration cfg = new Configuration();
        cfg.setString("foo", "bar");
        cfg.setInteger("the end (the int)", Integer.MAX_VALUE);
        ExecutionConfig execConfig = mock(ExecutionConfig.class);
        when(execConfig.getGlobalJobParameters()).thenReturn(new UnmodifiableConfiguration(cfg));
        TestDummyBolt testBolt = new TestDummyBolt();
        BoltWrapper<Object, Object> wrapper = new BoltWrapper<Object, Object>(testBolt);
        wrapper.setup(createMockStreamTask(execConfig), new StreamConfig(new Configuration()), mock(Output.class));
        wrapper.open();
        for (Entry<String, String> entry : cfg.toMap().entrySet()) {
            Assert.assertEquals(entry.getValue(), testBolt.config.get(entry.getKey()));
        }
    }
}

Example 2 with UnmodifiableConfiguration

use of org.apache.flink.configuration.UnmodifiableConfiguration in project flink by apache.

the class MiniCluster method start.

/**
	 * Starts the mini cluster, based on the configured properties.
	 * 
	 * @throws Exception This method passes on any exception that occurs during the startup of
	 *                   the mini cluster.
	 */
public void start() throws Exception {
    synchronized (lock) {
        checkState(!running, "FlinkMiniCluster is already running");
        LOG.info("Starting Flink Mini Cluster");
        LOG.debug("Using configuration {}", config);
        final Configuration configuration = new UnmodifiableConfiguration(config.generateConfiguration());
        final Time rpcTimeout = config.getRpcTimeout();
        final int numJobManagers = config.getNumJobManagers();
        final int numTaskManagers = config.getNumTaskManagers();
        final int numResourceManagers = config.getNumResourceManagers();
        final boolean singleRpc = config.getUseSingleRpcSystem();
        try {
            LOG.info("Starting Metrics Registry");
            metricRegistry = createMetricRegistry(configuration);
            RpcService[] jobManagerRpcServices = new RpcService[numJobManagers];
            RpcService[] taskManagerRpcServices = new RpcService[numTaskManagers];
            RpcService[] resourceManagerRpcServices = new RpcService[numResourceManagers];
            // bring up all the RPC services
            LOG.info("Starting RPC Service(s)");
            // we always need the 'commonRpcService' for auxiliary calls
            commonRpcService = createRpcService(configuration, rpcTimeout, false, null);
            if (singleRpc) {
                // set that same RPC service for all JobManagers and TaskManagers
                for (int i = 0; i < numJobManagers; i++) {
                    jobManagerRpcServices[i] = commonRpcService;
                }
                for (int i = 0; i < numTaskManagers; i++) {
                    taskManagerRpcServices[i] = commonRpcService;
                }
                for (int i = 0; i < numResourceManagers; i++) {
                    resourceManagerRpcServices[i] = commonRpcService;
                }
                this.resourceManagerRpcServices = null;
                this.jobManagerRpcServices = null;
                this.taskManagerRpcServices = null;
            } else {
                // start a new service per component, possibly with custom bind addresses
                final String jobManagerBindAddress = config.getJobManagerBindAddress();
                final String taskManagerBindAddress = config.getTaskManagerBindAddress();
                final String resourceManagerBindAddress = config.getResourceManagerBindAddress();
                for (int i = 0; i < numJobManagers; i++) {
                    jobManagerRpcServices[i] = createRpcService(configuration, rpcTimeout, true, jobManagerBindAddress);
                }
                for (int i = 0; i < numTaskManagers; i++) {
                    taskManagerRpcServices[i] = createRpcService(configuration, rpcTimeout, true, taskManagerBindAddress);
                }
                for (int i = 0; i < numResourceManagers; i++) {
                    resourceManagerRpcServices[i] = createRpcService(configuration, rpcTimeout, true, resourceManagerBindAddress);
                }
                this.jobManagerRpcServices = jobManagerRpcServices;
                this.taskManagerRpcServices = taskManagerRpcServices;
                this.resourceManagerRpcServices = resourceManagerRpcServices;
            }
            // create the high-availability services
            LOG.info("Starting high-availability services");
            haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(configuration);
            heartbeatServices = HeartbeatServices.fromConfiguration(configuration);
            // bring up the ResourceManager(s)
            LOG.info("Starting {} ResourceManger(s)", numResourceManagers);
            resourceManagerRunners = startResourceManagers(configuration, haServices, metricRegistry, numResourceManagers, resourceManagerRpcServices);
            // bring up the TaskManager(s) for the mini cluster
            LOG.info("Starting {} TaskManger(s)", numTaskManagers);
            taskManagerRunners = startTaskManagers(configuration, haServices, metricRegistry, numTaskManagers, taskManagerRpcServices);
            // bring up the dispatcher that launches JobManagers when jobs submitted
            LOG.info("Starting job dispatcher(s) for {} JobManger(s)", numJobManagers);
            jobDispatcher = new MiniClusterJobDispatcher(configuration, haServices, heartbeatServices, metricRegistry, numJobManagers, jobManagerRpcServices);
        } catch (Exception e) {
            // cleanup everything
            try {
                shutdownInternally();
            } catch (Exception ee) {
                e.addSuppressed(ee);
            }
            throw e;
        }
        // now officially mark this as running
        running = true;
        LOG.info("Flink Mini Cluster started successfully");
    }
}