Examples with Configuration org.apache.flink.configuration.Configuration used on opensource projects

Example 36 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class GroupReduceNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
    origProps.addForwardedField(0, 1);
    origProps.addForwardedField(2, 2);
    origProps.addForwardedField(3, 4);
    origProps.addForwardedField(6, 0);
    origProps.addReadFields(new FieldSet(0, 2, 4, 7));
    GroupReduceOperatorBase<?, ?, ?> op = mock(GroupReduceOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getParameters()).thenReturn(new Configuration());
    GroupReduceNode node = new GroupReduceNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
}

Example 37 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class BootstrapTools method parseDynamicProperties.

/**
	 * Parse the dynamic properties (passed on the command line).
	 */
public static Configuration parseDynamicProperties(CommandLine cmd) {
    final Configuration config = new Configuration();
    String[] values = cmd.getOptionValues(DYNAMIC_PROPERTIES_OPT);
    if (values != null) {
        for (String value : values) {
            String[] pair = value.split("=", 2);
            if (pair.length == 1) {
                config.setString(pair[0], Boolean.TRUE.toString());
            } else if (pair.length == 2) {
                config.setString(pair[0], pair[1]);
            }
        }
    }
    return config;
}

Example 38 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class NettyClientServerSslTest method testInvalidSslConfiguration.

/**
	 * Verify failure on invalid ssl configuration
	 *
	 */
@Test
public void testInvalidSslConfiguration() throws Exception {
    NettyProtocol protocol = new NettyProtocol() {

        @Override
        public ChannelHandler[] getServerChannelHandlers() {
            return new ChannelHandler[0];
        }

        @Override
        public ChannelHandler[] getClientChannelHandlers() {
            return new ChannelHandler[0];
        }
    };
    Configuration config = createSslConfig();
    // Modify the keystore password to an incorrect one
    config.setString(ConfigConstants.SECURITY_SSL_KEYSTORE_PASSWORD, "invalidpassword");
    NettyConfig nettyConfig = new NettyConfig(InetAddress.getLoopbackAddress(), NetUtils.getAvailablePort(), NettyTestUtil.DEFAULT_SEGMENT_SIZE, 1, config);
    NettyTestUtil.NettyServerAndClient serverAndClient = null;
    try {
        serverAndClient = NettyTestUtil.initServerAndClient(protocol, nettyConfig);
        Assert.fail("Created server and client from invalid configuration");
    } catch (Exception e) {
    // Exception should be thrown as expected
    }
    NettyTestUtil.shutdown(serverAndClient);
}

Example 39 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class NettyConnectionManagerTest method testManualConfiguration.

/**
	 * Tests that the number of arenas and threads can be configured manually.
	 */
@Test
public void testManualConfiguration() throws Exception {
    // Expected numbers
    int numberOfArenas = 1;
    int numberOfClientThreads = 3;
    int numberOfServerThreads = 4;
    // Expected number of threads
    Configuration flinkConfig = new Configuration();
    flinkConfig.setInteger(NettyConfig.NUM_ARENAS, numberOfArenas);
    flinkConfig.setInteger(NettyConfig.NUM_THREADS_CLIENT, 3);
    flinkConfig.setInteger(NettyConfig.NUM_THREADS_SERVER, 4);
    NettyConfig config = new NettyConfig(InetAddress.getLocalHost(), NetUtils.getAvailablePort(), 1024, 1337, flinkConfig);
    NettyConnectionManager connectionManager = new NettyConnectionManager(config);
    connectionManager.start(mock(ResultPartitionProvider.class), mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class));
    assertEquals(numberOfArenas, connectionManager.getBufferPool().getNumberOfArenas());
    {
        // Client event loop group
        Bootstrap boostrap = connectionManager.getClient().getBootstrap();
        EventLoopGroup group = boostrap.group();
        Field f = group.getClass().getSuperclass().getSuperclass().getDeclaredField("children");
        f.setAccessible(true);
        Object[] eventExecutors = (Object[]) f.get(group);
        assertEquals(numberOfClientThreads, eventExecutors.length);
    }
    {
        // Server event loop group
        ServerBootstrap bootstrap = connectionManager.getServer().getBootstrap();
        EventLoopGroup group = bootstrap.group();
        Field f = group.getClass().getSuperclass().getSuperclass().getDeclaredField("children");
        f.setAccessible(true);
        Object[] eventExecutors = (Object[]) f.get(group);
        assertEquals(numberOfServerThreads, eventExecutors.length);
    }
    {
        // Server child event loop group
        ServerBootstrap bootstrap = connectionManager.getServer().getBootstrap();
        EventLoopGroup group = bootstrap.childGroup();
        Field f = group.getClass().getSuperclass().getSuperclass().getDeclaredField("children");
        f.setAccessible(true);
        Object[] eventExecutors = (Object[]) f.get(group);
        assertEquals(numberOfServerThreads, eventExecutors.length);
    }
}

Example 40 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class NettyConnectionManagerTest method testMatchingNumberOfArenasAndThreadsAsDefault.

/**
	 * Tests that the number of arenas and number of threads of the client and
	 * server are set to the same number, that is the number of configured
	 * task slots.
	 */
@Test
public void testMatchingNumberOfArenasAndThreadsAsDefault() throws Exception {
    // Expected number of arenas and threads
    int numberOfSlots = 2;
    NettyConfig config = new NettyConfig(InetAddress.getLocalHost(), NetUtils.getAvailablePort(), 1024, numberOfSlots, new Configuration());
    NettyConnectionManager connectionManager = new NettyConnectionManager(config);
    connectionManager.start(mock(ResultPartitionProvider.class), mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class));
    assertEquals(numberOfSlots, connectionManager.getBufferPool().getNumberOfArenas());
    {
        // Client event loop group
        Bootstrap boostrap = connectionManager.getClient().getBootstrap();
        EventLoopGroup group = boostrap.group();
        Field f = group.getClass().getSuperclass().getSuperclass().getDeclaredField("children");
        f.setAccessible(true);
        Object[] eventExecutors = (Object[]) f.get(group);
        assertEquals(numberOfSlots, eventExecutors.length);
    }
    {
        // Server event loop group
        ServerBootstrap bootstrap = connectionManager.getServer().getBootstrap();
        EventLoopGroup group = bootstrap.group();
        Field f = group.getClass().getSuperclass().getSuperclass().getDeclaredField("children");
        f.setAccessible(true);
        Object[] eventExecutors = (Object[]) f.get(group);
        assertEquals(numberOfSlots, eventExecutors.length);
    }
    {
        // Server child event loop group
        ServerBootstrap bootstrap = connectionManager.getServer().getBootstrap();
        EventLoopGroup group = bootstrap.childGroup();
        Field f = group.getClass().getSuperclass().getSuperclass().getDeclaredField("children");
        f.setAccessible(true);
        Object[] eventExecutors = (Object[]) f.get(group);
        assertEquals(numberOfSlots, eventExecutors.length);
    }
}