Examples with Configuration org.apache.flink.configuration.Configuration used on opensource projects

Example 81 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class ZooKeeperUtilTest method testZooKeeperEnsembleConnectStringConfiguration.

@Test
public void testZooKeeperEnsembleConnectStringConfiguration() throws Exception {
    // ZooKeeper does not like whitespace in the quorum connect String.
    String actual, expected;
    Configuration conf = new Configuration();
    {
        expected = "localhost:2891";
        setQuorum(conf, expected);
        actual = ZooKeeperUtils.getZooKeeperEnsemble(conf);
        assertEquals(expected, actual);
        // with leading and trailing whitespace
        setQuorum(conf, " localhost:2891 ");
        actual = ZooKeeperUtils.getZooKeeperEnsemble(conf);
        assertEquals(expected, actual);
        // whitespace after port
        setQuorum(conf, "localhost :2891");
        actual = ZooKeeperUtils.getZooKeeperEnsemble(conf);
        assertEquals(expected, actual);
    }
    {
        expected = "localhost:2891,localhost:2891";
        setQuorum(conf, "localhost:2891,localhost:2891");
        actual = ZooKeeperUtils.getZooKeeperEnsemble(conf);
        assertEquals(expected, actual);
        setQuorum(conf, "localhost:2891, localhost:2891");
        actual = ZooKeeperUtils.getZooKeeperEnsemble(conf);
        assertEquals(expected, actual);
        setQuorum(conf, "localhost :2891, localhost:2891");
        actual = ZooKeeperUtils.getZooKeeperEnsemble(conf);
        assertEquals(expected, actual);
        setQuorum(conf, " localhost:2891, localhost:2891 ");
        actual = ZooKeeperUtils.getZooKeeperEnsemble(conf);
        assertEquals(expected, actual);
    }
}

Example 82 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class RemoteStreamEnvironment method executeRemotely.

/**
	 * Executes the remote job.
	 * 
	 * @param streamGraph
	 *            Stream Graph to execute
	 * @param jarFiles
	 * 			  List of jar file URLs to ship to the cluster
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException {
    if (LOG.isInfoEnabled()) {
        LOG.info("Running remotely at {}:{}", host, port);
    }
    ClassLoader usercodeClassLoader = JobWithJars.buildUserCodeClassLoader(jarFiles, globalClasspaths, getClass().getClassLoader());
    Configuration configuration = new Configuration();
    configuration.addAll(this.clientConfiguration);
    configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, host);
    configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, port);
    ClusterClient client;
    try {
        client = new StandaloneClusterClient(configuration);
        client.setPrintStatusDuringExecution(getConfig().isSysoutLoggingEnabled());
    } catch (Exception e) {
        throw new ProgramInvocationException("Cannot establish connection to JobManager: " + e.getMessage(), e);
    }
    try {
        return client.run(streamGraph, jarFiles, globalClasspaths, usercodeClassLoader).getJobExecutionResult();
    } catch (ProgramInvocationException e) {
        throw e;
    } catch (Exception e) {
        String term = e.getMessage() == null ? "." : (": " + e.getMessage());
        throw new ProgramInvocationException("The program execution failed" + term, e);
    } finally {
        client.shutdown();
    }
}

Example 83 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class ContinuousFileReaderOperator method open.

@Override
public void open() throws Exception {
    super.open();
    checkState(this.reader == null, "The reader is already initialized.");
    checkState(this.serializer != null, "The serializer has not been set. " + "Probably the setOutputType() was not called. Please report it.");
    this.format.setRuntimeContext(getRuntimeContext());
    this.format.configure(new Configuration());
    this.checkpointLock = getContainingTask().getCheckpointLock();
    // set the reader context based on the time characteristic
    final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic();
    final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval();
    this.readerContext = StreamSourceContexts.getSourceContext(timeCharacteristic, getProcessingTimeService(), checkpointLock, getContainingTask().getStreamStatusMaintainer(), output, watermarkInterval, -1);
    // and initialize the split reading thread
    this.reader = new SplitReader<>(format, serializer, readerContext, checkpointLock, restoredReaderState);
    this.restoredReaderState = null;
    this.reader.start();
}

Example 84 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class AbstractStreamOperator method setup.

// ------------------------------------------------------------------------
//  Life Cycle
// ------------------------------------------------------------------------
@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
    this.container = containingTask;
    this.config = config;
    this.metrics = container.getEnvironment().getMetricGroup().addOperator(config.getOperatorName());
    this.output = new CountingOutput(output, ((OperatorMetricGroup) this.metrics).getIOMetricGroup().getNumRecordsOutCounter());
    if (config.isChainStart()) {
        ((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseInputMetricsForTask();
    }
    if (config.isChainEnd()) {
        ((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseOutputMetricsForTask();
    }
    Configuration taskManagerConfig = container.getEnvironment().getTaskManagerInfo().getConfiguration();
    int historySize = taskManagerConfig.getInteger(ConfigConstants.METRICS_LATENCY_HISTORY_SIZE, ConfigConstants.DEFAULT_METRICS_LATENCY_HISTORY_SIZE);
    if (historySize <= 0) {
        LOG.warn("{} has been set to a value equal or below 0: {}. Using default.", ConfigConstants.METRICS_LATENCY_HISTORY_SIZE, historySize);
        historySize = ConfigConstants.DEFAULT_METRICS_LATENCY_HISTORY_SIZE;
    }
    latencyGauge = this.metrics.gauge("latency", new LatencyGauge(historySize));
    this.runtimeContext = new StreamingRuntimeContext(this, container.getEnvironment(), container.getAccumulatorMap());
    stateKeySelector1 = config.getStatePartitioner(0, getUserCodeClassloader());
    stateKeySelector2 = config.getStatePartitioner(1, getUserCodeClassloader());
}

Example 85 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class AbstractUdfStreamOperatorLifecycleTest method testLifeCycleCancel.

@Test
public void testLifeCycleCancel() throws Exception {
    ACTUAL_ORDER_TRACKING.clear();
    Configuration taskManagerConfig = new Configuration();
    StreamConfig cfg = new StreamConfig(new Configuration());
    MockSourceFunction srcFun = new MockSourceFunction();
    cfg.setStreamOperator(new LifecycleTrackingStreamSource(srcFun, false));
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);
    task.startTaskThread();
    LifecycleTrackingStreamSource.runStarted.await();
    // this should cancel the task even though it is blocked on runFinished
    task.cancelExecution();
    // wait for clean termination
    task.getExecutingThread().join();
    assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
}