Examples with DataflowWorkerHarnessOptions org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions used on opensource projects

Example 6 with DataflowWorkerHarnessOptions

use of org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions in project beam by apache.

the class StreamingDataflowWorker method main.

public static void main(String[] args) throws Exception {
    JvmInitializers.runOnStartup();
    DataflowWorkerHarnessHelper.initializeLogging(StreamingDataflowWorker.class);
    DataflowWorkerHarnessOptions options = DataflowWorkerHarnessHelper.initializeGlobalStateAndPipelineOptions(StreamingDataflowWorker.class);
    DataflowWorkerHarnessHelper.configureLogging(options);
    checkArgument(options.isStreaming(), "%s instantiated with options indicating batch use", StreamingDataflowWorker.class.getName());
    checkArgument(!DataflowRunner.hasExperiment(options, "beam_fn_api"), "%s cannot be main() class with beam_fn_api enabled", StreamingDataflowWorker.class.getSimpleName());
    // Create a non fnapi registry
    SdkHarnessRegistry sdkHarnessRegistry = SdkHarnessRegistries.emptySdkHarnessRegistry();
    StreamingDataflowWorker worker = StreamingDataflowWorker.fromDataflowWorkerHarnessOptions(options, sdkHarnessRegistry);
    // Use the MetricsLogger container which is used by BigQueryIO to periodically log process-wide
    // metrics.
    MetricsEnvironment.setProcessWideContainer(new MetricsLogger(null));
    JvmInitializers.runBeforeProcessing(options);
    worker.startStatusPages();
    worker.start();
}

Example 7 with DataflowWorkerHarnessOptions

use of org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions in project beam by apache.

the class DataflowWorkerHarnessHelperTest method testLoggingConfiguration.

@Test
public void testLoggingConfiguration() throws Exception {
    DataflowWorkerHarnessOptions pipelineOptions = PipelineOptionsFactory.as(DataflowWorkerHarnessOptions.class);
    pipelineOptions.setJobId(JOB_ID);
    pipelineOptions.setWorkerId(WORKER_ID);
    String serializedOptions = new ObjectMapper().writeValueAsString(pipelineOptions);
    File file = tmpFolder.newFile();
    Files.write(Paths.get(file.getPath()), serializedOptions.getBytes(StandardCharsets.UTF_8));
    System.setProperty("sdk_pipeline_options_file", file.getPath());
    DataflowWorkerHarnessOptions generatedOptions = DataflowWorkerHarnessHelper.initializeGlobalStateAndPipelineOptions(DataflowBatchWorkerHarnessTest.class);
    // Assert that the returned options are correct.
    assertThat(generatedOptions.getJobId(), equalTo(JOB_ID));
    assertThat(generatedOptions.getWorkerId(), equalTo(WORKER_ID));
    // Assert that the global logging configuration was properly initialized.
    assertThat(DataflowWorkerLoggingMDC.getJobId(), equalTo(JOB_ID));
    assertThat(DataflowWorkerLoggingMDC.getWorkerId(), equalTo(WORKER_ID));
}

Example 8 with DataflowWorkerHarnessOptions

use of org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions in project beam by apache.

the class WorkerPipelineOptionsFactoryTest method testCreationWithPipelineOptionsFile.

@Test
public void testCreationWithPipelineOptionsFile() throws Exception {
    File file = tmpFolder.newFile();
    String jsonOptions = "{\"options\":{\"numWorkers\":1000}}";
    Files.write(Paths.get(file.getPath()), jsonOptions.getBytes(StandardCharsets.UTF_8));
    System.getProperties().putAll(ImmutableMap.<String, String>builder().put("worker_id", "test_worker_id_2").put("job_id", "test_job_id_2").put("sdk_pipeline_options_file", file.getPath()).build());
    // testing deprecated functionality
    @SuppressWarnings("deprecation") DataflowWorkerHarnessOptions options = WorkerPipelineOptionsFactory.createFromSystemProperties();
    assertEquals("test_worker_id_2", options.getWorkerId());
    assertEquals("test_job_id_2", options.getJobId());
    assertEquals(1000, options.getNumWorkers());
}