Examples with PipelineResult org.apache.beam.sdk.PipelineResult used on opensource projects

Example 1 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class TrafficMaxLaneFlow method main.

/**
   * Sets up and starts streaming pipeline.
   *
   * @throws IOException if there is a problem setting up resources
   */
public static void main(String[] args) throws IOException {
    TrafficMaxLaneFlowOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(TrafficMaxLaneFlowOptions.class);
    options.setBigQuerySchema(FormatMaxesFn.getSchema());
    // Using ExampleUtils to set up required resources.
    ExampleUtils exampleUtils = new ExampleUtils(options);
    exampleUtils.setup();
    Pipeline pipeline = Pipeline.create(options);
    TableReference tableRef = new TableReference();
    tableRef.setProjectId(options.getProject());
    tableRef.setDatasetId(options.getBigQueryDataset());
    tableRef.setTableId(options.getBigQueryTable());
    pipeline.apply("ReadLines", new ReadFileAndExtractTimestamps(options.getInputFile())).apply(ParDo.of(new ExtractFlowInfoFn())).apply(Window.<KV<String, LaneInfo>>into(SlidingWindows.of(Duration.standardMinutes(options.getWindowDuration())).every(Duration.standardMinutes(options.getWindowSlideEvery())))).apply(new MaxLaneFlow()).apply(BigQueryIO.writeTableRows().to(tableRef).withSchema(FormatMaxesFn.getSchema()));
    // Run the pipeline.
    PipelineResult result = pipeline.run();
    // ExampleUtils will try to cancel the pipeline and the injector before the program exists.
    exampleUtils.waitToFinish(result);
}

Example 2 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class TrafficRoutes method main.

/**
   * Sets up and starts streaming pipeline.
   *
   * @throws IOException if there is a problem setting up resources
   */
public static void main(String[] args) throws IOException {
    TrafficRoutesOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(TrafficRoutesOptions.class);
    options.setBigQuerySchema(FormatStatsFn.getSchema());
    // Using ExampleUtils to set up required resources.
    ExampleUtils exampleUtils = new ExampleUtils(options);
    exampleUtils.setup();
    Pipeline pipeline = Pipeline.create(options);
    TableReference tableRef = new TableReference();
    tableRef.setProjectId(options.getProject());
    tableRef.setDatasetId(options.getBigQueryDataset());
    tableRef.setTableId(options.getBigQueryTable());
    pipeline.apply("ReadLines", new ReadFileAndExtractTimestamps(options.getInputFile())).apply(ParDo.of(new ExtractStationSpeedFn())).apply(Window.<KV<String, StationSpeed>>into(SlidingWindows.of(Duration.standardMinutes(options.getWindowDuration())).every(Duration.standardMinutes(options.getWindowSlideEvery())))).apply(new TrackSpeed()).apply(BigQueryIO.writeTableRows().to(tableRef).withSchema(FormatStatsFn.getSchema()));
    // Run the pipeline.
    PipelineResult result = pipeline.run();
    // ExampleUtils will try to cancel the pipeline and the injector before the program exists.
    exampleUtils.waitToFinish(result);
}

Example 3 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsTest method testAllCommittedMetrics.

@Category({ ValidatesRunner.class, UsesCommittedMetrics.class, UsesCounterMetrics.class, UsesDistributionMetrics.class, UsesGaugeMetrics.class })
@Test
public void testAllCommittedMetrics() {
    PipelineResult result = runPipelineWithMetrics();
    MetricQueryResults metrics = queryTestMetrics(result);
    assertAllMetrics(metrics, true);
}

Example 4 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsTest method testAttemptedDistributionMetrics.

@Category({ ValidatesRunner.class, UsesAttemptedMetrics.class, UsesDistributionMetrics.class })
@Test
public void testAttemptedDistributionMetrics() {
    PipelineResult result = runPipelineWithMetrics();
    MetricQueryResults metrics = queryTestMetrics(result);
    assertDistributionMetrics(metrics, false);
}

Example 5 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsTest method testUnboundedSourceMetrics.

@Test
@Category({ ValidatesRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class })
public void testUnboundedSourceMetrics() {
    long numElements = 1000;
    // Use withMaxReadTime to force unbounded mode.
    pipeline.apply(GenerateSequence.from(0).to(numElements).withMaxReadTime(Duration.standardDays(1)));
    PipelineResult pipelineResult = pipeline.run();
    MetricQueryResults metrics = pipelineResult.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(ELEMENTS_READ.namespace(), ELEMENTS_READ.name())).build());
    assertThat(metrics.counters(), hasItem(attemptedMetricsResult(ELEMENTS_READ.namespace(), ELEMENTS_READ.name(), "Read(UnboundedCountingSource)", 1000L)));
}