Examples with PipelineResult org.apache.beam.sdk.PipelineResult used on opensource projects

Example 61 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class BigQueryIOIT method testWrite.

private void testWrite(BigQueryIO.Write<byte[]> writeIO, String metricName) {
    Pipeline pipeline = Pipeline.create(options);
    BigQueryIO.Write.Method method = BigQueryIO.Write.Method.valueOf(options.getWriteMethod());
    pipeline.apply("Read from source", Read.from(new SyntheticBoundedSource(sourceOptions))).apply("Gather time", ParDo.of(new TimeMonitor<>(NAMESPACE, metricName))).apply("Map records", ParDo.of(new MapKVToV())).apply("Write to BQ", writeIO.to(tableQualifier).withCustomGcsTempLocation(ValueProvider.StaticValueProvider.of(tempRoot)).withMethod(method).withSchema(new TableSchema().setFields(Collections.singletonList(new TableFieldSchema().setName("data").setType("BYTES")))));
    PipelineResult pipelineResult = pipeline.run();
    pipelineResult.waitUntilFinish();
    extractAndPublishTime(pipelineResult, metricName);
}

Example 62 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class BigQueryIOIT method testRead.

private void testRead() {
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply("Read from BQ", BigQueryIO.readTableRows().from(tableQualifier)).apply("Gather time", ParDo.of(new TimeMonitor<>(NAMESPACE, READ_TIME_METRIC_NAME)));
    PipelineResult result = pipeline.run();
    result.waitUntilFinish();
    extractAndPublishTime(result, READ_TIME_METRIC_NAME);
}

Example 63 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsReaderTest method doesntThrowIllegalStateExceptionWhenThereIsNoMetricFound.

@Test
public void doesntThrowIllegalStateExceptionWhenThereIsNoMetricFound() {
    PipelineResult result = testPipeline.run();
    MetricsReader reader = new MetricsReader(result, NAMESPACE);
    reader.getCounterMetric("nonexistent");
}

Example 64 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsReaderTest method testCounterMetricReceivedFromPipelineResult.

@Test
public void testCounterMetricReceivedFromPipelineResult() {
    List<Integer> sampleInputData = Arrays.asList(1, 1, 1, 1, 1);
    createTestPipeline(sampleInputData, new MonitorWithCounter());
    PipelineResult result = testPipeline.run();
    MetricsReader reader = new MetricsReader(result, NAMESPACE);
    assertEquals(5, reader.getCounterMetric("counter"));
}

Example 65 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class TpcdsRun method call.

@Override
public TpcdsRunResult call() {
    TpcdsRunResult tpcdsRunResult;
    try {
        PipelineResult pipelineResult = pipeline.run();
        long startTimeStamp = System.currentTimeMillis();
        State state = pipelineResult.waitUntilFinish();
        long endTimeStamp = System.currentTimeMillis();
        // Make sure to set the job status to be successful only when pipelineResult's final state is
        // DONE.
        boolean isSuccessful = state == State.DONE;
        tpcdsRunResult = new TpcdsRunResult(isSuccessful, startTimeStamp, endTimeStamp, pipeline.getOptions(), pipelineResult);
    } catch (Exception e) {
        // If the pipeline execution failed, return a result with failed status but don't interrupt
        // other threads.
        e.printStackTrace();
        tpcdsRunResult = new TpcdsRunResult(false, 0, 0, pipeline.getOptions(), null);
    }
    return tpcdsRunResult;
}