use of org.apache.beam.sdk.PipelineResult in project beam by apache.
the class BigQueryIOIT method testWrite.
private void testWrite(BigQueryIO.Write<byte[]> writeIO, String metricName) {
Pipeline pipeline = Pipeline.create(options);
BigQueryIO.Write.Method method = BigQueryIO.Write.Method.valueOf(options.getWriteMethod());
pipeline.apply("Read from source", Read.from(new SyntheticBoundedSource(sourceOptions))).apply("Gather time", ParDo.of(new TimeMonitor<>(NAMESPACE, metricName))).apply("Map records", ParDo.of(new MapKVToV())).apply("Write to BQ", writeIO.to(tableQualifier).withCustomGcsTempLocation(ValueProvider.StaticValueProvider.of(tempRoot)).withMethod(method).withSchema(new TableSchema().setFields(Collections.singletonList(new TableFieldSchema().setName("data").setType("BYTES")))));
PipelineResult pipelineResult = pipeline.run();
pipelineResult.waitUntilFinish();
extractAndPublishTime(pipelineResult, metricName);
}
use of org.apache.beam.sdk.PipelineResult in project beam by apache.
the class BigQueryIOIT method testRead.
private void testRead() {
Pipeline pipeline = Pipeline.create(options);
pipeline.apply("Read from BQ", BigQueryIO.readTableRows().from(tableQualifier)).apply("Gather time", ParDo.of(new TimeMonitor<>(NAMESPACE, READ_TIME_METRIC_NAME)));
PipelineResult result = pipeline.run();
result.waitUntilFinish();
extractAndPublishTime(result, READ_TIME_METRIC_NAME);
}
use of org.apache.beam.sdk.PipelineResult in project beam by apache.
the class MetricsReaderTest method doesntThrowIllegalStateExceptionWhenThereIsNoMetricFound.
@Test
public void doesntThrowIllegalStateExceptionWhenThereIsNoMetricFound() {
PipelineResult result = testPipeline.run();
MetricsReader reader = new MetricsReader(result, NAMESPACE);
reader.getCounterMetric("nonexistent");
}
use of org.apache.beam.sdk.PipelineResult in project beam by apache.
the class MetricsReaderTest method testCounterMetricReceivedFromPipelineResult.
@Test
public void testCounterMetricReceivedFromPipelineResult() {
List<Integer> sampleInputData = Arrays.asList(1, 1, 1, 1, 1);
createTestPipeline(sampleInputData, new MonitorWithCounter());
PipelineResult result = testPipeline.run();
MetricsReader reader = new MetricsReader(result, NAMESPACE);
assertEquals(5, reader.getCounterMetric("counter"));
}
use of org.apache.beam.sdk.PipelineResult in project beam by apache.
the class TpcdsRun method call.
@Override
public TpcdsRunResult call() {
TpcdsRunResult tpcdsRunResult;
try {
PipelineResult pipelineResult = pipeline.run();
long startTimeStamp = System.currentTimeMillis();
State state = pipelineResult.waitUntilFinish();
long endTimeStamp = System.currentTimeMillis();
// Make sure to set the job status to be successful only when pipelineResult's final state is
// DONE.
boolean isSuccessful = state == State.DONE;
tpcdsRunResult = new TpcdsRunResult(isSuccessful, startTimeStamp, endTimeStamp, pipeline.getOptions(), pipelineResult);
} catch (Exception e) {
// If the pipeline execution failed, return a result with failed status but don't interrupt
// other threads.
e.printStackTrace();
tpcdsRunResult = new TpcdsRunResult(false, 0, 0, pipeline.getOptions(), null);
}
return tpcdsRunResult;
}
Aggregations