Examples with SyntheticStep org.apache.beam.sdk.io.synthetic.SyntheticStep used on opensource projects

Example 1 with SyntheticStep

use of org.apache.beam.sdk.io.synthetic.SyntheticStep in project beam by apache.

the class CoGroupByKeyLoadTest method loadTest.

@Override
void loadTest() throws IOException {
    SyntheticSourceOptions coSourceOptions = fromJsonString(options.getCoSourceOptions(), SyntheticSourceOptions.class);
    Optional<SyntheticStep> syntheticStep = createStep(options.getStepOptions());
    PCollection<KV<byte[], byte[]>> input = pipeline.apply("Read input", readFromSource(sourceOptions));
    input = input.apply("Collect start time metrics (input)", ParDo.of(runtimeMonitor));
    input = applyWindowing(input);
    input = applyStepIfPresent(input, "Synthetic step for input", syntheticStep);
    PCollection<KV<byte[], byte[]>> coInput = pipeline.apply("Read co-input", readFromSource(coSourceOptions));
    coInput = coInput.apply("Collect start time metrics (co-input)", ParDo.of(runtimeMonitor));
    coInput = applyWindowing(coInput, options.getCoInputWindowDurationSec());
    coInput = applyStepIfPresent(coInput, "Synthetic step for co-input", syntheticStep);
    KeyedPCollectionTuple.of(INPUT_TAG, input).and(CO_INPUT_TAG, coInput).apply("CoGroupByKey", CoGroupByKey.create()).apply("Ungroup and reiterate", ParDo.of(new UngroupAndReiterate(options.getIterations()))).apply("Collect total bytes", ParDo.of(new ByteMonitor(METRICS_NAMESPACE, "totalBytes.count"))).apply("Collect end time metrics", ParDo.of(runtimeMonitor));
}