Examples with WindowingStrategy org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy used on opensource projects

Example 1 with WindowingStrategy

use of org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy in project beam by apache.

the class WindowingStrategyTranslation method fromProto.

/**
 * Converts from {@link RunnerApi.WindowingStrategy} to the SDK's {@link WindowingStrategy} using
 * the provided components to dereferences identifiers found in the proto.
 */
public static WindowingStrategy<?, ?> fromProto(RunnerApi.WindowingStrategy proto, RehydratedComponents components) throws InvalidProtocolBufferException {
    FunctionSpec windowFnSpec = proto.getWindowFn();
    WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec);
    TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
    AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
    Trigger trigger = TriggerTranslation.fromProto(proto.getTrigger());
    ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
    Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
    OnTimeBehavior onTimeBehavior = fromProto(proto.getOnTimeBehavior());
    String environmentId = proto.getEnvironmentId();
    return WindowingStrategy.of(windowFn).withAllowedLateness(allowedLateness).withMode(accumulationMode).withTrigger(trigger).withTimestampCombiner(timestampCombiner).withClosingBehavior(closingBehavior).withOnTimeBehavior(onTimeBehavior).withEnvironmentId(environmentId);
}

Example 2 with WindowingStrategy

use of org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy in project beam by apache.

the class DataflowPipelineTranslator method serializeWindowingStrategy.

private static byte[] serializeWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy, PipelineOptions options) {
    try {
        SdkComponents sdkComponents = SdkComponents.create();
        String workerHarnessContainerImageURL = DataflowRunner.getContainerImageForJob(options.as(DataflowPipelineOptions.class));
        RunnerApi.Environment defaultEnvironmentForDataflow = Environments.createDockerEnvironment(workerHarnessContainerImageURL);
        sdkComponents.registerEnvironment(defaultEnvironmentForDataflow);
        return WindowingStrategyTranslation.toMessageProto(windowingStrategy, sdkComponents).toByteArray();
    } catch (Exception e) {
        throw new RuntimeException(String.format("Unable to format windowing strategy %s as bytes", windowingStrategy), e);
    }
}

Example 3 with WindowingStrategy

use of org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy in project beam by apache.

the class PipelineValidator method validateComponents.

private static void validateComponents(String context, Components components, Set<String> requirements) {
    {
        Map<String, String> uniqueNamesById = Maps.newHashMap();
        for (String transformId : components.getTransformsMap().keySet()) {
            PTransform transform = components.getTransformsOrThrow(transformId);
            String previousId = uniqueNamesById.put(transform.getUniqueName(), transformId);
            // A transform is allowed to not have unique_name set, but, obviously,
            // there can be only one such transform with an empty name.
            // It's allowed for the (only) root transform to have the empty unique_name.
            checkArgument(previousId == null, "%s: Transforms %s and %s both have unique_name \"%s\"", context, transformId, previousId, transform.getUniqueName());
            validateTransform(transformId, transform, components, requirements);
        }
    }
    {
        Map<String, String> uniqueNamesById = Maps.newHashMap();
        for (String pcollectionId : components.getPcollectionsMap().keySet()) {
            PCollection pc = components.getPcollectionsOrThrow(pcollectionId);
            checkArgument(!pc.getUniqueName().isEmpty(), "%s: PCollection %s does not have a unique_name set", context, pcollectionId);
            String previousId = uniqueNamesById.put(pc.getUniqueName(), pcollectionId);
            checkArgument(previousId == null, "%s: PCollections %s and %s both have unique_name \"%s\"", context, pcollectionId, previousId, pc.getUniqueName());
            checkArgument(components.containsCoders(pc.getCoderId()), "%s: PCollection %s uses unknown coder %s", context, pcollectionId, pc.getCoderId());
            checkArgument(components.containsWindowingStrategies(pc.getWindowingStrategyId()), "%s: PCollection %s uses unknown windowing strategy %s", context, pcollectionId, pc.getWindowingStrategyId());
        }
    }
    for (String strategyId : components.getWindowingStrategiesMap().keySet()) {
        WindowingStrategy strategy = components.getWindowingStrategiesOrThrow(strategyId);
        checkArgument(components.containsCoders(strategy.getWindowCoderId()), "%s: WindowingStrategy %s uses unknown coder %s", context, strategyId, strategy.getWindowCoderId());
    }
    for (String coderId : components.getCodersMap().keySet()) {
        for (String componentCoderId : components.getCodersOrThrow(coderId).getComponentCoderIdsList()) {
            checkArgument(components.containsCoders(componentCoderId), "%s: Coder %s uses unknown component coder %s", context, coderId, componentCoderId);
        }
    }
}

Example 4 with WindowingStrategy

use of org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy in project beam by apache.

the class PCollectionTranslationTest method testEncodeDecodeFields.

@Test
public void testEncodeDecodeFields() throws Exception {
    SdkComponents sdkComponents = SdkComponents.create();
    sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java"));
    RunnerApi.PCollection protoCollection = PCollectionTranslation.toProto(testCollection, sdkComponents);
    RehydratedComponents protoComponents = RehydratedComponents.forComponents(sdkComponents.toComponents());
    Coder<?> decodedCoder = protoComponents.getCoder(protoCollection.getCoderId());
    WindowingStrategy<?, ?> decodedStrategy = protoComponents.getWindowingStrategy(protoCollection.getWindowingStrategyId());
    IsBounded decodedIsBounded = PCollectionTranslation.isBounded(protoCollection);
    assertThat(decodedCoder, equalTo(testCollection.getCoder()));
    assertThat(decodedStrategy, equalTo(testCollection.getWindowingStrategy().withEnvironmentId(sdkComponents.getOnlyEnvironmentId()).fixDefaults()));
    assertThat(decodedIsBounded, equalTo(testCollection.isBounded()));
}

Example 5 with WindowingStrategy

use of org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy in project beam by apache.

the class WindowUtils method getWindowStrategy.

/**
 * Get {@link WindowingStrategy} of given collection id from {@link RunnerApi.Components}.
 */
public static WindowingStrategy<?, BoundedWindow> getWindowStrategy(String collectionId, RunnerApi.Components components) {
    RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(components);
    RunnerApi.WindowingStrategy windowingStrategyProto = components.getWindowingStrategiesOrThrow(components.getPcollectionsOrThrow(collectionId).getWindowingStrategyId());
    WindowingStrategy<?, ?> windowingStrategy;
    try {
        windowingStrategy = WindowingStrategyTranslation.fromProto(windowingStrategyProto, rehydratedComponents);
    } catch (Exception e) {
        throw new IllegalStateException(String.format("Unable to hydrate GroupByKey windowing strategy %s.", windowingStrategyProto), e);
    }
    @SuppressWarnings("unchecked") WindowingStrategy<?, BoundedWindow> ret = (WindowingStrategy<?, BoundedWindow>) windowingStrategy;
    return ret;
}