Examples with AccumulationMode org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode used on opensource projects

Example 1 with AccumulationMode

use of org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode in project beam by apache.

the class WindowingStrategyTranslation method fromProto.

/**
   * Converts from {@link RunnerApi.WindowingStrategy} to the SDK's {@link WindowingStrategy} using
   * the provided components to dereferences identifiers found in the proto.
   */
public static WindowingStrategy<?, ?> fromProto(RunnerApi.WindowingStrategy proto, Components components) throws InvalidProtocolBufferException {
    SdkFunctionSpec windowFnSpec = proto.getWindowFn();
    WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec);
    TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
    AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
    Trigger trigger = TriggerTranslation.fromProto(proto.getTrigger());
    ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
    Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
    return WindowingStrategy.of(windowFn).withAllowedLateness(allowedLateness).withMode(accumulationMode).withTrigger(trigger).withTimestampCombiner(timestampCombiner).withClosingBehavior(closingBehavior);
}

Example 2 with AccumulationMode

use of org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode in project beam by apache.

the class TriggerStateMachineTester method forTrigger.

public static <W extends BoundedWindow> SimpleTriggerStateMachineTester<W> forTrigger(TriggerStateMachine stateMachine, WindowFn<Object, W> windowFn) throws Exception {
    ExecutableTriggerStateMachine executableTriggerStateMachine = ExecutableTriggerStateMachine.create(stateMachine);
    // Merging requires accumulation mode or early firings can break up a session.
    // Not currently an issue with the tester (because we never GC) but we don't want
    // mystery failures due to violating this need.
    AccumulationMode mode = windowFn.isNonMerging() ? AccumulationMode.DISCARDING_FIRED_PANES : AccumulationMode.ACCUMULATING_FIRED_PANES;
    return new SimpleTriggerStateMachineTester<>(executableTriggerStateMachine, windowFn, Duration.ZERO);
}

Example 3 with AccumulationMode

use of org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode in project beam by apache.

the class TriggerStateMachineTester method forAdvancedTrigger.

public static <InputT, W extends BoundedWindow> TriggerStateMachineTester<InputT, W> forAdvancedTrigger(TriggerStateMachine stateMachine, WindowFn<Object, W> windowFn) throws Exception {
    ExecutableTriggerStateMachine executableTriggerStateMachine = ExecutableTriggerStateMachine.create(stateMachine);
    // Merging requires accumulation mode or early firings can break up a session.
    // Not currently an issue with the tester (because we never GC) but we don't want
    // mystery failures due to violating this need.
    AccumulationMode mode = windowFn.isNonMerging() ? AccumulationMode.DISCARDING_FIRED_PANES : AccumulationMode.ACCUMULATING_FIRED_PANES;
    return new TriggerStateMachineTester<>(executableTriggerStateMachine, windowFn, Duration.ZERO);
}

Example 4 with AccumulationMode

use of org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode in project beam by apache.

the class WindowingStrategyTranslation method fromProto.

/**
 * Converts from {@link RunnerApi.WindowingStrategy} to the SDK's {@link WindowingStrategy} using
 * the provided components to dereferences identifiers found in the proto.
 */
public static WindowingStrategy<?, ?> fromProto(RunnerApi.WindowingStrategy proto, RehydratedComponents components) throws InvalidProtocolBufferException {
    FunctionSpec windowFnSpec = proto.getWindowFn();
    WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec);
    TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
    AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
    Trigger trigger = TriggerTranslation.fromProto(proto.getTrigger());
    ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
    Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
    OnTimeBehavior onTimeBehavior = fromProto(proto.getOnTimeBehavior());
    String environmentId = proto.getEnvironmentId();
    return WindowingStrategy.of(windowFn).withAllowedLateness(allowedLateness).withMode(accumulationMode).withTrigger(trigger).withTimestampCombiner(timestampCombiner).withClosingBehavior(closingBehavior).withOnTimeBehavior(onTimeBehavior).withEnvironmentId(environmentId);
}

Example 5 with AccumulationMode

use of org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode in project beam by apache.

the class JoinTest method testBuild_OptionalWindowing.

@Test
public void testBuild_OptionalWindowing() {
    final Pipeline pipeline = TestUtils.createTestPipeline();
    final PCollection<String> left = TestUtils.createMockDataset(pipeline, TypeDescriptors.strings());
    final PCollection<String> right = TestUtils.createMockDataset(pipeline, TypeDescriptors.strings());
    final PCollection<KV<Integer, String>> joined = Join.named("Join1").of(left, right).by(String::length, String::length).using((String l, String r, Collector<String> c) -> c.collect(l + r)).applyIf(true, b -> b.windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1))).triggeredBy(AfterWatermark.pastEndOfWindow()).accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)).output();
    final Join join = (Join) TestUtils.getProducer(joined);
    assertTrue(join.getWindow().isPresent());
    final Window<?> window = (Window) join.getWindow().get();
    assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), window.getWindowFn());
    assertEquals(AfterWatermark.pastEndOfWindow(), WindowDesc.of(window).getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, WindowDesc.of(window).getAccumulationMode());
}