Examples with WindowDesc org.apache.beam.sdk.transforms.windowing.WindowDesc used on opensource projects

Example 1 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class CountByKeyTest method testWindow_applyIf.

@Test
public void testWindow_applyIf() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<KV<String, Long>> counted = CountByKey.named("CountByKey1").of(dataset).keyBy(s -> s).applyIf(true, b -> b.windowBy(windowing).triggeredBy(trigger).discardingFiredPanes()).output();
    final CountByKey count = (CountByKey) TestUtils.getProducer(counted);
    assertTrue(count.getWindow().isPresent());
    final WindowDesc<?> desc = WindowDesc.of((Window<?>) count.getWindow().get());
    assertEquals(windowing, desc.getWindowFn());
    assertEquals(trigger, desc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, desc.getAccumulationMode());
}

Example 2 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class ReduceByKeyTest method testBuild.

@Test
public void testBuild() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<KV<String, Long>> reduced = ReduceByKey.named("ReduceByKey1").of(dataset).keyBy(s -> s).valueBy(s -> 1L).combineBy(Sums.ofLongs()).windowBy(windowing).triggeredBy(trigger).discardingFiredPanes().withAllowedLateness(Duration.standardSeconds(1000)).output();
    final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);
    assertTrue(reduce.getName().isPresent());
    assertEquals("ReduceByKey1", reduce.getName().get());
    assertNotNull(reduce.getKeyExtractor());
    assertNotNull(reduce.getValueExtractor());
    assertTrue(reduce.isCombineFnStyle());
    assertNotNull(reduce.getAccumulatorFactory());
    assertNotNull(reduce.getAccumulate());
    assertNotNull(reduce.getAccumulatorType());
    assertNotNull(reduce.getMergeAccumulators());
    assertNotNull(reduce.getOutputFn());
    assertTrue(reduce.getWindow().isPresent());
    @SuppressWarnings("unchecked") final WindowDesc<?> windowDesc = WindowDesc.of((Window) reduce.getWindow().get());
    assertEquals(windowing, windowDesc.getWindowFn());
    assertEquals(trigger, windowDesc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
    assertEquals(Duration.standardSeconds(1000), windowDesc.getAllowedLateness());
}

Example 3 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class TopPerKeyTest method testBuild.

@Test
public void testBuild() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<Triple<String, Long, Long>> result = TopPerKey.named("TopPerKey1").of(dataset).keyBy(s -> s).valueBy(s -> 1L).scoreBy(s -> 1L).windowBy(windowing).triggeredBy(trigger).discardingFiredPanes().withAllowedLateness(Duration.millis(1000)).output();
    final TopPerKey tpk = (TopPerKey) TestUtils.getProducer(result);
    assertTrue(tpk.getName().isPresent());
    assertEquals("TopPerKey1", tpk.getName().get());
    assertNotNull(tpk.getKeyExtractor());
    assertNotNull(tpk.getValueExtractor());
    assertNotNull(tpk.getScoreExtractor());
    assertTrue(tpk.getWindow().isPresent());
    @SuppressWarnings("unchecked") final WindowDesc<?> windowDesc = WindowDesc.of((Window) tpk.getWindow().get());
    assertEquals(windowing, windowDesc.getWindowFn());
    assertEquals(trigger, windowDesc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
    assertEquals(Duration.millis(1000), windowDesc.getAllowedLateness());
}

Example 4 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class CountByKeyTest method testBuild.

@Test
public void testBuild() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<KV<String, Long>> counted = CountByKey.named("CountByKey1").of(dataset).keyBy(s -> s).windowBy(windowing).triggeredBy(trigger).discardingFiredPanes().withAllowedLateness(Duration.millis(1000)).output();
    final CountByKey count = (CountByKey) TestUtils.getProducer(counted);
    assertTrue(count.getName().isPresent());
    assertEquals("CountByKey1", count.getName().get());
    assertNotNull(count.getKeyExtractor());
    assertTrue(count.getWindow().isPresent());
    final WindowDesc<?> desc = WindowDesc.of((Window<?>) count.getWindow().get());
    assertEquals(windowing, desc.getWindowFn());
    assertEquals(trigger, desc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, desc.getAccumulationMode());
    assertEquals(Duration.millis(1000), desc.getAllowedLateness());
}