Search in sources :

Example 1 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class CountByKeyTest method testWindow_applyIf.

@Test
public void testWindow_applyIf() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<KV<String, Long>> counted = CountByKey.named("CountByKey1").of(dataset).keyBy(s -> s).applyIf(true, b -> b.windowBy(windowing).triggeredBy(trigger).discardingFiredPanes()).output();
    final CountByKey count = (CountByKey) TestUtils.getProducer(counted);
    assertTrue(count.getWindow().isPresent());
    final WindowDesc<?> desc = WindowDesc.of((Window<?>) count.getWindow().get());
    assertEquals(windowing, desc.getWindowFn());
    assertEquals(trigger, desc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, desc.getAccumulationMode());
}
Also used : KV(org.apache.beam.sdk.values.KV) TypeDescriptor(org.apache.beam.sdk.values.TypeDescriptor) AccumulationMode(org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode) Assert.assertNotNull(org.junit.Assert.assertNotNull) WindowDesc(org.apache.beam.sdk.transforms.windowing.WindowDesc) Duration(org.joda.time.Duration) RunWith(org.junit.runner.RunWith) Assert.assertTrue(org.junit.Assert.assertTrue) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) Test(org.junit.Test) JUnit4(org.junit.runners.JUnit4) PCollection(org.apache.beam.sdk.values.PCollection) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) Assert.assertFalse(org.junit.Assert.assertFalse) TypeDescriptors(org.apache.beam.sdk.values.TypeDescriptors) Window(org.apache.beam.sdk.transforms.windowing.Window) Assert.assertEquals(org.junit.Assert.assertEquals) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) KV(org.apache.beam.sdk.values.KV) Test(org.junit.Test)

Example 2 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class ReduceByKeyTest method testBuild.

@Test
public void testBuild() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<KV<String, Long>> reduced = ReduceByKey.named("ReduceByKey1").of(dataset).keyBy(s -> s).valueBy(s -> 1L).combineBy(Sums.ofLongs()).windowBy(windowing).triggeredBy(trigger).discardingFiredPanes().withAllowedLateness(Duration.standardSeconds(1000)).output();
    final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);
    assertTrue(reduce.getName().isPresent());
    assertEquals("ReduceByKey1", reduce.getName().get());
    assertNotNull(reduce.getKeyExtractor());
    assertNotNull(reduce.getValueExtractor());
    assertTrue(reduce.isCombineFnStyle());
    assertNotNull(reduce.getAccumulatorFactory());
    assertNotNull(reduce.getAccumulate());
    assertNotNull(reduce.getAccumulatorType());
    assertNotNull(reduce.getMergeAccumulators());
    assertNotNull(reduce.getOutputFn());
    assertTrue(reduce.getWindow().isPresent());
    @SuppressWarnings("unchecked") final WindowDesc<?> windowDesc = WindowDesc.of((Window) reduce.getWindow().get());
    assertEquals(windowing, windowDesc.getWindowFn());
    assertEquals(trigger, windowDesc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
    assertEquals(Duration.standardSeconds(1000), windowDesc.getAllowedLateness());
}
Also used : KV(org.apache.beam.sdk.values.KV) TypeDescriptor(org.apache.beam.sdk.values.TypeDescriptor) Duration(org.joda.time.Duration) RunWith(org.junit.runner.RunWith) ArrayList(java.util.ArrayList) Assert.assertSame(org.junit.Assert.assertSame) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) TypePropagationAssert(org.apache.beam.sdk.extensions.euphoria.core.client.type.TypePropagationAssert) Window(org.apache.beam.sdk.transforms.windowing.Window) Sums(org.apache.beam.sdk.extensions.euphoria.core.client.util.Sums) AccumulationMode(org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode) Assert.assertNotNull(org.junit.Assert.assertNotNull) WindowDesc(org.apache.beam.sdk.transforms.windowing.WindowDesc) Assert.assertTrue(org.junit.Assert.assertTrue) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) Lists(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists) Test(org.junit.Test) JUnit4(org.junit.runners.JUnit4) PCollection(org.apache.beam.sdk.values.PCollection) Collectors(java.util.stream.Collectors) List(java.util.List) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) Assert.assertFalse(org.junit.Assert.assertFalse) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) TypeDescriptors(org.apache.beam.sdk.values.TypeDescriptors) Assert.assertEquals(org.junit.Assert.assertEquals) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) KV(org.apache.beam.sdk.values.KV) Test(org.junit.Test)

Example 3 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class TopPerKeyTest method testBuild.

@Test
public void testBuild() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<Triple<String, Long, Long>> result = TopPerKey.named("TopPerKey1").of(dataset).keyBy(s -> s).valueBy(s -> 1L).scoreBy(s -> 1L).windowBy(windowing).triggeredBy(trigger).discardingFiredPanes().withAllowedLateness(Duration.millis(1000)).output();
    final TopPerKey tpk = (TopPerKey) TestUtils.getProducer(result);
    assertTrue(tpk.getName().isPresent());
    assertEquals("TopPerKey1", tpk.getName().get());
    assertNotNull(tpk.getKeyExtractor());
    assertNotNull(tpk.getValueExtractor());
    assertNotNull(tpk.getScoreExtractor());
    assertTrue(tpk.getWindow().isPresent());
    @SuppressWarnings("unchecked") final WindowDesc<?> windowDesc = WindowDesc.of((Window) tpk.getWindow().get());
    assertEquals(windowing, windowDesc.getWindowFn());
    assertEquals(trigger, windowDesc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
    assertEquals(Duration.millis(1000), windowDesc.getAllowedLateness());
}
Also used : Triple(org.apache.beam.sdk.extensions.euphoria.core.client.util.Triple) AccumulationMode(org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode) Triple(org.apache.beam.sdk.extensions.euphoria.core.client.util.Triple) Assert.assertNotNull(org.junit.Assert.assertNotNull) WindowDesc(org.apache.beam.sdk.transforms.windowing.WindowDesc) Duration(org.joda.time.Duration) RunWith(org.junit.runner.RunWith) Assert.assertTrue(org.junit.Assert.assertTrue) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) Test(org.junit.Test) JUnit4(org.junit.runners.JUnit4) PCollection(org.apache.beam.sdk.values.PCollection) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) Assert.assertFalse(org.junit.Assert.assertFalse) TypeDescriptors(org.apache.beam.sdk.values.TypeDescriptors) Window(org.apache.beam.sdk.transforms.windowing.Window) Assert.assertEquals(org.junit.Assert.assertEquals) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) Test(org.junit.Test)

Example 4 with WindowDesc

use of org.apache.beam.sdk.transforms.windowing.WindowDesc in project beam by apache.

the class CountByKeyTest method testBuild.

@Test
public void testBuild() {
    final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
    final FixedWindows windowing = FixedWindows.of(org.joda.time.Duration.standardHours(1));
    final DefaultTrigger trigger = DefaultTrigger.of();
    final PCollection<KV<String, Long>> counted = CountByKey.named("CountByKey1").of(dataset).keyBy(s -> s).windowBy(windowing).triggeredBy(trigger).discardingFiredPanes().withAllowedLateness(Duration.millis(1000)).output();
    final CountByKey count = (CountByKey) TestUtils.getProducer(counted);
    assertTrue(count.getName().isPresent());
    assertEquals("CountByKey1", count.getName().get());
    assertNotNull(count.getKeyExtractor());
    assertTrue(count.getWindow().isPresent());
    final WindowDesc<?> desc = WindowDesc.of((Window<?>) count.getWindow().get());
    assertEquals(windowing, desc.getWindowFn());
    assertEquals(trigger, desc.getTrigger());
    assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, desc.getAccumulationMode());
    assertEquals(Duration.millis(1000), desc.getAllowedLateness());
}
Also used : KV(org.apache.beam.sdk.values.KV) TypeDescriptor(org.apache.beam.sdk.values.TypeDescriptor) AccumulationMode(org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode) Assert.assertNotNull(org.junit.Assert.assertNotNull) WindowDesc(org.apache.beam.sdk.transforms.windowing.WindowDesc) Duration(org.joda.time.Duration) RunWith(org.junit.runner.RunWith) Assert.assertTrue(org.junit.Assert.assertTrue) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) Test(org.junit.Test) JUnit4(org.junit.runners.JUnit4) PCollection(org.apache.beam.sdk.values.PCollection) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) Assert.assertFalse(org.junit.Assert.assertFalse) TypeDescriptors(org.apache.beam.sdk.values.TypeDescriptors) Window(org.apache.beam.sdk.transforms.windowing.Window) Assert.assertEquals(org.junit.Assert.assertEquals) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) KV(org.apache.beam.sdk.values.KV) Test(org.junit.Test)

Aggregations

DefaultTrigger (org.apache.beam.sdk.transforms.windowing.DefaultTrigger)4 FixedWindows (org.apache.beam.sdk.transforms.windowing.FixedWindows)4 Window (org.apache.beam.sdk.transforms.windowing.Window)4 WindowDesc (org.apache.beam.sdk.transforms.windowing.WindowDesc)4 PCollection (org.apache.beam.sdk.values.PCollection)4 TypeDescriptors (org.apache.beam.sdk.values.TypeDescriptors)4 AccumulationMode (org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode)4 Duration (org.joda.time.Duration)4 Assert.assertEquals (org.junit.Assert.assertEquals)4 Assert.assertFalse (org.junit.Assert.assertFalse)4 Assert.assertNotNull (org.junit.Assert.assertNotNull)4 Assert.assertTrue (org.junit.Assert.assertTrue)4 Test (org.junit.Test)4 RunWith (org.junit.runner.RunWith)4 JUnit4 (org.junit.runners.JUnit4)4 KV (org.apache.beam.sdk.values.KV)3 TypeDescriptor (org.apache.beam.sdk.values.TypeDescriptor)3 ArrayList (java.util.ArrayList)1 List (java.util.List)1 Collectors (java.util.stream.Collectors)1