Examples with BagState org.apache.beam.sdk.state.BagState used on opensource projects

Example 1 with BagState

use of org.apache.beam.sdk.state.BagState in project beam by apache.

the class ParDoTest method testBagStateCoderInferenceFailure.

@Test
@Category({ ValidatesRunner.class, UsesStatefulParDo.class })
public void testBagStateCoderInferenceFailure() throws Exception {
    final String stateId = "foo";
    Coder<MyInteger> myIntegerCoder = MyIntegerCoder.of();
    DoFn<KV<String, Integer>, List<MyInteger>> fn = new DoFn<KV<String, Integer>, List<MyInteger>>() {

        @StateId(stateId)
        private final StateSpec<BagState<MyInteger>> bufferState = StateSpecs.bag();

        @ProcessElement
        public void processElement(ProcessContext c, @StateId(stateId) BagState<MyInteger> state) {
            Iterable<MyInteger> currentValue = state.read();
            state.add(new MyInteger(c.element().getValue()));
            if (Iterables.size(state.read()) >= 4) {
                List<MyInteger> sorted = Lists.newArrayList(currentValue);
                Collections.sort(sorted);
                c.output(sorted);
            }
        }
    };
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("Unable to infer a coder for BagState and no Coder was specified.");
    pipeline.apply(Create.of(KV.of("hello", 97), KV.of("hello", 42), KV.of("hello", 84), KV.of("hello", 12))).apply(ParDo.of(fn)).setCoder(ListCoder.of(myIntegerCoder));
    pipeline.run();
}

Example 2 with BagState

use of org.apache.beam.sdk.state.BagState in project beam by apache.

the class ParDoTest method testBagStateSideInput.

@Test
@Category({ ValidatesRunner.class, UsesStatefulParDo.class })
public void testBagStateSideInput() {
    final PCollectionView<List<Integer>> listView = pipeline.apply("Create list for side input", Create.of(2, 1, 0)).apply(View.<Integer>asList());
    final String stateId = "foo";
    DoFn<KV<String, Integer>, List<Integer>> fn = new DoFn<KV<String, Integer>, List<Integer>>() {

        @StateId(stateId)
        private final StateSpec<BagState<Integer>> bufferState = StateSpecs.bag(VarIntCoder.of());

        @ProcessElement
        public void processElement(ProcessContext c, @StateId(stateId) BagState<Integer> state) {
            Iterable<Integer> currentValue = state.read();
            state.add(c.element().getValue());
            if (Iterables.size(state.read()) >= 4) {
                List<Integer> sorted = Lists.newArrayList(currentValue);
                Collections.sort(sorted);
                c.output(sorted);
                List<Integer> sideSorted = Lists.newArrayList(c.sideInput(listView));
                Collections.sort(sideSorted);
                c.output(sideSorted);
            }
        }
    };
    PCollection<List<Integer>> output = pipeline.apply("Create main input", Create.of(KV.of("hello", 97), KV.of("hello", 42), KV.of("hello", 84), KV.of("hello", 12))).apply(ParDo.of(fn).withSideInputs(listView));
    PAssert.that(output).containsInAnyOrder(Lists.newArrayList(12, 42, 84, 97), Lists.newArrayList(0, 1, 2));
    pipeline.run();
}

Example 3 with BagState

use of org.apache.beam.sdk.state.BagState in project beam by apache.

the class ParDoTest method testBagStateCoderInference.

@Test
@Category({ ValidatesRunner.class, UsesStatefulParDo.class })
public void testBagStateCoderInference() {
    final String stateId = "foo";
    Coder<MyInteger> myIntegerCoder = MyIntegerCoder.of();
    pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
    DoFn<KV<String, Integer>, List<MyInteger>> fn = new DoFn<KV<String, Integer>, List<MyInteger>>() {

        @StateId(stateId)
        private final StateSpec<BagState<MyInteger>> bufferState = StateSpecs.bag();

        @ProcessElement
        public void processElement(ProcessContext c, @StateId(stateId) BagState<MyInteger> state) {
            Iterable<MyInteger> currentValue = state.read();
            state.add(new MyInteger(c.element().getValue()));
            if (Iterables.size(state.read()) >= 4) {
                List<MyInteger> sorted = Lists.newArrayList(currentValue);
                Collections.sort(sorted);
                c.output(sorted);
            }
        }
    };
    PCollection<List<MyInteger>> output = pipeline.apply(Create.of(KV.of("hello", 97), KV.of("hello", 42), KV.of("hello", 84), KV.of("hello", 12))).apply(ParDo.of(fn)).setCoder(ListCoder.of(myIntegerCoder));
    PAssert.that(output).containsInAnyOrder(Lists.newArrayList(new MyInteger(12), new MyInteger(42), new MyInteger(84), new MyInteger(97)));
    pipeline.run();
}

Example 4 with BagState

use of org.apache.beam.sdk.state.BagState in project beam by apache.

the class CopyOnAccessInMemoryStateInternalsTest method testCommitWithoutUnderlying.

@Test
public void testCommitWithoutUnderlying() {
    CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
    StateNamespace namespace = new StateNamespaceForTest("foo");
    StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of());
    BagState<String> stringBag = internals.state(namespace, bagTag);
    assertThat(stringBag.read(), emptyIterable());
    stringBag.add("bar");
    stringBag.add("baz");
    assertThat(stringBag.read(), containsInAnyOrder("baz", "bar"));
    internals.commit();
    BagState<String> reReadStringBag = internals.state(namespace, bagTag);
    assertThat(reReadStringBag.read(), containsInAnyOrder("baz", "bar"));
    assertThat(internals.isEmpty(), is(false));
}

Example 5 with BagState

use of org.apache.beam.sdk.state.BagState in project beam by apache.

the class CopyOnAccessInMemoryStateInternalsTest method testCommitWithOverwrittenUnderlying.

@Test
public void testCommitWithOverwrittenUnderlying() {
    CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
    CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
    StateNamespace namespace = new StateNamespaceForTest("foo");
    StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of());
    BagState<String> stringBag = underlying.state(namespace, bagTag);
    assertThat(stringBag.read(), emptyIterable());
    stringBag.add("bar");
    stringBag.add("baz");
    BagState<String> internalsState = internals.state(namespace, bagTag);
    internalsState.add("eggs");
    internalsState.add("ham");
    internalsState.add("0x00ff00");
    internalsState.add("&");
    internals.commit();
    BagState<String> reReadInternalState = internals.state(namespace, bagTag);
    assertThat(reReadInternalState.read(), containsInAnyOrder("bar", "baz", "0x00ff00", "eggs", "&", "ham"));
    BagState<String> reReadUnderlyingState = underlying.state(namespace, bagTag);
    assertThat(reReadUnderlyingState.read(), containsInAnyOrder("bar", "baz"));
}