Examples with StringSerde org.apache.apex.malhar.lib.utils.serde.StringSerde used on opensource projects

Example 6 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableMapImplTest method serializationBufferTest.

@Test
@Parameters({ "TimeUnifiedManagedState" })
public void serializationBufferTest(String opt) {
    SerializationBuffer keyBuffer = null;
    SerializationBuffer valueBuffer = null;
    SerializationBuffer currentBuffer;
    setup(opt);
    SpillableMapImplForTest<String, String> map;
    if (te == null) {
        map = new SpillableMapImplForTest<>(store, ID1, 0L, new StringSerde(), new StringSerde());
    } else {
        map = new SpillableMapImplForTest<>(store, ID1, new StringSerde(), new StringSerde(), te);
    }
    store.setup(testMeta.operatorContext);
    map.setup(testMeta.operatorContext);
    long windowId = 0L;
    store.beginWindow(windowId);
    map.beginWindow(windowId);
    map.put("a", "1");
    map.endWindow();
    store.endWindow();
    currentBuffer = map.serdeManager.getKeyBufferForWrite();
    Assert.assertTrue(currentBuffer != keyBuffer);
    keyBuffer = currentBuffer;
    currentBuffer = map.serdeManager.getValueBuffer();
    Assert.assertTrue(currentBuffer != valueBuffer);
    valueBuffer = currentBuffer;
    ++windowId;
    store.beginWindow(windowId);
    map.beginWindow(windowId);
    // each put use different key to make sure use the different bucket
    map.put("b", "2");
    map.endWindow();
    store.endWindow();
    currentBuffer = map.serdeManager.getKeyBufferForWrite();
    Assert.assertTrue(currentBuffer != keyBuffer);
    keyBuffer = currentBuffer;
    currentBuffer = map.serdeManager.getValueBuffer();
    Assert.assertTrue(currentBuffer != valueBuffer);
    valueBuffer = currentBuffer;
    map.teardown();
    store.teardown();
}

Example 7 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListImplTest method simpleAddGetAndSetTest1Helper.

public void simpleAddGetAndSetTest1Helper(SpillableStateStore store) {
    SpillableArrayListImpl<String> list = new SpillableArrayListImpl<>(0L, ID1, store, new StringSerde(), 1);
    store.setup(testMeta.operatorContext);
    list.setup(testMeta.operatorContext);
    long windowId = 0L;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkOutOfBounds(list, 0);
    Assert.assertEquals(0, list.size());
    list.add("a");
    SpillableTestUtils.checkOutOfBounds(list, 1);
    Assert.assertEquals(1, list.size());
    Assert.assertEquals("a", list.get(0));
    list.addAll(Lists.newArrayList("a", "b", "c"));
    Assert.assertEquals(4, list.size());
    Assert.assertEquals("a", list.get(0));
    Assert.assertEquals("a", list.get(1));
    Assert.assertEquals("b", list.get(2));
    Assert.assertEquals("c", list.get(3));
    SpillableTestUtils.checkOutOfBounds(list, 4);
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkValue(store, 0L, ID1, 0, Lists.newArrayList("a"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 1, Lists.newArrayList("a"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 2, Lists.newArrayList("b"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 3, Lists.newArrayList("c"));
    Assert.assertEquals(4, list.size());
    Assert.assertEquals("a", list.get(0));
    Assert.assertEquals("a", list.get(1));
    Assert.assertEquals("b", list.get(2));
    Assert.assertEquals("c", list.get(3));
    list.add("tt");
    list.add("ab");
    list.add("99");
    list.add("oo");
    Assert.assertEquals("tt", list.get(4));
    Assert.assertEquals("ab", list.get(5));
    Assert.assertEquals("99", list.get(6));
    Assert.assertEquals("oo", list.get(7));
    list.set(1, "111");
    Assert.assertEquals("a", list.get(0));
    Assert.assertEquals("111", list.get(1));
    Assert.assertEquals("b", list.get(2));
    Assert.assertEquals("c", list.get(3));
    Assert.assertEquals("tt", list.get(4));
    Assert.assertEquals("ab", list.get(5));
    Assert.assertEquals("99", list.get(6));
    Assert.assertEquals("oo", list.get(7));
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkValue(store, 0L, ID1, 0, Lists.newArrayList("a"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 1, Lists.newArrayList("111"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 2, Lists.newArrayList("b"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 3, Lists.newArrayList("c"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 4, Lists.newArrayList("tt"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 5, Lists.newArrayList("ab"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 6, Lists.newArrayList("99"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 7, Lists.newArrayList("oo"));
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    list.teardown();
    store.teardown();
}

Example 8 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListImplTest method simpleAddGetAndSetTest3Helper.

private void simpleAddGetAndSetTest3Helper(SpillableStateStore store) {
    SpillableArrayListImpl<String> list = new SpillableArrayListImpl<>(0L, ID1, store, new StringSerde(), 3);
    store.setup(testMeta.operatorContext);
    list.setup(testMeta.operatorContext);
    long windowId = 0L;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkOutOfBounds(list, 0);
    Assert.assertEquals(0, list.size());
    list.add("a");
    SpillableTestUtils.checkOutOfBounds(list, 1);
    Assert.assertEquals(1, list.size());
    Assert.assertEquals("a", list.get(0));
    list.addAll(Lists.newArrayList("a", "b", "c", "d", "e", "f", "g"));
    Assert.assertEquals(8, list.size());
    Assert.assertEquals("a", list.get(0));
    Assert.assertEquals("a", list.get(1));
    Assert.assertEquals("b", list.get(2));
    Assert.assertEquals("c", list.get(3));
    Assert.assertEquals("d", list.get(4));
    Assert.assertEquals("e", list.get(5));
    Assert.assertEquals("f", list.get(6));
    Assert.assertEquals("g", list.get(7));
    SpillableTestUtils.checkOutOfBounds(list, 20);
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkValue(store, 0L, ID1, 0, Lists.newArrayList("a", "a", "b"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 1, Lists.newArrayList("c", "d", "e"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 2, Lists.newArrayList("f", "g"));
    Assert.assertEquals(8, list.size());
    Assert.assertEquals("a", list.get(0));
    Assert.assertEquals("a", list.get(1));
    Assert.assertEquals("b", list.get(2));
    Assert.assertEquals("c", list.get(3));
    Assert.assertEquals("d", list.get(4));
    Assert.assertEquals("e", list.get(5));
    Assert.assertEquals("f", list.get(6));
    Assert.assertEquals("g", list.get(7));
    list.add("tt");
    list.add("ab");
    list.add("99");
    list.add("oo");
    Assert.assertEquals("tt", list.get(8));
    Assert.assertEquals("ab", list.get(9));
    Assert.assertEquals("99", list.get(10));
    Assert.assertEquals("oo", list.get(11));
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkValue(store, 0L, ID1, 0, Lists.newArrayList("a", "a", "b"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 1, Lists.newArrayList("c", "d", "e"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 2, Lists.newArrayList("f", "g", "tt"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 3, Lists.newArrayList("ab", "99", "oo"));
    list.set(1, "111");
    list.set(3, "222");
    list.set(5, "333");
    list.set(11, "444");
    Assert.assertEquals("a", list.get(0));
    Assert.assertEquals("111", list.get(1));
    Assert.assertEquals("b", list.get(2));
    Assert.assertEquals("222", list.get(3));
    Assert.assertEquals("d", list.get(4));
    Assert.assertEquals("333", list.get(5));
    Assert.assertEquals("f", list.get(6));
    Assert.assertEquals("g", list.get(7));
    Assert.assertEquals("tt", list.get(8));
    Assert.assertEquals("ab", list.get(9));
    Assert.assertEquals("99", list.get(10));
    Assert.assertEquals("444", list.get(11));
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkValue(store, 0L, ID1, 0, Lists.newArrayList("a", "111", "b"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 1, Lists.newArrayList("222", "d", "333"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 2, Lists.newArrayList("f", "g", "tt"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 3, Lists.newArrayList("ab", "99", "444"));
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    list.teardown();
    store.teardown();
}

Example 9 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListMultimapImplTest method recoveryTestWithManagedState.

@Test
public void recoveryTestWithManagedState() {
    SpillableStateStore store = testMeta.store;
    SpillableArrayListMultimapImpl<String, String> map = new SpillableArrayListMultimapImpl<>(store, ID1, 0L, new StringSerde(), new StringSerde());
    store.setup(testMeta.operatorContext);
    map.setup(testMeta.operatorContext);
    long nextWindowId = 0L;
    nextWindowId = simpleMultiKeyTestHelper(store, map, "a", nextWindowId);
    long activationWindow = nextWindowId;
    store.beforeCheckpoint(nextWindowId);
    SpillableArrayListMultimapImpl<String, String> clonedMap = KryoCloneUtils.cloneObject(map);
    store.checkpointed(nextWindowId);
    store.committed(nextWindowId);
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    List<String> list1 = map.get("a");
    Assert.assertEquals(12, list1.size());
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("111", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("222", list1.get(3));
    Assert.assertEquals("d", list1.get(4));
    Assert.assertEquals("333", list1.get(5));
    Assert.assertEquals("f", list1.get(6));
    Assert.assertEquals("g", list1.get(7));
    Assert.assertEquals("tt", list1.get(8));
    Assert.assertEquals("ab", list1.get(9));
    Assert.assertEquals("99", list1.get(10));
    Assert.assertEquals("444", list1.get(11));
    list1.add("111");
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("111", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("222", list1.get(3));
    Assert.assertEquals("d", list1.get(4));
    Assert.assertEquals("333", list1.get(5));
    Assert.assertEquals("f", list1.get(6));
    Assert.assertEquals("g", list1.get(7));
    Assert.assertEquals("tt", list1.get(8));
    Assert.assertEquals("ab", list1.get(9));
    Assert.assertEquals("99", list1.get(10));
    Assert.assertEquals("444", list1.get(11));
    Assert.assertEquals("111", list1.get(12));
    Assert.assertEquals(13, list1.size());
    map.endWindow();
    store.endWindow();
    map.teardown();
    store.teardown();
    map = clonedMap;
    store = map.getStore();
    Attribute.AttributeMap.DefaultAttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_PATH, testMeta.applicationPath);
    attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, activationWindow);
    OperatorContext context = mockOperatorContext(testMeta.operatorContext.getId(), attributes);
    store.setup(context);
    map.setup(context);
    nextWindowId = activationWindow + 1;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    StringSerde serdeString = new StringSerde();
    SerializationBuffer buffer = new SerializationBuffer(new WindowedBlockStream());
    serdeString.serialize("a", buffer);
    Slice keySlice = buffer.toSlice();
    byte[] keyBytes = SliceUtils.concatenate(ID1, keySlice.toByteArray());
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "111", "b", "222", "d", "333", "f", "g", "tt", "ab", "99", "444"));
    Assert.assertEquals(1, map.size());
    Assert.assertEquals(12, map.get("a").size());
    map.endWindow();
    store.endWindow();
    map.teardown();
    store.teardown();
}

Example 10 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListMultimapImplTest method simpleMultiKeyTestHelper.

public void simpleMultiKeyTestHelper(SpillableStateStore store) {
    SpillableArrayListMultimapImpl<String, String> map = new SpillableArrayListMultimapImpl<String, String>(store, ID1, 0L, new StringSerde(), new StringSerde());
    store.setup(testMeta.operatorContext);
    map.setup(testMeta.operatorContext);
    long nextWindowId = 0L;
    nextWindowId = simpleMultiKeyTestHelper(store, map, "a", nextWindowId);
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    Assert.assertEquals(1, map.size());
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    nextWindowId = simpleMultiKeyTestHelper(store, map, "b", nextWindowId);
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    Assert.assertEquals(2, map.size());
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    simpleMultiKeyTestHelper(store, map, "c", nextWindowId);
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    Assert.assertEquals(3, map.size());
    map.endWindow();
    store.endWindow();
    map.teardown();
    store.teardown();
}