Examples with StringSerde org.apache.apex.malhar.lib.utils.serde.StringSerde used on opensource projects

Example 11 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListMultimapImplTest method simpleMultiKeyTestHelper.

public long simpleMultiKeyTestHelper(SpillableStateStore store, SpillableArrayListMultimapImpl<String, String> map, String key, long nextWindowId) {
    StringSerde serdeString = new StringSerde();
    IntSerde serdeInt = new IntSerde();
    SerializationBuffer buffer = new SerializationBuffer(new WindowedBlockStream());
    serdeString.serialize(key, buffer);
    Slice keySlice = buffer.toSlice();
    byte[] keyBytes = SliceUtils.concatenate(ID1, keySlice.toByteArray());
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    Assert.assertNull(map.get(key));
    Assert.assertFalse(map.containsKey(key));
    map.put(key, "a");
    Assert.assertTrue(map.containsKey(key));
    List<String> list1 = map.get(key);
    Assert.assertEquals(1, list1.size());
    Assert.assertEquals("a", list1.get(0));
    list1.addAll(Lists.newArrayList("a", "b", "c", "d", "e", "f", "g"));
    Assert.assertEquals(8, list1.size());
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("a", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("c", list1.get(3));
    Assert.assertEquals("d", list1.get(4));
    Assert.assertEquals("e", list1.get(5));
    Assert.assertEquals("f", list1.get(6));
    Assert.assertEquals("g", list1.get(7));
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    SpillableTestUtils.checkValue(store, 0L, SliceUtils.concatenate(keyBytes, SpillableArrayListMultimapImpl.SIZE_KEY_SUFFIX), 8, 0, serdeInt);
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "a", "b", "c", "d", "e", "f", "g"));
    List<String> list2 = map.get(key);
    Assert.assertEquals(8, list2.size());
    Assert.assertEquals("a", list2.get(0));
    Assert.assertEquals("a", list2.get(1));
    Assert.assertEquals("b", list2.get(2));
    Assert.assertEquals("c", list2.get(3));
    Assert.assertEquals("d", list2.get(4));
    Assert.assertEquals("e", list2.get(5));
    Assert.assertEquals("f", list2.get(6));
    Assert.assertEquals("g", list2.get(7));
    list2.add("tt");
    list2.add("ab");
    list2.add("99");
    list2.add("oo");
    Assert.assertEquals("tt", list2.get(8));
    Assert.assertEquals("ab", list2.get(9));
    Assert.assertEquals("99", list2.get(10));
    Assert.assertEquals("oo", list2.get(11));
    Assert.assertEquals(12, list2.size());
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    Assert.assertEquals(12, list2.size());
    SpillableTestUtils.checkValue(store, 0L, SliceUtils.concatenate(keyBytes, SpillableArrayListMultimapImpl.SIZE_KEY_SUFFIX), 12, 0, serdeInt);
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "a", "b", "c", "d", "e", "f", "g", "tt", "ab", "99", "oo"));
    List<String> list3 = map.get(key);
    list3.set(1, "111");
    list3.set(3, "222");
    list3.set(5, "333");
    list3.set(11, "444");
    Assert.assertEquals("a", list3.get(0));
    Assert.assertEquals("111", list3.get(1));
    Assert.assertEquals("b", list3.get(2));
    Assert.assertEquals("222", list3.get(3));
    Assert.assertEquals("d", list3.get(4));
    Assert.assertEquals("333", list3.get(5));
    Assert.assertEquals("f", list3.get(6));
    Assert.assertEquals("g", list3.get(7));
    Assert.assertEquals("tt", list3.get(8));
    Assert.assertEquals("ab", list3.get(9));
    Assert.assertEquals("99", list3.get(10));
    Assert.assertEquals("444", list3.get(11));
    Assert.assertEquals(12, list2.size());
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    SpillableTestUtils.checkValue(store, 0L, SliceUtils.concatenate(keyBytes, SpillableArrayListMultimapImpl.SIZE_KEY_SUFFIX), 12, 0, serdeInt);
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "111", "b", "222", "d", "333", "f", "g", "tt", "ab", "99", "444"));
    map.endWindow();
    store.endWindow();
    return nextWindowId;
}

Example 12 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableMapImplTest method recoveryWithManagedStateTest.

@Test
@Parameters({ "ManagedState", "TimeUnifiedManagedState" })
public void recoveryWithManagedStateTest(String opt) throws Exception {
    setup(opt);
    StringSerde sss = new StringSerde();
    SpillableMapImpl<String, String> map1 = null;
    if (te == null) {
        map1 = new SpillableMapImpl<>(store, ID1, 0L, sss, sss);
    } else {
        map1 = new SpillableMapImpl<>(store, ID1, sss, sss, te);
    }
    store.setup(testMeta.operatorContext);
    map1.setup(testMeta.operatorContext);
    store.beginWindow(0);
    map1.beginWindow(0);
    map1.put("x", "1");
    map1.put("y", "2");
    map1.put("z", "3");
    map1.put("zz", "33");
    Assert.assertEquals(4, map1.size());
    map1.endWindow();
    store.endWindow();
    store.beginWindow(1);
    map1.beginWindow(1);
    Assert.assertEquals(4, map1.size());
    map1.put("x", "4");
    map1.put("y", "5");
    map1.remove("zz");
    Assert.assertEquals(3, map1.size());
    map1.endWindow();
    store.endWindow();
    store.beforeCheckpoint(1);
    store.checkpointed(1);
    SpillableMapImpl<String, String> clonedMap1 = KryoCloneUtils.cloneObject(map1);
    store.beginWindow(2);
    map1.beginWindow(2);
    Assert.assertEquals(3, map1.size());
    map1.put("x", "6");
    map1.put("y", "7");
    map1.put("w", "8");
    Assert.assertEquals(4, map1.size());
    map1.endWindow();
    store.endWindow();
    // simulating crash here
    map1.teardown();
    store.teardown();
    Attribute.AttributeMap.DefaultAttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_PATH, testMeta.applicationPath);
    attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 1L);
    OperatorContext context = mockOperatorContext(testMeta.operatorContext.getId(), attributes);
    map1 = clonedMap1;
    map1.getStore().setup(context);
    map1.setup(testMeta.operatorContext);
    map1.getStore().beginWindow(2);
    map1.beginWindow(2);
    Assert.assertEquals(3, map1.size());
    Assert.assertEquals("4", map1.get("x"));
    Assert.assertEquals("5", map1.get("y"));
    Assert.assertEquals("3", map1.get("z"));
    map1.endWindow();
    map1.getStore().endWindow();
    map1.teardown();
}

Example 13 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableMapImplTest method multiMapPerBucketTest.

@Test
@Parameters({ "InMem", "ManagedState", "TimeUnifiedManagedState" })
public void multiMapPerBucketTest(String opt) {
    setup(opt);
    StringSerde sss = new StringSerde();
    SpillableMapImpl<String, String> map1 = null;
    SpillableMapImpl<String, String> map2 = null;
    if (te == null) {
        map1 = new SpillableMapImpl<>(store, ID1, 0L, sss, sss);
        map2 = new SpillableMapImpl<>(store, ID2, 0L, sss, sss);
    } else {
        map1 = new SpillableMapImpl<>(store, ID1, sss, sss, te);
        map2 = new SpillableMapImpl<>(store, ID2, sss, sss, te);
    }
    store.setup(testMeta.operatorContext);
    map1.setup(testMeta.operatorContext);
    map2.setup(testMeta.operatorContext);
    long windowId = 0L;
    store.beginWindow(windowId);
    map1.beginWindow(windowId);
    map2.beginWindow(windowId);
    map1.put("a", "1");
    Assert.assertEquals("1", map1.get("a"));
    Assert.assertEquals(null, map2.get("a"));
    map2.put("a", "a1");
    Assert.assertEquals("1", map1.get("a"));
    Assert.assertEquals("a1", map2.get("a"));
    map1.put("b", "2");
    map2.put("c", "3");
    Assert.assertEquals("1", map1.get("a"));
    Assert.assertEquals("2", map1.get("b"));
    Assert.assertEquals("a1", map2.get("a"));
    Assert.assertEquals(null, map2.get("b"));
    Assert.assertEquals("3", map2.get("c"));
    map1.endWindow();
    map2.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    windowId++;
    store.beginWindow(windowId);
    map1.beginWindow(windowId);
    map2.beginWindow(windowId);
    multiValueCheck(new String[] { "a", "b" }, ID1, new String[] { "1", "2" });
    multiValueCheck(new String[] { "a", "b", "c" }, ID2, new String[] { "a1", null, "3" });
    map1.remove("a");
    Assert.assertEquals(null, map1.get("a"));
    Assert.assertEquals("a1", map2.get("a"));
    map1.endWindow();
    map2.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    windowId++;
    store.beginWindow(windowId);
    map1.beginWindow(windowId);
    map2.beginWindow(windowId);
    multiValueCheck(new String[] { "a" }, ID1, new String[] { null });
    multiValueCheck(new String[] { "a" }, ID2, new String[] { "a1" });
    map1.endWindow();
    map2.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    map1.teardown();
    map2.teardown();
    store.teardown();
}

Example 14 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class DefaultBucketTest method testFreeMemory.

@Test
public void testFreeMemory() throws IOException {
    DefaultBucket.setDisableBloomFilterByDefault(false);
    testMeta.defaultBucket.setup(testMeta.managedStateContext);
    testGetFromReader();
    long initSize = testMeta.defaultBucket.getSizeInBytes();
    // The temporary memory generated by get was not managed by bucket, only put was managed by bucket
    SerializationBuffer buffer = new SerializationBuffer(testMeta.defaultBucket.getKeyStream());
    byte[] keyPrefix = new byte[] { 0 };
    String key = "1";
    String value = "2";
    AffixSerde<String> keySerde = new AffixSerde<>(keyPrefix, new StringSerde(), null);
    StringSerde valueSerde = new StringSerde();
    testMeta.defaultBucket.getKeyStream().beginWindow(1);
    testMeta.defaultBucket.getValueStream().beginWindow(1);
    keySerde.serialize(key, buffer);
    Slice keySlice = buffer.toSlice();
    valueSerde.serialize(value, buffer);
    Slice valueSlice = buffer.toSlice();
    testMeta.defaultBucket.put(keySlice, 1, valueSlice);
    testMeta.defaultBucket.getKeyStream().endWindow();
    testMeta.defaultBucket.getValueStream().endWindow();
    long currentSize = testMeta.defaultBucket.getSizeInBytes();
    testMeta.defaultBucket.freeMemory(Long.MAX_VALUE);
    // call this method to invoke the release memory
    testMeta.defaultBucket.get(keySlice, -1, ReadSource.MEMORY);
    long sizeFreed = currentSize - testMeta.defaultBucket.getSizeInBytes();
    Assert.assertEquals("size freed", initSize, sizeFreed);
    Assert.assertEquals("existing size", currentSize - initSize, testMeta.defaultBucket.getSizeInBytes());
    testMeta.defaultBucket.teardown();
}