Examples with StringSerde org.apache.apex.malhar.lib.utils.serde.StringSerde used on opensource projects

Example 1 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class GenericSerdePerformanceTest method testCompareSerdeForString.

@Test
public void testCompareSerdeForString() {
    long beginTime = System.currentTimeMillis();
    testSerdeForString(new GenericSerde<String>(String.class));
    long genericSerdeCost = System.currentTimeMillis() - beginTime;
    logger.info("Generic Serde cost for String: {}", genericSerdeCost);
    beginTime = System.currentTimeMillis();
    testSerdeForString(new StringSerde());
    long stringSerdeCost = System.currentTimeMillis() - beginTime;
    logger.info("String Serde cost for String: {}", stringSerdeCost);
    beginTime = System.currentTimeMillis();
    Kryo kryo = new Kryo();
    for (int i = 0; i < serdeDataSize; ++i) {
        kryo.writeObject(buffer, "" + random.nextInt(1000));
        buffer.toSlice();
    }
    buffer.release();
    long kryoSerdeCost = System.currentTimeMillis() - beginTime;
    logger.info("Kryo Serde cost for String: {}", kryoSerdeCost);
}

Example 2 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableDSBenchmarkTest method testSpillableMap.

@Test
public void testSpillableMap() {
    byte[] ID1 = new byte[] { (byte) 1 };
    ManagedStateSpillableStateStore store = new ManagedStateSpillableStateStore();
    ((TFileImpl.DTFileImpl) store.getFileAccess()).setBasePath("target/temp");
    StringSerde keySerde = createKeySerde();
    Serde<String> valueSerde = createValueSerde();
    SpillableMapImpl<String, String> map = new SpillableMapImpl<String, String>(store, ID1, 0L, keySerde, valueSerde);
    store.setup(testMeta.operatorContext);
    map.setup(testMeta.operatorContext);
    final long startTime = System.currentTimeMillis();
    long windowId = 0;
    store.beginWindow(++windowId);
    map.beginWindow(windowId);
    int outputTimes = 0;
    for (int i = 0; i < loopCount; ++i) {
        putEntry(map);
        if (i % tuplesPerWindow == 0) {
            map.endWindow();
            store.endWindow();
            if (i % (tuplesPerWindow * checkPointWindows) == 0) {
                store.beforeCheckpoint(windowId);
                if (windowId > commitDelays) {
                    store.committed(windowId - commitDelays);
                }
            }
            // next window
            store.beginWindow(++windowId);
            map.beginWindow(windowId);
        }
        long spentTime = System.currentTimeMillis() - startTime;
        if (spentTime > outputTimes * 5000) {
            ++outputTimes;
            logger.info("Total Statistics: Spent {} mills for {} operation. average/second: {}", spentTime, i, i * 1000 / spentTime);
            checkEnvironment();
        }
    }
    long spentTime = System.currentTimeMillis() - startTime;
    logger.info("Spent {} mills for {} operation. average: {}", spentTime, loopCount, loopCount / spentTime);
}

Example 3 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListImplTest method recoveryManagedStateTest.

@Test
public void recoveryManagedStateTest() {
    SpillableStateStore store = testMeta.store;
    SpillableArrayListImpl<String> list = new SpillableArrayListImpl<>(0L, ID1, store, new StringSerde(), 3);
    store.setup(testMeta.operatorContext);
    list.setup(testMeta.operatorContext);
    long windowId = 0L;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    SpillableTestUtils.checkOutOfBounds(list, 0);
    list.add("a");
    list.addAll(Lists.newArrayList("a", "b", "c", "d", "e", "f", "g"));
    Assert.assertEquals(8, list.size());
    list.endWindow();
    store.endWindow();
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    list.add("tt");
    list.add("ab");
    list.add("99");
    list.add("oo");
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    list.set(1, "111");
    list.set(3, "222");
    list.set(5, "333");
    list.set(11, "444");
    list.endWindow();
    store.endWindow();
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    list.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    long activationWindow = windowId;
    SpillableArrayListImpl<String> clonedList = KryoCloneUtils.cloneObject(list);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    list.set(1, "111111");
    list.set(3, "222222");
    list.add("xyz");
    list.endWindow();
    store.endWindow();
    list.teardown();
    store.teardown();
    Attribute.AttributeMap.DefaultAttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_PATH, testMeta.applicationPath);
    attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, activationWindow);
    OperatorContext context = mockOperatorContext(testMeta.operatorContext.getId(), attributes);
    list = clonedList;
    store = clonedList.getStore();
    store.setup(context);
    list.setup(context);
    windowId = activationWindow + 1L;
    store.beginWindow(windowId);
    list.beginWindow(windowId);
    Assert.assertEquals("a", list.get(0));
    Assert.assertEquals("111", list.get(1));
    Assert.assertEquals("b", list.get(2));
    Assert.assertEquals("222", list.get(3));
    Assert.assertEquals("d", list.get(4));
    Assert.assertEquals("333", list.get(5));
    Assert.assertEquals("f", list.get(6));
    Assert.assertEquals("g", list.get(7));
    Assert.assertEquals("tt", list.get(8));
    Assert.assertEquals("ab", list.get(9));
    Assert.assertEquals("99", list.get(10));
    Assert.assertEquals("444", list.get(11));
    Assert.assertEquals(12, list.size());
    list.endWindow();
    store.endWindow();
    list.teardown();
    store.teardown();
}

Example 4 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListImplTest method simpleMultiListTestHelper.

public void simpleMultiListTestHelper(SpillableStateStore store) {
    SpillableArrayListImpl<String> list1 = new SpillableArrayListImpl<>(0L, ID1, store, new StringSerde(), 1);
    SpillableArrayListImpl<String> list2 = new SpillableArrayListImpl<>(0L, ID2, store, new StringSerde(), 1);
    store.setup(testMeta.operatorContext);
    list1.setup(testMeta.operatorContext);
    list2.setup(testMeta.operatorContext);
    long windowId = 0L;
    store.beginWindow(windowId);
    list1.beginWindow(windowId);
    list2.beginWindow(windowId);
    SpillableTestUtils.checkOutOfBounds(list1, 0);
    Assert.assertEquals(0, list1.size());
    list1.add("a");
    SpillableTestUtils.checkOutOfBounds(list2, 0);
    list2.add("2a");
    SpillableTestUtils.checkOutOfBounds(list1, 1);
    SpillableTestUtils.checkOutOfBounds(list2, 1);
    Assert.assertEquals(1, list1.size());
    Assert.assertEquals(1, list2.size());
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("2a", list2.get(0));
    list1.addAll(Lists.newArrayList("a", "b", "c"));
    list2.addAll(Lists.newArrayList("2a", "2b"));
    Assert.assertEquals(4, list1.size());
    Assert.assertEquals(3, list2.size());
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("a", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("c", list1.get(3));
    Assert.assertEquals("2a", list2.get(0));
    Assert.assertEquals("2a", list2.get(1));
    Assert.assertEquals("2b", list2.get(2));
    SpillableTestUtils.checkOutOfBounds(list1, 4);
    SpillableTestUtils.checkOutOfBounds(list2, 3);
    list1.endWindow();
    list2.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    SpillableTestUtils.checkValue(store, 0L, ID1, 0, Lists.newArrayList("a"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 1, Lists.newArrayList("a"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 2, Lists.newArrayList("b"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 3, Lists.newArrayList("c"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 0, Lists.newArrayList("2a"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 1, Lists.newArrayList("2a"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 2, Lists.newArrayList("2b"));
    windowId++;
    store.beginWindow(windowId);
    list1.beginWindow(windowId);
    list2.beginWindow(windowId);
    Assert.assertEquals(4, list1.size());
    Assert.assertEquals(3, list2.size());
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("a", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("c", list1.get(3));
    Assert.assertEquals("2a", list2.get(0));
    Assert.assertEquals("2a", list2.get(1));
    Assert.assertEquals("2b", list2.get(2));
    list1.add("tt");
    list1.add("ab");
    list1.add("99");
    list1.add("oo");
    list2.add("2tt");
    list2.add("2ab");
    Assert.assertEquals("tt", list1.get(4));
    Assert.assertEquals("ab", list1.get(5));
    Assert.assertEquals("99", list1.get(6));
    Assert.assertEquals("oo", list1.get(7));
    Assert.assertEquals("2tt", list2.get(3));
    Assert.assertEquals("2ab", list2.get(4));
    list1.set(1, "111");
    list2.set(1, "2111");
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("111", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("c", list1.get(3));
    Assert.assertEquals("tt", list1.get(4));
    Assert.assertEquals("ab", list1.get(5));
    Assert.assertEquals("99", list1.get(6));
    Assert.assertEquals("oo", list1.get(7));
    Assert.assertEquals("2a", list2.get(0));
    Assert.assertEquals("2111", list2.get(1));
    Assert.assertEquals("2b", list2.get(2));
    Assert.assertEquals("2tt", list2.get(3));
    Assert.assertEquals("2ab", list2.get(4));
    list1.endWindow();
    list2.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    windowId++;
    store.beginWindow(windowId);
    list1.beginWindow(windowId);
    list2.beginWindow(windowId);
    SpillableTestUtils.checkValue(store, 0L, ID1, 0, Lists.newArrayList("a"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 1, Lists.newArrayList("111"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 2, Lists.newArrayList("b"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 3, Lists.newArrayList("c"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 4, Lists.newArrayList("tt"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 5, Lists.newArrayList("ab"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 6, Lists.newArrayList("99"));
    SpillableTestUtils.checkValue(store, 0L, ID1, 7, Lists.newArrayList("oo"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 0, Lists.newArrayList("2a"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 1, Lists.newArrayList("2111"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 2, Lists.newArrayList("2b"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 3, Lists.newArrayList("2tt"));
    SpillableTestUtils.checkValue(store, 0L, ID2, 4, Lists.newArrayList("2ab"));
    list1.endWindow();
    list2.endWindow();
    store.endWindow();
    store.beforeCheckpoint(windowId);
    store.checkpointed(windowId);
    store.committed(windowId);
    list1.teardown();
    list2.teardown();
    store.teardown();
}

Example 5 with StringSerde

use of org.apache.apex.malhar.lib.utils.serde.StringSerde in project apex-malhar by apache.

the class SpillableArrayListMultimapImplTest method testLoad.

@Test
public void testLoad() {
    Random random = new Random();
    final int keySize = 1000000;
    final int valueSize = 100000000;
    final int numOfEntry = 100000;
    SpillableStateStore store = testMeta.store;
    SpillableArrayListMultimapImpl<String, String> multimap = new SpillableArrayListMultimapImpl<>(this.testMeta.store, ID1, 0L, new StringSerde(), new StringSerde());
    Attribute.AttributeMap.DefaultAttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_PATH, testMeta.applicationPath);
    OperatorContext context = mockOperatorContext(testMeta.operatorContext.getId(), attributes);
    store.setup(context);
    multimap.setup(context);
    store.beginWindow(1);
    multimap.beginWindow(1);
    for (int i = 0; i < numOfEntry; ++i) {
        multimap.put(String.valueOf(random.nextInt(keySize)), String.valueOf(random.nextInt(valueSize)));
    }
    multimap.endWindow();
    store.endWindow();
}