Examples with SerializationBuffer org.apache.apex.malhar.lib.utils.serde.SerializationBuffer used on opensource projects

Example 1 with SerializationBuffer

use of org.apache.apex.malhar.lib.utils.serde.SerializationBuffer in project apex-malhar by apache.

the class SpillableMapImplTest method serializationBufferTest.

@Test
@Parameters({ "TimeUnifiedManagedState" })
public void serializationBufferTest(String opt) {
    SerializationBuffer keyBuffer = null;
    SerializationBuffer valueBuffer = null;
    SerializationBuffer currentBuffer;
    setup(opt);
    SpillableMapImplForTest<String, String> map;
    if (te == null) {
        map = new SpillableMapImplForTest<>(store, ID1, 0L, new StringSerde(), new StringSerde());
    } else {
        map = new SpillableMapImplForTest<>(store, ID1, new StringSerde(), new StringSerde(), te);
    }
    store.setup(testMeta.operatorContext);
    map.setup(testMeta.operatorContext);
    long windowId = 0L;
    store.beginWindow(windowId);
    map.beginWindow(windowId);
    map.put("a", "1");
    map.endWindow();
    store.endWindow();
    currentBuffer = map.serdeManager.getKeyBufferForWrite();
    Assert.assertTrue(currentBuffer != keyBuffer);
    keyBuffer = currentBuffer;
    currentBuffer = map.serdeManager.getValueBuffer();
    Assert.assertTrue(currentBuffer != valueBuffer);
    valueBuffer = currentBuffer;
    ++windowId;
    store.beginWindow(windowId);
    map.beginWindow(windowId);
    // each put use different key to make sure use the different bucket
    map.put("b", "2");
    map.endWindow();
    store.endWindow();
    currentBuffer = map.serdeManager.getKeyBufferForWrite();
    Assert.assertTrue(currentBuffer != keyBuffer);
    keyBuffer = currentBuffer;
    currentBuffer = map.serdeManager.getValueBuffer();
    Assert.assertTrue(currentBuffer != valueBuffer);
    valueBuffer = currentBuffer;
    map.teardown();
    store.teardown();
}

Example 2 with SerializationBuffer

use of org.apache.apex.malhar.lib.utils.serde.SerializationBuffer in project apex-malhar by apache.

the class FSWindowDataManager method setup.

@Override
public void setup(Context.OperatorContext context) {
    serializationBuffer = new SerializationBuffer(new WindowedBlockStream());
    operatorId = context.getId();
    if (isStatePathRelativeToAppPath) {
        fullStatePath = context.getValue(DAG.APPLICATION_PATH) + Path.SEPARATOR + statePath;
    } else {
        fullStatePath = statePath;
    }
    try {
        fileContext = FileContextUtils.getFileContext(fullStatePath);
        setupWals(context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 3 with SerializationBuffer

use of org.apache.apex.malhar.lib.utils.serde.SerializationBuffer in project apex-malhar by apache.

the class SpillableArrayListMultimapImplTest method recoveryTestWithManagedState.

@Test
public void recoveryTestWithManagedState() {
    SpillableStateStore store = testMeta.store;
    SpillableArrayListMultimapImpl<String, String> map = new SpillableArrayListMultimapImpl<>(store, ID1, 0L, new StringSerde(), new StringSerde());
    store.setup(testMeta.operatorContext);
    map.setup(testMeta.operatorContext);
    long nextWindowId = 0L;
    nextWindowId = simpleMultiKeyTestHelper(store, map, "a", nextWindowId);
    long activationWindow = nextWindowId;
    store.beforeCheckpoint(nextWindowId);
    SpillableArrayListMultimapImpl<String, String> clonedMap = KryoCloneUtils.cloneObject(map);
    store.checkpointed(nextWindowId);
    store.committed(nextWindowId);
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    List<String> list1 = map.get("a");
    Assert.assertEquals(12, list1.size());
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("111", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("222", list1.get(3));
    Assert.assertEquals("d", list1.get(4));
    Assert.assertEquals("333", list1.get(5));
    Assert.assertEquals("f", list1.get(6));
    Assert.assertEquals("g", list1.get(7));
    Assert.assertEquals("tt", list1.get(8));
    Assert.assertEquals("ab", list1.get(9));
    Assert.assertEquals("99", list1.get(10));
    Assert.assertEquals("444", list1.get(11));
    list1.add("111");
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("111", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("222", list1.get(3));
    Assert.assertEquals("d", list1.get(4));
    Assert.assertEquals("333", list1.get(5));
    Assert.assertEquals("f", list1.get(6));
    Assert.assertEquals("g", list1.get(7));
    Assert.assertEquals("tt", list1.get(8));
    Assert.assertEquals("ab", list1.get(9));
    Assert.assertEquals("99", list1.get(10));
    Assert.assertEquals("444", list1.get(11));
    Assert.assertEquals("111", list1.get(12));
    Assert.assertEquals(13, list1.size());
    map.endWindow();
    store.endWindow();
    map.teardown();
    store.teardown();
    map = clonedMap;
    store = map.getStore();
    Attribute.AttributeMap.DefaultAttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_PATH, testMeta.applicationPath);
    attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, activationWindow);
    OperatorContext context = mockOperatorContext(testMeta.operatorContext.getId(), attributes);
    store.setup(context);
    map.setup(context);
    nextWindowId = activationWindow + 1;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    StringSerde serdeString = new StringSerde();
    SerializationBuffer buffer = new SerializationBuffer(new WindowedBlockStream());
    serdeString.serialize("a", buffer);
    Slice keySlice = buffer.toSlice();
    byte[] keyBytes = SliceUtils.concatenate(ID1, keySlice.toByteArray());
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "111", "b", "222", "d", "333", "f", "g", "tt", "ab", "99", "444"));
    Assert.assertEquals(1, map.size());
    Assert.assertEquals(12, map.get("a").size());
    map.endWindow();
    store.endWindow();
    map.teardown();
    store.teardown();
}

Example 4 with SerializationBuffer

use of org.apache.apex.malhar.lib.utils.serde.SerializationBuffer in project apex-malhar by apache.

the class SpillableArrayListMultimapImplTest method simpleMultiKeyTestHelper.

public long simpleMultiKeyTestHelper(SpillableStateStore store, SpillableArrayListMultimapImpl<String, String> map, String key, long nextWindowId) {
    StringSerde serdeString = new StringSerde();
    IntSerde serdeInt = new IntSerde();
    SerializationBuffer buffer = new SerializationBuffer(new WindowedBlockStream());
    serdeString.serialize(key, buffer);
    Slice keySlice = buffer.toSlice();
    byte[] keyBytes = SliceUtils.concatenate(ID1, keySlice.toByteArray());
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    Assert.assertNull(map.get(key));
    Assert.assertFalse(map.containsKey(key));
    map.put(key, "a");
    Assert.assertTrue(map.containsKey(key));
    List<String> list1 = map.get(key);
    Assert.assertEquals(1, list1.size());
    Assert.assertEquals("a", list1.get(0));
    list1.addAll(Lists.newArrayList("a", "b", "c", "d", "e", "f", "g"));
    Assert.assertEquals(8, list1.size());
    Assert.assertEquals("a", list1.get(0));
    Assert.assertEquals("a", list1.get(1));
    Assert.assertEquals("b", list1.get(2));
    Assert.assertEquals("c", list1.get(3));
    Assert.assertEquals("d", list1.get(4));
    Assert.assertEquals("e", list1.get(5));
    Assert.assertEquals("f", list1.get(6));
    Assert.assertEquals("g", list1.get(7));
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    SpillableTestUtils.checkValue(store, 0L, SliceUtils.concatenate(keyBytes, SpillableArrayListMultimapImpl.SIZE_KEY_SUFFIX), 8, 0, serdeInt);
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "a", "b", "c", "d", "e", "f", "g"));
    List<String> list2 = map.get(key);
    Assert.assertEquals(8, list2.size());
    Assert.assertEquals("a", list2.get(0));
    Assert.assertEquals("a", list2.get(1));
    Assert.assertEquals("b", list2.get(2));
    Assert.assertEquals("c", list2.get(3));
    Assert.assertEquals("d", list2.get(4));
    Assert.assertEquals("e", list2.get(5));
    Assert.assertEquals("f", list2.get(6));
    Assert.assertEquals("g", list2.get(7));
    list2.add("tt");
    list2.add("ab");
    list2.add("99");
    list2.add("oo");
    Assert.assertEquals("tt", list2.get(8));
    Assert.assertEquals("ab", list2.get(9));
    Assert.assertEquals("99", list2.get(10));
    Assert.assertEquals("oo", list2.get(11));
    Assert.assertEquals(12, list2.size());
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    Assert.assertEquals(12, list2.size());
    SpillableTestUtils.checkValue(store, 0L, SliceUtils.concatenate(keyBytes, SpillableArrayListMultimapImpl.SIZE_KEY_SUFFIX), 12, 0, serdeInt);
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "a", "b", "c", "d", "e", "f", "g", "tt", "ab", "99", "oo"));
    List<String> list3 = map.get(key);
    list3.set(1, "111");
    list3.set(3, "222");
    list3.set(5, "333");
    list3.set(11, "444");
    Assert.assertEquals("a", list3.get(0));
    Assert.assertEquals("111", list3.get(1));
    Assert.assertEquals("b", list3.get(2));
    Assert.assertEquals("222", list3.get(3));
    Assert.assertEquals("d", list3.get(4));
    Assert.assertEquals("333", list3.get(5));
    Assert.assertEquals("f", list3.get(6));
    Assert.assertEquals("g", list3.get(7));
    Assert.assertEquals("tt", list3.get(8));
    Assert.assertEquals("ab", list3.get(9));
    Assert.assertEquals("99", list3.get(10));
    Assert.assertEquals("444", list3.get(11));
    Assert.assertEquals(12, list2.size());
    map.endWindow();
    store.endWindow();
    nextWindowId++;
    store.beginWindow(nextWindowId);
    map.beginWindow(nextWindowId);
    SpillableTestUtils.checkValue(store, 0L, SliceUtils.concatenate(keyBytes, SpillableArrayListMultimapImpl.SIZE_KEY_SUFFIX), 12, 0, serdeInt);
    SpillableTestUtils.checkValue(store, 0L, keyBytes, 0, Lists.<String>newArrayList("a", "111", "b", "222", "d", "333", "f", "g", "tt", "ab", "99", "444"));
    map.endWindow();
    store.endWindow();
    return nextWindowId;
}

Example 5 with SerializationBuffer

use of org.apache.apex.malhar.lib.utils.serde.SerializationBuffer in project apex-malhar by apache.

the class SliceBloomFilterTest method testBloomFilterForInt.

public void testBloomFilterForInt(int span) {
    double expectedFalseProbability = 0.3;
    SerializationBuffer buffer = SerializationBuffer.READ_BUFFER;
    SliceBloomFilter bloomFilter = new SliceBloomFilter(loop, expectedFalseProbability);
    for (int i = 0; i < loop; i++) {
        if (i % span == 0) {
            buffer.writeInt(i);
            bloomFilter.put(buffer.toSlice());
        }
    }
    buffer.getWindowedBlockStream().releaseAllFreeMemory();
    int falsePositive = 0;
    for (int i = 0; i < loop; i++) {
        buffer.writeInt(i);
        if (!bloomFilter.mightContain(buffer.toSlice())) {
            Assert.assertTrue(i % span != 0);
        } else {
            // BF says its present
            if (i % 2 != 0) {
                // But was not there
                falsePositive++;
            }
        }
    }
    buffer.getWindowedBlockStream().releaseAllFreeMemory();
    // Verify false positive prob
    double falsePositiveProb = falsePositive;
    falsePositiveProb /= loop;
    Assert.assertTrue(falsePositiveProb <= expectedFalseProbability);
    for (int i = 0; i < loop; i++) {
        if (i % span == 0) {
            buffer.writeInt(i);
            Assert.assertTrue(bloomFilter.mightContain(buffer.toSlice()));
        }
    }
    buffer.getWindowedBlockStream().releaseAllFreeMemory();
}