Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 81 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SerializedListAccumulator method add.

public void add(T value, TypeSerializer<T> serializer) throws IOException {
    try {
        ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(outStream);
        serializer.serialize(value, out);
        localValue.add(outStream.toByteArray());
    } catch (IOException e) {
        throw new IOException("Failed to serialize value '" + value + '\'', e);
    }
}

Example 82 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class TypeSerializerSerializationUtilTest method testSerializerAndConfigPairsSerializationWithSerializerDeserializationFailures.

/**
 * Verifies resilience to serializer deserialization failures when writing and reading
 * serializer and config snapshot pairs.
 */
@Test
public void testSerializerAndConfigPairsSerializationWithSerializerDeserializationFailures() throws Exception {
    TestIntSerializer serializer = new TestIntSerializer();
    List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs = Arrays.asList(new Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>(serializer, serializer.snapshotConfiguration()));
    byte[] serializedSerializersAndConfigs;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience(new DataOutputViewStreamWrapper(out), serializersAndConfigs);
        serializedSerializersAndConfigs = out.toByteArray();
    }
    Set<String> cnfThrowingClassnames = new HashSet<>();
    cnfThrowingClassnames.add(TestIntSerializer.class.getName());
    List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> restored;
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedSerializersAndConfigs)) {
        restored = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(new DataInputViewStreamWrapper(in), new ArtificialCNFExceptionThrowingClassLoader(Thread.currentThread().getContextClassLoader(), cnfThrowingClassnames));
    }
    Assert.assertEquals(1, restored.size());
    Assert.assertTrue(restored.get(0).f0 instanceof UnloadableDummyTypeSerializer);
    Assert.assertThat(restored.get(0).f1, Matchers.instanceOf(SimpleTypeSerializerSnapshot.class));
}

Example 83 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class EnumSerializerTest method testConfigurationSnapshotSerialization.

@Test
public void testConfigurationSnapshotSerialization() throws Exception {
    EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class);
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), serializer.snapshotConfiguration(), serializer);
        serializedConfig = out.toByteArray();
    }
    TypeSerializerSnapshot<PublicEnum> restoredConfig;
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), serializer);
    }
    TypeSerializerSchemaCompatibility<PublicEnum> compatResult = restoredConfig.resolveSchemaCompatibility(serializer);
    assertTrue(compatResult.isCompatibleAsIs());
    assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
    assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
    assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
    assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
    assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
    assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
    assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues()));
}

Example 84 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class NFATest method testNFASerialization.

@Test
public void testNFASerialization() throws Exception {
    Pattern<Event, ?> pattern1 = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 1858562682635302605L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("a");
        }
    }).followedByAny("middle").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("b");
        }
    }).oneOrMore().optional().allowCombinations().followedByAny("end").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("d");
        }
    });
    Pattern<Event, ?> pattern2 = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 1858562682635302605L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("a");
        }
    }).notFollowedBy("not").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = -6085237016591726715L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("c");
        }
    }).followedByAny("middle").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("b");
        }
    }).oneOrMore().optional().allowCombinations().followedByAny("end").where(new IterativeCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value, IterativeCondition.Context<Event> ctx) throws Exception {
            double sum = 0.0;
            for (Event e : ctx.getEventsForPattern("middle")) {
                sum += e.getPrice();
            }
            return sum > 5.0;
        }
    });
    Pattern<Event, ?> pattern3 = Pattern.<Event>begin("start").notFollowedBy("not").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = -6085237016591726715L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("c");
        }
    }).followedByAny("middle").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("b");
        }
    }).oneOrMore().allowCombinations().followedByAny("end").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("d");
        }
    });
    List<Pattern<Event, ?>> patterns = new ArrayList<>();
    patterns.add(pattern1);
    patterns.add(pattern2);
    patterns.add(pattern3);
    for (Pattern<Event, ?> p : patterns) {
        NFA<Event> nfa = compile(p, false);
        Event a = new Event(40, "a", 1.0);
        Event b = new Event(41, "b", 2.0);
        Event c = new Event(42, "c", 3.0);
        Event b1 = new Event(41, "b", 3.0);
        Event b2 = new Event(41, "b", 4.0);
        Event b3 = new Event(41, "b", 5.0);
        Event d = new Event(43, "d", 4.0);
        NFAState nfaState = nfa.createInitialNFAState();
        NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build();
        nfaTestHarness.consumeRecord(new StreamRecord<>(a, 1));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b, 2));
        nfaTestHarness.consumeRecord(new StreamRecord<>(c, 3));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b1, 4));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b2, 5));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b3, 6));
        nfaTestHarness.consumeRecord(new StreamRecord<>(d, 7));
        nfaTestHarness.consumeRecord(new StreamRecord<>(a, 8));
        NFAStateSerializer serializer = new NFAStateSerializer();
        // serialize
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        serializer.serialize(nfaState, new DataOutputViewStreamWrapper(baos));
        baos.close();
        // copy
        ByteArrayInputStream in = new ByteArrayInputStream(baos.toByteArray());
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        serializer.duplicate().copy(new DataInputViewStreamWrapper(in), new DataOutputViewStreamWrapper(out));
        in.close();
        out.close();
        // deserialize
        ByteArrayInputStream bais = new ByteArrayInputStream(out.toByteArray());
        NFAState copy = serializer.duplicate().deserialize(new DataInputViewStreamWrapper(bais));
        bais.close();
        assertEquals(nfaState, copy);
    }
}

Example 85 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class ImmutableListStateTest method serializeInitValue.

/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
    TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);
    // write the same as RocksDB writes lists, with one ',' separator
    for (int i = 0; i < toSerialize.size(); i++) {
        serializer.serialize(toSerialize.get(i), view);
        if (i < toSerialize.size() - 1) {
            view.writeByte(',');
        }
    }
    view.flush();
    return baos.toByteArray();
}