Examples with ListSerializer org.apache.flink.api.common.typeutils.base.ListSerializer used on opensource projects

Example 16 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class StateDescriptorPassingTest method validateListStateDescriptorConfigured.

private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
    OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
    WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
    StateDescriptor<?, ?> descr = op.getStateDescriptor();
    assertTrue(descr instanceof ListStateDescriptor);
    ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr;
    // this would be the first statement to fail if state descriptors were not properly
    // initialized
    TypeSerializer<?> serializer = listDescr.getSerializer();
    assertTrue(serializer instanceof ListSerializer);
    TypeSerializer<?> elementSerializer = listDescr.getElementSerializer();
    assertTrue(elementSerializer instanceof KryoSerializer);
    Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo();
    assertTrue("serializer registration was not properly passed on", kryo.getSerializer(File.class) instanceof JavaSerializer);
}

Example 17 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class WindowRankProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    // compile comparator
    sortKeyComparator = generatedSortKeyComparator.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ListSerializer<RowData> listSerializer = new ListSerializer<>(inputSerializer);
    MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>("window_rank", sortKeySerializer, listSerializer);
    MapState<RowData, List<RowData>> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, mapStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowMapState<>((InternalMapState<RowData, Long, RowData, List<RowData>>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.reuseRankRow = new GenericRowData(1);
    this.currentProgress = Long.MIN_VALUE;
}

Example 18 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by splunk.

the class ListStateDescriptorTest method testListStateDescriptor.

@Test
public void testListStateDescriptor() throws Exception {
    TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());
    ListStateDescriptor<String> descr = new ListStateDescriptor<>("testName", serializer);
    assertEquals("testName", descr.getName());
    assertNotNull(descr.getSerializer());
    assertTrue(descr.getSerializer() instanceof ListSerializer);
    assertNotNull(descr.getElementSerializer());
    assertEquals(serializer, descr.getElementSerializer());
    ListStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr);
    assertEquals("testName", copy.getName());
    assertNotNull(copy.getSerializer());
    assertTrue(copy.getSerializer() instanceof ListSerializer);
    assertNotNull(copy.getElementSerializer());
    assertEquals(serializer, copy.getElementSerializer());
}

Example 19 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by splunk.

the class HeapListState method getSerializedValue.

@Override
public byte[] getSerializedValue(final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception {
    Preconditions.checkNotNull(serializedKeyAndNamespace);
    Preconditions.checkNotNull(safeKeySerializer);
    Preconditions.checkNotNull(safeNamespaceSerializer);
    Preconditions.checkNotNull(safeValueSerializer);
    Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);
    List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);
    if (result == null) {
        return null;
    }
    final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);
    // write the same as RocksDB writes lists, with one ',' separator
    for (int i = 0; i < result.size(); i++) {
        dupSerializer.serialize(result.get(i), view);
        if (i < result.size() - 1) {
            view.writeByte(',');
        }
    }
    view.flush();
    return baos.toByteArray();
}

Example 20 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by splunk.

the class RocksDBListState method migrateSerializedValue.

@Override
public void migrateSerializedValue(DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<List<V>> priorSerializer, TypeSerializer<List<V>> newSerializer) throws StateMigrationException {
    Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
    Preconditions.checkArgument(newSerializer instanceof ListSerializer);
    TypeSerializer<V> priorElementSerializer = ((ListSerializer<V>) priorSerializer).getElementSerializer();
    TypeSerializer<V> newElementSerializer = ((ListSerializer<V>) newSerializer).getElementSerializer();
    try {
        while (serializedOldValueInput.available() > 0) {
            V element = ListDelimitedSerializer.deserializeNextElement(serializedOldValueInput, priorElementSerializer);
            newElementSerializer.serialize(element, serializedMigratedValueOutput);
            if (serializedOldValueInput.available() > 0) {
                serializedMigratedValueOutput.write(DELIMITER);
            }
        }
    } catch (Exception e) {
        throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
    }
}