Examples with ListSerializer org.apache.flink.api.common.typeutils.base.ListSerializer used on opensource projects

Example 1 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class ListStateDescriptorTest method testValueStateDescriptorLazySerializer.

@Test
public void testValueStateDescriptorLazySerializer() throws Exception {
    // some different registered value
    ExecutionConfig cfg = new ExecutionConfig();
    cfg.registerKryoType(TaskInfo.class);
    ListStateDescriptor<Path> descr = new ListStateDescriptor<Path>("testName", Path.class);
    try {
        descr.getSerializer();
        fail("should cause an exception");
    } catch (IllegalStateException ignored) {
    }
    descr.initializeSerializerUnlessSet(cfg);
    assertNotNull(descr.getSerializer());
    assertTrue(descr.getSerializer() instanceof ListSerializer);
    assertNotNull(descr.getElementSerializer());
    assertTrue(descr.getElementSerializer() instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) descr.getElementSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
}

Example 2 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class HeapListState method getSerializedValue.

@Override
public byte[] getSerializedValue(final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception {
    Preconditions.checkNotNull(serializedKeyAndNamespace);
    Preconditions.checkNotNull(safeKeySerializer);
    Preconditions.checkNotNull(safeNamespaceSerializer);
    Preconditions.checkNotNull(safeValueSerializer);
    Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);
    List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);
    if (result == null) {
        return null;
    }
    final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);
    // write the same as RocksDB writes lists, with one ',' separator
    for (int i = 0; i < result.size(); i++) {
        dupSerializer.serialize(result.get(i), view);
        if (i < result.size() - 1) {
            view.writeByte(',');
        }
    }
    view.flush();
    return baos.toByteArray();
}

Example 3 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class RocksDBListState method migrateSerializedValue.

@Override
public void migrateSerializedValue(DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<List<V>> priorSerializer, TypeSerializer<List<V>> newSerializer) throws StateMigrationException {
    Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
    Preconditions.checkArgument(newSerializer instanceof ListSerializer);
    TypeSerializer<V> priorElementSerializer = ((ListSerializer<V>) priorSerializer).getElementSerializer();
    TypeSerializer<V> newElementSerializer = ((ListSerializer<V>) newSerializer).getElementSerializer();
    try {
        while (serializedOldValueInput.available() > 0) {
            V element = ListDelimitedSerializer.deserializeNextElement(serializedOldValueInput, priorElementSerializer);
            newElementSerializer.serialize(element, serializedMigratedValueOutput);
            if (serializedOldValueInput.available() > 0) {
                serializedMigratedValueOutput.write(DELIMITER);
            }
        }
    } catch (Exception e) {
        throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
    }
}

Example 4 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class RocksDbTtlCompactFiltersManager method configCompactFilter.

public void configCompactFilter(@Nonnull StateDescriptor<?, ?> stateDesc, TypeSerializer<?> stateSerializer) {
    StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
    if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
        FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
        Preconditions.checkNotNull(compactionFilterFactory);
        long ttl = ttlConfig.getTtl().toMilliseconds();
        StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy = ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
        Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
        long queryTimeAfterNumEntries = rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();
        FlinkCompactionFilter.Config config;
        if (stateDesc instanceof ListStateDescriptor) {
            TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
            int len = elemSerializer.getLength();
            if (len > 0) {
                config = FlinkCompactionFilter.Config.createForFixedElementList(ttl, queryTimeAfterNumEntries, // plus one byte for list element delimiter
                len + 1);
            } else {
                config = FlinkCompactionFilter.Config.createForList(ttl, queryTimeAfterNumEntries, new ListElementFilterFactory<>(elemSerializer.duplicate()));
            }
        } else if (stateDesc instanceof MapStateDescriptor) {
            config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
        } else {
            config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
        }
        compactionFilterFactory.configure(config);
    }
}

Example 5 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class StreamingRuntimeContextTest method testListStateInstantiation.

@Test
public void testListStateInstantiation() throws Exception {
    final ExecutionConfig config = new ExecutionConfig();
    config.registerKryoType(Path.class);
    final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
    StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);
    ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
    context.getListState(descr);
    ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
    TypeSerializer<?> serializer = descrIntercepted.getSerializer();
    // check that the Path class is really registered, i.e., the execution config was applied
    assertTrue(serializer instanceof ListSerializer);
    TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
    assertTrue(elementSerializer instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}