Examples with ListSerializer org.apache.flink.api.common.typeutils.base.ListSerializer used on opensource projects

Example 6 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink-mirror by flink-ci.

the class StreamingRuntimeContextTest method testListStateInstantiation.

@Test
public void testListStateInstantiation() throws Exception {
    final ExecutionConfig config = new ExecutionConfig();
    config.registerKryoType(Path.class);
    final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
    StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);
    ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
    context.getListState(descr);
    ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
    TypeSerializer<?> serializer = descrIntercepted.getSerializer();
    // check that the Path class is really registered, i.e., the execution config was applied
    assertTrue(serializer instanceof ListSerializer);
    TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
    assertTrue(elementSerializer instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}

Example 7 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink-mirror by flink-ci.

the class StateDescriptorPassingTest method validateListStateDescriptorConfigured.

private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
    OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
    WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
    StateDescriptor<?, ?> descr = op.getStateDescriptor();
    assertTrue(descr instanceof ListStateDescriptor);
    ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr;
    // this would be the first statement to fail if state descriptors were not properly
    // initialized
    TypeSerializer<?> serializer = listDescr.getSerializer();
    assertTrue(serializer instanceof ListSerializer);
    TypeSerializer<?> elementSerializer = listDescr.getElementSerializer();
    assertTrue(elementSerializer instanceof KryoSerializer);
    Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo();
    assertTrue("serializer registration was not properly passed on", kryo.getSerializer(File.class) instanceof JavaSerializer);
}

Example 8 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink-mirror by flink-ci.

the class WindowRankProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    // compile comparator
    sortKeyComparator = generatedSortKeyComparator.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ListSerializer<RowData> listSerializer = new ListSerializer<>(inputSerializer);
    MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>("window_rank", sortKeySerializer, listSerializer);
    MapState<RowData, List<RowData>> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, mapStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowMapState<>((InternalMapState<RowData, Long, RowData, List<RowData>>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.reuseRankRow = new GenericRowData(1);
    this.currentProgress = Long.MIN_VALUE;
}

Example 9 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink-mirror by flink-ci.

the class RocksDBListState method migrateSerializedValue.

@Override
public void migrateSerializedValue(DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<List<V>> priorSerializer, TypeSerializer<List<V>> newSerializer) throws StateMigrationException {
    Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
    Preconditions.checkArgument(newSerializer instanceof ListSerializer);
    TypeSerializer<V> priorElementSerializer = ((ListSerializer<V>) priorSerializer).getElementSerializer();
    TypeSerializer<V> newElementSerializer = ((ListSerializer<V>) newSerializer).getElementSerializer();
    try {
        while (serializedOldValueInput.available() > 0) {
            V element = ListDelimitedSerializer.deserializeNextElement(serializedOldValueInput, priorElementSerializer);
            newElementSerializer.serialize(element, serializedMigratedValueOutput);
            if (serializedOldValueInput.available() > 0) {
                serializedMigratedValueOutput.write(DELIMITER);
            }
        }
    } catch (Exception e) {
        throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
    }
}

Example 10 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink-mirror by flink-ci.

the class RocksDbTtlCompactFiltersManager method configCompactFilter.

public void configCompactFilter(@Nonnull StateDescriptor<?, ?> stateDesc, TypeSerializer<?> stateSerializer) {
    StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
    if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
        FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
        Preconditions.checkNotNull(compactionFilterFactory);
        long ttl = ttlConfig.getTtl().toMilliseconds();
        StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy = ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
        Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
        long queryTimeAfterNumEntries = rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();
        FlinkCompactionFilter.Config config;
        if (stateDesc instanceof ListStateDescriptor) {
            TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
            int len = elemSerializer.getLength();
            if (len > 0) {
                config = FlinkCompactionFilter.Config.createForFixedElementList(ttl, queryTimeAfterNumEntries, // plus one byte for list element delimiter
                len + 1);
            } else {
                config = FlinkCompactionFilter.Config.createForList(ttl, queryTimeAfterNumEntries, new ListElementFilterFactory<>(elemSerializer.duplicate()));
            }
        } else if (stateDesc instanceof MapStateDescriptor) {
            config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
        } else {
            config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
        }
        compactionFilterFactory.configure(config);
    }
}