Examples with ListSerializer org.apache.flink.api.common.typeutils.base.ListSerializer used on opensource projects

Example 11 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by splunk.

the class RocksDbTtlCompactFiltersManager method configCompactFilter.

public void configCompactFilter(@Nonnull StateDescriptor<?, ?> stateDesc, TypeSerializer<?> stateSerializer) {
    StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
    if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
        FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
        Preconditions.checkNotNull(compactionFilterFactory);
        long ttl = ttlConfig.getTtl().toMilliseconds();
        StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy = ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
        Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
        long queryTimeAfterNumEntries = rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();
        FlinkCompactionFilter.Config config;
        if (stateDesc instanceof ListStateDescriptor) {
            TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
            int len = elemSerializer.getLength();
            if (len > 0) {
                config = FlinkCompactionFilter.Config.createForFixedElementList(ttl, queryTimeAfterNumEntries, // plus one byte for list element delimiter
                len + 1);
            } else {
                config = FlinkCompactionFilter.Config.createForList(ttl, queryTimeAfterNumEntries, new ListElementFilterFactory<>(elemSerializer.duplicate()));
            }
        } else if (stateDesc instanceof MapStateDescriptor) {
            config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
        } else {
            config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
        }
        compactionFilterFactory.configure(config);
    }
}

Example 12 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by splunk.

the class WindowRankProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    // compile comparator
    sortKeyComparator = generatedSortKeyComparator.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ListSerializer<RowData> listSerializer = new ListSerializer<>(inputSerializer);
    MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>("window_rank", sortKeySerializer, listSerializer);
    MapState<RowData, List<RowData>> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, mapStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowMapState<>((InternalMapState<RowData, Long, RowData, List<RowData>>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.reuseRankRow = new GenericRowData(1);
    this.currentProgress = Long.MIN_VALUE;
}

Example 13 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by splunk.

the class StreamingRuntimeContextTest method testListStateInstantiation.

@Test
public void testListStateInstantiation() throws Exception {
    final ExecutionConfig config = new ExecutionConfig();
    config.registerKryoType(Path.class);
    final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
    StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);
    ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
    context.getListState(descr);
    ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
    TypeSerializer<?> serializer = descrIntercepted.getSerializer();
    // check that the Path class is really registered, i.e., the execution config was applied
    assertTrue(serializer instanceof ListSerializer);
    TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
    assertTrue(elementSerializer instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}

Example 14 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class ListStateDescriptorTest method testValueStateDescriptorEagerSerializer.

@Test
public void testValueStateDescriptorEagerSerializer() throws Exception {
    TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());
    ListStateDescriptor<String> descr = new ListStateDescriptor<String>("testName", serializer);
    assertEquals("testName", descr.getName());
    assertNotNull(descr.getSerializer());
    assertTrue(descr.getSerializer() instanceof ListSerializer);
    assertNotNull(descr.getElementSerializer());
    assertEquals(serializer, descr.getElementSerializer());
    ListStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr);
    assertEquals("testName", copy.getName());
    assertNotNull(copy.getSerializer());
    assertTrue(copy.getSerializer() instanceof ListSerializer);
    assertNotNull(copy.getElementSerializer());
    assertEquals(serializer, copy.getElementSerializer());
}

Example 15 with ListSerializer

use of org.apache.flink.api.common.typeutils.base.ListSerializer in project flink by apache.

the class ListStateDescriptorTest method testListStateDescriptor.

@Test
public void testListStateDescriptor() throws Exception {
    TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());
    ListStateDescriptor<String> descr = new ListStateDescriptor<>("testName", serializer);
    assertEquals("testName", descr.getName());
    assertNotNull(descr.getSerializer());
    assertTrue(descr.getSerializer() instanceof ListSerializer);
    assertNotNull(descr.getElementSerializer());
    assertEquals(serializer, descr.getElementSerializer());
    ListStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr);
    assertEquals("testName", copy.getName());
    assertNotNull(copy.getSerializer());
    assertTrue(copy.getSerializer() instanceof ListSerializer);
    assertNotNull(copy.getElementSerializer());
    assertEquals(serializer, copy.getElementSerializer());
}