Examples with LongSerializer org.apache.flink.api.common.typeutils.base.LongSerializer used on opensource projects

Example 6 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class StreamTaskNetworkInputTest method testSnapshotAfterEndOfPartition.

@Test
public void testSnapshotAfterEndOfPartition() throws Exception {
    int numInputChannels = 1;
    int channelId = 0;
    int checkpointId = 0;
    VerifyRecordsDataOutput<Long> output = new VerifyRecordsDataOutput<>();
    LongSerializer inSerializer = LongSerializer.INSTANCE;
    StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
    StreamTaskInput<Long> input = new StreamTaskNetworkInput<>(new CheckpointedInputGate(inputGate.getInputGate(), SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", new DummyCheckpointInvokable(), SystemClock.getInstance(), false, inputGate.getInputGate()), new SyncMailboxExecutor()), inSerializer, ioManager, new StatusWatermarkValve(numInputChannels), 0);
    inputGate.sendEvent(new CheckpointBarrier(checkpointId, 0L, CheckpointOptions.forCheckpointWithDefaultLocation().toUnaligned()), channelId);
    inputGate.sendElement(new StreamRecord<>(42L), channelId);
    assertHasNextElement(input, output);
    assertHasNextElement(input, output);
    assertEquals(1, output.getNumberOfEmittedRecords());
    // send EndOfPartitionEvent and ensure that deserializer has been released
    inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelId);
    input.emitNext(output);
    // now snapshot all inflight buffers
    CompletableFuture<Void> completableFuture = input.prepareSnapshot(ChannelStateWriter.NO_OP, checkpointId);
    completableFuture.join();
}

Example 7 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class StreamTaskNetworkInputTest method testReleasingDeserializerTimely.

@Test
public void testReleasingDeserializerTimely() throws Exception {
    int numInputChannels = 2;
    LongSerializer inSerializer = LongSerializer.INSTANCE;
    StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
    DataOutput<Long> output = new NoOpDataOutput<>();
    Map<InputChannelInfo, TestRecordDeserializer> deserializers = createDeserializers(inputGate.getInputGate());
    Map<InputChannelInfo, TestRecordDeserializer> copiedDeserializers = new HashMap<>(deserializers);
    StreamTaskInput<Long> input = new TestStreamTaskNetworkInput(inputGate, inSerializer, numInputChannels, deserializers);
    for (InputChannelInfo channelInfo : inputGate.getInputGate().getChannelInfos()) {
        assertNotNull(deserializers.get(channelInfo));
        inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelInfo.getInputChannelIdx());
        input.emitNext(output);
        assertTrue(copiedDeserializers.get(channelInfo).isCleared());
        assertNull(deserializers.get(channelInfo));
    }
}

Example 8 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class WindowRankProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    // compile comparator
    sortKeyComparator = generatedSortKeyComparator.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ListSerializer<RowData> listSerializer = new ListSerializer<>(inputSerializer);
    MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>("window_rank", sortKeySerializer, listSerializer);
    MapState<RowData, List<RowData>> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, mapStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowMapState<>((InternalMapState<RowData, Long, RowData, List<RowData>>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.reuseRankRow = new GenericRowData(1);
    this.currentProgress = Long.MIN_VALUE;
}

Example 9 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class RowTimeWindowDeduplicateProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ValueStateDescriptor<RowData> valueStateDescriptor = new ValueStateDescriptor<>("window_deduplicate", inputSerializer);
    ValueState<RowData> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, valueStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowValueState<>((InternalValueState<RowData, Long, RowData>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.currentProgress = Long.MIN_VALUE;
}