Examples with LongSerializer org.apache.flink.api.common.typeutils.base.LongSerializer used on opensource projects

Example 1 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class MigrationUtils method deserializeComputationStates.

static <T> Queue<ComputationState> deserializeComputationStates(org.apache.flink.cep.nfa.SharedBuffer<T> sharedBuffer, TypeSerializer<T> eventSerializer, DataInputView source) throws IOException {
    Queue<ComputationState> computationStates = new LinkedList<>();
    StringSerializer stateNameSerializer = StringSerializer.INSTANCE;
    LongSerializer timestampSerializer = LongSerializer.INSTANCE;
    DeweyNumber.DeweyNumberSerializer versionSerializer = DeweyNumber.DeweyNumberSerializer.INSTANCE;
    int computationStateNo = source.readInt();
    for (int i = 0; i < computationStateNo; i++) {
        String state = stateNameSerializer.deserialize(source);
        String prevState = stateNameSerializer.deserialize(source);
        long timestamp = timestampSerializer.deserialize(source);
        DeweyNumber version = versionSerializer.deserialize(source);
        long startTimestamp = timestampSerializer.deserialize(source);
        int counter = source.readInt();
        T event = null;
        if (source.readBoolean()) {
            event = eventSerializer.deserialize(source);
        }
        NodeId nodeId;
        EventId startEventId;
        if (prevState != null) {
            nodeId = sharedBuffer.getNodeId(prevState, timestamp, counter, event);
            startEventId = sharedBuffer.getStartEventId(version.getRun());
        } else {
            nodeId = null;
            startEventId = null;
        }
        computationStates.add(ComputationState.createState(state, nodeId, version, startTimestamp, startEventId));
    }
    return computationStates;
}

Example 2 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class StreamTaskNetworkInputTest method testInputStatusAfterEndOfRecovery.

@Test
public void testInputStatusAfterEndOfRecovery() throws Exception {
    int numInputChannels = 2;
    LongSerializer inSerializer = LongSerializer.INSTANCE;
    StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
    DataOutput<Long> output = new NoOpDataOutput<>();
    Map<InputChannelInfo, TestRecordDeserializer> deserializers = createDeserializers(inputGate.getInputGate());
    StreamTaskInput<Long> input = new TestStreamTaskNetworkInput(inputGate, inSerializer, numInputChannels, deserializers);
    inputGate.sendElement(new StreamRecord<>(42L), 0);
    assertThat(input.emitNext(output), equalTo(DataInputStatus.MORE_AVAILABLE));
    inputGate.sendEvent(EndOfChannelStateEvent.INSTANCE, 0);
    assertThat(input.emitNext(output), equalTo(DataInputStatus.MORE_AVAILABLE));
    inputGate.sendEvent(EndOfChannelStateEvent.INSTANCE, 1);
    assertThat(input.emitNext(output), equalTo(DataInputStatus.END_OF_RECOVERY));
}

Example 3 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class WindowJoinOperator method open.

@Override
public void open() throws Exception {
    super.open();
    this.collector = new TimestampedCollector<>(output);
    collector.eraseTimestamp();
    final LongSerializer windowSerializer = LongSerializer.INSTANCE;
    InternalTimerService<Long> internalTimerService = getInternalTimerService("window-timers", windowSerializer, this);
    this.windowTimerService = new WindowTimerServiceImpl(internalTimerService, shiftTimeZone);
    // init join condition
    JoinCondition condition = generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader());
    this.joinCondition = new JoinConditionWithNullFilters(condition, filterNullKeys, this);
    this.joinCondition.setRuntimeContext(getRuntimeContext());
    this.joinCondition.open(new Configuration());
    // init state
    ListStateDescriptor<RowData> leftRecordStateDesc = new ListStateDescriptor<>(LEFT_RECORDS_STATE_NAME, leftSerializer);
    ListState<RowData> leftListState = getOrCreateKeyedState(windowSerializer, leftRecordStateDesc);
    this.leftWindowState = new WindowListState<>((InternalListState<RowData, Long, RowData>) leftListState);
    ListStateDescriptor<RowData> rightRecordStateDesc = new ListStateDescriptor<>(RIGHT_RECORDS_STATE_NAME, rightSerializer);
    ListState<RowData> rightListState = getOrCreateKeyedState(windowSerializer, rightRecordStateDesc);
    this.rightWindowState = new WindowListState<>((InternalListState<RowData, Long, RowData>) rightListState);
    // metrics
    this.leftNumLateRecordsDropped = metrics.counter(LEFT_LATE_ELEMENTS_DROPPED_METRIC_NAME);
    this.leftLateRecordsDroppedRate = metrics.meter(LEFT_LATE_ELEMENTS_DROPPED_RATE_METRIC_NAME, new MeterView(leftNumLateRecordsDropped));
    this.rightNumLateRecordsDropped = metrics.counter(RIGHT_LATE_ELEMENTS_DROPPED_METRIC_NAME);
    this.rightLateRecordsDroppedRate = metrics.meter(RIGHT_LATE_ELEMENTS_DROPPED_RATE_METRIC_NAME, new MeterView(rightNumLateRecordsDropped));
    this.watermarkLatency = metrics.gauge(WATERMARK_LATENCY_METRIC_NAME, () -> {
        long watermark = windowTimerService.currentWatermark();
        if (watermark < 0) {
            return 0L;
        } else {
            return windowTimerService.currentProcessingTime() - watermark;
        }
    });
}

Example 4 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class StateBackendBenchmarkUtils method createHeapKeyedStateBackend.

private static HeapKeyedStateBackend<Long> createHeapKeyedStateBackend(File rootDir) throws IOException {
    File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
    KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
    int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
    ExecutionConfig executionConfig = new ExecutionConfig();
    HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
    HeapKeyedStateBackendBuilder<Long> backendBuilder = new HeapKeyedStateBackendBuilder<>(null, new LongSerializer(), Thread.currentThread().getContextClassLoader(), numberOfKeyGroups, keyGroupRange, executionConfig, TtlTimeProvider.DEFAULT, LatencyTrackingStateConfig.disabled(), Collections.emptyList(), AbstractStateBackend.getCompressionDecorator(executionConfig), new LocalRecoveryConfig(null), priorityQueueSetFactory, false, new CloseableRegistry());
    return backendBuilder.build();
}

Example 5 with LongSerializer

use of org.apache.flink.api.common.typeutils.base.LongSerializer in project flink by apache.

the class AbstractWindowAggProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ValueState<RowData> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, new ValueStateDescriptor<>("window-aggs", accSerializer));
    this.windowState = new WindowValueState<>((InternalValueState<RowData, Long, RowData>) state);
    this.clockService = ClockService.of(ctx.getTimerService());
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.aggregator = genAggsHandler.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    this.aggregator.open(new PerWindowStateDataViewStore(ctx.getKeyedStateBackend(), namespaceSerializer, ctx.getRuntimeContext()));
    this.windowBuffer = windowBufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, isEventTime, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.currentProgress = Long.MIN_VALUE;
    this.nextTriggerProgress = Long.MIN_VALUE;
}