Search in sources :

Example 1 with WindowTimerServiceImpl

use of org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl in project flink by apache.

the class WindowJoinOperator method open.

@Override
public void open() throws Exception {
    super.open();
    this.collector = new TimestampedCollector<>(output);
    collector.eraseTimestamp();
    final LongSerializer windowSerializer = LongSerializer.INSTANCE;
    InternalTimerService<Long> internalTimerService = getInternalTimerService("window-timers", windowSerializer, this);
    this.windowTimerService = new WindowTimerServiceImpl(internalTimerService, shiftTimeZone);
    // init join condition
    JoinCondition condition = generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader());
    this.joinCondition = new JoinConditionWithNullFilters(condition, filterNullKeys, this);
    this.joinCondition.setRuntimeContext(getRuntimeContext());
    this.joinCondition.open(new Configuration());
    // init state
    ListStateDescriptor<RowData> leftRecordStateDesc = new ListStateDescriptor<>(LEFT_RECORDS_STATE_NAME, leftSerializer);
    ListState<RowData> leftListState = getOrCreateKeyedState(windowSerializer, leftRecordStateDesc);
    this.leftWindowState = new WindowListState<>((InternalListState<RowData, Long, RowData>) leftListState);
    ListStateDescriptor<RowData> rightRecordStateDesc = new ListStateDescriptor<>(RIGHT_RECORDS_STATE_NAME, rightSerializer);
    ListState<RowData> rightListState = getOrCreateKeyedState(windowSerializer, rightRecordStateDesc);
    this.rightWindowState = new WindowListState<>((InternalListState<RowData, Long, RowData>) rightListState);
    // metrics
    this.leftNumLateRecordsDropped = metrics.counter(LEFT_LATE_ELEMENTS_DROPPED_METRIC_NAME);
    this.leftLateRecordsDroppedRate = metrics.meter(LEFT_LATE_ELEMENTS_DROPPED_RATE_METRIC_NAME, new MeterView(leftNumLateRecordsDropped));
    this.rightNumLateRecordsDropped = metrics.counter(RIGHT_LATE_ELEMENTS_DROPPED_METRIC_NAME);
    this.rightLateRecordsDroppedRate = metrics.meter(RIGHT_LATE_ELEMENTS_DROPPED_RATE_METRIC_NAME, new MeterView(rightNumLateRecordsDropped));
    this.watermarkLatency = metrics.gauge(WATERMARK_LATENCY_METRIC_NAME, () -> {
        long watermark = windowTimerService.currentWatermark();
        if (watermark < 0) {
            return 0L;
        } else {
            return windowTimerService.currentProcessingTime() - watermark;
        }
    });
}
Also used : LongSerializer(org.apache.flink.api.common.typeutils.base.LongSerializer) Configuration(org.apache.flink.configuration.Configuration) ListStateDescriptor(org.apache.flink.api.common.state.ListStateDescriptor) WindowTimerServiceImpl(org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl) MeterView(org.apache.flink.metrics.MeterView) GeneratedJoinCondition(org.apache.flink.table.runtime.generated.GeneratedJoinCondition) JoinCondition(org.apache.flink.table.runtime.generated.JoinCondition) JoinConditionWithNullFilters(org.apache.flink.table.runtime.operators.join.JoinConditionWithNullFilters) GenericRowData(org.apache.flink.table.data.GenericRowData) RowData(org.apache.flink.table.data.RowData) JoinedRowData(org.apache.flink.table.data.utils.JoinedRowData) InternalListState(org.apache.flink.runtime.state.internal.InternalListState)

Example 2 with WindowTimerServiceImpl

use of org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl in project flink by apache.

the class AbstractWindowAggProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ValueState<RowData> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, new ValueStateDescriptor<>("window-aggs", accSerializer));
    this.windowState = new WindowValueState<>((InternalValueState<RowData, Long, RowData>) state);
    this.clockService = ClockService.of(ctx.getTimerService());
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.aggregator = genAggsHandler.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    this.aggregator.open(new PerWindowStateDataViewStore(ctx.getKeyedStateBackend(), namespaceSerializer, ctx.getRuntimeContext()));
    this.windowBuffer = windowBufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, isEventTime, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.currentProgress = Long.MIN_VALUE;
    this.nextTriggerProgress = Long.MIN_VALUE;
}
Also used : RowData(org.apache.flink.table.data.RowData) JoinedRowData(org.apache.flink.table.data.utils.JoinedRowData) InternalValueState(org.apache.flink.runtime.state.internal.InternalValueState) LongSerializer(org.apache.flink.api.common.typeutils.base.LongSerializer) JoinedRowData(org.apache.flink.table.data.utils.JoinedRowData) PerWindowStateDataViewStore(org.apache.flink.table.runtime.dataview.PerWindowStateDataViewStore) WindowTimerServiceImpl(org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl)

Example 3 with WindowTimerServiceImpl

use of org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl in project flink by apache.

the class WindowRankProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    // compile comparator
    sortKeyComparator = generatedSortKeyComparator.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ListSerializer<RowData> listSerializer = new ListSerializer<>(inputSerializer);
    MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>("window_rank", sortKeySerializer, listSerializer);
    MapState<RowData, List<RowData>> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, mapStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowMapState<>((InternalMapState<RowData, Long, RowData, List<RowData>>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.reuseRankRow = new GenericRowData(1);
    this.currentProgress = Long.MIN_VALUE;
}
Also used : ListSerializer(org.apache.flink.api.common.typeutils.base.ListSerializer) LongSerializer(org.apache.flink.api.common.typeutils.base.LongSerializer) MapStateDescriptor(org.apache.flink.api.common.state.MapStateDescriptor) WindowTimerServiceImpl(org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl) GenericRowData(org.apache.flink.table.data.GenericRowData) RowData(org.apache.flink.table.data.RowData) JoinedRowData(org.apache.flink.table.data.utils.JoinedRowData) InternalMapState(org.apache.flink.runtime.state.internal.InternalMapState) JoinedRowData(org.apache.flink.table.data.utils.JoinedRowData) GenericRowData(org.apache.flink.table.data.GenericRowData) ArrayList(java.util.ArrayList) List(java.util.List)

Example 4 with WindowTimerServiceImpl

use of org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl in project flink by apache.

the class RowTimeWindowDeduplicateProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ValueStateDescriptor<RowData> valueStateDescriptor = new ValueStateDescriptor<>("window_deduplicate", inputSerializer);
    ValueState<RowData> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, valueStateDescriptor);
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.windowState = new WindowValueState<>((InternalValueState<RowData, Long, RowData>) state);
    this.windowBuffer = bufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, true, shiftTimeZone);
    this.currentProgress = Long.MIN_VALUE;
}
Also used : ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) RowData(org.apache.flink.table.data.RowData) InternalValueState(org.apache.flink.runtime.state.internal.InternalValueState) LongSerializer(org.apache.flink.api.common.typeutils.base.LongSerializer) WindowTimerServiceImpl(org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl)

Aggregations

LongSerializer (org.apache.flink.api.common.typeutils.base.LongSerializer)4 RowData (org.apache.flink.table.data.RowData)4 WindowTimerServiceImpl (org.apache.flink.table.runtime.operators.window.slicing.WindowTimerServiceImpl)4 JoinedRowData (org.apache.flink.table.data.utils.JoinedRowData)3 InternalValueState (org.apache.flink.runtime.state.internal.InternalValueState)2 GenericRowData (org.apache.flink.table.data.GenericRowData)2 ArrayList (java.util.ArrayList)1 List (java.util.List)1 ListStateDescriptor (org.apache.flink.api.common.state.ListStateDescriptor)1 MapStateDescriptor (org.apache.flink.api.common.state.MapStateDescriptor)1 ValueStateDescriptor (org.apache.flink.api.common.state.ValueStateDescriptor)1 ListSerializer (org.apache.flink.api.common.typeutils.base.ListSerializer)1 Configuration (org.apache.flink.configuration.Configuration)1 MeterView (org.apache.flink.metrics.MeterView)1 InternalListState (org.apache.flink.runtime.state.internal.InternalListState)1 InternalMapState (org.apache.flink.runtime.state.internal.InternalMapState)1 PerWindowStateDataViewStore (org.apache.flink.table.runtime.dataview.PerWindowStateDataViewStore)1 GeneratedJoinCondition (org.apache.flink.table.runtime.generated.GeneratedJoinCondition)1 JoinCondition (org.apache.flink.table.runtime.generated.JoinCondition)1 JoinConditionWithNullFilters (org.apache.flink.table.runtime.operators.join.JoinConditionWithNullFilters)1