use of org.apache.flink.runtime.state.KeyGroupsList in project beam by apache.
the class DedupingOperator method snapshotState.
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
// copy from AbstractStreamOperator
if (getKeyedStateBackend() != null) {
KeyedStateCheckpointOutputStream out;
try {
out = context.getRawKeyedOperatorStateOutput();
} catch (Exception exception) {
throw new Exception("Could not open raw keyed operator state stream for " + getOperatorName() + '.', exception);
}
try {
KeyGroupsList allKeyGroups = out.getKeyGroupList();
for (int keyGroupIdx : allKeyGroups) {
out.startNewKeyGroup(keyGroupIdx);
DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out);
// if (this instanceof KeyGroupCheckpointedOperator)
snapshotKeyGroupState(keyGroupIdx, dov);
}
} catch (Exception exception) {
throw new Exception("Could not write timer service of " + getOperatorName() + " to checkpoint state stream.", exception);
} finally {
try {
out.close();
} catch (Exception closeException) {
LOG.warn("Could not close raw keyed operator state stream for {}. This " + "might have prevented deleting some state data.", getOperatorName(), closeException);
}
}
}
}
use of org.apache.flink.runtime.state.KeyGroupsList in project beam by apache.
the class DoFnOperator method snapshotState.
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
// copy from AbstractStreamOperator
if (getKeyedStateBackend() != null) {
KeyedStateCheckpointOutputStream out;
try {
out = context.getRawKeyedOperatorStateOutput();
} catch (Exception exception) {
throw new Exception("Could not open raw keyed operator state stream for " + getOperatorName() + '.', exception);
}
try {
KeyGroupsList allKeyGroups = out.getKeyGroupList();
for (int keyGroupIdx : allKeyGroups) {
out.startNewKeyGroup(keyGroupIdx);
DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out);
// if (this instanceof KeyGroupCheckpointedOperator)
snapshotKeyGroupState(keyGroupIdx, dov);
// Maybe this is a normal DoFn that has no timerService
if (keyCoder != null) {
timerService.snapshotTimersForKeyGroup(dov, keyGroupIdx);
}
}
} catch (Exception exception) {
throw new Exception("Could not write timer service of " + getOperatorName() + " to checkpoint state stream.", exception);
} finally {
try {
out.close();
} catch (Exception closeException) {
LOG.warn("Could not close raw keyed operator state stream for {}. This " + "might have prevented deleting some state data.", getOperatorName(), closeException);
}
}
}
}
use of org.apache.flink.runtime.state.KeyGroupsList in project flink by apache.
the class AbstractStreamOperator method snapshotState.
/**
* Stream operators with state, which want to participate in a snapshot need to override this hook method.
*
* @param context context that provides information and means required for taking a snapshot
*/
public void snapshotState(StateSnapshotContext context) throws Exception {
if (getKeyedStateBackend() != null) {
KeyedStateCheckpointOutputStream out;
try {
out = context.getRawKeyedOperatorStateOutput();
} catch (Exception exception) {
throw new Exception("Could not open raw keyed operator state stream for " + getOperatorName() + '.', exception);
}
try {
KeyGroupsList allKeyGroups = out.getKeyGroupList();
for (int keyGroupIdx : allKeyGroups) {
out.startNewKeyGroup(keyGroupIdx);
timeServiceManager.snapshotStateForKeyGroup(new DataOutputViewStreamWrapper(out), keyGroupIdx);
}
} catch (Exception exception) {
throw new Exception("Could not write timer service of " + getOperatorName() + " to checkpoint state stream.", exception);
} finally {
try {
out.close();
} catch (Exception closeException) {
LOG.warn("Could not close raw keyed operator state stream for {}. This " + "might have prevented deleting some state data.", getOperatorName(), closeException);
}
}
}
}
use of org.apache.flink.runtime.state.KeyGroupsList in project flink by apache.
the class HeapInternalTimerServiceTest method testKeyGroupStartIndexSetting.
@Test
public void testKeyGroupStartIndexSetting() {
int startKeyGroupIdx = 7;
int endKeyGroupIdx = 21;
KeyGroupsList testKeyGroupList = new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> service = new HeapInternalTimerService<>(testKeyGroupList.getNumberOfKeyGroups(), testKeyGroupList, keyContext, processingTimeService);
Assert.assertEquals(startKeyGroupIdx, service.getLocalKeyGroupRangeStartIdx());
}
use of org.apache.flink.runtime.state.KeyGroupsList in project flink by apache.
the class InternalTimeServiceManagerImpl method snapshotToRawKeyedState.
// //////////////// Fault Tolerance Methods ///////////////////
@Override
public void snapshotToRawKeyedState(KeyedStateCheckpointOutputStream out, String operatorName) throws Exception {
try {
KeyGroupsList allKeyGroups = out.getKeyGroupList();
for (int keyGroupIdx : allKeyGroups) {
out.startNewKeyGroup(keyGroupIdx);
snapshotStateForKeyGroup(new DataOutputViewStreamWrapper(out), keyGroupIdx);
}
} catch (Exception exception) {
throw new Exception("Could not write timer service of " + operatorName + " to checkpoint state stream.", exception);
} finally {
try {
out.close();
} catch (Exception closeException) {
LOG.warn("Could not close raw keyed operator state stream for {}. This " + "might have prevented deleting some state data.", operatorName, closeException);
}
}
}
Aggregations