use of io.siddhi.core.util.snapshot.state.StateHolder in project siddhi by wso2.
the class SiddhiAppContext method generateStateHolder.
public StateHolder generateStateHolder(String name, StateFactory stateFactory, boolean unSafe) {
if (stateFactory != null) {
StateHolder stateHolder;
if (unSafe) {
stateHolder = new SingleStateHolder(stateFactory);
} else {
stateHolder = new SingleSyncStateHolder(stateFactory);
}
if (SnapshotService.getSkipStateStorageThreadLocal().get() == null || !SnapshotService.getSkipStateStorageThreadLocal().get()) {
Map<String, StateHolder> stateHolderMap = getSnapshotService().getStateHolderMap(SiddhiConstants.PARTITION_ID_DEFAULT, SiddhiConstants.PARTITION_ID_DEFAULT);
stateHolderMap.put(name + "-" + idGenerator.createNewId(), stateHolder);
}
return stateHolder;
} else {
return new EmptyStateHolder();
}
}
use of io.siddhi.core.util.snapshot.state.StateHolder in project siddhi by wso2.
the class SiddhiQueryContext method generateStateHolder.
public StateHolder generateStateHolder(String name, boolean groupBy, StateFactory stateFactory, boolean unSafe) {
if (stateFactory != null) {
StateHolder stateHolder;
if (unSafe) {
if (partitioned || groupBy) {
stateHolder = new PartitionStateHolder(stateFactory);
} else {
stateHolder = new SingleStateHolder(stateFactory);
}
} else {
if (partitioned || groupBy) {
stateHolder = new PartitionSyncStateHolder(stateFactory);
} else {
stateHolder = new SingleSyncStateHolder(stateFactory);
}
}
if (SnapshotService.getSkipStateStorageThreadLocal().get() == null || !SnapshotService.getSkipStateStorageThreadLocal().get()) {
Map<String, StateHolder> stateHolderMap = siddhiAppContext.getSnapshotService().getStateHolderMap(partitionId, this.getName());
stateHolderMap.put(idGenerator.createNewId() + "-" + name, stateHolder);
}
stateful = true;
return stateHolder;
} else {
return new EmptyStateHolder();
}
}
use of io.siddhi.core.util.snapshot.state.StateHolder in project siddhi by wso2.
the class PartitionRuntimeImpl method initPartition.
public void initPartition() {
PartitionState state = stateHolder.getState();
try {
Long time = state.partitionKeys.get(SiddhiAppContext.getPartitionFlowId());
if (time == null) {
synchronized (state) {
time = state.partitionKeys.get(SiddhiAppContext.getPartitionFlowId());
if (time == null) {
for (QueryRuntime queryRuntime : queryRuntimeList) {
((QueryRuntimeImpl) queryRuntime).initPartition();
}
}
state.partitionKeys.put(SiddhiAppContext.getPartitionFlowId(), siddhiAppContext.getTimestampGenerator().currentTime());
}
} else {
state.partitionKeys.put(SiddhiAppContext.getPartitionFlowId(), siddhiAppContext.getTimestampGenerator().currentTime());
}
} finally {
stateHolder.returnState(state);
}
if (purgingEnabled) {
siddhiAppContext.getScheduledExecutorService().scheduleWithFixedDelay(new Runnable() {
@Override
public void run() {
long currentTime = siddhiAppContext.getTimestampGenerator().currentTime();
PartitionState state = stateHolder.getState();
try {
synchronized (state) {
HashMap<String, Long> partitions = new HashMap<>(state.partitionKeys);
for (Map.Entry<String, Long> partition : partitions.entrySet()) {
if (partition.getValue() + purgeIdlePeriod < currentTime) {
state.partitionKeys.remove(partition.getKey());
SiddhiAppContext.startPartitionFlow(partition.getKey());
try {
for (QueryRuntime queryRuntime : queryRuntimeList) {
Map<String, StateHolder> elementHolderMap = siddhiAppContext.getSnapshotService().getStateHolderMap(partitionName, queryRuntime.getQueryId());
for (StateHolder stateHolder : elementHolderMap.values()) {
stateHolder.cleanGroupByStates();
}
}
} finally {
SiddhiAppContext.stopPartitionFlow();
}
}
}
}
} finally {
stateHolder.returnState(state);
}
}
}, purgeExecutionInterval, purgeExecutionInterval, TimeUnit.MILLISECONDS);
}
}
use of io.siddhi.core.util.snapshot.state.StateHolder in project siddhi by wso2.
the class SnapshotService method incrementalSnapshot.
public IncrementalSnapshot incrementalSnapshot() {
try {
SnapshotRequest.requestForFullSnapshot(false);
Map<String, Map<String, byte[]>> incrementalSnapshotMap = new HashMap<>();
Map<String, Map<String, byte[]>> incrementalBaseSnapshotMap = new HashMap<>();
Map<String, Map<String, byte[]>> periodicSnapshotMap = new HashMap<>();
if (log.isDebugEnabled()) {
log.debug("Taking snapshot ...");
}
try {
threadBarrier.lock();
waitForSystemStabilization();
for (Map.Entry<String, PartitionIdStateHolder> partitionIdState : partitionIdStates.entrySet()) {
for (Map.Entry<String, ElementStateHolder> queryState : partitionIdState.getValue().queryStateHolderMap.entrySet()) {
for (Map.Entry<String, StateHolder> elementState : queryState.getValue().elementHolderMap.entrySet()) {
Map<String, Map<String, State>> partitionKeyStates = elementState.getValue().getAllStates();
try {
for (Map.Entry<String, Map<String, State>> partitionKeyState : partitionKeyStates.entrySet()) {
for (Map.Entry<String, State> groupByKeyState : partitionKeyState.getValue().entrySet()) {
State state = groupByKeyState.getValue();
Map<String, Object> itemStates = state.snapshot();
if (itemStates != null) {
Map<String, Object> itemSnapshotsIncremental = new HashMap<>();
Map<String, Object> itemSnapshotsIncrementalBase = new HashMap<>();
Map<String, Object> itemSnapshotsPeriodic = new HashMap<>();
for (Map.Entry<String, Object> itemState : itemStates.entrySet()) {
if (itemState.getValue() instanceof Snapshot) {
if (((Snapshot) itemState.getValue()).isIncrementalSnapshot()) {
itemSnapshotsIncremental.put(itemState.getKey(), itemState.getValue());
} else {
itemSnapshotsIncrementalBase.put(itemState.getKey(), itemState.getValue());
}
} else {
itemSnapshotsPeriodic.put(itemState.getKey(), itemState.getValue());
}
}
if (!itemSnapshotsIncremental.isEmpty()) {
addToSnapshotIncrements(incrementalSnapshotMap, partitionIdState, queryState, elementState, partitionKeyState, groupByKeyState, itemSnapshotsIncremental);
}
if (!itemSnapshotsIncrementalBase.isEmpty()) {
addToSnapshotIncrements(incrementalBaseSnapshotMap, partitionIdState, queryState, elementState, partitionKeyState, groupByKeyState, itemSnapshotsIncrementalBase);
}
if (!itemSnapshotsPeriodic.isEmpty()) {
addToSnapshotIncrements(periodicSnapshotMap, partitionIdState, queryState, elementState, partitionKeyState, groupByKeyState, itemSnapshotsPeriodic);
}
}
}
}
} finally {
elementState.getValue().returnAllStates(partitionKeyStates);
}
}
}
}
} finally {
threadBarrier.unlock();
}
if (log.isDebugEnabled()) {
log.debug("Snapshot taken for Siddhi app '" + siddhiAppContext.getName() + "'");
}
IncrementalSnapshot snapshot = new IncrementalSnapshot();
if (!incrementalSnapshotMap.isEmpty()) {
snapshot.setIncrementalState(incrementalSnapshotMap);
}
if (!incrementalBaseSnapshotMap.isEmpty()) {
snapshot.setIncrementalStateBase(incrementalBaseSnapshotMap);
}
if (!periodicSnapshotMap.isEmpty()) {
snapshot.setPeriodicState(periodicSnapshotMap);
}
return snapshot;
} finally {
SnapshotRequest.requestForFullSnapshot(false);
}
}
use of io.siddhi.core.util.snapshot.state.StateHolder in project siddhi by wso2.
the class SnapshotService method restore.
public void restore(byte[] snapshot) throws CannotRestoreSiddhiAppStateException {
if (snapshot == null) {
throw new CannotRestoreSiddhiAppStateException("Restoring of Siddhi app " + siddhiAppContext.getName() + " failed due to no snapshot.");
}
Map<String, Map<String, Map<String, Map<String, Map<String, Object>>>>> fullSnapshot = (Map<String, Map<String, Map<String, Map<String, Map<String, Object>>>>>) ByteSerializer.byteToObject(snapshot, siddhiAppContext);
if (fullSnapshot == null) {
throw new CannotRestoreSiddhiAppStateException("Restoring of Siddhi app " + siddhiAppContext.getName() + " failed due to invalid snapshot.");
}
try {
threadBarrier.lock();
waitForSystemStabilization();
try {
// cleaning old group by states
cleanGroupByStates();
// restore data
for (Map.Entry<String, Map<String, Map<String, Map<String, Map<String, Object>>>>> partitionIdSnapshot : fullSnapshot.entrySet()) {
PartitionIdStateHolder partitionStateHolder = partitionIdStates.get(partitionIdSnapshot.getKey());
if (partitionStateHolder == null) {
continue;
}
for (Map.Entry<String, Map<String, Map<String, Map<String, Object>>>> partitionGroupByKeySnapshot : partitionIdSnapshot.getValue().entrySet()) {
for (Map.Entry<String, Map<String, Map<String, Object>>> querySnapshot : partitionGroupByKeySnapshot.getValue().entrySet()) {
ElementStateHolder elementStateHolder = partitionStateHolder.queryStateHolderMap.get(querySnapshot.getKey());
if (elementStateHolder == null) {
continue;
}
for (Map.Entry<String, Map<String, Object>> elementSnapshot : querySnapshot.getValue().entrySet()) {
StateHolder stateHolder = elementStateHolder.elementHolderMap.get(elementSnapshot.getKey());
if (stateHolder == null) {
continue;
}
try {
String partitionKey = null;
String groupByKey = null;
if (partitionGroupByKeySnapshot.getKey() != null) {
String[] keys = partitionGroupByKeySnapshot.getKey().split("--");
if (keys.length == 2) {
if (!keys[0].equals("null")) {
partitionKey = keys[0];
}
if (!keys[1].equals("null")) {
groupByKey = keys[1];
}
}
}
SiddhiAppContext.startPartitionFlow(partitionKey);
SiddhiAppContext.startGroupByFlow(groupByKey);
State state = stateHolder.getState();
try {
if (state == null) {
continue;
}
Map<String, Object> snapshotRestores = new HashMap<>();
for (Map.Entry<String, Object> itemSnapshot : elementSnapshot.getValue().entrySet()) {
if (itemSnapshot.getValue() instanceof Snapshot) {
SnapshotStateList snapshotStateList = new SnapshotStateList();
snapshotStateList.putSnapshotState(0L, (Snapshot) itemSnapshot.getValue());
snapshotRestores.put(itemSnapshot.getKey(), snapshotStateList);
} else {
snapshotRestores.put(itemSnapshot.getKey(), itemSnapshot.getValue());
}
}
state.restore(snapshotRestores);
} finally {
stateHolder.returnState(state);
}
} finally {
SiddhiAppContext.stopPartitionFlow();
SiddhiAppContext.stopGroupByFlow();
}
}
}
}
}
} catch (Throwable t) {
throw new CannotRestoreSiddhiAppStateException("Restoring of Siddhi app " + siddhiAppContext.getName() + " not completed properly because content of Siddhi " + "app has changed since last state persistence. Clean persistence store for a " + "fresh deployment.", t);
}
} finally {
threadBarrier.unlock();
}
}
Aggregations