Examples with Snapshot io.siddhi.core.util.snapshot.state.Snapshot used on opensource projects

Example 11 with Snapshot

use of io.siddhi.core.util.snapshot.state.Snapshot in project siddhi by wso2.

the class SnapshotableEventQueueTestCase method incrementalPersistenceTest1.

@Test
public void incrementalPersistenceTest1() throws InterruptedException, IOException, ClassNotFoundException {
    MetaStreamEvent metaStreamEvent = new MetaStreamEvent();
    metaStreamEvent.addOutputData(new Attribute("symbol", Attribute.Type.STRING));
    metaStreamEvent.addOutputData(new Attribute("price", Attribute.Type.FLOAT));
    metaStreamEvent.addOutputData(new Attribute("volume", Attribute.Type.LONG));
    StreamEventCloner streamEventCloner = new StreamEventCloner(metaStreamEvent, new StreamEventFactory(metaStreamEvent));
    SnapshotableStreamEventQueue snapshotableStreamEventQueue = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
    StreamEvent streamEvent = new StreamEvent(metaStreamEvent.getBeforeWindowData().size(), metaStreamEvent.getOnAfterWindowData().size(), metaStreamEvent.getOutputData().size());
    streamEvent.setOutputData(new Object[] { "IBM", 500.6f, 1 });
    for (int i = 0; i < 20; i++) {
        streamEvent.getOutputData()[2] = i;
        snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
    }
    HashMap<Long, String> snapshots = new HashMap<>();
    Snapshot snapshot1 = snapshotableStreamEventQueue.getSnapshot();
    StreamEvent streamEvents = (StreamEvent) snapshot1.getState();
    Assert.assertTrue(streamEvents != null);
    snapshots.put(3L, toString(snapshot1));
    for (int i = 20; i < 40; i++) {
        streamEvent.getOutputData()[2] = i;
        snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
    }
    Snapshot snapshot2 = snapshotableStreamEventQueue.getSnapshot();
    ArrayList<Operation> operationLog = (ArrayList<Operation>) snapshot2.getState();
    Assert.assertTrue(operationLog != null);
    snapshots.put(4L, toString(snapshot2));
    for (int i = 40; i < 80; i++) {
        streamEvent.getOutputData()[2] = i;
        snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
    }
    Snapshot snapshot3 = snapshotableStreamEventQueue.getSnapshot();
    operationLog = (ArrayList<Operation>) snapshot3.getState();
    Assert.assertTrue(operationLog != null);
    snapshots.put(5L, toString(snapshot3));
    SnapshotableStreamEventQueue snapshotableStreamEventQueue2 = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
    SnapshotStateList snapshotStateList = new SnapshotStateList();
    for (Map.Entry<Long, String> entry : snapshots.entrySet()) {
        snapshotStateList.putSnapshotState(entry.getKey(), (Snapshot) fromString(entry.getValue()));
    }
    snapshotableStreamEventQueue2.restore(snapshotStateList);
    Assert.assertEquals(snapshotableStreamEventQueue, snapshotableStreamEventQueue2);
    for (int i = 80; i < 130; i++) {
        streamEvent.getOutputData()[2] = i;
        snapshotableStreamEventQueue.add(streamEventCloner.copyStreamEvent(streamEvent));
    }
    Snapshot snapshot4 = snapshotableStreamEventQueue.getSnapshot();
    streamEvents = (StreamEvent) snapshot4.getState();
    Assert.assertTrue(streamEvents != null);
    snapshots = new HashMap<>();
    snapshots.put(6L, toString(snapshot4));
    SnapshotableStreamEventQueue snapshotableStreamEventQueue3 = new SnapshotableStreamEventQueue(new StreamEventClonerHolder(streamEventCloner));
    snapshotStateList = new SnapshotStateList();
    for (Map.Entry<Long, String> entry : snapshots.entrySet()) {
        snapshotStateList.putSnapshotState(entry.getKey(), (Snapshot) fromString(entry.getValue()));
    }
    snapshotableStreamEventQueue3.restore(snapshotStateList);
    snapshotableStreamEventQueue.reset();
    Assert.assertEquals(snapshotableStreamEventQueue, snapshotableStreamEventQueue3);
}

Example 12 with Snapshot

use of io.siddhi.core.util.snapshot.state.Snapshot in project siddhi by wso2.

the class SnapshotService method restoreRevision.

public void restoreRevision(String revision) throws CannotRestoreSiddhiAppStateException {
    PersistenceStore persistenceStore = siddhiAppContext.getSiddhiContext().getPersistenceStore();
    IncrementalPersistenceStore incrementalPersistenceStore = siddhiAppContext.getSiddhiContext().getIncrementalPersistenceStore();
    String siddhiAppName = siddhiAppContext.getName();
    if (persistenceStore != null) {
        if (log.isDebugEnabled()) {
            log.debug("Restoring revision: " + revision + " ...");
        }
        byte[] snapshot = persistenceStore.load(siddhiAppContext.getName(), revision);
        if (snapshot != null) {
            restore(snapshot);
            if (log.isDebugEnabled()) {
                log.debug("Restored revision: " + revision);
            }
        } else {
            if (log.isDebugEnabled()) {
                log.debug("No data found for revision: " + revision);
            }
            throw new PersistenceStoreException("No data found for revision: " + revision);
        }
    } else if (incrementalPersistenceStore != null) {
        if (log.isDebugEnabled()) {
            log.debug("Restoring revision: " + revision + " ...");
        }
        IncrementalSnapshotInfo restoreSnapshotInfo = PersistenceHelper.convertRevision(revision);
        List<IncrementalSnapshotInfo> incrementalSnapshotInfos = incrementalPersistenceStore.getListOfRevisionsToLoad(restoreSnapshotInfo.getTime(), restoreSnapshotInfo.getSiddhiAppId());
        if (incrementalSnapshotInfos != null) {
            incrementalSnapshotInfos.sort(new Comparator<IncrementalSnapshotInfo>() {

                @Override
                public int compare(IncrementalSnapshotInfo o1, IncrementalSnapshotInfo o2) {
                    int results = o1.getId().compareTo(o2.getId());
                    if (results == 0) {
                        results = Long.compare(o2.getTime(), o1.getTime());
                        if (results == 0) {
                            return o2.getType().compareTo(o1.getType());
                        }
                    }
                    return results;
                }
            });
            String lastId = null;
            boolean baseFound = false;
            boolean perioicFound = false;
            for (Iterator<IncrementalSnapshotInfo> iterator = incrementalSnapshotInfos.iterator(); iterator.hasNext(); ) {
                IncrementalSnapshotInfo snapshotInfo = iterator.next();
                if (snapshotInfo.getId().equals(lastId)) {
                    if (baseFound && (snapshotInfo.getType() == IncrementalSnapshotInfo.SnapshotType.BASE || snapshotInfo.getType() == IncrementalSnapshotInfo.SnapshotType.INCREMENT)) {
                        iterator.remove();
                    } else if (perioicFound && snapshotInfo.getType() == IncrementalSnapshotInfo.SnapshotType.PERIODIC) {
                        iterator.remove();
                    } else if (snapshotInfo.getType() == IncrementalSnapshotInfo.SnapshotType.BASE) {
                        baseFound = true;
                    } else if (snapshotInfo.getType() == IncrementalSnapshotInfo.SnapshotType.PERIODIC) {
                        perioicFound = true;
                    }
                } else {
                    baseFound = snapshotInfo.getType() == IncrementalSnapshotInfo.SnapshotType.BASE;
                    perioicFound = snapshotInfo.getType() == IncrementalSnapshotInfo.SnapshotType.PERIODIC;
                }
                lastId = snapshotInfo.getId();
            }
            Map<String, Map<String, Map<String, Map<Long, Map<IncrementalSnapshotInfo, byte[]>>>>> incrementalState = new HashMap<>();
            for (IncrementalSnapshotInfo snapshotInfo : incrementalSnapshotInfos) {
                Map<String, Map<String, Map<Long, Map<IncrementalSnapshotInfo, byte[]>>>> incrementalStateByPartitionGroupByKey = incrementalState.computeIfAbsent(snapshotInfo.getPartitionId(), k -> new TreeMap<>());
                Map<String, Map<Long, Map<IncrementalSnapshotInfo, byte[]>>> incrementalStateByTime = incrementalStateByPartitionGroupByKey.computeIfAbsent(snapshotInfo.getPartitionGroupByKey(), k -> new TreeMap<>());
                Map<Long, Map<IncrementalSnapshotInfo, byte[]>> idByTime = incrementalStateByTime.computeIfAbsent(snapshotInfo.getId(), k -> new TreeMap<>());
                Map<IncrementalSnapshotInfo, byte[]> incrementalStateByInfo = idByTime.computeIfAbsent(snapshotInfo.getTime(), k -> new HashMap<>());
                incrementalStateByInfo.put(snapshotInfo, incrementalPersistenceStore.load(snapshotInfo));
            }
            restore(incrementalState);
            if (log.isDebugEnabled()) {
                log.debug("Restored revision: " + revision);
            }
        } else {
            if (log.isDebugEnabled()) {
                log.debug("No data found for revision: " + revision);
            }
            throw new PersistenceStoreException("No data found for revision: " + revision);
        }
    } else {
        throw new NoPersistenceStoreException("No persistence store assigned for siddhi app " + siddhiAppName);
    }
}

Example 13 with Snapshot

use of io.siddhi.core.util.snapshot.state.Snapshot in project siddhi by wso2.

the class SnapshotService method restoreIncrementalSnapshot.

private void restoreIncrementalSnapshot(PartitionIdStateHolder partitionIdStateHolder, Map<String, Map<Long, Map<IncrementalSnapshotInfo, byte[]>>> incrementalStateByTime) {
    if (incrementalStateByTime != null) {
        String id = null;
        State state = null;
        StateHolder stateHolder = null;
        Map<String, Object> deserializedStateMap = null;
        try {
            for (Iterator<Map.Entry<String, Map<Long, Map<IncrementalSnapshotInfo, byte[]>>>> iterator = incrementalStateByTime.entrySet().iterator(); iterator.hasNext(); ) {
                Map.Entry<String, Map<Long, Map<IncrementalSnapshotInfo, byte[]>>> incrementalStateByTimeEntry = iterator.next();
                iterator.remove();
                for (Iterator<Map.Entry<Long, Map<IncrementalSnapshotInfo, byte[]>>> partitionGroupByKeyIterator = incrementalStateByTimeEntry.getValue().entrySet().iterator(); partitionGroupByKeyIterator.hasNext(); ) {
                    Map.Entry<Long, Map<IncrementalSnapshotInfo, byte[]>> partitionGroupByKeyStateByTimeEntry = partitionGroupByKeyIterator.next();
                    partitionGroupByKeyIterator.remove();
                    for (Iterator<Map.Entry<IncrementalSnapshotInfo, byte[]>> iterator1 = partitionGroupByKeyStateByTimeEntry.getValue().entrySet().iterator(); iterator1.hasNext(); ) {
                        Map.Entry<IncrementalSnapshotInfo, byte[]> incrementalStateByInfoEntry = iterator1.next();
                        iterator1.remove();
                        IncrementalSnapshotInfo incrementalSnapshotInfo = incrementalStateByInfoEntry.getKey();
                        Map<String, Object> singleIncrementSnapshot = (Map<String, Object>) ByteSerializer.byteToObject(incrementalStateByInfoEntry.getValue(), siddhiAppContext);
                        if (singleIncrementSnapshot != null) {
                            if (!incrementalSnapshotInfo.getId().equals(id)) {
                                if (id != null) {
                                    state.restore(deserializedStateMap);
                                    SiddhiAppContext.startPartitionFlow(id);
                                    try {
                                        stateHolder.returnState(state);
                                    } finally {
                                        SiddhiAppContext.stopPartitionFlow();
                                    }
                                    id = null;
                                    state = null;
                                    stateHolder = null;
                                    deserializedStateMap = null;
                                }
                                ElementStateHolder elementStateHolder = partitionIdStateHolder.queryStateHolderMap.get(incrementalSnapshotInfo.getQueryName());
                                if (elementStateHolder == null) {
                                    continue;
                                }
                                stateHolder = elementStateHolder.elementHolderMap.get(incrementalSnapshotInfo.getElementId());
                                if (stateHolder == null) {
                                    continue;
                                }
                                String partitionKey = null;
                                String groupByKey = null;
                                String[] keys = incrementalSnapshotInfo.getPartitionGroupByKey().split("--");
                                if (keys.length == 2) {
                                    if (!keys[0].equals("null")) {
                                        partitionKey = keys[0];
                                    }
                                    if (!keys[1].equals("null")) {
                                        groupByKey = keys[1];
                                    }
                                }
                                SiddhiAppContext.startPartitionFlow(partitionKey);
                                SiddhiAppContext.startGroupByFlow(groupByKey);
                                try {
                                    state = stateHolder.getState();
                                } finally {
                                    SiddhiAppContext.stopGroupByFlow();
                                    SiddhiAppContext.stopPartitionFlow();
                                }
                                if (state != null) {
                                    id = incrementalSnapshotInfo.getId();
                                    deserializedStateMap = new HashMap<>();
                                }
                            }
                            if (state != null) {
                                for (Map.Entry<String, Object> singleIncrementSnapshotEntry : singleIncrementSnapshot.entrySet()) {
                                    if (singleIncrementSnapshotEntry.getValue() instanceof Snapshot) {
                                        Snapshot snapshot = (Snapshot) singleIncrementSnapshotEntry.getValue();
                                        SnapshotStateList snapshotStateList = (SnapshotStateList) deserializedStateMap.computeIfAbsent(singleIncrementSnapshotEntry.getKey(), k -> new SnapshotStateList());
                                        if (snapshot.isIncrementalSnapshot()) {
                                            snapshotStateList.putSnapshotState(partitionGroupByKeyStateByTimeEntry.getKey(), snapshot);
                                        } else {
                                            snapshotStateList.getSnapshotStates().clear();
                                            snapshotStateList.putSnapshotState(partitionGroupByKeyStateByTimeEntry.getKey(), snapshot);
                                        }
                                    } else {
                                        deserializedStateMap.put(singleIncrementSnapshotEntry.getKey(), singleIncrementSnapshotEntry.getValue());
                                    }
                                }
                            }
                        }
                    }
                }
                if (id != null) {
                    state.restore(deserializedStateMap);
                    SiddhiAppContext.startPartitionFlow(id);
                    try {
                        stateHolder.returnState(state);
                    } finally {
                        SiddhiAppContext.stopPartitionFlow();
                    }
                    id = null;
                    state = null;
                    stateHolder = null;
                    deserializedStateMap = null;
                }
            }
        } finally {
            if (id != null && stateHolder != null && state != null) {
                SiddhiAppContext.startPartitionFlow(id);
                try {
                    stateHolder.returnState(state);
                } finally {
                    SiddhiAppContext.stopPartitionFlow();
                }
                id = null;
                state = null;
                stateHolder = null;
            }
        }
    }
}

Example 14 with Snapshot

use of io.siddhi.core.util.snapshot.state.Snapshot in project siddhi by wso2.

the class SnapshotService method fullSnapshot.

public byte[] fullSnapshot() {
    try {
        SnapshotRequest.requestForFullSnapshot(true);
        Map<String, Map<String, Map<String, Map<String, Map<String, Object>>>>> fullSnapshot = new HashMap<>();
        byte[] serializedFullState = null;
        if (log.isDebugEnabled()) {
            log.debug("Taking snapshot ...");
        }
        try {
            threadBarrier.lock();
            waitForSystemStabilization();
            for (Map.Entry<String, PartitionIdStateHolder> partitionIdState : partitionIdStates.entrySet()) {
                for (Map.Entry<String, ElementStateHolder> queryState : partitionIdState.getValue().queryStateHolderMap.entrySet()) {
                    for (Map.Entry<String, StateHolder> elementState : queryState.getValue().elementHolderMap.entrySet()) {
                        Map<String, Map<String, State>> partitionKeyStates = elementState.getValue().getAllStates();
                        try {
                            for (Map.Entry<String, Map<String, State>> partitionKeyState : partitionKeyStates.entrySet()) {
                                for (Map.Entry<String, State> groupByKeyState : partitionKeyState.getValue().entrySet()) {
                                    String partitionAndGroupByKey = partitionKeyState.getKey() + "--" + groupByKeyState.getKey();
                                    State state = groupByKeyState.getValue();
                                    Map<String, Object> itemStates = state.snapshot();
                                    if (itemStates != null) {
                                        Map<String, Object> itemSnapshots = new HashMap<>();
                                        for (Map.Entry<String, Object> itemState : itemStates.entrySet()) {
                                            if (itemState.getValue() instanceof Snapshot) {
                                                if (((Snapshot) itemState.getValue()).isIncrementalSnapshot()) {
                                                    throw new NoPersistenceStoreException("No incremental " + "persistence store exist to store incremental " + "snapshot of siddhiApp:'" + siddhiAppContext.getName() + "' subElement:'" + queryState.getKey() + "' elementId:'" + elementState.getKey() + "' partitionKey:'" + partitionKeyState.getKey() + "' groupByKey:'" + groupByKeyState.getKey() + "' and itemKey:'" + itemState.getKey() + "'");
                                                } else {
                                                    itemSnapshots.put(itemState.getKey(), itemState.getValue());
                                                }
                                            } else {
                                                itemSnapshots.put(itemState.getKey(), itemState.getValue());
                                            }
                                        }
                                        Map<String, Map<String, Map<String, Map<String, Object>>>> partitionIdSnapshot = fullSnapshot.computeIfAbsent(partitionIdState.getKey(), k -> new HashMap<>());
                                        Map<String, Map<String, Map<String, Object>>> partitionGroupByKeySnapshot = partitionIdSnapshot.computeIfAbsent(partitionAndGroupByKey, k -> new HashMap<>());
                                        Map<String, Map<String, Object>> querySnapshot = partitionGroupByKeySnapshot.computeIfAbsent(queryState.getKey(), k -> new HashMap<>());
                                        Map<String, Object> elementSnapshot = querySnapshot.get(elementState.getKey());
                                        if (elementSnapshot == null) {
                                            querySnapshot.put(elementState.getKey(), itemSnapshots);
                                        } else {
                                            throw new SiddhiAppRuntimeException("Duplicate state exist for " + "siddhiApp:'" + siddhiAppContext.getName() + "' partitionKey:'" + partitionKeyState.getKey() + "' groupByKey:'" + groupByKeyState.getKey() + "' subElement:'" + queryState.getKey() + "' elementId:'" + elementState.getKey() + "'");
                                        }
                                    }
                                }
                            }
                        } finally {
                            elementState.getValue().returnAllStates(partitionKeyStates);
                        }
                    }
                }
            }
            if (log.isDebugEnabled()) {
                log.debug("Snapshot serialization started ...");
            }
            serializedFullState = ByteSerializer.objectToByte(fullSnapshot, siddhiAppContext);
            if (log.isDebugEnabled()) {
                log.debug("Snapshot serialization finished.");
            }
        } finally {
            threadBarrier.unlock();
        }
        if (log.isDebugEnabled()) {
            log.debug("Snapshot taken for Siddhi app '" + siddhiAppContext.getName() + "'");
        }
        return serializedFullState;
    } finally {
        SnapshotRequest.requestForFullSnapshot(false);
    }
}