Examples with AsyncStorageAgent org.apache.apex.common.util.AsyncStorageAgent used on opensource projects

Example 1 with AsyncStorageAgent

use of org.apache.apex.common.util.AsyncStorageAgent in project apex-core by apache.

the class Node method checkpoint.

void checkpoint(long windowId) {
    if (!context.stateless) {
        if (operator instanceof Operator.CheckpointNotificationListener) {
            ((Operator.CheckpointNotificationListener) operator).beforeCheckpoint(windowId);
        }
        StorageAgent ba = context.getValue(OperatorContext.STORAGE_AGENT);
        if (ba != null) {
            try {
                checkpointStats = new Stats.CheckpointStats();
                checkpointStats.checkpointStartTime = System.currentTimeMillis();
                ba.save(operator, id, windowId);
                if (ba instanceof AsyncStorageAgent) {
                    AsyncStorageAgent asyncStorageAgent = (AsyncStorageAgent) ba;
                    if (!asyncStorageAgent.isSyncCheckpoint()) {
                        if (PROCESSING_MODE != ProcessingMode.EXACTLY_ONCE) {
                            CheckpointWindowInfo checkpointWindowInfo = new CheckpointWindowInfo();
                            checkpointWindowInfo.windowId = windowId;
                            checkpointWindowInfo.applicationWindowCount = applicationWindowCount;
                            checkpointWindowInfo.checkpointWindowCount = checkpointWindowCount;
                            CheckpointHandler checkpointHandler = new CheckpointHandler();
                            checkpointHandler.agent = asyncStorageAgent;
                            checkpointHandler.operatorId = id;
                            checkpointHandler.windowId = windowId;
                            checkpointHandler.stats = checkpointStats;
                            FutureTask<Stats.CheckpointStats> futureTask = new FutureTask<>(checkpointHandler);
                            taskQueue.add(new Pair<>(futureTask, checkpointWindowInfo));
                            executorService.submit(futureTask);
                            checkpoint = null;
                            checkpointStats = null;
                            return;
                        } else {
                            asyncStorageAgent.flush(id, windowId);
                        }
                    }
                }
                checkpointStats.checkpointTime = System.currentTimeMillis() - checkpointStats.checkpointStartTime;
            } catch (IOException ie) {
                try {
                    logger.warn("Rolling back checkpoint {} for Operator {} due to the exception {}", Codec.getStringWindowId(windowId), operator, ie);
                    ba.delete(id, windowId);
                } catch (IOException ex) {
                    logger.warn("Error while rolling back checkpoint", ex);
                }
                throw new RuntimeException(ie);
            }
        }
    }
    calculateNextCheckpointWindow();
    dagCheckpointOffsetCount = 0;
    checkpoint = new Checkpoint(windowId, applicationWindowCount, checkpointWindowCount);
    if (operator instanceof Operator.CheckpointListener) {
        ((Operator.CheckpointListener) operator).checkpointed(windowId);
    }
}

Example 2 with AsyncStorageAgent

use of org.apache.apex.common.util.AsyncStorageAgent in project apex-core by apache.

the class PhysicalPlan method initCheckpoint.

private void initCheckpoint(PTOperator oper, Operator oo, Checkpoint checkpoint) {
    try {
        LOG.debug("Writing activation checkpoint {} {} {}", checkpoint, oper, oo);
        long windowId = oper.isOperatorStateLess() ? Stateless.WINDOW_ID : checkpoint.windowId;
        StorageAgent agent = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
        agent.save(oo, oper.id, windowId);
        if (agent instanceof AsyncStorageAgent) {
            ((AsyncStorageAgent) agent).flush(oper.id, windowId);
        }
    } catch (IOException e) {
        // inconsistent state, no recovery option, requires shutdown
        throw new IllegalStateException("Failed to write operator state after partition change " + oper, e);
    }
    oper.setRecoveryCheckpoint(checkpoint);
    if (!Checkpoint.INITIAL_CHECKPOINT.equals(checkpoint)) {
        oper.checkpoints.add(checkpoint);
    }
}