Examples with StorageAgent com.datatorrent.api.StorageAgent used on opensource projects

Example 6 with StorageAgent

use of com.datatorrent.api.StorageAgent in project apex-core by apache.

the class PhysicalPlan method syncCheckpoints.

/**
   * Read available checkpoints from storage agent for all operators.
   * @param startTime
   * @param currentTime
   * @throws IOException
   */
public void syncCheckpoints(long startTime, long currentTime) throws IOException {
    for (PTOperator oper : getAllOperators().values()) {
        StorageAgent sa = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
        long[] windowIds = sa.getWindowIds(oper.getId());
        Arrays.sort(windowIds);
        oper.checkpoints.clear();
        for (long wid : windowIds) {
            if (wid != Stateless.WINDOW_ID) {
                oper.addCheckpoint(wid, startTime);
            }
        }
    }
}

Example 7 with StorageAgent

use of com.datatorrent.api.StorageAgent in project apex-core by apache.

the class PhysicalPlan method initCheckpoint.

private void initCheckpoint(PTOperator oper, Operator oo, Checkpoint checkpoint) {
    try {
        LOG.debug("Writing activation checkpoint {} {} {}", checkpoint, oper, oo);
        long windowId = oper.isOperatorStateLess() ? Stateless.WINDOW_ID : checkpoint.windowId;
        StorageAgent agent = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
        agent.save(oo, oper.id, windowId);
        if (agent instanceof AsyncStorageAgent) {
            ((AsyncStorageAgent) agent).flush(oper.id, windowId);
        }
    } catch (IOException e) {
        // inconsistent state, no recovery option, requires shutdown
        throw new IllegalStateException("Failed to write operator state after partition change " + oper, e);
    }
    oper.setRecoveryCheckpoint(checkpoint);
    if (!Checkpoint.INITIAL_CHECKPOINT.equals(checkpoint)) {
        oper.checkpoints.add(checkpoint);
    }
}