Examples with UnifierDeployInfo com.datatorrent.stram.api.OperatorDeployInfo.UnifierDeployInfo used on opensource projects

Example 1 with UnifierDeployInfo

use of com.datatorrent.stram.api.OperatorDeployInfo.UnifierDeployInfo in project apex-core by apache.

the class StreamingContainerAgent method createOperatorDeployInfo.

/**
   * Create deploy info for operator.
   * <p>
   *
   * @return {@link com.datatorrent.stram.api.OperatorDeployInfo}
   */
private OperatorDeployInfo createOperatorDeployInfo(PTOperator oper) {
    OperatorDeployInfo ndi;
    if (oper.isUnifier()) {
        UnifierDeployInfo udi = new UnifierDeployInfo();
        /* the constructor auto sets the type */
        try {
            udi.operatorAttributes = oper.getUnifiedOperatorMeta().getAttributes().clone();
        } catch (CloneNotSupportedException ex) {
            throw new RuntimeException("Cannot clone unifier attributes", ex);
        }
        ndi = udi;
    } else {
        ndi = new OperatorDeployInfo();
        Operator operator = oper.getOperatorMeta().getOperator();
        if (operator instanceof InputOperator) {
            ndi.type = OperatorType.INPUT;
            if (!oper.getInputs().isEmpty()) {
                //we check if any input port is connected which would make it a Generic operator.
                for (PTOperator.PTInput ptInput : oper.getInputs()) {
                    if (ptInput.logicalStream != null && ptInput.logicalStream.getSource() != null) {
                        ndi.type = OperatorType.GENERIC;
                        break;
                    }
                }
            }
        } else {
            ndi.type = OperatorType.GENERIC;
        }
    }
    Checkpoint checkpoint = oper.getRecoveryCheckpoint();
    ProcessingMode pm = oper.getOperatorMeta().getValue(OperatorContext.PROCESSING_MODE);
    if (pm == ProcessingMode.AT_MOST_ONCE || pm == ProcessingMode.EXACTLY_ONCE) {
        // TODO: following should be handled in the container at deploy time
        // for exactly once container should also purge previous checkpoint
        // whenever new checkpoint is written.
        StorageAgent agent = oper.getOperatorMeta().getAttributes().get(OperatorContext.STORAGE_AGENT);
        if (agent == null) {
            agent = initCtx.getValue(OperatorContext.STORAGE_AGENT);
        }
        // pick checkpoint most recently written
        try {
            long[] windowIds = agent.getWindowIds(oper.getId());
            long checkpointId = Stateless.WINDOW_ID;
            for (long windowId : windowIds) {
                if (windowId > checkpointId) {
                    checkpointId = windowId;
                }
            }
            if (checkpoint == null || checkpoint.windowId != checkpointId) {
                checkpoint = new Checkpoint(checkpointId, 0, 0);
            }
        } catch (Exception e) {
            throw new RuntimeException("Failed to determine checkpoint window id " + oper, e);
        }
    }
    LOG.debug("{} recovery checkpoint {}", oper, checkpoint);
    ndi.checkpoint = checkpoint;
    ndi.name = oper.getOperatorMeta().getName();
    ndi.id = oper.getId();
    try {
        // clone map before modifying it
        ndi.contextAttributes = oper.getOperatorMeta().getAttributes().clone();
    } catch (CloneNotSupportedException ex) {
        throw new RuntimeException("Cannot clone operator attributes", ex);
    }
    if (oper.isOperatorStateLess()) {
        ndi.contextAttributes.put(OperatorContext.STATELESS, true);
    }
    return ndi;
}

Example 2 with UnifierDeployInfo

use of com.datatorrent.stram.api.OperatorDeployInfo.UnifierDeployInfo in project apex-core by apache.

the class StreamingContainer method deployNodes.

private void deployNodes(List<OperatorDeployInfo> nodeList) throws IOException {
    for (OperatorDeployInfo ndi : nodeList) {
        StorageAgent backupAgent = getValue(OperatorContext.STORAGE_AGENT, ndi);
        assert (backupAgent != null);
        Context parentContext;
        if (ndi instanceof UnifierDeployInfo) {
            OperatorContext unifiedOperatorContext = new OperatorContext(0, ndi.name, ((UnifierDeployInfo) ndi).operatorAttributes, containerContext);
            parentContext = new PortContext(ndi.inputs.get(0).contextAttributes, unifiedOperatorContext);
            massageUnifierDeployInfo(ndi);
        } else {
            parentContext = containerContext;
        }
        OperatorContext ctx = new OperatorContext(ndi.id, ndi.name, ndi.contextAttributes, parentContext);
        ctx.attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, ndi.checkpoint.windowId);
        logger.debug("Restoring operator {} to checkpoint {} stateless={}.", ndi.id, Codec.getStringWindowId(ndi.checkpoint.windowId), ctx.stateless);
        Node<?> node = Node.retrieveNode(backupAgent.load(ndi.id, ctx.stateless ? Stateless.WINDOW_ID : ndi.checkpoint.windowId), ctx, ndi.type);
        node.currentWindowId = ndi.checkpoint.windowId;
        node.applicationWindowCount = ndi.checkpoint.applicationWindowCount;
        node.firstWindowMillis = firstWindowMillis;
        node.windowWidthMillis = windowWidthMillis;
        node.setId(ndi.id);
        nodes.put(ndi.id, node);
        logger.debug("Marking operator {} as deployed.", node);
    }
}