Examples with OperatorRequest com.datatorrent.api.StatsListener.OperatorRequest used on opensource projects

Example 1 with OperatorRequest

use of com.datatorrent.api.StatsListener.OperatorRequest in project apex-core by apache.

the class StreamingContainer method processHeartbeatResponse.

public void processHeartbeatResponse(ContainerHeartbeatResponse rsp) {
    if (rsp.nodeRequests != null) {
        nodeRequests = rsp.nodeRequests;
    }
    if (rsp.committedWindowId != lastCommittedWindowId) {
        lastCommittedWindowId = rsp.committedWindowId;
        if (bufferServer != null) {
            // One Window before the committed Window is kept in the Buffer Server, for historical reasons.
            // Jira for that issue is APEXCORE-479
            bufferServer.purge(lastCommittedWindowId - 1);
        }
        OperatorRequest nr = null;
        for (Entry<Integer, Node<?>> e : nodes.entrySet()) {
            final Thread thread = e.getValue().context.getThread();
            if (thread == null || !thread.isAlive()) {
                continue;
            }
            if (e.getValue().getOperator() instanceof Operator.CheckpointListener) {
                if (nr == null) {
                    nr = new OperatorRequest() {

                        @Override
                        public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException {
                            ((Operator.CheckpointListener) operator).committed(lastCommittedWindowId);
                            return null;
                        }
                    };
                }
                e.getValue().context.request(nr);
            }
        }
    }
    if (rsp.undeployRequest != null) {
        logger.info("Undeploy request: {}", rsp.undeployRequest);
        processNodeRequests(false);
        Map<Integer, Node<?>> undeployNodes = undeploy(rsp.undeployRequest);
        undeployNodes.entrySet().removeIf((entry) -> !isReuseOperator(entry.getValue()));
        reuseOpNodes.putAll(undeployNodes);
    }
    if (rsp.shutdown != null) {
        logger.info("Received shutdown request type {}", rsp.shutdown);
        if (rsp.shutdown == StreamingContainerUmbilicalProtocol.ShutdownType.ABORT) {
            processNodeRequests(false);
            this.exitHeartbeatLoop = true;
            return;
        } else if (rsp.shutdown == StreamingContainerUmbilicalProtocol.ShutdownType.WAIT_TERMINATE) {
            stopInputNodes();
        }
    }
    if (rsp.deployRequest != null) {
        logger.info("Deploy request: {}", rsp.deployRequest);
        try {
            deploy(rsp.deployRequest);
        } catch (Exception e) {
            logger.error("deploy request failed", e);
            try {
                umbilical.log(this.containerId, "deploy request failed: " + rsp.deployRequest + " " + ExceptionUtils.getStackTrace(e));
            } catch (IOException ioe) {
                logger.debug("Fail to log", ioe);
            }
            this.exitHeartbeatLoop = true;
            throw new IllegalStateException("Deploy request failed: " + rsp.deployRequest, e);
        }
    }
    processNodeRequests(true);
}

Example 2 with OperatorRequest

use of com.datatorrent.api.StatsListener.OperatorRequest in project apex-core by apache.

the class TupleRecorderCollection method startRecording.

private void startRecording(String id, final Node<?> node, int operatorId, final String portName, long numWindows) {
    PortMappingDescriptor descriptor = node.getPortMappingDescriptor();
    OperatorIdPortNamePair operatorIdPortNamePair = new OperatorIdPortNamePair(operatorId, portName);
    // check any recording conflict
    boolean conflict = false;
    if (containsKey(new OperatorIdPortNamePair(operatorId, null))) {
        conflict = true;
    } else if (portName == null) {
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : descriptor.inputPorts.entrySet()) {
            if (containsKey(new OperatorIdPortNamePair(operatorId, entry.getKey()))) {
                conflict = true;
                break;
            }
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : descriptor.outputPorts.entrySet()) {
            if (containsKey(new OperatorIdPortNamePair(operatorId, entry.getKey()))) {
                conflict = true;
                break;
            }
        }
    } else {
        if (containsKey(operatorIdPortNamePair)) {
            conflict = true;
        }
    }
    if (!conflict) {
        logger.debug("Executing start recording request for {}", operatorIdPortNamePair);
        if (wsClient != null) {
            try {
                wsClient.openConnection();
            } catch (Exception e) {
                logger.warn("Cannot establish websocket connection to uri {}", wsClient.getUri(), e);
            }
        }
        TupleRecorder tupleRecorder = new TupleRecorder(id, appId);
        tupleRecorder.setWebSocketClient(wsClient);
        HashMap<String, Sink<Object>> sinkMap = new HashMap<>();
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : descriptor.inputPorts.entrySet()) {
            String streamId = getDeclaredStreamId(operatorId, entry.getKey());
            if (streamId == null) {
                streamId = portName + "_implicit_stream";
            }
            if (entry.getValue().context != null && (portName == null || entry.getKey().equals(portName))) {
                logger.debug("Adding recorder sink to input port {}, stream {}", entry.getKey(), streamId);
                tupleRecorder.addInputPortInfo(entry.getKey(), streamId);
                sinkMap.put(entry.getKey(), tupleRecorder.newSink(entry.getKey()));
            }
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : descriptor.outputPorts.entrySet()) {
            String streamId = getDeclaredStreamId(operatorId, entry.getKey());
            if (streamId == null) {
                streamId = portName + "_implicit_stream";
            }
            if (portName == null || entry.getKey().equals(portName)) {
                logger.debug("Adding recorder sink to output port {}, stream {}", entry.getKey(), streamId);
                tupleRecorder.addOutputPortInfo(entry.getKey(), streamId);
                sinkMap.put(entry.getKey(), tupleRecorder.newSink(entry.getKey()));
            }
        }
        if (!sinkMap.isEmpty()) {
            logger.debug("Started recording on {} through {}", operatorIdPortNamePair, System.identityHashCode(this));
            String basePath = appPath + "/recordings/" + operatorId + "/" + tupleRecorder.getId();
            tupleRecorder.getStorage().setBasePath(basePath);
            tupleRecorder.getStorage().setBytesPerPartFile(tupleRecordingPartFileSize);
            tupleRecorder.getStorage().setMillisPerPartFile(tupleRecordingPartFileTimeMillis);
            node.addSinks(sinkMap);
            tupleRecorder.setup(node.getOperator(), codecs);
            put(operatorIdPortNamePair, tupleRecorder);
            if (numWindows > 0) {
                tupleRecorder.setNumWindows(numWindows, new Runnable() {

                    @Override
                    public void run() {
                        node.context.request(new OperatorRequest() {

                            @Override
                            public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException {
                                stopRecording(node, operatorId, portName);
                                return null;
                            }
                        });
                    }
                });
            }
        } else {
            logger.warn("Tuple recording request ignored because operator is not connected on the specified port.");
        }
    } else {
        logger.error("Operator id {} is already being recorded.", operatorId);
    }
}

Example 3 with OperatorRequest

use of com.datatorrent.api.StatsListener.OperatorRequest in project apex-core by apache.

the class PhysicalPlan method onStatusUpdate.

public void onStatusUpdate(PTOperator oper) {
    for (StatsListenerWithContext l : oper.statsListeners) {
        final StatsListener.Response rsp = l.processStats(oper.stats, statsListenerContext);
        if (rsp != null) {
            // LOG.debug("Response to processStats = {}", rsp.repartitionRequired);
            oper.loadIndicator = rsp.loadIndicator;
            if (rsp.repartitionRequired) {
                final OperatorMeta om = oper.getOperatorMeta();
                // concurrent heartbeat processing
                if (this.pendingRepartition.putIfAbsent(om, om) != null) {
                    LOG.debug("Skipping repartitioning for {} load {}", oper, oper.loadIndicator);
                } else {
                    LOG.debug("Scheduling repartitioning for {} load {}", oper, oper.loadIndicator);
                    // hand over to monitor thread
                    Runnable r = new Runnable() {

                        @Override
                        public void run() {
                            redoPartitions(logicalToPTOperator.get(om), rsp.repartitionNote);
                            pendingRepartition.remove(om);
                        }
                    };
                    ctx.dispatch(r);
                }
            }
            if (rsp.operatorRequests != null) {
                for (OperatorRequest cmd : rsp.operatorRequests) {
                    StramToNodeRequest request = new StramToNodeRequest();
                    request.operatorId = oper.getId();
                    request.requestType = StramToNodeRequest.RequestType.CUSTOM;
                    request.cmd = cmd;
                    ctx.addOperatorRequest(oper, request);
                }
            }
            // for backward compatibility
            if (rsp.operatorCommands != null) {
                for (@SuppressWarnings("deprecation") com.datatorrent.api.StatsListener.OperatorCommand cmd : rsp.operatorCommands) {
                    StramToNodeRequest request = new StramToNodeRequest();
                    request.operatorId = oper.getId();
                    request.requestType = StramToNodeRequest.RequestType.CUSTOM;
                    OperatorCommandConverter converter = new OperatorCommandConverter();
                    converter.cmd = cmd;
                    request.cmd = converter;
                    ctx.addOperatorRequest(oper, request);
                }
            }
        }
    }
}

Example 4 with OperatorRequest

use of com.datatorrent.api.StatsListener.OperatorRequest in project apex-core by apache.

the class Node method handleRequests.

protected void handleRequests(long windowId) {
    /*
     * we prefer to cater to requests at the end of the window boundary.
     */
    try {
        BlockingQueue<OperatorRequest> requests = context.getRequests();
        int size;
        StatsListener.OperatorResponse response;
        if ((size = requests.size()) > 0) {
            while (size-- > 0) {
                // logger.debug("endwindow: " + t.getWindowId() + " lastprocessed: " + context.getLastProcessedWindowId());
                response = requests.remove().execute(operator, context.getId(), windowId);
                if (response != null) {
                    commandResponse.add(response);
                }
            }
        }
    } catch (Error er) {
        throw er;
    } catch (RuntimeException re) {
        throw re;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 5 with OperatorRequest

use of com.datatorrent.api.StatsListener.OperatorRequest in project apex-core by apache.

the class StreamingContainer method processNodeRequests.

private void processNodeRequests(boolean flagInvalid) {
    for (StramToNodeRequest req : nodeRequests) {
        if (req.isDeleted()) {
            continue;
        }
        if (req instanceof StramToNodeChangeLoggersRequest) {
            handleChangeLoggersRequest((StramToNodeChangeLoggersRequest) req);
            continue;
        }
        Node<?> node = nodes.get(req.getOperatorId());
        if (node == null) {
            logger.warn("Node for operator {} is not found, probably not deployed yet", req.getOperatorId());
            continue;
        }
        Thread thread = node.context.getThread();
        if (thread == null || !thread.isAlive()) {
            if (flagInvalid) {
                logger.warn("Received request with invalid operator id {} ({})", req.getOperatorId(), req);
                req.setDeleted(true);
            }
        } else {
            logger.debug("request received: {}", req);
            OperatorRequest requestExecutor = requestFactory.getRequestExecutor(nodes.get(req.operatorId), req);
            if (requestExecutor != null) {
                node.context.request(requestExecutor);
            } else {
                logger.warn("No executor identified for the request {}", req);
            }
            req.setDeleted(true);
        }
    }
}