Examples with NodeIdentifier org.apache.nifi.cluster.protocol.NodeIdentifier used on opensource projects

Example 51 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class TestJaxbProtocolUtils method testRoundTripClusterWorkloadResponse.

@Test
public void testRoundTripClusterWorkloadResponse() throws JAXBException {
    final ClusterWorkloadResponseMessage msg = new ClusterWorkloadResponseMessage();
    final Map<NodeIdentifier, NodeWorkload> expectedNodeWorkloads = new HashMap<>();
    IntStream.range(1, 4).forEach(i -> {
        final String hostname = "node" + i;
        final NodeIdentifier nodeId = new NodeIdentifier(hostname, hostname, 8080, hostname, 8081, hostname, 8082, 8083, false);
        final NodeWorkload workload = new NodeWorkload();
        workload.setReportedTimestamp(System.currentTimeMillis() - 1000);
        workload.setSystemStartTime(System.currentTimeMillis());
        workload.setActiveThreadCount(i);
        workload.setFlowFileCount(i * 10);
        workload.setFlowFileBytes(i * 10 * 1024);
        expectedNodeWorkloads.put(nodeId, workload);
    });
    msg.setNodeWorkloads(expectedNodeWorkloads);
    // Marshall.
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    JaxbProtocolUtils.JAXB_CONTEXT.createMarshaller().marshal(msg, baos);
    // Un-marshall.
    final Object unmarshalled = JaxbProtocolUtils.JAXB_CONTEXT.createUnmarshaller().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
    assertTrue(unmarshalled instanceof ClusterWorkloadResponseMessage);
    // Assert result.
    final ClusterWorkloadResponseMessage response = (ClusterWorkloadResponseMessage) unmarshalled;
    assertEquals(expectedNodeWorkloads.size(), response.getNodeWorkloads().size());
    response.getNodeWorkloads().entrySet().stream().forEach(entry -> {
        assertTrue(expectedNodeWorkloads.containsKey(entry.getKey()));
        final NodeWorkload w = entry.getValue();
        NodeWorkload expectedW = expectedNodeWorkloads.get(entry.getKey());
        assertEquals(expectedW.getActiveThreadCount(), w.getActiveThreadCount());
        assertEquals(expectedW.getReportedTimestamp(), w.getReportedTimestamp());
        assertEquals(expectedW.getSystemStartTime(), w.getSystemStartTime());
        assertEquals(expectedW.getFlowFileBytes(), w.getFlowFileBytes());
        assertEquals(expectedW.getFlowFileCount(), w.getFlowFileCount());
    });
}

Example 52 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class AbstractHeartbeatMonitor method processHeartbeat.

private void processHeartbeat(final NodeHeartbeat heartbeat) {
    final NodeIdentifier nodeId = heartbeat.getNodeIdentifier();
    // Do not process heartbeat if it's blocked by firewall.
    if (clusterCoordinator.isBlockedByFirewall(nodeId.getSocketAddress())) {
        clusterCoordinator.reportEvent(nodeId, Severity.WARNING, "Firewall blocked received heartbeat. Issuing disconnection request.");
        // request node to disconnect
        clusterCoordinator.requestNodeDisconnect(nodeId, DisconnectionCode.BLOCKED_BY_FIREWALL, "Blocked by Firewall");
        removeHeartbeat(nodeId);
        return;
    }
    final NodeConnectionStatus connectionStatus = clusterCoordinator.getConnectionStatus(nodeId);
    if (connectionStatus == null) {
        // Unknown node. Issue reconnect request
        clusterCoordinator.reportEvent(nodeId, Severity.INFO, "Received heartbeat from unknown node. Removing heartbeat and requesting that node connect to cluster.");
        removeHeartbeat(nodeId);
        clusterCoordinator.requestNodeConnect(nodeId, null);
        return;
    }
    final NodeConnectionState connectionState = connectionStatus.getState();
    if (heartbeat.getConnectionStatus().getState() != NodeConnectionState.CONNECTED && connectionState == NodeConnectionState.CONNECTED) {
        // Cluster Coordinator believes that node is connected, but node does not believe so.
        clusterCoordinator.reportEvent(nodeId, Severity.WARNING, "Received heartbeat from node that thinks it is not yet part of the cluster," + "though the Cluster Coordinator thought it was (node claimed state was " + heartbeat.getConnectionStatus().getState() + "). Marking as Disconnected and requesting that Node reconnect to cluster");
        clusterCoordinator.requestNodeConnect(nodeId, null);
        return;
    }
    if (NodeConnectionState.DISCONNECTED == connectionState) {
        // ignore heartbeats from nodes disconnected by means other than lack of heartbeat, unless it is
        // the only node. We allow it if it is the only node because if we have a one-node cluster, then
        // we cannot manually reconnect it.
        final DisconnectionCode disconnectionCode = connectionStatus.getDisconnectCode();
        // Determine whether or not the node should be allowed to be in the cluster still, depending on its reason for disconnection.
        switch(disconnectionCode) {
            case LACK_OF_HEARTBEAT:
            case UNABLE_TO_COMMUNICATE:
            case NOT_YET_CONNECTED:
            case STARTUP_FAILURE:
                {
                    clusterCoordinator.reportEvent(nodeId, Severity.INFO, "Received heartbeat from node previously " + "disconnected due to " + disconnectionCode + ". Issuing reconnection request.");
                    clusterCoordinator.requestNodeConnect(nodeId, null);
                    break;
                }
            default:
                {
                    // disconnected nodes should not heartbeat, so we need to issue a disconnection request.
                    logger.info("Ignoring received heartbeat from disconnected node " + nodeId + ".  Issuing disconnection request.");
                    clusterCoordinator.requestNodeDisconnect(nodeId, disconnectionCode, connectionStatus.getDisconnectReason());
                    removeHeartbeat(nodeId);
                    break;
                }
        }
        return;
    }
    if (NodeConnectionState.DISCONNECTING == connectionStatus.getState()) {
        // ignore spurious heartbeat
        removeHeartbeat(nodeId);
        return;
    }
    // first heartbeat causes status change from connecting to connected
    if (NodeConnectionState.CONNECTING == connectionState) {
        final Long connectionRequestTime = connectionStatus.getConnectionRequestTime();
        if (connectionRequestTime != null && heartbeat.getTimestamp() < connectionRequestTime) {
            clusterCoordinator.reportEvent(nodeId, Severity.INFO, "Received heartbeat but ignoring because it was reported before the node was last asked to reconnect.");
            removeHeartbeat(nodeId);
            return;
        }
        // connection complete
        clusterCoordinator.finishNodeConnection(nodeId);
        clusterCoordinator.reportEvent(nodeId, Severity.INFO, "Received first heartbeat from connecting node. Node connected.");
    }
}

Example 53 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class ClusterProtocolHeartbeatMonitor method onStart.

@Override
public void onStart() {
    // We don't know what the heartbeats look like for the nodes, since we were just elected to monitoring
    // them. However, the map may be filled with old heartbeats. So we clear the heartbeats and populate the
    // map with new heartbeats set to the current time and using the currently known status. We do this so
    // that if we go the required amount of time without receiving a heartbeat, we do know to mark the node
    // as disconnected.
    heartbeatMessages.clear();
    for (final NodeIdentifier nodeId : clusterCoordinator.getNodeIdentifiers()) {
        final NodeHeartbeat heartbeat = new StandardNodeHeartbeat(nodeId, System.currentTimeMillis(), clusterCoordinator.getConnectionStatus(nodeId), 0, 0L, 0, System.currentTimeMillis());
        heartbeatMessages.put(nodeId, heartbeat);
    }
}

Example 54 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class ClusterProtocolHeartbeatMonitor method handleHeartbeat.

private ProtocolMessage handleHeartbeat(final HeartbeatMessage msg) {
    final HeartbeatMessage heartbeatMsg = msg;
    final Heartbeat heartbeat = heartbeatMsg.getHeartbeat();
    final NodeIdentifier nodeId = heartbeat.getNodeIdentifier();
    final NodeConnectionStatus connectionStatus = heartbeat.getConnectionStatus();
    final byte[] payloadBytes = heartbeat.getPayload();
    final HeartbeatPayload payload = HeartbeatPayload.unmarshal(payloadBytes);
    final int activeThreadCount = payload.getActiveThreadCount();
    final int flowFileCount = (int) payload.getTotalFlowFileCount();
    final long flowFileBytes = payload.getTotalFlowFileBytes();
    final long systemStartTime = payload.getSystemStartTime();
    final NodeHeartbeat nodeHeartbeat = new StandardNodeHeartbeat(nodeId, System.currentTimeMillis(), connectionStatus, flowFileCount, flowFileBytes, activeThreadCount, systemStartTime);
    heartbeatMessages.put(heartbeat.getNodeIdentifier(), nodeHeartbeat);
    logger.debug("Received new heartbeat from {}", nodeId);
    // Formulate a List of differences between our view of the cluster topology and the node's view
    // and send that back to the node so that it is in-sync with us
    List<NodeConnectionStatus> nodeStatusList = payload.getClusterStatus();
    if (nodeStatusList == null) {
        nodeStatusList = Collections.emptyList();
    }
    final List<NodeConnectionStatus> updatedStatuses = getUpdatedStatuses(nodeStatusList);
    final HeartbeatResponseMessage responseMessage = new HeartbeatResponseMessage();
    responseMessage.setUpdatedNodeStatuses(updatedStatuses);
    if (!getClusterCoordinator().isFlowElectionComplete()) {
        responseMessage.setFlowElectionMessage(getClusterCoordinator().getFlowElectionStatus());
    }
    return responseMessage;
}

Example 55 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class AbstractSingleDTOEndpoint method merge.

@Override
public final NodeResponse merge(final URI uri, final String method, final Set<NodeResponse> successfulResponses, final Set<NodeResponse> problematicResponses, final NodeResponse clientResponse) {
    if (!canHandle(uri, method)) {
        throw new IllegalArgumentException("Cannot use Endpoint Mapper of type " + getClass().getSimpleName() + " to map responses for URI " + uri + ", HTTP Method " + method);
    }
    final EntityType responseEntity = clientResponse.getClientResponse().readEntity(getEntityClass());
    final DtoType dto = getDto(responseEntity);
    final Map<NodeIdentifier, DtoType> dtoMap = new HashMap<>();
    for (final NodeResponse nodeResponse : successfulResponses) {
        final EntityType nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().readEntity(getEntityClass());
        final DtoType nodeDto = getDto(nodeResponseEntity);
        dtoMap.put(nodeResponse.getNodeId(), nodeDto);
    }
    mergeResponses(dto, dtoMap, successfulResponses, problematicResponses);
    return new NodeResponse(clientResponse, responseEntity);
}