Examples with NodeIdentifier org.apache.nifi.cluster.protocol.NodeIdentifier used on opensource projects

Example 46 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class ClusterConnectionIT method testNodeInheritsClusterTopologyOnHeartbeat.

@Test(timeout = 60000)
public void testNodeInheritsClusterTopologyOnHeartbeat() throws InterruptedException {
    final Node node1 = cluster.createNode();
    final Node node2 = cluster.createNode();
    final Node node3 = cluster.createNode();
    cluster.waitUntilAllNodesConnected(10, TimeUnit.SECONDS);
    final Node coordinator = cluster.waitForClusterCoordinator(10, TimeUnit.SECONDS);
    final NodeIdentifier node4NotReallyInCluster = new NodeIdentifier(UUID.randomUUID().toString(), "localhost", 9283, "localhost", 9284, "localhost", 9285, null, false, null);
    final Map<NodeIdentifier, NodeConnectionStatus> replacementStatuses = new HashMap<>();
    replacementStatuses.put(node1.getIdentifier(), new NodeConnectionStatus(node1.getIdentifier(), DisconnectionCode.USER_DISCONNECTED));
    replacementStatuses.put(node4NotReallyInCluster, new NodeConnectionStatus(node4NotReallyInCluster, NodeConnectionState.CONNECTING));
    // reset coordinator status so that other nodes with get its now-fake view of the cluster
    coordinator.getClusterCoordinator().resetNodeStatuses(replacementStatuses);
    final List<NodeConnectionStatus> expectedStatuses = coordinator.getClusterCoordinator().getConnectionStatuses();
    // give nodes a bit to heartbeat in. We need to wait long enough that each node heartbeats.
    // But we need to not wait more than 8 seconds because that's when nodes start getting kicked out.
    Thread.sleep(6000L);
    for (final Node node : new Node[] { node1, node2, node3 }) {
        assertEquals(expectedStatuses, node.getClusterCoordinator().getConnectionStatuses());
    }
}

Example 47 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class UserGroupEntityMergerTest method testMergeAccessPolicy.

@Test
public void testMergeAccessPolicy() throws Exception {
    final NodeIdentifier node1 = new NodeIdentifier("node-1", "host-1", 8080, "host-1", 19998, null, null, null, false);
    final NodeIdentifier node2 = new NodeIdentifier("node-2", "host-2", 8081, "host-2", 19999, null, null, null, false);
    final PermissionsDTO permissed = new PermissionsDTO();
    permissed.setCanRead(true);
    permissed.setCanWrite(true);
    final TenantDTO user1DTO = new TenantDTO();
    user1DTO.setId("user-1");
    final TenantEntity user1Entity = new TenantEntity();
    user1Entity.setPermissions(permissed);
    user1Entity.setId(user1DTO.getId());
    user1Entity.setComponent(user1DTO);
    final TenantDTO user2DTO = new TenantDTO();
    user1DTO.setId("user-2");
    final TenantEntity user2Entity = new TenantEntity();
    user2Entity.setPermissions(permissed);
    user2Entity.setId(user2DTO.getId());
    user2Entity.setComponent(user2DTO);
    final AccessPolicyDTO policy1DTO = new AccessPolicyDTO();
    policy1DTO.setId("policy-1");
    final AccessPolicyEntity policy1Entity = new AccessPolicyEntity();
    policy1Entity.setPermissions(permissed);
    policy1Entity.setId(policy1DTO.getId());
    policy1Entity.setComponent(policy1DTO);
    final AccessPolicyDTO policy2DTO = new AccessPolicyDTO();
    policy2DTO.setId("policy-2");
    final AccessPolicyEntity policy2Entity = new AccessPolicyEntity();
    policy2Entity.setPermissions(permissed);
    policy2Entity.setId(policy2DTO.getId());
    policy2Entity.setComponent(policy2DTO);
    final UserGroupDTO userGroup1DTO = new UserGroupDTO();
    userGroup1DTO.setId("user-1");
    userGroup1DTO.setAccessPolicies(Stream.of(policy1Entity, policy2Entity).collect(Collectors.toSet()));
    userGroup1DTO.setUsers(Stream.of(user2Entity).collect(Collectors.toSet()));
    final UserGroupEntity userGroup1Entity = new UserGroupEntity();
    userGroup1Entity.setPermissions(permissed);
    userGroup1Entity.setId(userGroup1DTO.getId());
    userGroup1Entity.setComponent(userGroup1DTO);
    final UserGroupDTO userGroup2DTO = new UserGroupDTO();
    userGroup2DTO.setId("user-2");
    userGroup2DTO.setAccessPolicies(Stream.of(policy1Entity).collect(Collectors.toSet()));
    userGroup2DTO.setUsers(Stream.of(user1Entity, user2Entity).collect(Collectors.toSet()));
    final UserGroupEntity userGroup2Entity = new UserGroupEntity();
    userGroup2Entity.setPermissions(permissed);
    userGroup2Entity.setId(userGroup2DTO.getId());
    userGroup2Entity.setComponent(userGroup2DTO);
    final Map<NodeIdentifier, UserGroupEntity> nodeMap = new HashMap<>();
    nodeMap.put(node1, userGroup1Entity);
    nodeMap.put(node2, userGroup2Entity);
    final UserGroupEntityMerger merger = new UserGroupEntityMerger();
    merger.merge(userGroup1Entity, nodeMap);
    assertEquals(1, userGroup1DTO.getUsers().size());
    assertTrue(userGroup1DTO.getAccessPolicies().contains(policy1Entity));
    assertEquals(1, userGroup1DTO.getUsers().size());
    assertTrue(userGroup1DTO.getUsers().contains(user2Entity));
}

Example 48 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class SocketProtocolListener method dispatchRequest.

@Override
public void dispatchRequest(final Socket socket) {
    String hostname = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        hostname = socket.getInetAddress().getHostName();
        final String requestId = UUID.randomUUID().toString();
        logger.debug("Received request {} from {}", requestId, hostname);
        String requestorDn = getRequestorDN(socket);
        // unmarshall message
        final ProtocolMessageUnmarshaller<ProtocolMessage> unmarshaller = protocolContext.createUnmarshaller();
        final ByteCountingInputStream countingIn = new ByteCountingInputStream(socket.getInputStream());
        InputStream wrappedInStream = countingIn;
        if (logger.isDebugEnabled()) {
            // don't buffer more than 1 MB of the message
            final int maxMsgBuffer = 1024 * 1024;
            final CopyingInputStream copyingInputStream = new CopyingInputStream(wrappedInStream, maxMsgBuffer);
            wrappedInStream = copyingInputStream;
        }
        final ProtocolMessage request;
        try {
            request = unmarshaller.unmarshal(wrappedInStream);
        } finally {
            if (logger.isDebugEnabled() && wrappedInStream instanceof CopyingInputStream) {
                final CopyingInputStream copyingInputStream = (CopyingInputStream) wrappedInStream;
                byte[] receivedMessage = copyingInputStream.getBytesRead();
                logger.debug("Received message: " + new String(receivedMessage));
            }
        }
        request.setRequestorDN(requestorDn);
        // dispatch message to handler
        ProtocolHandler desiredHandler = null;
        final Collection<ProtocolHandler> handlers = getHandlers();
        for (final ProtocolHandler handler : handlers) {
            if (handler.canHandle(request)) {
                desiredHandler = handler;
                break;
            }
        }
        // if no handler found, throw exception; otherwise handle request
        if (desiredHandler == null) {
            logger.error("Received request of type {} but none of the following Protocol Handlers were able to process the request: {}", request.getType(), handlers);
            throw new ProtocolException("No handler assigned to handle message type: " + request.getType());
        } else {
            final ProtocolMessage response = desiredHandler.handle(request);
            if (response != null) {
                try {
                    logger.debug("Sending response for request {}", requestId);
                    // marshal message to output stream
                    final ProtocolMessageMarshaller<ProtocolMessage> marshaller = protocolContext.createMarshaller();
                    marshaller.marshal(response, socket.getOutputStream());
                } catch (final IOException ioe) {
                    throw new ProtocolException("Failed marshalling protocol message in response to message type: " + request.getType() + " due to " + ioe, ioe);
                }
            }
        }
        stopWatch.stop();
        final NodeIdentifier nodeId = getNodeIdentifier(request);
        final String from = nodeId == null ? hostname : nodeId.toString();
        logger.info("Finished processing request {} (type={}, length={} bytes) from {} in {}", requestId, request.getType(), countingIn.getBytesRead(), from, stopWatch.getDuration());
    } catch (final IOException | ProtocolException e) {
        logger.warn("Failed processing protocol message from " + hostname + " due to " + e, e);
        if (bulletinRepository != null) {
            final Bulletin bulletin = BulletinFactory.createBulletin("Clustering", "WARNING", String.format("Failed to process protocol message from %s due to: %s", hostname, e.toString()));
            bulletinRepository.addBulletin(bulletin);
        }
    }
}

Example 49 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class StandardClusterCoordinationProtocolSender method notifyNodeStatusChange.

@Override
public void notifyNodeStatusChange(final Set<NodeIdentifier> nodesToNotify, final NodeStatusChangeMessage msg) {
    if (nodesToNotify.isEmpty()) {
        return;
    }
    final int numThreads = Math.min(nodesToNotify.size(), maxThreadsPerRequest);
    final byte[] msgBytes;
    try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
        final ProtocolMessageMarshaller<ProtocolMessage> marshaller = protocolContext.createMarshaller();
        marshaller.marshal(msg, baos);
        msgBytes = baos.toByteArray();
    } catch (final IOException e) {
        throw new ProtocolException("Failed to marshal NodeStatusChangeMessage", e);
    }
    final ExecutorService executor = Executors.newFixedThreadPool(numThreads, new ThreadFactory() {

        private final AtomicInteger counter = new AtomicInteger(0);

        @Override
        public Thread newThread(final Runnable r) {
            final Thread thread = Executors.defaultThreadFactory().newThread(r);
            thread.setDaemon(true);
            thread.setName("Notify Cluster of Node Status Change-" + counter.incrementAndGet());
            return thread;
        }
    });
    for (final NodeIdentifier nodeId : nodesToNotify) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try (final Socket socket = createSocket(nodeId, true)) {
                    // marshal message to output stream
                    socket.getOutputStream().write(msgBytes);
                } catch (final IOException ioe) {
                    throw new ProtocolException("Failed to send Node Status Change message to " + nodeId, ioe);
                }
                logger.debug("Notified {} of status change {}", nodeId, msg);
            }
        });
    }
    executor.shutdown();
    try {
        executor.awaitTermination(10, TimeUnit.DAYS);
    } catch (final InterruptedException ie) {
        throw new ProtocolException(ie);
    }
}

Example 50 with NodeIdentifier

use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.

the class TestJaxbProtocolUtils method testRoundTripConnectionStatusResponse.

@Test
public void testRoundTripConnectionStatusResponse() throws JAXBException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final NodeConnectionStatusResponseMessage msg = new NodeConnectionStatusResponseMessage();
    final NodeIdentifier nodeId = new NodeIdentifier("id", "localhost", 8000, "localhost", 8001, "localhost", 8002, 8003, true);
    final NodeConnectionStatus nodeStatus = new NodeConnectionStatus(nodeId, DisconnectionCode.NOT_YET_CONNECTED);
    msg.setNodeConnectionStatus(nodeStatus);
    JaxbProtocolUtils.JAXB_CONTEXT.createMarshaller().marshal(msg, baos);
    final Object unmarshalled = JaxbProtocolUtils.JAXB_CONTEXT.createUnmarshaller().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
    assertTrue(unmarshalled instanceof NodeConnectionStatusResponseMessage);
    final NodeConnectionStatusResponseMessage unmarshalledMsg = (NodeConnectionStatusResponseMessage) unmarshalled;
    final NodeConnectionStatus unmarshalledStatus = unmarshalledMsg.getNodeConnectionStatus();
    assertEquals(nodeStatus, unmarshalledStatus);
}