Examples with NoClusterCoordinatorException org.apache.nifi.cluster.exception.NoClusterCoordinatorException used on opensource projects

Example 1 with NoClusterCoordinatorException

use of org.apache.nifi.cluster.exception.NoClusterCoordinatorException in project nifi by apache.

the class CuratorNodeProtocolSender method getServiceAddress.

@Override
protected synchronized InetSocketAddress getServiceAddress() throws IOException {
    if (coordinatorAddress != null) {
        return coordinatorAddress;
    }
    final RetryPolicy retryPolicy = new RetryNTimes(0, 0);
    final CuratorFramework curatorClient = CuratorFrameworkFactory.newClient(zkConfig.getConnectString(), zkConfig.getSessionTimeoutMillis(), zkConfig.getConnectionTimeoutMillis(), retryPolicy);
    curatorClient.start();
    try {
        // Get coordinator address and add watcher to change who we are heartbeating to if the value changes.
        final byte[] coordinatorAddressBytes = curatorClient.getData().usingWatcher(new Watcher() {

            @Override
            public void process(final WatchedEvent event) {
                coordinatorAddress = null;
            }
        }).forPath(coordinatorPath);
        if (coordinatorAddressBytes == null || coordinatorAddressBytes.length == 0) {
            throw new NoClusterCoordinatorException("No node has yet been elected Cluster Coordinator. Cannot establish connection to cluster yet.");
        }
        final String address = new String(coordinatorAddressBytes, StandardCharsets.UTF_8);
        final String[] splits = address.split(":");
        if (splits.length != 2) {
            final String message = String.format("Attempted to determine Cluster Coordinator address. Zookeeper indicates " + "that address is %s, but this is not in the expected format of <hostname>:<port>", address);
            logger.error(message);
            throw new ProtocolException(message);
        }
        logger.info("Determined that Cluster Coordinator is located at {}; will use this address for sending heartbeat messages", address);
        final String hostname = splits[0];
        final int port;
        try {
            port = Integer.parseInt(splits[1]);
            if (port < 1 || port > 65535) {
                throw new NumberFormatException("Port must be in the range of 1 - 65535 but got " + port);
            }
        } catch (final NumberFormatException nfe) {
            final String message = String.format("Attempted to determine Cluster Coordinator address. Zookeeper indicates " + "that address is %s, but the port is not a valid port number", address);
            logger.error(message);
            throw new ProtocolException(message);
        }
        final InetSocketAddress socketAddress = InetSocketAddress.createUnresolved(hostname, port);
        coordinatorAddress = socketAddress;
        return socketAddress;
    } catch (final NoNodeException nne) {
        logger.info("No node has yet been elected Cluster Coordinator. Cannot establish connection to cluster yet.");
        throw new NoClusterCoordinatorException("No node has yet been elected Cluster Coordinator. Cannot establish connection to cluster yet.");
    } catch (final NoClusterCoordinatorException ncce) {
        throw ncce;
    } catch (Exception e) {
        throw new IOException("Unable to determine Cluster Coordinator from ZooKeeper", e);
    } finally {
        curatorClient.close();
    }
}

Example 2 with NoClusterCoordinatorException

use of org.apache.nifi.cluster.exception.NoClusterCoordinatorException in project nifi by apache.

the class LeaderElectionNodeProtocolSender method getServiceAddress.

@Override
protected InetSocketAddress getServiceAddress() throws IOException {
    final String address = electionManager.getLeader(ClusterRoles.CLUSTER_COORDINATOR);
    if (StringUtils.isEmpty(address)) {
        throw new NoClusterCoordinatorException("No node has yet been elected Cluster Coordinator. Cannot establish connection to cluster yet.");
    }
    final String[] splits = address.split(":");
    if (splits.length != 2) {
        final String message = String.format("Attempted to determine Cluster Coordinator address. Zookeeper indicates " + "that address is %s, but this is not in the expected format of <hostname>:<port>", address);
        logger.error(message);
        throw new ProtocolException(message);
    }
    logger.info("Determined that Cluster Coordinator is located at {}; will use this address for sending heartbeat messages", address);
    final String hostname = splits[0];
    final int port;
    try {
        port = Integer.parseInt(splits[1]);
        if (port < 1 || port > 65535) {
            throw new NumberFormatException("Port must be in the range of 1 - 65535 but got " + port);
        }
    } catch (final NumberFormatException nfe) {
        final String message = String.format("Attempted to determine Cluster Coordinator address. Zookeeper indicates " + "that address is %s, but the port is not a valid port number", address);
        logger.error(message);
        throw new ProtocolException(message);
    }
    final InetSocketAddress socketAddress = InetSocketAddress.createUnresolved(hostname, port);
    return socketAddress;
}

Example 3 with NoClusterCoordinatorException

use of org.apache.nifi.cluster.exception.NoClusterCoordinatorException in project nifi by apache.

the class NodeClusterCoordinator method getElectedActiveCoordinatorNode.

private NodeIdentifier getElectedActiveCoordinatorNode(final boolean warnOnError) {
    final String electedNodeAddress;
    try {
        electedNodeAddress = getElectedActiveCoordinatorAddress();
    } catch (final NoClusterCoordinatorException ncce) {
        logger.debug("There is currently no elected active Cluster Coordinator");
        return null;
    } catch (final IOException ioe) {
        if (warnOnError) {
            logger.warn("Failed to determine which node is elected active Cluster Coordinator. There may be no coordinator currently: " + ioe);
            if (logger.isDebugEnabled()) {
                logger.warn("", ioe);
            }
        }
        return null;
    }
    if (electedNodeAddress == null) {
        logger.debug("There is currently no elected active Cluster Coordinator");
        return null;
    }
    final int colonLoc = electedNodeAddress.indexOf(':');
    if (colonLoc < 1) {
        if (warnOnError) {
            logger.warn("Failed to determine which node is elected active Cluster Coordinator: ZooKeeper reports the address as {}, but this is not a valid address", electedNodeAddress);
        }
        return null;
    }
    final String electedNodeHostname = electedNodeAddress.substring(0, colonLoc);
    final String portString = electedNodeAddress.substring(colonLoc + 1);
    final int electedNodePort;
    try {
        electedNodePort = Integer.parseInt(portString);
    } catch (final NumberFormatException nfe) {
        if (warnOnError) {
            logger.warn("Failed to determine which node is elected active Cluster Coordinator: ZooKeeper reports the address as {}, but this is not a valid address", electedNodeAddress);
        }
        return null;
    }
    final Set<NodeIdentifier> connectedNodeIds = getNodeIdentifiers();
    final NodeIdentifier electedNodeId = connectedNodeIds.stream().filter(nodeId -> nodeId.getSocketAddress().equals(electedNodeHostname) && nodeId.getSocketPort() == electedNodePort).findFirst().orElse(null);
    if (electedNodeId == null && warnOnError) {
        logger.debug("Failed to determine which node is elected active Cluster Coordinator: ZooKeeper reports the address as {}," + "but there is no node with this address. Will attempt to communicate with node to determine its information", electedNodeAddress);
        try {
            final NodeConnectionStatus connectionStatus = senderListener.requestNodeConnectionStatus(electedNodeHostname, electedNodePort);
            logger.debug("Received NodeConnectionStatus {}", connectionStatus);
            if (connectionStatus == null) {
                return null;
            }
            final NodeConnectionStatus existingStatus = this.nodeStatuses.putIfAbsent(connectionStatus.getNodeIdentifier(), connectionStatus);
            if (existingStatus == null) {
                return connectionStatus.getNodeIdentifier();
            } else {
                return existingStatus.getNodeIdentifier();
            }
        } catch (final Exception e) {
            logger.warn("Failed to determine which node is elected active Cluster Coordinator: ZooKeeper reports the address as {}, but there is no node with this address. " + "Attempted to determine the node's information but failed to retrieve its information due to {}", electedNodeAddress, e.toString());
            if (logger.isDebugEnabled()) {
                logger.warn("", e);
            }
        }
    }
    return electedNodeId;
}

Example 4 with NoClusterCoordinatorException

use of org.apache.nifi.cluster.exception.NoClusterCoordinatorException in project nifi by apache.

the class StandardFlowService method connect.

private ConnectionResponse connect(final boolean retryOnCommsFailure, final boolean retryIndefinitely, final DataFlow dataFlow) throws ConnectionException {
    readLock.lock();
    try {
        logger.info("Connecting Node: " + nodeId);
        // create connection request message
        final ConnectionRequest request = new ConnectionRequest(nodeId, dataFlow);
        final ConnectionRequestMessage requestMsg = new ConnectionRequestMessage();
        requestMsg.setConnectionRequest(request);
        // send connection request to cluster manager
        /*
             * Try to get a current copy of the cluster's dataflow from the manager
             * for ten times, sleeping between attempts. Ten times should be
             * enough because the manager will register the node as connecting
             * and therefore, no other changes to the cluster flow can occur.
             *
             * However, the manager needs to obtain a current data flow within
             * maxAttempts * tryLaterSeconds or else the node will fail to startup.
             */
        final int maxAttempts = 10;
        ConnectionResponse response = null;
        for (int i = 0; i < maxAttempts || retryIndefinitely; i++) {
            try {
                response = senderListener.requestConnection(requestMsg).getConnectionResponse();
                if (response.shouldTryLater()) {
                    logger.info("Requested by cluster coordinator to retry connection in " + response.getTryLaterSeconds() + " seconds with explanation: " + response.getRejectionReason());
                    try {
                        Thread.sleep(response.getTryLaterSeconds() * 1000);
                    } catch (final InterruptedException ie) {
                        // we were interrupted, so finish quickly
                        Thread.currentThread().interrupt();
                        break;
                    }
                } else if (response.getRejectionReason() != null) {
                    logger.warn("Connection request was blocked by cluster coordinator with the explanation: " + response.getRejectionReason());
                    // set response to null and treat a firewall blockage the same as getting no response from manager
                    response = null;
                    break;
                } else {
                    // we received a successful connection response from manager
                    break;
                }
            } catch (final NoClusterCoordinatorException ncce) {
                logger.warn("There is currently no Cluster Coordinator. This often happens upon restart of NiFi when running an embedded ZooKeeper. Will register this node " + "to become the active Cluster Coordinator and will attempt to connect to cluster again");
                controller.registerForClusterCoordinator(true);
                try {
                    Thread.sleep(1000L);
                } catch (final InterruptedException ie) {
                    Thread.currentThread().interrupt();
                    break;
                }
            } catch (final Exception pe) {
                // could not create a socket and communicate with manager
                logger.warn("Failed to connect to cluster due to: " + pe);
                if (logger.isDebugEnabled()) {
                    logger.warn("", pe);
                }
                if (retryOnCommsFailure) {
                    try {
                        Thread.sleep(response == null ? 5000 : response.getTryLaterSeconds());
                    } catch (final InterruptedException ie) {
                        Thread.currentThread().interrupt();
                        break;
                    }
                } else {
                    break;
                }
            }
        }
        if (response == null) {
            // if response is null, then either we had IO problems or we were blocked by firewall or we couldn't determine manager's address
            return response;
        } else if (response.shouldTryLater()) {
            // if response indicates we should try later, then coordinator was unable to service our request. Just load local flow and move on.
            // when the cluster coordinator is able to service requests, this node's heartbeat will trigger the cluster coordinator to reach
            // out to this node and re-connect to the cluster.
            logger.info("Received a 'try again' response from Cluster Coordinator when attempting to connect to cluster with explanation '" + response.getRejectionReason() + "'. However, the maximum number of retries have already completed. Will load local flow and connect to the cluster when able.");
            return null;
        } else {
            // persist node uuid and index returned by NCM and return the response to the caller
            try {
                // Ensure that we have registered our 'cluster node configuration' state key
                final Map<String, String> map = Collections.singletonMap(NODE_UUID, response.getNodeIdentifier().getId());
                controller.getStateManagerProvider().getStateManager(CLUSTER_NODE_CONFIG).setState(map, Scope.LOCAL);
            } catch (final IOException ioe) {
                logger.warn("Received successful response from Cluster Manager but failed to persist state about the Node's Unique Identifier and the Node's Index. " + "This node may be assigned a different UUID when the node is restarted.", ioe);
            }
            return response;
        }
    } finally {
        readLock.unlock();
    }
}

Example 5 with NoClusterCoordinatorException

use of org.apache.nifi.cluster.exception.NoClusterCoordinatorException in project nifi by apache.

the class StandardNiFiContentAccess method getContent.

@Override
public DownloadableContent getContent(final ContentRequestContext request) {
    // if clustered, send request to cluster manager
    if (properties.isClustered() && clusterCoordinator != null && clusterCoordinator.isConnected()) {
        // get the URI
        URI dataUri;
        try {
            dataUri = new URI(request.getDataUri());
        } catch (final URISyntaxException use) {
            throw new ClusterRequestException(use);
        }
        // set the request parameters
        final MultivaluedMap<String, String> parameters = new MultivaluedHashMap();
        parameters.add(CLIENT_ID_PARAM, request.getClientId());
        // set the headers
        final Map<String, String> headers = new HashMap<>();
        // ensure we were able to detect the cluster node id
        if (request.getClusterNodeId() == null) {
            throw new IllegalArgumentException("Unable to determine the which node has the content.");
        }
        // get the target node and ensure it exists
        final NodeIdentifier nodeId = clusterCoordinator.getNodeIdentifier(request.getClusterNodeId());
        // replicate the request to the cluster coordinator, indicating the target node
        NodeResponse nodeResponse;
        try {
            headers.put(RequestReplicator.REPLICATION_TARGET_NODE_UUID_HEADER, nodeId.getId());
            final NodeIdentifier coordinatorNode = clusterCoordinator.getElectedActiveCoordinatorNode();
            if (coordinatorNode == null) {
                throw new NoClusterCoordinatorException();
            }
            final Set<NodeIdentifier> coordinatorNodes = Collections.singleton(coordinatorNode);
            nodeResponse = requestReplicator.replicate(coordinatorNodes, HttpMethod.GET, dataUri, parameters, headers, false, true).awaitMergedResponse();
        } catch (InterruptedException e) {
            throw new IllegalClusterStateException("Interrupted while waiting for a response from node");
        }
        final Response clientResponse = nodeResponse.getClientResponse();
        final MultivaluedMap<String, String> responseHeaders = clientResponse.getStringHeaders();
        // ensure an appropriate response
        if (Response.Status.NOT_FOUND.getStatusCode() == clientResponse.getStatusInfo().getStatusCode()) {
            throw new ResourceNotFoundException(clientResponse.readEntity(String.class));
        } else if (Response.Status.FORBIDDEN.getStatusCode() == clientResponse.getStatusInfo().getStatusCode() || Response.Status.UNAUTHORIZED.getStatusCode() == clientResponse.getStatusInfo().getStatusCode()) {
            throw new AccessDeniedException(clientResponse.readEntity(String.class));
        } else if (Response.Status.OK.getStatusCode() != clientResponse.getStatusInfo().getStatusCode()) {
            throw new IllegalStateException(clientResponse.readEntity(String.class));
        }
        // get the file name
        final String contentDisposition = responseHeaders.getFirst("Content-Disposition");
        final String filename = StringUtils.substringBetween(contentDisposition, "filename=\"", "\"");
        // get the content type
        final String contentType = responseHeaders.getFirst("Content-Type");
        // create the downloadable content
        return new DownloadableContent(filename, contentType, nodeResponse.getInputStream());
    } else {
        // example URIs:
        // http://localhost:8080/nifi-api/provenance/events/{id}/content/{input|output}
        // http://localhost:8080/nifi-api/flowfile-queues/{uuid}/flowfiles/{uuid}/content
        // get just the context path for comparison
        final String dataUri = StringUtils.substringAfter(request.getDataUri(), "/nifi-api");
        if (StringUtils.isBlank(dataUri)) {
            throw new IllegalArgumentException("The specified data reference URI is not valid.");
        }
        // flowfile listing content
        final Matcher flowFileMatcher = FLOWFILE_CONTENT_URI_PATTERN.matcher(dataUri);
        if (flowFileMatcher.matches()) {
            final String connectionId = flowFileMatcher.group(1);
            final String flowfileId = flowFileMatcher.group(2);
            return getFlowFileContent(connectionId, flowfileId, dataUri);
        }
        // provenance event content
        final Matcher provenanceMatcher = PROVENANCE_CONTENT_URI_PATTERN.matcher(dataUri);
        if (provenanceMatcher.matches()) {
            try {
                final Long eventId = Long.parseLong(provenanceMatcher.group(1));
                final ContentDirection direction = ContentDirection.valueOf(provenanceMatcher.group(2).toUpperCase());
                return getProvenanceEventContent(eventId, dataUri, direction);
            } catch (final IllegalArgumentException iae) {
                throw new IllegalArgumentException("The specified data reference URI is not valid.");
            }
        }
        // invalid uri
        throw new IllegalArgumentException("The specified data reference URI is not valid.");
    }
}