Examples with PeerStatus org.apache.nifi.remote.PeerStatus used on opensource projects

Example 11 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class EndpointConnectionPool method getEndpointConnection.

public EndpointConnection getEndpointConnection(final TransferDirection direction, final SiteToSiteClientConfig config) throws IOException {
    // 
    // Attempt to get a connection state that already exists for this URL.
    // 
    FlowFileCodec codec = null;
    CommunicationsSession commsSession = null;
    SocketClientProtocol protocol = null;
    EndpointConnection connection;
    Peer peer = null;
    final URI clusterUrl;
    try {
        clusterUrl = siteInfoProvider.getActiveClusterUrl();
    } catch (final IOException ioe) {
        throw new UnreachableClusterException("Unable to refresh details from any of the configured remote instances.", ioe);
    }
    do {
        final List<EndpointConnection> addBack = new ArrayList<>();
        logger.debug("{} getting next peer status", this);
        final PeerStatus peerStatus = peerSelector.getNextPeerStatus(direction);
        logger.debug("{} next peer status = {}", this, peerStatus);
        if (peerStatus == null) {
            return null;
        }
        final PeerDescription peerDescription = peerStatus.getPeerDescription();
        BlockingQueue<EndpointConnection> connectionQueue = connectionQueueMap.get(peerDescription);
        if (connectionQueue == null) {
            connectionQueue = new LinkedBlockingQueue<>();
            BlockingQueue<EndpointConnection> existing = connectionQueueMap.putIfAbsent(peerDescription, connectionQueue);
            if (existing != null) {
                connectionQueue = existing;
            }
        }
        try {
            connection = connectionQueue.poll();
            logger.debug("{} Connection State for {} = {}", this, clusterUrl, connection);
            final String portId = getPortIdentifier(direction);
            if (connection == null && !addBack.isEmpty()) {
                // all available connections have been penalized.
                logger.debug("{} all Connections for {} are penalized; returning no Connection", this, portId);
                return null;
            }
            if (connection != null && connection.getPeer().isPenalized(portId)) {
                // we have a connection, but it's penalized. We want to add it back to the queue
                // when we've found one to use.
                addBack.add(connection);
                continue;
            }
            // if we can't get an existing Connection, create one
            if (connection == null) {
                logger.debug("{} No Connection available for Port {}; creating new Connection", this, portId);
                protocol = new SocketClientProtocol();
                protocol.setDestination(new IdEnrichedRemoteDestination(remoteDestination, portId));
                protocol.setEventReporter(eventReporter);
                final long penalizationMillis = remoteDestination.getYieldPeriod(TimeUnit.MILLISECONDS);
                try {
                    logger.debug("{} Establishing site-to-site connection with {}", this, peerStatus);
                    commsSession = establishSiteToSiteConnection(peerStatus);
                } catch (final IOException ioe) {
                    peerSelector.penalize(peerStatus.getPeerDescription(), penalizationMillis);
                    throw ioe;
                }
                final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
                final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
                try {
                    logger.debug("{} Negotiating protocol", this);
                    RemoteResourceInitiator.initiateResourceNegotiation(protocol, dis, dos);
                } catch (final HandshakeException e) {
                    try {
                        commsSession.close();
                    } catch (final IOException ioe) {
                        throw e;
                    }
                }
                final String peerUrl = "nifi://" + peerDescription.getHostname() + ":" + peerDescription.getPort();
                peer = new Peer(peerDescription, commsSession, peerUrl, clusterUrl.toString());
                // set properties based on config
                if (config != null) {
                    protocol.setTimeout((int) config.getTimeout(TimeUnit.MILLISECONDS));
                    protocol.setPreferredBatchCount(config.getPreferredBatchCount());
                    protocol.setPreferredBatchSize(config.getPreferredBatchSize());
                    protocol.setPreferredBatchDuration(config.getPreferredBatchDuration(TimeUnit.MILLISECONDS));
                }
                // perform handshake
                try {
                    logger.debug("{} performing handshake", this);
                    protocol.handshake(peer);
                    // handle error cases
                    if (protocol.isDestinationFull()) {
                        logger.warn("{} {} indicates that port {}'s destination is full; penalizing peer", this, peer, config.getPortName() == null ? config.getPortIdentifier() : config.getPortName());
                        peerSelector.penalize(peer, penalizationMillis);
                        try {
                            peer.close();
                        } catch (final IOException ioe) {
                        }
                        continue;
                    } else if (protocol.isPortInvalid()) {
                        peerSelector.penalize(peer, penalizationMillis);
                        cleanup(protocol, peer);
                        throw new PortNotRunningException(peer.toString() + " indicates that port " + portId + " is not running");
                    } else if (protocol.isPortUnknown()) {
                        peerSelector.penalize(peer, penalizationMillis);
                        cleanup(protocol, peer);
                        throw new UnknownPortException(peer.toString() + " indicates that port " + portId + " is not known");
                    }
                    // negotiate the FlowFileCodec to use
                    logger.debug("{} negotiating codec", this);
                    codec = protocol.negotiateCodec(peer);
                    logger.debug("{} negotiated codec is {}", this, codec);
                } catch (final PortNotRunningException | UnknownPortException e) {
                    throw e;
                } catch (final Exception e) {
                    peerSelector.penalize(peer, penalizationMillis);
                    cleanup(protocol, peer);
                    final String message = String.format("%s failed to communicate with %s due to %s", this, peer == null ? clusterUrl : peer, e.toString());
                    error(logger, eventReporter, message);
                    if (logger.isDebugEnabled()) {
                        logger.error("", e);
                    }
                    throw e;
                }
                connection = new EndpointConnection(peer, protocol, codec);
            } else {
                final long lastTimeUsed = connection.getLastTimeUsed();
                final long millisSinceLastUse = System.currentTimeMillis() - lastTimeUsed;
                if (commsTimeout > 0L && millisSinceLastUse >= commsTimeout) {
                    cleanup(connection.getSocketClientProtocol(), connection.getPeer());
                    connection = null;
                } else {
                    codec = connection.getCodec();
                    peer = connection.getPeer();
                    commsSession = peer.getCommunicationsSession();
                    protocol = connection.getSocketClientProtocol();
                }
            }
        } catch (final Throwable t) {
            if (commsSession != null) {
                try {
                    commsSession.close();
                } catch (final IOException ioe) {
                }
            }
            throw t;
        } finally {
            if (!addBack.isEmpty()) {
                connectionQueue.addAll(addBack);
                addBack.clear();
            }
        }
    } while (connection == null || codec == null || commsSession == null || protocol == null);
    activeConnections.add(connection);
    return connection;
}

Example 12 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class EndpointConnectionPool method fetchRemotePeerStatuses.

@Override
public Set<PeerStatus> fetchRemotePeerStatuses(final PeerDescription peerDescription) throws IOException {
    final String hostname = peerDescription.getHostname();
    final int port = peerDescription.getPort();
    final URI clusterUrl = siteInfoProvider.getActiveClusterUrl();
    final PeerDescription clusterPeerDescription = new PeerDescription(hostname, port, clusterUrl.toString().startsWith("https://"));
    final CommunicationsSession commsSession = establishSiteToSiteConnection(hostname, port);
    final Peer peer = new Peer(clusterPeerDescription, commsSession, "nifi://" + hostname + ":" + port, clusterUrl.toString());
    final SocketClientProtocol clientProtocol = new SocketClientProtocol();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    RemoteResourceInitiator.initiateResourceNegotiation(clientProtocol, dis, dos);
    clientProtocol.setTimeout(commsTimeout);
    if (clientProtocol.getVersionNegotiator().getVersion() < 5) {
        String portId = getPortIdentifier(TransferDirection.RECEIVE);
        if (portId == null) {
            portId = getPortIdentifier(TransferDirection.SEND);
        }
        if (portId == null) {
            peer.close();
            throw new IOException("Failed to determine the identifier of port " + remoteDestination.getName());
        }
        clientProtocol.handshake(peer, portId);
    } else {
        clientProtocol.handshake(peer, null);
    }
    final Set<PeerStatus> peerStatuses = clientProtocol.getPeerStatuses(peer);
    try {
        clientProtocol.shutdown(peer);
    } catch (final IOException e) {
        final String message = String.format("%s Failed to shutdown protocol when updating list of peers due to %s", this, e.toString());
        warn(logger, eventReporter, message);
        if (logger.isDebugEnabled()) {
            logger.warn("", e);
        }
    }
    try {
        peer.close();
    } catch (final IOException e) {
        final String message = String.format("%s Failed to close resources when updating list of peers due to %s", this, e.toString());
        warn(logger, eventReporter, message);
        if (logger.isDebugEnabled()) {
            logger.warn("", e);
        }
    }
    return peerStatuses;
}

Example 13 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class TestPeerSelector method testFetchRemotePeerStatuses.

/**
 * This test simulates a failure scenario of a remote NiFi cluster. It confirms that:
 * <ol>
 *     <li>PeerSelector uses the bootstrap node to fetch remote peer statuses at the initial attempt</li>
 *     <li>PeerSelector uses one of query-able nodes lastly fetched successfully</li>
 *     <li>PeerSelector can refresh remote peer statuses even if the bootstrap node is down</li>
 *     <li>PeerSelector returns null as next peer when there's no peer available</li>
 *     <li>PeerSelector always tries to fetch peer statuses at least from the bootstrap node, so that it can
 *     recover when the node gets back online</li>
 * </ol>
 */
@Test
public void testFetchRemotePeerStatuses() throws IOException {
    final Set<PeerStatus> peerStatuses = new HashSet<>();
    final PeerDescription bootstrapNode = new PeerDescription("Node1", 1111, true);
    final PeerDescription node2 = new PeerDescription("Node2", 2222, true);
    final PeerStatus bootstrapNodeStatus = new PeerStatus(bootstrapNode, 10, true);
    final PeerStatus node2Status = new PeerStatus(node2, 10, true);
    peerStatuses.add(bootstrapNodeStatus);
    peerStatuses.add(node2Status);
    final PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    final PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final UnitTestSystemTime systemTime = new UnitTestSystemTime();
    peerSelector.setSystemTime(systemTime);
    doReturn(bootstrapNode).when(peerStatusProvider).getBootstrapPeerDescription();
    doAnswer(invocation -> {
        final PeerDescription peerFetchStatusesFrom = invocation.getArgumentAt(0, PeerDescription.class);
        if (peerStatuses.stream().filter(ps -> ps.getPeerDescription().equals(peerFetchStatusesFrom)).collect(Collectors.toSet()).size() > 0) {
            // If the remote peer is running, then return available peer statuses.
            return peerStatuses;
        }
        throw new IOException("Connection refused. " + peerFetchStatusesFrom + " is not running.");
    }).when(peerStatusProvider).fetchRemotePeerStatuses(any(PeerDescription.class));
    // 1st attempt. It uses the bootstrap node.
    peerSelector.refreshPeers();
    PeerStatus peerStatus = peerSelector.getNextPeerStatus(TransferDirection.RECEIVE);
    assertNotNull(peerStatus);
    // Proceed time so that peer selector refresh statuses.
    peerStatuses.remove(bootstrapNodeStatus);
    systemTime.offset += TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES) + 1;
    // 2nd attempt.
    peerSelector.refreshPeers();
    peerStatus = peerSelector.getNextPeerStatus(TransferDirection.RECEIVE);
    assertNotNull(peerStatus);
    assertEquals("Node2 should be returned since node 2 is the only available node.", node2, peerStatus.getPeerDescription());
    // Proceed time so that peer selector refresh statuses.
    systemTime.offset += TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES) + 1;
    // 3rd attempt.
    peerSelector.refreshPeers();
    peerStatus = peerSelector.getNextPeerStatus(TransferDirection.RECEIVE);
    assertNotNull(peerStatus);
    assertEquals("Node2 should be returned since node 2 is the only available node.", node2, peerStatus.getPeerDescription());
    // Remove node2 to simulate that it goes down. There's no available node at this point.
    peerStatuses.remove(node2Status);
    systemTime.offset += TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES) + 1;
    peerSelector.refreshPeers();
    peerStatus = peerSelector.getNextPeerStatus(TransferDirection.RECEIVE);
    assertNull("PeerSelector should return null as next peer status, since there's no available peer", peerStatus);
    // Add node1 back. PeerSelector should be able to fetch peer statuses because it always tries to fetch at least from the bootstrap node.
    peerStatuses.add(bootstrapNodeStatus);
    systemTime.offset += TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES) + 1;
    peerSelector.refreshPeers();
    peerStatus = peerSelector.getNextPeerStatus(TransferDirection.RECEIVE);
    assertEquals("Node1 should be returned since node 1 is the only available node.", bootstrapNode, peerStatus.getPeerDescription());
}

Example 14 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class PeerSelector method getNextPeerStatus.

/**
 * Return status of a peer that will be used for the next communication.
 * The peer with less workload will be selected with higher probability.
 * @param direction the amount of workload is calculated based on transaction direction,
 *                  for SEND, a peer with less flow files is preferred,
 *                  for RECEIVE, a peer with more flow files is preferred
 * @return a selected peer, if there is no available peer or all peers are penalized, then return null
 */
public PeerStatus getNextPeerStatus(final TransferDirection direction) {
    List<PeerStatus> peerList = peerStatuses;
    if (isPeerRefreshNeeded(peerList)) {
        peerRefreshLock.lock();
        try {
            // now that we have the lock, check again that we need to refresh (because another thread
            // could have been refreshing while we were waiting for the lock).
            peerList = peerStatuses;
            if (isPeerRefreshNeeded(peerList)) {
                try {
                    peerList = createPeerStatusList(direction);
                } catch (final Exception e) {
                    final String message = String.format("%s Failed to update list of peers due to %s", this, e.toString());
                    warn(logger, eventReporter, message);
                    if (logger.isDebugEnabled()) {
                        logger.warn("", e);
                    }
                }
                this.peerStatuses = peerList;
                peerRefreshTime = systemTime.currentTimeMillis();
            }
        } finally {
            peerRefreshLock.unlock();
        }
    }
    if (peerList == null || peerList.isEmpty()) {
        return null;
    }
    PeerStatus peerStatus;
    for (int i = 0; i < peerList.size(); i++) {
        final long idx = peerIndex.getAndIncrement();
        final int listIndex = (int) (idx % peerList.size());
        peerStatus = peerList.get(listIndex);
        if (isPenalized(peerStatus)) {
            logger.debug("{} {} is penalized; will not communicate with this peer", this, peerStatus);
        } else {
            return peerStatus;
        }
    }
    logger.debug("{} All peers appear to be penalized; returning null", this);
    return null;
}

Example 15 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class PeerSelector method fetchRemotePeerStatuses.

private Set<PeerStatus> fetchRemotePeerStatuses() throws IOException {
    final Set<PeerDescription> peersToRequestClusterInfoFrom = new HashSet<>();
    // Look at all of the peers that we fetched last time.
    final Set<PeerStatus> lastFetched = lastFetchedQueryablePeers;
    if (lastFetched != null && !lastFetched.isEmpty()) {
        lastFetched.stream().map(peer -> peer.getPeerDescription()).forEach(desc -> peersToRequestClusterInfoFrom.add(desc));
    }
    // Always add the configured node info to the list of peers to communicate with
    peersToRequestClusterInfoFrom.add(peerStatusProvider.getBootstrapPeerDescription());
    logger.debug("Fetching remote peer statuses from: {}", peersToRequestClusterInfoFrom);
    Exception lastFailure = null;
    for (final PeerDescription peerDescription : peersToRequestClusterInfoFrom) {
        try {
            final Set<PeerStatus> statuses = peerStatusProvider.fetchRemotePeerStatuses(peerDescription);
            lastFetchedQueryablePeers = statuses.stream().filter(p -> p.isQueryForPeers()).collect(Collectors.toSet());
            return statuses;
        } catch (final Exception e) {
            logger.warn("Could not communicate with {}:{} to determine which nodes exist in the remote NiFi cluster, due to {}", peerDescription.getHostname(), peerDescription.getPort(), e.toString());
            lastFailure = e;
        }
    }
    final IOException ioe = new IOException("Unable to communicate with remote NiFi cluster in order to determine which nodes exist in the remote cluster");
    if (lastFailure != null) {
        ioe.addSuppressed(lastFailure);
    }
    throw ioe;
}