Examples with SiteToSiteRestApiClient org.apache.nifi.remote.util.SiteToSiteRestApiClient used on opensource projects

Example 1 with SiteToSiteRestApiClient

use of org.apache.nifi.remote.util.SiteToSiteRestApiClient in project nifi by apache.

the class HttpClient method fetchRemotePeerStatuses.

@Override
public Set<PeerStatus> fetchRemotePeerStatuses(PeerDescription peerDescription) throws IOException {
    // Each node should has the same URL structure and network reach-ability with the proxy configuration.
    try (final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter())) {
        final String scheme = peerDescription.isSecure() ? "https" : "http";
        apiClient.setBaseUrl(scheme, peerDescription.getHostname(), peerDescription.getPort());
        final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS);
        apiClient.setConnectTimeoutMillis(timeoutMillis);
        apiClient.setReadTimeoutMillis(timeoutMillis);
        apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS));
        apiClient.setLocalAddress(config.getLocalAddress());
        final Collection<PeerDTO> peers = apiClient.getPeers();
        if (peers == null || peers.size() == 0) {
            throw new IOException("Couldn't get any peer to communicate with. " + apiClient.getBaseUrl() + " returned zero peers.");
        }
        // was added in NiFi 1.0.0, which means that peer-to-peer queries are always allowed.
        return peers.stream().map(p -> new PeerStatus(new PeerDescription(p.getHostname(), p.getPort(), p.isSecure()), p.getFlowFileCount(), true)).collect(Collectors.toSet());
    }
}

Example 2 with SiteToSiteRestApiClient

use of org.apache.nifi.remote.util.SiteToSiteRestApiClient in project nifi by apache.

the class HttpClient method createTransaction.

@Override
public Transaction createTransaction(final TransferDirection direction) throws HandshakeException, PortNotRunningException, ProtocolException, UnknownPortException, IOException {
    final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS);
    PeerStatus peerStatus;
    while ((peerStatus = peerSelector.getNextPeerStatus(direction)) != null) {
        logger.debug("peerStatus={}", peerStatus);
        final CommunicationsSession commSession = new HttpCommunicationsSession();
        final String nodeApiUrl = resolveNodeApiUrl(peerStatus.getPeerDescription());
        final StringBuilder clusterUrls = new StringBuilder();
        config.getUrls().forEach(url -> {
            if (clusterUrls.length() > 0) {
                clusterUrls.append(",");
                clusterUrls.append(url);
            }
        });
        final Peer peer = new Peer(peerStatus.getPeerDescription(), commSession, nodeApiUrl, clusterUrls.toString());
        final int penaltyMillis = (int) config.getPenalizationPeriod(TimeUnit.MILLISECONDS);
        String portId = config.getPortIdentifier();
        if (StringUtils.isEmpty(portId)) {
            portId = siteInfoProvider.getPortIdentifier(config.getPortName(), direction);
            if (StringUtils.isEmpty(portId)) {
                peer.close();
                throw new IOException("Failed to determine the identifier of port " + config.getPortName());
            }
        }
        final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter());
        apiClient.setBaseUrl(peer.getUrl());
        apiClient.setConnectTimeoutMillis(timeoutMillis);
        apiClient.setReadTimeoutMillis(timeoutMillis);
        apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS));
        apiClient.setLocalAddress(config.getLocalAddress());
        apiClient.setCompress(config.isUseCompression());
        apiClient.setRequestExpirationMillis(config.getIdleConnectionExpiration(TimeUnit.MILLISECONDS));
        apiClient.setBatchCount(config.getPreferredBatchCount());
        apiClient.setBatchSize(config.getPreferredBatchSize());
        apiClient.setBatchDurationMillis(config.getPreferredBatchDuration(TimeUnit.MILLISECONDS));
        final String transactionUrl;
        try {
            transactionUrl = apiClient.initiateTransaction(direction, portId);
            commSession.setUserDn(apiClient.getTrustedPeerDn());
        } catch (final Exception e) {
            apiClient.close();
            logger.warn("Penalizing a peer {} due to {}", peer, e.toString());
            peerSelector.penalize(peer, penaltyMillis);
            // Following exceptions will be thrown even if we tried other peers, so throw it.
            if (e instanceof UnknownPortException || e instanceof PortNotRunningException || e instanceof HandshakeException) {
                throw e;
            }
            logger.debug("Continue trying other peers...");
            continue;
        }
        // We found a valid peer to communicate with.
        final Integer transactionProtocolVersion = apiClient.getTransactionProtocolVersion();
        final HttpClientTransaction transaction = new HttpClientTransaction(transactionProtocolVersion, peer, direction, config.isUseCompression(), portId, penaltyMillis, config.getEventReporter()) {

            @Override
            protected void close() throws IOException {
                try {
                    super.close();
                } finally {
                    activeTransactions.remove(this);
                }
            }
        };
        try {
            transaction.initialize(apiClient, transactionUrl);
        } catch (final Exception e) {
            transaction.error();
            throw e;
        }
        activeTransactions.add(transaction);
        return transaction;
    }
    logger.info("Couldn't find a valid peer to communicate with.");
    return null;
}

Example 3 with SiteToSiteRestApiClient

use of org.apache.nifi.remote.util.SiteToSiteRestApiClient in project nifi by apache.

the class TestSiteInfoProvider method testConnectException.

@Test
public void testConnectException() throws Exception {
    final Set<String> expectedClusterUrl = new LinkedHashSet<>(Arrays.asList(new String[] { "http://node1:8443, http://node2:8443" }));
    final SiteInfoProvider siteInfoProvider = spy(new SiteInfoProvider());
    siteInfoProvider.setClusterUrls(expectedClusterUrl);
    final ControllerDTO controllerDTO = new ControllerDTO();
    controllerDTO.setInputPorts(Collections.emptySet());
    controllerDTO.setOutputPorts(Collections.emptySet());
    controllerDTO.setRemoteSiteListeningPort(8081);
    controllerDTO.setRemoteSiteHttpListeningPort(8080);
    controllerDTO.setSiteToSiteSecure(false);
    // SiteInfoProvider uses SiteToSIteRestApiClient to get ControllerDTO.
    doAnswer(invocation -> {
        final SiteToSiteRestApiClient apiClient = mock(SiteToSiteRestApiClient.class);
        when(apiClient.getController(eq(expectedClusterUrl))).thenThrow(new IOException("Connection refused."));
        return apiClient;
    }).when(siteInfoProvider).createSiteToSiteRestApiClient(any(), any());
    try {
        siteInfoProvider.getSiteToSitePort();
        fail();
    } catch (IOException e) {
    }
    try {
        siteInfoProvider.getActiveClusterUrl();
        fail();
    } catch (IOException e) {
    }
}

Example 4 with SiteToSiteRestApiClient

use of org.apache.nifi.remote.util.SiteToSiteRestApiClient in project nifi by apache.

the class TestHttpClientTransaction method testSendTwoFlowFiles.

@Test
public void testSendTwoFlowFiles() throws IOException {
    SiteToSiteRestApiClient apiClient = mock(SiteToSiteRestApiClient.class);
    final String transactionUrl = "http://www.example.com/data-transfer/input-ports/portId/transactions/transactionId";
    doNothing().when(apiClient).openConnectionForSend(eq("portId"), any(Peer.class));
    // Emulate that server returns correct checksum.
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            HttpCommunicationsSession commSession = (HttpCommunicationsSession) invocation.getArguments()[0];
            commSession.setChecksum("3359812065");
            return null;
        }
    }).when(apiClient).finishTransferFlowFiles(any(CommunicationsSession.class));
    TransactionResultEntity resultEntity = new TransactionResultEntity();
    resultEntity.setResponseCode(ResponseCode.TRANSACTION_FINISHED.getCode());
    doReturn(resultEntity).when(apiClient).commitTransferFlowFiles(eq(transactionUrl), eq(CONFIRM_TRANSACTION));
    ByteArrayOutputStream serverResponseBos = new ByteArrayOutputStream();
    ByteArrayInputStream serverResponse = new ByteArrayInputStream(serverResponseBos.toByteArray());
    ByteArrayOutputStream clientRequest = new ByteArrayOutputStream();
    HttpClientTransaction transaction = getClientTransaction(serverResponse, clientRequest, apiClient, TransferDirection.SEND, transactionUrl);
    execSendTwoFlowFiles(transaction);
    InputStream sentByClient = new ByteArrayInputStream(clientRequest.toByteArray());
    DataPacket packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 1", readContents(packetByClient));
    packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 2", readContents(packetByClient));
    assertEquals(-1, sentByClient.read());
    verify(apiClient).commitTransferFlowFiles(transactionUrl, CONFIRM_TRANSACTION);
}

Example 5 with SiteToSiteRestApiClient

use of org.apache.nifi.remote.util.SiteToSiteRestApiClient in project nifi by apache.

the class TestHttpClientTransaction method testSendWithInvalidChecksum.

@Test
public void testSendWithInvalidChecksum() throws IOException {
    SiteToSiteRestApiClient apiClient = mock(SiteToSiteRestApiClient.class);
    final String transactionUrl = "http://www.example.com/data-transfer/input-ports/portId/transactions/transactionId";
    doNothing().when(apiClient).openConnectionForSend(eq(transactionUrl), any(Peer.class));
    // Emulate that server returns incorrect checksum.
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            HttpCommunicationsSession commSession = (HttpCommunicationsSession) invocation.getArguments()[0];
            commSession.setChecksum("Different checksum");
            return null;
        }
    }).when(apiClient).finishTransferFlowFiles(any(CommunicationsSession.class));
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            TransactionResultEntity serverResult = new TransactionResultEntity();
            serverResult.setResponseCode(ResponseCode.CANCEL_TRANSACTION.getCode());
            return serverResult;
        }
    }).when(apiClient).commitTransferFlowFiles(eq(transactionUrl), eq(ResponseCode.BAD_CHECKSUM));
    ByteArrayOutputStream serverResponseBos = new ByteArrayOutputStream();
    ByteArrayInputStream serverResponse = new ByteArrayInputStream(serverResponseBos.toByteArray());
    ByteArrayOutputStream clientRequest = new ByteArrayOutputStream();
    HttpClientTransaction transaction = getClientTransaction(serverResponse, clientRequest, apiClient, TransferDirection.SEND, transactionUrl);
    execSendWithInvalidChecksum(transaction);
    InputStream sentByClient = new ByteArrayInputStream(clientRequest.toByteArray());
    DataPacket packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 1", readContents(packetByClient));
    packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 2", readContents(packetByClient));
    assertEquals(-1, sentByClient.read());
    verify(apiClient).commitTransferFlowFiles(transactionUrl, ResponseCode.BAD_CHECKSUM);
}