Examples with DataPacket org.apache.nifi.remote.protocol.DataPacket used on opensource projects

Example 16 with DataPacket

use of org.apache.nifi.remote.protocol.DataPacket in project nifi by apache.

the class TestSocketClientTransaction method testSendOneFlowFile.

@Test
public void testSendOneFlowFile() throws IOException {
    ByteArrayOutputStream serverResponseBos = new ByteArrayOutputStream();
    DataOutputStream serverResponse = new DataOutputStream(serverResponseBos);
    ResponseCode.CONFIRM_TRANSACTION.writeResponse(serverResponse, "2946083981");
    ResponseCode.TRANSACTION_FINISHED.writeResponse(serverResponse);
    ByteArrayInputStream bis = new ByteArrayInputStream(serverResponseBos.toByteArray());
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    SocketClientTransaction transaction = getClientTransaction(bis, bos, TransferDirection.SEND);
    execSendOneFlowFile(transaction);
    // Verify what client has sent.
    DataInputStream sentByClient = new DataInputStream(new ByteArrayInputStream(bos.toByteArray()));
    assertEquals(RequestType.SEND_FLOWFILES, RequestType.readRequestType(sentByClient));
    DataPacket packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 1", readContents(packetByClient));
    Response endOfDataResponse = Response.read(sentByClient);
    assertEquals(ResponseCode.FINISH_TRANSACTION, endOfDataResponse.getCode());
    Response confirmResponse = Response.read(sentByClient);
    assertEquals(ResponseCode.CONFIRM_TRANSACTION, confirmResponse.getCode());
    assertEquals(-1, sentByClient.read());
}

Example 17 with DataPacket

use of org.apache.nifi.remote.protocol.DataPacket in project apex-malhar by apache.

the class AbstractNiFiInputOperator method emitTuples.

@Override
public void emitTuples() {
    // clear the recovered list, and return until we have no more recovered data
    if (recoveredTuples.size() > 0) {
        emitTuples(recoveredTuples);
        recoveredTuples.clear();
        return;
    }
    // no recovered data so start a transaction and pull new data
    try {
        final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);
        if (transaction == null) {
            LOGGER.warn("A transaction could not be created, returning...");
            return;
        }
        DataPacket dataPacket = transaction.receive();
        if (dataPacket == null) {
            transaction.confirm();
            transaction.complete();
            LOGGER.debug("No data available to pull, returning and will try again...");
            return;
        }
        // read all of the available data packets and convert to the given type
        final List<T> tuples = new ArrayList<>();
        do {
            tuples.add(createTuple(dataPacket));
            dataPacket = transaction.receive();
        } while (dataPacket != null);
        // confirm all of the expected data was received by comparing check-sums, does not complete the transaction
        transaction.confirm();
        // ensure we have the data saved before proceeding in case anything goes wrong
        currentWindowTuples.addAll(tuples);
        windowDataManager.save(currentWindowTuples, currentWindowId);
        // we now have the data saved so we can complete the transaction
        transaction.complete();
        // delegate to sub-classes to emit the tuples
        emitTuples(tuples);
    } catch (IOException e) {
        DTThrowable.rethrow(e);
    }
}

Example 18 with DataPacket

use of org.apache.nifi.remote.protocol.DataPacket in project nifi by apache.

the class StandardRemoteGroupPort method transferFlowFiles.

private int transferFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session, final FlowFile firstFlowFile) throws IOException, ProtocolException {
    FlowFile flowFile = firstFlowFile;
    try {
        final String userDn = transaction.getCommunicant().getDistinguishedName();
        final long startSendingNanos = System.nanoTime();
        final StopWatch stopWatch = new StopWatch(true);
        long bytesSent = 0L;
        final SiteToSiteClientConfig siteToSiteClientConfig = getSiteToSiteClient().getConfig();
        final long maxBatchBytes = siteToSiteClientConfig.getPreferredBatchSize();
        final int maxBatchCount = siteToSiteClientConfig.getPreferredBatchCount();
        final long preferredBatchDuration = siteToSiteClientConfig.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
        final long maxBatchDuration = preferredBatchDuration > 0 ? preferredBatchDuration : BATCH_SEND_NANOS;
        final Set<FlowFile> flowFilesSent = new HashSet<>();
        boolean continueTransaction = true;
        while (continueTransaction) {
            final long startNanos = System.nanoTime();
            // call codec.encode within a session callback so that we have the InputStream to read the FlowFile
            final FlowFile toWrap = flowFile;
            session.read(flowFile, new InputStreamCallback() {

                @Override
                public void process(final InputStream in) throws IOException {
                    final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
                    transaction.send(dataPacket);
                }
            });
            final long transferNanos = System.nanoTime() - startNanos;
            final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);
            flowFilesSent.add(flowFile);
            bytesSent += flowFile.getSize();
            logger.debug("{} Sent {} to {}", this, flowFile, transaction.getCommunicant().getUrl());
            final String transitUri = transaction.getCommunicant().createTransitUri(flowFile.getAttribute(CoreAttributes.UUID.key()));
            session.getProvenanceReporter().send(flowFile, transitUri, "Remote DN=" + userDn, transferMillis, false);
            session.remove(flowFile);
            final long sendingNanos = System.nanoTime() - startSendingNanos;
            if (maxBatchCount > 0 && flowFilesSent.size() >= maxBatchCount) {
                flowFile = null;
            } else if (maxBatchBytes > 0 && bytesSent >= maxBatchBytes) {
                flowFile = null;
            } else if (sendingNanos >= maxBatchDuration) {
                flowFile = null;
            } else {
                flowFile = session.get();
            }
            continueTransaction = (flowFile != null);
        }
        transaction.confirm();
        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        stopWatch.stop();
        final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesSent);
        transaction.complete();
        session.commit();
        final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
        logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[] { this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate });
        return flowFilesSent.size();
    } catch (final Exception e) {
        session.rollback();
        throw e;
    }
}

Example 19 with DataPacket

use of org.apache.nifi.remote.protocol.DataPacket in project nifi by apache.

the class StandardRemoteGroupPort method receiveFlowFiles.

private int receiveFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session) throws IOException, ProtocolException {
    final String userDn = transaction.getCommunicant().getDistinguishedName();
    final StopWatch stopWatch = new StopWatch(true);
    final Set<FlowFile> flowFilesReceived = new HashSet<>();
    long bytesReceived = 0L;
    while (true) {
        final long start = System.nanoTime();
        final DataPacket dataPacket = transaction.receive();
        if (dataPacket == null) {
            break;
        }
        FlowFile flowFile = session.create();
        flowFile = session.putAllAttributes(flowFile, dataPacket.getAttributes());
        final Communicant communicant = transaction.getCommunicant();
        final String host = StringUtils.isEmpty(communicant.getHost()) ? "unknown" : communicant.getHost();
        final String port = communicant.getPort() < 0 ? "unknown" : String.valueOf(communicant.getPort());
        final Map<String, String> attributes = new HashMap<>(2);
        attributes.put(SiteToSiteAttributes.S2S_HOST.key(), host);
        attributes.put(SiteToSiteAttributes.S2S_ADDRESS.key(), host + ":" + port);
        flowFile = session.putAllAttributes(flowFile, attributes);
        flowFile = session.importFrom(dataPacket.getData(), flowFile);
        final long receiveNanos = System.nanoTime() - start;
        flowFilesReceived.add(flowFile);
        String sourceFlowFileIdentifier = dataPacket.getAttributes().get(CoreAttributes.UUID.key());
        if (sourceFlowFileIdentifier == null) {
            sourceFlowFileIdentifier = "<Unknown Identifier>";
        }
        final String transitUri = transaction.getCommunicant().createTransitUri(sourceFlowFileIdentifier);
        session.getProvenanceReporter().receive(flowFile, transitUri, "urn:nifi:" + sourceFlowFileIdentifier, "Remote DN=" + userDn, TimeUnit.NANOSECONDS.toMillis(receiveNanos));
        session.transfer(flowFile, Relationship.ANONYMOUS);
        bytesReceived += dataPacket.getSize();
    }
    // Confirm that what we received was the correct data.
    transaction.confirm();
    // Commit the session so that we have persisted the data
    session.commit();
    transaction.complete();
    if (!flowFilesReceived.isEmpty()) {
        stopWatch.stop();
        final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
        final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesReceived);
        logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[] { this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate });
    }
    return flowFilesReceived.size();
}

Example 20 with DataPacket

use of org.apache.nifi.remote.protocol.DataPacket in project nifi by apache.

the class TestStandardRemoteGroupPort method testReceiveHttp.

@Test
public void testReceiveHttp() throws Exception {
    setupMock(SiteToSiteTransportProtocol.HTTP, TransferDirection.RECEIVE);
    setupMockProcessSession();
    final String peerUrl = "https://node1.example.com:8080/nifi";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 8080, true);
    final HttpCommunicationsSession commsSession = new HttpCommunicationsSession();
    commsSession.setUserDn("nifi.node1.example.com");
    final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
    final String flowFileEndpointUri = "https://node1.example.com:8080/nifi-api/output-ports/port-id/transactions/transaction-id/flow-files";
    doReturn(peer).when(transaction).getCommunicant();
    commsSession.setDataTransferUrl(flowFileEndpointUri);
    final Map<String, String> attributes = new HashMap<>();
    final byte[] dataPacketContents = "DataPacket Contents".getBytes();
    final ByteArrayInputStream dataPacketInputStream = new ByteArrayInputStream(dataPacketContents);
    final DataPacket dataPacket = new StandardDataPacket(attributes, dataPacketInputStream, dataPacketContents.length);
    // Return null when it gets called second time.
    doReturn(dataPacket).doReturn(null).when(transaction).receive();
    port.onTrigger(processContext, processSession);
    // Assert provenance.
    final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
    assertEquals(1, provenanceEvents.size());
    final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
    assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
    assertEquals(flowFileEndpointUri, provenanceEvent.getTransitUri());
    assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
    // Assert received flow files.
    processSession.assertAllFlowFilesTransferred(Relationship.ANONYMOUS);
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(Relationship.ANONYMOUS);
    assertEquals(1, flowFiles.size());
    final MockFlowFile flowFile = flowFiles.get(0);
    flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_HOST.key(), peer.getHost());
    flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_ADDRESS.key(), peer.getHost() + ":" + peer.getPort());
}