Examples with Transaction org.apache.nifi.remote.Transaction used on opensource projects

Example 1 with Transaction

use of org.apache.nifi.remote.Transaction in project apex-malhar by apache.

the class AbstractNiFiOutputOperator method processTuples.

/**
 * Send the given batch of tuples to NiFi in a transaction, using the provided builder to
 * first convert each tuple into a NiFiDataPacket.
 *
 * @param tuples a list of tuples to process
 */
protected void processTuples(List<T> tuples) {
    if (tuples == null || tuples.size() == 0) {
        return;
    }
    // create a transaction and send the data packets
    try {
        final Transaction transaction = client.createTransaction(TransferDirection.SEND);
        if (transaction == null) {
            throw new IllegalStateException("Unable to create a NiFi Transaction to send data");
        }
        // convert each tuple to a NiFiDataPacket using the provided builder
        for (T tuple : tuples) {
            NiFiDataPacket dp = dataPacketBuilder.createNiFiDataPacket(tuple);
            transaction.send(dp.getContent(), dp.getAttributes());
        }
        transaction.confirm();
        transaction.complete();
    } catch (IOException ioe) {
        DTThrowable.rethrow(ioe);
    }
}

Example 2 with Transaction

use of org.apache.nifi.remote.Transaction in project apex-malhar by apache.

the class AbstractNiFiInputOperator method emitTuples.

@Override
public void emitTuples() {
    // clear the recovered list, and return until we have no more recovered data
    if (recoveredTuples.size() > 0) {
        emitTuples(recoveredTuples);
        recoveredTuples.clear();
        return;
    }
    // no recovered data so start a transaction and pull new data
    try {
        final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);
        if (transaction == null) {
            LOGGER.warn("A transaction could not be created, returning...");
            return;
        }
        DataPacket dataPacket = transaction.receive();
        if (dataPacket == null) {
            transaction.confirm();
            transaction.complete();
            LOGGER.debug("No data available to pull, returning and will try again...");
            return;
        }
        // read all of the available data packets and convert to the given type
        final List<T> tuples = new ArrayList<>();
        do {
            tuples.add(createTuple(dataPacket));
            dataPacket = transaction.receive();
        } while (dataPacket != null);
        // confirm all of the expected data was received by comparing check-sums, does not complete the transaction
        transaction.confirm();
        // ensure we have the data saved before proceeding in case anything goes wrong
        currentWindowTuples.addAll(tuples);
        windowDataManager.save(currentWindowTuples, currentWindowId);
        // we now have the data saved so we can complete the transaction
        transaction.complete();
        // delegate to sub-classes to emit the tuples
        emitTuples(tuples);
    } catch (IOException e) {
        DTThrowable.rethrow(e);
    }
}

Example 3 with Transaction

use of org.apache.nifi.remote.Transaction in project flink by apache.

the class NiFiSink method invoke.

@Override
public void invoke(T value) throws Exception {
    final NiFiDataPacket niFiDataPacket = builder.createNiFiDataPacket(value, getRuntimeContext());
    final Transaction transaction = client.createTransaction(TransferDirection.SEND);
    if (transaction == null) {
        throw new IllegalStateException("Unable to create a NiFi Transaction to send data");
    }
    transaction.send(niFiDataPacket.getContent(), niFiDataPacket.getAttributes());
    transaction.confirm();
    transaction.complete();
}

Example 4 with Transaction

use of org.apache.nifi.remote.Transaction in project flink by apache.

the class NiFiSource method run.

@Override
public void run(SourceContext<NiFiDataPacket> ctx) throws Exception {
    while (isRunning) {
        final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);
        if (transaction == null) {
            LOG.warn("A transaction could not be created, waiting and will try again...");
            try {
                Thread.sleep(waitTimeMs);
            } catch (InterruptedException ignored) {
            }
            continue;
        }
        DataPacket dataPacket = transaction.receive();
        if (dataPacket == null) {
            transaction.confirm();
            transaction.complete();
            LOG.debug("No data available to pull, waiting and will try again...");
            try {
                Thread.sleep(waitTimeMs);
            } catch (InterruptedException ignored) {
            }
            continue;
        }
        final List<NiFiDataPacket> niFiDataPackets = new ArrayList<>();
        do {
            // Read the data into a byte array and wrap it along with the attributes
            // into a NiFiDataPacket.
            final InputStream inStream = dataPacket.getData();
            final byte[] data = new byte[(int) dataPacket.getSize()];
            StreamUtils.fillBuffer(inStream, data);
            final Map<String, String> attributes = dataPacket.getAttributes();
            niFiDataPackets.add(new StandardNiFiDataPacket(data, attributes));
            dataPacket = transaction.receive();
        } while (dataPacket != null);
        // Confirm transaction to verify the data
        transaction.confirm();
        for (NiFiDataPacket dp : niFiDataPackets) {
            ctx.collect(dp);
        }
        transaction.complete();
    }
}