Examples with StopWatch org.apache.nifi.util.StopWatch used on opensource projects

Example 6 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class GetAzureEventHub method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final BlockingQueue<String> partitionIds = this.partitionNames;
    final String partitionId = partitionIds.poll();
    if (partitionId == null) {
        getLogger().debug("No partitions available");
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    try {
        final Iterable<EventData> receivedEvents = receiveEvents(context, partitionId);
        if (receivedEvents == null) {
            return;
        }
        for (final EventData eventData : receivedEvents) {
            if (null != eventData) {
                final Map<String, String> attributes = new HashMap<>();
                FlowFile flowFile = session.create();
                final EventData.SystemProperties systemProperties = eventData.getSystemProperties();
                if (null != systemProperties) {
                    attributes.put("eventhub.enqueued.timestamp", String.valueOf(systemProperties.getEnqueuedTime()));
                    attributes.put("eventhub.offset", systemProperties.getOffset());
                    attributes.put("eventhub.sequence", String.valueOf(systemProperties.getSequenceNumber()));
                }
                attributes.put("eventhub.name", context.getProperty(EVENT_HUB_NAME).getValue());
                attributes.put("eventhub.partition", partitionId);
                flowFile = session.putAllAttributes(flowFile, attributes);
                flowFile = session.write(flowFile, out -> {
                    out.write(eventData.getBytes());
                });
                session.transfer(flowFile, REL_SUCCESS);
                final String namespace = context.getProperty(NAMESPACE).getValue();
                final String eventHubName = context.getProperty(EVENT_HUB_NAME).getValue();
                final String consumerGroup = context.getProperty(CONSUMER_GROUP).getValue();
                final String serviceBusEndPoint = context.getProperty(SERVICE_BUS_ENDPOINT).getValue();
                final String transitUri = "amqps://" + namespace + serviceBusEndPoint + "/" + eventHubName + "/ConsumerGroups/" + consumerGroup + "/Partitions/" + partitionId;
                session.getProvenanceReporter().receive(flowFile, transitUri, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
            }
        }
    } finally {
        partitionIds.offer(partitionId);
    }
}

Example 7 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class AbstractEnrichIP method onScheduled.

@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
    final String dbFileString = context.getProperty(GEO_DATABASE_FILE).getValue();
    final File dbFile = new File(dbFileString);
    final StopWatch stopWatch = new StopWatch(true);
    final DatabaseReader reader = new DatabaseReader.Builder(dbFile).build();
    stopWatch.stop();
    getLogger().info("Completed loading of Maxmind Database.  Elapsed time was {} milliseconds.", new Object[] { stopWatch.getDuration(TimeUnit.MILLISECONDS) });
    databaseReaderRef.set(reader);
}

Example 8 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class GeoEnrichIP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final DatabaseReader dbReader = databaseReaderRef.get();
    final String ipAttributeName = context.getProperty(IP_ADDRESS_ATTRIBUTE).evaluateAttributeExpressions(flowFile).getValue();
    final String ipAttributeValue = flowFile.getAttribute(ipAttributeName);
    if (StringUtils.isEmpty(ipAttributeName)) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("FlowFile '{}' attribute '{}' was empty. Routing to failure", new Object[] { flowFile, IP_ADDRESS_ATTRIBUTE.getDisplayName() });
        return;
    }
    InetAddress inetAddress = null;
    CityResponse response = null;
    try {
        inetAddress = InetAddress.getByName(ipAttributeValue);
    } catch (final IOException ioe) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Could not resolve the IP for value '{}', contained within the attribute '{}' in " + "FlowFile '{}'. This is usually caused by issue resolving the appropriate DNS record or " + "providing the processor with an invalid IP address ", new Object[] { ipAttributeValue, IP_ADDRESS_ATTRIBUTE.getDisplayName(), flowFile }, ioe);
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    try {
        response = dbReader.city(inetAddress);
        stopWatch.stop();
    } catch (final IOException | GeoIp2Exception ex) {
        // Note IOException is captured again as dbReader also makes InetAddress.getByName() calls.
        // Most name or IP resolutions failure should have been triggered in the try loop above but
        // environmental conditions may trigger errors during the second resolution as well.
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Failure while trying to find enrichment data for {} due to {}", new Object[] { flowFile, ex }, ex);
        return;
    }
    if (response == null) {
        session.transfer(flowFile, REL_NOT_FOUND);
        return;
    }
    final Map<String, String> attrs = new HashMap<>();
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.lookup.micros").toString(), String.valueOf(stopWatch.getDuration(TimeUnit.MICROSECONDS)));
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.city").toString(), response.getCity().getName());
    final Double latitude = response.getLocation().getLatitude();
    if (latitude != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.latitude").toString(), latitude.toString());
    }
    final Double longitude = response.getLocation().getLongitude();
    if (longitude != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.longitude").toString(), longitude.toString());
    }
    final Integer accuracy = response.getLocation().getAccuracyRadius();
    if (accuracy != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".accuracy").toString(), String.valueOf(accuracy));
    }
    int i = 0;
    for (final Subdivision subd : response.getSubdivisions()) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.subdivision.").append(i).toString(), subd.getName());
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.subdivision.isocode.").append(i).toString(), subd.getIsoCode());
        i++;
    }
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.country").toString(), response.getCountry().getName());
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.country.isocode").toString(), response.getCountry().getIsoCode());
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.postalcode").toString(), response.getPostal().getCode());
    flowFile = session.putAllAttributes(flowFile, attrs);
    session.transfer(flowFile, REL_FOUND);
}

Example 9 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class ISPEnrichIP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final DatabaseReader dbReader = databaseReaderRef.get();
    final String ipAttributeName = context.getProperty(IP_ADDRESS_ATTRIBUTE).evaluateAttributeExpressions(flowFile).getValue();
    final String ipAttributeValue = flowFile.getAttribute(ipAttributeName);
    if (StringUtils.isEmpty(ipAttributeName)) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("FlowFile '{}' attribute '{}' was empty. Routing to failure", new Object[] { flowFile, IP_ADDRESS_ATTRIBUTE.getDisplayName() });
        return;
    }
    InetAddress inetAddress = null;
    IspResponse response = null;
    try {
        inetAddress = InetAddress.getByName(ipAttributeValue);
    } catch (final IOException ioe) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Could not resolve the IP for value '{}', contained within the attribute '{}' in " + "FlowFile '{}'. This is usually caused by issue resolving the appropriate DNS record or " + "providing the processor with an invalid IP address ", new Object[] { ipAttributeValue, IP_ADDRESS_ATTRIBUTE.getDisplayName(), flowFile }, ioe);
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    try {
        response = dbReader.isp(inetAddress);
        stopWatch.stop();
    } catch (final IOException | GeoIp2Exception ex) {
        // Note IOException is captured again as dbReader also makes InetAddress.getByName() calls.
        // Most name or IP resolutions failure should have been triggered in the try loop above but
        // environmental conditions may trigger errors during the second resolution as well.
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Failure while trying to find enrichment data for {} due to {}", new Object[] { flowFile, ex }, ex);
        return;
    }
    if (response == null) {
        session.transfer(flowFile, REL_NOT_FOUND);
        return;
    }
    final Map<String, String> attrs = new HashMap<>();
    attrs.put(new StringBuilder(ipAttributeName).append(".isp.lookup.micros").toString(), String.valueOf(stopWatch.getDuration(TimeUnit.MICROSECONDS)));
    // seem like good option to "final int asn ..." as with the other returned data.
    if (!(response.getAutonomousSystemNumber() == null)) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.asn").toString(), String.valueOf(response.getAutonomousSystemNumber()));
    }
    final String asnOrg = response.getAutonomousSystemOrganization();
    if (asnOrg != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.asn.organization").toString(), asnOrg);
    }
    final String ispName = response.getIsp();
    if (ispName != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.name").toString(), ispName);
    }
    final String organisation = response.getOrganization();
    if (organisation != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.organization").toString(), organisation);
    }
    flowFile = session.putAllAttributes(flowFile, attrs);
    session.transfer(flowFile, REL_FOUND);
}

Example 10 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class PublishMQTT method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowfile = session.get();
    if (flowfile == null) {
        return;
    }
    if (mqttClient == null || !mqttClient.isConnected()) {
        logger.info("Was disconnected from client or was never connected, attempting to connect.");
        try {
            reconnect();
        } catch (MqttException e) {
            context.yield();
            session.transfer(flowfile, REL_FAILURE);
            logger.error("MQTT client is disconnected and re-connecting failed. Transferring FlowFile to fail and yielding", e);
            return;
        }
    }
    // get the MQTT topic
    String topic = context.getProperty(PROP_TOPIC).evaluateAttributeExpressions(flowfile).getValue();
    if (topic == null || topic.isEmpty()) {
        logger.warn("Evaluation of the topic property returned null or evaluated to be empty, routing to failure");
        session.transfer(flowfile, REL_FAILURE);
        return;
    }
    // do the read
    final byte[] messageContent = new byte[(int) flowfile.getSize()];
    session.read(flowfile, new InputStreamCallback() {

        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, messageContent, true);
        }
    });
    int qos = context.getProperty(PROP_QOS).evaluateAttributeExpressions(flowfile).asInteger();
    final MqttMessage mqttMessage = new MqttMessage(messageContent);
    mqttMessage.setQos(qos);
    mqttMessage.setPayload(messageContent);
    mqttMessage.setRetained(context.getProperty(PROP_RETAIN).evaluateAttributeExpressions(flowfile).asBoolean());
    try {
        mqttClientConnectLock.readLock().lock();
        final StopWatch stopWatch = new StopWatch(true);
        try {
            /*
                 * Underlying method waits for the message to publish (according to set QoS), so it executes synchronously:
                 *     MqttClient.java:361 aClient.publish(topic, message, null, null).waitForCompletion(getTimeToWait());
                 */
            mqttClient.publish(topic, mqttMessage);
        } finally {
            mqttClientConnectLock.readLock().unlock();
        }
        session.getProvenanceReporter().send(flowfile, broker, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowfile, REL_SUCCESS);
    } catch (MqttException me) {
        logger.error("Failed to publish message.", me);
        session.transfer(flowfile, REL_FAILURE);
    }
}