Examples with FlowFile org.apache.nifi.flowfile.FlowFile used on opensource projects

Example 76 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class GeoEnrichIP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final DatabaseReader dbReader = databaseReaderRef.get();
    final String ipAttributeName = context.getProperty(IP_ADDRESS_ATTRIBUTE).evaluateAttributeExpressions(flowFile).getValue();
    final String ipAttributeValue = flowFile.getAttribute(ipAttributeName);
    if (StringUtils.isEmpty(ipAttributeName)) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("FlowFile '{}' attribute '{}' was empty. Routing to failure", new Object[] { flowFile, IP_ADDRESS_ATTRIBUTE.getDisplayName() });
        return;
    }
    InetAddress inetAddress = null;
    CityResponse response = null;
    try {
        inetAddress = InetAddress.getByName(ipAttributeValue);
    } catch (final IOException ioe) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Could not resolve the IP for value '{}', contained within the attribute '{}' in " + "FlowFile '{}'. This is usually caused by issue resolving the appropriate DNS record or " + "providing the processor with an invalid IP address ", new Object[] { ipAttributeValue, IP_ADDRESS_ATTRIBUTE.getDisplayName(), flowFile }, ioe);
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    try {
        response = dbReader.city(inetAddress);
        stopWatch.stop();
    } catch (final IOException | GeoIp2Exception ex) {
        // Note IOException is captured again as dbReader also makes InetAddress.getByName() calls.
        // Most name or IP resolutions failure should have been triggered in the try loop above but
        // environmental conditions may trigger errors during the second resolution as well.
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Failure while trying to find enrichment data for {} due to {}", new Object[] { flowFile, ex }, ex);
        return;
    }
    if (response == null) {
        session.transfer(flowFile, REL_NOT_FOUND);
        return;
    }
    final Map<String, String> attrs = new HashMap<>();
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.lookup.micros").toString(), String.valueOf(stopWatch.getDuration(TimeUnit.MICROSECONDS)));
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.city").toString(), response.getCity().getName());
    final Double latitude = response.getLocation().getLatitude();
    if (latitude != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.latitude").toString(), latitude.toString());
    }
    final Double longitude = response.getLocation().getLongitude();
    if (longitude != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.longitude").toString(), longitude.toString());
    }
    final Integer accuracy = response.getLocation().getAccuracyRadius();
    if (accuracy != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".accuracy").toString(), String.valueOf(accuracy));
    }
    int i = 0;
    for (final Subdivision subd : response.getSubdivisions()) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.subdivision.").append(i).toString(), subd.getName());
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.subdivision.isocode.").append(i).toString(), subd.getIsoCode());
        i++;
    }
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.country").toString(), response.getCountry().getName());
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.country.isocode").toString(), response.getCountry().getIsoCode());
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.postalcode").toString(), response.getPostal().getCode());
    flowFile = session.putAllAttributes(flowFile, attrs);
    session.transfer(flowFile, REL_FOUND);
}

Example 77 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class ISPEnrichIP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final DatabaseReader dbReader = databaseReaderRef.get();
    final String ipAttributeName = context.getProperty(IP_ADDRESS_ATTRIBUTE).evaluateAttributeExpressions(flowFile).getValue();
    final String ipAttributeValue = flowFile.getAttribute(ipAttributeName);
    if (StringUtils.isEmpty(ipAttributeName)) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("FlowFile '{}' attribute '{}' was empty. Routing to failure", new Object[] { flowFile, IP_ADDRESS_ATTRIBUTE.getDisplayName() });
        return;
    }
    InetAddress inetAddress = null;
    IspResponse response = null;
    try {
        inetAddress = InetAddress.getByName(ipAttributeValue);
    } catch (final IOException ioe) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Could not resolve the IP for value '{}', contained within the attribute '{}' in " + "FlowFile '{}'. This is usually caused by issue resolving the appropriate DNS record or " + "providing the processor with an invalid IP address ", new Object[] { ipAttributeValue, IP_ADDRESS_ATTRIBUTE.getDisplayName(), flowFile }, ioe);
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    try {
        response = dbReader.isp(inetAddress);
        stopWatch.stop();
    } catch (final IOException | GeoIp2Exception ex) {
        // Note IOException is captured again as dbReader also makes InetAddress.getByName() calls.
        // Most name or IP resolutions failure should have been triggered in the try loop above but
        // environmental conditions may trigger errors during the second resolution as well.
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Failure while trying to find enrichment data for {} due to {}", new Object[] { flowFile, ex }, ex);
        return;
    }
    if (response == null) {
        session.transfer(flowFile, REL_NOT_FOUND);
        return;
    }
    final Map<String, String> attrs = new HashMap<>();
    attrs.put(new StringBuilder(ipAttributeName).append(".isp.lookup.micros").toString(), String.valueOf(stopWatch.getDuration(TimeUnit.MICROSECONDS)));
    // seem like good option to "final int asn ..." as with the other returned data.
    if (!(response.getAutonomousSystemNumber() == null)) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.asn").toString(), String.valueOf(response.getAutonomousSystemNumber()));
    }
    final String asnOrg = response.getAutonomousSystemOrganization();
    if (asnOrg != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.asn.organization").toString(), asnOrg);
    }
    final String ispName = response.getIsp();
    if (ispName != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.name").toString(), ispName);
    }
    final String organisation = response.getOrganization();
    if (organisation != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.organization").toString(), organisation);
    }
    flowFile = session.putAllAttributes(flowFile, attrs);
    session.transfer(flowFile, REL_FOUND);
}

Example 78 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class QueryWhois method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
    List<FlowFile> flowFiles = session.get(batchSize);
    if (flowFiles == null || flowFiles.isEmpty()) {
        context.yield();
        return;
    }
    // Build query
    String buildString = "";
    final String queryType = context.getProperty(WHOIS_QUERY_TYPE).getValue();
    // Verify the the protocol mode and craft the "begin" pseudo-command, otherwise just the query type
    buildString = context.getProperty(BULK_PROTOCOL).getValue().equals(BEGIN_END.getValue()) ? buildString.concat("begin ") : buildString.concat("");
    // Append the query type
    buildString = context.getProperty(WHOIS_QUERY_TYPE).isSet() ? buildString.concat(queryType + " ") : buildString.concat("");
    // A new line is required when working on Begin/End
    buildString = context.getProperty(BULK_PROTOCOL).getValue().equals(BEGIN_END.getValue()) ? buildString.concat("\n") : buildString.concat("");
    // append the values
    for (FlowFile flowFile : flowFiles) {
        final String evaluatedInput = context.getProperty(QUERY_INPUT).evaluateAttributeExpressions(flowFile).getValue();
        buildString = buildString + evaluatedInput + "\n";
    }
    // Verify the the protocol mode and craft the "end" pseudo-command, otherwise just the query type
    buildString = context.getProperty(BULK_PROTOCOL).getValue().equals(BEGIN_END.getValue()) ? buildString.concat("end") : buildString.concat("");
    final String queryParser = context.getProperty(QUERY_PARSER).getValue();
    final String queryRegex = context.getProperty(QUERY_PARSER_INPUT).getValue();
    final int keyLookup = context.getProperty(KEY_GROUP).asInteger();
    final int whoisTimeout = context.getProperty(WHOIS_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    final String whoisServer = context.getProperty(WHOIS_SERVER).getValue();
    final int whoisPort = context.getProperty(WHOIS_SERVER_PORT).asInteger();
    final List<FlowFile> flowFilesMatched = new ArrayList<FlowFile>();
    final List<FlowFile> flowFilesNotMatched = new ArrayList<FlowFile>();
    String result = doLookup(whoisServer, whoisPort, whoisTimeout, buildString);
    if (StringUtils.isEmpty(result)) {
        // If nothing was returned, let the processor continue its life and transfer the batch to REL_NOT_FOUND
        session.transfer(flowFiles, REL_NOT_FOUND);
        return;
    } else {
        // Run as normal
        for (FlowFile flowFile : flowFiles) {
            // Check the batchSize. If 1, run normal parser
            if (batchSize == 1) {
                Map<String, String> parsedResults = parseResponse(null, result, queryParser, queryRegex, "whois");
                if (parsedResults.isEmpty()) {
                    // parsedResults didn't return anything valid, sending to not found.
                    flowFilesNotMatched.add(flowFile);
                } else {
                    // Still, extraction is needed
                    flowFile = session.putAllAttributes(flowFile, parsedResults);
                    flowFilesMatched.add(flowFile);
                // Finished processing single result
                }
            } else {
                // Otherwise call the multiline parser and get the row map;
                final Map<String, Map<String, String>> rowMap = parseBatchResponse(result, queryParser, queryRegex, keyLookup, "whois").rowMap();
                // Identify the flowfile Lookupvalue and search against the rowMap
                String ffLookupValue = context.getProperty(QUERY_INPUT).evaluateAttributeExpressions(flowFile).getValue();
                if (rowMap.containsKey(ffLookupValue)) {
                    // flowfile Lookup Value is contained within the results, get the properties and add to matched list
                    flowFile = session.putAllAttributes(flowFile, rowMap.get(ffLookupValue));
                    flowFilesMatched.add(flowFile);
                } else {
                    // otherwise add to Not Matched
                    flowFilesNotMatched.add(flowFile);
                }
            }
        }
    }
    // each to its relationship
    if (flowFilesMatched.size() > 0) {
        // Sending the resulting flowfile (with attributes) to REL_FOUND
        session.transfer(flowFilesMatched, REL_FOUND);
    }
    if (flowFilesNotMatched.size() > 0) {
        // Sending whatetver didn't match to REL_NOT_FOUND
        session.transfer(flowFilesNotMatched, REL_NOT_FOUND);
    }
}

Example 79 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestGeoEnrichIP method successfulMaxMindResponseShouldFlowToFoundRelationshipWhenLatAndLongAreNotSet.

@Test
public void successfulMaxMindResponseShouldFlowToFoundRelationshipWhenLatAndLongAreNotSet() throws Exception {
    testRunner.setProperty(GeoEnrichIP.GEO_DATABASE_FILE, "./");
    testRunner.setProperty(GeoEnrichIP.IP_ADDRESS_ATTRIBUTE, "ip");
    final CityResponse cityResponse = getNullLatAndLongCityResponse();
    when(databaseReader.city(InetAddress.getByName("1.2.3.4"))).thenReturn(cityResponse);
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("ip", "1.2.3.4");
    testRunner.enqueue(new byte[0], attributes);
    testRunner.run();
    List<MockFlowFile> notFound = testRunner.getFlowFilesForRelationship(GeoEnrichIP.REL_NOT_FOUND);
    List<MockFlowFile> found = testRunner.getFlowFilesForRelationship(GeoEnrichIP.REL_FOUND);
    assertEquals(0, notFound.size());
    assertEquals(1, found.size());
    FlowFile finishedFound = found.get(0);
    assertNotNull(finishedFound.getAttribute("ip.geo.lookup.micros"));
    assertEquals("Minneapolis", finishedFound.getAttribute("ip.geo.city"));
    assertNull(finishedFound.getAttribute("ip.geo.latitude"));
    assertNull(finishedFound.getAttribute("ip.geo.longitude"));
    assertEquals("Minnesota", finishedFound.getAttribute("ip.geo.subdivision.0"));
    assertEquals("MN", finishedFound.getAttribute("ip.geo.subdivision.isocode.0"));
    assertNull(finishedFound.getAttribute("ip.geo.subdivision.1"));
    assertEquals("TT", finishedFound.getAttribute("ip.geo.subdivision.isocode.1"));
    assertEquals("United States of America", finishedFound.getAttribute("ip.geo.country"));
    assertEquals("US", finishedFound.getAttribute("ip.geo.country.isocode"));
    assertEquals("55401", finishedFound.getAttribute("ip.geo.postalcode"));
}

Example 80 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestISPEnrichIP method successfulMaxMindResponseShouldFlowToFoundRelationship.

@Test
public void successfulMaxMindResponseShouldFlowToFoundRelationship() throws Exception {
    testRunner.setProperty(ISPEnrichIP.GEO_DATABASE_FILE, "./");
    testRunner.setProperty(ISPEnrichIP.IP_ADDRESS_ATTRIBUTE, "ip");
    final IspResponse ispResponse = getIspResponse();
    when(databaseReader.isp(InetAddress.getByName("1.2.3.4"))).thenReturn(ispResponse);
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("ip", "1.2.3.4");
    testRunner.enqueue(new byte[0], attributes);
    testRunner.run();
    List<MockFlowFile> notFound = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_NOT_FOUND);
    List<MockFlowFile> found = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_FOUND);
    assertEquals(0, notFound.size());
    assertEquals(1, found.size());
    FlowFile finishedFound = found.get(0);
    assertNotNull(finishedFound.getAttribute("ip.isp.lookup.micros"));
    assertEquals("Apache NiFi - Test ISP", finishedFound.getAttribute("ip.isp.name"));
    assertEquals("Apache NiFi - Test Organization", finishedFound.getAttribute("ip.isp.organization"));
    assertEquals("1337", finishedFound.getAttribute("ip.isp.asn"));
    assertEquals("Apache NiFi - Test Chocolate", finishedFound.getAttribute("ip.isp.asn.organization"));
}