Examples with IpAddressType datawave.data.type.IpAddressType used on opensource projects

Example 1 with IpAddressType

use of datawave.data.type.IpAddressType in project datawave by NationalSecurityAgency.

the class NormalizationFunctions method ipv4.

public static String ipv4(Object ipField) throws NormalizationException {
    String ip = ValueTuple.getNormalizedStringValue(ipField);
    int index = ip.indexOf("..*");
    // is it a wildcard?
    if (index != -1) {
        String temp = ip.substring(0, index);
        if (log.isDebugEnabled()) {
            log.debug("wildcard ip: " + temp);
        }
        // zero padd temp and return with the .* on the end
        String[] octets = StringUtils.split(temp, ".");
        for (int i = 0; i < octets.length; i++) {
            int oct = Integer.parseInt(octets[i]);
            octets[i] = String.format("%03d", oct);
            if (log.isDebugEnabled()) {
                log.debug("octets[i]: " + octets[i] + "  oct: " + oct);
            }
        }
        ip = StringUtils.join(octets, ".") + "..*";
    } else {
        IpAddressType normalizer = new IpAddressType();
        ip = normalizer.normalize(ip);
    }
    return ip;
}

Example 2 with IpAddressType

use of datawave.data.type.IpAddressType in project datawave by NationalSecurityAgency.

the class ExpandMultiNormalizedTermsTest method testFilterFunctionNormalization.

@Test
public void testFilterFunctionNormalization() throws ParseException {
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.put("IP", new IpAddressType());
    helper.setIndexedFields(dataTypes.keySet());
    helper.setIndexOnlyFields(dataTypes.keySet());
    helper.addTermFrequencyFields(dataTypes.keySet());
    config.setQueryFieldsDatatypes(dataTypes);
    String original = "filter:includeRegex(IP, '1\\.2\\.3\\..*')";
    String expected = "filter:includeRegex(IP, '1\\\\.2\\\\.3\\\\..*')";
    expandTerms(original, expected);
}

Example 3 with IpAddressType

use of datawave.data.type.IpAddressType in project datawave by NationalSecurityAgency.

the class ExpandMultiNormalizedTerms method expandNodeForNormalizers.

/**
 * @param node
 * @param data
 * @return
 */
protected JexlNode expandNodeForNormalizers(JexlNode node, Object data) {
    JexlNode nodeToReturn = node;
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op != null) {
        final String fieldName = op.deconstructIdentifier();
        final Object literal = op.getLiteralValue();
        // Get all of the indexed or normalized dataTypes for the field name
        Set<Type<?>> dataTypes = Sets.newHashSet(config.getQueryFieldsDatatypes().get(fieldName));
        dataTypes.addAll(config.getNormalizedFieldsDatatypes().get(fieldName));
        // Catch the case of the user entering FIELD == null
        if (!dataTypes.isEmpty() && null != literal) {
            try {
                String term = literal.toString();
                Set<String> normalizedTerms = Sets.newHashSet();
                // Build up a set of normalized terms using each normalizer
                for (Type<?> normalizer : dataTypes) {
                    try {
                        if (node instanceof ASTNRNode || node instanceof ASTERNode) {
                            normalizedTerms.add(normalizer.normalizeRegex(term));
                        } else {
                            normalizedTerms.add(normalizer.normalize(term));
                        }
                        log.debug("normalizedTerms=" + normalizedTerms);
                    } catch (IpAddressNormalizer.Exception ipex) {
                        try {
                            String[] lowHi = ((IpAddressType) normalizer).normalizeCidrToRange(term);
                            // node was FIELD == 'cidr'
                            // change to FIELD >= low and FIELD <= hi
                            JexlNode geNode = JexlNodeFactory.buildNode(new ASTGENode(ParserTreeConstants.JJTGENODE), fieldName, lowHi[0]);
                            JexlNode leNode = JexlNodeFactory.buildNode(new ASTLENode(ParserTreeConstants.JJTLENODE), fieldName, lowHi[1]);
                            // now link em up
                            return BoundedRange.create(JexlNodeFactory.createAndNode(Arrays.asList(geNode, leNode)));
                        } catch (Exception ex) {
                            if (log.isTraceEnabled()) {
                                log.trace("Could not normalize " + term + " as cidr notation with: " + normalizer.getClass());
                            }
                        }
                    // this could be CIDR notation, attempt to expand the node to the cidr range
                    } catch (Exception ne) {
                        if (log.isTraceEnabled()) {
                            log.trace("Could not normalize " + term + " using " + normalizer.getClass());
                        }
                    }
                }
                if (normalizedTerms.size() > 1) {
                    nodeToReturn = JexlNodeFactory.createNodeTreeFromFieldValues(ContainerType.OR_NODE, node, node, fieldName, normalizedTerms);
                } else if (1 == normalizedTerms.size()) {
                    // If there is only one term, we don't need to make an OR
                    nodeToReturn = JexlNodeFactory.buildUntypedNewLiteralNode(node, fieldName, normalizedTerms.iterator().next());
                } else {
                    // If we couldn't map anything, return a copy
                    nodeToReturn = JexlNodeFactory.buildUntypedNewLiteralNode(node, fieldName, literal);
                }
            } catch (Exception e) {
                QueryException qe = new QueryException(DatawaveErrorCode.NODE_EXPANSION_ERROR, e, MessageFormat.format("Node: {0}, Datatypes: {1}", PrintingVisitor.formattedQueryString(node), dataTypes));
                log.error(qe);
                throw new DatawaveFatalQueryException(qe);
            }
        }
    }
    return nodeToReturn;
}