Examples with MarkingFunctions datawave.marking.MarkingFunctions used on opensource projects

Example 1 with MarkingFunctions

use of datawave.marking.MarkingFunctions in project datawave by NationalSecurityAgency.

the class GroupingTest method testCountingMap.

@Test
public void testCountingMap() {
    MarkingFunctions markingFunctions = new MarkingFunctions.Default();
    GroupingTransform.GroupCountingHashMap map = new GroupingTransform.GroupCountingHashMap(markingFunctions);
    GroupingTypeAttribute attr1 = new GroupingTypeAttribute(new LcType("FOO"), new Key("FOO"), true);
    attr1.setColumnVisibility(new ColumnVisibility("A"));
    map.add(Collections.singleton(attr1));
    GroupingTypeAttribute attr2 = new GroupingTypeAttribute(new LcType("FOO"), new Key("FOO"), true);
    attr2.setColumnVisibility(new ColumnVisibility("B"));
    map.add(Collections.singleton(attr2));
    GroupingTypeAttribute attr3 = new GroupingTypeAttribute(new LcType("BAR"), new Key("BAR"), true);
    attr3.setColumnVisibility(new ColumnVisibility("C"));
    map.add(Collections.singleton(attr3));
    log.debug("map is: " + map);
    for (Map.Entry<Collection<GroupingTypeAttribute<?>>, Integer> entry : map.entrySet()) {
        // the first and only one
        Attribute<?> attr = entry.getKey().iterator().next();
        int count = entry.getValue();
        if (attr.getData().toString().equals("FOO")) {
            Assert.assertEquals(2, count);
            Assert.assertEquals(new ColumnVisibility("A&B"), attr.getColumnVisibility());
        } else if (attr.getData().toString().equals("BAR")) {
            Assert.assertEquals(1, count);
            Assert.assertEquals(new ColumnVisibility("C"), attr.getColumnVisibility());
        }
    }
}

Example 2 with MarkingFunctions

use of datawave.marking.MarkingFunctions in project datawave by NationalSecurityAgency.

the class ValueToAttributesTest method testComposites.

@Test
public void testComposites() {
    CompositeMetadata compositeMetadata = new CompositeMetadata();
    for (String ingestType : new String[] { "test", "pilot", "work", "beep", "tw" }) {
        compositeMetadata.setCompositeFieldMappingByType(ingestType, "MAKE_COLOR", Arrays.asList("MAKE", "COLOR"));
        compositeMetadata.setCompositeFieldMappingByType(ingestType, "COLOR_WHEELS", Arrays.asList("MAKE", "COLOR"));
    }
    TypeMetadata typeMetadata = new TypeMetadata("MAKE:[beep:datawave.data.type.LcNoDiacriticsType];MAKE_COLOR:[beep:datawave.data.type.NoOpType];START_DATE:[beep:datawave.data.type.DateType];TYPE_NOEVAL:[beep:datawave.data.type.LcNoDiacriticsType];IP_ADDR:[beep:datawave.data.type.IpAddressType];WHEELS:[beep:datawave.data.type.LcNoDiacriticsType,datawave.data.type.NumberType];COLOR:[beep:datawave.data.type.LcNoDiacriticsType];COLOR_WHEELS:[beep:datawave.data.type.NoOpType];TYPE:[beep:datawave.data.type.LcNoDiacriticsType]");
    MarkingFunctions markingFunctions = new MarkingFunctions.Default();
    ValueToAttributes valueToAttributes = new ValueToAttributes(compositeMetadata, typeMetadata, null, markingFunctions, true);
}

Example 3 with MarkingFunctions

use of datawave.marking.MarkingFunctions in project datawave by NationalSecurityAgency.

the class ErrorDataTypeHandler method setup.

@Override
public void setup(TaskAttemptContext context) {
    markingFunctions = MarkingFunctions.Factory.createMarkingFunctions();
    IngestConfiguration ingestConfiguration = IngestConfigurationFactory.getIngestConfiguration();
    markingsHelper = ingestConfiguration.getMarkingsHelper(context.getConfiguration(), TypeRegistry.getType(TypeRegistry.ERROR_PREFIX));
    tableName = ConfigurationHelper.isNull(context.getConfiguration(), ERROR_TABLE_NAME, String.class);
    this.conf = context.getConfiguration();
    try {
        Map<String, String> defaultMarkings = markingsHelper.getDefaultMarkings();
        defaultVisibility = flatten(markingFunctions.translateToColumnVisibility(defaultMarkings));
    } catch (MarkingFunctions.Exception e) {
        throw new IllegalArgumentException("Failed to convert default markings to a ColumnVisibility.", e);
    }
    // Initialize a UID builder based on the configuration
    uidBuilder = UID.builder(conf);
    log.info("ErrorDataTypeHandler configured.");
}

Example 4 with MarkingFunctions

use of datawave.marking.MarkingFunctions in project datawave by NationalSecurityAgency.

the class GroupingTestWithModel method testCountingMapAgain.

@Test
public void testCountingMapAgain() {
    MarkingFunctions markingFunctions = new MarkingFunctions.Default();
    GroupingTransform.GroupCountingHashMap map = new GroupingTransform.GroupCountingHashMap(markingFunctions);
    GroupingTypeAttribute<?> attr1a = new GroupingTypeAttribute(new LcType("FOO"), new Key("NAME"), true);
    attr1a.setColumnVisibility(new ColumnVisibility("A"));
    GroupingTypeAttribute<?> attr1b = new GroupingTypeAttribute(new NumberType("5"), new Key("AGE"), true);
    attr1b.setColumnVisibility(new ColumnVisibility("C"));
    Set<GroupingTypeAttribute<?>> seta = Sets.newHashSet(attr1a, attr1b);
    map.add(seta);
    GroupingTypeAttribute<?> attr2a = new GroupingTypeAttribute(new LcType("FOO"), new Key("NAME"), true);
    attr2a.setColumnVisibility(new ColumnVisibility("B"));
    GroupingTypeAttribute<?> attr2b = new GroupingTypeAttribute(new NumberType("5"), new Key("AGE"), true);
    attr2b.setColumnVisibility(new ColumnVisibility("D"));
    Set<GroupingTypeAttribute<?>> setb = Sets.newHashSet(attr2a, attr2b);
    map.add(setb);
    // even though the ColumnVisibilities are different, the 2 collections seta and setb are 'equal' and generate the same hashCode
    Assert.assertEquals(seta.hashCode(), setb.hashCode());
    Assert.assertEquals(seta, setb);
    GroupingTypeAttribute attr3a = new GroupingTypeAttribute(new LcType("BAR"), new Key("NAME"), true);
    attr3a.setColumnVisibility(new ColumnVisibility("C"));
    GroupingTypeAttribute attr3b = new GroupingTypeAttribute(new NumberType("6"), new Key("AGE"), true);
    attr3b.setColumnVisibility(new ColumnVisibility("D"));
    map.add(Sets.newHashSet(attr3a, attr3b));
    log.debug("map is: " + map);
    for (Map.Entry<Collection<GroupingTypeAttribute<?>>, Integer> entry : map.entrySet()) {
        for (Attribute<?> attr : entry.getKey()) {
            int count = entry.getValue();
            if (attr.getData().toString().equals("FOO")) {
                Assert.assertEquals(2, count);
                // the ColumnVisibility for the key was changed to the merged value of the 2 items that were added to the map
                Assert.assertEquals(new ColumnVisibility("A&B"), attr.getColumnVisibility());
            } else if (attr.getData().toString().equals("5")) {
                Assert.assertEquals(2, count);
                // the ColumnVisibility for the key was changed to the merged value of the 2 items that were added to the map
                Assert.assertEquals(new ColumnVisibility("C&D"), attr.getColumnVisibility());
            } else if (attr.getData().toString().equals("BAR")) {
                Assert.assertEquals(1, count);
                Assert.assertEquals(new ColumnVisibility("C"), attr.getColumnVisibility());
            } else if (attr.getData().toString().equals("6")) {
                Assert.assertEquals(1, count);
                Assert.assertEquals(new ColumnVisibility("D"), attr.getColumnVisibility());
            }
        }
    }
}

Example 5 with MarkingFunctions

use of datawave.marking.MarkingFunctions in project datawave by NationalSecurityAgency.

the class GroupingTestWithModel method testCountingMap.

@Test
public void testCountingMap() {
    MarkingFunctions markingFunctions = new MarkingFunctions.Default();
    GroupingTransform.GroupCountingHashMap map = new GroupingTransform.GroupCountingHashMap(markingFunctions);
    GroupingTypeAttribute attr1 = new GroupingTypeAttribute(new LcType("FOO"), new Key("FOO"), true);
    attr1.setColumnVisibility(new ColumnVisibility("A"));
    map.add(Collections.singleton(attr1));
    GroupingTypeAttribute attr2 = new GroupingTypeAttribute(new LcType("FOO"), new Key("FOO"), true);
    attr2.setColumnVisibility(new ColumnVisibility("B"));
    map.add(Collections.singleton(attr2));
    GroupingTypeAttribute attr3 = new GroupingTypeAttribute(new LcType("BAR"), new Key("BAR"), true);
    attr3.setColumnVisibility(new ColumnVisibility("C"));
    map.add(Collections.singleton(attr3));
    log.debug("map is: " + map);
    for (Map.Entry<Collection<GroupingTypeAttribute<?>>, Integer> entry : map.entrySet()) {
        // the first and only one
        Attribute<?> attr = entry.getKey().iterator().next();
        int count = entry.getValue();
        if (attr.getData().toString().equals("FOO")) {
            Assert.assertEquals(2, count);
            Assert.assertEquals(new ColumnVisibility("A&B"), attr.getColumnVisibility());
        } else if (attr.getData().toString().equals("BAR")) {
            Assert.assertEquals(1, count);
            Assert.assertEquals(new ColumnVisibility("C"), attr.getColumnVisibility());
        }
    }
}