Examples with StandaloneStatusReporter datawave.ingest.test.StandaloneStatusReporter used on opensource projects

Example 1 with StandaloneStatusReporter

use of datawave.ingest.test.StandaloneStatusReporter in project datawave by NationalSecurityAgency.

the class ColumnBasedHandlerTestUtil method processEvent.

public static void processEvent(DataTypeHandler<Text> handler, ExtendedDataTypeHandler<Text, BulkIngestKey, Value> edgeHandler, RawRecordContainer event, int expectedShardKeys, int expectedShardIndexKeys, int expectedShardReverseIndexKeys, int expectedEdgeKeys, boolean printKeysOnlyOnFail) {
    Assert.assertNotNull("Event was null.", event);
    Multimap<String, NormalizedContentInterface> eventFields = handler.getHelper(event.getDataType()).getEventFields(event);
    VirtualIngest vHelper = (VirtualIngest) handler.getHelper(event.getDataType());
    Multimap<String, NormalizedContentInterface> virtualFields = vHelper.getVirtualFields(eventFields);
    for (Map.Entry<String, NormalizedContentInterface> v : virtualFields.entries()) {
        eventFields.put(v.getKey(), v.getValue());
    }
    Multimap<BulkIngestKey, Value> results = handler.processBulk(new Text(), event, eventFields, new MockStatusReporter());
    Set<Key> shardKeys = new HashSet<>();
    Set<Key> shardIndexKeys = new HashSet<>();
    Set<Key> shardReverseIndexKeys = new HashSet<>();
    Set<Key> edgeKeys = new HashSet<>();
    Map<Text, Integer> countMap = Maps.newHashMap();
    for (BulkIngestKey k : results.keySet()) {
        Text tableName = k.getTableName();
        if (countMap.containsKey(tableName)) {
            countMap.put(tableName, countMap.get(tableName) + 1);
        } else {
            countMap.put(tableName, 1);
        }
    }
    for (Map.Entry<BulkIngestKey, Value> e : results.entries()) {
        BulkIngestKey bik = e.getKey();
        if (log.isDebugEnabled() && isDocumentKey(bik.getKey())) {
            log.debug("Found Document Key: " + bik.getKey());
            log.debug("value:\n" + e.getValue());
        }
        if (bik.getTableName().equals(shardTableName)) {
            shardKeys.add(bik.getKey());
        } else if (bik.getTableName().equals(shardIndexTableName)) {
            shardIndexKeys.add(bik.getKey());
        } else if (bik.getTableName().equals(shardReverseIndexTableName)) {
            shardReverseIndexKeys.add(bik.getKey());
        } else {
            Assert.fail("unknown table: " + bik.getTableName() + " key: " + bik.getKey());
        }
    }
    // Process edges
    countMap.put(edgeTableName, 0);
    if (null != edgeHandler) {
        MyCachingContextWriter contextWriter = new MyCachingContextWriter();
        StandaloneTaskAttemptContext<Text, RawRecordContainerImpl, BulkIngestKey, Value> ctx = new StandaloneTaskAttemptContext<>(((RawRecordContainerImpl) event).getConf(), new StandaloneStatusReporter());
        try {
            contextWriter.setup(ctx.getConfiguration(), false);
            edgeHandler.process(null, event, eventFields, ctx, contextWriter);
            contextWriter.commit(ctx);
            for (Map.Entry<BulkIngestKey, Value> entry : contextWriter.getCache().entries()) {
                if (entry.getKey().getTableName().equals(edgeTableName)) {
                    edgeKeys.add(entry.getKey().getKey());
                }
                if (countMap.containsKey(entry.getKey().getTableName())) {
                    countMap.put(entry.getKey().getTableName(), countMap.get(entry.getKey().getTableName()) + 1);
                } else {
                    countMap.put(entry.getKey().getTableName(), 1);
                }
            }
        } catch (Throwable t) {
            log.error("Error during edge processing", t);
            throw new RuntimeException(t);
        }
    }
    Set<String> keyPrint = new TreeSet<>();
    for (Key k : shardKeys) {
        keyPrint.add("shard key: " + k.getRow() + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    // check index keys
    for (Key k : shardIndexKeys) {
        keyPrint.add("shardIndex key: " + k.getRow() + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    // check reverse index keys
    for (Key k : shardReverseIndexKeys) {
        keyPrint.add("reverseShardIndex key: " + k.getRow() + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    // check edge keys
    for (Key k : edgeKeys) {
        keyPrint.add("edge key: " + k.getRow().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    try {
        if (!printKeysOnlyOnFail) {
            for (String keyString : keyPrint) {
                log.info(keyString.trim());
            }
        }
        Assert.assertTrue(countMap.get(shardTableName) == expectedShardKeys && countMap.get(shardIndexTableName) == expectedShardIndexKeys && countMap.get(shardReverseIndexTableName) == expectedShardReverseIndexKeys && countMap.get(edgeTableName) == expectedEdgeKeys);
    } catch (AssertionError ae) {
        if (printKeysOnlyOnFail) {
            for (String keyString : keyPrint) {
                log.info(keyString.trim());
            }
        }
        Assert.fail(String.format("Expected: %s shard, %s index, %s reverse index, and %s edge keys.\nFound: %s, %s, %s, and %s respectively", expectedShardKeys, expectedShardIndexKeys, expectedShardReverseIndexKeys, expectedEdgeKeys, countMap.get(shardTableName), countMap.get(shardIndexTableName), countMap.get(shardReverseIndexTableName), countMap.get(edgeTableName)));
    }
}

Example 2 with StandaloneStatusReporter

use of datawave.ingest.test.StandaloneStatusReporter in project datawave by NationalSecurityAgency.

the class AccumuloSetup method ingestTestData.

private void ingestTestData(Configuration conf, TestFileLoader loader) throws IOException, InterruptedException {
    log.debug("------------- ingestTestData -------------");
    Path tmpPath = new Path(tempFolder.toURI());
    // To prevent periodic test cases failing, added "---" prefix for UUID for test cases to support queries with _ANYFIELD_ starting with particular
    // letters.
    Path seqFile = new Path(tmpPath, "---" + UUID.randomUUID().toString());
    TaskAttemptID id = new TaskAttemptID("testJob", 0, TaskType.MAP, 0, 0);
    TaskAttemptContext context = new TaskAttemptContextImpl(conf, id);
    try (final RawLocalFileSystem rfs = createSequenceFile(conf, seqFile, loader)) {
        InputSplit split = new FileSplit(seqFile, 0, rfs.pathToFile(seqFile).length(), null);
        EventSequenceFileRecordReader<LongWritable> rr = new EventSequenceFileRecordReader<>();
        rr.initialize(split, context);
        Path ocPath = new Path(tmpPath, "oc");
        OutputCommitter oc = new FileOutputCommitter(ocPath, context);
        rfs.deleteOnExit(ocPath);
        StandaloneStatusReporter sr = new StandaloneStatusReporter();
        EventMapper<LongWritable, RawRecordContainer, Text, Mutation> mapper = new EventMapper<>();
        MapContext<LongWritable, RawRecordContainer, Text, Mutation> mapContext = new MapContextImpl<>(conf, id, rr, this.recordWriter, oc, sr, split);
        Mapper<LongWritable, RawRecordContainer, Text, Mutation>.Context con = new WrappedMapper<LongWritable, RawRecordContainer, Text, Mutation>().getMapContext(mapContext);
        mapper.run(con);
        mapper.cleanup(con);
    } finally {
        this.recordWriter.close(context);
    }
}

Example 3 with StandaloneStatusReporter

use of datawave.ingest.test.StandaloneStatusReporter in project datawave by NationalSecurityAgency.

the class ColumnBasedHandlerTestUtil method processEvent.

public static void processEvent(DataTypeHandler<Text> handler, ExtendedDataTypeHandler<Text, BulkIngestKey, Value> edgeHandler, RawRecordContainer event, int expectedShardKeys, int expectedShardIndexKeys, int expectedShardReverseIndexKeys, int expectedEdgeKeys, boolean printKeysOnlyOnFail) {
    Assert.assertNotNull("Event was null.", event);
    Multimap<String, NormalizedContentInterface> eventFields = handler.getHelper(event.getDataType()).getEventFields(event);
    VirtualIngest vHelper = (VirtualIngest) handler.getHelper(event.getDataType());
    Multimap<String, NormalizedContentInterface> virtualFields = vHelper.getVirtualFields(eventFields);
    for (Map.Entry<String, NormalizedContentInterface> v : virtualFields.entries()) {
        eventFields.put(v.getKey(), v.getValue());
    }
    if (vHelper instanceof CompositeIngest) {
        CompositeIngest compIngest = (CompositeIngest) vHelper;
        Multimap<String, NormalizedContentInterface> compositeFields = compIngest.getCompositeFields(eventFields);
        for (String fieldName : compositeFields.keySet()) {
            // if this is an overloaded event field, we are replacing the existing data
            if (compIngest.isOverloadedCompositeField(fieldName))
                eventFields.removeAll(fieldName);
            eventFields.putAll(fieldName, compositeFields.get(fieldName));
        }
    }
    Multimap<BulkIngestKey, Value> results = handler.processBulk(new Text(), event, eventFields, new MockStatusReporter());
    Set<Key> shardKeys = new HashSet<>();
    Set<Key> shardIndexKeys = new HashSet<>();
    Set<Key> shardReverseIndexKeys = new HashSet<>();
    Set<Key> edgeKeys = new HashSet<>();
    Map<Text, Integer> countMap = Maps.newHashMap();
    for (BulkIngestKey k : results.keySet()) {
        Text tableName = k.getTableName();
        if (countMap.containsKey(tableName)) {
            countMap.put(tableName, countMap.get(tableName) + 1);
        } else {
            countMap.put(tableName, 1);
        }
    }
    for (Map.Entry<BulkIngestKey, Value> e : results.entries()) {
        BulkIngestKey bik = e.getKey();
        if (log.isDebugEnabled() && isDocumentKey(bik.getKey())) {
            log.debug("Found Document Key: " + bik.getKey());
            log.debug("value:\n" + e.getValue());
        }
        if (bik.getTableName().equals(shardTableName)) {
            shardKeys.add(bik.getKey());
        } else if (bik.getTableName().equals(shardIndexTableName)) {
            shardIndexKeys.add(bik.getKey());
        } else if (bik.getTableName().equals(shardReverseIndexTableName)) {
            shardReverseIndexKeys.add(bik.getKey());
        } else {
            Assert.fail("unknown table: " + bik.getTableName() + " key: " + bik.getKey());
        }
    }
    // Process edges
    countMap.put(edgeTableName, 0);
    if (null != edgeHandler) {
        MyCachingContextWriter contextWriter = new MyCachingContextWriter();
        StandaloneTaskAttemptContext<Text, RawRecordContainerImpl, BulkIngestKey, Value> ctx = new StandaloneTaskAttemptContext<>(((RawRecordContainerImpl) event).getConf(), new StandaloneStatusReporter());
        try {
            contextWriter.setup(ctx.getConfiguration(), false);
            edgeHandler.process(null, event, eventFields, ctx, contextWriter);
            contextWriter.commit(ctx);
            for (Map.Entry<BulkIngestKey, Value> entry : contextWriter.getCache().entries()) {
                if (entry.getKey().getTableName().equals(edgeTableName)) {
                    edgeKeys.add(entry.getKey().getKey());
                }
                if (countMap.containsKey(entry.getKey().getTableName())) {
                    countMap.put(entry.getKey().getTableName(), countMap.get(entry.getKey().getTableName()) + 1);
                } else {
                    countMap.put(entry.getKey().getTableName(), 1);
                }
            }
        } catch (Throwable t) {
            log.error("Error during edge processing", t);
            throw new RuntimeException(t);
        }
    }
    Set<String> keyPrint = new TreeSet<>();
    for (Key k : shardKeys) {
        keyPrint.add("shard key: " + k.getRow() + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    // check index keys
    for (Key k : shardIndexKeys) {
        keyPrint.add("shardIndex key: " + k.getRow() + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    // check reverse index keys
    for (Key k : shardReverseIndexKeys) {
        keyPrint.add("reverseShardIndex key: " + k.getRow() + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    // check edge keys
    for (Key k : edgeKeys) {
        keyPrint.add("edge key: " + k.getRow().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnFamily().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnQualifier().toString().replaceAll(NB, "%00;") + " ::: " + k.getColumnVisibility() + " ::: " + k.getTimestamp() + "\n");
    }
    try {
        if (!printKeysOnlyOnFail) {
            for (String keyString : keyPrint) {
                log.info(keyString.trim());
            }
        }
        if (expectedShardKeys > 0)
            Assert.assertEquals((int) countMap.get(shardTableName), expectedShardKeys);
        if (expectedShardIndexKeys > 0)
            Assert.assertEquals((int) countMap.get(shardIndexTableName), expectedShardIndexKeys);
        if (expectedShardReverseIndexKeys > 0)
            Assert.assertEquals((int) countMap.get(shardReverseIndexTableName), expectedShardReverseIndexKeys);
        if (expectedEdgeKeys > 0)
            Assert.assertEquals((int) countMap.get(edgeTableName), expectedEdgeKeys);
    } catch (AssertionError ae) {
        if (printKeysOnlyOnFail) {
            for (String keyString : keyPrint) {
                log.info(keyString.trim());
            }
        }
        Assert.fail(String.format("Expected: %s shard, %s index, %s reverse index, and %s edge keys.\nFound: %s, %s, %s, and %s respectively", expectedShardKeys, expectedShardIndexKeys, expectedShardReverseIndexKeys, expectedEdgeKeys, countMap.get(shardTableName), countMap.get(shardIndexTableName), countMap.get(shardReverseIndexTableName), countMap.get(edgeTableName)));
    }
}