Examples with ReplaceRequest org.apache.metron.indexing.dao.update.ReplaceRequest used on opensource projects

Example 1 with ReplaceRequest

use of org.apache.metron.indexing.dao.update.ReplaceRequest in project metron by apache.

the class ElasticsearchUpdateIntegrationTest method test.

@Test
public void test() throws Exception {
    List<Map<String, Object>> inputData = new ArrayList<>();
    for (int i = 0; i < 10; ++i) {
        final String name = "message" + i;
        inputData.add(new HashMap<String, Object>() {

            {
                put("source:type", SENSOR_NAME);
                put("name", name);
                put("timestamp", System.currentTimeMillis());
                put(Constants.GUID, name);
            }
        });
    }
    es.add(index, SENSOR_NAME, Iterables.transform(inputData, m -> {
        try {
            return JSONUtils.INSTANCE.toJSON(m, true);
        } catch (JsonProcessingException e) {
            throw new IllegalStateException(e.getMessage(), e);
        }
    }));
    List<Map<String, Object>> docs = null;
    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
        docs = es.getAllIndexedDocs(index, SENSOR_NAME + "_doc");
        if (docs.size() >= 10) {
            break;
        }
    }
    Assert.assertEquals(10, docs.size());
    // modify the first message and add a new field
    {
        Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) {

            {
                put("new-field", "metron");
            }
        };
        String guid = "" + message0.get(Constants.GUID);
        dao.replace(new ReplaceRequest() {

            {
                setReplacement(message0);
                setGuid(guid);
                setSensorType(SENSOR_NAME);
            }
        }, Optional.empty());
        Assert.assertEquals(1, table.size());
        Document doc = dao.getLatest(guid, SENSOR_NAME);
        Assert.assertEquals(message0, doc.getDocument());
        {
            // ensure hbase is up to date
            Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
            Result r = table.get(g);
            NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
            Assert.assertEquals(1, columns.size());
            Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue()), JSONUtils.MAP_SUPPLIER));
        }
        {
            // ensure ES is up-to-date
            long cnt = 0;
            for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
                docs = es.getAllIndexedDocs(index, SENSOR_NAME + "_doc");
                cnt = docs.stream().filter(d -> message0.get("new-field").equals(d.get("new-field"))).count();
            }
            Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
        }
    }
    // modify the same message and modify the new field
    {
        Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) {

            {
                put("new-field", "metron2");
            }
        };
        String guid = "" + message0.get(Constants.GUID);
        dao.replace(new ReplaceRequest() {

            {
                setReplacement(message0);
                setGuid(guid);
                setSensorType(SENSOR_NAME);
            }
        }, Optional.empty());
        Assert.assertEquals(1, table.size());
        Document doc = dao.getLatest(guid, SENSOR_NAME);
        Assert.assertEquals(message0, doc.getDocument());
        {
            // ensure hbase is up to date
            Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
            Result r = table.get(g);
            NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
            Assert.assertEquals(2, columns.size());
            Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue()), JSONUtils.MAP_SUPPLIER));
            Assert.assertNotEquals(message0, JSONUtils.INSTANCE.load(new String(columns.firstEntry().getValue()), JSONUtils.MAP_SUPPLIER));
        }
        {
            // ensure ES is up-to-date
            long cnt = 0;
            for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
                docs = es.getAllIndexedDocs(index, SENSOR_NAME + "_doc");
                cnt = docs.stream().filter(d -> message0.get("new-field").equals(d.get("new-field"))).count();
            }
            Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
        }
    }
}