Examples with AuditBatchQueue org.apache.ranger.audit.queue.AuditBatchQueue used on opensource projects

Example 1 with AuditBatchQueue

use of org.apache.ranger.audit.queue.AuditBatchQueue in project ranger by apache.

the class AuditProviderFactory method getProviderFromConfig.

private AuditHandler getProviderFromConfig(Properties props, String propPrefix, String providerName, AuditHandler consumer) {
    AuditHandler provider = null;
    String className = MiscUtil.getStringProperty(props, propPrefix + "." + BaseAuditHandler.PROP_CLASS_NAME);
    if (className != null && !className.isEmpty()) {
        try {
            Class<?> handlerClass = Class.forName(className);
            if (handlerClass.isAssignableFrom(AuditQueue.class)) {
                // Queue class needs consumer
                handlerClass.getDeclaredConstructor(AuditHandler.class).newInstance(consumer);
            } else {
                provider = (AuditHandler) Class.forName(className).newInstance();
            }
        } catch (Exception e) {
            LOG.fatal("Can't instantiate audit class for providerName=" + providerName + ", className=" + className + ", propertyPrefix=" + propPrefix, e);
        }
    } else {
        if (providerName.equals("file")) {
            provider = new FileAuditDestination();
        } else if (providerName.equalsIgnoreCase("hdfs")) {
            provider = new HDFSAuditDestination();
        } else if (providerName.equals("solr")) {
            provider = new SolrAuditDestination();
        } else if (providerName.equals("kafka")) {
            provider = new KafkaAuditProvider();
        } else if (providerName.equals("db")) {
            provider = new DBAuditDestination();
        } else if (providerName.equals("log4j")) {
            provider = new Log4JAuditDestination();
        } else if (providerName.equals("batch")) {
            provider = new AuditBatchQueue(consumer);
        } else if (providerName.equals("async")) {
            provider = new AuditAsyncQueue(consumer);
        } else {
            LOG.error("Provider name doesn't have any class associated with it. providerName=" + providerName + ", propertyPrefix=" + propPrefix);
        }
    }
    if (provider != null && provider instanceof AuditQueue) {
        if (consumer == null) {
            LOG.fatal("consumer can't be null for AuditQueue. queue=" + provider.getName() + ", propertyPrefix=" + propPrefix);
            provider = null;
        }
    }
    return provider;
}

Example 2 with AuditBatchQueue

use of org.apache.ranger.audit.queue.AuditBatchQueue in project ranger by apache.

the class TestAuditQueue method testAuditBatchQueueByTime.

@Test
public void testAuditBatchQueueByTime() {
    logger.debug("testAuditBatchQueue()...");
    int messageToSend = 10;
    String basePropName = "testAuditBatchQueueByTime_" + MiscUtil.generateUniqueId();
    // Deliberately big size
    int batchSize = messageToSend * 2;
    int queueSize = messageToSend * 2;
    // e.g (1000/10 * 3) = 300
    int intervalMS = (1000 / messageToSend) * 3;
    // ms
    // e.g. 1000/10 + 3 = 103ms
    int pauseMS = 1000 / messageToSend + 3;
    int expectedBatchSize = (messageToSend * pauseMS) / intervalMS + 1;
    Properties props = new Properties();
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_SIZE, "" + batchSize);
    props.put(basePropName + "." + AuditQueue.PROP_QUEUE_SIZE, "" + queueSize);
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_INTERVAL, "" + intervalMS);
    TestConsumer testConsumer = new TestConsumer();
    AuditBatchQueue queue = new AuditBatchQueue(testConsumer);
    queue.init(props, basePropName);
    queue.start();
    for (int i = 0; i < messageToSend; i++) {
        queue.log(createEvent());
        try {
            Thread.sleep(pauseMS);
        } catch (InterruptedException e) {
        // ignore
        }
    }
    // Let's wait for second
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    queue.waitToComplete();
    queue.stop();
    queue.waitToComplete();
    assertEquals("Total count", messageToSend, testConsumer.getCountTotal());
    assertEquals("Total sum", messageToSend, testConsumer.getSumTotal());
    assertEquals("Total batch", expectedBatchSize, testConsumer.getBatchCount());
    assertNull("Event not in sequnce", testConsumer.isInSequence());
}

Example 3 with AuditBatchQueue

use of org.apache.ranger.audit.queue.AuditBatchQueue in project ranger by apache.

the class TestAuditQueue method testAuditBatchQueueDestDownFlipFlop.

@Test
public void testAuditBatchQueueDestDownFlipFlop() {
    logger.debug("testAuditBatchQueueDestDownFlipFlop()...");
    int messageToSend = 10;
    String basePropName = "testAuditBatchQueueDestDownFlipFlop_" + MiscUtil.generateUniqueId();
    int batchSize = messageToSend / 3;
    int queueSize = messageToSend * 2;
    // Deliberately big interval
    int intervalMS = 5000;
    Properties props = new Properties();
    props.put(basePropName + "." + BaseAuditHandler.PROP_NAME, "testAuditBatchQueueDestDownFlipFlop_" + MiscUtil.generateUniqueId());
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_SIZE, "" + batchSize);
    props.put(basePropName + "." + AuditQueue.PROP_QUEUE_SIZE, "" + queueSize);
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_INTERVAL, "" + intervalMS);
    // Enable File Spooling
    int destRetryMS = 10;
    props.put(basePropName + "." + AuditQueue.PROP_FILE_SPOOL_ENABLE, "" + true);
    props.put(basePropName + "." + AuditFileSpool.PROP_FILE_SPOOL_LOCAL_DIR, "target");
    props.put(basePropName + "." + AuditFileSpool.PROP_FILE_SPOOL_DEST_RETRY_MS, "" + destRetryMS);
    TestConsumer testConsumer = new TestConsumer();
    testConsumer.isDown = false;
    AuditBatchQueue queue = new AuditBatchQueue(testConsumer);
    queue.init(props, basePropName);
    queue.start();
    try {
        queue.log(createEvent());
        queue.log(createEvent());
        queue.log(createEvent());
        Thread.sleep(1000);
        testConsumer.isDown = true;
        Thread.sleep(1000);
        queue.log(createEvent());
        queue.log(createEvent());
        queue.log(createEvent());
        Thread.sleep(1000);
        testConsumer.isDown = false;
        Thread.sleep(1000);
        queue.log(createEvent());
        queue.log(createEvent());
        queue.log(createEvent());
        Thread.sleep(1000);
        testConsumer.isDown = true;
        Thread.sleep(1000);
        queue.log(createEvent());
        Thread.sleep(1000);
        testConsumer.isDown = false;
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    // ignore
    }
    // Let's wait for second
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    queue.waitToComplete(5000);
    queue.stop();
    queue.waitToComplete();
    assertEquals("Total count", messageToSend, testConsumer.getCountTotal());
    assertEquals("Total sum", messageToSend, testConsumer.getSumTotal());
    assertNull("Event not in sequnce", testConsumer.isInSequence());
}

Example 4 with AuditBatchQueue

use of org.apache.ranger.audit.queue.AuditBatchQueue in project ranger by apache.

the class TestAuditQueue method testAuditBatchQueueBySize.

@Test
public void testAuditBatchQueueBySize() {
    logger.debug("testAuditBatchQueue()...");
    int messageToSend = 10;
    String basePropName = "testAuditBatchQueueBySize_" + MiscUtil.generateUniqueId();
    int batchSize = messageToSend / 3;
    int expectedBatchSize = batchSize + (batchSize * 3 < messageToSend ? 1 : 0);
    int queueSize = messageToSend * 2;
    // Deliberately big interval
    int intervalMS = messageToSend * 100;
    Properties props = new Properties();
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_SIZE, "" + batchSize);
    props.put(basePropName + "." + AuditQueue.PROP_QUEUE_SIZE, "" + queueSize);
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_INTERVAL, "" + intervalMS);
    TestConsumer testConsumer = new TestConsumer();
    AuditBatchQueue queue = new AuditBatchQueue(testConsumer);
    queue.init(props, basePropName);
    queue.start();
    for (int i = 0; i < messageToSend; i++) {
        queue.log(createEvent());
    }
    // Let's wait for second
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    queue.waitToComplete();
    queue.stop();
    queue.waitToComplete();
    assertEquals("Total count", messageToSend, testConsumer.getCountTotal());
    assertEquals("Total sum", messageToSend, testConsumer.getSumTotal());
    assertEquals("Total batch", expectedBatchSize, testConsumer.getBatchCount());
    assertNull("Event not in sequnce", testConsumer.isInSequence());
}

Example 5 with AuditBatchQueue

use of org.apache.ranger.audit.queue.AuditBatchQueue in project ranger by apache.

the class TestAuditQueue method testAuditBatchQueueDestDownRestart.

/**
 * See if we recover after restart
 */
@Test
public void testAuditBatchQueueDestDownRestart() {
    logger.debug("testAuditBatchQueueDestDownRestart()...");
    int messageToSend = 10;
    String basePropName = "testAuditBatchQueueDestDownRestart_" + MiscUtil.generateUniqueId();
    int batchSize = messageToSend / 3;
    int queueSize = messageToSend * 2;
    // Deliberately big interval
    int intervalMS = 3000;
    int maxArchivedFiles = 1;
    Properties props = new Properties();
    props.put(basePropName + "." + BaseAuditHandler.PROP_NAME, "testAuditBatchQueueDestDownRestart_" + MiscUtil.generateUniqueId());
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_SIZE, "" + batchSize);
    props.put(basePropName + "." + AuditQueue.PROP_QUEUE_SIZE, "" + queueSize);
    props.put(basePropName + "." + AuditQueue.PROP_BATCH_INTERVAL, "" + intervalMS);
    // Enable File Spooling
    int destRetryMS = 10;
    props.put(basePropName + "." + AuditQueue.PROP_FILE_SPOOL_ENABLE, "" + true);
    props.put(basePropName + "." + AuditFileSpool.PROP_FILE_SPOOL_LOCAL_DIR, "target");
    props.put(basePropName + "." + AuditFileSpool.PROP_FILE_SPOOL_DEST_RETRY_MS, "" + destRetryMS);
    props.put(basePropName + "." + AuditFileSpool.PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT, "" + maxArchivedFiles);
    TestConsumer testConsumer = new TestConsumer();
    testConsumer.isDown = true;
    AuditBatchQueue queue = new AuditBatchQueue(testConsumer);
    queue.init(props, basePropName);
    queue.start();
    for (int i = 0; i < messageToSend; i++) {
        queue.log(createEvent());
    }
    // Let's wait for second or two
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    queue.waitToComplete(5000);
    queue.stop();
    queue.waitToComplete();
    testConsumer.isDown = true;
    // Let's wait for second or two
    try {
        Thread.sleep(5000);
    } catch (InterruptedException e) {
    // ignore
    }
    // Let's now recreate the objects
    testConsumer = new TestConsumer();
    queue = new AuditBatchQueue(testConsumer);
    queue.init(props, basePropName);
    queue.start();
    // Let's wait for second
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    queue.waitToComplete(5000);
    queue.stop();
    queue.waitToComplete();
    assertEquals("Total count", messageToSend, testConsumer.getCountTotal());
    assertEquals("Total sum", messageToSend, testConsumer.getSumTotal());
    assertNull("Event not in sequnce", testConsumer.isInSequence());
}