Examples with AuditAsyncQueue org.apache.ranger.audit.queue.AuditAsyncQueue used on opensource projects

Example 1 with AuditAsyncQueue

use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.

the class AuditProviderFactory method init.

public synchronized void init(Properties props, String appType) {
    LOG.info("AuditProviderFactory: initializing..");
    if (mInitDone) {
        LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
    }
    mInitDone = true;
    componentAppType = appType;
    MiscUtil.setApplicationType(appType);
    boolean isEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_ENABLED_PROP, false);
    boolean isAuditToDbEnabled = MiscUtil.getBooleanProperty(props, AUDIT_DB_IS_ENABLED_PROP, false);
    boolean isAuditToHdfsEnabled = MiscUtil.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false);
    boolean isAuditToLog4jEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
    boolean isAuditToKafkaEnabled = MiscUtil.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false);
    boolean isAuditToSolrEnabled = MiscUtil.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false);
    boolean isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false);
    List<AuditHandler> providers = new ArrayList<AuditHandler>();
    for (Object propNameObj : props.keySet()) {
        LOG.info("AUDIT PROPERTY: " + propNameObj.toString() + "=" + props.getProperty(propNameObj.toString()));
    }
    // Process new audit configurations
    List<String> destNameList = new ArrayList<String>();
    for (Object propNameObj : props.keySet()) {
        String propName = propNameObj.toString();
        if (!propName.startsWith(AUDIT_DEST_BASE)) {
            continue;
        }
        String destName = propName.substring(AUDIT_DEST_BASE.length() + 1);
        List<String> splits = MiscUtil.toArray(destName, ".");
        if (splits.size() > 1) {
            continue;
        }
        String value = props.getProperty(propName);
        if (value.equalsIgnoreCase("enable") || value.equalsIgnoreCase("enabled") || value.equalsIgnoreCase("true")) {
            destNameList.add(destName);
            LOG.info("Audit destination " + propName + " is set to " + value);
        }
    }
    for (String destName : destNameList) {
        String destPropPrefix = AUDIT_DEST_BASE + "." + destName;
        AuditHandler destProvider = getProviderFromConfig(props, destPropPrefix, destName, null);
        if (destProvider != null) {
            destProvider.init(props, destPropPrefix);
            String queueName = MiscUtil.getStringProperty(props, destPropPrefix + "." + AuditQueue.PROP_QUEUE);
            if (queueName == null || queueName.isEmpty()) {
                LOG.info(destPropPrefix + "." + AuditQueue.PROP_QUEUE + " is not set. Setting queue to batch for " + destName);
                queueName = "batch";
            }
            LOG.info("queue for " + destName + " is " + queueName);
            if (queueName != null && !queueName.isEmpty() && !queueName.equalsIgnoreCase("none")) {
                String queuePropPrefix = destPropPrefix + "." + queueName;
                AuditHandler queueProvider = getProviderFromConfig(props, queuePropPrefix, queueName, destProvider);
                if (queueProvider != null) {
                    if (queueProvider instanceof AuditQueue) {
                        AuditQueue qProvider = (AuditQueue) queueProvider;
                        qProvider.init(props, queuePropPrefix);
                        providers.add(queueProvider);
                    } else {
                        LOG.fatal("Provider queue doesn't extend AuditQueue. Destination=" + destName + " can't be created. queueName=" + queueName);
                    }
                } else {
                    LOG.fatal("Queue provider for destination " + destName + " can't be created. queueName=" + queueName);
                }
            } else {
                LOG.info("Audit destination " + destProvider.getName() + " added to provider list");
                providers.add(destProvider);
            }
        }
    }
    if (providers.size() > 0) {
        LOG.info("Using v3 audit configuration");
        AuditHandler consumer = providers.get(0);
        if (providers.size() > 1) {
            // If there are more than one destination, then we need multi
            // destination to process it in parallel
            LOG.info("MultiDestAuditProvider is used. Destination count=" + providers.size());
            MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
            multiDestProvider.init(props);
            multiDestProvider.addAuditProviders(providers);
            consumer = multiDestProvider;
        }
        // Let's see if Summary is enabled, then summarize before sending it
        // downstream
        String propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX;
        boolean summaryEnabled = MiscUtil.getBooleanProperty(props, propPrefix + "." + "summary" + "." + "enabled", false);
        AuditSummaryQueue summaryQueue = null;
        if (summaryEnabled) {
            LOG.info("AuditSummaryQueue is enabled");
            summaryQueue = new AuditSummaryQueue(consumer);
            summaryQueue.init(props, propPrefix);
            consumer = summaryQueue;
        } else {
            LOG.info("AuditSummaryQueue is disabled");
        }
        if (!isAuditFileCacheProviderEnabled) {
            // Create the AsysnQueue
            AuditAsyncQueue asyncQueue = new AuditAsyncQueue(consumer);
            propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "async";
            asyncQueue.init(props, propPrefix);
            asyncQueue.setParentPath(componentAppType);
            mProvider = asyncQueue;
            LOG.info("Starting audit queue " + mProvider.getName());
            mProvider.start();
        } else {
            // Assign AsyncQueue to AuditFileCacheProvider
            AuditFileCacheProvider auditFileCacheProvider = new AuditFileCacheProvider(consumer);
            propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "filecache";
            auditFileCacheProvider.init(props, propPrefix);
            auditFileCacheProvider.setParentPath(componentAppType);
            mProvider = auditFileCacheProvider;
            LOG.info("Starting Audit File Cache Provider " + mProvider.getName());
            mProvider.start();
        }
    } else {
        LOG.info("No v3 audit configuration found. Trying v2 audit configurations");
        if (!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled || providers.size() == 0)) {
            LOG.info("AuditProviderFactory: Audit not enabled..");
            mProvider = getDefaultProvider();
            return;
        }
        if (isAuditToDbEnabled) {
            LOG.info("DbAuditProvider is enabled");
            DbAuditProvider dbProvider = new DbAuditProvider();
            boolean isAuditToDbAsync = MiscUtil.getBooleanProperty(props, DbAuditProvider.AUDIT_DB_IS_ASYNC_PROP, false);
            if (isAuditToDbAsync) {
                int maxQueueSize = MiscUtil.getIntProperty(props, DbAuditProvider.AUDIT_DB_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
                int maxFlushInterval = MiscUtil.getIntProperty(props, DbAuditProvider.AUDIT_DB_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
                AsyncAuditProvider asyncProvider = new AsyncAuditProvider("DbAuditProvider", maxQueueSize, maxFlushInterval, dbProvider);
                providers.add(asyncProvider);
            } else {
                providers.add(dbProvider);
            }
        }
        if (isAuditToHdfsEnabled) {
            LOG.info("HdfsAuditProvider is enabled");
            HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
            boolean isAuditToHdfsAsync = MiscUtil.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false);
            if (isAuditToHdfsAsync) {
                int maxQueueSize = MiscUtil.getIntProperty(props, HdfsAuditProvider.AUDIT_HDFS_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
                int maxFlushInterval = MiscUtil.getIntProperty(props, HdfsAuditProvider.AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
                AsyncAuditProvider asyncProvider = new AsyncAuditProvider("HdfsAuditProvider", maxQueueSize, maxFlushInterval, hdfsProvider);
                providers.add(asyncProvider);
            } else {
                providers.add(hdfsProvider);
            }
        }
        if (isAuditToKafkaEnabled) {
            LOG.info("KafkaAuditProvider is enabled");
            KafkaAuditProvider kafkaProvider = new KafkaAuditProvider();
            kafkaProvider.init(props);
            if (kafkaProvider.isAsync()) {
                AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MyKafkaAuditProvider", 1000, 1000, kafkaProvider);
                providers.add(asyncProvider);
            } else {
                providers.add(kafkaProvider);
            }
        }
        if (isAuditToSolrEnabled) {
            LOG.info("SolrAuditProvider is enabled");
            SolrAuditProvider solrProvider = new SolrAuditProvider();
            solrProvider.init(props);
            if (solrProvider.isAsync()) {
                AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", 1000, 1000, solrProvider);
                providers.add(asyncProvider);
            } else {
                providers.add(solrProvider);
            }
        }
        if (isAuditToLog4jEnabled) {
            Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
            boolean isAuditToLog4jAsync = MiscUtil.getBooleanProperty(props, Log4jAuditProvider.AUDIT_LOG4J_IS_ASYNC_PROP, false);
            if (isAuditToLog4jAsync) {
                int maxQueueSize = MiscUtil.getIntProperty(props, Log4jAuditProvider.AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
                int maxFlushInterval = MiscUtil.getIntProperty(props, Log4jAuditProvider.AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
                AsyncAuditProvider asyncProvider = new AsyncAuditProvider("Log4jAuditProvider", maxQueueSize, maxFlushInterval, log4jProvider);
                providers.add(asyncProvider);
            } else {
                providers.add(log4jProvider);
            }
        }
        if (providers.size() == 0) {
            mProvider = getDefaultProvider();
        } else if (providers.size() == 1) {
            mProvider = providers.get(0);
        } else {
            MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
            multiDestProvider.addAuditProviders(providers);
            mProvider = multiDestProvider;
        }
        mProvider.init(props);
        mProvider.start();
    }
    installJvmSutdownHook(props);
}

Example 2 with AuditAsyncQueue

use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.

the class AuditProviderFactory method getProviderFromConfig.

private AuditHandler getProviderFromConfig(Properties props, String propPrefix, String providerName, AuditHandler consumer) {
    AuditHandler provider = null;
    String className = MiscUtil.getStringProperty(props, propPrefix + "." + BaseAuditHandler.PROP_CLASS_NAME);
    if (className != null && !className.isEmpty()) {
        try {
            Class<?> handlerClass = Class.forName(className);
            if (handlerClass.isAssignableFrom(AuditQueue.class)) {
                // Queue class needs consumer
                handlerClass.getDeclaredConstructor(AuditHandler.class).newInstance(consumer);
            } else {
                provider = (AuditHandler) Class.forName(className).newInstance();
            }
        } catch (Exception e) {
            LOG.fatal("Can't instantiate audit class for providerName=" + providerName + ", className=" + className + ", propertyPrefix=" + propPrefix, e);
        }
    } else {
        if (providerName.equals("file")) {
            provider = new FileAuditDestination();
        } else if (providerName.equalsIgnoreCase("hdfs")) {
            provider = new HDFSAuditDestination();
        } else if (providerName.equals("solr")) {
            provider = new SolrAuditDestination();
        } else if (providerName.equals("kafka")) {
            provider = new KafkaAuditProvider();
        } else if (providerName.equals("db")) {
            provider = new DBAuditDestination();
        } else if (providerName.equals("log4j")) {
            provider = new Log4JAuditDestination();
        } else if (providerName.equals("batch")) {
            provider = new AuditBatchQueue(consumer);
        } else if (providerName.equals("async")) {
            provider = new AuditAsyncQueue(consumer);
        } else {
            LOG.error("Provider name doesn't have any class associated with it. providerName=" + providerName + ", propertyPrefix=" + propPrefix);
        }
    }
    if (provider != null && provider instanceof AuditQueue) {
        if (consumer == null) {
            LOG.fatal("consumer can't be null for AuditQueue. queue=" + provider.getName() + ", propertyPrefix=" + propPrefix);
            provider = null;
        }
    }
    return provider;
}

Example 3 with AuditAsyncQueue

use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.

the class TestAuditQueue method testMultipleQueue.

@Test
public void testMultipleQueue() {
    logger.debug("testAuditAsyncQueue()...");
    int destCount = 3;
    TestConsumer[] testConsumer = new TestConsumer[destCount];
    MultiDestAuditProvider multiQueue = new MultiDestAuditProvider();
    for (int i = 0; i < destCount; i++) {
        testConsumer[i] = new TestConsumer();
        multiQueue.addAuditProvider(testConsumer[i]);
    }
    AuditAsyncQueue queue = new AuditAsyncQueue(multiQueue);
    Properties props = new Properties();
    queue.init(props);
    queue.start();
    int messageToSend = 10;
    for (int i = 0; i < messageToSend; i++) {
        queue.log(createEvent());
    }
    queue.stop();
    queue.waitToComplete();
    // Let's wait for second
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    // ignore
    }
    for (int i = 0; i < destCount; i++) {
        assertEquals("consumer" + i, messageToSend, testConsumer[i].getCountTotal());
        assertEquals("consumer" + i, messageToSend, testConsumer[i].getSumTotal());
    }
}

Example 4 with AuditAsyncQueue

use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.

the class TestAuditQueue method testAuditAsyncQueue.

@Test
public void testAuditAsyncQueue() {
    logger.debug("testAuditAsyncQueue()...");
    TestConsumer testConsumer = new TestConsumer();
    AuditAsyncQueue queue = new AuditAsyncQueue(testConsumer);
    Properties props = new Properties();
    queue.init(props);
    queue.start();
    int messageToSend = 10;
    for (int i = 0; i < messageToSend; i++) {
        queue.log(createEvent());
    }
    queue.stop();
    queue.waitToComplete();
    // Let's wait for second
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    // ignore
    }
    assertEquals(messageToSend, testConsumer.getCountTotal());
    assertEquals(messageToSend, testConsumer.getSumTotal());
    assertNull("Event not in sequnce", testConsumer.isInSequence());
}