use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.
the class AuditProviderFactory method init.
public synchronized void init(Properties props, String appType) {
LOG.info("AuditProviderFactory: initializing..");
if (mInitDone) {
LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
}
mInitDone = true;
componentAppType = appType;
MiscUtil.setApplicationType(appType);
boolean isEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_ENABLED_PROP, false);
boolean isAuditToDbEnabled = MiscUtil.getBooleanProperty(props, AUDIT_DB_IS_ENABLED_PROP, false);
boolean isAuditToHdfsEnabled = MiscUtil.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false);
boolean isAuditToLog4jEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
boolean isAuditToKafkaEnabled = MiscUtil.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false);
boolean isAuditToSolrEnabled = MiscUtil.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false);
boolean isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false);
List<AuditHandler> providers = new ArrayList<AuditHandler>();
for (Object propNameObj : props.keySet()) {
LOG.info("AUDIT PROPERTY: " + propNameObj.toString() + "=" + props.getProperty(propNameObj.toString()));
}
// Process new audit configurations
List<String> destNameList = new ArrayList<String>();
for (Object propNameObj : props.keySet()) {
String propName = propNameObj.toString();
if (!propName.startsWith(AUDIT_DEST_BASE)) {
continue;
}
String destName = propName.substring(AUDIT_DEST_BASE.length() + 1);
List<String> splits = MiscUtil.toArray(destName, ".");
if (splits.size() > 1) {
continue;
}
String value = props.getProperty(propName);
if (value.equalsIgnoreCase("enable") || value.equalsIgnoreCase("enabled") || value.equalsIgnoreCase("true")) {
destNameList.add(destName);
LOG.info("Audit destination " + propName + " is set to " + value);
}
}
for (String destName : destNameList) {
String destPropPrefix = AUDIT_DEST_BASE + "." + destName;
AuditHandler destProvider = getProviderFromConfig(props, destPropPrefix, destName, null);
if (destProvider != null) {
destProvider.init(props, destPropPrefix);
String queueName = MiscUtil.getStringProperty(props, destPropPrefix + "." + AuditQueue.PROP_QUEUE);
if (queueName == null || queueName.isEmpty()) {
LOG.info(destPropPrefix + "." + AuditQueue.PROP_QUEUE + " is not set. Setting queue to batch for " + destName);
queueName = "batch";
}
LOG.info("queue for " + destName + " is " + queueName);
if (queueName != null && !queueName.isEmpty() && !queueName.equalsIgnoreCase("none")) {
String queuePropPrefix = destPropPrefix + "." + queueName;
AuditHandler queueProvider = getProviderFromConfig(props, queuePropPrefix, queueName, destProvider);
if (queueProvider != null) {
if (queueProvider instanceof AuditQueue) {
AuditQueue qProvider = (AuditQueue) queueProvider;
qProvider.init(props, queuePropPrefix);
providers.add(queueProvider);
} else {
LOG.fatal("Provider queue doesn't extend AuditQueue. Destination=" + destName + " can't be created. queueName=" + queueName);
}
} else {
LOG.fatal("Queue provider for destination " + destName + " can't be created. queueName=" + queueName);
}
} else {
LOG.info("Audit destination " + destProvider.getName() + " added to provider list");
providers.add(destProvider);
}
}
}
if (providers.size() > 0) {
LOG.info("Using v3 audit configuration");
AuditHandler consumer = providers.get(0);
if (providers.size() > 1) {
// If there are more than one destination, then we need multi
// destination to process it in parallel
LOG.info("MultiDestAuditProvider is used. Destination count=" + providers.size());
MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
multiDestProvider.init(props);
multiDestProvider.addAuditProviders(providers);
consumer = multiDestProvider;
}
// Let's see if Summary is enabled, then summarize before sending it
// downstream
String propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX;
boolean summaryEnabled = MiscUtil.getBooleanProperty(props, propPrefix + "." + "summary" + "." + "enabled", false);
AuditSummaryQueue summaryQueue = null;
if (summaryEnabled) {
LOG.info("AuditSummaryQueue is enabled");
summaryQueue = new AuditSummaryQueue(consumer);
summaryQueue.init(props, propPrefix);
consumer = summaryQueue;
} else {
LOG.info("AuditSummaryQueue is disabled");
}
if (!isAuditFileCacheProviderEnabled) {
// Create the AsysnQueue
AuditAsyncQueue asyncQueue = new AuditAsyncQueue(consumer);
propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "async";
asyncQueue.init(props, propPrefix);
asyncQueue.setParentPath(componentAppType);
mProvider = asyncQueue;
LOG.info("Starting audit queue " + mProvider.getName());
mProvider.start();
} else {
// Assign AsyncQueue to AuditFileCacheProvider
AuditFileCacheProvider auditFileCacheProvider = new AuditFileCacheProvider(consumer);
propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "filecache";
auditFileCacheProvider.init(props, propPrefix);
auditFileCacheProvider.setParentPath(componentAppType);
mProvider = auditFileCacheProvider;
LOG.info("Starting Audit File Cache Provider " + mProvider.getName());
mProvider.start();
}
} else {
LOG.info("No v3 audit configuration found. Trying v2 audit configurations");
if (!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled || providers.size() == 0)) {
LOG.info("AuditProviderFactory: Audit not enabled..");
mProvider = getDefaultProvider();
return;
}
if (isAuditToDbEnabled) {
LOG.info("DbAuditProvider is enabled");
DbAuditProvider dbProvider = new DbAuditProvider();
boolean isAuditToDbAsync = MiscUtil.getBooleanProperty(props, DbAuditProvider.AUDIT_DB_IS_ASYNC_PROP, false);
if (isAuditToDbAsync) {
int maxQueueSize = MiscUtil.getIntProperty(props, DbAuditProvider.AUDIT_DB_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
int maxFlushInterval = MiscUtil.getIntProperty(props, DbAuditProvider.AUDIT_DB_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
AsyncAuditProvider asyncProvider = new AsyncAuditProvider("DbAuditProvider", maxQueueSize, maxFlushInterval, dbProvider);
providers.add(asyncProvider);
} else {
providers.add(dbProvider);
}
}
if (isAuditToHdfsEnabled) {
LOG.info("HdfsAuditProvider is enabled");
HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
boolean isAuditToHdfsAsync = MiscUtil.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false);
if (isAuditToHdfsAsync) {
int maxQueueSize = MiscUtil.getIntProperty(props, HdfsAuditProvider.AUDIT_HDFS_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
int maxFlushInterval = MiscUtil.getIntProperty(props, HdfsAuditProvider.AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
AsyncAuditProvider asyncProvider = new AsyncAuditProvider("HdfsAuditProvider", maxQueueSize, maxFlushInterval, hdfsProvider);
providers.add(asyncProvider);
} else {
providers.add(hdfsProvider);
}
}
if (isAuditToKafkaEnabled) {
LOG.info("KafkaAuditProvider is enabled");
KafkaAuditProvider kafkaProvider = new KafkaAuditProvider();
kafkaProvider.init(props);
if (kafkaProvider.isAsync()) {
AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MyKafkaAuditProvider", 1000, 1000, kafkaProvider);
providers.add(asyncProvider);
} else {
providers.add(kafkaProvider);
}
}
if (isAuditToSolrEnabled) {
LOG.info("SolrAuditProvider is enabled");
SolrAuditProvider solrProvider = new SolrAuditProvider();
solrProvider.init(props);
if (solrProvider.isAsync()) {
AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", 1000, 1000, solrProvider);
providers.add(asyncProvider);
} else {
providers.add(solrProvider);
}
}
if (isAuditToLog4jEnabled) {
Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
boolean isAuditToLog4jAsync = MiscUtil.getBooleanProperty(props, Log4jAuditProvider.AUDIT_LOG4J_IS_ASYNC_PROP, false);
if (isAuditToLog4jAsync) {
int maxQueueSize = MiscUtil.getIntProperty(props, Log4jAuditProvider.AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
int maxFlushInterval = MiscUtil.getIntProperty(props, Log4jAuditProvider.AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
AsyncAuditProvider asyncProvider = new AsyncAuditProvider("Log4jAuditProvider", maxQueueSize, maxFlushInterval, log4jProvider);
providers.add(asyncProvider);
} else {
providers.add(log4jProvider);
}
}
if (providers.size() == 0) {
mProvider = getDefaultProvider();
} else if (providers.size() == 1) {
mProvider = providers.get(0);
} else {
MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
multiDestProvider.addAuditProviders(providers);
mProvider = multiDestProvider;
}
mProvider.init(props);
mProvider.start();
}
installJvmSutdownHook(props);
}
use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.
the class AuditProviderFactory method getProviderFromConfig.
private AuditHandler getProviderFromConfig(Properties props, String propPrefix, String providerName, AuditHandler consumer) {
AuditHandler provider = null;
String className = MiscUtil.getStringProperty(props, propPrefix + "." + BaseAuditHandler.PROP_CLASS_NAME);
if (className != null && !className.isEmpty()) {
try {
Class<?> handlerClass = Class.forName(className);
if (handlerClass.isAssignableFrom(AuditQueue.class)) {
// Queue class needs consumer
handlerClass.getDeclaredConstructor(AuditHandler.class).newInstance(consumer);
} else {
provider = (AuditHandler) Class.forName(className).newInstance();
}
} catch (Exception e) {
LOG.fatal("Can't instantiate audit class for providerName=" + providerName + ", className=" + className + ", propertyPrefix=" + propPrefix, e);
}
} else {
if (providerName.equals("file")) {
provider = new FileAuditDestination();
} else if (providerName.equalsIgnoreCase("hdfs")) {
provider = new HDFSAuditDestination();
} else if (providerName.equals("solr")) {
provider = new SolrAuditDestination();
} else if (providerName.equals("kafka")) {
provider = new KafkaAuditProvider();
} else if (providerName.equals("db")) {
provider = new DBAuditDestination();
} else if (providerName.equals("log4j")) {
provider = new Log4JAuditDestination();
} else if (providerName.equals("batch")) {
provider = new AuditBatchQueue(consumer);
} else if (providerName.equals("async")) {
provider = new AuditAsyncQueue(consumer);
} else {
LOG.error("Provider name doesn't have any class associated with it. providerName=" + providerName + ", propertyPrefix=" + propPrefix);
}
}
if (provider != null && provider instanceof AuditQueue) {
if (consumer == null) {
LOG.fatal("consumer can't be null for AuditQueue. queue=" + provider.getName() + ", propertyPrefix=" + propPrefix);
provider = null;
}
}
return provider;
}
use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.
the class TestAuditQueue method testMultipleQueue.
@Test
public void testMultipleQueue() {
logger.debug("testAuditAsyncQueue()...");
int destCount = 3;
TestConsumer[] testConsumer = new TestConsumer[destCount];
MultiDestAuditProvider multiQueue = new MultiDestAuditProvider();
for (int i = 0; i < destCount; i++) {
testConsumer[i] = new TestConsumer();
multiQueue.addAuditProvider(testConsumer[i]);
}
AuditAsyncQueue queue = new AuditAsyncQueue(multiQueue);
Properties props = new Properties();
queue.init(props);
queue.start();
int messageToSend = 10;
for (int i = 0; i < messageToSend; i++) {
queue.log(createEvent());
}
queue.stop();
queue.waitToComplete();
// Let's wait for second
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// ignore
}
for (int i = 0; i < destCount; i++) {
assertEquals("consumer" + i, messageToSend, testConsumer[i].getCountTotal());
assertEquals("consumer" + i, messageToSend, testConsumer[i].getSumTotal());
}
}
use of org.apache.ranger.audit.queue.AuditAsyncQueue in project ranger by apache.
the class TestAuditQueue method testAuditAsyncQueue.
@Test
public void testAuditAsyncQueue() {
logger.debug("testAuditAsyncQueue()...");
TestConsumer testConsumer = new TestConsumer();
AuditAsyncQueue queue = new AuditAsyncQueue(testConsumer);
Properties props = new Properties();
queue.init(props);
queue.start();
int messageToSend = 10;
for (int i = 0; i < messageToSend; i++) {
queue.log(createEvent());
}
queue.stop();
queue.waitToComplete();
// Let's wait for second
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// ignore
}
assertEquals(messageToSend, testConsumer.getCountTotal());
assertEquals(messageToSend, testConsumer.getSumTotal());
assertNull("Event not in sequnce", testConsumer.isInSequence());
}
Aggregations