Examples with XXAccessAudit org.apache.ranger.entity.XXAccessAudit used on opensource projects

Example 1 with XXAccessAudit

use of org.apache.ranger.entity.XXAccessAudit in project ranger by apache.

the class DbToSolrMigrationUtil method migrateAuditDbLogsToSolr.

public void migrateAuditDbLogsToSolr() {
    System.out.println("Migration process is started..");
    long maxXXAccessAuditID = daoManager.getXXAccessAudit().getMaxIdOfXXAccessAudit();
    if (maxXXAccessAuditID == 0) {
        logger.info("Access Audit log does not exist.");
        System.out.println("Access Audit log does not exist in db.");
        return;
    }
    long maxMigratedID = 0;
    try {
        maxMigratedID = readMigrationStatusFile(CHECK_FILE_NAME);
    } catch (IOException ex) {
        logger.error("Failed to read migration status from file " + CHECK_FILE_NAME, ex);
    }
    logger.info("ID of the last available audit log: " + maxXXAccessAuditID);
    if (maxMigratedID > 0) {
        logger.info("ID of the last migrated audit log: " + maxMigratedID);
    }
    if (maxMigratedID >= maxXXAccessAuditID) {
        logger.info("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID);
        System.out.println("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID);
        return;
    }
    String db_flavor = AppConstants.getLabelFor_DatabaseFlavor(RangerBizUtil.getDBFlavor());
    logger.info("DB flavor: " + db_flavor);
    List<String> columnList = daoManager.getXXAccessAudit().getColumnNames(db_flavor);
    int auditTableVersion = 4;
    if (columnList != null) {
        if (columnList.contains("tags")) {
            auditTableVersion = 6;
        } else if (columnList.contains("seq_num") && columnList.contains("event_count") && columnList.contains("event_dur_ms")) {
            auditTableVersion = 5;
        }
    }
    logger.info("Columns Name:" + columnList);
    long maxRowsPerBatch = 10000;
    // To ceil the actual division result i.e noOfBatches=maxXXAccessAuditID/maxRowsPerBatch
    long noOfBatches = ((maxXXAccessAuditID - maxMigratedID) + maxRowsPerBatch - 1) / maxRowsPerBatch;
    long rangeStart = maxMigratedID;
    long rangeEnd = maxXXAccessAuditID - maxMigratedID <= maxRowsPerBatch ? maxXXAccessAuditID : rangeStart + maxRowsPerBatch;
    long startTimeInMS = 0;
    long timeTaken = 0;
    long lastMigratedID = 0;
    long totalMigratedLogs = 0;
    for (long index = 1; index <= noOfBatches; index++) {
        logger.info("Batch " + index + " of total " + noOfBatches);
        System.out.println("Processing batch " + index + " of total " + noOfBatches);
        startTimeInMS = System.currentTimeMillis();
        // rangeStart and rangeEnd both exclusive, if we add +1 in maxRange
        if (auditTableVersion == 4) {
            List<XXAccessAuditV4> xXAccessAuditV4List = daoManager.getXXAccessAudit().getByIdRangeV4(rangeStart, rangeEnd + 1);
            if (!CollectionUtils.isEmpty(xXAccessAuditV4List)) {
                for (XXAccessAuditV4 xXAccessAudit : xXAccessAuditV4List) {
                    if (xXAccessAudit != null) {
                        try {
                            send2solr(xXAccessAudit);
                            lastMigratedID = xXAccessAudit.getId();
                            totalMigratedLogs++;
                        } catch (Throwable e) {
                            logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
                            writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
                            logger.info("Stopping migration process!");
                            System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
                            System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
                            return;
                        }
                    }
                }
            }
        } else if (auditTableVersion == 5) {
            List<XXAccessAuditV5> xXAccessAuditV5List = daoManager.getXXAccessAudit().getByIdRangeV5(rangeStart, rangeEnd + 1);
            if (!CollectionUtils.isEmpty(xXAccessAuditV5List)) {
                for (XXAccessAuditV5 xXAccessAudit : xXAccessAuditV5List) {
                    if (xXAccessAudit != null) {
                        try {
                            send2solr(xXAccessAudit);
                            lastMigratedID = xXAccessAudit.getId();
                            totalMigratedLogs++;
                        } catch (Throwable e) {
                            logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
                            writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
                            logger.info("Stopping migration process!");
                            System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
                            System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
                            return;
                        }
                    }
                }
            }
        } else if (auditTableVersion == 6) {
            List<XXAccessAudit> xXAccessAuditV6List = daoManager.getXXAccessAudit().getByIdRangeV6(rangeStart, rangeEnd + 1);
            if (!CollectionUtils.isEmpty(xXAccessAuditV6List)) {
                for (XXAccessAudit xXAccessAudit : xXAccessAuditV6List) {
                    if (xXAccessAudit != null) {
                        try {
                            send2solr(xXAccessAudit);
                            lastMigratedID = xXAccessAudit.getId();
                            totalMigratedLogs++;
                        } catch (Throwable e) {
                            logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
                            writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
                            logger.info("Stopping migration process!");
                            System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
                            System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
                            return;
                        }
                    }
                }
            }
        }
        timeTaken = (System.currentTimeMillis() - startTimeInMS);
        logger.info("Batch #" + index + ": time taken:" + timeTaken + " ms");
        if (rangeEnd < maxXXAccessAuditID) {
            writeMigrationStatusFile(rangeEnd, CHECK_FILE_NAME);
        } else {
            writeMigrationStatusFile(maxXXAccessAuditID, CHECK_FILE_NAME);
        }
        rangeStart = rangeEnd;
        rangeEnd = rangeEnd + maxRowsPerBatch;
    }
    if (totalMigratedLogs > 0) {
        System.out.println("Total Number of Migrated Audit logs:" + totalMigratedLogs);
        logger.info("Total Number of Migrated Audit logs:" + totalMigratedLogs);
    }
    if (solrClient != null) {
        try {
            solrClient.close();
        } catch (IOException e) {
            logger.error("Error while closing solr connection", e);
        } finally {
            solrClient = null;
        }
    }
    System.out.println("Migration process finished!!");
}

Example 2 with XXAccessAudit

use of org.apache.ranger.entity.XXAccessAudit in project ranger by apache.

the class XAccessAuditService method searchXAccessAudits.

/**
 * @param searchCriteria
 * @return
 */
public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
    VXAccessAuditList returnList = new VXAccessAuditList();
    List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>();
    List<XXAccessAudit> resultList = (List<XXAccessAudit>) searchResources(searchCriteria, searchFields, sortFields, returnList);
    final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true);
    // Iterate over the result list and create the return list
    for (XXAccessAudit gjXAccessAudit : resultList) {
        VXAccessAudit vXAccessAudit = populateViewBean(gjXAccessAudit);
        if (vXAccessAudit != null) {
            if (StringUtils.equalsIgnoreCase(vXAccessAudit.getAclEnforcer(), RangerHadoopConstants.DEFAULT_XASECURE_MODULE_ACL_NAME)) {
                vXAccessAudit.setAclEnforcer(RangerHadoopConstants.DEFAULT_RANGER_MODULE_ACL_NAME);
            }
            if (!hiveQueryVisibility && "hive".equalsIgnoreCase(vXAccessAudit.getServiceType())) {
                vXAccessAudit.setRequestData(null);
            } else if ("hive".equalsIgnoreCase(vXAccessAudit.getServiceType()) && ("grant".equalsIgnoreCase(vXAccessAudit.getAccessType()) || "revoke".equalsIgnoreCase(vXAccessAudit.getAccessType()))) {
                try {
                    vXAccessAudit.setRequestData(java.net.URLDecoder.decode(vXAccessAudit.getRequestData(), "UTF-8"));
                } catch (UnsupportedEncodingException e) {
                    logger.warn("Error while encoding request data");
                }
            }
            xAccessAuditList.add(vXAccessAudit);
        }
    }
    returnList.setVXAccessAudits(xAccessAuditList);
    return returnList;
}