Examples with XXAccessAuditV4 org.apache.ranger.entity.XXAccessAuditV4 used on opensource projects

Example 1 with XXAccessAuditV4

use of org.apache.ranger.entity.XXAccessAuditV4 in project ranger by apache.

the class DbToSolrMigrationUtil method migrateAuditDbLogsToSolr.

public void migrateAuditDbLogsToSolr() {
    System.out.println("Migration process is started..");
    long maxXXAccessAuditID = daoManager.getXXAccessAudit().getMaxIdOfXXAccessAudit();
    if (maxXXAccessAuditID == 0) {
        logger.info("Access Audit log does not exist.");
        System.out.println("Access Audit log does not exist in db.");
        return;
    }
    long maxMigratedID = 0;
    try {
        maxMigratedID = readMigrationStatusFile(CHECK_FILE_NAME);
    } catch (IOException ex) {
        logger.error("Failed to read migration status from file " + CHECK_FILE_NAME, ex);
    }
    logger.info("ID of the last available audit log: " + maxXXAccessAuditID);
    if (maxMigratedID > 0) {
        logger.info("ID of the last migrated audit log: " + maxMigratedID);
    }
    if (maxMigratedID >= maxXXAccessAuditID) {
        logger.info("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID);
        System.out.println("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID);
        return;
    }
    String db_flavor = AppConstants.getLabelFor_DatabaseFlavor(RangerBizUtil.getDBFlavor());
    logger.info("DB flavor: " + db_flavor);
    List<String> columnList = daoManager.getXXAccessAudit().getColumnNames(db_flavor);
    int auditTableVersion = 4;
    if (columnList != null) {
        if (columnList.contains("tags")) {
            auditTableVersion = 6;
        } else if (columnList.contains("seq_num") && columnList.contains("event_count") && columnList.contains("event_dur_ms")) {
            auditTableVersion = 5;
        }
    }
    logger.info("Columns Name:" + columnList);
    long maxRowsPerBatch = 10000;
    // To ceil the actual division result i.e noOfBatches=maxXXAccessAuditID/maxRowsPerBatch
    long noOfBatches = ((maxXXAccessAuditID - maxMigratedID) + maxRowsPerBatch - 1) / maxRowsPerBatch;
    long rangeStart = maxMigratedID;
    long rangeEnd = maxXXAccessAuditID - maxMigratedID <= maxRowsPerBatch ? maxXXAccessAuditID : rangeStart + maxRowsPerBatch;
    long startTimeInMS = 0;
    long timeTaken = 0;
    long lastMigratedID = 0;
    long totalMigratedLogs = 0;
    for (long index = 1; index <= noOfBatches; index++) {
        logger.info("Batch " + index + " of total " + noOfBatches);
        System.out.println("Processing batch " + index + " of total " + noOfBatches);
        startTimeInMS = System.currentTimeMillis();
        // rangeStart and rangeEnd both exclusive, if we add +1 in maxRange
        if (auditTableVersion == 4) {
            List<XXAccessAuditV4> xXAccessAuditV4List = daoManager.getXXAccessAudit().getByIdRangeV4(rangeStart, rangeEnd + 1);
            if (!CollectionUtils.isEmpty(xXAccessAuditV4List)) {
                for (XXAccessAuditV4 xXAccessAudit : xXAccessAuditV4List) {
                    if (xXAccessAudit != null) {
                        try {
                            send2solr(xXAccessAudit);
                            lastMigratedID = xXAccessAudit.getId();
                            totalMigratedLogs++;
                        } catch (Throwable e) {
                            logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
                            writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
                            logger.info("Stopping migration process!");
                            System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
                            System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
                            return;
                        }
                    }
                }
            }
        } else if (auditTableVersion == 5) {
            List<XXAccessAuditV5> xXAccessAuditV5List = daoManager.getXXAccessAudit().getByIdRangeV5(rangeStart, rangeEnd + 1);
            if (!CollectionUtils.isEmpty(xXAccessAuditV5List)) {
                for (XXAccessAuditV5 xXAccessAudit : xXAccessAuditV5List) {
                    if (xXAccessAudit != null) {
                        try {
                            send2solr(xXAccessAudit);
                            lastMigratedID = xXAccessAudit.getId();
                            totalMigratedLogs++;
                        } catch (Throwable e) {
                            logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
                            writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
                            logger.info("Stopping migration process!");
                            System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
                            System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
                            return;
                        }
                    }
                }
            }
        } else if (auditTableVersion == 6) {
            List<XXAccessAudit> xXAccessAuditV6List = daoManager.getXXAccessAudit().getByIdRangeV6(rangeStart, rangeEnd + 1);
            if (!CollectionUtils.isEmpty(xXAccessAuditV6List)) {
                for (XXAccessAudit xXAccessAudit : xXAccessAuditV6List) {
                    if (xXAccessAudit != null) {
                        try {
                            send2solr(xXAccessAudit);
                            lastMigratedID = xXAccessAudit.getId();
                            totalMigratedLogs++;
                        } catch (Throwable e) {
                            logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
                            writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
                            logger.info("Stopping migration process!");
                            System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
                            System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
                            return;
                        }
                    }
                }
            }
        }
        timeTaken = (System.currentTimeMillis() - startTimeInMS);
        logger.info("Batch #" + index + ": time taken:" + timeTaken + " ms");
        if (rangeEnd < maxXXAccessAuditID) {
            writeMigrationStatusFile(rangeEnd, CHECK_FILE_NAME);
        } else {
            writeMigrationStatusFile(maxXXAccessAuditID, CHECK_FILE_NAME);
        }
        rangeStart = rangeEnd;
        rangeEnd = rangeEnd + maxRowsPerBatch;
    }
    if (totalMigratedLogs > 0) {
        System.out.println("Total Number of Migrated Audit logs:" + totalMigratedLogs);
        logger.info("Total Number of Migrated Audit logs:" + totalMigratedLogs);
    }
    if (solrClient != null) {
        try {
            solrClient.close();
        } catch (IOException e) {
            logger.error("Error while closing solr connection", e);
        } finally {
            solrClient = null;
        }
    }
    System.out.println("Migration process finished!!");
}