use of org.apache.ranger.entity.XXAccessAuditV5 in project ranger by apache.
the class DbToSolrMigrationUtil method migrateAuditDbLogsToSolr.
public void migrateAuditDbLogsToSolr() {
System.out.println("Migration process is started..");
long maxXXAccessAuditID = daoManager.getXXAccessAudit().getMaxIdOfXXAccessAudit();
if (maxXXAccessAuditID == 0) {
logger.info("Access Audit log does not exist.");
System.out.println("Access Audit log does not exist in db.");
return;
}
long maxMigratedID = 0;
try {
maxMigratedID = readMigrationStatusFile(CHECK_FILE_NAME);
} catch (IOException ex) {
logger.error("Failed to read migration status from file " + CHECK_FILE_NAME, ex);
}
logger.info("ID of the last available audit log: " + maxXXAccessAuditID);
if (maxMigratedID > 0) {
logger.info("ID of the last migrated audit log: " + maxMigratedID);
}
if (maxMigratedID >= maxXXAccessAuditID) {
logger.info("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID);
System.out.println("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID);
return;
}
String db_flavor = AppConstants.getLabelFor_DatabaseFlavor(RangerBizUtil.getDBFlavor());
logger.info("DB flavor: " + db_flavor);
List<String> columnList = daoManager.getXXAccessAudit().getColumnNames(db_flavor);
int auditTableVersion = 4;
if (columnList != null) {
if (columnList.contains("tags")) {
auditTableVersion = 6;
} else if (columnList.contains("seq_num") && columnList.contains("event_count") && columnList.contains("event_dur_ms")) {
auditTableVersion = 5;
}
}
logger.info("Columns Name:" + columnList);
long maxRowsPerBatch = 10000;
// To ceil the actual division result i.e noOfBatches=maxXXAccessAuditID/maxRowsPerBatch
long noOfBatches = ((maxXXAccessAuditID - maxMigratedID) + maxRowsPerBatch - 1) / maxRowsPerBatch;
long rangeStart = maxMigratedID;
long rangeEnd = maxXXAccessAuditID - maxMigratedID <= maxRowsPerBatch ? maxXXAccessAuditID : rangeStart + maxRowsPerBatch;
long startTimeInMS = 0;
long timeTaken = 0;
long lastMigratedID = 0;
long totalMigratedLogs = 0;
for (long index = 1; index <= noOfBatches; index++) {
logger.info("Batch " + index + " of total " + noOfBatches);
System.out.println("Processing batch " + index + " of total " + noOfBatches);
startTimeInMS = System.currentTimeMillis();
// rangeStart and rangeEnd both exclusive, if we add +1 in maxRange
if (auditTableVersion == 4) {
List<XXAccessAuditV4> xXAccessAuditV4List = daoManager.getXXAccessAudit().getByIdRangeV4(rangeStart, rangeEnd + 1);
if (!CollectionUtils.isEmpty(xXAccessAuditV4List)) {
for (XXAccessAuditV4 xXAccessAudit : xXAccessAuditV4List) {
if (xXAccessAudit != null) {
try {
send2solr(xXAccessAudit);
lastMigratedID = xXAccessAudit.getId();
totalMigratedLogs++;
} catch (Throwable e) {
logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
logger.info("Stopping migration process!");
System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
return;
}
}
}
}
} else if (auditTableVersion == 5) {
List<XXAccessAuditV5> xXAccessAuditV5List = daoManager.getXXAccessAudit().getByIdRangeV5(rangeStart, rangeEnd + 1);
if (!CollectionUtils.isEmpty(xXAccessAuditV5List)) {
for (XXAccessAuditV5 xXAccessAudit : xXAccessAuditV5List) {
if (xXAccessAudit != null) {
try {
send2solr(xXAccessAudit);
lastMigratedID = xXAccessAudit.getId();
totalMigratedLogs++;
} catch (Throwable e) {
logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
logger.info("Stopping migration process!");
System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
return;
}
}
}
}
} else if (auditTableVersion == 6) {
List<XXAccessAudit> xXAccessAuditV6List = daoManager.getXXAccessAudit().getByIdRangeV6(rangeStart, rangeEnd + 1);
if (!CollectionUtils.isEmpty(xXAccessAuditV6List)) {
for (XXAccessAudit xXAccessAudit : xXAccessAuditV6List) {
if (xXAccessAudit != null) {
try {
send2solr(xXAccessAudit);
lastMigratedID = xXAccessAudit.getId();
totalMigratedLogs++;
} catch (Throwable e) {
logger.error("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.", e);
writeMigrationStatusFile(lastMigratedID, CHECK_FILE_NAME);
logger.info("Stopping migration process!");
System.out.println("Error while writing audit log id '" + xXAccessAudit.getId() + "' to Solr.");
System.out.println("Migration process failed, Please refer ranger_db_patch.log file.");
return;
}
}
}
}
}
timeTaken = (System.currentTimeMillis() - startTimeInMS);
logger.info("Batch #" + index + ": time taken:" + timeTaken + " ms");
if (rangeEnd < maxXXAccessAuditID) {
writeMigrationStatusFile(rangeEnd, CHECK_FILE_NAME);
} else {
writeMigrationStatusFile(maxXXAccessAuditID, CHECK_FILE_NAME);
}
rangeStart = rangeEnd;
rangeEnd = rangeEnd + maxRowsPerBatch;
}
if (totalMigratedLogs > 0) {
System.out.println("Total Number of Migrated Audit logs:" + totalMigratedLogs);
logger.info("Total Number of Migrated Audit logs:" + totalMigratedLogs);
}
if (solrClient != null) {
try {
solrClient.close();
} catch (IOException e) {
logger.error("Error while closing solr connection", e);
} finally {
solrClient = null;
}
}
System.out.println("Migration process finished!!");
}
Aggregations