use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.
the class BootstrapSrcDBEventReader method readEventsForSource.
private EventReaderSummary readEventsForSource(OracleTriggerMonitoredSourceInfo sourceInfo, long maxScn) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, IOException {
int retryMax = _numRetries;
int numRetry = 0;
Connection conn = null;
PreparedStatement pstmt = null;
ResultSet rs = null;
KeyType keyType = _pKeyTypeMap.get(sourceInfo.getEventView());
String keyName = _pKeyNameMap.get(sourceInfo.getEventView());
String sql = _eventQueryMap.get(sourceInfo.getEventView());
String endSrcKey = _endSrcKeyMap.get(sourceInfo.getEventView());
if (sql == null) {
sql = generateEventQuery2(sourceInfo, keyName, keyType, getPKIndex(sourceInfo), getQueryHint(sourceInfo));
}
LOG.info("Chunked Query for Source (" + sourceInfo + ") is :" + sql);
LOG.info("EndSrcKey for source (" + sourceInfo + ") is :" + endSrcKey);
PrimaryKeyTxn endKeyTxn = null;
if ((null != endSrcKey) && (!endSrcKey.trim().isEmpty())) {
if (KeyType.LONG == keyType)
endKeyTxn = new PrimaryKeyTxn(new Long(endSrcKey));
else
endKeyTxn = new PrimaryKeyTxn(endSrcKey);
}
long timestamp = System.currentTimeMillis();
int numRowsFetched = 0;
long totalEventSize = 0;
long timeStart = System.currentTimeMillis();
long checkpointInterval = _commitInterval;
boolean done = false;
long lastTime = timeStart;
long numRows = 0;
PrimaryKeyTxn pKey = null;
String minKeySQL = generateMinKeyQuery(sourceInfo, keyName);
String srcName = sourceInfo.getEventView();
LOG.info("Bootstrapping for Source :" + srcName);
String lastKey = _lastKeys.get(sourceInfo.getEventView());
File f = _keyTxnFilesMap.get(srcName);
FileWriter oStream = new FileWriter(f, f.exists());
BufferedWriter keyTxnWriter = new BufferedWriter(oStream, _keyTxnBufferSizeMap.get(srcName));
_bootstrapSeedWriter.startEvents();
RateMonitor seedingRate = new RateMonitor("Seeding Rate");
RateMonitor queryRate = new RateMonitor("Query Rate");
seedingRate.start();
seedingRate.suspend();
queryRate.start();
queryRate.suspend();
boolean isException = false;
long totProcessTime = 0;
try {
conn = _dataSource.getConnection();
pstmt = conn.prepareStatement(sql);
if (_enableNumRowsQuery)
numRows = getNumRows(conn, getTableName(sourceInfo));
else
numRows = -1;
long currRowId = _lastRows.get(sourceInfo.getEventView());
/**
* First Key to be seeded will be decided in the following order:
* 1. Use bootstrap_seeder_state's last srcKey as the key for the first chunk.
* 2. If (1) is empty, use passed-in begin srcKey.
* 3. If (2) is also empty, use Oracle's minKey as the first Chunk Key.
*/
if (null == lastKey) {
lastKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
LOG.info("No last Src Key available in bootstrap_seeder_state for source (" + sourceInfo + ". Trying beginSrc Key from config :" + lastKey);
}
if ((null == lastKey) || (lastKey.trim().isEmpty())) {
if (KeyType.LONG == keyType)
pKey = new PrimaryKeyTxn(executeAndGetLong(minKeySQL));
else
pKey = new PrimaryKeyTxn(executeAndGetString(minKeySQL));
} else {
if (KeyType.LONG == keyType)
pKey = new PrimaryKeyTxn(Long.parseLong(lastKey));
else
pKey = new PrimaryKeyTxn(lastKey);
}
PrimaryKeyTxn lastRoundKeyTxn = new PrimaryKeyTxn(pKey);
PrimaryKeyTxn lastKeyTxn = new PrimaryKeyTxn(pKey);
long numUniqueKeysThisRound = 0;
boolean first = true;
_rate.resume();
while (!done) {
LOG.info("MinKey being used for this round:" + pKey);
numUniqueKeysThisRound = 0;
try {
lastRoundKeyTxn.copyFrom(pKey);
if (KeyType.LONG == keyType) {
pstmt.setLong(1, pKey.getKey());
} else {
String key = pKey.getKeyStr();
pstmt.setString(1, key);
}
pstmt.setLong(2, _numRowsPerQuery);
pstmt.setFetchSize(_numRowsPrefetch);
if (_oraclePreparedStatementClass.isInstance(pstmt)) {
try {
_setLobPrefetchSizeMethod.invoke(pstmt, _LOBPrefetchSize);
} catch (Exception e) {
throw new EventCreationException("Unable to set Lob Prefetch size" + e.getMessage());
}
}
LOG.info("Executing Oracle Query :" + sql + ". Key: " + pKey + ",NumRows: " + _numRowsPerQuery);
queryRate.resume();
rs = pstmt.executeQuery();
queryRate.suspend();
LOG.info("Total Query Latency :" + queryRate.getDuration() / 1000000000L);
long totLatency = 0;
long txnId = 0;
int numRowsThisRound = 0;
seedingRate.resume();
while (rs.next()) {
_rate.tick();
seedingRate.tick();
currRowId++;
txnId = rs.getLong(2);
if (KeyType.LONG == keyType) {
pKey.setKeyTxn(rs.getLong(1), txnId);
} else {
String key = rs.getString(1);
pKey.setKeyStrTxn(key, txnId);
}
//Write TXN to file
pKey.writeTo(keyTxnWriter);
//LOG.info("TXNId is :" + txnId + ",RowId is :" + currRowId);
long start = System.nanoTime();
long eventSize = sourceInfo.getFactory().createAndAppendEvent(maxScn, timestamp, rs, _bootstrapSeedWriter, false, null);
long latency = System.nanoTime() - start;
totLatency += latency;
totalEventSize += eventSize;
totProcessTime += (totLatency / 1000 * 1000);
numRowsFetched++;
numRowsThisRound++;
if (lastKeyTxn.compareKey(pKey) != 0) {
numUniqueKeysThisRound++;
lastKeyTxn.copyFrom(pKey);
}
if (numRowsFetched % checkpointInterval == 0) {
// Commit this batch and reinit
_bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
keyTxnWriter.flush();
_bootstrapSeedWriter.startEvents();
long procTime = totLatency / 1000000000;
long currTime = System.currentTimeMillis();
long diff = (currTime - lastTime) / 1000;
long timeSinceStart = (currTime - timeStart) / 1000;
double currRate = _rate.getRate();
currRate = (currRate <= 0) ? 1 : currRate;
if (_enableNumRowsQuery) {
double remTime = (numRows - currRowId) / (currRate);
LOG.info("Processed " + checkpointInterval + " rows in " + diff + " seconds, Processing Time (seconds) so far :" + (procTime) + ",Seconds elapsed since start :" + (timeSinceStart) + ",Approx Seconds remaining :" + remTime + ",Overall Row Rate:" + _rate.getRate() + "(" + seedingRate.getRate() + ")" + ",NumRows Fetched so far:" + numRowsFetched + ". TotalEventSize :" + totalEventSize);
} else {
LOG.info("Processed " + checkpointInterval + " rows in " + diff + " seconds, Processing Time (seconds) so far :" + (procTime) + ",Seconds elapsed since start :" + (timeSinceStart) + ",Overall Row Rate:" + _rate.getRate() + "(" + seedingRate.getRate() + ")" + ",NumRows Fetched so far:" + numRowsFetched + ". TotalEventSize :" + totalEventSize);
}
lastTime = currTime;
}
if ((null != endKeyTxn) && (endKeyTxn.compareKey(lastKeyTxn) < 0)) {
LOG.info("Seeding to be stopped for current source as it has completed seeding upto endSrckey :" + endKeyTxn + ", Current SrcKey :" + lastKeyTxn);
break;
}
}
seedingRate.suspend();
if ((numRowsThisRound <= 1) || ((numRowsThisRound < _numRowsPerQuery) && (numUniqueKeysThisRound <= 1))) {
LOG.info("Seeding Done for source :" + sourceInfo.getEventView() + ", numRowsThisRound :" + numRowsThisRound + ", _numRowsPerQuery :" + _numRowsPerQuery + ", numUniqueKeys :" + numUniqueKeysThisRound);
done = true;
} else if ((numRowsThisRound == _numRowsPerQuery) && (numUniqueKeysThisRound <= 1)) {
String msg = "Seeding stuck at infinte loop for source : " + sourceInfo.getEventView() + ", numRowsThisRound :" + numRowsThisRound + ", _numRowsPerQuery :" + _numRowsPerQuery + ", numUniqueKeys :" + numUniqueKeysThisRound + ", lastChunkKey :" + lastRoundKeyTxn;
LOG.error(msg);
throw new DatabusException(msg);
} else if (null != endKeyTxn) {
if (endKeyTxn.compareKey(lastKeyTxn) < 0) {
LOG.info("Seeding stopped for source :" + sourceInfo.getEventView() + ", as it has completed seeding upto the endSrckey :" + endKeyTxn + ", numRowsThisRound :" + numRowsThisRound + ", _numRowsPerQuery :" + _numRowsPerQuery + ", numUniqueKeys :" + numUniqueKeysThisRound + " , Current SrcKey :" + lastKeyTxn);
done = true;
}
}
if (currRowId > 0 && (!first || done)) {
//Since next time, we will read the last seen record again
currRowId--;
}
LOG.info("about to call end events with currRowId = " + currRowId);
first = false;
_bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
isException = false;
} catch (SQLException ex) {
LOG.error("Got SQLException for source (" + sourceInfo + ")", ex);
_bootstrapSeedWriter.rollbackEvents();
numRetry++;
isException = true;
if (numRetry >= retryMax) {
throw new DatabusException("Error: Reached max retries for reading/processing bootstrap", ex);
}
} finally {
DBHelper.close(rs);
rs = null;
}
}
} catch (DatabusException ex) {
isException = true;
throw ex;
} finally {
DBHelper.close(rs, pstmt, conn);
keyTxnWriter.close();
rs = null;
_rate.suspend();
if (!isException) {
dedupeKeyTxnFile(_keyTxnFilesMap.get(srcName), keyType);
}
}
long timeEnd = System.currentTimeMillis();
long elapsedMin = (timeEnd - timeStart) / (MILLISEC_TO_MIN);
LOG.info("Processed " + numRowsFetched + " rows of Source: " + sourceInfo.getSourceName() + " in " + elapsedMin + " minutes");
return new EventReaderSummary(sourceInfo.getSourceId(), sourceInfo.getSourceName(), -1, numRowsFetched, totalEventSize, (timeEnd - timeStart), totProcessTime, 0, 0, 0);
}
use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.
the class BootstrapSrcDBEventReader method validate.
public void validate() throws Exception {
if (null == _pKeyTypeMap)
throw new Exception("_pKeyTypeMap cannot be null !!");
if (null == _pKeyNameMap)
throw new Exception("_pKeyNameMap cannot be null !!");
boolean isNullIndex = (null == _pKeyIndexMap);
boolean isNullQueryHint = (null == _queryHintMap);
if (isNullIndex && isNullQueryHint)
throw new Exception("Index and Query Hint both cannot be null !!");
for (OracleTriggerMonitoredSourceInfo s : _sources) {
if (null == _pKeyTypeMap.get(s.getEventView()))
throw new Exception("pKey Type for Source (" + s.getEventView() + ") not provided !!");
if (null == _pKeyNameMap.get(s.getEventView()))
throw new Exception("pKey Name for Source (" + s.getEventView() + ") not provided !!");
if ((isNullIndex || (null == _pKeyIndexMap.get(s.getEventView()))) && (isNullQueryHint || (null == _queryHintMap.get(s.getEventView())))) {
throw new Exception("Both pkey Index and Query Hint for source (" + s.getEventView() + ") not provided !!");
}
}
}
use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.
the class BootstrapDBSeeder method setBootstrapSourceStatus.
/*
* Initialize Bootstrap Sources entry
*/
public void setBootstrapSourceStatus(int status) throws SQLException {
Connection conn = null;
PreparedStatement stmt = null;
try {
String sql = "insert into bootstrap_sources (id, src, status) values (?, ?, ?) on duplicate key update id = ?, src= ?, status=?";
conn = getConnection();
stmt = conn.prepareStatement(sql);
for (OracleTriggerMonitoredSourceInfo sourceInfo : _sources) {
stmt.setInt(1, sourceInfo.getSourceId());
stmt.setString(2, sourceInfo.getSourceName());
stmt.setInt(3, status);
stmt.setInt(4, sourceInfo.getSourceId());
stmt.setString(5, sourceInfo.getSourceName());
stmt.setInt(6, status);
stmt.executeUpdate();
}
conn.commit();
} catch (SQLException sqlEx) {
LOG.error("Got Exception while initializing Bootstrap Sources !!", sqlEx);
if (conn != null)
conn.rollback();
throw sqlEx;
} finally {
DBHelper.close(stmt);
}
}
use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.
the class BootstrapDBSeeder method updateBootstrapStateTable.
private void updateBootstrapStateTable(long scn, String table) throws SQLException {
String upsertSQL = getBootstrapStateInsertStmt(table);
Connection conn = null;
PreparedStatement stmt = null;
try {
conn = getConnection();
stmt = conn.prepareStatement(upsertSQL);
for (OracleTriggerMonitoredSourceInfo srcInfo : _sources) {
stmt.setInt(1, srcInfo.getSourceId());
stmt.setInt(2, 0);
stmt.setLong(3, scn);
stmt.setLong(4, 0);
stmt.setInt(5, 0);
stmt.setLong(6, scn);
stmt.setLong(7, 0);
stmt.executeUpdate();
}
} finally {
DBHelper.close(stmt);
}
}
use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.
the class BootstrapAuditMain method main.
/**
* @param args
*/
public static void main(String[] args) throws Exception {
BootstrapSeederMain.init(args);
BootstrapSeederMain.StaticConfig staticConfig = BootstrapSeederMain.getStaticConfig();
int interval = staticConfig.getController().getCommitInterval();
int sourceChunkSize = staticConfig.getController().getNumRowsPerQuery();
List<OracleTriggerMonitoredSourceInfo> sources = BootstrapSeederMain.getSources();
BootstrapDBSeeder seeder = BootstrapSeederMain.getSeeder();
BootstrapSrcDBEventReader seedController = BootstrapSeederMain.getReader();
Map<String, String> pKeyNameMap = seedController.getpKeyNameMap();
Map<String, DbusEventKey.KeyType> pKeyTypeMap = seedController.getpKeyTypeMap();
for (OracleTriggerMonitoredSourceInfo source : sources) {
short srcId = source.getSourceId();
new ConcurrentHashMap<Long, ResultSetEntry>();
OracleTableReader oracleReader = null;
MySQLTableReader mySQLReader = null;
try {
SchemaRegistryService schemaRegistry = FileSystemSchemaRegistryService.build(staticConfig.getSchemaRegistry().getFileSystem());
Map<Short, String> schemaSet = schemaRegistry.fetchAllSchemaVersionsBySourceName(source.getSourceName());
VersionedSchemaSet vSchemaSet = new VersionedSchemaSet();
Iterator<Map.Entry<Short, String>> it = schemaSet.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Short, String> pairs = it.next();
Schema s = Schema.parse(pairs.getValue());
VersionedSchema vs = new VersionedSchema(s.getFullName(), pairs.getKey(), s, null);
vSchemaSet.add(vs);
}
/* Try and identify the schema key */
VersionedSchema vschema = schemaRegistry.fetchLatestVersionedSchemaBySourceName(source.getSourceName());
Schema schema = Schema.parse(vschema.getSchema().toString());
LOG.info("Schema =" + vschema.getSchema() + "version=" + vschema.getVersion() + " name=" + vschema.getSchemaBaseName());
/* Determine type of field txn */
Field txnFieldType = schema.getField("txn");
if (txnFieldType == null) {
throw new Exception("Unable to find field called 'txn'. Cannot proceeed\n");
}
Type txnType = SchemaHelper.getAnyType(txnFieldType);
/*
* Determine primary key of schema. This is assumed to be invariant
* across versions
*/
String keyOverrideName = SchemaHelper.getMetaField(schema, "pk");
String keyColumnName = "key";
if (null != keyOverrideName) {
keyColumnName = keyOverrideName;
}
Field pkeyField = schema.getField(keyColumnName);
if (null == pkeyField) {
keyColumnName = "id";
pkeyField = schema.getField("id");
}
if (null == pkeyField) {
throw new Exception("Unable to get the primary key for schema. Schema is :" + schema);
}
DbusEventAvroDecoder decoder = new DbusEventAvroDecoder(vSchemaSet);
BootstrapAuditTester auditor = new BootstrapAuditTester(schema, BootstrapSrcDBEventReader.getTableName(source));
List<BootstrapAuditTester> auditors = new ArrayList<BootstrapAuditTester>();
auditors.add(auditor);
oracleReader = new OracleTableReader(BootstrapSeederMain.getDataStore().getConnection(), BootstrapSrcDBEventReader.getTableName(source), pkeyField, SchemaHelper.getMetaField(pkeyField, "dbFieldName"), SchemaHelper.getAnyType(pkeyField), sourceChunkSize, seedController.getPKIndex(source), seedController.getQueryHint(source));
mySQLReader = new MySQLTableReader(seeder.getConnection(), // THis is the primary
seeder.getTableName(srcId), // THis is the primary
pkeyField, // THis is the primary
"id", // bootstrapDB
SchemaHelper.getAnyType(pkeyField), interval);
double samplePct = BootstrapSeederMain.getValidationSamplePct();
TableComparator comparator = new TableComparator(oracleReader, mySQLReader, auditor, decoder, interval, pKeyNameMap.get(source.getEventView()), pKeyTypeMap.get(source.getEventView()), txnType, samplePct);
boolean success = false;
if (BootstrapSeederMain.getValidationType().equals("point")) {
success = comparator.compareRecordsPoint();
} else if (BootstrapSeederMain.getValidationType().equals("pointBs")) {
success = comparator.compareRecordsPointBs();
} else {
success = comparator.compareRecordsNew();
}
if (success)
LOG.info("Audit completed successfully");
else
LOG.error("Audit FAILED !!! ");
} catch (Exception ex) {
LOG.error("Caught an exception ex", ex);
throw ex;
} finally {
if (null != oracleReader)
oracleReader.close();
}
}
DBHelper.close(seeder.getConnection());
}
Aggregations