Examples with DatabusException com.linkedin.databus2.core.DatabusException used on opensource projects

Example 66 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class ORListener method frameAvroRecord.

private void frameAvroRecord(long tableId, BinlogEventV4Header bh, List<Row> rl, final DbusOpcode doc) {
    try {
        final long timestampInNanos = bh.getTimestamp() * 1000000L;
        final long scn = scn(_currFileNum, (int) bh.getPosition());
        final boolean isReplicated = false;
        final TableMapEvent tme = _tableMapEvents.get(tableId);
        String tableName = tme.getDatabaseName().toString().toLowerCase() + "." + tme.getTableName().toString().toLowerCase();
        VersionedSchema vs = _schemaRegistryService.fetchLatestVersionedSchemaBySourceName(_tableUriToSrcNameMap.get(tableName));
        Schema schema = vs.getSchema();
        if (_log.isDebugEnabled())
            _log.debug("File Number :" + _currFileNum + ", Position :" + (int) bh.getPosition() + ", SCN =" + scn);
        for (Row r : rl) {
            List<Column> cl = r.getColumns();
            GenericRecord gr = new GenericData.Record(schema);
            generateAvroEvent(schema, cl, gr);
            List<KeyPair> kps = generateKeyPair(cl, schema);
            DbChangeEntry db = new DbChangeEntry(scn, timestampInNanos, gr, doc, isReplicated, schema, kps);
            _transaction.getPerSourceTransaction(_tableUriToSrcIdMap.get(tableName)).mergeDbChangeEntrySet(db);
        }
    } catch (NoSuchSchemaException ne) {
        throw new DatabusRuntimeException(ne);
    } catch (DatabusException de) {
        throw new DatabusRuntimeException(de);
    }
}

Example 67 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class ORListener method insertFieldIntoRecord.

/**
   * Given the following :
   * 1. A row data as a map of (dbFieldName, Column) data)
   * 2. A generic record to populate
   * 3. dbFieldName
   * 4. avroFieldName
   *
   * The method locates the Column for the dbFieldName, extracts the data as a Java Object,
   * inserts into the generic record with avroFieldName.name() as the key
   */
private void insertFieldIntoRecord(Map<String, Column> eventFields, GenericRecord record, String dbFieldName, Schema.Field avroField) throws DatabusException {
    String f = avroField.name();
    Column fieldValue = eventFields.get(f);
    boolean isFieldNull = (fieldValue == null);
    Object fieldValueObj = null;
    try {
        if (!isFieldNull)
            fieldValueObj = orToAvroType(fieldValue, avroField);
        else
            fieldValueObj = null;
        record.put(avroField.name(), fieldValueObj);
    } catch (DatabusException e) {
        _log.error("Unable to process field: " + avroField.name());
        throw e;
    }
    return;
}

Example 68 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class BootstrapSrcDBEventReader method readEventsForSource.

private EventReaderSummary readEventsForSource(OracleTriggerMonitoredSourceInfo sourceInfo, long maxScn) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, IOException {
    int retryMax = _numRetries;
    int numRetry = 0;
    Connection conn = null;
    PreparedStatement pstmt = null;
    ResultSet rs = null;
    KeyType keyType = _pKeyTypeMap.get(sourceInfo.getEventView());
    String keyName = _pKeyNameMap.get(sourceInfo.getEventView());
    String sql = _eventQueryMap.get(sourceInfo.getEventView());
    String endSrcKey = _endSrcKeyMap.get(sourceInfo.getEventView());
    if (sql == null) {
        sql = generateEventQuery2(sourceInfo, keyName, keyType, getPKIndex(sourceInfo), getQueryHint(sourceInfo));
    }
    LOG.info("Chunked  Query for Source (" + sourceInfo + ") is :" + sql);
    LOG.info("EndSrcKey for source (" + sourceInfo + ") is :" + endSrcKey);
    PrimaryKeyTxn endKeyTxn = null;
    if ((null != endSrcKey) && (!endSrcKey.trim().isEmpty())) {
        if (KeyType.LONG == keyType)
            endKeyTxn = new PrimaryKeyTxn(new Long(endSrcKey));
        else
            endKeyTxn = new PrimaryKeyTxn(endSrcKey);
    }
    long timestamp = System.currentTimeMillis();
    int numRowsFetched = 0;
    long totalEventSize = 0;
    long timeStart = System.currentTimeMillis();
    long checkpointInterval = _commitInterval;
    boolean done = false;
    long lastTime = timeStart;
    long numRows = 0;
    PrimaryKeyTxn pKey = null;
    String minKeySQL = generateMinKeyQuery(sourceInfo, keyName);
    String srcName = sourceInfo.getEventView();
    LOG.info("Bootstrapping for Source :" + srcName);
    String lastKey = _lastKeys.get(sourceInfo.getEventView());
    File f = _keyTxnFilesMap.get(srcName);
    FileWriter oStream = new FileWriter(f, f.exists());
    BufferedWriter keyTxnWriter = new BufferedWriter(oStream, _keyTxnBufferSizeMap.get(srcName));
    _bootstrapSeedWriter.startEvents();
    RateMonitor seedingRate = new RateMonitor("Seeding Rate");
    RateMonitor queryRate = new RateMonitor("Query Rate");
    seedingRate.start();
    seedingRate.suspend();
    queryRate.start();
    queryRate.suspend();
    boolean isException = false;
    long totProcessTime = 0;
    try {
        conn = _dataSource.getConnection();
        pstmt = conn.prepareStatement(sql);
        if (_enableNumRowsQuery)
            numRows = getNumRows(conn, getTableName(sourceInfo));
        else
            numRows = -1;
        long currRowId = _lastRows.get(sourceInfo.getEventView());
        /**
			 * First Key to be seeded will be decided in the following order:
			 * 1. Use bootstrap_seeder_state's last srcKey as the key for the first chunk.
			 * 2. If (1) is empty, use passed-in begin srcKey.
			 * 3. If (2) is also empty, use Oracle's minKey as the first Chunk Key.
			 */
        if (null == lastKey) {
            lastKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
            LOG.info("No last Src Key available in bootstrap_seeder_state for source (" + sourceInfo + ". Trying beginSrc Key from config :" + lastKey);
        }
        if ((null == lastKey) || (lastKey.trim().isEmpty())) {
            if (KeyType.LONG == keyType)
                pKey = new PrimaryKeyTxn(executeAndGetLong(minKeySQL));
            else
                pKey = new PrimaryKeyTxn(executeAndGetString(minKeySQL));
        } else {
            if (KeyType.LONG == keyType)
                pKey = new PrimaryKeyTxn(Long.parseLong(lastKey));
            else
                pKey = new PrimaryKeyTxn(lastKey);
        }
        PrimaryKeyTxn lastRoundKeyTxn = new PrimaryKeyTxn(pKey);
        PrimaryKeyTxn lastKeyTxn = new PrimaryKeyTxn(pKey);
        long numUniqueKeysThisRound = 0;
        boolean first = true;
        _rate.resume();
        while (!done) {
            LOG.info("MinKey being used for this round:" + pKey);
            numUniqueKeysThisRound = 0;
            try {
                lastRoundKeyTxn.copyFrom(pKey);
                if (KeyType.LONG == keyType) {
                    pstmt.setLong(1, pKey.getKey());
                } else {
                    String key = pKey.getKeyStr();
                    pstmt.setString(1, key);
                }
                pstmt.setLong(2, _numRowsPerQuery);
                pstmt.setFetchSize(_numRowsPrefetch);
                if (_oraclePreparedStatementClass.isInstance(pstmt)) {
                    try {
                        _setLobPrefetchSizeMethod.invoke(pstmt, _LOBPrefetchSize);
                    } catch (Exception e) {
                        throw new EventCreationException("Unable to set Lob Prefetch size" + e.getMessage());
                    }
                }
                LOG.info("Executing Oracle Query :" + sql + ". Key: " + pKey + ",NumRows: " + _numRowsPerQuery);
                queryRate.resume();
                rs = pstmt.executeQuery();
                queryRate.suspend();
                LOG.info("Total Query Latency :" + queryRate.getDuration() / 1000000000L);
                long totLatency = 0;
                long txnId = 0;
                int numRowsThisRound = 0;
                seedingRate.resume();
                while (rs.next()) {
                    _rate.tick();
                    seedingRate.tick();
                    currRowId++;
                    txnId = rs.getLong(2);
                    if (KeyType.LONG == keyType) {
                        pKey.setKeyTxn(rs.getLong(1), txnId);
                    } else {
                        String key = rs.getString(1);
                        pKey.setKeyStrTxn(key, txnId);
                    }
                    //Write TXN to file
                    pKey.writeTo(keyTxnWriter);
                    //LOG.info("TXNId is :" + txnId + ",RowId is :" + currRowId);
                    long start = System.nanoTime();
                    long eventSize = sourceInfo.getFactory().createAndAppendEvent(maxScn, timestamp, rs, _bootstrapSeedWriter, false, null);
                    long latency = System.nanoTime() - start;
                    totLatency += latency;
                    totalEventSize += eventSize;
                    totProcessTime += (totLatency / 1000 * 1000);
                    numRowsFetched++;
                    numRowsThisRound++;
                    if (lastKeyTxn.compareKey(pKey) != 0) {
                        numUniqueKeysThisRound++;
                        lastKeyTxn.copyFrom(pKey);
                    }
                    if (numRowsFetched % checkpointInterval == 0) {
                        // Commit this batch and reinit
                        _bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
                        keyTxnWriter.flush();
                        _bootstrapSeedWriter.startEvents();
                        long procTime = totLatency / 1000000000;
                        long currTime = System.currentTimeMillis();
                        long diff = (currTime - lastTime) / 1000;
                        long timeSinceStart = (currTime - timeStart) / 1000;
                        double currRate = _rate.getRate();
                        currRate = (currRate <= 0) ? 1 : currRate;
                        if (_enableNumRowsQuery) {
                            double remTime = (numRows - currRowId) / (currRate);
                            LOG.info("Processed " + checkpointInterval + " rows in " + diff + " seconds, Processing Time (seconds) so far :" + (procTime) + ",Seconds elapsed since start :" + (timeSinceStart) + ",Approx Seconds remaining :" + remTime + ",Overall Row Rate:" + _rate.getRate() + "(" + seedingRate.getRate() + ")" + ",NumRows Fetched so far:" + numRowsFetched + ". TotalEventSize :" + totalEventSize);
                        } else {
                            LOG.info("Processed " + checkpointInterval + " rows in " + diff + " seconds, Processing Time (seconds) so far :" + (procTime) + ",Seconds elapsed since start :" + (timeSinceStart) + ",Overall Row Rate:" + _rate.getRate() + "(" + seedingRate.getRate() + ")" + ",NumRows Fetched so far:" + numRowsFetched + ". TotalEventSize :" + totalEventSize);
                        }
                        lastTime = currTime;
                    }
                    if ((null != endKeyTxn) && (endKeyTxn.compareKey(lastKeyTxn) < 0)) {
                        LOG.info("Seeding to be stopped for current source as it has completed seeding upto endSrckey :" + endKeyTxn + ", Current SrcKey :" + lastKeyTxn);
                        break;
                    }
                }
                seedingRate.suspend();
                if ((numRowsThisRound <= 1) || ((numRowsThisRound < _numRowsPerQuery) && (numUniqueKeysThisRound <= 1))) {
                    LOG.info("Seeding Done for source :" + sourceInfo.getEventView() + ", numRowsThisRound :" + numRowsThisRound + ", _numRowsPerQuery :" + _numRowsPerQuery + ", numUniqueKeys :" + numUniqueKeysThisRound);
                    done = true;
                } else if ((numRowsThisRound == _numRowsPerQuery) && (numUniqueKeysThisRound <= 1)) {
                    String msg = "Seeding stuck at infinte loop for source : " + sourceInfo.getEventView() + ", numRowsThisRound :" + numRowsThisRound + ", _numRowsPerQuery :" + _numRowsPerQuery + ", numUniqueKeys :" + numUniqueKeysThisRound + ", lastChunkKey :" + lastRoundKeyTxn;
                    LOG.error(msg);
                    throw new DatabusException(msg);
                } else if (null != endKeyTxn) {
                    if (endKeyTxn.compareKey(lastKeyTxn) < 0) {
                        LOG.info("Seeding stopped for source :" + sourceInfo.getEventView() + ", as it has completed seeding upto the endSrckey :" + endKeyTxn + ", numRowsThisRound :" + numRowsThisRound + ", _numRowsPerQuery :" + _numRowsPerQuery + ", numUniqueKeys :" + numUniqueKeysThisRound + " , Current SrcKey :" + lastKeyTxn);
                        done = true;
                    }
                }
                if (currRowId > 0 && (!first || done)) {
                    //Since next time, we will read the last seen record again
                    currRowId--;
                }
                LOG.info("about to call end events with currRowId = " + currRowId);
                first = false;
                _bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
                isException = false;
            } catch (SQLException ex) {
                LOG.error("Got SQLException for source (" + sourceInfo + ")", ex);
                _bootstrapSeedWriter.rollbackEvents();
                numRetry++;
                isException = true;
                if (numRetry >= retryMax) {
                    throw new DatabusException("Error: Reached max retries for reading/processing bootstrap", ex);
                }
            } finally {
                DBHelper.close(rs);
                rs = null;
            }
        }
    } catch (DatabusException ex) {
        isException = true;
        throw ex;
    } finally {
        DBHelper.close(rs, pstmt, conn);
        keyTxnWriter.close();
        rs = null;
        _rate.suspend();
        if (!isException) {
            dedupeKeyTxnFile(_keyTxnFilesMap.get(srcName), keyType);
        }
    }
    long timeEnd = System.currentTimeMillis();
    long elapsedMin = (timeEnd - timeStart) / (MILLISEC_TO_MIN);
    LOG.info("Processed " + numRowsFetched + " rows of Source: " + sourceInfo.getSourceName() + " in " + elapsedMin + " minutes");
    return new EventReaderSummary(sourceInfo.getSourceId(), sourceInfo.getSourceName(), -1, numRowsFetched, totalEventSize, (timeEnd - timeStart), totProcessTime, 0, 0, 0);
}

Example 69 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class MysqlMaxSCNHandler method createConnectionPool.

private static ComboPooledDataSource createConnectionPool(StaticConfig staticConfig) throws DatabusException {
    ComboPooledDataSource cpds = new ComboPooledDataSource();
    try {
        cpds.setDriverClass(staticConfig.getDriverClass());
    } catch (PropertyVetoException e) {
        throw new DatabusException("Unable to create connection pool", e);
    }
    cpds.setJdbcUrl(staticConfig.getJdbcUrl());
    cpds.setUser(staticConfig.getDbUser());
    cpds.setPassword(staticConfig.getDbPassword());
    //TODO : Default connection pool setting, make it configurable
    return cpds;
}

Example 70 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class MysqlMaxSCNHandler method loadInitialValue.

private void loadInitialValue(ComboPooledDataSource cpds) throws DatabusException {
    Statement stmt = null;
    Connection connection = null;
    try {
        connection = cpds.getConnection();
        stmt = connection.createStatement();
        ResultSet rs = stmt.executeQuery(staticConfig.getGetSCNQuery());
        if (!rs.isBeforeFirst()) {
            LOG.info("Initial max SCN does not exist in datastore. Defaulting to initial value from configuration: " + staticConfig.getInitVal());
            _scn.set(staticConfig.getInitVal());
            writeScnToDataStore();
            return;
        }
        while (rs.next()) {
            _scn.set(rs.getLong(staticConfig.getScnColumnName()));
        }
        rs.close();
    } catch (SQLException e) {
        throw new DatabusException("unable to load initial SCN value", e);
    } finally {
        if (stmt != null) {
            try {
                stmt.close();
            } catch (SQLException e) {
            }
        }
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
            }
        }
    }
}