Examples with DatabusException com.linkedin.databus2.core.DatabusException used on opensource projects

Example 21 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class GoldenGateEventProducer method locateScnInTrailFile.

/**
   * Given an xml directory and prefix, the method identifies the file which has the scn (_scn from event producer class)
   * and returns an inputstream reader pointing to the scn location. If the scn is not found:
   * 1. If scn less than what is present in the trail file directory (minimum) - throws a fatal exception.
   * 2. If exact scn is not found, but it's greater than the minimum scn in the trail file directory, it returns the closest scn greater than _scn (from the event producer class).
   * This methods reads and modifies the _scn from the event producer class.
   * @param xmlDir The directory where the trail files are located
   * @param xmlPrefix The prefix of the xml trail files, eg. x4
   * @return
   * @throws IOException
   * @throws DatabusException
   */
private ConcurrentAppendableCompositeFileInputStream locateScnInTrailFile(String xmlDir, String xmlPrefix) throws Exception {
    ConcurrentAppendableCompositeFileInputStream compositeInputStream = null;
    TrailFilePositionSetter.FilePositionResult filePositionResult = null;
    TrailFilePositionSetter trailFilePositionSetter = null;
    while (compositeInputStream == null) {
        _log.info("Requesting trail file position setter for scn: " + _scn.get());
        trailFilePositionSetter = new TrailFilePositionSetter(xmlDir, xmlPrefix, getName());
        filePositionResult = trailFilePositionSetter.locateFilePosition(_scn.get(), new GGXMLTrailTransactionFinder());
        _log.info("File position at : " + filePositionResult);
        switch(filePositionResult.getStatus()) {
            case ERROR:
                _log.fatal("Unable to locate the scn in the trail file.");
                throw new DatabusException("Unable to find the given scn " + _scn.get() + " in the trail files");
            case NO_TXNS_FOUND:
                //If the latest scn is not found in the trail files, then use the earliest scn.
                if (_scn.get() == TrailFilePositionSetter.USE_LATEST_SCN) {
                    _log.info("Switching from USE_LATEST_SCN to USE_EARLIEST_SCN because no trail files were not found");
                    _scn.set(TrailFilePositionSetter.USE_EARLIEST_SCN);
                }
                //TODO sleep get configuration for sleep time
                long noTxnsFoundSleepTime = 500;
                _log.info("NO_TXNS_FOUND, sleeping for " + noTxnsFoundSleepTime + " ms before retrying");
                Thread.sleep(noTxnsFoundSleepTime);
                break;
            case EXACT_SCN_NOT_FOUND:
                {
                    _log.info("Exact SCN was not found, the closest scn found was: " + filePositionResult.getTxnPos().getMinScn());
                    compositeInputStream = new ConcurrentAppendableCompositeFileInputStream(xmlDir, filePositionResult.getTxnPos().getFile(), filePositionResult.getTxnPos().getFileOffset(), new TrailFilePositionSetter.FileFilter(new File(xmlDir), xmlPrefix), false);
                    long foundScn = filePositionResult.getTxnPos().getMaxScn();
                    /**
           * If exact scn is not found, the trail file position setter returns the next immediate available scn, i.e., the contract guarantees
           * a scn always greater than the given scn (foundscn > _scn). We use the _scn (requested scn to be found) as the prevScn to start the event buffer.
           * And the scn found as the current scn(first event in the relay).
           */
                    if (foundScn <= _scn.get())
                        throw new DatabusException("EXACT_SCN_NOT_FOUND, but foundScn is <= _scn ");
                    _startPrevScn.set(_scn.get());
                    _log.info("Changing current scn from " + _scn.get() + " to " + foundScn);
                    _log.info("Planning to use prevScn " + _startPrevScn);
                    _scn.set(foundScn);
                    break;
                }
            case FOUND:
                {
                    _log.info("Exact SCN was  found" + filePositionResult.getTxnPos().getMaxScn());
                    compositeInputStream = new ConcurrentAppendableCompositeFileInputStream(xmlDir, filePositionResult.getTxnPos().getFile(), filePositionResult.getTxnPos().getFileOffset(), new TrailFilePositionSetter.FileFilter(new File(xmlDir), xmlPrefix), false);
                    /**
           * The trail file position setter returns FOUND in two cases:
           * 1. MaxScn was given as input.
           * 2. Earliest or Latest scn was given as input.
           * For both the cases, we set the prevScn to the foundScn-1 and the foundScn as the currentScn.
           */
                    long foundScn = filePositionResult.getTxnPos().getMaxScn();
                    //Assert that if maxScn was requested, the trail file position setter has returned the exact scn (It has returned FOUND).
                    if (_scn.get() >= 0 && _scn.get() != foundScn) {
                        throw new DatabusException("The exact scn was not found, but the trail file position setter has returned FOUND!");
                    }
                    _startPrevScn.set(foundScn - 1);
                    _scn.set(foundScn);
                    break;
                }
            default:
                throw new DatabusException("Unhandled file position result in switch case, terminating producer.");
        }
    }
    if (filePositionResult == null) {
        _log.info(trailFilePositionSetter);
        throw new DatabusException("file position Result returned by TrailFilePositionSetter is null!");
    }
    if (_scn.get() <= 0) {
        _log.info("The scn is <=0, using scn from file position setter:" + filePositionResult);
        _scn.set(filePositionResult.getTxnPos().getMaxScn());
    }
    return compositeInputStream;
}

Example 22 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class GoldenGateEventProducer method buildGGMonitoredSourceInfo.

public GGMonitoredSourceInfo buildGGMonitoredSourceInfo(LogicalSourceStaticConfig sourceConfig, PhysicalSourceStaticConfig pConfig) throws DatabusException, InvalidConfigException {
    // udpate partition mapping
    PartitionFunction partitionFunction = GGEventGenerationFactory.buildPartitionFunction(sourceConfig);
    _partitionFunctionHashMap.put((int) sourceConfig.getId(), partitionFunction);
    EventSourceStatistics statisticsBean = new EventSourceStatistics(sourceConfig.getName());
    GGMonitoredSourceInfo sourceInfo = new GGMonitoredSourceInfo(sourceConfig.getId(), sourceConfig.getName(), statisticsBean);
    registerMbeans(sourceInfo);
    return sourceInfo;
}

Example 23 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class GoldenGateEventProducer method addEventToBuffer.

/**
   *
   * @param dbUpdates  The dbUpdates present in the current transaction
   * @param ti The meta information about the transaction. (See TransactionInfo class for more details).
   * @throws DatabusException
   * @throws UnsupportedKeyException
   */
protected void addEventToBuffer(List<TransactionState.PerSourceTransactionalUpdate> dbUpdates, TransactionInfo ti) throws DatabusException, UnsupportedKeyException {
    if (dbUpdates.size() == 0)
        throw new DatabusException("Cannot handle empty dbUpdates");
    long scn = ti.getScn();
    long timestamp = ti.getTransactionTimeStampNs();
    EventSourceStatistics globalStats = getSource(GLOBAL_SOURCE_ID).getStatisticsBean();
    /**
     * We skip the start scn of the relay, we have already added a EOP for this SCN in the buffer.
     * Why is this not a problem ?
     * There are two cases:
     * 1. When we use the earliest/latest scn if there is no maxScn (We don't really have a start point). So it's really OK to miss the first event.
     * 2. If it's the maxSCN, then event was already seen by the relay.
     */
    if (scn == _startPrevScn.get()) {
        _log.info("Skipping this transaction, EOP already send for this event");
        return;
    }
    getEventBuffer().startEvents();
    int eventsInTransactionCount = 0;
    List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();
    for (int i = 0; i < dbUpdates.size(); ++i) {
        GenericRecord record = null;
        TransactionState.PerSourceTransactionalUpdate perSourceUpdate = dbUpdates.get(i);
        short sourceId = (short) perSourceUpdate.getSourceId();
        // prepare stats collection per source
        EventSourceStatistics perSourceStats = getSource(sourceId).getStatisticsBean();
        Iterator<DbUpdateState.DBUpdateImage> dbUpdateIterator = perSourceUpdate.getDbUpdatesSet().iterator();
        int eventsInDbUpdate = 0;
        long dbUpdatesEventsSize = 0;
        long startDbUpdatesMs = System.currentTimeMillis();
        while (//TODO verify if there is any case where we need to rollback.
        dbUpdateIterator.hasNext()) {
            DbUpdateState.DBUpdateImage dbUpdate = dbUpdateIterator.next();
            //Construct the Databus Event key, determine the key type and construct the key
            Object keyObj = obtainKey(dbUpdate);
            DbusEventKey eventKey = new DbusEventKey(keyObj);
            //Get the logicalparition id
            PartitionFunction partitionFunction = _partitionFunctionHashMap.get((int) sourceId);
            short lPartitionId = partitionFunction.getPartition(eventKey);
            record = dbUpdate.getGenericRecord();
            //Write the event to the buffer
            if (record == null)
                throw new DatabusException("Cannot write event to buffer because record = " + record);
            if (record.getSchema() == null)
                throw new DatabusException("The record does not have a schema (null schema)");
            try {
                //Collect stats on number of dbUpdates for one source
                eventsInDbUpdate++;
                //Count of all the events in the current transaction
                eventsInTransactionCount++;
                // Serialize the row
                ByteArrayOutputStream bos = new ByteArrayOutputStream();
                Encoder encoder = new BinaryEncoder(bos);
                GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(record.getSchema());
                writer.write(record, encoder);
                byte[] serializedValue = bos.toByteArray();
                //Get the md5 for the schema
                SchemaId schemaId = SchemaId.createWithMd5(dbUpdate.getSchema());
                //Determine the operation type and convert to dbus opcode
                DbusOpcode opCode;
                if (dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.INSERT || dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.UPDATE) {
                    opCode = DbusOpcode.UPSERT;
                    if (_log.isDebugEnabled())
                        _log.debug("The event with scn " + scn + " is INSERT/UPDATE");
                } else if (dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.DELETE) {
                    opCode = DbusOpcode.DELETE;
                    if (_log.isDebugEnabled())
                        _log.debug("The event with scn " + scn + " is DELETE");
                } else {
                    throw new DatabusException("Unknown opcode from dbUpdate for event with scn:" + scn);
                }
                //Construct the dbusEvent info
                DbusEventInfo dbusEventInfo = new DbusEventInfo(opCode, scn, (short) _pConfig.getId(), lPartitionId, timestamp, sourceId, schemaId.getByteArray(), serializedValue, false, false);
                dbusEventInfo.setReplicated(dbUpdate.isReplicated());
                perSourceStats.addEventCycle(1, ti.getTransactionTimeRead(), serializedValue.length, scn);
                globalStats.addEventCycle(1, ti.getTransactionTimeRead(), serializedValue.length, scn);
                long tsEnd = System.currentTimeMillis();
                perSourceStats.addTimeOfLastDBAccess(tsEnd);
                globalStats.addTimeOfLastDBAccess(tsEnd);
                //Append to the event buffer
                getEventBuffer().appendEvent(eventKey, dbusEventInfo, _statsCollector);
                _rc.incrementEventCount();
                dbUpdatesEventsSize += serializedValue.length;
            } catch (IOException io) {
                perSourceStats.addError();
                globalStats.addEmptyEventCycle();
                _log.error("Cannot create byte stream payload: " + dbUpdates.get(i).getSourceId());
            }
        }
        long endDbUpdatesMs = System.currentTimeMillis();
        long dbUpdatesElapsedTimeMs = endDbUpdatesMs - startDbUpdatesMs;
        // Log Event Summary at logical source level
        EventReaderSummary summary = new EventReaderSummary(sourceId, _monitoredSources.get(sourceId).getSourceName(), scn, eventsInDbUpdate, dbUpdatesEventsSize, -1L, /* Not supported */
        dbUpdatesElapsedTimeMs, timestamp, timestamp, -1L);
        if (_eventsLog.isInfoEnabled()) {
            _eventsLog.info(summary.toString());
        }
        summaries.add(summary);
        if (_log.isDebugEnabled())
            _log.debug("There are " + eventsInDbUpdate + " events seen in the current dbUpdate");
    }
    // Log Event Summary at Physical source level
    ReadEventCycleSummary summary = new ReadEventCycleSummary(_pConfig.getName(), summaries, scn, -1);
    if (_eventsLog.isInfoEnabled()) {
        _eventsLog.info(summary.toString());
    }
    _log.info("Writing " + eventsInTransactionCount + " events from transaction with scn: " + scn);
    if (scn <= 0)
        throw new DatabusException("Unable to write events to buffer because of negative/zero scn: " + scn);
    getEventBuffer().endEvents(scn, _statsCollector);
    _scn.set(scn);
    if (getMaxScnReaderWriter() != null) {
        try {
            getMaxScnReaderWriter().saveMaxScn(_scn.get());
        } catch (DatabusException e) {
            _log.error("Cannot save scn = " + _scn + " for physical source = " + getName(), e);
        }
    }
}

Example 24 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class GGXMLTrailTransactionFinder method processEnd.

/**
   * When the transaction end is seen, this should be called to save SCN
   * @throws DatabusException
   */
private void processEnd() throws DatabusException {
    if (!_beginTxnSeen) {
        _currTxnStr.setLength(0);
        return;
    }
    _maxScn = Long.valueOf(-1);
    _minScn = Long.MAX_VALUE;
    try {
        if (!_enableRegex) {
            xpathQuery();
        } else {
            regexQuery();
        }
    } catch (DatabusTrailFileParseException ex) {
        LOG.warn("empty/corrupted txn (" + ex.getMessage() + "); resetting invalid _txnPos (" + _txnPos + ") to _prevTxnPos (" + _prevTxnPos + ")");
        _txnPos.copyFrom(_prevTxnPos);
        // TODO:  wire into metrics/monitoring (need accessor plus whatever lies on caller's end)
        ++_numInvalidTxnsSeen;
        return;
    }
    _txnPos.setMaxScn(_maxScn);
    _txnPos.setMinScn(_minScn);
    _txnEndSeen = true;
    _numTxnsSeen++;
    if (!_firstTxnSeen) {
        if (// common case:  need to try previous trail file instead
        (_targetScn >= 0) && (_targetScn < _minScn))
            throw new DatabusException("SinceSCN is less than MinScn available in trail file. Requested SinceSCN is :" + _targetScn + " but found only : " + _minScn + " in Location " + _txnPos);
    }
    _firstTxnSeen = true;
    _beginTxnSeen = false;
    if (LOG.isDebugEnabled()) {
        LOG.debug("Seen Txn : " + _txnPos);
    }
}

Example 25 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class RelayEventProducer method getCheckpoint.

protected Checkpoint getCheckpoint(long sinceSCN, MaxSCNReaderWriter scnReaderWriter) {
    long scn = sinceSCN;
    if ((scn < 0) && (_scnReaderWriter != null)) {
        try {
            scn = _scnReaderWriter.getMaxScn();
        } catch (DatabusException e) {
            LOG.info("Cannot read persisted SCN " + e);
            scn = -1;
        }
    }
    // return no cp if unable to read from saved SCN
    if (scn <= 0) {
        return null;
    }
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    // always have greater than semantic
    cp.setWindowOffset(-1);
    cp.setWindowScn(scn);
    return cp;
}