Examples with DatabusException com.linkedin.databus2.core.DatabusException used on opensource projects

Example 96 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class ColumnsState method insertFieldIntoRecord.

/**
   * The method takes the given field(avro+oracle name), fetches the value from the eventFields map and inserts it into the record.
   * In addition the function also constructs the primary keys for the given record
   * @param eventFields Map containing dbFieldName => fieldValue
   * @param record The record to insert the field value into
   * @param pkFieldName The name of primaryKey field (comma seperated list)
   * @param pk The primary key object stored as an class object
   * @param field The current field being processed(Avro)
   * @param databaseFieldName Field being processed(oracle name)
   * @return
   * @throws Exception
   */
private String insertFieldIntoRecord(HashMap<String, ColumnState.EventField> eventFields, GenericRecord record, String pkFieldName, PrimaryKey pk, Schema.Field field, String databaseFieldName) throws DatabusException {
    String fieldValue = eventFields.get(databaseFieldName).getVal();
    boolean isFieldNull = eventFields.get(databaseFieldName).isNull();
    Object fieldValueObj = null;
    try {
        if (!isFieldNull)
            fieldValueObj = GGEventGenerationFactory.stringToAvroType(fieldValue, field);
        else
            fieldValueObj = null;
        record.put(field.name(), fieldValueObj);
    } catch (DatabusException e) {
        LOG.error("Unable to process field: " + field.name());
        throw e;
    }
    constructPkeys(eventFields, pkFieldName, pk, field, databaseFieldName, fieldValueObj);
    return fieldValue;
}

Example 97 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class DbUpdateState method onStartElement.

@Override
public void onStartElement(StateMachine stateMachine, XMLStreamReader xmlStreamReader) throws DatabusException, XMLStreamException {
    _currentStateType = STATETYPE.STARTELEMENT;
    _opType = DBUpdateImage.OpType.UNKNOWN;
    boolean isUpdate = false;
    boolean isDelete = false;
    boolean isPreImage = false;
    for (int i = 0; i < xmlStreamReader.getAttributeCount(); i++) {
        if (xmlStreamReader.getAttributeName(i).getLocalPart().equals(TABLEATTR))
            _currentTable = xmlStreamReader.getAttributeValue(i);
        else if (xmlStreamReader.getAttributeName(i).getLocalPart().equalsIgnoreCase(UPDATEATTRNAME) && xmlStreamReader.getAttributeValue(i).equalsIgnoreCase(UPDATEVAL)) {
            _opType = DBUpdateImage.OpType.UPDATE;
        } else if (xmlStreamReader.getAttributeName(i).getLocalPart().equalsIgnoreCase(DELETEATTRNAME) && xmlStreamReader.getAttributeValue(i).equalsIgnoreCase(DELETEVAL)) {
            _opType = DBUpdateImage.OpType.DELETE;
        } else if (xmlStreamReader.getAttributeName(i).getLocalPart().equalsIgnoreCase(INSERTATTRNAME) && xmlStreamReader.getAttributeValue(i).equalsIgnoreCase(INSERTVAL)) {
            _opType = DBUpdateImage.OpType.INSERT;
        } else if (xmlStreamReader.getAttributeName(i).getLocalPart().equalsIgnoreCase(PREIMAGEATTRNAME) && xmlStreamReader.getAttributeValue(i).equalsIgnoreCase(PREIMAGEVAL)) {
            isPreImage = true;
        }
    }
    if (//This is the pre-image of the row, we can skip this
    isPreImage) {
        if (LOG.isDebugEnabled())
            LOG.debug("Skipping current dbUpdate because it's a preimage");
        skipCurrentDBupdate(stateMachine, xmlStreamReader);
        return;
    }
    if (_currentTable == null || _currentTable.length() == 0) {
        LOG.fatal("PROBLEM WITH XML: Dbupdate does not have any table name associated with it, stopping ");
        throw new DatabusException("Dbupdate does not have any table name associated with it, stopping");
    }
    Schema schema = StateMachineHelper.tableToSchema(_currentTable, stateMachine.getTableToSourceNameMap(), stateMachine.getSchemaRegistryService());
    if (schema == null) {
        if (LOG.isDebugEnabled())
            LOG.debug("This source is not configured (couldn't find namespace). Skipping to tokens, to capture scn for empty DBUpdate");
        /**
       * State jump from DBUPDATE-> TOKENS. (we skip COLUMNS).
       * At this point, we can't capture this update (because the tableName -> namespace configuration is not found), we are still interested in knowing
       * the SCN associated with the current dbUpdate. The SCNs are captured because, if it's a slow source, we need to insert EOP at some threshold (This is done by the
       * goldengate event producer). If the transaction has no updates relevant to the tables we are interested in, this will be passed to the transaction callback as an "empty" transaction with just SCNs contained.
       */
        skipToTokens(stateMachine, xmlStreamReader);
        setNextStateProcessor(stateMachine, xmlStreamReader);
        return;
    }
    stateMachine.columnsState.setCurrentSchema(schema);
    stateMachine.columnsState.setKeyPairs(new ArrayList<ColumnsState.KeyPair>());
    xmlStreamReader.nextTag();
    setNextStateProcessor(stateMachine, xmlStreamReader);
}

Example 98 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class DbUpdateState method onEndElement.

/**
   * clean up the state and generate the required data structures
   * @param stateMachine
   * @param xmlStreamReader
   */
@Override
public void onEndElement(StateMachine stateMachine, XMLStreamReader xmlStreamReader) throws DatabusException, XMLStreamException {
    _currentStateType = STATETYPE.ENDELEMENT;
    //TODO construct a data structure that will hold key, csn and the avro
    GenericRecord record = stateMachine.columnsState.getGenericRecord();
    long scn = stateMachine.tokensState.getScn();
    if (_scn < scn) {
        _scn = scn;
        if (LOG.isDebugEnabled())
            LOG.debug("Setting current DbUpdates scn to " + _scn);
    }
    /**
     * The record is empty, this will happen when we have skipped the columns state
     */
    if (record == null) {
        if (LOG.isDebugEnabled())
            LOG.debug("Unable to process the current dbUpdate (record was found to be empty), skipping the dbUpdate");
        onError(stateMachine, xmlStreamReader);
        return;
    }
    if (scn == TokenState.ERRORSCN) {
        LOG.error("Unable to find scn for the given dbUpdate");
        throw new DatabusException("Unable to find scn for the given dbUpdate, terminating the parser");
    }
    //check if the current columns state has seen any missing elements, if it has, then print the current scn associated with the dbupdate for debugging purpose
    if (stateMachine.columnsState.isSeenMissingFields()) {
        LOG.error("There were missing fields seen in Columns section and the corresponding scn is : " + stateMachine.tokensState.getScn());
    }
    Boolean isReplicated = false;
    if (stateMachine.getReplicationBitConfig().getSourceType() == ReplicationBitSetterStaticConfig.SourceType.COLUMN)
        isReplicated = stateMachine.columnsState.isReplicated();
    else if (stateMachine.getReplicationBitConfig().getSourceType() == ReplicationBitSetterStaticConfig.SourceType.TOKEN)
        isReplicated = stateMachine.tokensState.isReplicated();
    else if (stateMachine.getReplicationBitConfig().getSourceType() == ReplicationBitSetterStaticConfig.SourceType.NONE)
        isReplicated = false;
    else
        throw new DatabusException("Unknown source type specified in replicationBitConfig, expected COLUMN or TOKEN or NONE");
    DBUpdateImage eventImage = new DBUpdateImage(stateMachine.columnsState.getKeyPairs(), stateMachine.tokensState.getScn(), stateMachine.columnsState.getGenericRecord(), stateMachine.columnsState.getCurrentSchema(), _opType, isReplicated);
    Integer sourceId = stateMachine.getTableToSourceId().get(_currentTable);
    if (sourceId == null) {
        LOG.error("The table " + _currentTable + " does not have a sourceId, the current dbUpdate cannot be processed.");
        onError(stateMachine, xmlStreamReader);
        return;
    }
    if (getHashSet(sourceId) == null) {
        LOG.error("The hashset is empty, cannot proceed without a valid hashset");
        throw new DatabusException("Error while creating hashset for storing dbUpdates");
    }
    //The equals method of the hashset is overridden to compare only the key, thereby ensuring the latest update on the key
    if (getHashSet(sourceId) != null && getHashSet(sourceId).contains(eventImage))
        getHashSet(sourceId).remove(eventImage);
    getHashSet(sourceId).add(eventImage);
    stateMachine.columnsState.cleanUpState(stateMachine, xmlStreamReader);
    stateMachine.tokensState.cleanUpState(stateMachine, xmlStreamReader);
    xmlStreamReader.nextTag();
    setNextStateProcessor(stateMachine, xmlStreamReader);
}

Example 99 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class TrailFilePositionSetter method locateFilePosition.

/**
   *
   * This is an optimized version of searching the trail file position. It does a lookup from latest file quickly skipping trail files whose first transaction has
   * SCN newer than the requested SCN.
   *
   * Steps:
   * 1. Get all the list of trail files
   * 2. Iterate the trail files from the latest to earliest trail file until exhausted or successful, do
   *    a) FindResult = Call FindPosition() on the current file
   *    b) If FindResult was successful( FOUND (exact scn is present) or EXACT_SCN_NOT_FOUND (scn with both lower and higher SCn is seen but not exact) and the found txn was not the first transaction seen, then return
   *    c) Otherwise on the first transaction, reset and look at the previous file
   *
   * This method is quick because if the currentTrailFile's SCN has higher than requested SCN or transaction found was the first one, then it fails fast (after first transaction) so that lookup happens on the previous file.
   * In EI/Prod, each trail file is in the order of 50 MB and this jumping should save a lot of time.
   *
   * Reason for continuing to scan if the requested transaction is the first one :
   *        In each round of scanning we start from one trail file if the first transaction read has SCN == requestedScn, then we
   *        would still need to look at previous file as there could be txns with same SCN. So, we need to locate the first txn
   *        which has this SCN. If there is no previous file present (earliest txn matches requested SCN), then we return error.
   * @param scn : SCN to locate
   * @param callback TransactionSCNFinderCallback to parse and store offsets.
   * @return FilePositionResult of the locate operation
   * @throws IOException if issues with File operations.
   */
public synchronized FilePositionResult locateFilePosition(long scn, TransactionSCNFinderCallback callback) throws IOException {
    TrailFileNotifier notifier = new TrailFileNotifier(_dir, _filter, null, 0, null);
    List<File> orderedTrailFiles = notifier.getCandidateTrailFiles();
    _log.info("Initial set of Trail Files :" + orderedTrailFiles);
    if ((null == orderedTrailFiles) || orderedTrailFiles.isEmpty()) {
        return FilePositionResult.createNoTxnsFoundResult();
    }
    FilePositionResult res = null;
    if (scn == USE_EARLIEST_SCN) {
        res = getFilePosition(scn, callback);
    } else {
        for (int i = orderedTrailFiles.size() - 1; i >= 0; i--) {
            callback.reset();
            File startFile = orderedTrailFiles.get(i);
            _log.info("Locating the SCN (" + scn + ") starting from the trail file :" + startFile);
            res = getFilePosition(scn, callback, startFile.getName());
            _log.info("Result of the location operation for SCN (" + scn + ") starting from trail file (" + startFile + ") is : " + res);
            // If this is the first txn scanned, we need to go to previous file
            if (((res.getStatus() == Status.EXACT_SCN_NOT_FOUND) || (res.getStatus() == Status.FOUND)) && // TxnRank will be 0 if this is the
            (res.getTxnPos().getTxnRank() > 0)) // first txn scanned
            {
                break;
            }
            if ((0 == i) && (res != null)) {
                ScnTxnPos scnTxnPos = res.getTxnPos();
                if ((scnTxnPos != null) && (scnTxnPos.getTxnRank() <= 0)) {
                    return FilePositionResult.createErrorResult(new DatabusException("A transaction with scn less than requested SCN was not found. " + "Without this txn, we cannot identify if all transactions for requested SCN " + "have been located. Requested SinceSCN is :" + scn));
                }
            }
        }
    }
    return res;
}

Example 100 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class OracleRelayFactory method createRelay.

@Override
public HttpRelay createRelay() throws DatabusException {
    try {
        SchemaRegistryService schemaRegistry = _schemaRegistryFactory.createSchemaRegistry();
        HttpRelay.StaticConfig relayStaticConfig = _relayConfigBuilder.build();
        getSourcesIdNameRegistry().updateFromIdNamePairs(relayStaticConfig.getSourceIds());
        PhysicalSourceConfigBuilder psourceConfBuilder = new PhysicalSourceConfigBuilder(_physicalSourcesConfigFiles);
        return new HttpRelay(relayStaticConfig, psourceConfBuilder.build(), getSourcesIdNameRegistry(), schemaRegistry);
    } catch (IOException ioe) {
        throw new DatabusException("OracleRelay instantiation error: " + ioe.getMessage(), ioe);
    }
}