Examples with DBUpdateImage com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage used on opensource projects

Example 6 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class TestGoldenGateEventProducer method generateUpdatesForSchema2.

// aux method to generate updates
private List<TransactionState.PerSourceTransactionalUpdate> generateUpdatesForSchema2(short[] sourceIds, Map<String, String> keyVals, long scn) throws DatabusException {
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    String keyName = "name1";
    String valName = "name2";
    Schema.Type keyType = Schema.Type.RECORD;
    Schema s = Schema.parse(avroSchema2);
    for (short sourceId : sourceIds) {
        for (String k : keyVals.keySet()) {
            ColumnsState.KeyPair kp = new ColumnsState.KeyPair(k, keyType);
            ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
            keyPairs.add(kp);
            GenericRecord gr = new GenericData.Record(s);
            gr.put(keyName, k);
            gr.put(valName, keyVals.get(k));
            DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
            db.add(dbi);
            TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
            dbUpdates.add(dbUpdate);
        }
    }
    return dbUpdates;
}

Example 7 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class TestGoldenGateEventProducer method generateUpdates.

// aux method to generate updates
private List<TransactionState.PerSourceTransactionalUpdate> generateUpdates(short[] sourceIds, List<String> keys, long scn) throws DatabusException {
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    String keyName = "name";
    Schema.Type keyType = Schema.Type.RECORD;
    Schema s = Schema.parse(avroSchema);
    for (short sourceId : sourceIds) {
        for (String k : keys) {
            ColumnsState.KeyPair kp = new ColumnsState.KeyPair(k, keyType);
            ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
            keyPairs.add(kp);
            GenericRecord gr = new GenericData.Record(s);
            gr.put(keyName, k);
            DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
            db.add(dbi);
            TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
            dbUpdates.add(dbUpdate);
        }
    }
    return dbUpdates;
}

Example 8 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class DbUpdateState method onEndElement.

/**
 * clean up the state and generate the required data structures
 * @param stateMachine
 * @param xmlStreamReader
 */
@Override
public void onEndElement(StateMachine stateMachine, XMLStreamReader xmlStreamReader) throws DatabusException, XMLStreamException {
    _currentStateType = STATETYPE.ENDELEMENT;
    // TODO construct a data structure that will hold key, csn and the avro
    GenericRecord record = stateMachine.columnsState.getGenericRecord();
    long scn = stateMachine.tokensState.getScn();
    if (_scn < scn) {
        _scn = scn;
        if (LOG.isDebugEnabled())
            LOG.debug("Setting current DbUpdates scn to " + _scn);
    }
    /**
     * The record is empty, this will happen when we have skipped the columns state
     */
    if (record == null) {
        if (LOG.isDebugEnabled())
            LOG.debug("Unable to process the current dbUpdate (record was found to be empty), skipping the dbUpdate");
        onError(stateMachine, xmlStreamReader);
        return;
    }
    if (scn == TokenState.ERRORSCN) {
        LOG.error("Unable to find scn for the given dbUpdate");
        throw new DatabusException("Unable to find scn for the given dbUpdate, terminating the parser");
    }
    // check if the current columns state has seen any missing elements, if it has, then print the current scn associated with the dbupdate for debugging purpose
    if (stateMachine.columnsState.isSeenMissingFields()) {
        LOG.error("There were missing fields seen in Columns section and the corresponding scn is : " + stateMachine.tokensState.getScn());
    }
    Boolean isReplicated = false;
    if (stateMachine.getReplicationBitConfig().getSourceType() == ReplicationBitSetterStaticConfig.SourceType.COLUMN)
        isReplicated = stateMachine.columnsState.isReplicated();
    else if (stateMachine.getReplicationBitConfig().getSourceType() == ReplicationBitSetterStaticConfig.SourceType.TOKEN)
        isReplicated = stateMachine.tokensState.isReplicated();
    else if (stateMachine.getReplicationBitConfig().getSourceType() == ReplicationBitSetterStaticConfig.SourceType.NONE)
        isReplicated = false;
    else
        throw new DatabusException("Unknown source type specified in replicationBitConfig, expected COLUMN or TOKEN or NONE");
    DBUpdateImage eventImage = new DBUpdateImage(stateMachine.columnsState.getKeyPairs(), stateMachine.tokensState.getScn(), stateMachine.columnsState.getGenericRecord(), stateMachine.columnsState.getCurrentSchema(), _opType, isReplicated);
    Integer sourceId = stateMachine.getTableToSourceId().get(_currentTable);
    if (sourceId == null) {
        LOG.error("The table " + _currentTable + " does not have a sourceId, the current dbUpdate cannot be processed.");
        onError(stateMachine, xmlStreamReader);
        return;
    }
    if (getHashSet(sourceId) == null) {
        LOG.error("The hashset is empty, cannot proceed without a valid hashset");
        throw new DatabusException("Error while creating hashset for storing dbUpdates");
    }
    // The equals method of the hashset is overridden to compare only the key, thereby ensuring the latest update on the key
    if (getHashSet(sourceId) != null && getHashSet(sourceId).contains(eventImage))
        getHashSet(sourceId).remove(eventImage);
    getHashSet(sourceId).add(eventImage);
    stateMachine.columnsState.cleanUpState(stateMachine, xmlStreamReader);
    stateMachine.tokensState.cleanUpState(stateMachine, xmlStreamReader);
    xmlStreamReader.nextTag();
    setNextStateProcessor(stateMachine, xmlStreamReader);
}