Examples with DBUpdateImage com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage used on opensource projects

Example 1 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}

Example 2 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class TestParser method checkNonEmptyFields.

public boolean checkNonEmptyFields(List<TransactionState.PerSourceTransactionalUpdate> dbUpdates) {
    for (TransactionState.PerSourceTransactionalUpdate dbUpdate : dbUpdates) {
        Set<DbUpdateState.DBUpdateImage> DBUpdateImage = dbUpdate.getDbUpdatesSet();
        Iterator<DbUpdateState.DBUpdateImage> it = DBUpdateImage.iterator();
        while (it.hasNext()) {
            DbUpdateState.DBUpdateImage image = it.next();
            GenericRecord record = image.getGenericRecord();
            Iterator<Schema.Field> fieldIt = record.getSchema().getFields().iterator();
            while (fieldIt.hasNext()) {
                String fieldName = fieldIt.next().name();
                if (record.get(fieldName) == null)
                    return false;
            }
        }
    }
    return true;
}

Example 3 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class OpenReplicatorAvroEventFactory method obtainKey.

/**
 * Given a DBImage, returns the key
 * If it is a single key, it returns the object if it is LONG /INT / STRING
 * For compound key, it casts the fields as String, delimits the fields and returns the appended string
 * @param dbChangeEntry The post-image of the event
 * @return Actual key object
 * @throws DatabusException
 */
private Object obtainKey(DbChangeEntry dbChangeEntry) throws DatabusException {
    if (null == dbChangeEntry) {
        throw new DatabusException("DBUpdateImage is null");
    }
    List<KeyPair> pairs = dbChangeEntry.getPkeys();
    if (null == pairs || pairs.size() == 0) {
        throw new DatabusException("There do not seem to be any keys");
    }
    if (pairs.size() == 1) {
        Object key = pairs.get(0).getKey();
        Schema.Type pKeyType = pairs.get(0).getKeyType();
        Object keyObj = null;
        if (pKeyType == Schema.Type.INT) {
            if (key instanceof Integer) {
                keyObj = key;
            } else {
                throw new DatabusException("Schema.Type does not match actual key type (INT) " + key.getClass().getName());
            }
        } else if (pKeyType == Schema.Type.LONG) {
            if (key instanceof Long) {
                keyObj = key;
            } else {
                throw new DatabusException("Schema.Type does not match actual key type (LONG) " + key.getClass().getName());
            }
            keyObj = key;
        } else {
            keyObj = key;
        }
        return keyObj;
    } else {
        // Treat multiple keys as a separate case to avoid unnecessary casts
        Iterator<KeyPair> li = pairs.iterator();
        StringBuilder compositeKey = new StringBuilder();
        while (li.hasNext()) {
            KeyPair kp = li.next();
            Schema.Type pKeyType = kp.getKeyType();
            Object key = kp.getKey();
            if (pKeyType == Schema.Type.INT) {
                if (key instanceof Integer)
                    compositeKey.append(kp.getKey().toString());
                else
                    throw new DatabusException("Schema.Type does not match actual key type (INT) " + key.getClass().getName());
            } else if (pKeyType == Schema.Type.LONG) {
                if (key instanceof Long)
                    compositeKey.append(key.toString());
                else
                    throw new DatabusException("Schema.Type does not match actual key type (LONG) " + key.getClass().getName());
            } else {
                compositeKey.append(key);
            }
            if (li.hasNext()) {
                // Add the delimiter for all keys except the last key
                compositeKey.append(DbusConstants.COMPOUND_KEY_DELIMITER);
            }
        }
        return compositeKey.toString();
    }
}

Example 4 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class GoldenGateEventProducer method obtainKey.

/**
 * Given a DBImage, returns the key
 * If it is a single key, it returns the object if it is LONG /INT / STRING
 * For compound key, it casts the fields as String, delimits the fields and returns the appended string
 * @param dbUpdate The post-image of the event
 * @return Actual key object
 * @throws DatabusException
 */
protected static Object obtainKey(DbUpdateState.DBUpdateImage dbUpdate) throws DatabusException {
    if (null == dbUpdate) {
        throw new DatabusException("DBUpdateImage is null");
    }
    List<KeyPair> pairs = dbUpdate.getKeyPairs();
    if (null == pairs || pairs.size() == 0) {
        throw new DatabusException("There do not seem to be any keys");
    }
    if (pairs.size() == 1) {
        Object key = dbUpdate.getKeyPairs().get(0).getKey();
        Schema.Type pKeyType = dbUpdate.getKeyPairs().get(0).getKeyType();
        Object keyObj = null;
        if (pKeyType == Schema.Type.INT) {
            if (key instanceof Integer) {
                keyObj = key;
            } else {
                throw new DatabusException("Schema.Type does not match actual key type (INT) " + key.getClass().getName());
            }
        } else if (pKeyType == Schema.Type.LONG) {
            if (key instanceof Long) {
                keyObj = key;
            } else {
                throw new DatabusException("Schema.Type does not match actual key type (LONG) " + key.getClass().getName());
            }
            keyObj = key;
        } else {
            keyObj = key;
        }
        return keyObj;
    } else {
        // Treat multiple keys as a separate case to avoid unnecessary casts
        Iterator<KeyPair> li = pairs.iterator();
        String compositeKey = "";
        while (li.hasNext()) {
            KeyPair kp = li.next();
            Schema.Type pKeyType = kp.getKeyType();
            Object key = kp.getKey();
            if (pKeyType == Schema.Type.INT) {
                if (key instanceof Integer)
                    compositeKey += kp.getKey().toString();
                else
                    throw new DatabusException("Schema.Type does not match actual key type (INT) " + key.getClass().getName());
            } else if (pKeyType == Schema.Type.LONG) {
                if (key instanceof Long)
                    compositeKey += key.toString();
                else
                    throw new DatabusException("Schema.Type does not match actual key type (LONG) " + key.getClass().getName());
            } else {
                compositeKey += key;
            }
            if (li.hasNext()) {
                // Add the delimiter for all keys except the last key
                compositeKey += DbusConstants.COMPOUND_KEY_DELIMITER;
            }
        }
        return compositeKey;
    }
}

Example 5 with DBUpdateImage

use of com.linkedin.databus2.ggParser.XmlStateMachine.DbUpdateState.DBUpdateImage in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBufferRateControl.

private void testAddEventToBufferRateControl(long throttleDurationInSecs) throws InvalidConfigException, UnsupportedKeyException, DatabusException, NoSuchFieldException, IllegalAccessException {
    // 1 event per second required. Send 5 events. Must have 4 sleeps.
    long rate = 1;
    int numEvents = 5;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, throttleDurationInSecs);
    long scn = 10;
    DbusEventBufferAppendable mb = createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    // enable if want to run with mocked timer
    // run_with_mock_timer(gg);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    // name1 is the only key
    ColumnsState.KeyPair kp1 = new ColumnsState.KeyPair(new String("name1"), Schema.Type.RECORD);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(numEvents);
    keyPairs.add(kp1);
    Schema s = Schema.parse(avroSchema2);
    GenericRecord gr1 = new GenericData.Record(s);
    gr1.put("name1", "phani1");
    gr1.put("name2", "boris1");
    GenericRecord gr2 = new GenericData.Record(s);
    gr2.put("name1", "phani2");
    gr2.put("name2", "boris2");
    GenericRecord gr3 = new GenericData.Record(s);
    gr3.put("name1", "phani3");
    gr3.put("name2", "boris3");
    GenericRecord gr4 = new GenericData.Record(s);
    gr4.put("name1", "phani4");
    gr4.put("name2", "boris4");
    GenericRecord gr5 = new GenericData.Record(s);
    gr5.put("name1", "phani5");
    gr5.put("name2", "boris5");
    DBUpdateImage dbi1 = new DBUpdateImage(keyPairs, scn, gr1, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi2 = new DBUpdateImage(keyPairs, scn, gr2, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi3 = new DBUpdateImage(keyPairs, scn, gr3, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi4 = new DBUpdateImage(keyPairs, scn, gr4, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    DBUpdateImage dbi5 = new DBUpdateImage(keyPairs, scn, gr5, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi1);
    db.add(dbi2);
    db.add(dbi3);
    db.add(dbi4);
    db.add(dbi5);
    // For a given transaction, and logical source : only 1 update ( the last one succeeds )
    Assert.assertEquals(1, db.size());
    // Generate 5 transactions with the same update
    for (int i = 0; i < numEvents; i++) {
        List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
        TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
        dbUpdates.add(dbUpdate);
        long timestamp = 60;
        gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
        scn++;
    }
    // It may not sleep the very first time as 1 second may have elapsed from when the rate control got started to when event in
    // getting inserted. Subsequently, expect rate control to kick in
    long numSleeps = Math.min(numEvents, throttleDurationInSecs);
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), numSleeps);
    gg.getRateControl().resetNumSleeps();
    return;
}