Search in sources :

Example 1 with Row

use of com.google.code.or.common.glossary.Row in project databus by linkedin.

the class ORListener method updateRows.

private void updateRows(UpdateRowsEvent ure) {
    if (_ignoreSource) {
        LOG.info("Ignoring update rows for " + _curSourceName);
        return;
    }
    List<Pair<Row>> lp = ure.getRows();
    List<Row> lr = new ArrayList<Row>(lp.size());
    for (Pair<Row> pr : lp) {
        Row r = pr.getAfter();
        lr.add(r);
    }
    if (lr.size() > 0) {
        LOG.info("UPDATE " + _curSourceName + ": " + lr.size());
        frameAvroRecord(ure.getTableId(), ure.getHeader(), lr, DbusOpcode.UPSERT);
    }
}
Also used : ArrayList(java.util.ArrayList) Row(com.google.code.or.common.glossary.Row) KeyPair(com.linkedin.databus2.producers.ds.KeyPair) Pair(com.google.code.or.common.glossary.Pair)

Example 2 with Row

use of com.google.code.or.common.glossary.Row in project databus by linkedin.

the class ORListener method updateRows.

private void updateRows(UpdateRowsEventV2 ure) {
    if (_ignoreSource) {
        LOG.info("Ignoring update rows for " + _curSourceName);
        return;
    }
    List<Pair<Row>> lp = ure.getRows();
    List<Row> lr = new ArrayList<Row>(lp.size());
    for (Pair<Row> pr : lp) {
        Row r = pr.getAfter();
        lr.add(r);
    }
    if (lr.size() > 0) {
        LOG.info("UPDATE " + _curSourceName + ": " + lr.size());
        frameAvroRecord(ure.getTableId(), ure.getHeader(), lr, DbusOpcode.UPSERT);
    }
}
Also used : ArrayList(java.util.ArrayList) Row(com.google.code.or.common.glossary.Row) KeyPair(com.linkedin.databus2.producers.ds.KeyPair) Pair(com.google.code.or.common.glossary.Pair)

Example 3 with Row

use of com.google.code.or.common.glossary.Row in project databus by linkedin.

the class ORListener method frameAvroRecord.

private void frameAvroRecord(long tableId, BinlogEventV4Header bh, List<Row> rl, final DbusOpcode doc) {
    try {
        final long timestampInNanos = bh.getTimestamp() * 1000000L;
        final long scn = scn(_currFileNum, (int) bh.getPosition());
        final boolean isReplicated = false;
        final TableMapEvent tme = _tableMapEvents.get(tableId);
        String tableName = tme.getDatabaseName().toString().toLowerCase() + "." + tme.getTableName().toString().toLowerCase();
        VersionedSchema vs = _schemaRegistryService.fetchLatestVersionedSchemaBySourceName(_tableUriToSrcNameMap.get(tableName));
        Schema schema = vs.getSchema();
        if (_log.isDebugEnabled())
            _log.debug("File Number :" + _currFileNum + ", Position :" + (int) bh.getPosition() + ", SCN =" + scn);
        for (Row r : rl) {
            List<Column> cl = r.getColumns();
            GenericRecord gr = new GenericData.Record(schema);
            generateAvroEvent(schema, cl, gr);
            List<KeyPair> kps = generateKeyPair(cl, schema);
            DbChangeEntry db = new DbChangeEntry(scn, timestampInNanos, gr, doc, isReplicated, schema, kps);
            _transaction.getPerSourceTransaction(_tableUriToSrcIdMap.get(tableName)).mergeDbChangeEntrySet(db);
        }
    } catch (NoSuchSchemaException ne) {
        throw new DatabusRuntimeException(ne);
    } catch (DatabusException de) {
        throw new DatabusRuntimeException(de);
    }
}
Also used : TableMapEvent(com.google.code.or.binlog.impl.event.TableMapEvent) KeyPair(com.linkedin.databus2.producers.ds.KeyPair) Schema(org.apache.avro.Schema) PrimaryKeySchema(com.linkedin.databus2.producers.ds.PrimaryKeySchema) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) DbChangeEntry(com.linkedin.databus2.producers.ds.DbChangeEntry) NoSuchSchemaException(com.linkedin.databus2.schemas.NoSuchSchemaException) DatabusException(com.linkedin.databus2.core.DatabusException) FloatColumn(com.google.code.or.common.glossary.column.FloatColumn) SetColumn(com.google.code.or.common.glossary.column.SetColumn) DatetimeColumn(com.google.code.or.common.glossary.column.DatetimeColumn) EnumColumn(com.google.code.or.common.glossary.column.EnumColumn) TimestampColumn(com.google.code.or.common.glossary.column.TimestampColumn) TimeColumn(com.google.code.or.common.glossary.column.TimeColumn) BitColumn(com.google.code.or.common.glossary.column.BitColumn) LongColumn(com.google.code.or.common.glossary.column.LongColumn) YearColumn(com.google.code.or.common.glossary.column.YearColumn) Datetime2Column(com.google.code.or.common.glossary.column.Datetime2Column) DecimalColumn(com.google.code.or.common.glossary.column.DecimalColumn) DoubleColumn(com.google.code.or.common.glossary.column.DoubleColumn) LongLongColumn(com.google.code.or.common.glossary.column.LongLongColumn) TinyColumn(com.google.code.or.common.glossary.column.TinyColumn) Column(com.google.code.or.common.glossary.Column) StringColumn(com.google.code.or.common.glossary.column.StringColumn) Int24Column(com.google.code.or.common.glossary.column.Int24Column) BlobColumn(com.google.code.or.common.glossary.column.BlobColumn) NullColumn(com.google.code.or.common.glossary.column.NullColumn) ShortColumn(com.google.code.or.common.glossary.column.ShortColumn) DateColumn(com.google.code.or.common.glossary.column.DateColumn) GenericRecord(org.apache.avro.generic.GenericRecord) Row(com.google.code.or.common.glossary.Row) GenericRecord(org.apache.avro.generic.GenericRecord) DatabusRuntimeException(com.linkedin.databus.core.DatabusRuntimeException)

Aggregations

Row (com.google.code.or.common.glossary.Row)3 KeyPair (com.linkedin.databus2.producers.ds.KeyPair)3 Pair (com.google.code.or.common.glossary.Pair)2 ArrayList (java.util.ArrayList)2 TableMapEvent (com.google.code.or.binlog.impl.event.TableMapEvent)1 Column (com.google.code.or.common.glossary.Column)1 BitColumn (com.google.code.or.common.glossary.column.BitColumn)1 BlobColumn (com.google.code.or.common.glossary.column.BlobColumn)1 DateColumn (com.google.code.or.common.glossary.column.DateColumn)1 Datetime2Column (com.google.code.or.common.glossary.column.Datetime2Column)1 DatetimeColumn (com.google.code.or.common.glossary.column.DatetimeColumn)1 DecimalColumn (com.google.code.or.common.glossary.column.DecimalColumn)1 DoubleColumn (com.google.code.or.common.glossary.column.DoubleColumn)1 EnumColumn (com.google.code.or.common.glossary.column.EnumColumn)1 FloatColumn (com.google.code.or.common.glossary.column.FloatColumn)1 Int24Column (com.google.code.or.common.glossary.column.Int24Column)1 LongColumn (com.google.code.or.common.glossary.column.LongColumn)1 LongLongColumn (com.google.code.or.common.glossary.column.LongLongColumn)1 NullColumn (com.google.code.or.common.glossary.column.NullColumn)1 SetColumn (com.google.code.or.common.glossary.column.SetColumn)1