Search in sources :

Example 1 with NoSuchSchemaException

use of com.linkedin.databus2.schemas.NoSuchSchemaException in project databus by linkedin.

the class OracleEventProducerFactory method buildOracleMonitoredSourceInfo.

public OracleTriggerMonitoredSourceInfo buildOracleMonitoredSourceInfo(LogicalSourceStaticConfig sourceConfig, PhysicalSourceStaticConfig pConfig, SchemaRegistryService schemaRegistryService) throws DatabusException, EventCreationException, UnsupportedKeyException, InvalidConfigException {
    String schema = null;
    try {
        schema = schemaRegistryService.fetchLatestSchemaBySourceName(sourceConfig.getName());
    } catch (NoSuchSchemaException e) {
        throw new InvalidConfigException("Unable to load the schema for source (" + sourceConfig.getName() + ").");
    }
    if (schema == null) {
        throw new InvalidConfigException("Unable to load the schema for source (" + sourceConfig.getName() + ").");
    }
    _log.info("Loading schema for source id " + sourceConfig.getId() + ": " + schema);
    String eventViewSchema;
    String eventView;
    if (sourceConfig.getUri().indexOf('.') != -1) {
        String[] parts = sourceConfig.getUri().split("\\.");
        eventViewSchema = parts[0];
        eventView = parts[1];
    } else {
        eventViewSchema = null;
        eventView = sourceConfig.getUri();
    }
    if (eventView.toLowerCase().startsWith("sy$")) {
        eventView = eventView.substring(3);
    }
    PartitionFunction partitionFunction = buildPartitionFunction(sourceConfig);
    EventFactory factory = createEventFactory(eventViewSchema, eventView, sourceConfig, pConfig, schema, partitionFunction);
    EventSourceStatistics statisticsBean = new EventSourceStatistics(sourceConfig.getName());
    OracleTriggerMonitoredSourceInfo sourceInfo = new OracleTriggerMonitoredSourceInfo(sourceConfig.getId(), sourceConfig.getName(), eventViewSchema, eventView, factory, statisticsBean, sourceConfig.getRegularQueryHints(), sourceConfig.getChunkedTxnQueryHints(), sourceConfig.getChunkedScnQueryHints(), sourceConfig.isSkipInfinityScn());
    return sourceInfo;
}
Also used : ConstantPartitionFunction(com.linkedin.databus2.producers.ConstantPartitionFunction) PartitionFunction(com.linkedin.databus2.producers.PartitionFunction) NoSuchSchemaException(com.linkedin.databus2.schemas.NoSuchSchemaException) OracleAvroGenericEventFactory(com.linkedin.databus2.producers.db.OracleAvroGenericEventFactory) EventFactory(com.linkedin.databus2.producers.db.EventFactory) InvalidConfigException(com.linkedin.databus.core.util.InvalidConfigException) EventSourceStatistics(com.linkedin.databus.monitoring.mbean.EventSourceStatistics) OracleTriggerMonitoredSourceInfo(com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo)

Example 2 with NoSuchSchemaException

use of com.linkedin.databus2.schemas.NoSuchSchemaException in project databus by linkedin.

the class OpenReplicatorEventProducerServiceProvider method buildEventFactory.

public OpenReplicatorAvroEventFactory buildEventFactory(LogicalSourceStaticConfig sourceConfig, PhysicalSourceStaticConfig pConfig, SchemaRegistryService schemaRegistryService) throws DatabusException, EventCreationException, UnsupportedKeyException, InvalidConfigException {
    String schema = null;
    try {
        schema = schemaRegistryService.fetchLatestSchemaBySourceName(sourceConfig.getName());
    } catch (NoSuchSchemaException e) {
        throw new InvalidConfigException("Unable to load the schema for source (" + sourceConfig.getName() + ").");
    }
    if (schema == null) {
        throw new InvalidConfigException("Unable to load the schema for source (" + sourceConfig.getName() + ").");
    }
    LOG.info("Loading schema for source id " + sourceConfig.getId() + ": " + schema);
    String eventViewSchema;
    String eventView;
    if (sourceConfig.getUri().indexOf('.') != -1) {
        String[] parts = sourceConfig.getUri().split("\\.");
        eventViewSchema = parts[0];
        eventView = parts[1];
    } else {
        eventViewSchema = null;
        eventView = sourceConfig.getUri();
    }
    PartitionFunction partitionFunction = buildPartitionFunction(sourceConfig);
    OpenReplicatorAvroEventFactory factory = createEventFactory(eventViewSchema, eventView, sourceConfig, pConfig, schema, partitionFunction);
    return factory;
}
Also used : PartitionFunction(com.linkedin.databus2.producers.PartitionFunction) NoSuchSchemaException(com.linkedin.databus2.schemas.NoSuchSchemaException) InvalidConfigException(com.linkedin.databus.core.util.InvalidConfigException)

Example 3 with NoSuchSchemaException

use of com.linkedin.databus2.schemas.NoSuchSchemaException in project databus by linkedin.

the class ORListener method frameAvroRecord.

private void frameAvroRecord(long tableId, BinlogEventV4Header bh, List<Row> rl, final DbusOpcode doc) {
    try {
        final long timestampInNanos = bh.getTimestamp() * 1000000L;
        final long scn = scn(_currFileNum, (int) bh.getPosition());
        final boolean isReplicated = false;
        final TableMapEvent tme = _tableMapEvents.get(tableId);
        String tableName = tme.getDatabaseName().toString().toLowerCase() + "." + tme.getTableName().toString().toLowerCase();
        VersionedSchema vs = _schemaRegistryService.fetchLatestVersionedSchemaBySourceName(_tableUriToSrcNameMap.get(tableName));
        Schema schema = vs.getSchema();
        if (_log.isDebugEnabled())
            _log.debug("File Number :" + _currFileNum + ", Position :" + (int) bh.getPosition() + ", SCN =" + scn);
        for (Row r : rl) {
            List<Column> cl = r.getColumns();
            GenericRecord gr = new GenericData.Record(schema);
            generateAvroEvent(vs, cl, gr);
            List<KeyPair> kps = generateKeyPair(gr, vs);
            DbChangeEntry db = new DbChangeEntry(scn, timestampInNanos, gr, doc, isReplicated, schema, kps);
            _transaction.getPerSourceTransaction(_tableUriToSrcIdMap.get(tableName)).mergeDbChangeEntrySet(db);
        }
    } catch (NoSuchSchemaException ne) {
        throw new DatabusRuntimeException(ne);
    } catch (DatabusException de) {
        throw new DatabusRuntimeException(de);
    }
}
Also used : TableMapEvent(com.google.code.or.binlog.impl.event.TableMapEvent) KeyPair(com.linkedin.databus2.producers.ds.KeyPair) Schema(org.apache.avro.Schema) PrimaryKeySchema(com.linkedin.databus2.producers.ds.PrimaryKeySchema) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) VersionedSchema(com.linkedin.databus2.schemas.VersionedSchema) DbChangeEntry(com.linkedin.databus2.producers.ds.DbChangeEntry) NoSuchSchemaException(com.linkedin.databus2.schemas.NoSuchSchemaException) DatabusException(com.linkedin.databus2.core.DatabusException) LongColumn(com.google.code.or.common.glossary.column.LongColumn) LongLongColumn(com.google.code.or.common.glossary.column.LongLongColumn) TinyColumn(com.google.code.or.common.glossary.column.TinyColumn) Column(com.google.code.or.common.glossary.Column) Int24Column(com.google.code.or.common.glossary.column.Int24Column) NullColumn(com.google.code.or.common.glossary.column.NullColumn) ShortColumn(com.google.code.or.common.glossary.column.ShortColumn) GenericRecord(org.apache.avro.generic.GenericRecord) Row(com.google.code.or.common.glossary.Row) GenericRecord(org.apache.avro.generic.GenericRecord) DatabusRuntimeException(com.linkedin.databus.core.DatabusRuntimeException)

Aggregations

NoSuchSchemaException (com.linkedin.databus2.schemas.NoSuchSchemaException)3 InvalidConfigException (com.linkedin.databus.core.util.InvalidConfigException)2 PartitionFunction (com.linkedin.databus2.producers.PartitionFunction)2 TableMapEvent (com.google.code.or.binlog.impl.event.TableMapEvent)1 Column (com.google.code.or.common.glossary.Column)1 Row (com.google.code.or.common.glossary.Row)1 Int24Column (com.google.code.or.common.glossary.column.Int24Column)1 LongColumn (com.google.code.or.common.glossary.column.LongColumn)1 LongLongColumn (com.google.code.or.common.glossary.column.LongLongColumn)1 NullColumn (com.google.code.or.common.glossary.column.NullColumn)1 ShortColumn (com.google.code.or.common.glossary.column.ShortColumn)1 TinyColumn (com.google.code.or.common.glossary.column.TinyColumn)1 DatabusRuntimeException (com.linkedin.databus.core.DatabusRuntimeException)1 EventSourceStatistics (com.linkedin.databus.monitoring.mbean.EventSourceStatistics)1 DatabusException (com.linkedin.databus2.core.DatabusException)1 ConstantPartitionFunction (com.linkedin.databus2.producers.ConstantPartitionFunction)1 EventFactory (com.linkedin.databus2.producers.db.EventFactory)1 OracleAvroGenericEventFactory (com.linkedin.databus2.producers.db.OracleAvroGenericEventFactory)1 OracleTriggerMonitoredSourceInfo (com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo)1 DbChangeEntry (com.linkedin.databus2.producers.ds.DbChangeEntry)1