Examples with VersionedSchema com.linkedin.databus2.schemas.VersionedSchema used on opensource projects

Example 6 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class DbusEventAvroDecoder method getGenericRecord.

@Override
public GenericRecord getGenericRecord(DbusEvent e, GenericRecord reuse) {
    byte[] md5 = new byte[16];
    e.schemaId(md5);
    SchemaId schemaId = new SchemaId(md5);
    VersionedSchema writerSchema = _schemaSet.getById(schemaId);
    if (null == writerSchema) {
        LOG.error("Unable to find schema for id " + schemaId + "; event = " + e);
        throw new DatabusRuntimeException("No schema available to decode event " + e);
    }
    ByteBuffer valueBuffer = e.value();
    byte[] valueBytes = null;
    if (valueBuffer.hasArray()) {
        valueBytes = valueBuffer.array();
    } else {
        valueBytes = new byte[valueBuffer.remaining()];
        valueBuffer.get(valueBytes);
    }
    return getGenericRecord(valueBytes, writerSchema.getSchema(), reuse);
}

Example 7 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class DbusEventAvroDecoder method getMetadata.

/**
   * Deserializes the metadata (if any) of a Databus event to an Avro GenericRecord.  This method
   * is for INTERNAL USE ONLY (by Espresso and Databus).  It is NOT a stable API and may change
   * without warning!
   *
   * @param e       the Databus event whose metadata is to be decoded
   * @param reuse   an existing {@link org.apache.avro.generic.GenericRecord} object where the
   *                deserialized values will be written to. The object can be <b>null</b>, in
   *                which case a new object will be allocated.
   * @return {@link org.apache.avro.generic.GenericRecord} object with the deserialized data, or
   *         null if no metadata exists.  Returned in <b>reuse</b> if provided, else in a newly
   *         allocated object.
   * @throws DatabusRuntimeException if event contains metadata but schema to decode it is missing
   */
public GenericRecord getMetadata(DbusEvent e, GenericRecord reuse) {
    DbusEventPart metadataPart = e.getPayloadMetadataPart();
    ByteBuffer dataBuffer = null;
    if (null == metadataPart || null == (dataBuffer = metadataPart.getData()) || dataBuffer.remaining() <= 0) {
        LOG.debug("No metadata for event " + e);
        return null;
    }
    VersionedSchema schema = getMetadataSchema(metadataPart);
    if (null == schema) {
        throw new DatabusRuntimeException("No schema available to decode metadata for event " + e);
    }
    byte[] dataBytes = null;
    if (dataBuffer.hasArray()) {
        dataBytes = dataBuffer.array();
    } else {
        dataBytes = new byte[dataBuffer.remaining()];
        try {
            dataBuffer.get(dataBytes);
        } catch (BufferUnderflowException ex) {
            LOG.error("metadata buffer error (remaining = " + dataBuffer.remaining() + ") for event " + e, ex);
            return null;
        }
    }
    return getGenericRecord(dataBytes, schema.getSchema(), reuse);
}

Example 8 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class DbusEventAvroDecoder method getPayloadSchema.

@Override
public /**
   * @param e DatabusEvent
   * @return Avro Schema, sourceName, version tuple describing the payload data appearing in 'e'.
   */
VersionedSchema getPayloadSchema(DbusEvent e) {
    byte[] md5 = new byte[16];
    e.schemaId(md5);
    SchemaId schemaId = new SchemaId(md5);
    VersionedSchema writerSchema = _schemaSet.getById(schemaId);
    return writerSchema;
}

Example 9 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class GenericDispatcher method doCheckStartSource.

protected boolean doCheckStartSource(DispatcherState curState, Long eventSrcId, SchemaId schemaId) {
    boolean success = true;
    if (eventSrcId >= 0) {
        IdNamePair source = curState.getSources().get(eventSrcId);
        if (null == source) {
            _log.error("Unable to find source: srcid=" + eventSrcId);
            success = false;
        } else {
            VersionedSchema verSchema = curState.getSchemaSet().getLatestVersionByName(source.getName());
            VersionedSchema exactSchema = _schemaIdCheck ? curState.getSchemaSet().getById(schemaId) : null;
            if (null == verSchema) {
                _log.error("Unable to find schema: srcid=" + source.getId() + " name=" + source.getName());
                success = false;
            } else if (_schemaIdCheck && null == exactSchema) {
                _log.error("Unable to find schema: srcid=" + source.getId() + " name=" + source.getName() + " schemaId=" + schemaId);
                success = false;
            } else if (verSchema.getSchema() != curState.getCurrentSourceSchema()) {
                curState.switchToStartStreamSource(source, verSchema.getSchema());
                success = doStartStreamSource(curState);
            }
        }
    }
    return success;
}

Example 10 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class TestGenericDispatcher method testMetadataSchema.

@Test
public void testMetadataSchema() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testMetadataSchema");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
    //svsConsumer.getLog().setLevel(Level.DEBUG);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher, "testMetadataSchema-dispatcher");
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    //add meta data schema
    byte[] crc32 = { 0x01, 0x02, 0x03, 0x04 };
    List<RegisterResponseMetadataEntry> lMeta = new ArrayList<RegisterResponseMetadataEntry>();
    lMeta.add(new RegisterResponseMetadataEntry((short) 1, META1_SCHEMA_STR, crc32));
    lMeta.add(new RegisterResponseMetadataEntry((short) 2, META2_SCHEMA_STR, crc32));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap, lMeta));
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L, null);
    //check standard execution of callbacks
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("incorrect amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("incorrect amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    //check metadata schemas
    EventCountingConsumer myCons = (EventCountingConsumer) mockConsumer;
    VersionedSchema metadataSchema = myCons.getMetadataSchema();
    Assert.assertTrue(null != metadataSchema);
    log.info("Metadata VersionedSchema = " + metadataSchema);
    Assert.assertEquals(metadataSchema.getVersion(), 2);
    Assert.assertEquals(metadataSchema.getSchemaBaseName(), SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
    verifyNoLocks(log, eventsBuf);
    log.info("end\n");
}