Examples with DbusEventPart com.linkedin.databus.core.DbusEventPart used on opensource projects

Example 1 with DbusEventPart

use of com.linkedin.databus.core.DbusEventPart in project databus by linkedin.

the class DbusEventAvroDecoder method getMetadata.

/**
 * Deserializes the metadata (if any) of a Databus event to an Avro GenericRecord.  This method
 * is for INTERNAL USE ONLY (by Espresso and Databus).  It is NOT a stable API and may change
 * without warning!
 *
 * @param e       the Databus event whose metadata is to be decoded
 * @param reuse   an existing {@link org.apache.avro.generic.GenericRecord} object where the
 *                deserialized values will be written to. The object can be <b>null</b>, in
 *                which case a new object will be allocated.
 * @return {@link org.apache.avro.generic.GenericRecord} object with the deserialized data, or
 *         null if no metadata exists.  Returned in <b>reuse</b> if provided, else in a newly
 *         allocated object.
 * @throws DatabusRuntimeException if event contains metadata but schema to decode it is missing
 */
public GenericRecord getMetadata(DbusEvent e, GenericRecord reuse) {
    DbusEventPart metadataPart = e.getPayloadMetadataPart();
    ByteBuffer dataBuffer = null;
    if (null == metadataPart || null == (dataBuffer = metadataPart.getData()) || dataBuffer.remaining() <= 0) {
        LOG.debug("No metadata for event " + e);
        return null;
    }
    VersionedSchema schema = getMetadataSchema(metadataPart);
    if (null == schema) {
        throw new DatabusRuntimeException("No schema available to decode metadata for event " + e);
    }
    byte[] dataBytes = null;
    if (dataBuffer.hasArray()) {
        dataBytes = dataBuffer.array();
    } else {
        dataBytes = new byte[dataBuffer.remaining()];
        try {
            dataBuffer.get(dataBytes);
        } catch (BufferUnderflowException ex) {
            LOG.error("metadata buffer error (remaining = " + dataBuffer.remaining() + ") for event " + e, ex);
            return null;
        }
    }
    return getGenericRecord(dataBytes, schema.getSchema(), reuse);
}

Example 2 with DbusEventPart

use of com.linkedin.databus.core.DbusEventPart in project databus by linkedin.

the class TestInternalMetadata method testGetMetadata_UnhappyPath_BadSchema.

/**
 * Verifies that getMetadata() returns null if there's a mismatch between the event's metadata
 * and the metadata schema whose signature/checksum is specified in the event header.
 */
@Test
public void testGetMetadata_UnhappyPath_BadSchema() throws Exception {
    LOG.info("starting testGetMetadata_UnhappyPath_BadSchema()");
    // build the event's metadata and then the event
    DbusEventPart metadataPart = createMetadataPart();
    DbusEvent event = createEvent(metadataPart);
    // create a metadata schema set with a schema that claims to match the event's
    // metadata but doesn't actually
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, // METADATA_SCHEMA_VERSION
    metadataPart.getSchemaVersion(), // METADATA_SCHEMA_CHECKSUM
    new SchemaId(metadataPart.getSchemaDigest()), INCORRECT_METADATA_SCHEMA, // preserve original string
    true);
    // now create the decoder and attempt to use it to extract and decode the event's metadata
    DbusEventAvroDecoder eventDecoder = createDecoder(metadataSchemaSet);
    try {
        GenericRecord reuse = null;
        GenericRecord decodedMetadata = eventDecoder.getMetadata(event, reuse);
        Assert.assertNull(decodedMetadata, "getMetadata() should have returned null;");
    } catch (Exception ex) {
        Assert.fail("getMetadata() should not have thrown exception: " + ex);
    }
    LOG.info("leaving testGetMetadata_UnhappyPath_BadSchema()");
}

Example 3 with DbusEventPart

use of com.linkedin.databus.core.DbusEventPart in project databus by linkedin.

the class LoggingConsumer method doDataEvent.

private ConsumerCallbackResult doDataEvent(DbusEvent e, DbusEventDecoder eventDecoder, ConsumerCallbackResult result, boolean bootstrapOn) {
    RuntimeConfig rtConfig = getRuntimeConfig();
    if (!rtConfig.isEnabled())
        return result;
    // check for event validity as long as the option is NOT disabled
    if (rtConfig.isValidityCheckEnabled()) {
        if (!e.isValid()) {
            _log.error("invalid event received:");
            _log.error(e.toString());
        }
    }
    // for backwards compatibility
    if (bootstrapOn) {
        ++_bstEventsNum;
        if (_bstEventsNum % BOOTSTRAP_EVENT_LOG_FREQUENCY == 1) {
            final VersionedSchema payloadSchema = eventDecoder.getPayloadSchema(e);
            String schemaName = (null == payloadSchema) ? "unknown source: " + e.getSourceId() : payloadSchema.getSchema().getName();
            String keyStr = null;
            try {
                if (e.isKeyString()) {
                    keyStr = new String(e.keyBytes(), "UTF-8");
                } else if (e.isKeyNumber()) {
                    keyStr = Long.toString(e.key());
                } else if (e.isKeySchema()) {
                    // TODO Fix to use a decoder (DDSDBUS-2076)
                    DbusEventPart keyPart = e.getKeyPart();
                    keyStr = keyPart.toString();
                }
            } catch (UnsupportedEncodingException e1) {
                keyStr = "unsupported key encoding";
            } catch (RuntimeException ex) {
                keyStr = "key decoding error: " + ex;
            }
            String msg = String.format(BOOTSTRAP_EVENT_LOG_FORMAT, _bstEventsNum, e.sequence(), keyStr, schemaName);
            _log.log(rtConfig.getLogLevel(), msg);
        }
    }
    _currentWindowScn = e.sequence();
    if (_fileBasedCallback != null) {
        _fileBasedCallback.onEvent(e);
    }
    if (_staticConfig.isLogTypedValue()) {
        logTypedValue(e, eventDecoder, rtConfig, bootstrapOn ? "b:" : "s:");
    }
    updateEventStats(e);
    return result;
}

Example 4 with DbusEventPart

use of com.linkedin.databus.core.DbusEventPart in project databus by linkedin.

the class TestInternalMetadata method testGetMetadata_HappyPath.

/**
 * Verifies that getMetadata() returns the expected GenericRecord for the event's
 * metadata and that it has the expected fields and values in it.
 */
@Test
public void testGetMetadata_HappyPath() throws Exception {
    LOG.info("starting testGetMetadata_HappyPath()");
    // build the event's metadata and then the event
    DbusEventPart metadataPart = createMetadataPart();
    DbusEvent event = createEvent(metadataPart);
    // create a metadata schema set that correctly corresponds to the metadata
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, // METADATA_SCHEMA_VERSION
    metadataPart.getSchemaVersion(), // METADATA_SCHEMA_CHECKSUM
    new SchemaId(metadataPart.getSchemaDigest()), CORRECT_METADATA_SCHEMA, // preserve original string
    true);
    // now create the decoder and use it to extract and decode the event's metadata
    DbusEventAvroDecoder eventDecoder = createDecoder(metadataSchemaSet);
    try {
        GenericRecord reuse = null;
        GenericRecord decodedMetadata = eventDecoder.getMetadata(event, reuse);
        Assert.assertNotNull(decodedMetadata, "getMetadata() returned null GenericRecord;");
        Utf8 etag = (Utf8) decodedMetadata.get("etag");
        Assert.assertEquals(etag.toString(), "dunno what an etag is");
        Integer flags = (Integer) decodedMetadata.get("flags");
        Assert.assertEquals(flags, null, "expected flags to be null");
        Long expires = (Long) decodedMetadata.get("expires");
        Assert.assertNotNull(expires, "expected expires to have a value;");
        Assert.assertEquals(expires.longValue(), 1366150681);
        Utf8 nonexistentField = (Utf8) decodedMetadata.get("nonexistentField");
        Assert.assertNull(nonexistentField, "unexpected value for 'nonexistentField';");
    } catch (Exception ex) {
        Assert.fail("unexpected error decoding metadata: " + ex);
    }
    LOG.info("leaving testGetMetadata_HappyPath()");
}

Example 5 with DbusEventPart

use of com.linkedin.databus.core.DbusEventPart in project databus by linkedin.

the class TestInternalMetadata method testGetMetadata_UnhappyPath_MissingSchema.

/**
 * Verifies that getMetadata() throws an exception if the metadata schema specified
 * in the event header is unavailable.
 */
@Test
public void testGetMetadata_UnhappyPath_MissingSchema() throws Exception {
    LOG.info("starting testGetMetadata_UnhappyPath_MissingSchema()");
    // build the event's metadata and then the event
    DbusEventPart metadataPart = createMetadataPart();
    DbusEvent event = createEvent(metadataPart);
    // create an empty metadata schema set
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    // now create the decoder and attempt to use it to extract and decode the event's metadata
    DbusEventAvroDecoder eventDecoder = createDecoder(metadataSchemaSet);
    try {
        GenericRecord reuse = null;
        GenericRecord decodedMetadata = eventDecoder.getMetadata(event, reuse);
        Assert.fail("getMetadata() should have thrown exception");
    } catch (Exception ex) {
    // expected case:  event had metadata, but schema to decode it was missing
    }
    LOG.info("leaving testGetMetadata_UnhappyPath_MissingSchema()");
}