Examples with VersionedSchema com.linkedin.databus2.schemas.VersionedSchema used on opensource projects

Example 1 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class DevRelayConfigGenerator method generateRelayConfig.

public static void generateRelayConfig(String schemaRegistryLocation, String dbName, String uri, String outputDir, List<String> srcNames, SchemaMetaDataManager manager) throws Exception {
    PhysicalSourceConfig config = new PhysicalSourceConfig();
    FileSystemSchemaRegistryService s = FileSystemSchemaRegistryService.build(new FileSystemSchemaRegistryService.StaticConfig(new File(schemaRegistryLocation), 0, false, false));
    dbName = dbName.trim().toLowerCase();
    config.setName(dbName);
    config.setUri(uri);
    for (String srcName : srcNames) {
        VersionedSchema schema = null;
        schema = s.fetchLatestVersionedSchemaBySourceName(srcName);
        String dbObjectName = SchemaHelper.getMetaField(schema.getSchema(), "dbFieldName");
        LogicalSourceConfig c = new LogicalSourceConfig();
        c.setId(manager.getSrcId(srcName));
        c.setName(srcName);
        c.setUri(dbName + "." + dbObjectName);
        c.setPartitionFunction("constant:1");
        config.addSource(c);
    }
    DatabusRelaySourcesInFiles relaySourcesInFiles = new DatabusRelaySourcesInFiles(outputDir);
    relaySourcesInFiles.add(dbName, config);
    boolean success = relaySourcesInFiles.save();
    if (!success)
        throw new RuntimeException("Unable to create the dev relay config for DB :" + dbName);
}

Example 2 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class GenericRecordDtailPrinter method printEvent.

/**
   * @see com.linkedin.databus2.tools.dtail.DtailPrinter#printEvent(com.linkedin.databus.core.DbusEventInternalReadable, com.linkedin.databus.client.pub.DbusEventDecoder)
   */
@Override
public ConsumerCallbackResult printEvent(DbusEventInternalReadable e, DbusEventDecoder eventDecoder) {
    DbusEventAvroDecoder avroDecoder = (DbusEventAvroDecoder) eventDecoder;
    switch(_metadataOutput) {
        case NONE:
            GenericRecord payload = eventDecoder.getGenericRecord(e, null);
            return payload != null ? printGenericRecord(payload) : ConsumerCallbackResult.SUCCESS;
        case ONLY:
            GenericRecord metadata = avroDecoder.getMetadata(e, null);
            return null != metadata ? printGenericRecord(metadata) : ConsumerCallbackResult.SUCCESS;
        case INCLUDE:
            GenericRecord payload1 = avroDecoder.getGenericRecord(e, null);
            GenericRecord metadata1 = avroDecoder.getMetadata(e, null);
            Schema pschema = Schema.createUnion(Arrays.asList(avroDecoder.getPayloadSchema(e).getSchema(), Schema.create(Type.NULL)));
            Field pfield = new Field("payload", pschema, "payload", null);
            VersionedSchema metaschema = avroDecoder.getMetadataSchema(e);
            Schema mschema = null != metaschema ? Schema.createUnion(Arrays.asList(metaschema.getSchema(), Schema.create(Type.NULL))) : Schema.createUnion(Arrays.asList(Schema.create(Type.INT), Schema.create(Type.NULL)));
            Field mfield = new Field("metadata", mschema, "metadata", null);
            Schema combined = Schema.createRecord(Arrays.asList(pfield, mfield));
            GenericRecord r = new GenericData.Record(combined);
            r.put(0, payload1);
            r.put(1, metadata1);
            return printGenericRecord(r);
        default:
            LOG.error("unknown metadata output mode: " + _metadataOutput);
            return ConsumerCallbackResult.ERROR_FATAL;
    }
}

Example 3 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class RegisterRequestProcessor method getMetadataSchemas.

/**
   * Returns list of versioned metadata schemas.
   * TODO (DDSDBUS-2093):  implement this.
   */
private // IN
void getMetadataSchemas(// IN
SchemaRegistryService schemaRegistry, // OUT
ArrayList<RegisterResponseMetadataEntry> registeredMetadata) throws RequestProcessingException {
    Map<SchemaId, VersionedSchema> versionedSchemas = null;
    try {
        VersionedSchemaSet schemaSet = schemaRegistry.fetchAllMetadataSchemaVersions();
        if (schemaSet != null) {
            versionedSchemas = schemaSet.getAllVersionsWithSchemaId(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
        }
    } catch (DatabusException ie) {
        HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
        if (relayStatsCollector != null)
            relayStatsCollector.registerInvalidRegisterCall();
        throw new RequestProcessingException(ie);
    }
    if (versionedSchemas != null && !versionedSchemas.isEmpty()) {
        for (SchemaId id : versionedSchemas.keySet()) {
            VersionedSchema entry = versionedSchemas.get(id);
            if (entry.getOrigSchemaStr() == null) {
                throw new RequestProcessingException("Null schema string for metadata version " + entry.getVersion());
            }
            registeredMetadata.add(new RegisterResponseMetadataEntry((short) entry.getVersion(), entry.getOrigSchemaStr(), id.getByteArray()));
        }
    }
}

Example 4 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class BootstrapTableReader method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    BootstrapSeederMain.Config bsConf = new BootstrapSeederMain.Config();
    ConfigLoader<BootstrapSeederMain.StaticConfig> configLoader = new ConfigLoader<BootstrapSeederMain.StaticConfig>("databus.reader.", bsConf);
    _bsStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    Config qConf = new Config();
    ConfigLoader<StaticConfig> configLoader2 = new ConfigLoader<StaticConfig>("databus.query.", qConf);
    _queryStaticConfig = configLoader2.loadConfig(_sQueryConfigProps);
    SchemaRegistryService schemaRegistry = FileSystemSchemaRegistryService.build(_bsStaticConfig.getSchemaRegistry().getFileSystem());
    LOG.info("Schema = " + schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));
    _schema = Schema.parse(schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));
    VersionedSchema vs = new VersionedSchema(_schema.getFullName(), (short) 1, _schema, null);
    VersionedSchemaSet schemaSet = new VersionedSchemaSet();
    schemaSet.add(vs);
    _decoder = new DbusEventAvroDecoder(schemaSet);
}

Example 5 with VersionedSchema

use of com.linkedin.databus2.schemas.VersionedSchema in project databus by linkedin.

the class DbusEventAvroDecoder method getTypedValue.

@Override
public <T extends SpecificRecord> T getTypedValue(DbusEvent e, T reuse, Class<T> targetClass) {
    if (null == reuse) {
        try {
            reuse = targetClass.newInstance();
        } catch (InstantiationException e1) {
            LOG.error("getTypedValue class instantiation error (" + e1.getMessage() + ") for event " + e, e1);
            return null;
        } catch (IllegalAccessException e1) {
            LOG.error("getTypedValue access error (" + e1.getMessage() + ") for event " + e, e1);
            return null;
        }
    }
    byte[] md5 = new byte[16];
    e.schemaId(md5);
    SchemaId schemaId = new SchemaId(md5);
    VersionedSchema writerSchema = _schemaSet.getById(schemaId);
    if (null == writerSchema) {
        LOG.error("Unable to find schema for id " + schemaId + "; event = " + e);
        throw new DatabusRuntimeException("No schema available to decode event " + e);
    }
    ByteBuffer valueBuffer = e.value();
    byte[] valueBytes = new byte[valueBuffer.remaining()];
    valueBuffer.get(valueBytes);
    try {
        //JsonDecoder jsonDec = new JsonDecoder(sourceSchema.getSchema(),new ByteArrayInputStream(valueBytes));
        binDecoder.set(DecoderFactory.defaultFactory().createBinaryDecoder(valueBytes, binDecoder.get()));
        SpecificDatumReader<SpecificRecord> reader = new SpecificDatumReader<SpecificRecord>(writerSchema.getSchema(), reuse.getSchema());
        return targetClass.cast(reader.read(reuse, binDecoder.get()));
    } catch (IOException e1) {
        LOG.error("getTypedValue IO error (" + e1.getMessage() + ") for event " + e, e1);
    }
    return reuse;
}