Examples with SchemaRegistryService com.linkedin.databus2.schemas.SchemaRegistryService used on opensource projects

Example 1 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class BootstrapAvroFileSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapAvroFileEventReader(_sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sBootstrapBuffer);
}

Example 2 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class BootstrapSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    // Load the source configuration JSON file
    // File sourcesJson = new File("integration-test/config/sources-member2.json");
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    String sourceTypeStr = physicalSourceConfig.getReplBitSetter().getSourceType();
    if (SourceType.TOKEN.toString().equalsIgnoreCase(sourceTypeStr))
        throw new InvalidConfigException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
    // Create the OracleDataSource used to get DB connection(s)
    try {
        Class oracleDataSourceClass = OracleJarUtils.loadClass("oracle.jdbc.pool.OracleDataSource");
        Object ods = oracleDataSourceClass.newInstance();
        Method setURLMethod = oracleDataSourceClass.getMethod("setURL", String.class);
        setURLMethod.invoke(ods, uri);
        _sDataStore = (DataSource) ods;
    } catch (Exception e) {
        String errMsg = "Error creating a data source object ";
        LOG.error(errMsg, e);
        throw e;
    }
    // TODO: Need a better way than relaying on RelayFactory for generating MonitoredSourceInfo
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    Set<String> seenUris = new HashSet<String>();
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        String srcUri = sourceConfig.getUri();
        if (seenUris.contains(srcUri)) {
            String msg = "Uri (" + srcUri + ") is used for more than one sources. Currently Bootstrap Seeder cannot support seeding sources with the same URI together. Please have them run seperately !!";
            LOG.fatal(msg);
            throw new InvalidConfigException(msg);
        }
        seenUris.add(srcUri);
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapSrcDBEventReader(_sDataStore, _sBootstrapBuffer, _sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sSeeder.getLastKeys(), 0);
}

Example 3 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class BootstrapTableReader method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    BootstrapSeederMain.Config bsConf = new BootstrapSeederMain.Config();
    ConfigLoader<BootstrapSeederMain.StaticConfig> configLoader = new ConfigLoader<BootstrapSeederMain.StaticConfig>("databus.reader.", bsConf);
    _bsStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    Config qConf = new Config();
    ConfigLoader<StaticConfig> configLoader2 = new ConfigLoader<StaticConfig>("databus.query.", qConf);
    _queryStaticConfig = configLoader2.loadConfig(_sQueryConfigProps);
    SchemaRegistryService schemaRegistry = FileSystemSchemaRegistryService.build(_bsStaticConfig.getSchemaRegistry().getFileSystem());
    LOG.info("Schema = " + schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));
    _schema = Schema.parse(schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));
    VersionedSchema vs = new VersionedSchema(_schema.getFullName(), (short) 1, _schema, null);
    VersionedSchemaSet schemaSet = new VersionedSchemaSet();
    schemaSet.add(vs);
    _decoder = new DbusEventAvroDecoder(schemaSet);
}

Example 4 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class TestRegisterRequestProcessor method testNullSchemasInGetSchemas.

private void testNullSchemasInGetSchemas(final int protoVersion) throws Exception {
    LOG.info("Testing null return from fetchAllSchemaVersionsBySourceName() with protoversion " + protoVersion);
    Properties params = new Properties();
    final int srcId1 = 101;
    final String srcName1 = "source-101";
    if (protoVersion != 0) {
        params.setProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, Integer.toString(protoVersion));
    }
    params.setProperty(RegisterRequestProcessor.SOURCES_PARAM, Integer.toString(srcId1));
    final StringBuilder responseStr = new StringBuilder();
    ChunkedWritableByteChannel chunkedWritableByteChannel = EasyMock.createMock(ChunkedWritableByteChannel.class);
    // We should write out proto-version as 3 if none was specified in the input, otherwise match the proto version
    chunkedWritableByteChannel.addMetadata(EasyMock.eq(DatabusHttpHeaders.DBUS_CLIENT_RELAY_PROTOCOL_VERSION_HDR), protoVersion != 0 ? EasyMock.eq(protoVersion) : EasyMock.eq(3));
    EasyMock.expectLastCall().times(1);
    chunkedWritableByteChannel.write(EasyMock.anyObject(ByteBuffer.class));
    EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            Charset charset = Charset.forName("UTF-8");
            CharsetDecoder decoder = charset.newDecoder();
            responseStr.append(decoder.decode((ByteBuffer) EasyMock.getCurrentArguments()[0]));
            return responseStr.length();
        }
    });
    EasyMock.replay(chunkedWritableByteChannel);
    DatabusRequest mockReq = EasyMock.createMock(DatabusRequest.class);
    EasyMock.expect(mockReq.getParams()).andReturn(params).anyTimes();
    EasyMock.expect(mockReq.getResponseContent()).andReturn(chunkedWritableByteChannel);
    EasyMock.replay(mockReq);
    LogicalSource lsrc1 = new LogicalSource(srcId1, srcName1);
    SourceIdNameRegistry mockSrcIdReg = EasyMock.createMock(SourceIdNameRegistry.class);
    EasyMock.expect(mockSrcIdReg.getSource(srcId1)).andReturn(lsrc1).anyTimes();
    EasyMock.replay(mockSrcIdReg);
    SchemaRegistryService mockSchemaReg = EasyMock.createMock(SchemaRegistryService.class);
    EasyMock.expect(mockSchemaReg.fetchAllSchemaVersionsBySourceName(srcName1)).andReturn(null);
    EasyMock.replay(mockSchemaReg);
    HttpRelay mockRelay = EasyMock.createMock(HttpRelay.class);
    EasyMock.expect(mockRelay.getHttpStatisticsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(mockRelay.getSourcesIdNameRegistry()).andReturn(mockSrcIdReg).anyTimes();
    EasyMock.expect(mockRelay.getSchemaRegistryService()).andReturn(mockSchemaReg).anyTimes();
    EasyMock.replay(mockRelay);
    RegisterRequestProcessor reqProcessor = new RegisterRequestProcessor(null, mockRelay);
    reqProcessor.process(mockReq);
    ObjectMapper mapper = new ObjectMapper();
    List<RegisterResponseEntry> schemasList = mapper.readValue(responseStr.toString(), new TypeReference<List<RegisterResponseEntry>>() {
    });
    Map<Long, List<RegisterResponseEntry>> sourcesSchemasMap = RegisterResponseEntry.convertSchemaListToMap(schemasList);
    // There should be 1 entry in the map.
    Assert.assertEquals(0, sourcesSchemasMap.size());
    EasyMock.verify(mockRelay);
    EasyMock.verify(mockReq);
    EasyMock.verify(mockSchemaReg);
    EasyMock.verify(mockSrcIdReg);
}

Example 5 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class TestRegisterRequestProcessor method testRegisterReqProcessorVx.

// Test of happy path when the protocol version is specified as 2 or 3,
// or not specified at all.
// We should send out the source schemas only, and that too as a list.
private void testRegisterReqProcessorVx(final int protoVersion) throws Exception {
    LOG.info("Verifying happy path with protocol version: " + protoVersion);
    Properties params = new Properties();
    final int srcId1 = 101;
    final String srcName1 = "source-101";
    final String docSchema1 = "docSchema1";
    final String docSchema2 = "docSchema2";
    final short docSchemaV1 = 1;
    final short docSchemaV2 = 2;
    if (protoVersion != 0) {
        params.setProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, Integer.toString(protoVersion));
    }
    params.setProperty(RegisterRequestProcessor.SOURCES_PARAM, Integer.toString(srcId1));
    final StringBuilder responseStr = new StringBuilder();
    ChunkedWritableByteChannel chunkedWritableByteChannel = EasyMock.createMock(ChunkedWritableByteChannel.class);
    // We should write out proto-version as 3 if none was specified in the input, otherwise match the proto version
    chunkedWritableByteChannel.addMetadata(EasyMock.eq(DatabusHttpHeaders.DBUS_CLIENT_RELAY_PROTOCOL_VERSION_HDR), protoVersion != 0 ? EasyMock.eq(protoVersion) : EasyMock.eq(3));
    EasyMock.expectLastCall().times(1);
    chunkedWritableByteChannel.write(EasyMock.anyObject(ByteBuffer.class));
    EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            Charset charset = Charset.forName("UTF-8");
            CharsetDecoder decoder = charset.newDecoder();
            responseStr.append(decoder.decode((ByteBuffer) EasyMock.getCurrentArguments()[0]));
            return responseStr.length();
        }
    });
    EasyMock.replay(chunkedWritableByteChannel);
    DatabusRequest mockReq = EasyMock.createMock(DatabusRequest.class);
    EasyMock.expect(mockReq.getParams()).andReturn(params).anyTimes();
    EasyMock.expect(mockReq.getResponseContent()).andReturn(chunkedWritableByteChannel);
    EasyMock.replay(mockReq);
    LogicalSource lsrc1 = new LogicalSource(srcId1, srcName1);
    SourceIdNameRegistry mockSrcIdReg = EasyMock.createMock(SourceIdNameRegistry.class);
    EasyMock.expect(mockSrcIdReg.getSource(srcId1)).andReturn(lsrc1).anyTimes();
    EasyMock.replay(mockSrcIdReg);
    Map<Short, String> srcSchemaVersions = new HashMap<Short, String>();
    srcSchemaVersions.put(docSchemaV1, docSchema1);
    srcSchemaVersions.put(docSchemaV2, docSchema2);
    SchemaRegistryService mockSchemaReg = EasyMock.createMock(SchemaRegistryService.class);
    EasyMock.expect(mockSchemaReg.fetchAllSchemaVersionsBySourceName(srcName1)).andReturn(srcSchemaVersions).anyTimes();
    EasyMock.replay(mockSchemaReg);
    HttpRelay mockRelay = EasyMock.createMock(HttpRelay.class);
    EasyMock.expect(mockRelay.getHttpStatisticsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(mockRelay.getSourcesIdNameRegistry()).andReturn(mockSrcIdReg).anyTimes();
    EasyMock.expect(mockRelay.getSchemaRegistryService()).andReturn(mockSchemaReg).anyTimes();
    EasyMock.replay(mockRelay);
    RegisterRequestProcessor reqProcessor = new RegisterRequestProcessor(null, mockRelay);
    reqProcessor.process(mockReq);
    ObjectMapper mapper = new ObjectMapper();
    List<RegisterResponseEntry> schemasList = mapper.readValue(responseStr.toString(), new TypeReference<List<RegisterResponseEntry>>() {
    });
    Map<Long, List<RegisterResponseEntry>> sourcesSchemasMap = RegisterResponseEntry.convertSchemaListToMap(schemasList);
    // There should be 1 entry in the map.
    Assert.assertEquals(1, sourcesSchemasMap.size());
    Assert.assertEquals(2, sourcesSchemasMap.get(new Long(srcId1)).size());
    for (RegisterResponseEntry r : sourcesSchemasMap.get(new Long(srcId1))) {
        Assert.assertEquals(srcId1, r.getId());
        if (r.getVersion() == docSchemaV1) {
            Assert.assertEquals(docSchema1, r.getSchema());
        } else {
            Assert.assertEquals(docSchema2, r.getSchema());
        }
    }
    EasyMock.verify(mockRelay);
    EasyMock.verify(mockReq);
    EasyMock.verify(mockSchemaReg);
    EasyMock.verify(mockSrcIdReg);
}