Examples with SchemaRegistryService com.linkedin.databus2.schemas.SchemaRegistryService used on opensource projects

Example 16 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class BootstrapAuditMain method main.

/**
   * @param args
   */
public static void main(String[] args) throws Exception {
    BootstrapSeederMain.init(args);
    BootstrapSeederMain.StaticConfig staticConfig = BootstrapSeederMain.getStaticConfig();
    int interval = staticConfig.getController().getCommitInterval();
    int sourceChunkSize = staticConfig.getController().getNumRowsPerQuery();
    List<OracleTriggerMonitoredSourceInfo> sources = BootstrapSeederMain.getSources();
    BootstrapDBSeeder seeder = BootstrapSeederMain.getSeeder();
    BootstrapSrcDBEventReader seedController = BootstrapSeederMain.getReader();
    Map<String, String> pKeyNameMap = seedController.getpKeyNameMap();
    Map<String, DbusEventKey.KeyType> pKeyTypeMap = seedController.getpKeyTypeMap();
    for (OracleTriggerMonitoredSourceInfo source : sources) {
        short srcId = source.getSourceId();
        new ConcurrentHashMap<Long, ResultSetEntry>();
        OracleTableReader oracleReader = null;
        MySQLTableReader mySQLReader = null;
        try {
            SchemaRegistryService schemaRegistry = FileSystemSchemaRegistryService.build(staticConfig.getSchemaRegistry().getFileSystem());
            Map<Short, String> schemaSet = schemaRegistry.fetchAllSchemaVersionsBySourceName(source.getSourceName());
            VersionedSchemaSet vSchemaSet = new VersionedSchemaSet();
            Iterator<Map.Entry<Short, String>> it = schemaSet.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<Short, String> pairs = it.next();
                Schema s = Schema.parse(pairs.getValue());
                VersionedSchema vs = new VersionedSchema(s.getFullName(), pairs.getKey(), s, null);
                vSchemaSet.add(vs);
            }
            /* Try and identify the schema key */
            VersionedSchema vschema = schemaRegistry.fetchLatestVersionedSchemaBySourceName(source.getSourceName());
            Schema schema = Schema.parse(vschema.getSchema().toString());
            LOG.info("Schema =" + vschema.getSchema() + "version=" + vschema.getVersion() + " name=" + vschema.getSchemaBaseName());
            /* Determine type of field txn */
            Field txnFieldType = schema.getField("txn");
            if (txnFieldType == null) {
                throw new Exception("Unable to find field called 'txn'. Cannot proceeed\n");
            }
            Type txnType = SchemaHelper.getAnyType(txnFieldType);
            /*
         * Determine primary key of schema. This is assumed to be invariant
         * across versions
         */
            String keyOverrideName = SchemaHelper.getMetaField(schema, "pk");
            String keyColumnName = "key";
            if (null != keyOverrideName) {
                keyColumnName = keyOverrideName;
            }
            Field pkeyField = schema.getField(keyColumnName);
            if (null == pkeyField) {
                keyColumnName = "id";
                pkeyField = schema.getField("id");
            }
            if (null == pkeyField) {
                throw new Exception("Unable to get the primary key for schema. Schema is :" + schema);
            }
            DbusEventAvroDecoder decoder = new DbusEventAvroDecoder(vSchemaSet);
            BootstrapAuditTester auditor = new BootstrapAuditTester(schema, BootstrapSrcDBEventReader.getTableName(source));
            List<BootstrapAuditTester> auditors = new ArrayList<BootstrapAuditTester>();
            auditors.add(auditor);
            oracleReader = new OracleTableReader(BootstrapSeederMain.getDataStore().getConnection(), BootstrapSrcDBEventReader.getTableName(source), pkeyField, SchemaHelper.getMetaField(pkeyField, "dbFieldName"), SchemaHelper.getAnyType(pkeyField), sourceChunkSize, seedController.getPKIndex(source), seedController.getQueryHint(source));
            mySQLReader = new MySQLTableReader(seeder.getConnection(), // THis is the primary
            seeder.getTableName(srcId), // THis is the primary
            pkeyField, // THis is the primary
            "id", // bootstrapDB
            SchemaHelper.getAnyType(pkeyField), interval);
            double samplePct = BootstrapSeederMain.getValidationSamplePct();
            TableComparator comparator = new TableComparator(oracleReader, mySQLReader, auditor, decoder, interval, pKeyNameMap.get(source.getEventView()), pKeyTypeMap.get(source.getEventView()), txnType, samplePct);
            boolean success = false;
            if (BootstrapSeederMain.getValidationType().equals("point")) {
                success = comparator.compareRecordsPoint();
            } else if (BootstrapSeederMain.getValidationType().equals("pointBs")) {
                success = comparator.compareRecordsPointBs();
            } else {
                success = comparator.compareRecordsNew();
            }
            if (success)
                LOG.info("Audit completed successfully");
            else
                LOG.error("Audit FAILED !!! ");
        } catch (Exception ex) {
            LOG.error("Caught an exception ex", ex);
            throw ex;
        } finally {
            if (null != oracleReader)
                oracleReader.close();
        }
    }
    DBHelper.close(seeder.getConnection());
}

Example 17 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class TestRegisterRequestProcessor method testDatabusExceptionInGetSchemas.

private void testDatabusExceptionInGetSchemas(final int protoVersion) throws Exception {
    LOG.info("Testing DatabusException in getSchemas() call with protocol version " + protoVersion);
    Properties params = new Properties();
    final int srcId1 = 101;
    final String srcName1 = "source-101";
    final String docSchema1 = "docSchema1";
    final String docSchema2 = "docSchema2";
    final short docSchemaV1 = 1;
    final short docSchemaV2 = 2;
    if (protoVersion != 0) {
        params.setProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, Integer.toString(protoVersion));
    }
    params.setProperty(RegisterRequestProcessor.SOURCES_PARAM, Integer.toString(srcId1));
    final StringBuilder responseStr = new StringBuilder();
    ChunkedWritableByteChannel chunkedWritableByteChannel = EasyMock.createMock(ChunkedWritableByteChannel.class);
    chunkedWritableByteChannel.addMetadata(EasyMock.anyObject(String.class), EasyMock.anyInt());
    EasyMock.expectLastCall().times(1);
    chunkedWritableByteChannel.write(EasyMock.anyObject(ByteBuffer.class));
    EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            Charset charset = Charset.forName("UTF-8");
            CharsetDecoder decoder = charset.newDecoder();
            responseStr.append(decoder.decode((ByteBuffer) EasyMock.getCurrentArguments()[0]));
            return responseStr.length();
        }
    });
    EasyMock.replay(chunkedWritableByteChannel);
    DatabusRequest mockReq = EasyMock.createMock(DatabusRequest.class);
    EasyMock.expect(mockReq.getParams()).andReturn(params).anyTimes();
    EasyMock.replay(mockReq);
    LogicalSource lsrc1 = new LogicalSource(srcId1, srcName1);
    SourceIdNameRegistry mockSrcIdReg = EasyMock.createMock(SourceIdNameRegistry.class);
    EasyMock.expect(mockSrcIdReg.getSource(srcId1)).andReturn(lsrc1).anyTimes();
    EasyMock.replay(mockSrcIdReg);
    Map<Short, String> srcSchemaVersions = new HashMap<Short, String>();
    srcSchemaVersions.put(docSchemaV1, docSchema1);
    srcSchemaVersions.put(docSchemaV2, docSchema2);
    DatabusException expectedCause = new DatabusException("FakeException");
    SchemaRegistryService mockSchemaReg = EasyMock.createMock(SchemaRegistryService.class);
    EasyMock.expect(mockSchemaReg.fetchAllSchemaVersionsBySourceName(srcName1)).andThrow(expectedCause);
    EasyMock.replay(mockSchemaReg);
    HttpRelay mockRelay = EasyMock.createMock(HttpRelay.class);
    EasyMock.expect(mockRelay.getHttpStatisticsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(mockRelay.getSourcesIdNameRegistry()).andReturn(mockSrcIdReg).anyTimes();
    EasyMock.expect(mockRelay.getSchemaRegistryService()).andReturn(mockSchemaReg).anyTimes();
    EasyMock.replay(mockRelay);
    RegisterRequestProcessor reqProcessor = new RegisterRequestProcessor(null, mockRelay);
    boolean exceptionCaught = false;
    try {
        reqProcessor.process(mockReq);
    } catch (RequestProcessingException e) {
        Assert.assertEquals(expectedCause, e.getCause());
    }
    EasyMock.verify(mockRelay);
    EasyMock.verify(mockReq);
    EasyMock.verify(mockSchemaReg);
    EasyMock.verify(mockSrcIdReg);
}

Example 18 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class TestRegisterRequestProcessor method testV4RegisterRequestProcessor.

// Test the happy path where there are 2 versions of document schema of a table and one version of metadata
// schema in the registry.
@Test
public void testV4RegisterRequestProcessor() throws Exception {
    Properties params = new Properties();
    final int protoVersion = 4;
    final int srcId1 = 101;
    final String srcName1 = "source-101";
    final String docSchema1 = "docSchema1";
    final String docSchema2 = "docSchema2";
    final String metadataSchema1 = makeMetadataSchema(1);
    final String metadataSchema2 = makeMetadataSchema(2);
    final byte[] metaSchemaDigest1 = new byte[] { 32, 33, 34, 35 };
    final byte[] metaSchemaDigest2 = new byte[] { 35, 34, 33, 32 };
    final short docSchemaV1 = 1;
    final short docSchemaV2 = 2;
    final short metaSchemaV1 = 1;
    final short metaSchemaV2 = 2;
    params.setProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, Integer.toString(protoVersion));
    params.setProperty(RegisterRequestProcessor.SOURCES_PARAM, Integer.toString(srcId1));
    final StringBuilder responseStr = new StringBuilder();
    ChunkedWritableByteChannel chunkedWritableByteChannel = EasyMock.createMock(ChunkedWritableByteChannel.class);
    chunkedWritableByteChannel.addMetadata(EasyMock.eq(DatabusHttpHeaders.DBUS_CLIENT_RELAY_PROTOCOL_VERSION_HDR), EasyMock.eq(protoVersion));
    chunkedWritableByteChannel.write(EasyMock.anyObject(ByteBuffer.class));
    EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            Charset charset = Charset.forName("UTF-8");
            CharsetDecoder decoder = charset.newDecoder();
            responseStr.append(decoder.decode((ByteBuffer) EasyMock.getCurrentArguments()[0]));
            return responseStr.length();
        }
    });
    EasyMock.replay(chunkedWritableByteChannel);
    DatabusRequest mockReq = EasyMock.createMock(DatabusRequest.class);
    EasyMock.expect(mockReq.getParams()).andReturn(params).anyTimes();
    EasyMock.expect(mockReq.getResponseContent()).andReturn(chunkedWritableByteChannel);
    EasyMock.replay(mockReq);
    LogicalSource lsrc1 = new LogicalSource(srcId1, srcName1);
    SourceIdNameRegistry mockSrcIdReg = EasyMock.createMock(SourceIdNameRegistry.class);
    EasyMock.expect(mockSrcIdReg.getSource(srcId1)).andReturn(lsrc1).anyTimes();
    EasyMock.replay(mockSrcIdReg);
    Map<Short, String> srcSchemaVersions = new HashMap<Short, String>();
    srcSchemaVersions.put(docSchemaV1, docSchema1);
    srcSchemaVersions.put(docSchemaV2, docSchema2);
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, metaSchemaV1, new SchemaId(metaSchemaDigest1), metadataSchema1, true);
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, metaSchemaV2, new SchemaId(metaSchemaDigest2), metadataSchema2, true);
    SchemaRegistryService mockSchemaReg = EasyMock.createMock(SchemaRegistryService.class);
    EasyMock.expect(mockSchemaReg.fetchAllSchemaVersionsBySourceName(srcName1)).andReturn(srcSchemaVersions).anyTimes();
    EasyMock.expect(mockSchemaReg.fetchAllMetadataSchemaVersions()).andReturn(metadataSchemaSet).anyTimes();
    EasyMock.replay(mockSchemaReg);
    HttpRelay mockRelay = EasyMock.createMock(HttpRelay.class);
    EasyMock.expect(mockRelay.getHttpStatisticsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(mockRelay.getSourcesIdNameRegistry()).andReturn(mockSrcIdReg).anyTimes();
    EasyMock.expect(mockRelay.getSchemaRegistryService()).andReturn(mockSchemaReg).anyTimes();
    EasyMock.replay(mockRelay);
    RegisterRequestProcessor reqProcessor = new RegisterRequestProcessor(null, mockRelay);
    reqProcessor.process(mockReq);
    // Decode
    ObjectMapper mapper = new ObjectMapper();
    HashMap<String, List<Object>> responseMap = mapper.readValue(responseStr.toString(), new TypeReference<HashMap<String, List<Object>>>() {
    });
    Map<Long, List<RegisterResponseEntry>> sourcesSchemasMap = RegisterResponseEntry.createFromResponse(responseMap, RegisterResponseEntry.SOURCE_SCHEMAS_KEY, false);
    // There should be one entry in the map, which is a  list.
    Assert.assertEquals(1, sourcesSchemasMap.size());
    Assert.assertEquals(2, sourcesSchemasMap.get(new Long(srcId1)).size());
    for (RegisterResponseEntry r : sourcesSchemasMap.get(new Long(srcId1))) {
        Assert.assertEquals(srcId1, r.getId());
        if (r.getVersion() == docSchemaV1) {
            Assert.assertEquals(docSchema1, r.getSchema());
        } else {
            Assert.assertEquals(docSchema2, r.getSchema());
        }
    }
    Map<Long, List<RegisterResponseEntry>> keysSchemasMap = RegisterResponseEntry.createFromResponse(responseMap, RegisterResponseEntry.KEY_SCHEMAS_KEY, true);
    Assert.assertNull(keysSchemasMap);
    List<RegisterResponseMetadataEntry> metadataSchemasList = RegisterResponseMetadataEntry.createFromResponse(responseMap, RegisterResponseMetadataEntry.METADATA_SCHEMAS_KEY, true);
    // The response should contain the exact string that the schema registry has.
    Assert.assertEquals(2, metadataSchemasList.size());
    for (RegisterResponseMetadataEntry r : metadataSchemasList) {
        if (r.getVersion() == 1) {
            Assert.assertEquals(metadataSchema1, r.getSchema());
            Assert.assertTrue(Arrays.equals(metaSchemaDigest1, r.getCrc32()));
        } else {
            Assert.assertEquals(metadataSchema2, r.getSchema());
            Assert.assertTrue(Arrays.equals(metaSchemaDigest2, r.getCrc32()));
        }
    }
    EasyMock.verify(mockRelay);
    EasyMock.verify(mockReq);
    EasyMock.verify(mockSchemaReg);
    EasyMock.verify(mockSrcIdReg);
}

Example 19 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class OracleRelayFactory method createRelay.

@Override
public HttpRelay createRelay() throws DatabusException {
    try {
        SchemaRegistryService schemaRegistry = _schemaRegistryFactory.createSchemaRegistry();
        HttpRelay.StaticConfig relayStaticConfig = _relayConfigBuilder.build();
        getSourcesIdNameRegistry().updateFromIdNamePairs(relayStaticConfig.getSourceIds());
        PhysicalSourceConfigBuilder psourceConfBuilder = new PhysicalSourceConfigBuilder(_physicalSourcesConfigFiles);
        return new HttpRelay(relayStaticConfig, psourceConfBuilder.build(), getSourcesIdNameRegistry(), schemaRegistry);
    } catch (IOException ioe) {
        throw new DatabusException("OracleRelay instantiation error: " + ioe.getMessage(), ioe);
    }
}

Example 20 with SchemaRegistryService

use of com.linkedin.databus2.schemas.SchemaRegistryService in project databus by linkedin.

the class StateMachineHelper method tableToSchema.

/**
   * Given a table name, extracts the avro schema corresponding to the table. If the table is not found (i.e., the relay is not configured
   * to host this table), returns null.
   * @param currentTable The table for which you want the avro schema (e.g. PERSON.ADDRESSBOOKTABLE)
   * @param tableToSourceName The map which has the table to the namespace of the schema (e.g. PERSON.ADDRESSBOOKTABLE => com.linkedin.com.person.addressBook)
   * @param schemaRegistryService The schema registry which holds all the schemas from where the schemas are looked up based on namespace
   * @return
   * @throws DatabusException
   */
public static Schema tableToSchema(String currentTable, HashMap<String, String> tableToSourceName, SchemaRegistryService schemaRegistryService) {
    String sourceName = tableToSourceName.get(currentTable);
    Schema schema = null;
    try {
        VersionedSchema vSchema = schemaRegistryService.fetchLatestVersionedSchemaBySourceName(sourceName);
        if (vSchema != null)
            schema = vSchema.getSchema();
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        LOG.error("Unable to fetch the schema for the table: " + currentTable + " with source name: " + sourceName + " because of the following error : " + e, e);
        return null;
    }
    if (schema == null)
        return null;
    return schema;
}