use of com.linkedin.databus2.core.container.request.RegisterResponseMetadataEntry in project databus by linkedin.
the class RegisterRequestProcessor method getMetadataSchemas.
/**
* Returns list of versioned metadata schemas.
* TODO (DDSDBUS-2093): implement this.
*/
private // IN
void getMetadataSchemas(// IN
SchemaRegistryService schemaRegistry, // OUT
ArrayList<RegisterResponseMetadataEntry> registeredMetadata) throws RequestProcessingException {
Map<SchemaId, VersionedSchema> versionedSchemas = null;
try {
VersionedSchemaSet schemaSet = schemaRegistry.fetchAllMetadataSchemaVersions();
if (schemaSet != null) {
versionedSchemas = schemaSet.getAllVersionsWithSchemaId(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
}
} catch (DatabusException ie) {
HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
if (relayStatsCollector != null)
relayStatsCollector.registerInvalidRegisterCall();
throw new RequestProcessingException(ie);
}
if (versionedSchemas != null && !versionedSchemas.isEmpty()) {
for (SchemaId id : versionedSchemas.keySet()) {
VersionedSchema entry = versionedSchemas.get(id);
if (entry.getOrigSchemaStr() == null) {
throw new RequestProcessingException("Null schema string for metadata version " + entry.getVersion());
}
registeredMetadata.add(new RegisterResponseMetadataEntry((short) entry.getVersion(), entry.getOrigSchemaStr(), id.getByteArray()));
}
}
}
use of com.linkedin.databus2.core.container.request.RegisterResponseMetadataEntry in project databus by linkedin.
the class RegisterRequestProcessor method process.
@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
try {
// fail early if optional version param is included but isn't valid
// 2 and 3 are same for us; 4 is a superset only newer clients understand
int registerRequestProtocolVersion = 3;
String registerRequestProtocolVersionStr = request.getParams().getProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM);
if (registerRequestProtocolVersionStr != null) {
try {
registerRequestProtocolVersion = Integer.parseInt(registerRequestProtocolVersionStr);
} catch (NumberFormatException e) {
LOG.error("Could not parse /register request protocol version: " + registerRequestProtocolVersionStr);
throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, registerRequestProtocolVersionStr);
}
if (registerRequestProtocolVersion < 2 || registerRequestProtocolVersion > 4) {
LOG.error("Out-of-range /register request protocol version: " + registerRequestProtocolVersionStr);
throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, registerRequestProtocolVersionStr);
}
}
Collection<LogicalSource> logicalSources = null;
HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
String sources = request.getParams().getProperty(SOURCES_PARAM);
if (null == sources) {
// need to return all schemas, so first get all sources
logicalSources = _relay.getSourcesIdNameRegistry().getAllSources();
} else {
String[] sourceIds = sources.split(",");
logicalSources = new ArrayList<LogicalSource>(sourceIds.length);
for (String sourceId : sourceIds) {
int srcId;
String trimmedSourceId = sourceId.trim();
try {
srcId = Integer.valueOf(trimmedSourceId);
LogicalSource lsource = _relay.getSourcesIdNameRegistry().getSource(srcId);
if (null != lsource)
logicalSources.add(lsource);
else {
LOG.error("No source name for source id: " + srcId);
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
}
} catch (NumberFormatException nfe) {
if (relayStatsCollector != null) {
relayStatsCollector.registerInvalidRegisterCall();
}
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
}
}
}
SchemaRegistryService schemaRegistry = _relay.getSchemaRegistryService();
ArrayList<RegisterResponseEntry> registeredSources = new ArrayList<RegisterResponseEntry>(20);
for (LogicalSource lsource : logicalSources) {
getSchemas(schemaRegistry, lsource.getName(), lsource.getId(), sources, registeredSources);
}
// Note that, as of April 2013, the Espresso sandbox's schema registry
// (in JSON format) is 4.5 MB and growing. But 100 KB is probably OK
// for regular production cases.
StringWriter out = new StringWriter(102400);
ObjectMapper mapper = new ObjectMapper();
// any circumstances under which we might want to override this?
int registerResponseProtocolVersion = registerRequestProtocolVersion;
if (// DDSDBUS-2009
registerRequestProtocolVersion == 4) {
LOG.debug("Got version 4 /register request; fetching metadata schema.");
// Get (replication) metadata schema from registry; format it as list
// of schemas (multiple only if more than one version exists). Per
// https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Espresso+Metadata+Schema,
// name of replication metadata is simply "metadata".
ArrayList<RegisterResponseMetadataEntry> registeredMetadata = new ArrayList<RegisterResponseMetadataEntry>(2);
getMetadataSchemas(schemaRegistry, registeredMetadata);
// Set up the v4 response as a map: one entry is the existing list of source
// schemas, and the others (if present) are the new lists of metadata schema(s)
// and (TODO) key schemas.
HashMap<String, List<Object>> responseMap = new HashMap<String, List<Object>>(4);
responseMap.put(RegisterResponseEntry.SOURCE_SCHEMAS_KEY, (List<Object>) (List<?>) registeredSources);
if (registeredMetadata.size() > 0) {
LOG.debug("Sending v4 /register response with metadata schema.");
responseMap.put(RegisterResponseMetadataEntry.METADATA_SCHEMAS_KEY, (List<Object>) (List<?>) registeredMetadata);
} else {
LOG.debug("No metadata schema available; sending v4 /register response without.");
}
// TODO: figure out how to retrieve key schemas and include via RegisterResponseEntry.KEY_SCHEMAS_KEY
mapper.writeValue(out, responseMap);
} else // fall back to old style (v2/v3 response)
{
mapper.writeValue(out, registeredSources);
}
ChunkedWritableByteChannel responseContent = request.getResponseContent();
byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
responseContent.addMetadata(DatabusHttpHeaders.DBUS_CLIENT_RELAY_PROTOCOL_VERSION_HDR, registerResponseProtocolVersion);
responseContent.write(ByteBuffer.wrap(resultBytes));
if (null != relayStatsCollector) {
HttpStatisticsCollector connStatsCollector = (HttpStatisticsCollector) request.getParams().get(relayStatsCollector.getName());
if (null != connStatsCollector) {
connStatsCollector.registerRegisterCall(registeredSources);
} else {
relayStatsCollector.registerRegisterCall(registeredSources);
}
}
return request;
} catch (InvalidRequestParamValueException e) {
HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
if (null != relayStatsCollector)
relayStatsCollector.registerInvalidRegisterCall();
throw e;
}
}
use of com.linkedin.databus2.core.container.request.RegisterResponseMetadataEntry in project databus by linkedin.
the class DispatcherState method refreshSchemas.
private void refreshSchemas(List<RegisterResponseMetadataEntry> metadataSchemaList) {
final boolean isDebugEnabled = LOG.isDebugEnabled();
try {
for (Map.Entry<Long, List<RegisterResponseEntry>> e : _payloadSchemaMap.entrySet()) {
for (RegisterResponseEntry r : e.getValue()) {
final long id = r.getId();
String schemaName = null;
if (_sources.containsKey(id)) {
schemaName = _sources.get(r.getId()).getName();
} else {
LOG.error("Obtained a RegisterResponseEntry with schema that has no sourceId set. id = " + id);
continue;
}
String schema = r.getSchema();
if (_schemaSet.add(schemaName, r.getVersion(), schema)) {
LOG.info("Registering schema name=" + schemaName + " id=" + e.getKey().toString() + " version=" + r.getVersion());
if (isDebugEnabled) {
String msg = "Registering schema name=" + schemaName + " id=" + e.getKey().toString() + " version=" + r.getVersion() + ": " + schema;
DbusLogAccumulator.addLog(msg, LOG);
}
} else {
if (isDebugEnabled) {
String msg = "Schema already known: " + schemaName + " version " + r.getId();
DbusLogAccumulator.addLog(msg, LOG);
}
}
}
}
//Refresh metadata schema map
if ((metadataSchemaList != null) && !metadataSchemaList.isEmpty()) {
for (RegisterResponseMetadataEntry e : metadataSchemaList) {
SchemaId id = new SchemaId(e.getCrc32());
if (_metadataSchemasSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, e.getVersion(), id, e.getSchema())) {
LOG.info("Added metadata schema version " + e.getVersion() + ",schemaID=0x" + id);
} else {
if (isDebugEnabled) {
String msg = "Metadata schema version " + e.getVersion() + ",schemaId=0x" + id + " already exists";
DbusLogAccumulator.addLog(msg, LOG);
}
}
}
} else {
if (isDebugEnabled) {
String msg = "Metadata schema is empty";
DbusLogAccumulator.addLog(msg, LOG);
}
}
_eventDecoder = new DbusEventAvroDecoder(_schemaSet, _metadataSchemasSet);
} catch (Exception e) {
LOG.error("Error adding schema", e);
}
}
use of com.linkedin.databus2.core.container.request.RegisterResponseMetadataEntry in project databus by linkedin.
the class TestGenericDispatcher method testMetadataSchema.
@Test
public void testMetadataSchema() {
final Logger log = Logger.getLogger("TestGenericDispatcher.testMetadataSchema");
//log.setLevel(Level.DEBUG);
log.info("start");
int source1EventsNum = 2;
int source2EventsNum = 2;
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
//svsConsumer.getLog().setLevel(Level.DEBUG);
DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
Thread dispatcherThread = new Thread(dispatcher, "testMetadataSchema-dispatcher");
//dispatcherThread.setDaemon(true);
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
//add meta data schema
byte[] crc32 = { 0x01, 0x02, 0x03, 0x04 };
List<RegisterResponseMetadataEntry> lMeta = new ArrayList<RegisterResponseMetadataEntry>();
lMeta.add(new RegisterResponseMetadataEntry((short) 1, META1_SCHEMA_STR, crc32));
lMeta.add(new RegisterResponseMetadataEntry((short) 2, META2_SCHEMA_STR, crc32));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap, lMeta));
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
eventsBuf.endEvents(100L, null);
//check standard execution of callbacks
try {
Thread.sleep(2000);
} catch (InterruptedException ie) {
}
dispatcher.shutdown();
for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
}
assertEquals("incorrect amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
assertEquals("incorrect amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
//check metadata schemas
EventCountingConsumer myCons = (EventCountingConsumer) mockConsumer;
VersionedSchema metadataSchema = myCons.getMetadataSchema();
Assert.assertTrue(null != metadataSchema);
log.info("Metadata VersionedSchema = " + metadataSchema);
Assert.assertEquals(metadataSchema.getVersion(), 2);
Assert.assertEquals(metadataSchema.getSchemaBaseName(), SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
verifyNoLocks(log, eventsBuf);
log.info("end\n");
}
use of com.linkedin.databus2.core.container.request.RegisterResponseMetadataEntry in project databus by linkedin.
the class DummyRemoteExceptionHandler method initSchemaObjectsLists.
@SuppressWarnings("unchecked")
private static void initSchemaObjectsLists() {
List<RegisterResponseEntry> sourceSchemasList = new ArrayList<RegisterResponseEntry>();
sourceSchemasList.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
_sourceObjectsList = (List<Object>) (List<?>) sourceSchemasList;
List<RegisterResponseEntry> keySchemasList = new ArrayList<RegisterResponseEntry>();
keySchemasList.add(new RegisterResponseEntry(1L, (short) 7, KEY1_SCHEMA_STR));
_keyObjectsList = (List<Object>) (List<?>) keySchemasList;
List<RegisterResponseMetadataEntry> metadataSchemasList = new ArrayList<RegisterResponseMetadataEntry>();
byte[] tbdCrc32 = new byte[DbusEvent.CRC32_DIGEST_LEN];
final short schemaVersion = 5;
metadataSchemasList.add(new RegisterResponseMetadataEntry(schemaVersion, METADATA1_SCHEMA_STR, tbdCrc32));
_metadataObjectsList = (List<Object>) (List<?>) metadataSchemasList;
}
Aggregations