use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.
the class RegistrationIdGenerator method generateNewId.
/**
* Queries the existing database of registrations inside the client and generates a new id different from
* any of them
*
* @param prefix : A String prefix which can be specified to precede the id to be generated. Typically this
* is the name of the Consumer class
* @param subsSources : List of subscriptions the consumer is interested in
* @return RegistrationId : Generated based on the prefix _ 8 byte md5 hash ( subscriptions ) _ count
*/
public static RegistrationId generateNewId(String prefix, Collection<DatabusSubscription> subsSources) {
StringBuilder subscription = new StringBuilder();
for (DatabusSubscription ds : subsSources) {
if (ds != null)
subscription.append(ds.generateSubscriptionString());
}
String id = generateUniqueString(prefix, subscription.toString());
RegistrationId rid = new RegistrationId(id);
return rid;
}
use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.
the class RelayPullThread method buildSubsList.
/**
* Build the string of subs to be sent in the register call.
* */
private String buildSubsList(List<DatabusSubscription> subs, Map<String, IdNamePair> sourceNameMap) {
StringBuilder sb = new StringBuilder(128);
sb.append("[");
boolean first = true;
for (DatabusSubscription sub : subs) {
if (!first)
sb.append(',');
DatabusSubscription realSub = sub;
LogicalSource ls = sub.getLogicalSource();
if (!ls.idKnown() && !ls.isWildcard()) {
IdNamePair sourceEntry = sourceNameMap.get(ls.getName());
if (null == sourceEntry) {
//this should never happen
throw new RuntimeException("FATAL! unable to find logical source " + ls.getName() + " in " + sourceNameMap);
}
realSub = new DatabusSubscription(sub.getPhysicalSource(), sub.getPhysicalPartition(), new LogicalSourceId(new LogicalSource(sourceEntry.getId().intValue(), ls.getName()), sub.getLogicalPartition().getId()));
}
sb.append(realSub.toJsonString());
first = false;
}
sb.append("]");
return sb.toString();
}
use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.
the class DatabusSourcesConnection method composeName.
// figure out name for the connection - to be used in mbean
private String composeName(String id) {
StringBuilder shortSourcesListBuilder = new StringBuilder();
String separatorChar = "[";
for (DatabusSubscription sub : getSubscriptions()) {
shortSourcesListBuilder.append(separatorChar);
PhysicalPartition p = sub.getPhysicalPartition();
String sourceName = "AnySource";
LogicalSource source = sub.getLogicalPartition().getSource();
if (!source.isAllSourcesWildcard()) {
sourceName = source.getName();
int lastDotIdx = sourceName.lastIndexOf('.');
if (lastDotIdx >= 0)
sourceName = sourceName.substring(lastDotIdx + 1);
}
String partString = "AnyPPart_";
if (!p.isAnyPartitionWildcard()) {
partString = p.getId() + "_";
}
shortSourcesListBuilder.append(partString + sourceName);
separatorChar = "_";
}
shortSourcesListBuilder.append(']');
String shortSourcesList = shortSourcesListBuilder.toString();
return "conn" + shortSourcesList + (id == null ? "" : "_" + id);
}
use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.
the class ReadEventsRequestProcessor method process.
@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException, DatabusException {
boolean isDebug = LOG.isDebugEnabled();
try {
ObjectMapper objMapper = new ObjectMapper();
String checkpointString = request.getParams().getProperty(CHECKPOINT_PARAM, null);
String checkpointStringMult = request.getParams().getProperty(CHECKPOINT_PARAM_MULT, null);
int fetchSize = request.getRequiredIntParam(FETCH_SIZE_PARAM);
String formatStr = request.getRequiredStringParam(OUTPUT_FORMAT_PARAM);
Encoding enc = Encoding.valueOf(formatStr.toUpperCase());
String sourcesListStr = request.getParams().getProperty(SOURCES_PARAM, null);
String subsStr = request.getParams().getProperty(SUBS_PARAM, null);
String partitionInfoStr = request.getParams().getProperty(PARTITION_INFO_STRING);
String streamFromLatestSCNStr = request.getParams().getProperty(STREAM_FROM_LATEST_SCN);
String clientMaxEventVersionStr = request.getParams().getProperty(DatabusHttpHeaders.MAX_EVENT_VERSION);
int clientEventVersion = (clientMaxEventVersionStr != null) ? Integer.parseInt(clientMaxEventVersionStr) : DbusEventFactory.DBUS_EVENT_V1;
if (clientEventVersion < 0 || clientEventVersion == 1 || clientEventVersion > DbusEventFactory.DBUS_EVENT_V2) {
throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.MAX_EVENT_VERSION, clientMaxEventVersionStr);
}
if (null == sourcesListStr && null == subsStr) {
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM + "|" + SUBS_PARAM, "null");
}
//TODO for now we separte the code paths to limit the impact on existing Databus 2 deployments (DDSDBUS-79)
//We have to get rid of this eventually and have a single data path.
boolean v2Mode = null == subsStr;
DbusKeyCompositeFilter keyCompositeFilter = null;
if (null != partitionInfoStr) {
try {
Map<Long, DbusKeyFilter> fMap = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(partitionInfoStr);
keyCompositeFilter = new DbusKeyCompositeFilter();
keyCompositeFilter.setFilterMap(fMap);
if (isDebug)
LOG.debug("keyCompositeFilter is :" + keyCompositeFilter);
} catch (Exception ex) {
String msg = "Got exception while parsing partition Configs. PartitionInfo is:" + partitionInfoStr;
LOG.error(msg, ex);
throw new InvalidRequestParamValueException(COMMAND_NAME, PARTITION_INFO_STRING, partitionInfoStr);
}
}
boolean streamFromLatestSCN = false;
if (null != streamFromLatestSCNStr) {
streamFromLatestSCN = Boolean.valueOf(streamFromLatestSCNStr);
}
long start = System.currentTimeMillis();
List<DatabusSubscription> subs = null;
//parse source ids
SourceIdNameRegistry srcRegistry = _relay.getSourcesIdNameRegistry();
HashSet<Integer> sourceIds = new HashSet<Integer>();
if (null != sourcesListStr) {
String[] sourcesList = sourcesListStr.split(",");
for (String sourceId : sourcesList) {
try {
Integer srcId = Integer.valueOf(sourceId);
sourceIds.add(srcId);
} catch (NumberFormatException nfe) {
HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerInvalidStreamRequest();
}
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
}
}
}
//process explicit subscriptions and generate respective logical partition filters
NavigableSet<PhysicalPartitionKey> ppartKeys = null;
if (null != subsStr) {
List<DatabusSubscription.Builder> subsBuilder = null;
subsBuilder = objMapper.readValue(subsStr, new TypeReference<List<DatabusSubscription.Builder>>() {
});
subs = new ArrayList<DatabusSubscription>(subsBuilder.size());
for (DatabusSubscription.Builder subBuilder : subsBuilder) {
subs.add(subBuilder.build());
}
ppartKeys = new TreeSet<PhysicalPartitionKey>();
for (DatabusSubscription sub : subs) {
PhysicalPartition ppart = sub.getPhysicalPartition();
if (ppart.isAnyPartitionWildcard()) {
ppartKeys = _eventBuffer.getAllPhysicalPartitionKeys();
break;
} else {
ppartKeys.add(new PhysicalPartitionKey(ppart));
}
}
}
// Need to make sure that we don't have tests that send requests in this form.
if (subs != null && checkpointStringMult == null && checkpointString != null) {
throw new RequestProcessingException("Both Subscriptions and CheckpointMult should be present");
}
//convert source ids into subscriptions
if (null == subs)
subs = new ArrayList<DatabusSubscription>();
for (Integer srcId : sourceIds) {
LogicalSource lsource = srcRegistry.getSource(srcId);
if (lsource == null)
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, srcId.toString());
if (isDebug)
LOG.debug("registry returns " + lsource + " for srcid=" + srcId);
DatabusSubscription newSub = DatabusSubscription.createSimpleSourceSubscription(lsource);
subs.add(newSub);
}
DbusFilter ppartFilters = null;
if (subs.size() > 0) {
try {
ppartFilters = _eventBuffer.constructFilters(subs);
} catch (DatabusException de) {
throw new RequestProcessingException("unable to generate physical partitions filters:" + de.getMessage(), de);
}
}
ConjunctionDbusFilter filters = new ConjunctionDbusFilter();
// Source filter comes first
if (v2Mode)
filters.addFilter(new SourceDbusFilter(sourceIds));
else if (null != ppartFilters)
filters.addFilter(ppartFilters);
/*
// Key range filter comes next
if ((keyMin >0) && (keyMax > 0))
{
filters.addFilter(new KeyRangeFilter(keyMin, keyMax));
}
*/
if (null != keyCompositeFilter) {
filters.addFilter(keyCompositeFilter);
}
// need to update registerStreamRequest to support Mult checkpoint TODO (DDSDBUS-80)
// temp solution
// 3 options:
// 1. checkpointStringMult not null - generate checkpoint from it
// 2. checkpointStringMult null, checkpointString not null - create empty CheckpointMult
// and add create Checkpoint(checkpointString) and add it to cpMult;
// 3 both are null - create empty CheckpointMult and add empty Checkpoint to it for each ppartition
PhysicalPartition pPartition;
Checkpoint cp = null;
CheckpointMult cpMult = null;
if (checkpointStringMult != null) {
try {
cpMult = new CheckpointMult(checkpointStringMult);
} catch (InvalidParameterSpecException e) {
LOG.error("Invalid CheckpointMult:" + checkpointStringMult, e);
throw new InvalidRequestParamValueException("stream", "CheckpointMult", checkpointStringMult);
}
} else {
// there is no checkpoint - create an empty one
cpMult = new CheckpointMult();
Iterator<Integer> it = sourceIds.iterator();
while (it.hasNext()) {
Integer srcId = it.next();
pPartition = _eventBuffer.getPhysicalPartition(srcId);
if (pPartition == null)
throw new RequestProcessingException("unable to find physical partitions for source:" + srcId);
if (checkpointString != null) {
cp = new Checkpoint(checkpointString);
} else {
cp = new Checkpoint();
cp.setFlexible();
}
cpMult.addCheckpoint(pPartition, cp);
}
}
if (isDebug)
LOG.debug("checkpointStringMult = " + checkpointStringMult + ";singlecheckpointString=" + checkpointString + ";CPM=" + cpMult);
// of the server context.
if (cpMult.getCursorPartition() == null) {
cpMult.setCursorPartition(request.getCursorPartition());
}
if (isDebug) {
if (cpMult.getCursorPartition() != null) {
LOG.debug("Using physical paritition cursor " + cpMult.getCursorPartition());
}
}
// for registerStreamRequest we need a single Checkpoint (TODO - fix it) (DDSDBUS-81)
if (cp == null) {
Iterator<Integer> it = sourceIds.iterator();
if (it.hasNext()) {
Integer srcId = it.next();
pPartition = _eventBuffer.getPhysicalPartition(srcId);
cp = cpMult.getCheckpoint(pPartition);
} else {
cp = new Checkpoint();
cp.setFlexible();
}
}
if (null != checkpointString && isDebug)
LOG.debug("About to stream from cp: " + checkpointString.toString());
HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
HttpStatisticsCollector connHttpStatsCollector = null;
if (null != globalHttpStatsCollector) {
connHttpStatsCollector = (HttpStatisticsCollector) request.getParams().get(globalHttpStatsCollector.getName());
}
if (null != globalHttpStatsCollector)
globalHttpStatsCollector.registerStreamRequest(cp, sourceIds);
StatsCollectors<DbusEventsStatisticsCollector> statsCollectors = _relay.getOutBoundStatsCollectors();
try {
DbusEventBufferBatchReadable bufRead = v2Mode ? _eventBuffer.getDbusEventBufferBatchReadable(sourceIds, cpMult, statsCollectors) : _eventBuffer.getDbusEventBufferBatchReadable(cpMult, ppartKeys, statsCollectors);
int eventsRead = 0;
int minPendingEventSize = 0;
StreamEventsResult result = null;
bufRead.setClientMaxEventVersion(clientEventVersion);
if (v2Mode) {
result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
eventsRead = result.getNumEventsStreamed();
minPendingEventSize = result.getSizeOfPendingEvent();
if (isDebug) {
LOG.debug("Process: streamed " + eventsRead + " from sources " + Arrays.toString(sourceIds.toArray()));
//can be used for debugging to stream from a cp
LOG.debug("CP=" + cpMult);
}
//if (null != statsCollectors) statsCollectors.mergeStatsCollectors();
} else {
result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
eventsRead = result.getNumEventsStreamed();
minPendingEventSize = result.getSizeOfPendingEvent();
if (isDebug)
LOG.debug("Process: streamed " + eventsRead + " with subscriptions " + subs);
cpMult = bufRead.getCheckpointMult();
if (cpMult != null) {
request.setCursorPartition(cpMult.getCursorPartition());
}
}
if (eventsRead == 0 && minPendingEventSize > 0) {
// Append a header to indicate to the client that we do have at least one event to
// send, but it is too large to fit into client's offered buffer.
request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, minPendingEventSize);
LOG.debug("Returning 0 events but have pending event of size " + minPendingEventSize);
}
} catch (ScnNotFoundException snfe) {
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerScnNotFoundStreamResponse();
}
throw new RequestProcessingException(snfe);
} catch (OffsetNotFoundException snfe) {
LOG.error("OffsetNotFound", snfe);
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerScnNotFoundStreamResponse();
}
throw new RequestProcessingException(snfe);
}
if (null != connHttpStatsCollector) {
connHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
globalHttpStatsCollector.merge(connHttpStatsCollector);
connHttpStatsCollector.reset();
} else if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
}
} catch (InvalidRequestParamValueException e) {
HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerInvalidStreamRequest();
}
throw e;
}
return request;
}
use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.
the class DbusRangePartitionedFilterFactory method createServerSideFilter.
@Override
public DbusKeyCompositeFilterConfig createServerSideFilter(DbusClusterInfo cluster, DbusPartitionInfo partition) throws InvalidConfigException {
DbusKeyCompositeFilterConfig.Config compositeConfig = new DbusKeyCompositeFilterConfig.Config();
for (DatabusSubscription s : _subscriptions) {
KeyFilterConfigHolder.Config filterConfig = new KeyFilterConfigHolder.Config();
filterConfig.setType(PartitionType.RANGE.toString());
filterConfig.getRange().setSize(_rangeWidth);
filterConfig.getRange().setPartitions("[" + partition.getPartitionId() + "]");
compositeConfig.setFilter(s.getLogicalSource().getName(), filterConfig);
}
DbusKeyCompositeFilterConfig c = new DbusKeyCompositeFilterConfig(compositeConfig.build());
LOG.info("Generated Mod Partitioned Config for partition (" + partition + ") of cluster (" + cluster + ") is :" + c);
return c;
}
Aggregations