Examples with LogicalSource com.linkedin.databus.core.data_model.LogicalSource used on opensource projects

Example 1 with LogicalSource

use of com.linkedin.databus.core.data_model.LogicalSource in project databus by linkedin.

the class TestDbusEventBufferMult method setUpTest.

@BeforeTest
public void setUpTest() throws IOException, InvalidConfigException {
    LOG.info("Setting up Test");
    PhysicalSourceStaticConfig pStatConf1 = convertToPhysicalSourceConfig(_configSource1).build();
    PhysicalSourceStaticConfig pStatConf2 = convertToPhysicalSourceConfig(_configSource2).build();
    PhysicalSourceStaticConfig pStatConf3 = convertToPhysicalSourceConfig(_configSource3).build();
    _pConfigs = new PhysicalSourceStaticConfig[] { pStatConf1, pStatConf2, pStatConf3 };
    // generate testData
    int scn = 100;
    String srcName = "srcName";
    int srcId = 1;
    PhysicalSource pS = pStatConf1.getPhysicalSource();
    PhysicalPartition pP = pStatConf1.getPhysicalPartition();
    _events = new TestDbusEvent[20];
    LogicalPartition lP = new LogicalPartition((short) 0);
    for (int i = 0; i < _events.length; i++) {
        _events[i] = new TestDbusEvent(i, scn, new LogicalSource(srcId, srcName + srcId), pS, pP, lP);
        switch(i) {
            case 4:
                srcId = 2;
                break;
            case 9:
                srcId = 11;
                pS = pStatConf2.getPhysicalSource();
                pP = pStatConf2.getPhysicalPartition();
                break;
            case 14:
                srcId = 12;
                break;
        }
        if ((i & 1) == 1)
            scn++;
    }
    ;
}

Example 2 with LogicalSource

use of com.linkedin.databus.core.data_model.LogicalSource in project databus by linkedin.

the class SourceIdNameRegistry method updateFromIdNamePairs.

public void updateFromIdNamePairs(Collection<IdNamePair> newPairs) {
    ArrayList<LogicalSource> srcCollection = new ArrayList<LogicalSource>(newPairs.size());
    for (IdNamePair pair : newPairs) srcCollection.add(new LogicalSource(pair.getId().intValue(), pair.getName()));
    update(srcCollection);
}

Example 3 with LogicalSource

use of com.linkedin.databus.core.data_model.LogicalSource in project databus by linkedin.

the class SchemaMetaDataManager method updateAndGetNewSrcId.

public short updateAndGetNewSrcId(String dbName, String srcName) throws DatabusException {
    _maxSrcId++;
    dbName = dbName.toLowerCase(Locale.ENGLISH);
    if (null != _idNameRegistry.getSource(srcName))
        throw new DatabusException("Source Name (" + srcName + ") already available in the schema registry !!");
    TreeSet<String> s = null;
    if (_physicalToLogicalSrcMap.containsKey(dbName)) {
        s = _physicalToLogicalSrcMap.get(dbName);
    } else {
        s = new TreeSet<String>();
        _physicalToLogicalSrcMap.put(dbName, s);
    }
    s.add(srcName);
    IdNamePair pair = new IdNamePair(Long.valueOf(_maxSrcId), srcName);
    LogicalSource src = new LogicalSource(pair);
    List<LogicalSource> newSources = new ArrayList<LogicalSource>();
    newSources.add(src);
    _idNameRegistry.add(newSources);
    return _maxSrcId;
}

Example 4 with LogicalSource

use of com.linkedin.databus.core.data_model.LogicalSource in project databus by linkedin.

the class SourceIdNameRegistry method update.

// TODO Rename this method to 'replace'
public void update(Collection<LogicalSource> newPairs) {
    HashMap<String, LogicalSource> newNameIndex = new HashMap<String, LogicalSource>((int) (newPairs.size() * 1.3));
    HashMap<Integer, LogicalSource> newIdIndex = new HashMap<Integer, LogicalSource>((int) (newPairs.size() * 1.3));
    for (LogicalSource pair : newPairs) {
        newNameIndex.put(pair.getName(), pair);
        newIdIndex.put(pair.getId(), pair);
    }
    synchronized (this) {
        _nameIndex = newNameIndex;
        _idIndex = newIdIndex;
        if (LOG.isDebugEnabled()) {
            LOG.debug("sources updated: " + _idIndex.values());
        }
    }
}

Example 5 with LogicalSource

use of com.linkedin.databus.core.data_model.LogicalSource in project databus by linkedin.

the class RegisterRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    try {
        // fail early if optional version param is included but isn't valid
        // 2 and 3 are same for us; 4 is a superset only newer clients understand
        int registerRequestProtocolVersion = 3;
        String registerRequestProtocolVersionStr = request.getParams().getProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM);
        if (registerRequestProtocolVersionStr != null) {
            try {
                registerRequestProtocolVersion = Integer.parseInt(registerRequestProtocolVersionStr);
            } catch (NumberFormatException e) {
                LOG.error("Could not parse /register request protocol version: " + registerRequestProtocolVersionStr);
                throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, registerRequestProtocolVersionStr);
            }
            if (registerRequestProtocolVersion < 2 || registerRequestProtocolVersion > 4) {
                LOG.error("Out-of-range /register request protocol version: " + registerRequestProtocolVersionStr);
                throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, registerRequestProtocolVersionStr);
            }
        }
        Collection<LogicalSource> logicalSources = null;
        HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
        String sources = request.getParams().getProperty(SOURCES_PARAM);
        if (null == sources) {
            // need to return all schemas, so first get all sources
            logicalSources = _relay.getSourcesIdNameRegistry().getAllSources();
        } else {
            String[] sourceIds = sources.split(",");
            logicalSources = new ArrayList<LogicalSource>(sourceIds.length);
            for (String sourceId : sourceIds) {
                int srcId;
                String trimmedSourceId = sourceId.trim();
                try {
                    srcId = Integer.valueOf(trimmedSourceId);
                    LogicalSource lsource = _relay.getSourcesIdNameRegistry().getSource(srcId);
                    if (null != lsource)
                        logicalSources.add(lsource);
                    else {
                        LOG.error("No source name for source id: " + srcId);
                        throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
                    }
                } catch (NumberFormatException nfe) {
                    if (relayStatsCollector != null) {
                        relayStatsCollector.registerInvalidRegisterCall();
                    }
                    throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
                }
            }
        }
        SchemaRegistryService schemaRegistry = _relay.getSchemaRegistryService();
        ArrayList<RegisterResponseEntry> registeredSources = new ArrayList<RegisterResponseEntry>(20);
        for (LogicalSource lsource : logicalSources) {
            getSchemas(schemaRegistry, lsource.getName(), lsource.getId(), sources, registeredSources);
        }
        // Note that, as of April 2013, the Espresso sandbox's schema registry
        // (in JSON format) is 4.5 MB and growing.  But 100 KB is probably OK
        // for regular production cases.
        StringWriter out = new StringWriter(102400);
        ObjectMapper mapper = new ObjectMapper();
        // any circumstances under which we might want to override this?
        int registerResponseProtocolVersion = registerRequestProtocolVersion;
        if (// DDSDBUS-2009
        registerRequestProtocolVersion == 4) {
            LOG.debug("Got version 4 /register request; fetching metadata schema.");
            // Get (replication) metadata schema from registry; format it as list
            // of schemas (multiple only if more than one version exists).  Per
            // https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Espresso+Metadata+Schema,
            // name of replication metadata is simply "metadata".
            ArrayList<RegisterResponseMetadataEntry> registeredMetadata = new ArrayList<RegisterResponseMetadataEntry>(2);
            getMetadataSchemas(schemaRegistry, registeredMetadata);
            // Set up the v4 response as a map:  one entry is the existing list of source
            // schemas, and the others (if present) are the new lists of metadata schema(s)
            // and (TODO) key schemas.
            HashMap<String, List<Object>> responseMap = new HashMap<String, List<Object>>(4);
            responseMap.put(RegisterResponseEntry.SOURCE_SCHEMAS_KEY, (List<Object>) (List<?>) registeredSources);
            if (registeredMetadata.size() > 0) {
                LOG.debug("Sending v4 /register response with metadata schema.");
                responseMap.put(RegisterResponseMetadataEntry.METADATA_SCHEMAS_KEY, (List<Object>) (List<?>) registeredMetadata);
            } else {
                LOG.debug("No metadata schema available; sending v4 /register response without.");
            }
            // TODO:  figure out how to retrieve key schemas and include via RegisterResponseEntry.KEY_SCHEMAS_KEY
            mapper.writeValue(out, responseMap);
        } else // fall back to old style (v2/v3 response)
        {
            mapper.writeValue(out, registeredSources);
        }
        ChunkedWritableByteChannel responseContent = request.getResponseContent();
        byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
        responseContent.addMetadata(DatabusHttpHeaders.DBUS_CLIENT_RELAY_PROTOCOL_VERSION_HDR, registerResponseProtocolVersion);
        responseContent.write(ByteBuffer.wrap(resultBytes));
        if (null != relayStatsCollector) {
            HttpStatisticsCollector connStatsCollector = (HttpStatisticsCollector) request.getParams().get(relayStatsCollector.getName());
            if (null != connStatsCollector) {
                connStatsCollector.registerRegisterCall(registeredSources);
            } else {
                relayStatsCollector.registerRegisterCall(registeredSources);
            }
        }
        return request;
    } catch (InvalidRequestParamValueException e) {
        HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
        if (null != relayStatsCollector)
            relayStatsCollector.registerInvalidRegisterCall();
        throw e;
    }
}