Examples with PhysicalPartition com.linkedin.databus.core.data_model.PhysicalPartition used on opensource projects

Example 31 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class DbusEventBufferMult method addNewBuffer.

//CM API
/** add new buffer
   * also checks if any buffers should be removed
   * @throws InvalidConfigException */
public synchronized DbusEventBuffer addNewBuffer(PhysicalSourceStaticConfig pConfig, DbusEventBuffer.StaticConfig config) throws InvalidConfigException {
    long startTimeTs = System.nanoTime();
    if (config == null)
        throw new InvalidConfigException("config cannot be null for addNewBuffer");
    // see if a buffer for this mapping exists
    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    PhysicalPartitionKey pKey = new PhysicalPartitionKey(pPartition);
    //record pSource association to the buffer
    PhysicalSource pSource = pConfig.getPhysicalSource();
    Set<PhysicalSource> set = _partKey2PhysiscalSources.get(pKey);
    if (set == null) {
        set = new HashSet<PhysicalSource>();
        _partKey2PhysiscalSources.put(pKey, set);
    }
    set.add(pSource);
    DbusEventBuffer buf = _bufsMap.get(pKey);
    if (buf != null) {
        LOG.info("Adding new buffer. Buffer " + buf.hashCode() + " already exists for: " + pConfig);
    } else {
        if (pConfig.isDbusEventBufferSet()) {
            buf = new DbusEventBuffer(pConfig.getDbusEventBuffer(), pPartition, _eventFactory);
            LOG.info("Using- source specific event buffer config, the event buffer size allocated is: " + buf.getAllocatedSize());
        } else {
            buf = new DbusEventBuffer(config, pPartition, _eventFactory);
            LOG.info("Using- global event buffer config, the buffer size allocated is: " + buf.getAllocatedSize());
        }
        addBuffer(pConfig, buf);
    }
    buf.increaseRefCounter();
    // check if some buffers need to be removed
    deallocateRemovedBuffers(false);
    long endTimeTs = System.nanoTime();
    if (PERF_LOG.isDebugEnabled()) {
        PERF_LOG.debug("addNewBuffer took:" + (endTimeTs - startTimeTs) / _nanoSecsInMSec + "ms");
    }
    return buf;
}

Example 32 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class DbusEventBufferMult method constructFilters.

/**
   * Processes all {@link DatabusSubscription} and generates a filter to match events for any of
   * those subscriptions.
   */
public DbusFilter constructFilters(Collection<DatabusSubscription> subs) throws DatabusException {
    HashMap<PhysicalPartition, PhysicalPartitionDbusFilter> filterMap = null;
    for (DatabusSubscription sub : subs) {
        PhysicalPartition ppart = sub.getPhysicalPartition();
        if (sub.getLogicalSource().isWildcard()) {
            if (!ppart.isWildcard()) {
                if (null == filterMap)
                    filterMap = new HashMap<PhysicalPartition, PhysicalPartitionDbusFilter>(10);
                filterMap.put(ppart, new PhysicalPartitionDbusFilter(ppart, null));
            } else {
                LOG.warn("ignoring subscription with both physical partition and logical source wildcards");
            }
        } else {
            PhysicalPartitionDbusFilter ppartFilter = null != filterMap ? filterMap.get(ppart) : null;
            LogicalSourceAndPartitionDbusFilter logFilter = null;
            if (null == ppartFilter) {
                logFilter = new LogicalSourceAndPartitionDbusFilter();
                ppartFilter = new PhysicalPartitionDbusFilter(ppart, logFilter);
                if (null == filterMap)
                    filterMap = new HashMap<PhysicalPartition, PhysicalPartitionDbusFilter>(10);
                filterMap.put(ppart, ppartFilter);
            } else {
                logFilter = (LogicalSourceAndPartitionDbusFilter) ppartFilter.getNestedFilter();
            }
            if (null != logFilter)
                logFilter.addSourceCondition(sub.getLogicalPartition());
            else
                LOG.error("unexpected null filter for logical source");
        }
    }
    if (0 == filterMap.size())
        return AllowAllDbusFilter.THE_INSTANCE;
    else if (1 == filterMap.size()) {
        DbusFilter result = filterMap.entrySet().iterator().next().getValue();
        return result;
    } else {
        ConjunctionDbusFilter result = new ConjunctionDbusFilter();
        for (Map.Entry<PhysicalPartition, PhysicalPartitionDbusFilter> filterEntry : filterMap.entrySet()) {
            result.addFilter(filterEntry.getValue());
        }
        return result;
    }
}

Example 33 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class CheckpointMult method serialize.

/**
   * serialize CheckpointMult into the stream
   * @param outStream
   */
void serialize(OutputStream outStream) throws JsonGenerationException, JsonMappingException, IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // first convert checkpointmult into a map
    Map<String, String> map = new HashMap<String, String>();
    boolean debugEnabled = LOG.isDebugEnabled();
    for (Entry<PhysicalPartition, Checkpoint> e : _pPart2Checkpoint.entrySet()) {
        baos.reset();
        Checkpoint cp = e.getValue();
        cp.serialize(baos);
        String pPartJson = e.getKey().toJsonString();
        String cpStr = StringUtils.bytesToString(baos.toByteArray());
        map.put(pPartJson, cpStr);
        if (debugEnabled)
            LOG.debug("phSourId=" + e.getKey() + ";cp =" + cpStr);
    }
    _mapper.writeValue(outStream, map);
}

Example 34 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class TestDatabusRequest method testToString.

@Test
public void testToString() throws Exception {
    DatabusRequest req = makeRequest();
    // Coverage of toString().
    String jsonStr = req.toString();
    Map<String, String> jsonMap = _objectMapper.readValue(jsonStr.getBytes(Charset.defaultCharset()), new TypeReference<Map<String, String>>() {
    });
    Assert.assertEquals(reqName, jsonMap.get("name"));
    Assert.assertEquals(method.toString(), jsonMap.get("method"));
    Assert.assertEquals(propval1, jsonMap.get(prop1));
    Assert.assertEquals(propval2, jsonMap.get(prop2));
    Assert.assertEquals(4, jsonMap.size());
    // Coverage for null processor.
    Assert.assertEquals(req, req.call());
    // Coverqage for setCursorPartition
    PhysicalPartition pp = new PhysicalPartition(15, "SomePartition");
    req.setCursorPartition(pp);
    Assert.assertEquals(pp, req.getCursorPartition());
    Assert.assertFalse(req.cancel(true));
    Assert.assertFalse(req.cancel(false));
    Assert.assertEquals(req, req.get());
    Assert.assertEquals(req, req.get(89L, TimeUnit.HOURS));
    Assert.assertFalse(req.isCancelled());
    InetSocketAddress remoteAddress = (InetSocketAddress) req.getRemoteAddress();
    Assert.assertEquals(port, remoteAddress.getPort());
    Assert.assertEquals(InetAddress.getByAddress(ipAddr), remoteAddress.getAddress());
}

Example 35 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class DatabusRelayMain method pause.

@Override
public void pause() {
    for (Entry<PhysicalPartition, EventProducer> entry : _producers.entrySet()) {
        EventProducer producer = entry.getValue();
        if (null != producer) {
            if (producer.isRunning()) {
                producer.pause();
                LOG.info("EventProducer :" + producer.getName() + "  pause sent");
            } else if (producer.isPaused()) {
                LOG.info("EventProducer :" + producer.getName() + "  already paused");
            }
        }
    }
}