Examples with PhysicalPartition com.linkedin.databus.core.data_model.PhysicalPartition used on opensource projects

Example 21 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class TestCheckpointMult method makeCpMult.

private CheckpointMult makeCpMult() {
    // init test data
    ArrayList<Integer> al = new ArrayList<Integer>();
    al.add(WIN_OFFSET1);
    al.add(WIN_SCN1);
    pSrcIds.put(pSrcId1, al);
    al = new ArrayList<Integer>();
    al.add(WIN_OFFSET2);
    al.add(WIN_SCN2);
    pSrcIds.put(pSrcId2, al);
    CheckpointMult cpMult = new CheckpointMult();
    for (int id : pSrcIds.keySet()) {
        Checkpoint cp = new Checkpoint();
        cp.setWindowOffset(pSrcIds.get(id).get(0).longValue());
        cp.setWindowScn((long) pSrcIds.get(id).get(1));
        PhysicalPartition pPart = new PhysicalPartition(id, "name");
        cpMult.addCheckpoint(pPart, cp);
    }
    return cpMult;
}

Example 22 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class ClientStateRequestProcessor method getV3ClusterPartitions.

/**
   * Get the list of partitions hosted by this client for the V3 cluster.
   * 
   * @param cluster
   *          V3 CLuster for which we need to find out the partitions.
   * @return
   * @throws RequestProcessingException
   *           when unable to find the cluster.
   */
private Collection<PartitionInfo> getV3ClusterPartitions(String cluster) throws RequestProcessingException {
    DatabusV3MultiPartitionRegistration reg = getV3ClusterRegistration(cluster);
    List<PartitionInfo> partitions = new ArrayList<PartitionInfo>();
    Map<PhysicalPartition, DatabusV3Registration> regMap = reg.getPartionRegs();
    for (Entry<PhysicalPartition, DatabusV3Registration> e : regMap.entrySet()) {
        PartitionInfo p = new PartitionInfo(e.getKey().getId(), e.getValue().getRegistrationId());
        partitions.add(p);
    }
    return partitions;
}

Example 23 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class ClientStateRequestProcessor method getAllTopLevelV3Registrations.

/**
   * Returns all the top-level V3 registrations. Top-level registrations are those that
   * were created as a result of one of "registerXXX()" calls on databus-client. In the
   * case of multi-partition registrations (like MPRegistration, V3 CLB), only the parent
   * registration is considered the top-level registration. Per-partition (child)
   * registrations which were created as part of partition migration are NOT top-level
   * registrations.
   * 
   * @return collection of top-level registrations (V3)
   */
private Collection<RegInfo> getAllTopLevelV3Registrations() {
    /**
     * Important Note: There is an important implementation difference on which
     * registrations are stored in the global registration data-structure maintained by
     * the client (DatabusHttp[V3]ClientImpls) between V2 and V3.
     * 
     * 1. In the case of V2, only top-level registrations are stored in the global
     * data-structure (DatabusHttpClientImpl.regList 2. In the case of V3, all
     * registrations are stored in the global data-structure.
     * 
     * In the case of V3, this is needed so that all registrations can act on the relay
     * external view change. This can be refactored in the future by moving the
     * relay-external view change to registration impl ( reduce the complexity in
     * ClientImpl ). The V2 implementation did not have this logic and was following a
     * more intuitive structure of preserving the hierarchy.
     */
    Map<RegistrationId, RegInfo> regListMap = new HashMap<RegistrationId, RegInfo>();
    /**
     * The _client.getRegistrationIdMap() has all registrations in one place. Top-Level
     * Registrations = Only those registrations whose getParent() == null.
     */
    Map<RegistrationId, DatabusV3Registration> regMap = _client.getRegistrationIdMap();
    for (Entry<RegistrationId, DatabusV3Registration> e : regMap.entrySet()) {
        RegInfo regInfo = null;
        DatabusV3Registration r = e.getValue();
        // If not top-level, skip
        if (null != r.getParentRegistration()) {
            continue;
        }
        Map<DbusPartitionInfo, RegInfo> childR = null;
        if (r instanceof DatabusV3MultiPartitionRegistration) {
            // ass the children regs to parent.
            Map<PhysicalPartition, DatabusV3Registration> childRegs = ((DatabusV3MultiPartitionRegistration) r).getPartionRegs();
            childR = new HashMap<DbusPartitionInfo, RegInfo>();
            for (Entry<PhysicalPartition, DatabusV3Registration> e2 : childRegs.entrySet()) {
                childR.put(new DbusPartitionInfoImpl(e2.getKey().getId()), new RegInfo(e.getValue().getState().name(), e.getValue().getRegistrationId(), e.getValue().getStatus(), null, e.getValue().getSubscriptions()));
            }
        }
        regInfo = new RegInfo(r.getState().name(), r.getRegistrationId(), r.getStatus(), null, r.getSubscriptions(), true, childR);
        regListMap.put(e.getKey(), regInfo);
    }
    return regListMap.values();
}

Example 24 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class DatabusSourcesConnection method composeName.

// figure out name for the connection - to be used in mbean
private String composeName(String id) {
    StringBuilder shortSourcesListBuilder = new StringBuilder();
    String separatorChar = "[";
    for (DatabusSubscription sub : getSubscriptions()) {
        shortSourcesListBuilder.append(separatorChar);
        PhysicalPartition p = sub.getPhysicalPartition();
        String sourceName = "AnySource";
        LogicalSource source = sub.getLogicalPartition().getSource();
        if (!source.isAllSourcesWildcard()) {
            sourceName = source.getName();
            int lastDotIdx = sourceName.lastIndexOf('.');
            if (lastDotIdx >= 0)
                sourceName = sourceName.substring(lastDotIdx + 1);
        }
        String partString = "AnyPPart_";
        if (!p.isAnyPartitionWildcard()) {
            partString = p.getId() + "_";
        }
        shortSourcesListBuilder.append(partString + sourceName);
        separatorChar = "_";
    }
    shortSourcesListBuilder.append(']');
    String shortSourcesList = shortSourcesListBuilder.toString();
    return "conn" + shortSourcesList + (id == null ? "" : "_" + id);
}

Example 25 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class LoadDataEventsRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    String fileStr = request.getRequiredStringParam(FILE_PATH_PARAM);
    String startWinStr = request.getParams().getProperty(START_WINDOW_PARAM, "false");
    String physicalParitionParameter = request.getParams().getProperty(PHYSICAL_PARTITION_ID_PARAM);
    LOG.info(PHYSICAL_PARTITION_ID_PARAM + "=" + physicalParitionParameter);
    PhysicalPartition pPartition = PhysicalSourceStaticConfig.getDefaultPhysicalPartition();
    if (physicalParitionParameter != null) {
        // physical partition parameter format is PPName_PPId
        pPartition = PhysicalPartition.parsePhysicalPartitionString(physicalParitionParameter, "_");
    }
    boolean startWin = Boolean.valueOf(startWinStr);
    BufferedReader in = new BufferedReader(new FileReader(fileStr));
    try {
        //PhysicalPartition pPartition = new PhysicalPartition(physicalPartitionId);
        // TODO this should actually use DbusEventBufferAppendable (DDSDBUS-78)
        DbusEventBuffer buf = (DbusEventBuffer) _eventBuffer.getDbusEventBufferAppendable(pPartition);
        if (buf == null)
            throw new RequestProcessingException("cannot find buffer for ph. partion " + pPartition);
        if ((buf.getMinScn() < 0) && (buf.getPrevScn() < 0))
            buf.start(0);
        try {
            DbusEventsStatisticsCollector statsCollector = _relay.getInBoundStatsCollectors().getStatsCollector(pPartition.toSimpleString());
            int eventsAppended = 0;
            if (!((eventsAppended = DbusEventSerializable.appendToEventBuffer(in, buf, statsCollector, startWin)) > 0)) {
                throw new RequestProcessingException("event loading failed");
            }
            StringBuilder res = new StringBuilder(20);
            res.append("{\"eventsAppended\":");
            res.append(eventsAppended);
            res.append("}");
            request.getResponseContent().write(ByteBuffer.wrap(res.toString().getBytes(Charset.defaultCharset())));
        } catch (InvalidEventException iee) {
            throw new RequestProcessingException(iee);
        } catch (RuntimeException re) {
            LOG.error("runttime excception: " + re.getMessage(), re);
            throw re;
        }
    } finally {
        in.close();
    }
    return request;
}