Examples with LogicalPartition com.linkedin.databus.core.data_model.LogicalPartition used on opensource projects

Example 1 with LogicalPartition

use of com.linkedin.databus.core.data_model.LogicalPartition in project databus by linkedin.

the class TestDbusEventBufferMult method addEvents.

private void addEvents() {
    byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
    for (int i = 0; i < _events.length; i++) {
        TestDbusEvent e = _events[i];
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(e._phPartition);
        // two events per scn
        if (i % 2 == 0)
            buf.startEvents();
        DbusEventKey key = new DbusEventKey(e._key);
        //short lPartitionId = (short) (key.getLongKey() % Short.MAX_VALUE);
        String value = "" + i;
        short srcId = e._lSource.getId().shortValue();
        LogicalPartition lPart = e._lPartition;
        //System.out.print("appending events for i=" + i + "; lSource=" + srcId);
        short pPartitionId = _eventBufferMult.getPhysicalPartition(srcId, lPart).getId().shortValue();
        //System.out.println(";partitionid=" + pPartitionId);
        assertTrue(buf.appendEvent(key, pPartitionId, lPart.getId(), System.currentTimeMillis(), srcId, schema, value.getBytes(Charset.defaultCharset()), false, null));
        if (i % 2 != 0)
            buf.endEvents(e._scn, null);
    }
}

Example 2 with LogicalPartition

use of com.linkedin.databus.core.data_model.LogicalPartition in project databus by linkedin.

the class TestDbusEventBufferMult method setUpTest.

@BeforeTest
public void setUpTest() throws IOException, InvalidConfigException {
    LOG.info("Setting up Test");
    PhysicalSourceStaticConfig pStatConf1 = convertToPhysicalSourceConfig(_configSource1).build();
    PhysicalSourceStaticConfig pStatConf2 = convertToPhysicalSourceConfig(_configSource2).build();
    PhysicalSourceStaticConfig pStatConf3 = convertToPhysicalSourceConfig(_configSource3).build();
    _pConfigs = new PhysicalSourceStaticConfig[] { pStatConf1, pStatConf2, pStatConf3 };
    // generate testData
    int scn = 100;
    String srcName = "srcName";
    int srcId = 1;
    PhysicalSource pS = pStatConf1.getPhysicalSource();
    PhysicalPartition pP = pStatConf1.getPhysicalPartition();
    _events = new TestDbusEvent[20];
    LogicalPartition lP = new LogicalPartition((short) 0);
    for (int i = 0; i < _events.length; i++) {
        _events[i] = new TestDbusEvent(i, scn, new LogicalSource(srcId, srcName + srcId), pS, pP, lP);
        switch(i) {
            case 4:
                srcId = 2;
                break;
            case 9:
                srcId = 11;
                pS = pStatConf2.getPhysicalSource();
                pP = pStatConf2.getPhysicalPartition();
                break;
            case 14:
                srcId = 12;
                break;
        }
        if ((i & 1) == 1)
            scn++;
    }
    ;
}

Example 3 with LogicalPartition

use of com.linkedin.databus.core.data_model.LogicalPartition in project databus by linkedin.

the class TestDbusEventBufferMult method batchReading.

private void batchReading(Set<Integer> srcIds, final int expectEvents, final int batchFetchSize) throws IOException, ScnNotFoundException, DatabusException, OffsetNotFoundException {
    // for debug generate this line
    String inputSrcIds = Arrays.toString(srcIds.toArray());
    LOG.info("Reading events from " + inputSrcIds);
    // create checkpoints
    CheckpointMult cpMult = new CheckpointMult();
    for (PhysicalSourceStaticConfig pConf : _pConfigs) {
        PhysicalPartition pPart = pConf.getPhysicalPartition();
        Checkpoint cp;
        if (cpMult.getCheckpoint(pPart) == null) {
            // needs a new checkpoint
            cp = new Checkpoint();
            cp.setFlexible();
            cpMult.addCheckpoint(pPart, cp);
        }
    }
    DbusEventBufferBatchReadable read = _eventBufferMult.getDbusEventBufferBatchReadable(srcIds, cpMult, null);
    int totalRead = 0;
    int numEventsRead = Integer.MAX_VALUE;
    int numNonControlEventsRead = 0;
    int maxIterNum = 100, iterCount = 0;
    String prevEvent = "";
    while (numEventsRead > 0) {
        if (iterCount++ > maxIterNum) {
            fail("Checkpoint doesn't work - it is a never-ending loop");
        }
        ByteArrayOutputStream jsonOut = new ByteArrayOutputStream();
        WritableByteChannel jsonOutChannel = Channels.newChannel(jsonOut);
        numEventsRead = read.streamEvents(false, batchFetchSize, jsonOutChannel, Encoding.JSON_PLAIN_VALUE, new SourceDbusFilter(srcIds)).getNumEventsStreamed();
        totalRead += numEventsRead;
        LOG.info("read for " + inputSrcIds + ": " + numEventsRead + " events");
        byte[] jsonBytes = jsonOut.toByteArray();
        if (jsonBytes.length == 0)
            // nothing more to read
            break;
        String jsonString = new String(jsonBytes);
        String[] jsonStrings = jsonString.split("\n");
        assertEquals(jsonStrings.length, numEventsRead);
        ObjectMapper mapper = new ObjectMapper();
        for (int i = 0; i < jsonStrings.length; i++) {
            // verify what was written
            String evtStr = jsonStrings[i];
            if (evtStr.equals(prevEvent)) {
                // offered buffer is small. This check gets around the issue.
                continue;
            }
            prevEvent = evtStr;
            Map<String, Object> jsonMap = mapper.readValue(evtStr, new TypeReference<Map<String, Object>>() {
            });
            //assertEquals(jsonMap.size(), 10);
            Integer srcId = (Integer) jsonMap.get("srcId");
            if (!DbusEventUtils.isControlSrcId(srcId)) {
                // not a control message
                numNonControlEventsRead++;
                Integer physicalPartitionId = (Integer) jsonMap.get("physicalPartitionId");
                Integer logicalPartitionId = (Integer) jsonMap.get("logicalPartitionId");
                PhysicalPartition pPartition = _eventBufferMult.getPhysicalPartition(srcId, new LogicalPartition(logicalPartitionId.shortValue()));
                LOG.info("EVENT: " + jsonMap.toString());
                assertTrue(srcIds.contains(srcId), "src id " + srcId + " doesn't match to " + inputSrcIds);
                assertEquals(physicalPartitionId, pPartition.getId(), "physical partition id didn't match");
            } else {
                LOG.info("Control event: " + jsonMap.toString());
            }
        }
    }
    assertTrue(totalRead > numNonControlEventsRead);
    assertEquals(numNonControlEventsRead, expectEvents);
}

Example 4 with LogicalPartition

use of com.linkedin.databus.core.data_model.LogicalPartition in project databus by linkedin.

the class TestDbusEventBufferMult method testGetBuffer.

@Test
public void testGetBuffer() throws InvalidConfigException {
    // get buffer by physical partition
    createBufMult();
    addEvents();
    Integer pPartitionId = 100;
    // get buffer by partition id
    PhysicalPartition pPartition = new PhysicalPartition(pPartitionId, "multBufferTest1");
    DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(pPartition);
    assertNotNull(buf, "cannot get by pPartition" + pPartition);
    // get buffer by physical partition
    buf = _eventBufferMult.getDbusEventBufferAppendable(pPartition);
    assertNotNull(buf, "cannot get by pPartition" + pPartition);
    // get buffer by logical source id
    int lSrcId = 1;
    buf = _eventBufferMult.getDbusEventBufferAppendable(lSrcId);
    assertNotNull(buf, "cannot get by lSrcId" + lSrcId);
    // logical source 2 , default partition  - SAME BUFFER
    lSrcId = 2;
    // correct source name, default partition 0
    LogicalSource lSource = new LogicalSource(lSrcId, "srcName2");
    DbusEventBufferAppendable buf1 = _eventBufferMult.getDbusEventBufferAppendable(lSource);
    assertNotNull(buf1, "cannot get buffer by lSource " + lSource);
    assertTrue(buf1 == buf, "buf and buf1 should be the same buffer");
    // logical source, different logical partition 1 - SAME BUFFER
    LogicalPartition lPartition = new LogicalPartition((short) 1);
    buf1 = _eventBufferMult.getDbusEventBufferAppendable(lSource, lPartition);
    // should be the same buffer
    assertNotNull(buf1, "cannot get buffer by lSource " + lSource + ";lPartition =" + lPartition);
    assertTrue(buf1 == buf, "buf and buf1 should be the same buffer");
    // logical source, different logical partition 2 - DIFFERENT BUFFER
    lPartition = new LogicalPartition((short) 2);
    buf1 = _eventBufferMult.getDbusEventBufferAppendable(lSource, lPartition);
    assertNotNull(buf1, "cannot get buffer by lSource " + lSource + ";lPartition =" + lPartition);
    assertTrue(buf1 != buf, "buf and buf1 should not be the same buffer");
    // logical source, different logical partition 12 - DIFFERENT BUFFER (same as for lsr=2,lp=2)
    DbusEventBufferAppendable buf2 = _eventBufferMult.getDbusEventBufferAppendable(12);
    assertNotNull(buf2, "cannot get buffer by lSourceid " + 12);
    assertTrue(buf2 != buf, "buf and buf2 should not be the same buffer");
    assertTrue(buf1 == buf2, "buf2 and buf1 should be the same buffer");
}