Examples with PhysicalSourceStaticConfig com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig used on opensource projects

Example 26 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class DatabusRelayMain method initProducers.

public void initProducers() throws InvalidConfigException, DatabusException, EventCreationException, UnsupportedKeyException, SQLException, ProcessorRegistrationConflictException {
    LOG.info("initializing producers");
    for (PhysicalSourceStaticConfig pConfig : _pConfigs) {
        addOneProducer(pConfig);
    }
    this.setDbPullerStart(_relayStaticConfig.getStartDbPuller());
    LOG.info("done initializing producers");
}

Example 27 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestDbusEventBufferMult method addRemoveBufferMapping.

@Test
public void addRemoveBufferMapping() throws InvalidConfigException, DatabusException {
    createBufMult();
    PhysicalSourceStaticConfig pConfig = convertToPhysicalSourceConfig(_configSource2).build();
    PhysicalPartition pPart = _eventBufferMult.getPhysicalPartition(11);
    Assert.assertNotNull(pPart);
    DbusEventBuffer buf = _eventBufferMult.getDbusEventBuffer(pConfig.getSources()[0].getLogicalSource());
    Assert.assertNotNull(buf);
    DbusEventBuffer buf1 = _eventBufferMult.getOneBuffer(pPart);
    Assert.assertNotNull(buf1);
    Assert.assertTrue(buf == buf1, "same references to the buffer");
    // now remove the buffer and the mapping
    _eventBufferMult.removeBuffer(pConfig);
    _eventBufferMult.deallocateRemovedBuffers(true);
    // same should fail
    pPart = _eventBufferMult.getPhysicalPartition(11);
    Assert.assertNull(pPart);
    buf = _eventBufferMult.getDbusEventBuffer(pConfig.getSources()[0].getLogicalSource());
    Assert.assertNull(buf);
    buf1 = _eventBufferMult.getOneBuffer(pPart);
    Assert.assertNull(buf1);
}

Example 28 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestDbusEventBufferMult method verifyReadingLogicalPartitionWildcard.

@Test
public void verifyReadingLogicalPartitionWildcard() throws IOException, ScnNotFoundException, InvalidConfigException, DatabusException, OffsetNotFoundException {
    createBufMult();
    PhysicalSourceStaticConfig pStatConf1 = convertToPhysicalSourceConfig(_configSource1).build();
    PhysicalSourceStaticConfig pStatConf2 = convertToPhysicalSourceConfig(_configSource2).build();
    PhysicalPartition pP = pStatConf1.getPhysicalPartition();
    DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(pP);
    buf.startEvents();
    byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
    assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 1, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    buf.endEvents(100, null);
    pP = pStatConf2.getPhysicalPartition();
    buf = _eventBufferMult.getDbusEventBufferAppendable(pP);
    buf.startEvents();
    assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    buf.endEvents(200, null);
    Set<Integer> srcIds = new HashSet<Integer>(1);
    srcIds.add(2);
    // total expected events 3 - for pp100:lp=0,pp100:lp=1;pp101:lp=2
    batchReading(srcIds, 3);
}

Example 29 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestDbusEventBufferMult method addRemoveBuffers.

@Test
public void addRemoveBuffers() throws IOException, ScnNotFoundException, InvalidConfigException, DatabusException, OffsetNotFoundException {
    createBufMult();
    addEvents();
    Set<Integer> srcIds = new HashSet<Integer>(2);
    srcIds.add(1);
    // total expected events from source 1 is 5
    batchReading(srcIds, 5);
    PhysicalSourceStaticConfig pConfig = convertToPhysicalSourceConfig(_configSource3).build();
    LOG.info("one more buffer for " + pConfig);
    DbusEventBuffer buf = _eventBufferMult.addNewBuffer(pConfig, _config);
    buf.start(100);
    // add events to the new buffer
    byte[] schema = "ABCDEFGHIJKLMNOP".getBytes(Charset.defaultCharset());
    for (int i = 100; i < 110; i++) {
        if (i % 2 == 0)
            buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(i), pConfig.getPhysicalPartition().getId().shortValue(), // logical source id
        (short) 0, System.currentTimeMillis(), (short) (i < 105 ? 21 : 22), schema, ("" + i).getBytes(Charset.defaultCharset()), false, null));
        if ((i & 1) == 1)
            buf.endEvents(i);
    }
    // read from one old and one new buffer
    srcIds.clear();
    srcIds.add(12);
    srcIds.add(22);
    // total expected events 10 - 5 from source 12, and 5 from source 22
    batchReading(srcIds, 10);
    // remove a buf
    pConfig = convertToPhysicalSourceConfig(_configSource2).build();
    _eventBufferMult.removeBuffer(pConfig);
    _eventBufferMult.deallocateRemovedBuffers(true);
    srcIds.clear();
    srcIds.add(11);
    boolean failed = false;
    try {
        batchReading(srcIds, 5);
        // didn't fail
        failed = false;
    } catch (java.lang.AssertionError ae) {
        // expected
        failed = true;
    }
    Assert.assertTrue(failed, "should've failed for src 11, since the buffer was removed" + pConfig);
}

Example 30 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestDbusEventBufferMult method batchReading.

private void batchReading(Set<Integer> srcIds, final int expectEvents, final int batchFetchSize) throws IOException, ScnNotFoundException, DatabusException, OffsetNotFoundException {
    // for debug generate this line
    String inputSrcIds = Arrays.toString(srcIds.toArray());
    LOG.info("Reading events from " + inputSrcIds);
    // create checkpoints
    CheckpointMult cpMult = new CheckpointMult();
    for (PhysicalSourceStaticConfig pConf : _pConfigs) {
        PhysicalPartition pPart = pConf.getPhysicalPartition();
        Checkpoint cp;
        if (cpMult.getCheckpoint(pPart) == null) {
            // needs a new checkpoint
            cp = new Checkpoint();
            cp.setFlexible();
            cpMult.addCheckpoint(pPart, cp);
        }
    }
    DbusEventBufferBatchReadable read = _eventBufferMult.getDbusEventBufferBatchReadable(srcIds, cpMult, null);
    int totalRead = 0;
    int numEventsRead = Integer.MAX_VALUE;
    int numNonControlEventsRead = 0;
    int maxIterNum = 100, iterCount = 0;
    String prevEvent = "";
    while (numEventsRead > 0) {
        if (iterCount++ > maxIterNum) {
            fail("Checkpoint doesn't work - it is a never-ending loop");
        }
        ByteArrayOutputStream jsonOut = new ByteArrayOutputStream();
        WritableByteChannel jsonOutChannel = Channels.newChannel(jsonOut);
        numEventsRead = read.streamEvents(false, batchFetchSize, jsonOutChannel, Encoding.JSON_PLAIN_VALUE, new SourceDbusFilter(srcIds)).getNumEventsStreamed();
        totalRead += numEventsRead;
        LOG.info("read for " + inputSrcIds + ": " + numEventsRead + " events");
        byte[] jsonBytes = jsonOut.toByteArray();
        if (jsonBytes.length == 0)
            // nothing more to read
            break;
        String jsonString = new String(jsonBytes);
        String[] jsonStrings = jsonString.split("\n");
        assertEquals(jsonStrings.length, numEventsRead);
        ObjectMapper mapper = new ObjectMapper();
        for (int i = 0; i < jsonStrings.length; i++) {
            // verify what was written
            String evtStr = jsonStrings[i];
            if (evtStr.equals(prevEvent)) {
                // offered buffer is small. This check gets around the issue.
                continue;
            }
            prevEvent = evtStr;
            Map<String, Object> jsonMap = mapper.readValue(evtStr, new TypeReference<Map<String, Object>>() {
            });
            //assertEquals(jsonMap.size(), 10);
            Integer srcId = (Integer) jsonMap.get("srcId");
            if (!DbusEventUtils.isControlSrcId(srcId)) {
                // not a control message
                numNonControlEventsRead++;
                Integer physicalPartitionId = (Integer) jsonMap.get("physicalPartitionId");
                Integer logicalPartitionId = (Integer) jsonMap.get("logicalPartitionId");
                PhysicalPartition pPartition = _eventBufferMult.getPhysicalPartition(srcId, new LogicalPartition(logicalPartitionId.shortValue()));
                LOG.info("EVENT: " + jsonMap.toString());
                assertTrue(srcIds.contains(srcId), "src id " + srcId + " doesn't match to " + inputSrcIds);
                assertEquals(physicalPartitionId, pPartition.getId(), "physical partition id didn't match");
            } else {
                LOG.info("Control event: " + jsonMap.toString());
            }
        }
    }
    assertTrue(totalRead > numNonControlEventsRead);
    assertEquals(numNonControlEventsRead, expectEvents);
}