Examples with IdNamePair com.linkedin.databus.core.util.IdNamePair used on opensource projects

Example 31 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestGenericDispatcher method testMultiWindowsHappyPath.

@Test(groups = { "small", "functional" })
public void testMultiWindowsHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testMultiWindowsHappyPath");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 3;
    int source2EventsNum = 5;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    int windowsNum = 3;
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    int curEventNum = 1;
    for (int w = 0; w < windowsNum; ++w) {
        eventsBuf.startEvents();
        initBufferWithEvents(eventsBuf, curEventNum, source1EventsNum, (short) 1, keyCounts, srcidCounts);
        curEventNum += source1EventsNum;
        initBufferWithEvents(eventsBuf, curEventNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
        curEventNum += source2EventsNum;
        eventsBuf.endEvents(100L * (w + 1), null);
    }
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i < curEventNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", windowsNum * source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", windowsNum * source2EventsNum, srcidCounts.get((short) 2).intValue());
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 32 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestGenericDispatcher method testRollbackFailure.

@Test(groups = { "small", "functional" })
public /**
     * Test a relatively big window which forces a checkpoint which fails. Then trigger
     * a rollback. The dispatcher should shutdown. */
void testRollbackFailure() throws InvalidConfigException {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testRollbackFailure");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 100, (short) 1, null, null);
    eventsBuf.endEvents(100L, null);
    RollbackFailingConsumer mockConsumer = new RollbackFailingConsumer(new StateVerifyingStreamConsumer(null), LOG, 2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(mockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DatabusSourcesConnection.Config connCfgBuilder = new DatabusSourcesConnection.Config();
    connCfgBuilder.setConsumerParallelism(1);
    connCfgBuilder.setEventBuffer(_generic100KBufferConfig);
    connCfgBuilder.setFreeBufferThreshold(10000);
    connCfgBuilder.setConsumerTimeBudgetMs(1000);
    connCfgBuilder.setCheckpointThresholdPct(5.0);
    connCfgBuilder.getDispatcherRetries().setMaxRetryNum(10);
    DatabusSourcesConnection.StaticConfig connStaticCfg = connCfgBuilder.build();
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", connStaticCfg, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                LOG.info(dispatcher.getStatus().getStatus().toString());
                LOG.info(String.valueOf(dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN));
                return dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN;
            }
        }, "dispatcher shtudown", 50000, LOG);
    } finally {
        dispatcher.shutdown();
    }
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 33 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestGenericDispatcher method testMetadataSchema.

@Test
public void testMetadataSchema() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testMetadataSchema");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
    //svsConsumer.getLog().setLevel(Level.DEBUG);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher, "testMetadataSchema-dispatcher");
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    //add meta data schema
    byte[] crc32 = { 0x01, 0x02, 0x03, 0x04 };
    List<RegisterResponseMetadataEntry> lMeta = new ArrayList<RegisterResponseMetadataEntry>();
    lMeta.add(new RegisterResponseMetadataEntry((short) 1, META1_SCHEMA_STR, crc32));
    lMeta.add(new RegisterResponseMetadataEntry((short) 2, META2_SCHEMA_STR, crc32));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap, lMeta));
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L, null);
    //check standard execution of callbacks
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("incorrect amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("incorrect amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    //check metadata schemas
    EventCountingConsumer myCons = (EventCountingConsumer) mockConsumer;
    VersionedSchema metadataSchema = myCons.getMetadataSchema();
    Assert.assertTrue(null != metadataSchema);
    log.info("Metadata VersionedSchema = " + metadataSchema);
    Assert.assertEquals(metadataSchema.getVersion(), 2);
    Assert.assertEquals(metadataSchema.getSchemaBaseName(), SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
    verifyNoLocks(log, eventsBuf);
    log.info("end\n");
}

Example 34 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestGenericDispatcher method testDispatcherDiscrepancy.

@Test(groups = { "small", "functional" })
public void testDispatcherDiscrepancy() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testDispatcherDiscrepancy");
    log.setLevel(Level.INFO);
    log.info("start");
    final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
    //Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
    log.info("Creating sourcesMap");
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    // Not setting the ID for the schema
    RegisterResponseEntry re = new RegisterResponseEntry();
    re.setSchema(SOURCE2_SCHEMA_STR);
    re.setVersion((short) 1);
    l2.add(re);
    // Should add the third schema correctly
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    log.info("Switch to start dispatch events");
    DispatcherState ds = DispatcherState.create().addSources(sourcesMap.values());
    ds.getSchemaSet().clear();
    long initSize = 0, finalSize = 0;
    try {
        // the schemaSet inside DispatcherState is a static
        log.info("===Printing the decoder object's schema set\n" + ds.getEventDecoder().getSchemaSet());
        log.info("===Printing the decoder object's schema set basenames\n" + ds.getEventDecoder().getSchemaSet().getSchemaBaseNames());
        initSize = ds.getEventDecoder().getSchemaSet().size();
        log.info("initSize = " + initSize + " Schema base names = " + ds.getEventDecoder().getSchemaSet().getSchemaBaseNames());
    } catch (Exception e) {
    }
    ds.addSchemas(schemaMap);
    log.info("Schemas have been refreshed");
    finalSize = ds.getEventDecoder().getSchemaSet().getSchemaBaseNames().size();
    log.info("===Printing the decoder object's schema set\n" + ds.getEventDecoder().getSchemaSet());
    log.info("===Printing the decoder object's schema set basenames\n" + ds.getEventDecoder().getSchemaSet().getSchemaBaseNames());
    Assert.assertEquals(finalSize, 2 + initSize);
    Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
    log.info("end\n");
}

Example 35 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class SourceIdNameRegistry method updateFromIdNamePairs.

public void updateFromIdNamePairs(Collection<IdNamePair> newPairs) {
    ArrayList<LogicalSource> srcCollection = new ArrayList<LogicalSource>(newPairs.size());
    for (IdNamePair pair : newPairs) srcCollection.add(new LogicalSource(pair.getId().intValue(), pair.getName()));
    update(srcCollection);
}