Examples with EventHubSpoutConfig org.apache.storm.eventhubs.spout.EventHubSpoutConfig used on opensource projects

Example 1 with EventHubSpoutConfig

use of org.apache.storm.eventhubs.spout.EventHubSpoutConfig in project storm by apache.

the class TestTransactionalTridentEmitter method setUp.

@Before
public void setUp() throws Exception {
    EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "password", "namespace", "entityname", 16, "zookeeper");
    conf.setTopologyName("TestTopo");
    IEventHubReceiverFactory recvFactory = new IEventHubReceiverFactory() {

        @Override
        public IEventHubReceiver create(EventHubSpoutConfig config, String partitionId) {
            return new EventHubReceiverMock(partitionId);
        }
    };
    partition = new Partition(conf, "0");
    emitter = new TransactionalTridentEventHubEmitter(conf, batchSize, null, recvFactory);
    collectorMock = new TridentCollectorMock();
}

Example 2 with EventHubSpoutConfig

use of org.apache.storm.eventhubs.spout.EventHubSpoutConfig in project storm by apache.

the class AtMostOnceEventCount method createEventHubSpout.

@Override
protected EventHubSpout createEventHubSpout() {
    IPartitionManagerFactory pmFactory = new IPartitionManagerFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IPartitionManager create(EventHubSpoutConfig spoutConfig, String partitionId, IStateStore stateStore, IEventHubReceiver receiver) {
            return new SimplePartitionManager(spoutConfig, partitionId, stateStore, receiver);
        }
    };
    EventHubSpout eventHubSpout = new EventHubSpout(spoutConfig, null, pmFactory, null);
    return eventHubSpout;
}

Example 3 with EventHubSpoutConfig

use of org.apache.storm.eventhubs.spout.EventHubSpoutConfig in project storm by apache.

the class EventCount method readEHConfig.

protected void readEHConfig(String[] args) throws Exception {
    Properties properties = new Properties();
    if (args.length > 1) {
        properties.load(new FileReader(args[1]));
    } else {
        properties.load(EventCount.class.getClassLoader().getResourceAsStream("Config.properties"));
    }
    String username = properties.getProperty("eventhubspout.username");
    String password = properties.getProperty("eventhubspout.password");
    String namespaceName = properties.getProperty("eventhubspout.namespace");
    String entityPath = properties.getProperty("eventhubspout.entitypath");
    String targetFqnAddress = properties.getProperty("eventhubspout.targetfqnaddress");
    String zkEndpointAddress = properties.getProperty("zookeeper.connectionstring");
    int partitionCount = Integer.parseInt(properties.getProperty("eventhubspout.partitions.count"));
    int checkpointIntervalInSeconds = Integer.parseInt(properties.getProperty("eventhubspout.checkpoint.interval"));
    int receiverCredits = Integer.parseInt(properties.getProperty("eventhub.receiver.credits"));
    String maxPendingMsgsPerPartitionStr = properties.getProperty("eventhubspout.max.pending.messages.per.partition");
    if (maxPendingMsgsPerPartitionStr == null) {
        maxPendingMsgsPerPartitionStr = "1024";
    }
    int maxPendingMsgsPerPartition = Integer.parseInt(maxPendingMsgsPerPartitionStr);
    String enqueueTimeDiffStr = properties.getProperty("eventhub.receiver.filter.timediff");
    if (enqueueTimeDiffStr == null) {
        enqueueTimeDiffStr = "0";
    }
    int enqueueTimeDiff = Integer.parseInt(enqueueTimeDiffStr);
    long enqueueTimeFilter = 0;
    if (enqueueTimeDiff != 0) {
        enqueueTimeFilter = System.currentTimeMillis() - enqueueTimeDiff * 1000;
    }
    String consumerGroupName = properties.getProperty("eventhubspout.consumer.group.name");
    System.out.println("Eventhub spout config: ");
    System.out.println("  partition count: " + partitionCount);
    System.out.println("  checkpoint interval: " + checkpointIntervalInSeconds);
    System.out.println("  receiver credits: " + receiverCredits);
    spoutConfig = new EventHubSpoutConfig(username, password, namespaceName, entityPath, partitionCount, zkEndpointAddress, checkpointIntervalInSeconds, receiverCredits, maxPendingMsgsPerPartition, enqueueTimeFilter);
    if (targetFqnAddress != null) {
        spoutConfig.setTargetAddress(targetFqnAddress);
    }
    spoutConfig.setConsumerGroupName(consumerGroupName);
    //set the number of workers to be the same as partition number.
    //the idea is to have a spout and a partial count bolt co-exist in one
    //worker to avoid shuffling messages across workers in storm cluster.
    numWorkers = spoutConfig.getPartitionCount();
    if (args.length > 0) {
        //set topology name so that sample Trident topology can use it as stream name.
        spoutConfig.setTopologyName(args[0]);
    }
}