Examples with EventData org.apache.storm.eventhubs.spout.EventData used on opensource projects

Example 1 with EventData

use of org.apache.storm.eventhubs.spout.EventData in project storm by apache.

the class TransactionalTridentEventHubEmitter method emitPartitionBatchNew.

@Override
public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
    ITridentPartitionManager pm = getOrCreatePartitionManager(partition);
    String offset = Constants.DefaultStartingOffset;
    if (meta != null && meta.containsKey("nextOffset")) {
        offset = (String) meta.get("nextOffset");
    }
    //logger.info("emit for partition " + partition.getId() + ", offset=" + offset);
    String nextOffset = offset;
    List<EventData> listEvents = pm.receiveBatch(offset, batchSize);
    for (EventData ed : listEvents) {
        //update nextOffset;
        nextOffset = ed.getMessageId().getOffset();
        List<Object> tuples = spoutConfig.getEventDataScheme().deserialize(ed.getMessage());
        collector.emit(tuples);
    }
    //logger.info("emitted new batches: " + listEvents.size());
    Map newMeta = new HashMap();
    newMeta.put("offset", offset);
    newMeta.put("nextOffset", nextOffset);
    newMeta.put("count", "" + listEvents.size());
    return newMeta;
}

Example 2 with EventData

use of org.apache.storm.eventhubs.spout.EventData in project storm by apache.

the class TransactionalTridentEventHubEmitter method emitPartitionBatch.

@Override
public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
    String offset = (String) meta.get("offset");
    int count = Integer.parseInt((String) meta.get("count"));
    logger.info("re-emit for partition " + partition.getId() + ", offset=" + offset + ", count=" + count);
    ITridentPartitionManager pm = getOrCreatePartitionManager(partition);
    List<EventData> listEvents = pm.receiveBatch(offset, count);
    if (listEvents.size() != count) {
        logger.error("failed to refetch eventhub messages, new count=" + listEvents.size());
        return;
    }
    for (EventData ed : listEvents) {
        List<Object> tuples = spoutConfig.getEventDataScheme().deserialize(ed.getMessage());
        collector.emit(tuples);
    }
}

Example 3 with EventData

use of org.apache.storm.eventhubs.spout.EventData in project storm by apache.

the class EventHubReceiverMock method receive.

@Override
public EventData receive(long timeoutInMilliseconds) {
    if (isPaused) {
        return null;
    }
    currentOffset++;
    List<Section> body = new ArrayList<Section>();
    //the body of the message is "message" + currentOffset, e.g. "message123"
    body.add(new Data(new Binary(("message" + currentOffset).getBytes())));
    Message m = new Message(body);
    MessageId mid = new MessageId(partitionId, "" + currentOffset, currentOffset);
    EventData ed = new EventData(m, mid);
    return ed;
}

Example 4 with EventData

use of org.apache.storm.eventhubs.spout.EventData in project storm by apache.

the class PartitionManagerCallerMock method execute.

/**
   * Execute a sequence of calls to Partition Manager.
   * 
   * @param callSequence: is represented as a string of commands, 
   * e.g. "r,r,r,r,a1,f2,...". The commands are:
   * r[N]: receive() called N times
   * aX: ack(X)
   * fY: fail(Y)
   * 
   * @return the sequence of messages the receive call returns
   */
public String execute(String callSequence) {
    String[] cmds = callSequence.split(",");
    StringBuilder ret = new StringBuilder();
    for (String cmd : cmds) {
        if (cmd.startsWith("r")) {
            int count = 1;
            if (cmd.length() > 1) {
                count = Integer.parseInt(cmd.substring(1));
            }
            for (int i = 0; i < count; ++i) {
                EventData ed = pm.receive();
                if (ed == null) {
                    ret.append("null,");
                } else {
                    ret.append(ed.getMessageId().getOffset());
                    ret.append(",");
                }
            }
        } else if (cmd.startsWith("a")) {
            pm.ack(cmd.substring(1));
        } else if (cmd.startsWith("f")) {
            pm.fail(cmd.substring(1));
        }
    }
    if (ret.length() > 0) {
        ret.setLength(ret.length() - 1);
    }
    return ret.toString();
}