Examples with PubsubMessageToHopRowFn org.apache.hop.beam.core.fn.PubsubMessageToHopRowFn used on opensource projects

Example 1 with PubsubMessageToHopRowFn

use of org.apache.hop.beam.core.fn.PubsubMessageToHopRowFn in project hop by apache.

the class BeamSubscribeTransform method expand.

@Override
public PCollection<HopRow> expand(PBegin input) {
    try {
        if (rowMeta == null) {
            // Only initialize once on this node/vm
            // 
            BeamHop.init(transformPluginClasses, xpPluginClasses);
            rowMeta = JsonRowMeta.fromJson(rowMetaJson);
            inputCounter = Metrics.counter(Pipeline.METRIC_NAME_INPUT, transformName);
            writtenCounter = Metrics.counter(Pipeline.METRIC_NAME_WRITTEN, transformName);
            Metrics.counter(Pipeline.METRIC_NAME_INIT, transformName).inc();
        }
        // This stuff only outputs a single field.
        // It's either a Serializable or a String
        // 
        PCollection<HopRow> output;
        if (BeamDefaults.PUBSUB_MESSAGE_TYPE_STRING.equalsIgnoreCase(messageType)) {
            PubsubIO.Read<String> stringRead = PubsubIO.readStrings();
            if (StringUtils.isNotEmpty(subscription)) {
                stringRead = stringRead.fromSubscription(subscription);
            } else {
                stringRead = stringRead.fromTopic(topic);
            }
            PCollection<String> stringPCollection = stringRead.expand(input);
            output = stringPCollection.apply(transformName, ParDo.of(new StringToHopRowFn(transformName, rowMetaJson, transformPluginClasses, xpPluginClasses)));
        } else if (BeamDefaults.PUBSUB_MESSAGE_TYPE_MESSAGE.equalsIgnoreCase(messageType)) {
            PubsubIO.Read<PubsubMessage> messageRead = PubsubIO.readMessages();
            if (StringUtils.isNotEmpty(subscription)) {
                messageRead = messageRead.fromSubscription(subscription);
            } else {
                messageRead = messageRead.fromTopic(topic);
            }
            PCollection<PubsubMessage> messagesPCollection = messageRead.expand(input);
            output = messagesPCollection.apply(transformName, ParDo.of(new PubsubMessageToHopRowFn(transformName, rowMetaJson, transformPluginClasses, xpPluginClasses)));
        } else {
            throw new RuntimeException("Unsupported message type: " + messageType);
        }
        return output;
    } catch (Exception e) {
        Metrics.counter(Pipeline.METRIC_NAME_ERROR, transformName).inc();
        LOG.error("Error in beam subscribe transform", e);
        throw new RuntimeException("Error in beam subscribe transform", e);
    }
}