Examples with RegistrationId streams.components.utils.RegistrationId used on opensource projects

Example 1 with RegistrationId

use of streams.components.utils.RegistrationId in project compss by bsc-wdc.

the class COMPSsStream method announce.

/*
     * ****************************************************************************************************************
     * PRODUCER METHODS
     * ****************************************************************************************************************
     */
public RegistrationId announce() throws AnnounceException {
    // Create subscription id
    RegistrationId registrationId = new RegistrationId();
    // Create internal producer
    Properties properties = new Properties();
    try (FileInputStream fis = new FileInputStream(new File(PROPERTIES_FILE_PATH_PRODUCER))) {
        properties.load(fis);
    } catch (IOException ioe) {
        throw new AnnounceException("ERROR: Cannot open properties file for announcement", ioe);
    }
    KafkaProducer<String, String> producer = new KafkaProducer<>(properties);
    // Update producerId - producer relation
    this.producerIds2producers.put(registrationId.getProducerId(), producer);
    // Update subscriber - producerId relation
    List<String> producers = this.subscriberIds2producerIds.get(registrationId.getSubscriberId());
    if (producers == null) {
        producers = new LinkedList<>();
    }
    producers.add(registrationId.getProducerId());
    this.subscriberIds2producerIds.put(registrationId.getSubscriberId(), producers);
    this.producerIds2subscriberIds.put(registrationId.getProducerId(), registrationId.getSubscriberId());
    // Return the registration id
    return registrationId;
}

Example 2 with RegistrationId

use of streams.components.utils.RegistrationId in project compss by bsc-wdc.

the class Producer method sendMessages.

/**
 * Sends numMessages messages trough the given stream and closes it
 *
 * @param stream
 * @param numMessages
 * @return
 */
public static Integer sendMessages(COMPSsStream stream, int numMessages) {
    // Announce on stream
    if (DEBUG) {
        System.out.println("[LOG] Announcing stream write");
    }
    RegistrationId id = null;
    try {
        id = stream.announce();
    } catch (AnnounceException ae) {
        System.err.println("ERROR: Cannot announce publication");
        ae.printStackTrace();
        return -1;
    }
    // Send messages
    if (DEBUG) {
        System.out.println("[LOG] Sending " + numMessages + " messages");
    }
    for (int i = 0; i < numMessages; ++i) {
        // Send lots of regular messages with type message
        // Message of the form {"type":"message", t=12.234, k=1}
        String msg = String.format(Locale.ROOT, "{\"" + Messages.FIELD_TYPE + "\":\"" + Messages.TYPE_MESSAGE + "\", \"" + Messages.FIELD_T + "\":%.3f, \"" + Messages.FIELD_K + "\":%d}", System.nanoTime() * 1e-9, i);
        try {
            stream.publish(id, Topics.TOPIC_REGULAR_MESSAGES, msg);
        } catch (InvalidCredentialsException ice) {
            System.err.println("ERROR: Cannot send regular message");
            ice.printStackTrace();
            return -1;
        }
        // Every so often send a stats message
        if (i % FREQ_STATS_MSG == 0) {
            String statMsg = String.format(Locale.ROOT, "{\"" + Messages.FIELD_TYPE + "\":\"" + Messages.TYPE_STATS + "\", \"" + Messages.FIELD_T + "\":%.3f, \"" + Messages.FIELD_K + "\":%d}", System.nanoTime() * 1e-9, i);
            try {
                stream.publish(id, Topics.TOPIC_REGULAR_MESSAGES, statMsg);
            } catch (InvalidCredentialsException ice) {
                System.err.println("ERROR: Cannot send stat message");
                ice.printStackTrace();
                return -1;
            }
        }
        // Every so often send a sleep for a while
        if (i % FREQ_WAIT_TIME == 0) {
            try {
                Thread.sleep(WAIT_TIME);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
    }
    // Send end message
    if (DEBUG) {
        System.out.println("[LOG] Sending end messge");
    }
    String endMsg = String.format(Locale.ROOT, "{\"" + Messages.FIELD_TYPE + "\":\"" + Messages.TYPE_END + "\", \"" + Messages.FIELD_T + "\":%.3f, \"" + Messages.FIELD_K + "\":%d}", System.nanoTime() * 1e-9, 0);
    try {
        stream.publish(id, Topics.TOPIC_SYSTEM_MESSAGES, endMsg);
    } catch (InvalidCredentialsException ice) {
        System.err.println("ERROR: Cannot send end message");
        ice.printStackTrace();
        return -1;
    }
    // Close stream
    if (DEBUG) {
        System.out.println("[LOG] Closing stream");
    }
    try {
        stream.close(id);
    } catch (InvalidCredentialsException ice) {
        System.err.println("ERROR: Cannot close stream");
        ice.printStackTrace();
        return -1;
    }
    // All ok
    if (DEBUG) {
        System.out.println("[LOG] DONE");
    }
    return 0;
}

Example 3 with RegistrationId

use of streams.components.utils.RegistrationId in project compss by bsc-wdc.

the class COMPSsStream method subscribe.

/*
     * ****************************************************************************************************************
     * CONSUMER METHODS
     * ****************************************************************************************************************
     */
public RegistrationId subscribe(List<String> topics) throws SubscribeException {
    // Create subscription id
    RegistrationId registrationId = new RegistrationId();
    // Create internal consumer
    Properties properties = new Properties();
    try (FileInputStream fis = new FileInputStream(new File(PROPERTIES_FILE_PATH_CONSUMER))) {
        properties.load(fis);
    } catch (IOException ioe) {
        throw new SubscribeException("ERROR: Cannot open properties file for subscription", ioe);
    }
    if (properties.getProperty("group.id") == null) {
        properties.setProperty("group.id", "group-" + new Random().nextInt(100_000));
    }
    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(properties);
    consumer.subscribe(topics);
    // Update consumerId - consumer relation
    this.consumerIds2consumers.put(registrationId.getConsumerId(), consumer);
    // Update consumerId - topics relation
    this.consumerIds2topics.put(registrationId.getConsumerId(), topics);
    // Update subscriber - consumer relation
    List<String> consumers = this.subscriberIds2consumerIds.get(registrationId.getSubscriberId());
    if (consumers == null) {
        consumers = new LinkedList<>();
    }
    consumers.add(registrationId.getConsumerId());
    this.subscriberIds2consumerIds.put(registrationId.getSubscriberId(), consumers);
    this.consumerIds2subscriberIds.put(registrationId.getConsumerId(), registrationId.getSubscriberId());
    // Return the registration id
    return registrationId;
}

Example 4 with RegistrationId

use of streams.components.utils.RegistrationId in project compss by bsc-wdc.

the class Consumer method receiveMessages.

/**
 * Process the stream messages until and EOS is received
 *
 * @param stream
 * @return
 */
public static Result receiveMessages(COMPSsStream stream) {
    // Subscribe
    if (DEBUG) {
        System.out.println("[LOG] Subscribing to stream");
    }
    RegistrationId id;
    try {
        id = stream.subscribe(Topics.ALL_TOPICS);
    } catch (SubscribeException se) {
        System.err.println("ERROR: Cannot subscribe to stream");
        se.printStackTrace();
        return Result.generate(-1);
    }
    // Process stream
    if (DEBUG) {
        System.out.println("[LOG] Processing stream records");
    }
    try {
        boolean end = false;
        while (!end) {
            // Read all the records within the timeout
            // We don't care about timeout because we will receive an end message
            ConsumerRecords<String, String> records = stream.poll(id, MESSAGE_TIMEOUT);
            // Process the received records
            end = processRecords(records);
        }
    } catch (InvalidCredentialsException ice) {
        System.err.println("ERROR: Cannot receive messages from stream");
        ice.printStackTrace();
        return Result.generate(-2);
    } catch (ConsumerException ce) {
        System.err.println("ERROR: Cannot process received records");
        ce.printStackTrace();
        return Result.generate(-3);
    } finally {
        // Close stream
        if (DEBUG) {
            System.out.println("[LOG] Closing stream");
        }
        try {
            stream.unsubscribe(id);
        } catch (InvalidCredentialsException ice) {
            System.err.println("ERROR: Cannot unsubscribe from stream");
            ice.printStackTrace();
            return Result.generate(-4);
        }
    }
    // Set all ok on result
    if (DEBUG) {
        System.out.println("[LOG] DONE");
    }
    // Return result
    return Result.generate(0);
}