Examples with PulsarDriverParamException io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException used on opensource projects

Example 1 with PulsarDriverParamException

use of io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException in project nosqlbench by nosqlbench.

the class PulsarProducerOp method run.

@Override
public void run(Runnable timeTracker) {
    if (StringUtils.isBlank(msgPayload)) {
        throw new PulsarDriverParamException("Message payload (\"msg-value\") can't be empty!");
    }
    TypedMessageBuilder typedMessageBuilder;
    final Transaction transaction;
    if (useTransaction) {
        // if you are in a transaction you cannot set the schema per-message
        transaction = transactionSupplier.get();
        typedMessageBuilder = producer.newMessage(transaction);
    } else {
        transaction = null;
        typedMessageBuilder = producer.newMessage(pulsarSchema);
    }
    // set message key
    if (!StringUtils.isBlank(msgKey)) {
        typedMessageBuilder = typedMessageBuilder.key(msgKey);
    }
    // set message properties
    if (!msgProperties.isEmpty()) {
        typedMessageBuilder = typedMessageBuilder.properties(msgProperties);
    }
    // set message payload
    int messageSize;
    SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
    if (pulsarSchema instanceof KeyValueSchema) {
        // {KEY IN JSON}||{VALUE IN JSON}
        int separator = msgPayload.indexOf("}||{");
        if (separator < 0) {
            throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)");
        }
        String keyInput = msgPayload.substring(0, separator + 1);
        String valueInput = msgPayload.substring(separator + 3);
        KeyValueSchema keyValueSchema = (KeyValueSchema) pulsarSchema;
        org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
        GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro((GenericAvroSchema) keyValueSchema.getValueSchema(), avroSchema, valueInput);
        org.apache.avro.Schema avroSchemaForKey = getKeyAvroSchemaFromConfiguration();
        GenericRecord key = AvroUtil.GetGenericRecord_PulsarAvro((GenericAvroSchema) keyValueSchema.getKeySchema(), avroSchemaForKey, keyInput);
        typedMessageBuilder = typedMessageBuilder.value(new KeyValue(key, payload));
        // TODO: add a way to calculate the message size for KEY_VALUE messages
        messageSize = msgPayload.length();
    } else if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
        GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro((GenericAvroSchema) pulsarSchema, pulsarSchema.getSchemaInfo().getSchemaDefinition(), msgPayload);
        typedMessageBuilder = typedMessageBuilder.value(payload);
        // TODO: add a way to calculate the message size for AVRO messages
        messageSize = msgPayload.length();
    } else {
        byte[] array = msgPayload.getBytes(StandardCharsets.UTF_8);
        typedMessageBuilder = typedMessageBuilder.value(array);
        messageSize = array.length;
    }
    messageSizeHistogram.update(messageSize);
    bytesCounter.inc(messageSize);
    // TODO: add error handling with failed message production
    if (!asyncPulsarOp) {
        try {
            logger.trace("Sending message");
            typedMessageBuilder.send();
            if (useTransaction) {
                try (Timer.Context ctx = transactionCommitTimer.time()) {
                    transaction.commit().get();
                }
            }
            if (logger.isDebugEnabled()) {
                if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
                    org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
                    org.apache.avro.generic.GenericRecord avroGenericRecord = AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload);
                    logger.debug("({}) Sync message sent: msg-key={}; msg-properties={}; msg-payload={})", producer.getProducerName(), msgKey, msgProperties, avroGenericRecord.toString());
                } else {
                    logger.debug("({}) Sync message sent; msg-key={}; msg-properties={}; msg-payload={}", producer.getProducerName(), msgKey, msgProperties, msgPayload);
                }
            }
        } catch (PulsarClientException | ExecutionException | InterruptedException pce) {
            String errMsg = "Sync message sending failed: " + "key - " + msgKey + "; " + "properties - " + msgProperties + "; " + "payload - " + msgPayload;
            logger.trace(errMsg);
            throw new PulsarDriverUnexpectedException(errMsg);
        }
        timeTracker.run();
    } else {
        try {
            // we rely on blockIfQueueIsFull in order to throttle the request in this case
            CompletableFuture<?> future = typedMessageBuilder.sendAsync();
            if (useTransaction) {
                // add commit step
                future = future.thenCompose(msg -> {
                    Timer.Context ctx = transactionCommitTimer.time();
                    return transaction.commit().whenComplete((m, e) -> ctx.close()).thenApply(v -> msg);
                });
            }
            future.whenComplete((messageId, error) -> {
                if (logger.isDebugEnabled()) {
                    if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
                        org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
                        org.apache.avro.generic.GenericRecord avroGenericRecord = AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload);
                        logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={})", producer.getProducerName(), msgKey, msgProperties, avroGenericRecord.toString());
                    } else {
                        logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={}", producer.getProducerName(), msgKey, msgProperties, msgPayload);
                    }
                }
                timeTracker.run();
            }).exceptionally(ex -> {
                logger.error("Async message sending failed: " + "key - " + msgKey + "; " + "properties - " + msgProperties + "; " + "payload - " + msgPayload);
                pulsarActivity.asyncOperationFailed(ex);
                return null;
            });
        } catch (Exception e) {
            throw new PulsarDriverUnexpectedException(e);
        }
    }
}

Example 2 with PulsarDriverParamException

use of io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException in project nosqlbench by nosqlbench.

the class ReadyPulsarOp method resolve.

private LongFunction<PulsarOp> resolve() {
    String stmtOpType = lookupStaticParameter("optype", true, null);
    if (cmdTpl.containsKey("topic_url")) {
        throw new PulsarDriverParamException("[resolve()] \"topic_url\" parameter is not valid. Perhaps you mean \"topic_uri\"?");
    }
    // Doc-level parameter: topic_uri
    LongFunction<String> topicUriFunc = lookupParameterFunc(PulsarActivityUtil.DOC_LEVEL_PARAMS.TOPIC_URI.label);
    logger.info("topic_uri: {}", topicUriFunc.apply(0));
    // Doc-level parameter: async_api
    boolean useAsyncApi = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ASYNC_API.label, false, "false"));
    LongFunction<Boolean> asyncApiFunc = (l) -> useAsyncApi;
    logger.info("async_api: {}", useAsyncApi);
    // Doc-level parameter: use_transaction
    boolean useTransaction = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.USE_TRANSACTION.label, false, "false"));
    LongFunction<Boolean> useTransactionFunc = (l) -> useTransaction;
    logger.info("use_transaction: {}", useTransaction);
    // Doc-level parameter: admin_delop
    boolean adminDelOp = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ADMIN_DELOP.label, false, "false"));
    LongFunction<Boolean> adminDelOpFunc = (l) -> adminDelOp;
    logger.info("admin_delop: {}", adminDelOp);
    // Doc-level parameter: seq_tracking
    boolean seqTracking = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.SEQ_TRACKING.label, false, "false"));
    LongFunction<Boolean> seqTrackingFunc = (l) -> seqTracking;
    logger.info("seq_tracking: {}", seqTracking);
    // TODO: Collapse this pattern into a simple version and flatten out all call sites
    LongFunction<String> payloadRttFieldFunc = lookupParameterFunc(RTT_TRACKING_FIELD, false, "");
    logger.info("payload_rtt_field_func: {}", payloadRttFieldFunc.apply(0));
    // Admin operation: create/delete tenant
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TENANT.label)) {
        return resolveAdminTenant(clientSpace, asyncApiFunc, adminDelOpFunc);
    } else // Admin operation: create/delete namespace
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_NAMESPACE.label)) {
        return resolveAdminNamespace(clientSpace, asyncApiFunc, adminDelOpFunc);
    } else // Admin operation: create/delete topic
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TOPIC.label)) {
        return resolveAdminTopic(clientSpace, topicUriFunc, asyncApiFunc, adminDelOpFunc);
    } else // Regular/non-admin operation: single message sending (producer)
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_SEND.label)) {
        return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc);
    } else // Regular/non-admin operation: single message consuming from a single topic (consumer)
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_CONSUME.label)) {
        return resolveMsgConsume(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc, parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.NONE), payloadRttFieldFunc);
    } else // Regular/non-admin operation: single message consuming from multiple-topics (consumer)
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_MULTI_CONSUME.label)) {
        return resolveMultiTopicMsgConsume(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc, payloadRttFieldFunc);
    } else // Regular/non-admin operation: single message consuming a single topic (reader)
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_READ.label)) {
        return resolveMsgRead(clientSpace, topicUriFunc, asyncApiFunc);
    } else // Regular/non-admin operation: batch message processing - batch start
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_START.label)) {
        return resolveMsgBatchSendStart(clientSpace, topicUriFunc, asyncApiFunc);
    } else // Regular/non-admin operation: batch message processing - message sending (producer)
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND.label)) {
        return resolveMsgBatchSend(clientSpace, asyncApiFunc);
    } else // Regular/non-admin operation: batch message processing - batch send
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_END.label)) {
        return resolveMsgBatchSendEnd(clientSpace, asyncApiFunc);
    } else // Regular/non-admin operation: end-to-end message processing - sending message
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_SEND.label)) {
        return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc);
    } else // Regular/non-admin operation: end-to-end message processing - consuming message
    if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_CONSUME.label)) {
        return resolveMsgConsume(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc, parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.MESSAGE_PUBLISH_TIME), payloadRttFieldFunc);
    } else // Invalid operation type
    {
        throw new PulsarDriverUnsupportedOpException();
    }
}