Examples with SequencePlanner io.nosqlbench.engine.api.activityapi.planning.SequencePlanner used on opensource projects

Example 1 with SequencePlanner

use of io.nosqlbench.engine.api.activityapi.planning.SequencePlanner in project nosqlbench by nosqlbench.

the class StdoutActivity method initOpSequencer.

private OpSequence<StringBindings> initOpSequencer() {
    // List<StringBindingsTemplate> stringBindingsTemplates = new ArrayList<>();
    SequencerType sequencerType = SequencerType.valueOf(getParams().getOptionalString("seq").orElse("bucket"));
    SequencePlanner<StringBindings> sequencer = new SequencePlanner<>(sequencerType);
    String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
    List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
    String format = getParams().getOptionalString("format").orElse(null);
    if ((stmts.size() == 0 && stmtsDocList.getDocBindings().size() > 0) || format != null) {
        if (format != null && format.startsWith("diag")) {
            logger.info("Creating diagnostic log for resolver construction...");
            BindingsTemplate bt = new BindingsTemplate();
            stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
            String diagnostics = bt.getDiagnostics();
            getConsoleOut().println(diagnostics);
            getConsoleOut().flush();
            System.exit(2);
        } else {
            logger.info("Creating stdout statement template from bindings, since none is otherwise defined.");
            Set<String> activeBindingNames = new LinkedHashSet<>();
            String bindings = getActivityDef().getParams().getOptionalString("bindings").orElse("doc");
            activeBindingNames.addAll(stmtsDocList.getDocBindings().keySet());
            Pattern bindingsFilter = Pattern.compile(bindings.equalsIgnoreCase("doc") ? ".*" : bindings);
            Set<String> filteredBindingNames = new LinkedHashSet<>();
            activeBindingNames.stream().filter(n -> {
                if (bindingsFilter.matcher(n).matches()) {
                    logger.trace("bindings filter kept binding '" + n + "'");
                    return true;
                } else {
                    logger.trace("bindings filter removed binding '" + n + "'");
                    return false;
                }
            }).forEach(filteredBindingNames::add);
            activeBindingNames = filteredBindingNames;
            String generatedStmt = genStatementTemplate(activeBindingNames);
            BindingsTemplate bt = new BindingsTemplate();
            stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
            StringBindings sb = new StringBindings(generatedStmt, bt.getMap());
            sequencer.addOp(sb, 1L);
        }
    } else if (stmts.size() > 0) {
        for (OpTemplate stmt : stmts) {
            ParsedTemplate parsed = stmt.getParsed().orElseThrow();
            BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints());
            String statement = parsed.getPositionalStatement(Function.identity());
            Objects.requireNonNull(statement);
            if (!statement.endsWith("\n") && getParams().getOptionalBoolean("newline").orElse(true)) {
                statement = statement + "\n";
            }
            StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt().orElseThrow(), bt);
            StringBindings sb = sbt.resolve();
            sequencer.addOp(sb, stmt.getParamOrDefault("ratio", 1));
        }
    } else {
        logger.error("Unable to create a stdout statement if you have no active statements or bindings configured.");
    }
    OpSequence<StringBindings> opSequence = sequencer.resolve();
    return opSequence;
}

Example 2 with SequencePlanner

use of io.nosqlbench.engine.api.activityapi.planning.SequencePlanner in project nosqlbench by nosqlbench.

the class KafkaProducerActivity method initOpSequencer.

private OpSequence<KafkaStatement> initOpSequencer() {
    SequencerType sequencerType = SequencerType.valueOf(getParams().getOptionalString("seq").orElse("bucket"));
    SequencePlanner<KafkaStatement> sequencer = new SequencePlanner<>(sequencerType);
    String tagFilter = activityDef.getParams().getOptionalString("tags").orElse("");
    StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, activityDef.getParams(), "activities");
    List<OpTemplate> statements = stmtsDocList.getStmts(tagFilter);
    String format = getParams().getOptionalString("format").orElse(null);
    if (statements.size() > 0) {
        for (OpTemplate statement : statements) {
            sequencer.addOp(new KafkaStatement(statement, servers, clientId, schemaRegistryUrl), statement.getParamOrDefault("ratio", 1));
        }
    } else {
        logger.error("Unable to create a Kafka statement if you have no active statements.");
    }
    return sequencer.resolve();
}

Example 3 with SequencePlanner

use of io.nosqlbench.engine.api.activityapi.planning.SequencePlanner in project nosqlbench by nosqlbench.

the class MongoActivity method initOpSequencer.

OpSequence<ReadyMongoStatement> initOpSequencer() {
    SequencerType sequencerType = SequencerType.valueOf(activityDef.getParams().getOptionalString("seq").orElse("bucket"));
    SequencePlanner<ReadyMongoStatement> sequencer = new SequencePlanner<>(sequencerType);
    StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, activityDef.getParams(), "activities");
    String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
    TagFilter tagFilter = new TagFilter(tagfilter);
    stmtsDocList.getStmts().stream().map(tagFilter::matchesTaggedResult).forEach(r -> logger.info(r.getLog()));
    List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
    if (stmts.isEmpty()) {
        logger.error("No statements found for this activity");
    } else {
        for (OpTemplate stmt : stmts) {
            ParsedTemplate parsed = stmt.getParsed().orElseThrow();
            String statement = parsed.getPositionalStatement(Function.identity());
            Objects.requireNonNull(statement);
            sequencer.addOp(new ReadyMongoStatement(stmt), stmt.getParamOrDefault("ratio", 1));
        }
    }
    return sequencer.resolve();
}

Example 4 with SequencePlanner

use of io.nosqlbench.engine.api.activityapi.planning.SequencePlanner in project nosqlbench by nosqlbench.

the class SimpleActivity method createOpSequence.

/**
 * Given a function that can create an op of type <O> from an OpTemplate, generate
 * an indexed sequence of ready to call operations.
 *
 * This method uses the following conventions to derive the sequence:
 *
 * <OL>
 * <LI>If an 'op', 'stmt', or 'statement' parameter is provided, then it's value is
 * taken as the only provided statement.</LI>
 * <LI>If a 'yaml, or 'workload' parameter is provided, then the statements in that file
 * are taken with their ratios </LI>
 * <LI>Any provided tags filter is used to select only the statements which have matching
 * tags. If no tags are provided, then all the found statements are included.</LI>
 * <LI>The ratios and the 'seq' parameter are used to build a sequence of the ready operations,
 * where the sequence length is the sum of the ratios.</LI>
 * </OL>
 *
 * @param opinit A function to map an OpTemplate to the executable operation form required by
 *               the native driver for this activity.
 * @param <O>    A holder for an executable operation for the native driver used by this activity.
 * @return The sequence of operations as determined by filtering and ratios
 */
@Deprecated(forRemoval = true)
protected <O> OpSequence<OpDispenser<? extends O>> createOpSequence(Function<OpTemplate, OpDispenser<? extends O>> opinit) {
    String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
    // StrInterpolator interp = new StrInterpolator(activityDef);
    SequencerType sequencerType = getParams().getOptionalString("seq").map(SequencerType::valueOf).orElse(SequencerType.bucket);
    SequencePlanner<OpDispenser<? extends O>> planner = new SequencePlanner<>(sequencerType);
    StmtsDocList stmtsDocList = null;
    String workloadSource = "unspecified";
    Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
    Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
    if (stmt.isPresent()) {
        stmtsDocList = StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
        workloadSource = "commandline:" + stmt.get();
    } else if (op_yaml_loc.isPresent()) {
        stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
        workloadSource = "yaml:" + op_yaml_loc.get();
    }
    List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
    List<Long> ratios = new ArrayList<>(stmts.size());
    for (int i = 0; i < stmts.size(); i++) {
        OpTemplate opTemplate = stmts.get(i);
        long ratio = opTemplate.removeParamOrDefault("ratio", 1);
        ratios.add(ratio);
    }
    if (stmts.size() == 0) {
        throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
    }
    try {
        for (int i = 0; i < stmts.size(); i++) {
            long ratio = ratios.get(i);
            OpTemplate optemplate = stmts.get(i);
            OpDispenser<? extends O> driverSpecificReadyOp = opinit.apply(optemplate);
            planner.addOp(driverSpecificReadyOp, ratio);
        }
    } catch (Exception e) {
        throw new OpConfigError(e.getMessage(), workloadSource, e);
    }
    return planner.resolve();
}