Examples with CassandraJobStore com.netflix.titus.ext.cassandra.store.CassandraJobStore used on opensource projects

Example 1 with CassandraJobStore

use of com.netflix.titus.ext.cassandra.store.CassandraJobStore in project titus-control-plane by Netflix.

the class TestStoreLoadCommand method execute.

@Override
public void execute(CommandContext commandContext) {
    CommandLine commandLine = commandContext.getCommandLine();
    String keyspace = commandContext.getTargetKeySpace();
    Integer jobs = Integer.valueOf(commandLine.getOptionValue("jobs"));
    Integer tasks = Integer.valueOf(commandLine.getOptionValue("tasks"));
    Integer concurrency = Integer.valueOf(commandLine.getOptionValue("concurrency"));
    Integer iterations = Integer.valueOf(commandLine.getOptionValue("iterations"));
    Session session = commandContext.getTargetSession();
    boolean keyspaceExists = session.getCluster().getMetadata().getKeyspace(keyspace) != null;
    if (!keyspaceExists) {
        throw new IllegalStateException("Keyspace: " + keyspace + " does not exist. You must create it first.");
    }
    session.execute("USE " + keyspace);
    JobStore titusStore = new CassandraJobStore(CONFIGURATION, session, TitusRuntimes.internal());
    // Create jobs and tasks
    long jobStartTime = System.currentTimeMillis();
    List<Observable<Void>> createJobAndTasksObservables = new ArrayList<>();
    for (int i = 0; i < jobs; i++) {
        createJobAndTasksObservables.add(createJobAndTasksObservable(tasks, titusStore));
    }
    Observable.merge(createJobAndTasksObservables, concurrency).toBlocking().subscribe(none -> {
    }, e -> logger.error("Error creating jobs: ", e), () -> {
        logger.info("Created {} jobs with {} tasks in {}[ms]", jobs, tasks, System.currentTimeMillis() - jobStartTime);
    });
    // try loading jobs and tasks for i iterations
    long loadTotalTime = 0L;
    for (int i = 0; i < iterations; i++) {
        long loadStartTime = System.currentTimeMillis();
        List<Pair<Job, List<Task>>> pairs = new ArrayList<>();
        titusStore.init().andThen(titusStore.retrieveJobs().flatMap(retrievedJobsAndErrors -> {
            List<Job<?>> retrievedJobs = retrievedJobsAndErrors.getLeft();
            List<Observable<Pair<Job, List<Task>>>> retrieveTasksObservables = new ArrayList<>();
            for (Job job : retrievedJobs) {
                Observable<Pair<Job, List<Task>>> retrieveTasksObservable = titusStore.retrieveTasksForJob(job.getId()).map(taskList -> new Pair<>(job, taskList.getLeft()));
                retrieveTasksObservables.add(retrieveTasksObservable);
            }
            return Observable.merge(retrieveTasksObservables, MAX_RETRIEVE_TASK_CONCURRENCY);
        })).map(p -> {
            pairs.add(p);
            return null;
        }).toBlocking().subscribe(none -> {
        }, e -> logger.error("Failed to load jobs from cassandra with error: ", e), () -> {
        });
        long loadTime = System.currentTimeMillis() - loadStartTime;
        logger.info("Loaded {} jobs from cassandra in {}[ms]", pairs.size(), loadTime);
        loadTotalTime += loadTime;
    }
    logger.info("Average load time: {}[ms]", loadTotalTime / iterations);
}