Examples with ParallelActionSet org.apache.asterix.experiment.action.base.ParallelActionSet used on opensource projects

Example 6 with ParallelActionSet

use of org.apache.asterix.experiment.action.base.ParallelActionSet in project asterixdb by apache.

the class AbstractExperiment7Builder method doBuildDataGen.

@Override
protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception {
    int nDgens = 0;
    for (List<String> v : dgenPairs.values()) {
        nDgens += v.size();
    }
    final OrchestratorServer7 oServer = new OrchestratorServer7(orchPort, nDgens, nIntervals, new ProtocolActionBuilder(), this.lsAction);
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.start();
        }
    });
    ParallelActionSet dgenActions = new ParallelActionSet();
    int partition = 0;
    // run dgen
    for (String dgenHost : dgenPairs.keySet()) {
        final List<String> rcvrs = dgenPairs.get(dgenHost);
        final int p = partition;
        dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {

            @Override
            protected String getCommand() {
                String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
                String binary = "JAVA_HOME=" + javaHomePath + " " + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
                return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of", openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " ");
            }
        });
        partition += rcvrs.size();
    }
    seq.add(dgenActions);
    // wait until all dgen / queries are done
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.awaitFinished();
        }
    });
}

Example 7 with ParallelActionSet

use of org.apache.asterix.experiment.action.base.ParallelActionSet in project asterixdb by apache.

the class AbstractExperiment8Builder method doBuildDataGen.

@Override
protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception {
    //start datagen
    SequentialActionList[] protocolActions = new SequentialActionList[nIntervals];
    for (int i = 0; i < nIntervals; i++) {
        protocolActions[i] = new SequentialActionList();
        protocolActions[i].add(new SleepAction(10000));
        protocolActions[i].add(new RunRESTIOWaitAction(httpClient, restHost, restPort));
        protocolActions[i].add(new SleepAction(10000));
        doBuildProtocolAction(protocolActions[i], i);
    }
    int nDgens = 0;
    for (List<String> v : dgenPairs.values()) {
        nDgens += v.size();
    }
    final OrchestratorServer oServer = new OrchestratorServer(orchPort, nDgens, nIntervals, protocolActions);
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.start();
        }
    });
    ParallelActionSet dgenActions = new ParallelActionSet();
    int partition = 0;
    // run dgen
    for (String dgenHost : dgenPairs.keySet()) {
        final List<String> rcvrs = dgenPairs.get(dgenHost);
        final int p = partition;
        dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {

            @Override
            protected String getCommand() {
                String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
                String binary = "JAVA_HOME=" + javaHomePath + " " + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
                return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of", openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " ");
            }
        });
        partition += rcvrs.size();
    }
    seq.add(dgenActions);
    // wait until all dgen / queries are done
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.awaitFinished();
        }
    });
}

Example 8 with ParallelActionSet

use of org.apache.asterix.experiment.action.base.ParallelActionSet in project asterixdb by apache.

the class AbstractLSMBaseExperimentBuilder method doBuildDataGen.

protected void doBuildDataGen(SequentialActionList seq, final Map<String, List<String>> dgenPairs) throws Exception {
    //start datagen
    ParallelActionSet dgenActions = new ParallelActionSet();
    int partition = 0;
    for (String dgenHost : dgenPairs.keySet()) {
        final List<String> rcvrs = dgenPairs.get(dgenHost);
        final int p = partition;
        dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {

            @Override
            protected String getCommand() {
                String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
                String binary = "JAVA_HOME=" + javaHomePath + " " + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
                if (openStreetMapFilePath == null) {
                    return StringUtils.join(new String[] { binary, "-rcbi", "" + recordCountPerBatchDuringIngestionOnly, "-rcbq", "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly, "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-p", "" + p, "-d", "" + duration, ipPortPairs }, " ");
                } else {
                    return StringUtils.join(new String[] { binary, "-rcbi", "" + recordCountPerBatchDuringIngestionOnly, "-rcbq", "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly, "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-of", openStreetMapFilePath, "-p", "" + p, "-d", "" + duration, ipPortPairs }, " ");
                }
            }
        });
        partition += rcvrs.size();
    }
    seq.add(dgenActions);
}

Example 9 with ParallelActionSet

use of org.apache.asterix.experiment.action.base.ParallelActionSet in project asterixdb by apache.

the class AbstractPerfLoadBuilder method doBuild.

@Override
protected void doBuild(Experiment e) throws IOException, JAXBException {
    SequentialActionList execs = new SequentialActionList();
    String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
    String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
    //stop/delete/create instance
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new SleepAction(30000));
    execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, asterixConfigPath));
    //ddl statements
    execs.add(new SleepAction(15000));
    // TODO: implement retry handler
    execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMPerfConstants.BASE_TYPES)));
    doBuildDDL(execs);
    //prepare io state action in NC node(s)
    Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
    final Set<String> ncHosts = new HashSet<>();
    for (List<String> ncHostList : dgenPairs.values()) {
        for (String ncHost : ncHostList) {
            ncHosts.add(ncHost.split(":")[0]);
        }
    }
    if (statFile != null) {
        ParallelActionSet ioCountActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
                    return cmd;
                }
            });
        }
        execs.add(ioCountActions);
    }
    //prepare post ls action
    File file = new File(clusterConfigPath);
    JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
    Unmarshaller unmarshaller = ctx.createUnmarshaller();
    final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
    String[] storageRoots = cluster.getIodevices().split(",");
    //---------- main experiment body begins -----------
    //run DDL + Load
    execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(loadAQLFilePath))));
    //execute SQL++ Queries
    execs.add(new TimedAction(new RunSQLPPFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(querySQLPPFileName), localExperimentRoot.resolve(LSMPerfConstants.RESULT_FILE))));
    //kill io state action
    if (statFile != null) {
        ParallelActionSet ioCountKillActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
                    return cmd;
                }
            });
        }
        execs.add(ioCountKillActions);
    }
    //total record count
    execs.add(new SleepAction(10000));
    if (countFileName != null) {
        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
    }
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    //prepare to collect io state by putting the state file into asterix log dir
    if (statFile != null) {
        ParallelActionSet collectIOActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "cp " + statFile + " " + cluster.getLogDir();
                    return cmd;
                }
            });
        }
        execs.add(collectIOActions);
    }
    //collect cc and nc logs
    execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
    e.addBody(execs);
}