Search in sources :

Example 1 with AbstractRemoteExecutableAction

use of org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction in project asterixdb by apache.

the class AbstractSpatialIndexExperiment2Builder method doBuildDataGen.

@Override
protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception {
    int nDgens = 0;
    for (List<String> v : dgenPairs.values()) {
        nDgens += v.size();
    }
    final SpatialIndexExperiment2OrchestratorServer oServer = new SpatialIndexExperiment2OrchestratorServer(dataGenOrchPort, nDgens, nIntervals, queryGenOrchPort, nDgens);
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.start();
        }
    });
    //prepare data gen runner and query gen runner
    //note that dgenPairs.keySet() are used as query gen runners' ip address.
    ParallelActionSet dataAndQueryGenActions = new ParallelActionSet();
    int partition = 0;
    for (String dgenHost : dgenPairs.keySet()) {
        final List<String> rcvrs = dgenPairs.get(dgenHost);
        final int p = partition;
        //prepare data gen
        dataAndQueryGenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {

            @Override
            protected String getCommand() {
                String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
                String binary = "JAVA_HOME=" + javaHomePath + " " + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
                if (openStreetMapFilePath == null) {
                    return StringUtils.join(new String[] { binary, "-rcbi", "" + recordCountPerBatchDuringIngestionOnly, "-rcbq", "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly, "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, "-qd", "" + queryGenDuration, "-oh", dataGenOrchHost, "-op", "" + dataGenOrchPort, ipPortPairs }, " ");
                } else {
                    return StringUtils.join(new String[] { binary, "-rcbi", "" + recordCountPerBatchDuringIngestionOnly, "-rcbq", "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly, "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-of", openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, "-qd", "" + queryGenDuration, "-oh", dataGenOrchHost, "-op", "" + dataGenOrchPort, ipPortPairs }, " ");
                }
            }
        });
        //prepare query gen
        dataAndQueryGenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {

            @Override
            protected String getCommand() {
                String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
                String binary = "JAVA_HOME=" + javaHomePath + " " + localExperimentRoot.resolve("bin").resolve("querygenrunner").toString();
                if (openStreetMapFilePath == null) {
                    if (isIndexOnlyPlan) {
                        return StringUtils.join(new String[] { binary, "-iop", "-p", "" + p, "-qd", "" + queryGenDuration, "-qoh", "" + queryGenOrchHost, "-qop", "" + queryGenOrchPort, "-rh", restHost, "-rp", "" + restPort }, " ");
                    } else {
                        return StringUtils.join(new String[] { binary, "-p", "" + p, "-qd", "" + queryGenDuration, "-qoh", "" + queryGenOrchHost, "-qop", "" + queryGenOrchPort, "-rh", restHost, "-rp", "" + restPort }, " ");
                    }
                } else {
                    if (isIndexOnlyPlan) {
                        return StringUtils.join(new String[] { binary, "-iop", "-of", openStreetMapFilePath, "-p", "" + p, "-qd", "" + queryGenDuration, "-qoh", "" + queryGenOrchHost, "-qop", "" + queryGenOrchPort, "-rh", restHost, "-rp", "" + restPort }, " ");
                    } else {
                        return StringUtils.join(new String[] { binary, "-of", openStreetMapFilePath, "-p", "" + p, "-qd", "" + queryGenDuration, "-qoh", "" + queryGenOrchHost, "-qop", "" + queryGenOrchPort, "-rh", restHost, "-rp", "" + restPort }, " ");
                    }
                }
            }
        });
        partition += rcvrs.size();
    }
    seq.add(dataAndQueryGenActions);
    // wait until all dgen / queries are done
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.awaitFinished();
        }
    });
}
Also used : SpatialIndexExperiment2OrchestratorServer(org.apache.asterix.experiment.client.SpatialIndexExperiment2OrchestratorServer) AbstractRemoteExecutableAction(org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction) AbstractAction(org.apache.asterix.experiment.action.base.AbstractAction) ParallelActionSet(org.apache.asterix.experiment.action.base.ParallelActionSet)

Example 2 with AbstractRemoteExecutableAction

use of org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction in project asterixdb by apache.

the class AbstractSpatialIndexExperiment3PIdxLoadBuilder method doBuild.

@Override
protected void doBuild(Experiment e) throws Exception {
    SequentialActionList execs = new SequentialActionList();
    String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
    String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
    //stop/delete/create instance
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new SleepAction(30000));
    execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, asterixConfigPath));
    //ddl statements
    execs.add(new SleepAction(15000));
    // TODO: implement retry handler
    execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_TYPES)));
    doBuildDDL(execs);
    //prepare io state action in NC node(s)
    Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
    final Set<String> ncHosts = new HashSet<>();
    for (List<String> ncHostList : dgenPairs.values()) {
        for (String ncHost : ncHostList) {
            ncHosts.add(ncHost.split(":")[0]);
        }
    }
    if (statFile != null) {
        ParallelActionSet ioCountActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
                    return cmd;
                }
            });
        }
        execs.add(ioCountActions);
    }
    //prepare post ls action
    SequentialActionList postLSAction = new SequentialActionList();
    File file = new File(clusterConfigPath);
    JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
    Unmarshaller unmarshaller = ctx.createUnmarshaller();
    final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
    String[] storageRoots = cluster.getIodevices().split(",");
    for (String ncHost : ncHosts) {
        for (final String sRoot : storageRoots) {
            lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    return "ls -Rl " + sRoot;
                }
            });
            postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    return "ls -Rl " + sRoot;
                }
            });
        }
    }
    //---------- main experiment body begins -----------
    //load data into pidx 
    execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(loadAQLFilePath))));
    //kill io state action
    if (statFile != null) {
        ParallelActionSet ioCountKillActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
                    return cmd;
                }
            });
        }
        execs.add(ioCountKillActions);
    }
    //total record count
    execs.add(new SleepAction(10000));
    if (countFileName != null) {
        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
    }
    //add ls action
    execs.add(postLSAction);
    //kill asterix cc and nc
    ParallelActionSet killCmds = new ParallelActionSet();
    for (String ncHost : ncHosts) {
        killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
    }
    killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
    execs.add(killCmds);
    //stop asterix instance
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    //prepare to collect io state by putting the state file into asterix log dir
    if (statFile != null) {
        ParallelActionSet collectIOActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "cp " + statFile + " " + cluster.getLogDir();
                    return cmd;
                }
            });
        }
        execs.add(collectIOActions);
    }
    //collect cc and nc logs
    execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
    e.addBody(execs);
}
Also used : DeleteAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.DeleteAsterixManagixAction) StopAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction) Cluster(org.apache.asterix.event.schema.cluster.Cluster) JAXBContext(javax.xml.bind.JAXBContext) TimedAction(org.apache.asterix.experiment.action.derived.TimedAction) SleepAction(org.apache.asterix.experiment.action.derived.SleepAction) LogAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction) AbstractRemoteExecutableAction(org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction) ParallelActionSet(org.apache.asterix.experiment.action.base.ParallelActionSet) RemoteAsterixDriverKill(org.apache.asterix.experiment.action.derived.RemoteAsterixDriverKill) SequentialActionList(org.apache.asterix.experiment.action.base.SequentialActionList) RunAQLFileAction(org.apache.asterix.experiment.action.derived.RunAQLFileAction) ArrayList(java.util.ArrayList) SequentialActionList(org.apache.asterix.experiment.action.base.SequentialActionList) List(java.util.List) Unmarshaller(javax.xml.bind.Unmarshaller) File(java.io.File) CreateAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.CreateAsterixManagixAction) HashSet(java.util.HashSet)

Example 3 with AbstractRemoteExecutableAction

use of org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction in project asterixdb by apache.

the class AbstractSpatialIndexExperiment3SIdxCreateAndQueryBuilder method doBuild.

@Override
protected void doBuild(Experiment e) throws Exception {
    SequentialActionList execs = new SequentialActionList();
    String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
    String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
    //start asterix instance
    execs.add(new StartAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new SleepAction(30000));
    //prepare io state action in NC node(s)
    Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
    final Set<String> ncHosts = new HashSet<>();
    for (List<String> ncHostList : dgenPairs.values()) {
        for (String ncHost : ncHostList) {
            ncHosts.add(ncHost.split(":")[0]);
        }
    }
    if (statFile != null) {
        ParallelActionSet ioCountActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
                    return cmd;
                }
            });
        }
        execs.add(ioCountActions);
    }
    //prepare post ls action
    SequentialActionList postLSAction = new SequentialActionList();
    File file = new File(clusterConfigPath);
    JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
    Unmarshaller unmarshaller = ctx.createUnmarshaller();
    final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
    String[] storageRoots = cluster.getIodevices().split(",");
    for (String ncHost : ncHosts) {
        for (final String sRoot : storageRoots) {
            lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    return "ls -Rl " + sRoot;
                }
            });
            postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    return "ls -Rl " + sRoot;
                }
            });
        }
    }
    try {
        outputFilePath = openStreetMapFilePath.substring(0, openStreetMapFilePath.lastIndexOf(File.separator)) + File.separator + "QueryGenResult-" + getName() + "-" + Inet4Address.getLocalHost().getHostAddress() + ".txt";
        outputFos = ExperimentProfilerUtils.openOutputFile(outputFilePath);
    } catch (Exception e1) {
        e1.printStackTrace();
        return;
    }
    //delete all existing secondary indexes if any
    execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.dhbtreeLocation;", outputFos));
    execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.dhvbtreeLocation;", outputFos));
    execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.rtreeLocation;", outputFos));
    execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.shbtreeLocation;", outputFos));
    execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.sifLocation;", outputFos));
    //create secondary index 
    execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(createAQLFilePath), outputFos), outputFos));
    //run count query for cleaning up OS buffer cache
    if (countFileName != null) {
        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName), outputFos));
    }
    //run cache warm-up queries: run CACHE_WARM_UP_QUERY_COUNT select queries
    br = new BufferedReader(new FileReader(querySeedFilePath));
    radiusIter = 0;
    for (int i = 0; i < CACHE_WARM_UP_QUERY_COUNT; i++) {
        execs.add(getSelectQuery(isIndexOnlyPlan));
    }
    radiusIter = 0;
    //run queries for measurement: run SELECT_QUERY_COUNT select queries
    for (int i = 0; i < SELECT_QUERY_COUNT; i++) {
        execs.add(getSelectQuery(isIndexOnlyPlan));
    }
    radiusIter = 0;
    //run queries for measurement: run JOIN_QUERY_COUNT join queries
    for (int i = 0; i < JOIN_QUERY_COUNT; i++) {
        execs.add(getJoinQuery(isIndexOnlyPlan));
    }
    //---------- main experiment body ends -----------
    //kill io state action
    //        if (statFile != null) {
    //            ParallelActionSet ioCountKillActions = new ParallelActionSet();
    //            for (String ncHost : ncHosts) {
    //                ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
    //
    //                    @Override
    //                    protected String getCommand() {
    //                        String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
    //                        return cmd;
    //                    }
    //                });
    //            }
    //            execs.add(ioCountKillActions);
    //        }
    //add ls action
    execs.add(postLSAction);
    //kill asterix cc and nc
    ParallelActionSet killCmds = new ParallelActionSet();
    for (String ncHost : ncHosts) {
        killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
    }
    killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
    execs.add(killCmds);
    //stop asterix instance
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    //prepare to collect io state by putting the state file into asterix log dir
    if (statFile != null) {
        ParallelActionSet collectIOActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "cp " + statFile + " " + cluster.getLogDir();
                    return cmd;
                }
            });
        }
        execs.add(collectIOActions);
    }
    //collect profile information
    //        if (ExperimentProfiler.PROFILE_MODE) {
    //            if (!SpatialIndexProfiler.PROFILE_HOME_DIR.contentEquals(cluster.getLogDir())) {
    //                ParallelActionSet collectProfileInfo = new ParallelActionSet();
    //                for (String ncHost : ncHosts) {
    //                    collectProfileInfo.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
    //                        @Override
    //                        protected String getCommand() {
    //                            String cmd = "mv " + SpatialIndexProfiler.PROFILE_HOME_DIR + "*.txt " + cluster.getLogDir();
    //                            return cmd;
    //                        }
    //                    });
    //                }
    //                execs.add(collectProfileInfo);
    //            }
    //        }
    //collect cc and nc logs
    execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
    //get query result file
    final String queryResultFilePath = outputFilePath;
    execs.add(new AbstractRemoteExecutableAction(restHost, username, sshKeyLocation) {

        @Override
        protected String getCommand() {
            String cmd = "mv " + queryResultFilePath + " " + localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString();
            return cmd;
        }
    });
    //close the outputStream
    execs.add(new CloseOutputStreamAction(outputFos));
    e.addBody(execs);
}
Also used : StopAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction) JAXBContext(javax.xml.bind.JAXBContext) TimedAction(org.apache.asterix.experiment.action.derived.TimedAction) SleepAction(org.apache.asterix.experiment.action.derived.SleepAction) AbstractRemoteExecutableAction(org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction) StartAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.StartAsterixManagixAction) RunAQLStringAction(org.apache.asterix.experiment.action.derived.RunAQLStringAction) CloseOutputStreamAction(org.apache.asterix.experiment.action.derived.CloseOutputStreamAction) ArrayList(java.util.ArrayList) SequentialActionList(org.apache.asterix.experiment.action.base.SequentialActionList) List(java.util.List) FileReader(java.io.FileReader) Unmarshaller(javax.xml.bind.Unmarshaller) HashSet(java.util.HashSet) Cluster(org.apache.asterix.event.schema.cluster.Cluster) LogAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction) ParallelActionSet(org.apache.asterix.experiment.action.base.ParallelActionSet) IOException(java.io.IOException) RemoteAsterixDriverKill(org.apache.asterix.experiment.action.derived.RemoteAsterixDriverKill) SequentialActionList(org.apache.asterix.experiment.action.base.SequentialActionList) RunAQLFileAction(org.apache.asterix.experiment.action.derived.RunAQLFileAction) BufferedReader(java.io.BufferedReader) File(java.io.File)

Example 4 with AbstractRemoteExecutableAction

use of org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction in project asterixdb by apache.

the class AbstractExperiment9Builder method doBuildDataGen.

@Override
protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws IOException {
    int nDgens = 0;
    for (List<String> v : dgenPairs.values()) {
        nDgens += v.size();
    }
    final OrchestratorServer9 oServer = new OrchestratorServer9(orchPort, nDgens, nIntervals, new ProtocolActionBuilder());
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.start();
        }
    });
    ParallelActionSet dgenActions = new ParallelActionSet();
    int partition = 0;
    // run dgen
    for (String dgenHost : dgenPairs.keySet()) {
        final List<String> rcvrs = dgenPairs.get(dgenHost);
        final int p = partition;
        dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {

            @Override
            protected String getCommand() {
                String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
                String binary = "JAVA_HOME=" + javaHomePath + " " + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
                return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of", openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " ");
            }
        });
        partition += rcvrs.size();
    }
    seq.add(dgenActions);
    // wait until all dgen / queries are done
    seq.add(new AbstractAction() {

        @Override
        protected void doPerform() throws Exception {
            oServer.awaitFinished();
        }
    });
}
Also used : IProtocolActionBuilder(org.apache.asterix.experiment.client.OrchestratorServer9.IProtocolActionBuilder) OrchestratorServer9(org.apache.asterix.experiment.client.OrchestratorServer9) AbstractRemoteExecutableAction(org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction) AbstractAction(org.apache.asterix.experiment.action.base.AbstractAction) IOException(java.io.IOException) ParallelActionSet(org.apache.asterix.experiment.action.base.ParallelActionSet)

Example 5 with AbstractRemoteExecutableAction

use of org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction in project asterixdb by apache.

the class AbstractLSMBaseExperimentBuilder method doBuild.

@Override
protected void doBuild(Experiment e) throws Exception {
    SequentialActionList execs = new SequentialActionList();
    String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
    String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
    //create instance
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new SleepAction(30000));
    execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, asterixConfigPath));
    //run ddl statements
    execs.add(new SleepAction(15000));
    // TODO: implement retry handler
    execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_TYPES)));
    doBuildDDL(execs);
    execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_DIR).resolve(ingestFileName)));
    Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
    final Set<String> ncHosts = new HashSet<>();
    for (List<String> ncHostList : dgenPairs.values()) {
        for (String ncHost : ncHostList) {
            ncHosts.add(ncHost.split(":")[0]);
        }
    }
    if (statFile != null) {
        ParallelActionSet ioCountActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
                    return cmd;
                }
            });
        }
        execs.add(ioCountActions);
    }
    SequentialActionList postLSAction = new SequentialActionList();
    File file = new File(clusterConfigPath);
    JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
    Unmarshaller unmarshaller = ctx.createUnmarshaller();
    final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
    String[] storageRoots = cluster.getIodevices().split(",");
    for (String ncHost : ncHosts) {
        for (final String sRoot : storageRoots) {
            lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    return "ls -Rl " + sRoot;
                }
            });
            postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    return "ls -Rl " + sRoot;
                }
            });
        }
    }
    // main exp
    doBuildDataGen(execs, dgenPairs);
    //        if (statFile != null) {
    //            ParallelActionSet ioCountKillActions = new ParallelActionSet();
    //            for (String ncHost : ncHosts) {
    //                ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
    //
    //                    @Override
    //                    protected String getCommand() {
    //                        String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
    //                        return cmd;
    //                    }
    //                });
    //            }
    //            execs.add(ioCountKillActions);
    //        }
    execs.add(new SleepAction(10000));
    if (countFileName != null) {
        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
    }
    execs.add(postLSAction);
    doPost(execs);
    ParallelActionSet killCmds = new ParallelActionSet();
    for (String ncHost : ncHosts) {
        killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
    }
    //killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
    execs.add(killCmds);
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    if (statFile != null) {
        ParallelActionSet collectIOActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "cp " + statFile + " " + cluster.getLogDir();
                    return cmd;
                }
            });
        }
        execs.add(collectIOActions);
    }
    //collect profile information
    //        if (ExperimentProfiler.PROFILE_MODE) {
    //            if (!SpatialIndexProfiler.PROFILE_HOME_DIR.contentEquals(cluster.getLogDir())) {
    //                ParallelActionSet collectProfileInfo = new ParallelActionSet();
    //                for (String ncHost : ncHosts) {
    //                    collectProfileInfo.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
    //                        @Override
    //                        protected String getCommand() {
    //                            String cmd = "mv " + SpatialIndexProfiler.PROFILE_HOME_DIR + "*.txt " + cluster.getLogDir();
    //                            return cmd;
    //                        }
    //                    });
    //                }
    //                execs.add(collectProfileInfo);
    //            }
    //        }
    execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
    if (getName().contains("SpatialIndexExperiment2") || getName().contains("SpatialIndexExperiment5")) {
        //get query result file
        SequentialActionList getQueryResultFileActions = new SequentialActionList();
        final String queryResultFilePath = openStreetMapFilePath.substring(0, openStreetMapFilePath.lastIndexOf(File.separator)) + File.separator + "QueryGenResult-*.txt";
        for (final String qgenHost : dgenPairs.keySet()) {
            getQueryResultFileActions.add(new AbstractRemoteExecutableAction(restHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "scp " + username + "@" + qgenHost + ":" + queryResultFilePath + " " + localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString();
                    return cmd;
                }
            });
        }
        execs.add(getQueryResultFileActions);
    }
    e.addBody(execs);
}
Also used : DeleteAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.DeleteAsterixManagixAction) StopAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction) Cluster(org.apache.asterix.event.schema.cluster.Cluster) JAXBContext(javax.xml.bind.JAXBContext) SleepAction(org.apache.asterix.experiment.action.derived.SleepAction) LogAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction) AbstractRemoteExecutableAction(org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction) ParallelActionSet(org.apache.asterix.experiment.action.base.ParallelActionSet) RemoteAsterixDriverKill(org.apache.asterix.experiment.action.derived.RemoteAsterixDriverKill) SequentialActionList(org.apache.asterix.experiment.action.base.SequentialActionList) RunAQLFileAction(org.apache.asterix.experiment.action.derived.RunAQLFileAction) ArrayList(java.util.ArrayList) SequentialActionList(org.apache.asterix.experiment.action.base.SequentialActionList) List(java.util.List) Unmarshaller(javax.xml.bind.Unmarshaller) File(java.io.File) CreateAsterixManagixAction(org.apache.asterix.experiment.action.derived.ManagixActions.CreateAsterixManagixAction) HashSet(java.util.HashSet)

Aggregations

ParallelActionSet (org.apache.asterix.experiment.action.base.ParallelActionSet)8 AbstractRemoteExecutableAction (org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction)8 IOException (java.io.IOException)4 AbstractAction (org.apache.asterix.experiment.action.base.AbstractAction)4 SequentialActionList (org.apache.asterix.experiment.action.base.SequentialActionList)4 SleepAction (org.apache.asterix.experiment.action.derived.SleepAction)4 File (java.io.File)3 ArrayList (java.util.ArrayList)3 HashSet (java.util.HashSet)3 List (java.util.List)3 JAXBContext (javax.xml.bind.JAXBContext)3 Unmarshaller (javax.xml.bind.Unmarshaller)3 Cluster (org.apache.asterix.event.schema.cluster.Cluster)3 LogAsterixManagixAction (org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction)3 StopAsterixManagixAction (org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction)3 RemoteAsterixDriverKill (org.apache.asterix.experiment.action.derived.RemoteAsterixDriverKill)3 RunAQLFileAction (org.apache.asterix.experiment.action.derived.RunAQLFileAction)3 CreateAsterixManagixAction (org.apache.asterix.experiment.action.derived.ManagixActions.CreateAsterixManagixAction)2 DeleteAsterixManagixAction (org.apache.asterix.experiment.action.derived.ManagixActions.DeleteAsterixManagixAction)2 TimedAction (org.apache.asterix.experiment.action.derived.TimedAction)2