use of org.apache.asterix.experiment.action.derived.RunAQLFileAction in project asterixdb by apache.
the class AbstractSpatialIndexExperiment3PIdxLoadBuilder method doBuild.
@Override
protected void doBuild(Experiment e) throws Exception {
SequentialActionList execs = new SequentialActionList();
String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
//stop/delete/create instance
execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
execs.add(new SleepAction(30000));
execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, asterixConfigPath));
//ddl statements
execs.add(new SleepAction(15000));
// TODO: implement retry handler
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_TYPES)));
doBuildDDL(execs);
//prepare io state action in NC node(s)
Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
final Set<String> ncHosts = new HashSet<>();
for (List<String> ncHostList : dgenPairs.values()) {
for (String ncHost : ncHostList) {
ncHosts.add(ncHost.split(":")[0]);
}
}
if (statFile != null) {
ParallelActionSet ioCountActions = new ParallelActionSet();
for (String ncHost : ncHosts) {
ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
return cmd;
}
});
}
execs.add(ioCountActions);
}
//prepare post ls action
SequentialActionList postLSAction = new SequentialActionList();
File file = new File(clusterConfigPath);
JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
Unmarshaller unmarshaller = ctx.createUnmarshaller();
final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
String[] storageRoots = cluster.getIodevices().split(",");
for (String ncHost : ncHosts) {
for (final String sRoot : storageRoots) {
lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
return "ls -Rl " + sRoot;
}
});
postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
return "ls -Rl " + sRoot;
}
});
}
}
//---------- main experiment body begins -----------
//load data into pidx
execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(loadAQLFilePath))));
//kill io state action
if (statFile != null) {
ParallelActionSet ioCountKillActions = new ParallelActionSet();
for (String ncHost : ncHosts) {
ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
return cmd;
}
});
}
execs.add(ioCountKillActions);
}
//total record count
execs.add(new SleepAction(10000));
if (countFileName != null) {
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
}
//add ls action
execs.add(postLSAction);
//kill asterix cc and nc
ParallelActionSet killCmds = new ParallelActionSet();
for (String ncHost : ncHosts) {
killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
}
killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
execs.add(killCmds);
//stop asterix instance
execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
//prepare to collect io state by putting the state file into asterix log dir
if (statFile != null) {
ParallelActionSet collectIOActions = new ParallelActionSet();
for (String ncHost : ncHosts) {
collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "cp " + statFile + " " + cluster.getLogDir();
return cmd;
}
});
}
execs.add(collectIOActions);
}
//collect cc and nc logs
execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
e.addBody(execs);
}
use of org.apache.asterix.experiment.action.derived.RunAQLFileAction in project asterixdb by apache.
the class AbstractSpatialIndexExperiment3SIdxCreateAndQueryBuilder method doBuild.
@Override
protected void doBuild(Experiment e) throws Exception {
SequentialActionList execs = new SequentialActionList();
String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
//start asterix instance
execs.add(new StartAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
execs.add(new SleepAction(30000));
//prepare io state action in NC node(s)
Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
final Set<String> ncHosts = new HashSet<>();
for (List<String> ncHostList : dgenPairs.values()) {
for (String ncHost : ncHostList) {
ncHosts.add(ncHost.split(":")[0]);
}
}
if (statFile != null) {
ParallelActionSet ioCountActions = new ParallelActionSet();
for (String ncHost : ncHosts) {
ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
return cmd;
}
});
}
execs.add(ioCountActions);
}
//prepare post ls action
SequentialActionList postLSAction = new SequentialActionList();
File file = new File(clusterConfigPath);
JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
Unmarshaller unmarshaller = ctx.createUnmarshaller();
final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
String[] storageRoots = cluster.getIodevices().split(",");
for (String ncHost : ncHosts) {
for (final String sRoot : storageRoots) {
lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
return "ls -Rl " + sRoot;
}
});
postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
return "ls -Rl " + sRoot;
}
});
}
}
try {
outputFilePath = openStreetMapFilePath.substring(0, openStreetMapFilePath.lastIndexOf(File.separator)) + File.separator + "QueryGenResult-" + getName() + "-" + Inet4Address.getLocalHost().getHostAddress() + ".txt";
outputFos = ExperimentProfilerUtils.openOutputFile(outputFilePath);
} catch (Exception e1) {
e1.printStackTrace();
return;
}
//delete all existing secondary indexes if any
execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.dhbtreeLocation;", outputFos));
execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.dhvbtreeLocation;", outputFos));
execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.rtreeLocation;", outputFos));
execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.shbtreeLocation;", outputFos));
execs.add(new RunAQLStringAction(httpClient, restHost, restPort, "use dataverse experiments; drop index Tweets.sifLocation;", outputFos));
//create secondary index
execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(createAQLFilePath), outputFos), outputFos));
//run count query for cleaning up OS buffer cache
if (countFileName != null) {
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName), outputFos));
}
//run cache warm-up queries: run CACHE_WARM_UP_QUERY_COUNT select queries
br = new BufferedReader(new FileReader(querySeedFilePath));
radiusIter = 0;
for (int i = 0; i < CACHE_WARM_UP_QUERY_COUNT; i++) {
execs.add(getSelectQuery(isIndexOnlyPlan));
}
radiusIter = 0;
//run queries for measurement: run SELECT_QUERY_COUNT select queries
for (int i = 0; i < SELECT_QUERY_COUNT; i++) {
execs.add(getSelectQuery(isIndexOnlyPlan));
}
radiusIter = 0;
//run queries for measurement: run JOIN_QUERY_COUNT join queries
for (int i = 0; i < JOIN_QUERY_COUNT; i++) {
execs.add(getJoinQuery(isIndexOnlyPlan));
}
//---------- main experiment body ends -----------
//kill io state action
// if (statFile != null) {
// ParallelActionSet ioCountKillActions = new ParallelActionSet();
// for (String ncHost : ncHosts) {
// ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
//
// @Override
// protected String getCommand() {
// String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
// return cmd;
// }
// });
// }
// execs.add(ioCountKillActions);
// }
//add ls action
execs.add(postLSAction);
//kill asterix cc and nc
ParallelActionSet killCmds = new ParallelActionSet();
for (String ncHost : ncHosts) {
killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
}
killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
execs.add(killCmds);
//stop asterix instance
execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
//prepare to collect io state by putting the state file into asterix log dir
if (statFile != null) {
ParallelActionSet collectIOActions = new ParallelActionSet();
for (String ncHost : ncHosts) {
collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "cp " + statFile + " " + cluster.getLogDir();
return cmd;
}
});
}
execs.add(collectIOActions);
}
//collect profile information
// if (ExperimentProfiler.PROFILE_MODE) {
// if (!SpatialIndexProfiler.PROFILE_HOME_DIR.contentEquals(cluster.getLogDir())) {
// ParallelActionSet collectProfileInfo = new ParallelActionSet();
// for (String ncHost : ncHosts) {
// collectProfileInfo.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
// @Override
// protected String getCommand() {
// String cmd = "mv " + SpatialIndexProfiler.PROFILE_HOME_DIR + "*.txt " + cluster.getLogDir();
// return cmd;
// }
// });
// }
// execs.add(collectProfileInfo);
// }
// }
//collect cc and nc logs
execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
//get query result file
final String queryResultFilePath = outputFilePath;
execs.add(new AbstractRemoteExecutableAction(restHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "mv " + queryResultFilePath + " " + localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString();
return cmd;
}
});
//close the outputStream
execs.add(new CloseOutputStreamAction(outputFos));
e.addBody(execs);
}
use of org.apache.asterix.experiment.action.derived.RunAQLFileAction in project asterixdb by apache.
the class PresetClusterPerfBuilder method doBuild.
@Override
protected void doBuild(Experiment e) throws IOException, JAXBException {
SequentialActionList execs = new SequentialActionList();
//ddl statements
execs.add(new SleepAction(15000));
// TODO: implement retry handler
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMPerfConstants.BASE_TYPES)));
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("bench_3.aql")));
//---------- main experiment body begins -----------
//run DDL + Load
execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(loadAQLFilePath))));
//execute SQL++ Queries
execs.add(new TimedAction(new RunSQLPPFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(querySQLPPFileName), localExperimentRoot.resolve(LSMPerfConstants.RESULT_FILE))));
//---------- main experiment body ends -----------
//total record count
execs.add(new SleepAction(10000));
if (countFileName != null) {
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
}
e.addBody(execs);
}
use of org.apache.asterix.experiment.action.derived.RunAQLFileAction in project asterixdb by apache.
the class AbstractLSMBaseExperimentBuilder method doBuild.
@Override
protected void doBuild(Experiment e) throws Exception {
SequentialActionList execs = new SequentialActionList();
String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
//create instance
execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
execs.add(new SleepAction(30000));
execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, asterixConfigPath));
//run ddl statements
execs.add(new SleepAction(15000));
// TODO: implement retry handler
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_TYPES)));
doBuildDDL(execs);
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_DIR).resolve(ingestFileName)));
Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
final Set<String> ncHosts = new HashSet<>();
for (List<String> ncHostList : dgenPairs.values()) {
for (String ncHost : ncHostList) {
ncHosts.add(ncHost.split(":")[0]);
}
}
if (statFile != null) {
ParallelActionSet ioCountActions = new ParallelActionSet();
for (String ncHost : ncHosts) {
ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
return cmd;
}
});
}
execs.add(ioCountActions);
}
SequentialActionList postLSAction = new SequentialActionList();
File file = new File(clusterConfigPath);
JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
Unmarshaller unmarshaller = ctx.createUnmarshaller();
final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
String[] storageRoots = cluster.getIodevices().split(",");
for (String ncHost : ncHosts) {
for (final String sRoot : storageRoots) {
lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
return "ls -Rl " + sRoot;
}
});
postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
return "ls -Rl " + sRoot;
}
});
}
}
// main exp
doBuildDataGen(execs, dgenPairs);
// if (statFile != null) {
// ParallelActionSet ioCountKillActions = new ParallelActionSet();
// for (String ncHost : ncHosts) {
// ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
//
// @Override
// protected String getCommand() {
// String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
// return cmd;
// }
// });
// }
// execs.add(ioCountKillActions);
// }
execs.add(new SleepAction(10000));
if (countFileName != null) {
execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
}
execs.add(postLSAction);
doPost(execs);
ParallelActionSet killCmds = new ParallelActionSet();
for (String ncHost : ncHosts) {
killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
}
//killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
execs.add(killCmds);
execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
if (statFile != null) {
ParallelActionSet collectIOActions = new ParallelActionSet();
for (String ncHost : ncHosts) {
collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "cp " + statFile + " " + cluster.getLogDir();
return cmd;
}
});
}
execs.add(collectIOActions);
}
//collect profile information
// if (ExperimentProfiler.PROFILE_MODE) {
// if (!SpatialIndexProfiler.PROFILE_HOME_DIR.contentEquals(cluster.getLogDir())) {
// ParallelActionSet collectProfileInfo = new ParallelActionSet();
// for (String ncHost : ncHosts) {
// collectProfileInfo.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
// @Override
// protected String getCommand() {
// String cmd = "mv " + SpatialIndexProfiler.PROFILE_HOME_DIR + "*.txt " + cluster.getLogDir();
// return cmd;
// }
// });
// }
// execs.add(collectProfileInfo);
// }
// }
execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
if (getName().contains("SpatialIndexExperiment2") || getName().contains("SpatialIndexExperiment5")) {
//get query result file
SequentialActionList getQueryResultFileActions = new SequentialActionList();
final String queryResultFilePath = openStreetMapFilePath.substring(0, openStreetMapFilePath.lastIndexOf(File.separator)) + File.separator + "QueryGenResult-*.txt";
for (final String qgenHost : dgenPairs.keySet()) {
getQueryResultFileActions.add(new AbstractRemoteExecutableAction(restHost, username, sshKeyLocation) {
@Override
protected String getCommand() {
String cmd = "scp " + username + "@" + qgenHost + ":" + queryResultFilePath + " " + localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString();
return cmd;
}
});
}
execs.add(getQueryResultFileActions);
}
e.addBody(execs);
}
Aggregations