Examples with Cluster org.apache.asterix.event.schema.cluster.Cluster used on opensource projects

Example 16 with Cluster

use of org.apache.asterix.event.schema.cluster.Cluster in project asterixdb by apache.

the class AbstractPerfLoadBuilder method doBuild.

@Override
protected void doBuild(Experiment e) throws IOException, JAXBException {
    SequentialActionList execs = new SequentialActionList();
    String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(clusterConfigFileName).toString();
    String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR).resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
    //stop/delete/create instance
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    execs.add(new SleepAction(30000));
    execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, asterixConfigPath));
    //ddl statements
    execs.add(new SleepAction(15000));
    // TODO: implement retry handler
    execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMPerfConstants.BASE_TYPES)));
    doBuildDDL(execs);
    //prepare io state action in NC node(s)
    Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
    final Set<String> ncHosts = new HashSet<>();
    for (List<String> ncHostList : dgenPairs.values()) {
        for (String ncHost : ncHostList) {
            ncHosts.add(ncHost.split(":")[0]);
        }
    }
    if (statFile != null) {
        ParallelActionSet ioCountActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
                    return cmd;
                }
            });
        }
        execs.add(ioCountActions);
    }
    //prepare post ls action
    File file = new File(clusterConfigPath);
    JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
    Unmarshaller unmarshaller = ctx.createUnmarshaller();
    final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
    String[] storageRoots = cluster.getIodevices().split(",");
    //---------- main experiment body begins -----------
    //run DDL + Load
    execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(loadAQLFilePath))));
    //execute SQL++ Queries
    execs.add(new TimedAction(new RunSQLPPFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(querySQLPPFileName), localExperimentRoot.resolve(LSMPerfConstants.RESULT_FILE))));
    //kill io state action
    if (statFile != null) {
        ParallelActionSet ioCountKillActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
                    return cmd;
                }
            });
        }
        execs.add(ioCountKillActions);
    }
    //total record count
    execs.add(new SleepAction(10000));
    if (countFileName != null) {
        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
    }
    execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
    //prepare to collect io state by putting the state file into asterix log dir
    if (statFile != null) {
        ParallelActionSet collectIOActions = new ParallelActionSet();
        for (String ncHost : ncHosts) {
            collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {

                @Override
                protected String getCommand() {
                    String cmd = "cp " + statFile + " " + cluster.getLogDir();
                    return cmd;
                }
            });
        }
        execs.add(collectIOActions);
    }
    //collect cc and nc logs
    execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
    e.addBody(execs);
}

Example 17 with Cluster

use of org.apache.asterix.event.schema.cluster.Cluster in project asterixdb by apache.

the class ValidateConfig method validateCluster.

public boolean validateCluster(String clusterPath) throws Exception {
    boolean valid = true;
    Cluster cluster = null;
    File f = new File(clusterPath);
    List<String> ipAddresses = new ArrayList<String>();
    if (!f.exists() || !f.isFile()) {
        LOGGER.error(" Invalid path " + f.getAbsolutePath() + ERROR);
        valid = false;
    } else {
        cluster = EventUtil.getCluster(clusterPath);
        valid = valid & validateClusterProperties(cluster);
        Set<String> servers = new HashSet<String>();
        Set<String> serverIds = new HashSet<String>();
        servers.add(cluster.getMasterNode().getClusterIp());
        serverIds.add(cluster.getMasterNode().getId());
        MasterNode masterNode = cluster.getMasterNode();
        Node master = new Node(masterNode.getId(), masterNode.getClusterIp(), masterNode.getJavaHome(), masterNode.getLogDir(), null, null, null, null);
        ipAddresses.add(masterNode.getClusterIp());
        valid = valid & validateNodeConfiguration(master, cluster);
        for (Node node : cluster.getNode()) {
            servers.add(node.getClusterIp());
            if (serverIds.contains(node.getId())) {
                valid = false;
                LOGGER.error("Duplicate node id :" + node.getId() + ERROR);
            } else {
                valid = valid & validateNodeConfiguration(node, cluster);
                if (!ipAddresses.contains(node.getClusterIp())) {
                    ipAddresses.add(node.getClusterIp());
                }
            }
        }
        valid = valid & validateReplicationProperties(cluster);
    }
    if (valid) {
        String username = cluster.getUsername();
        if (username == null) {
            username = System.getProperty("user.name");
        }
        valid = checkPasswordLessSSHLogin(username, ipAddresses);
    }
    return valid;
}

Example 18 with Cluster

use of org.apache.asterix.event.schema.cluster.Cluster in project asterixdb by apache.

the class StartNodeConfig method execCommand.

@Override
protected void execCommand() throws Exception {
    InstallerDriver.initConfig(true);
    String asterixInstanceName = ((StartNodeConfig) config).name;
    AsterixInstance instance = AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE, State.ACTIVE, State.UNUSABLE);
    Cluster cluster = instance.getCluster();
    List<Pattern> pl = new ArrayList<Pattern>();
    AsterixRuntimeState runtimeState = VerificationUtil.getAsterixRuntimeState(instance);
    String[] nodesToBeAdded = ((StartNodeConfig) config).nodes.split(",");
    List<String> aliveNodes = new ArrayList<String>();
    for (ProcessInfo p : runtimeState.getProcesses()) {
        aliveNodes.add(p.getNodeId());
    }
    List<Node> clusterNodes = cluster.getNode();
    for (String n : nodesToBeAdded) {
        if (aliveNodes.contains(n)) {
            throw new InstallerException("Node: " + n + " is already alive");
        }
        for (Node node : clusterNodes) {
            if (n.equals(node.getId())) {
                String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
                Pattern createNC = PatternCreator.INSTANCE.createNCStartPattern(cluster.getMasterNode().getClusterIp(), node.getId(), asterixInstanceName + "_" + node.getId(), iodevices, false);
                pl.add(createNC);
                break;
            }
        }
    }
    Patterns patterns = new Patterns(pl);
    AsterixEventServiceClient client = AsterixEventService.getAsterixEventServiceClient(cluster);
    client.submit(patterns);
    runtimeState = VerificationUtil.getAsterixRuntimeState(instance);
    VerificationUtil.updateInstanceWithRuntimeDescription(instance, runtimeState, true);
    LOGGER.info(instance.getDescription(false));
    ServiceProvider.INSTANCE.getLookupService().updateAsterixInstance(instance);
}

Example 19 with Cluster

use of org.apache.asterix.event.schema.cluster.Cluster in project asterixdb by apache.

the class ConfigureConfig method configureCluster.

private void configureCluster(String dir, String file) throws JAXBException, PropertyException, FileNotFoundException {
    String clusterDir = InstallerDriver.getManagixHome() + File.separator + "clusters" + File.separator + dir;
    String localClusterPath = clusterDir + File.separator + file;
    if (!Files.exists(Paths.get(localClusterPath))) {
        return;
    }
    Cluster cluster = EventUtil.getCluster(localClusterPath);
    String workingDir = clusterDir + File.separator + "working_dir";
    cluster.setWorkingDir(new WorkingDir(workingDir, true));
    cluster.setIodevices(configureIoDevices(cluster.getIodevices(), workingDir));
    cluster.setLogDir(configureDirectory(cluster.getLogDir(), workingDir));
    cluster.setTxnLogDir(configureDirectory(cluster.getTxnLogDir(), workingDir));
    String javaHome = System.getProperty("java.home");
    if (javaHome.endsWith(File.separator + "jre")) {
        javaHome = javaHome.substring(0, javaHome.indexOf(File.separator + "jre"));
    }
    cluster.setJavaHome(javaHome);
    for (Node node : cluster.getNode()) {
        node.setIodevices(configureIoDevices(node.getIodevices(), workingDir));
        node.setLogDir(configureDirectory(node.getLogDir(), workingDir));
        node.setTxnLogDir(configureDirectory(node.getTxnLogDir(), workingDir));
    }
    JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
    Marshaller marshaller = ctx.createMarshaller();
    marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
    marshaller.marshal(cluster, new FileOutputStream(localClusterPath));
}

Example 20 with Cluster

use of org.apache.asterix.event.schema.cluster.Cluster in project asterixdb by apache.

the class ClusterManager method addNode.

@Override
public void addNode(ICcApplicationContext appCtx, Node node) throws AsterixException {
    try {
        Cluster cluster = ClusterProperties.INSTANCE.getCluster();
        List<Pattern> pattern = new ArrayList<>();
        String asterixInstanceName = appCtx.getMetadataProperties().getInstanceName();
        Patterns prepareNode = PatternCreator.INSTANCE.createPrepareNodePattern(asterixInstanceName, ClusterProperties.INSTANCE.getCluster(), node);
        cluster.getNode().add(node);
        client.submit(prepareNode);
        ExternalProperties externalProps = appCtx.getExternalProperties();
        AsterixEventServiceUtil.poulateClusterEnvironmentProperties(cluster, externalProps.getCCJavaParams(), externalProps.getNCJavaParams());
        pattern.clear();
        String ccHost = cluster.getMasterNode().getClusterIp();
        String hostId = node.getId();
        String nodeControllerId = asterixInstanceName + "_" + node.getId();
        String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
        Pattern startNC = PatternCreator.INSTANCE.createNCStartPattern(ccHost, hostId, nodeControllerId, iodevices, false);
        pattern.add(startNC);
        Patterns startNCPattern = new Patterns(pattern);
        client.submit(startNCPattern);
        removeNode(cluster.getSubstituteNodes().getNode(), node);
        AsterixInstance instance = lookupService.getAsterixInstance(cluster.getInstanceName());
        instance.getCluster().getNode().add(node);
        removeNode(instance.getCluster().getSubstituteNodes().getNode(), node);
        lookupService.updateAsterixInstance(instance);
    } catch (Exception e) {
        throw new AsterixException(e);
    }
}