Examples with ServiceRecord org.apache.hadoop.registry.client.types.ServiceRecord used on opensource projects

Example 26 with ServiceRecord

use of org.apache.hadoop.registry.client.types.ServiceRecord in project jstorm by alibaba.

the class JstormMaster method finish.

protected boolean finish() {
    // wait for completion.
    String appPath;
    while (!jstormMasterContext.done) {
        try {
            Thread.sleep(JOYConstants.HEARTBEAT_TIME_INTERVAL);
            appPath = RegistryUtils.servicePath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName, jstormMasterContext.appAttemptID.getApplicationId().toString());
            ServiceRecord app = new ServiceRecord();
            Date now = new Date();
            app.set(JOYConstants.APP_HEARTBEAT_TIME, String.valueOf(now.getTime()));
            registryOperations.bind(appPath, app, BindFlags.OVERWRITE);
        } catch (Exception ex) {
            LOG.error(ex);
        }
    }
    if (timelineClient != null) {
        publishApplicationAttemptEvent(timelineClient, jstormMasterContext.appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END, jstormMasterContext.domainId, appSubmitterUgi);
    }
    appPath = RegistryUtils.servicePath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName, jstormMasterContext.appAttemptID.getApplicationId().toString());
    try {
        registryOperations.delete(appPath, true);
        LOG.info("unRegister application' appPath:" + appPath);
    } catch (IOException e) {
        LOG.error("Failed to unRegister application's Registry", e);
    }
    // Join all launched threads
    for (Thread launchThread : launchThreads) {
        try {
            launchThread.join(JOYConstants.JOIN_THREAD_TIMEOUT);
        } catch (InterruptedException e) {
            LOG.info("Exception thrown in thread join: " + e.getMessage());
            e.printStackTrace();
        }
    }
    // When the application completes, it should stop all running containers
    LOG.info("Application completed. Stopping running containers");
    nmClientAsync.stop();
    // When the application completes, it should send a finish application
    // signal to the RM
    LOG.info("Application completed. Signalling finish to RM");
    FinalApplicationStatus appStatus;
    String appMessage = null;
    boolean success = true;
    if (jstormMasterContext.numFailedContainers.get() == 0 && jstormMasterContext.numCompletedContainers.get() == jstormMasterContext.numTotalContainers) {
        appStatus = FinalApplicationStatus.SUCCEEDED;
    } else {
        appStatus = FinalApplicationStatus.FAILED;
        appMessage = "Diagnostics." + ", total=" + jstormMasterContext.numTotalContainers + ", completed=" + jstormMasterContext.numCompletedContainers.get() + ", allocated=" + jstormMasterContext.numAllocatedContainers.get() + ", failed=" + jstormMasterContext.numFailedContainers.get();
        LOG.info(appMessage);
        success = false;
    }
    try {
        amRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
    } catch (YarnException ex) {
        LOG.error("Failed to unregister application", ex);
    } catch (IOException e) {
        LOG.error("Failed to unregister application", e);
    }
    amRMClient.stop();
    // Stop Timeline Client
    if (timelineClient != null) {
        timelineClient.stop();
    }
    return success;
}

Example 27 with ServiceRecord

use of org.apache.hadoop.registry.client.types.ServiceRecord in project jstorm by alibaba.

the class ContainersView method tryHostLock.

/**
     * see if anyone is updating host's port list, if not start , update this host itself
     * timeout is 45 seconds
     *
     * @throws InterruptedException
     * @throws IOException
     */
private void tryHostLock() throws Exception {
    String hostPath = getHostPath();
    //if path has created 60 seconds ago, then delete
    if (registryOperations.exists(hostPath)) {
        try {
            ServiceRecord host = registryOperations.resolve(hostPath);
            Long cTime = Long.parseLong(host.get("cTime", "0"));
            Date now = new Date();
            if (now.getTime() - cTime > 60 * 1000 || cTime > now.getTime())
                registryOperations.delete(hostPath, true);
        } catch (Exception ex) {
            LOG.error(ex);
        //                registryOperations.delete(hostPath, true);
        }
    }
    int failedCount = 45;
    while (!registryOperations.mknode(hostPath, true)) {
        Thread.sleep(1000);
        failedCount--;
        if (failedCount <= 0)
            break;
    }
    if (failedCount > 0) {
        ServiceRecord sr = new ServiceRecord();
        Date date = new Date();
        date.getTime();
        sr.set("cTime", String.valueOf(date.getTime()));
        registryOperations.bind(hostPath, sr, BindFlags.OVERWRITE);
        return;
    }
    throw new Exception("can't get host lock");
}

Example 28 with ServiceRecord

use of org.apache.hadoop.registry.client.types.ServiceRecord in project jstorm by alibaba.

the class SlotPortsView method getSetPortUsedBySupervisor.

/**
     * see if port is using by supervisor
     * if used return true, if not, add this port to registry and return false
     *
     * @param supervisorHost
     * @param slotCount
     * @return
     */
public List<String> getSetPortUsedBySupervisor(String supervisorHost, int slotCount) throws Exception {
    String appPath = RegistryUtils.serviceclassPath(JOYConstants.APP_TYPE, JOYConstants.EMPTY);
    String path = RegistryUtils.serviceclassPath(JOYConstants.APP_TYPE, instanceName);
    String containerPath = RegistryUtils.componentPath(JOYConstants.APP_TYPE, instanceName, containerId.getApplicationAttemptId().getApplicationId().toString(), containerId.toString());
    List<String> reList = new ArrayList<String>();
    try {
        List<ServiceRecord> hostContainers = new ArrayList<ServiceRecord>();
        Set<String> hostUsedPorts = new HashSet<String>();
        List<String> instanceNames = registryOperations.list(appPath);
        for (String instance : instanceNames) {
            String servicePath = RegistryUtils.serviceclassPath(JOYConstants.APP_TYPE, instance);
            List<String> apps = registryOperations.list(servicePath);
            for (String subapp : apps) {
                String subAppPath = RegistryUtils.servicePath(JOYConstants.APP_TYPE, instance, subapp);
                String componentsPath = subAppPath + JOYConstants.COMPONENTS;
                if (!registryOperations.exists(componentsPath))
                    continue;
                Map<String, ServiceRecord> containers = RegistryUtils.listServiceRecords(registryOperations, componentsPath);
                for (String container : containers.keySet()) {
                    ServiceRecord sr = containers.get(container);
                    LOG.info(sr.toString());
                    if (!sr.get(JOYConstants.HOST).equals(supervisorHost))
                        continue;
                    hostContainers.add(sr);
                    String[] portList = new String[] {};
                    if (sr.get(JOYConstants.PORT_LIST) != null)
                        portList = sr.get(JOYConstants.PORT_LIST).split(JOYConstants.COMMA);
                    for (String usedport : portList) {
                        hostUsedPorts.add(usedport);
                    }
                }
            }
        }
        //scan port range from 9000 to 15000
        for (int i = getMinPort(); i < getMaxPort(); i++) {
            if (JstormYarnUtils.isPortAvailable(supervisorHost, i)) {
                if (!hostUsedPorts.contains(String.valueOf(i)))
                    reList.add(String.valueOf(i));
            }
            if (reList.size() >= slotCount) {
                break;
            }
        }
        if (registryOperations.exists(containerPath)) {
            ServiceRecord sr = registryOperations.resolve(containerPath);
            String portListUpdate = JstormYarnUtils.join(reList, JOYConstants.COMMA, false);
            if (sr.get(JOYConstants.PORT_LIST) != null) {
                String[] portList = sr.get(JOYConstants.PORT_LIST).split(JOYConstants.COMMA);
                portListUpdate = JstormYarnUtils.join(portList, JOYConstants.COMMA, true) + JstormYarnUtils.join(reList, JOYConstants.COMMA, false);
            }
            sr.set(JOYConstants.PORT_LIST, portListUpdate);
            registryOperations.bind(containerPath, sr, BindFlags.OVERWRITE);
        } else {
            registryOperations.mknode(containerPath, true);
            ServiceRecord sr = new ServiceRecord();
            sr.set(JOYConstants.HOST, supervisorHost);
            String portListUpdate = JstormYarnUtils.join(reList, JOYConstants.COMMA, false);
            sr.set(JOYConstants.PORT_LIST, portListUpdate);
            sr.set(YarnRegistryAttributes.YARN_ID, containerId.toString());
            sr.description = JOYConstants.CONTAINER;
            sr.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
            registryOperations.bind(containerPath, sr, BindFlags.OVERWRITE);
        }
        return reList;
    } catch (Exception ex) {
        LOG.error(ex);
        throw ex;
    }
}

Example 29 with ServiceRecord

use of org.apache.hadoop.registry.client.types.ServiceRecord in project hadoop by apache.

the class RegistryAdminService method purge.

/**
   * Recursive operation to purge all matching records under a base path.
   * <ol>
   *   <li>Uses a depth first search</li>
   *   <li>A match is on ID and persistence policy, or, if policy==-1, any match</li>
   *   <li>If a record matches then it is deleted without any child searches</li>
   *   <li>Deletions will be asynchronous if a callback is provided</li>
   * </ol>
   *
   * The code is designed to be robust against parallel deletions taking place;
   * in such a case it will stop attempting that part of the tree. This
   * avoid the situation of more than 1 purge happening in parallel and
   * one of the purge operations deleteing the node tree above the other.
   * @param path base path
   * @param selector selector for the purge policy
   * @param purgePolicy what to do if there is a matching record with children
   * @param callback optional curator callback
   * @return the number of delete operations perfomed. As deletes may be for
   * everything under a path, this may be less than the number of records
   * actually deleted
   * @throws IOException problems
   * @throws PathIsNotEmptyDirectoryException if an entry cannot be deleted
   * as it has children and the purge policy is FailOnChildren
   */
@VisibleForTesting
public int purge(String path, NodeSelector selector, PurgePolicy purgePolicy, BackgroundCallback callback) throws IOException {
    boolean toDelete = false;
    // look at self to see if it has a service record
    Map<String, RegistryPathStatus> childEntries;
    Collection<RegistryPathStatus> entries;
    try {
        // list this path's children
        childEntries = RegistryUtils.statChildren(this, path);
        entries = childEntries.values();
    } catch (PathNotFoundException e) {
        // exit
        return 0;
    }
    try {
        RegistryPathStatus registryPathStatus = stat(path);
        ServiceRecord serviceRecord = resolve(path);
        // there is now an entry here.
        toDelete = selector.shouldSelect(path, registryPathStatus, serviceRecord);
    } catch (EOFException ignored) {
    // ignore
    } catch (InvalidRecordException ignored) {
    // ignore
    } catch (NoRecordException ignored) {
    // ignore
    } catch (PathNotFoundException e) {
        // exit
        return 0;
    }
    if (toDelete && !entries.isEmpty()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Match on record @ {} with children ", path);
        }
        // there's children
        switch(purgePolicy) {
            case SkipOnChildren:
                // don't do the deletion... continue to next record
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Skipping deletion");
                }
                toDelete = false;
                break;
            case PurgeAll:
                // mark for deletion
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Scheduling for deletion with children");
                }
                toDelete = true;
                entries = new ArrayList<RegistryPathStatus>(0);
                break;
            case FailOnChildren:
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Failing deletion operation");
                }
                throw new PathIsNotEmptyDirectoryException(path);
        }
    }
    int deleteOps = 0;
    if (toDelete) {
        try {
            zkDelete(path, true, callback);
        } catch (PathNotFoundException e) {
            // this is a no-op, and all children can be skipped
            return deleteOps;
        }
        deleteOps++;
    }
    // now go through the children
    for (RegistryPathStatus status : entries) {
        String childname = status.path;
        String childpath = RegistryPathUtils.join(path, childname);
        deleteOps += purge(childpath, selector, purgePolicy, callback);
    }
    return deleteOps;
}

Example 30 with ServiceRecord

use of org.apache.hadoop.registry.client.types.ServiceRecord in project hadoop by apache.

the class RegistryCli method bind.

public int bind(String[] args) {
    Option rest = OptionBuilder.withArgName("rest").hasArg().withDescription("rest Option").create("rest");
    Option webui = OptionBuilder.withArgName("webui").hasArg().withDescription("webui Option").create("webui");
    Option inet = OptionBuilder.withArgName("inet").withDescription("inet Option").create("inet");
    Option port = OptionBuilder.withArgName("port").hasArg().withDescription("port to listen on [9999]").create("p");
    Option host = OptionBuilder.withArgName("host").hasArg().withDescription("host name").create("h");
    Option apiOpt = OptionBuilder.withArgName("api").hasArg().withDescription("api").create("api");
    Options inetOption = new Options();
    inetOption.addOption(inet);
    inetOption.addOption(port);
    inetOption.addOption(host);
    inetOption.addOption(apiOpt);
    Options webuiOpt = new Options();
    webuiOpt.addOption(webui);
    webuiOpt.addOption(apiOpt);
    Options restOpt = new Options();
    restOpt.addOption(rest);
    restOpt.addOption(apiOpt);
    CommandLineParser parser = new GnuParser();
    ServiceRecord sr = new ServiceRecord();
    CommandLine line;
    if (args.length <= 1) {
        return usageError("Invalid syntax ", BIND_USAGE);
    }
    if (args[1].equals("-inet")) {
        int portNum;
        String hostName;
        String api;
        try {
            line = parser.parse(inetOption, args);
        } catch (ParseException exp) {
            return usageError("Invalid syntax " + exp.getMessage(), BIND_USAGE);
        }
        if (line.hasOption("inet") && line.hasOption("p") && line.hasOption("h") && line.hasOption("api")) {
            try {
                portNum = Integer.parseInt(line.getOptionValue("p"));
            } catch (NumberFormatException exp) {
                return usageError("Invalid Port - int required" + exp.getMessage(), BIND_USAGE);
            }
            hostName = line.getOptionValue("h");
            api = line.getOptionValue("api");
            sr.addExternalEndpoint(inetAddrEndpoint(api, ProtocolTypes.PROTOCOL_HADOOP_IPC, hostName, portNum));
        } else {
            return usageError("Missing options: must have host, port and api", BIND_USAGE);
        }
    } else if (args[1].equals("-webui")) {
        try {
            line = parser.parse(webuiOpt, args);
        } catch (ParseException exp) {
            return usageError("Invalid syntax " + exp.getMessage(), BIND_USAGE);
        }
        if (line.hasOption("webui") && line.hasOption("api")) {
            URI theUri;
            try {
                theUri = new URI(line.getOptionValue("webui"));
            } catch (URISyntaxException e) {
                return usageError("Invalid URI: " + e.getMessage(), BIND_USAGE);
            }
            sr.addExternalEndpoint(webEndpoint(line.getOptionValue("api"), theUri));
        } else {
            return usageError("Missing options: must have value for uri and api", BIND_USAGE);
        }
    } else if (args[1].equals("-rest")) {
        try {
            line = parser.parse(restOpt, args);
        } catch (ParseException exp) {
            return usageError("Invalid syntax " + exp.getMessage(), BIND_USAGE);
        }
        if (line.hasOption("rest") && line.hasOption("api")) {
            URI theUri = null;
            try {
                theUri = new URI(line.getOptionValue("rest"));
            } catch (URISyntaxException e) {
                return usageError("Invalid URI: " + e.getMessage(), BIND_USAGE);
            }
            sr.addExternalEndpoint(restEndpoint(line.getOptionValue("api"), theUri));
        } else {
            return usageError("Missing options: must have value for uri and api", BIND_USAGE);
        }
    } else {
        return usageError("Invalid syntax", BIND_USAGE);
    }
    @SuppressWarnings("unchecked") List<String> argsList = line.getArgList();
    if (argsList.size() != 2) {
        return usageError("bind requires exactly one path argument", BIND_USAGE);
    }
    if (!validatePath(argsList.get(1))) {
        return -1;
    }
    try {
        registry.bind(argsList.get(1), sr, BindFlags.OVERWRITE);
        return 0;
    } catch (Exception e) {
        syserr.println(analyzeException("bind", e, argsList));
    }
    return -1;
}