Examples with ServiceInstance org.apache.curator.x.discovery.ServiceInstance used on opensource projects

Example 16 with ServiceInstance

use of org.apache.curator.x.discovery.ServiceInstance in project drill by axbaretto.

the class ZKClusterCoordinator method updateEndpoints.

private synchronized void updateEndpoints() {
    try {
        // All active bits in the Zookeeper
        Collection<DrillbitEndpoint> newDrillbitSet = transform(discovery.queryForInstances(serviceName), new Function<ServiceInstance<DrillbitEndpoint>, DrillbitEndpoint>() {

            @Override
            public DrillbitEndpoint apply(ServiceInstance<DrillbitEndpoint> input) {
                return input.getPayload();
            }
        });
        // set of newly dead bits : original bits - new set of active bits.
        Set<DrillbitEndpoint> unregisteredBits = new HashSet<>();
        // Set of newly live bits : new set of active bits - original bits.
        Set<DrillbitEndpoint> registeredBits = new HashSet<>();
        // of new drillbit endpoints. Registered endpoints is set to newly live drillbit endpoints.
        for (DrillbitEndpoint endpoint : newDrillbitSet) {
            String endpointAddress = endpoint.getAddress();
            int endpointPort = endpoint.getUserPort();
            if (!endpointsMap.containsKey(new MultiKey(endpointAddress, endpointPort))) {
                registeredBits.add(endpoint);
            }
            endpointsMap.put(new MultiKey(endpointAddress, endpointPort), endpoint);
        }
        // Remove all the endpoints that are newly dead
        for (MultiKey key : endpointsMap.keySet()) {
            if (!newDrillbitSet.contains(endpointsMap.get(key))) {
                unregisteredBits.add(endpointsMap.get(key));
                endpointsMap.remove(key);
            }
        }
        endpoints = endpointsMap.values();
        if (logger.isDebugEnabled()) {
            StringBuilder builder = new StringBuilder();
            builder.append("Active drillbit set changed.  Now includes ");
            builder.append(newDrillbitSet.size());
            builder.append(" total bits. New active drillbits:\n");
            builder.append("Address | User Port | Control Port | Data Port | Version | State\n");
            for (DrillbitEndpoint bit : newDrillbitSet) {
                builder.append(bit.getAddress()).append(" | ");
                builder.append(bit.getUserPort()).append(" | ");
                builder.append(bit.getControlPort()).append(" | ");
                builder.append(bit.getDataPort()).append(" | ");
                builder.append(bit.getVersion()).append(" |");
                builder.append(bit.getState()).append(" | ");
                builder.append('\n');
            }
            logger.debug(builder.toString());
        }
        // Notify listeners of newly unregistered Drillbits.
        if (!unregisteredBits.isEmpty()) {
            drillbitUnregistered(unregisteredBits);
        }
        // Notify listeners of newly registered Drillbits.
        if (!registeredBits.isEmpty()) {
            drillbitRegistered(registeredBits);
        }
    } catch (Exception e) {
        logger.error("Failure while update Drillbit service location cache.", e);
    }
}

Example 17 with ServiceInstance

use of org.apache.curator.x.discovery.ServiceInstance in project drill by axbaretto.

the class ZKClusterCoordinator method updateEndpoints.

private synchronized void updateEndpoints() {
    try {
        Collection<DrillbitEndpoint> newDrillbitSet = transform(discovery.queryForInstances(serviceName), new Function<ServiceInstance<DrillbitEndpoint>, DrillbitEndpoint>() {

            @Override
            public DrillbitEndpoint apply(ServiceInstance<DrillbitEndpoint> input) {
                return input.getPayload();
            }
        });
        // set of newly dead bits : original bits - new set of active bits.
        Set<DrillbitEndpoint> unregisteredBits = new HashSet<>(endpoints);
        unregisteredBits.removeAll(newDrillbitSet);
        // Set of newly live bits : new set of active bits - original bits.
        Set<DrillbitEndpoint> registeredBits = new HashSet<>(newDrillbitSet);
        registeredBits.removeAll(endpoints);
        endpoints = newDrillbitSet;
        if (logger.isDebugEnabled()) {
            StringBuilder builder = new StringBuilder();
            builder.append("Active drillbit set changed.  Now includes ");
            builder.append(newDrillbitSet.size());
            builder.append(" total bits.");
            if (!newDrillbitSet.isEmpty()) {
                builder.append(" New active drillbits: \n");
            }
            for (DrillbitEndpoint bit : newDrillbitSet) {
                builder.append('\t');
                builder.append(bit.getAddress());
                builder.append(':');
                builder.append(bit.getUserPort());
                builder.append(':');
                builder.append(bit.getControlPort());
                builder.append(':');
                builder.append(bit.getDataPort());
                builder.append('\n');
            }
            logger.debug(builder.toString());
        }
        // Notify the drillbit listener for newly unregistered bits.
        if (!(unregisteredBits.isEmpty())) {
            drillbitUnregistered(unregisteredBits);
        }
        // Notify the drillbit listener for newly registered bits.
        if (!(registeredBits.isEmpty())) {
            drillbitRegistered(registeredBits);
        }
    } catch (Exception e) {
        logger.error("Failure while update Drillbit service location cache.", e);
    }
}

Example 18 with ServiceInstance

use of org.apache.curator.x.discovery.ServiceInstance in project oozie by apache.

the class ZKXLogStreamingService method collateLogs.

/**
 * Contacts each of the other Oozie servers, gets their logs for the job, collates them, and sends them to the user via the
 * Writer.  It will make sure to not read all of the log messages into memory at the same time to not use up the heap.  If there
 * is a problem talking to one of the other servers, it will ignore that server and prepend a message to the Writer about it.
 * For getting the logs from this server, it won't use the REST API and instead get them directly to be more efficient.
 *
 * @param logStreamer the XLogStreamer
 * @param startTime the job start time
 * @param endTime the job end time
 * @param writer the writer
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void collateLogs(XLogStreamer logStreamer, Date startTime, Date endTime, Writer writer) throws IOException {
    List<String> badOozies = new ArrayList<String>();
    List<ServiceInstance<Map>> oozies = null;
    try {
        oozies = zk.getAllMetaData();
    } catch (Exception ex) {
        throw new IOException("Issue communicating with ZooKeeper: " + ex.getMessage(), ex);
    }
    List<TimestampedMessageParser> parsers = new ArrayList<TimestampedMessageParser>(oozies.size());
    try {
        // Create a BufferedReader for getting the logs of each server and put them in a TimestampedMessageParser
        for (ServiceInstance<Map> oozie : oozies) {
            Map<String, String> oozieMeta = oozie.getPayload();
            String otherId = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_ID);
            // If it's this server, we can just get them directly
            if (otherId.equals(zk.getZKId())) {
                BufferedReader reader = logStreamer.makeReader(startTime, endTime);
                parsers.add(new TimestampedMessageParser(reader, logStreamer.getXLogFilter()));
            } else // If it's another server, we'll have to use the REST API
            {
                String otherUrl = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_URL);
                String jobId = logStreamer.getXLogFilter().getFilterParams().get(DagXLogInfoService.JOB);
                try {
                    // It's important that we specify ALL_SERVERS_PARAM=false in the GET request to prevent the other Oozie
                    // Server from trying aggregate logs from the other Oozie servers (and creating an infinite recursion)
                    final String url = otherUrl + "/v" + OozieClient.WS_PROTOCOL_VERSION + "/" + RestConstants.JOB + "/" + jobId + "?" + RestConstants.JOB_SHOW_PARAM + "=" + logStreamer.getLogType() + "&" + RestConstants.ALL_SERVER_REQUEST + "=false" + AuthUrlClient.getQueryParamString(logStreamer.getRequestParam());
                    // remove doAs from url to avoid failure while fetching
                    // logs in case of HA mode
                    String key = "doAs";
                    String[] value = null;
                    if (logStreamer.getRequestParam() != null) {
                        value = logStreamer.getRequestParam().get(key);
                    }
                    String urlWithoutdoAs = null;
                    if (value != null && value.length > 0 && value[0] != null && value[0].length() > 0) {
                        urlWithoutdoAs = url.replace("&" + key + "=" + URLEncoder.encode(value[0], "UTF-8"), "");
                    } else {
                        urlWithoutdoAs = url;
                    }
                    BufferedReader reader = AuthUrlClient.callServer(urlWithoutdoAs);
                    parsers.add(new SimpleTimestampedMessageParser(reader, logStreamer.getXLogFilter()));
                } catch (IOException ioe) {
                    log.warn("Failed to retrieve logs for job [" + jobId + "] from Oozie server with ID [" + otherId + "] at [" + otherUrl + "]; log information may be incomplete", ioe);
                    badOozies.add(otherId);
                }
            }
        }
        // If log param debug is set, we need to write start date and end date to outputstream.
        if (!StringUtils.isEmpty(logStreamer.getXLogFilter().getTruncatedMessage())) {
            writer.write(logStreamer.getXLogFilter().getTruncatedMessage());
        }
        if (logStreamer.getXLogFilter().isDebugMode()) {
            writer.write(logStreamer.getXLogFilter().getDebugMessage());
        }
        // Add a message about any servers we couldn't contact
        if (!badOozies.isEmpty()) {
            writer.write("Unable to contact the following Oozie Servers for logs (log information may be incomplete):\n");
            for (String badOozie : badOozies) {
                writer.write("     ");
                writer.write(badOozie);
                writer.write("\n");
            }
            writer.write("\n");
            writer.flush();
        }
        // If it's just the one server (this server), then we don't need to do any more processing and can just copy it directly
        if (parsers.size() == 1) {
            TimestampedMessageParser parser = parsers.get(0);
            parser.processRemaining(writer, logStreamer);
        } else {
            // Now that we have a Reader for each server to get the logs from that server, we have to collate them.  Within each
            // server, the logs should already be in the correct order, so we can take advantage of that.  We'll use the
            // BufferedReaders to read the messages from the logs of each server and put them in order without having to bring
            // every message into memory at the same time.
            TreeMap<String, TimestampedMessageParser> timestampMap = new TreeMap<String, TimestampedMessageParser>();
            // populate timestampMap with initial values
            for (TimestampedMessageParser parser : parsers) {
                if (parser.increment()) {
                    timestampMap.put(parser.getLastTimestamp(), parser);
                }
            }
            while (timestampMap.size() > 1) {
                // The first entry will be the earliest based on the timestamp (also removes it) from the map
                TimestampedMessageParser earliestParser = timestampMap.pollFirstEntry().getValue();
                // Write the message from that parser at that timestamp
                writer.write(earliestParser.getLastMessage());
                if (logStreamer.shouldFlushOutput(earliestParser.getLastMessage().length())) {
                    writer.flush();
                }
                // Increment that parser to read the next message
                if (earliestParser.increment()) {
                    // If it still has messages left, put it back in the map with the new last timestamp for it
                    timestampMap.put(earliestParser.getLastTimestamp(), earliestParser);
                }
            }
            // If there's only one parser left in the map, then we can simply copy the rest of its lines directly to be faster
            if (timestampMap.size() == 1) {
                TimestampedMessageParser parser = timestampMap.values().iterator().next();
                // don't forget the last message read by the parser
                writer.write(parser.getLastMessage());
                parser.processRemaining(writer, logStreamer);
            }
        }
    } finally {
        for (TimestampedMessageParser parser : parsers) {
            parser.closeReader();
        }
    }
}

Example 19 with ServiceInstance

use of org.apache.curator.x.discovery.ServiceInstance in project drill by apache.

the class ZKClusterCoordinator method updateEndpoints.

private synchronized void updateEndpoints() {
    try {
        // All active bits in the Zookeeper
        Collection<DrillbitEndpoint> newDrillbitSet = transform(discovery.queryForInstances(serviceName), new Function<ServiceInstance<DrillbitEndpoint>, DrillbitEndpoint>() {

            @Override
            public DrillbitEndpoint apply(ServiceInstance<DrillbitEndpoint> input) {
                return input.getPayload();
            }
        });
        // set of newly dead bits : original bits - new set of active bits.
        Set<DrillbitEndpoint> unregisteredBits = new HashSet<>();
        // Set of newly live bits : new set of active bits - original bits.
        Set<DrillbitEndpoint> registeredBits = new HashSet<>();
        // of new drillbit endpoints. Registered endpoints is set to newly live drillbit endpoints.
        for (DrillbitEndpoint endpoint : newDrillbitSet) {
            String endpointAddress = endpoint.getAddress();
            int endpointPort = endpoint.getUserPort();
            if (!endpointsMap.containsKey(new MultiKey(endpointAddress, endpointPort))) {
                registeredBits.add(endpoint);
            }
            endpointsMap.put(new MultiKey(endpointAddress, endpointPort), endpoint);
        }
        // Remove all the endpoints that are newly dead
        for (MultiKey key : endpointsMap.keySet()) {
            if (!newDrillbitSet.contains(endpointsMap.get(key))) {
                unregisteredBits.add(endpointsMap.get(key));
                endpointsMap.remove(key);
            }
        }
        endpoints = endpointsMap.values();
        if (logger.isDebugEnabled()) {
            StringBuilder builder = new StringBuilder();
            builder.append("Active drillbit set changed.  Now includes ");
            builder.append(newDrillbitSet.size());
            builder.append(" total bits. New active drillbits:\n");
            builder.append("Address | User Port | Control Port | Data Port | Version | State\n");
            for (DrillbitEndpoint bit : newDrillbitSet) {
                builder.append(bit.getAddress()).append(" | ");
                builder.append(bit.getUserPort()).append(" | ");
                builder.append(bit.getControlPort()).append(" | ");
                builder.append(bit.getDataPort()).append(" | ");
                builder.append(bit.getVersion()).append(" |");
                builder.append(bit.getState()).append(" | ");
                builder.append('\n');
            }
            logger.debug(builder.toString());
        }
        // Notify listeners of newly unregistered Drillbits.
        if (!unregisteredBits.isEmpty()) {
            drillbitUnregistered(unregisteredBits);
        }
        // Notify listeners of newly registered Drillbits.
        if (!registeredBits.isEmpty()) {
            drillbitRegistered(registeredBits);
        }
    } catch (Exception e) {
        logger.error("Failure while update Drillbit service location cache.", e);
    }
}