Examples with HBaseRpc org.hbase.async.HBaseRpc used on opensource projects

Example 1 with HBaseRpc

use of org.hbase.async.HBaseRpc in project opentsdb by OpenTSDB.

the class TextImporter method importFile.

private static int importFile(final HBaseClient client, final TSDB tsdb, final String path, final boolean skip_errors) throws IOException {
    final long start_time = System.nanoTime();
    long ping_start_time = start_time;
    final BufferedReader in = open(path);
    String line = null;
    int points = 0;
    try {
        final class Errback implements Callback<Object, Exception> {

            public Object call(final Exception arg) {
                if (arg instanceof PleaseThrottleException) {
                    final PleaseThrottleException e = (PleaseThrottleException) arg;
                    LOG.warn("Need to throttle, HBase isn't keeping up.", e);
                    throttle = true;
                    final HBaseRpc rpc = e.getFailedRpc();
                    if (rpc instanceof PutRequest) {
                        // Don't lose edits.
                        client.put((PutRequest) rpc);
                    }
                    return null;
                }
                LOG.error("Exception caught while processing file " + path, arg);
                System.exit(2);
                return arg;
            }

            public String toString() {
                return "importFile errback";
            }
        }
        ;
        final Errback errback = new Errback();
        LOG.info("reading from file:" + path);
        while ((line = in.readLine()) != null) {
            final String[] words = Tags.splitString(line, ' ');
            final String metric = words[0];
            if (metric.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid metric: " + metric);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid metric: " + metric);
                }
            }
            final long timestamp;
            try {
                timestamp = Tags.parseLong(words[1]);
                if (timestamp <= 0) {
                    if (skip_errors) {
                        LOG.error("invalid timestamp: " + timestamp);
                        LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                        continue;
                    } else {
                        throw new RuntimeException("invalid timestamp: " + timestamp);
                    }
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("invalid timestamp: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
            final String value = words[2];
            if (value.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid value: " + value);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid value: " + value);
                }
            }
            try {
                final HashMap<String, String> tags = new HashMap<String, String>();
                for (int i = 3; i < words.length; i++) {
                    if (!words[i].isEmpty()) {
                        Tags.parse(tags, words[i]);
                    }
                }
                final WritableDataPoints dp = getDataPoints(tsdb, metric, tags);
                Deferred<Object> d;
                if (Tags.looksLikeInteger(value)) {
                    d = dp.addPoint(timestamp, Tags.parseLong(value));
                } else {
                    // floating point value
                    d = dp.addPoint(timestamp, Float.parseFloat(value));
                }
                d.addErrback(errback);
                points++;
                if (points % 1000000 == 0) {
                    final long now = System.nanoTime();
                    ping_start_time = (now - ping_start_time) / 1000000;
                    LOG.info(String.format("... %d data points in %dms (%.1f points/s)", points, ping_start_time, (1000000 * 1000.0 / ping_start_time)));
                    ping_start_time = now;
                }
                if (throttle) {
                    LOG.info("Throttling...");
                    long throttle_time = System.nanoTime();
                    try {
                        d.joinUninterruptibly();
                    } catch (final Exception e) {
                        throw new RuntimeException("Should never happen", e);
                    }
                    throttle_time = System.nanoTime() - throttle_time;
                    if (throttle_time < 1000000000L) {
                        LOG.info("Got throttled for only " + throttle_time + "ns, sleeping a bit now");
                        try {
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            throw new RuntimeException("interrupted", e);
                        }
                    }
                    LOG.info("Done throttling...");
                    throttle = false;
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("Exception: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
        }
    } catch (RuntimeException e) {
        LOG.error("Exception caught while processing file " + path + " line=[" + line + "]", e);
        throw e;
    } finally {
        in.close();
    }
    final long time_delta = (System.nanoTime() - start_time) / 1000000;
    LOG.info(String.format("Processed %s in %d ms, %d data points" + " (%.1f points/s)", path, time_delta, points, (points * 1000.0 / time_delta)));
    return points;
}