Examples with WritableDataPoints net.opentsdb.core.WritableDataPoints used on opensource projects

Example 1 with WritableDataPoints

use of net.opentsdb.core.WritableDataPoints in project opentsdb by OpenTSDB.

the class TextImporter method importFile.

private static int importFile(final HBaseClient client, final TSDB tsdb, final String path, final boolean skip_errors) throws IOException {
    final long start_time = System.nanoTime();
    long ping_start_time = start_time;
    final BufferedReader in = open(path);
    String line = null;
    int points = 0;
    try {
        final class Errback implements Callback<Object, Exception> {

            public Object call(final Exception arg) {
                if (arg instanceof PleaseThrottleException) {
                    final PleaseThrottleException e = (PleaseThrottleException) arg;
                    LOG.warn("Need to throttle, HBase isn't keeping up.", e);
                    throttle = true;
                    final HBaseRpc rpc = e.getFailedRpc();
                    if (rpc instanceof PutRequest) {
                        // Don't lose edits.
                        client.put((PutRequest) rpc);
                    }
                    return null;
                }
                LOG.error("Exception caught while processing file " + path, arg);
                System.exit(2);
                return arg;
            }

            public String toString() {
                return "importFile errback";
            }
        }
        ;
        final Errback errback = new Errback();
        LOG.info("reading from file:" + path);
        while ((line = in.readLine()) != null) {
            final String[] words = Tags.splitString(line, ' ');
            final String metric = words[0];
            if (metric.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid metric: " + metric);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid metric: " + metric);
                }
            }
            final long timestamp;
            try {
                timestamp = Tags.parseLong(words[1]);
                if (timestamp <= 0) {
                    if (skip_errors) {
                        LOG.error("invalid timestamp: " + timestamp);
                        LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                        continue;
                    } else {
                        throw new RuntimeException("invalid timestamp: " + timestamp);
                    }
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("invalid timestamp: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
            final String value = words[2];
            if (value.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid value: " + value);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid value: " + value);
                }
            }
            try {
                final HashMap<String, String> tags = new HashMap<String, String>();
                for (int i = 3; i < words.length; i++) {
                    if (!words[i].isEmpty()) {
                        Tags.parse(tags, words[i]);
                    }
                }
                final WritableDataPoints dp = getDataPoints(tsdb, metric, tags);
                Deferred<Object> d;
                if (Tags.looksLikeInteger(value)) {
                    d = dp.addPoint(timestamp, Tags.parseLong(value));
                } else {
                    // floating point value
                    d = dp.addPoint(timestamp, Float.parseFloat(value));
                }
                d.addErrback(errback);
                points++;
                if (points % 1000000 == 0) {
                    final long now = System.nanoTime();
                    ping_start_time = (now - ping_start_time) / 1000000;
                    LOG.info(String.format("... %d data points in %dms (%.1f points/s)", points, ping_start_time, (1000000 * 1000.0 / ping_start_time)));
                    ping_start_time = now;
                }
                if (throttle) {
                    LOG.info("Throttling...");
                    long throttle_time = System.nanoTime();
                    try {
                        d.joinUninterruptibly();
                    } catch (final Exception e) {
                        throw new RuntimeException("Should never happen", e);
                    }
                    throttle_time = System.nanoTime() - throttle_time;
                    if (throttle_time < 1000000000L) {
                        LOG.info("Got throttled for only " + throttle_time + "ns, sleeping a bit now");
                        try {
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            throw new RuntimeException("interrupted", e);
                        }
                    }
                    LOG.info("Done throttling...");
                    throttle = false;
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("Exception: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
        }
    } catch (RuntimeException e) {
        LOG.error("Exception caught while processing file " + path + " line=[" + line + "]", e);
        throw e;
    } finally {
        in.close();
    }
    final long time_delta = (System.nanoTime() - start_time) / 1000000;
    LOG.info(String.format("Processed %s in %d ms, %d data points" + " (%.1f points/s)", path, time_delta, points, (points * 1000.0 / time_delta)));
    return points;
}

Example 2 with WritableDataPoints

use of net.opentsdb.core.WritableDataPoints in project opentsdb by OpenTSDB.

the class TextImporter method getDataPoints.

private static WritableDataPoints getDataPoints(final TSDB tsdb, final String metric, final HashMap<String, String> tags) {
    final String key = metric + tags;
    WritableDataPoints dp = datapoints.get(key);
    if (dp != null) {
        return dp;
    }
    dp = tsdb.newDataPoints();
    dp.setSeries(metric, tags);
    dp.setBatchImport(true);
    datapoints.put(key, dp);
    return dp;
}

Example 3 with WritableDataPoints

use of net.opentsdb.core.WritableDataPoints in project opentsdb by OpenTSDB.

the class TestTextImporter method before.

@Before
public void before() throws Exception {
    config = new Config(false);
    tsdb = new TSDB(client, config);
    storage = new MockBase(tsdb, client, true, true, true, true);
    storage.setFamily("t".getBytes(MockBase.ASCII()));
    // replace the "real" field objects with mocks
    Field met = tsdb.getClass().getDeclaredField("metrics");
    met.setAccessible(true);
    met.set(tsdb, metrics);
    Field tagk = tsdb.getClass().getDeclaredField("tag_names");
    tagk.setAccessible(true);
    tagk.set(tsdb, tag_names);
    Field tagv = tsdb.getClass().getDeclaredField("tag_values");
    tagv.setAccessible(true);
    tagv.set(tsdb, tag_values);
    PowerMockito.spy(TextImporter.class);
    // we need to purge the hash map before each unit test since it's a static
    // field
    datapoints.set(null, new HashMap<String, WritableDataPoints>());
    // mock UniqueId
    when(metrics.getId("sys.cpu.user")).thenReturn(new byte[] { 0, 0, 1 });
    when(metrics.getNameAsync(new byte[] { 0, 0, 1 })).thenReturn(Deferred.fromResult("sys.cpu.user"));
    when(metrics.getId("sys.cpu.system")).thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric"));
    when(metrics.getOrCreateId("sys.cpu.system")).thenThrow(new NoSuchUniqueName("sys.cpu.system", "metric"));
    when(metrics.getId("sys.cpu.nice")).thenReturn(new byte[] { 0, 0, 2 });
    when(metrics.getName(new byte[] { 0, 0, 2 })).thenReturn("sys.cpu.nice");
    when(tag_names.getId("host")).thenReturn(new byte[] { 0, 0, 1 });
    when(tag_names.getName(new byte[] { 0, 0, 1 })).thenReturn("host");
    when(tag_names.getOrCreateId("host")).thenReturn(new byte[] { 0, 0, 1 });
    when(tag_names.getId("fqdn")).thenThrow(new NoSuchUniqueName("dc", "tagk"));
    when(tag_names.getOrCreateId("fqdn")).thenThrow(new NoSuchUniqueName("dc", "tagk"));
    when(tag_values.getId("web01")).thenReturn(new byte[] { 0, 0, 1 });
    when(tag_values.getName(new byte[] { 0, 0, 1 })).thenReturn("web01");
    when(tag_values.getOrCreateId("web01")).thenReturn(new byte[] { 0, 0, 1 });
    when(tag_values.getId("web02")).thenReturn(new byte[] { 0, 0, 2 });
    when(tag_values.getName(new byte[] { 0, 0, 2 })).thenReturn("web02");
    when(tag_values.getOrCreateId("web02")).thenReturn(new byte[] { 0, 0, 2 });
    when(tag_values.getId("web03")).thenThrow(new NoSuchUniqueName("web03", "tagv"));
    when(tag_values.getOrCreateId("web03")).thenThrow(new NoSuchUniqueName("web03", "tagv"));
    when(metrics.width()).thenReturn((short) 3);
    when(tag_names.width()).thenReturn((short) 3);
    when(tag_values.width()).thenReturn((short) 3);
}