Examples with UIDMeta net.opentsdb.meta.UIDMeta used on opensource projects

Example 1 with UIDMeta

use of net.opentsdb.meta.UIDMeta in project opentsdb by OpenTSDB.

the class MetaSync method run.

/**
   * Loops through the entire TSDB data set and exits when complete.
   */
public void run() {
    // list of deferred calls used to act as a buffer
    final ArrayList<Deferred<Boolean>> storage_calls = new ArrayList<Deferred<Boolean>>();
    final Deferred<Object> result = new Deferred<Object>();
    final class ErrBack implements Callback<Object, Exception> {

        @Override
        public Object call(Exception e) throws Exception {
            Throwable ex = e;
            while (ex.getClass().equals(DeferredGroupException.class)) {
                if (ex.getCause() == null) {
                    LOG.warn("Unable to get to the root cause of the DGE");
                    break;
                }
                ex = ex.getCause();
            }
            LOG.error("Sync thread failed with exception", ex);
            result.callback(null);
            return null;
        }
    }
    final ErrBack err_back = new ErrBack();
    /**
     * Called when we have encountered a previously un-processed UIDMeta object.
     * This callback will update the "created" timestamp of the UIDMeta and
     * store the update, replace corrupted metas and update search plugins.
     */
    final class UidCB implements Callback<Deferred<Boolean>, UIDMeta> {

        private final UniqueIdType type;

        private final byte[] uid;

        private final long timestamp;

        /**
       * Constructor that initializes the local callback
       * @param type The type of UIDMeta we're dealing with
       * @param uid The UID of the meta object as a byte array
       * @param timestamp The timestamp of the timeseries when this meta
       * was first detected
       */
        public UidCB(final UniqueIdType type, final byte[] uid, final long timestamp) {
            this.type = type;
            this.uid = uid;
            this.timestamp = timestamp;
        }

        /**
       * A nested class called after fetching a UID name to use when creating a
       * new UIDMeta object if the previous object was corrupted. Also pushes
       * the meta off to the search plugin.
       */
        final class UidNameCB implements Callback<Deferred<Boolean>, String> {

            @Override
            public Deferred<Boolean> call(final String name) throws Exception {
                UIDMeta new_meta = new UIDMeta(type, uid, name);
                new_meta.setCreated(timestamp);
                tsdb.indexUIDMeta(new_meta);
                LOG.info("Replacing corrupt UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "]");
                return new_meta.syncToStorage(tsdb, true);
            }
        }

        @Override
        public Deferred<Boolean> call(final UIDMeta meta) throws Exception {
            // otherwise it's probably an accurate timestamp
            if (meta.getCreated() > (timestamp + 3600) || meta.getCreated() == 0) {
                LOG.info("Updating UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "]");
                meta.setCreated(timestamp);
                // consider it corrupt and replace it with a new object
                if (meta.getUID() == null || meta.getUID().isEmpty() || meta.getType() == null) {
                    return tsdb.getUidName(type, uid).addCallbackDeferring(new UidNameCB());
                } else {
                    // the meta was good, just needed a timestamp update so sync to
                    // search and storage
                    tsdb.indexUIDMeta(meta);
                    LOG.info("Syncing valid UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "]");
                    return meta.syncToStorage(tsdb, false);
                }
            } else {
                LOG.debug("UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "] is up to date in storage");
                return Deferred.fromResult(true);
            }
        }
    }
    /**
     * Called to handle a previously unprocessed TSMeta object. This callback
     * will update the "created" timestamp, create a new TSMeta object if
     * missing, and update search plugins.
     */
    final class TSMetaCB implements Callback<Deferred<Boolean>, TSMeta> {

        private final String tsuid_string;

        private final byte[] tsuid;

        private final long timestamp;

        /**
       * Default constructor
       * @param tsuid ID of the timeseries
       * @param timestamp The timestamp when the first data point was recorded
       */
        public TSMetaCB(final byte[] tsuid, final long timestamp) {
            this.tsuid = tsuid;
            tsuid_string = UniqueId.uidToString(tsuid);
            this.timestamp = timestamp;
        }

        @Override
        public Deferred<Boolean> call(final TSMeta meta) throws Exception {
            /** Called to process the new meta through the search plugin and tree code */
            final class IndexCB implements Callback<Deferred<Boolean>, TSMeta> {

                @Override
                public Deferred<Boolean> call(final TSMeta new_meta) throws Exception {
                    tsdb.indexTSMeta(new_meta);
                    // pass through the trees
                    return tsdb.processTSMetaThroughTrees(new_meta);
                }
            }
            /** Called to load the newly created meta object for passage onto the
         * search plugin and tree builder if configured
         */
            final class GetCB implements Callback<Deferred<Boolean>, Boolean> {

                @Override
                public final Deferred<Boolean> call(final Boolean exists) throws Exception {
                    if (exists) {
                        return TSMeta.getTSMeta(tsdb, tsuid_string).addCallbackDeferring(new IndexCB());
                    } else {
                        return Deferred.fromResult(false);
                    }
                }
            }
            /** Errback on the store new call to catch issues */
            class ErrBack implements Callback<Object, Exception> {

                public Object call(final Exception e) throws Exception {
                    LOG.warn("Failed creating meta for: " + tsuid + " with exception: ", e);
                    return null;
                }
            }
            // new one
            if (meta == null) {
                /**
           * Called after successfully creating a TSMeta counter and object,
           * used to convert the deferred long to a boolean so it can be
           * combined with other calls for waiting.
           */
                final class CreatedCB implements Callback<Deferred<Boolean>, Long> {

                    @Override
                    public Deferred<Boolean> call(Long value) throws Exception {
                        LOG.info("Created counter and meta for timeseries [" + tsuid_string + "]");
                        return Deferred.fromResult(true);
                    }
                }
                /**
           * Called after checking to see if the counter exists and is used
           * to determine if we should create a new counter AND meta or just a
           * new meta
           */
                final class CounterCB implements Callback<Deferred<Boolean>, Boolean> {

                    @Override
                    public Deferred<Boolean> call(final Boolean exists) throws Exception {
                        if (!exists) {
                            // here or in the local callback
                            return TSMeta.incrementAndGetCounter(tsdb, tsuid).addCallbackDeferring(new CreatedCB());
                        } else {
                            TSMeta new_meta = new TSMeta(tsuid, timestamp);
                            tsdb.indexTSMeta(new_meta);
                            LOG.info("Counter exists but meta was null, creating meta data " + "for timeseries [" + tsuid_string + "]");
                            return new_meta.storeNew(tsdb).addCallbackDeferring(new GetCB()).addErrback(new ErrBack());
                        }
                    }
                }
                // improperly before the meta is flushed to storage.
                return TSMeta.counterExistsInStorage(tsdb, tsuid).addCallbackDeferring(new CounterCB());
            }
            // corrupted
            if (meta.getTSUID() == null || meta.getTSUID().isEmpty()) {
                LOG.warn("Replacing corrupt meta data for timeseries [" + tsuid_string + "]");
                TSMeta new_meta = new TSMeta(tsuid, timestamp);
                tsdb.indexTSMeta(new_meta);
                return new_meta.storeNew(tsdb).addCallbackDeferring(new GetCB()).addErrback(new ErrBack());
            } else {
                // hour otherwise it's probably an accurate timestamp
                if (meta.getCreated() > (timestamp + 3600) || meta.getCreated() == 0) {
                    meta.setCreated(timestamp);
                    tsdb.indexTSMeta(meta);
                    LOG.info("Updated created timestamp for timeseries [" + tsuid_string + "]");
                    return meta.syncToStorage(tsdb, false);
                }
                LOG.debug("TSUID [" + tsuid_string + "] is up to date in storage");
                return Deferred.fromResult(false);
            }
        }
    }
    /**
     * Scanner callback that recursively loops through all of the data point
     * rows. Note that we don't process the actual data points, just the row
     * keys.
     */
    final class MetaScanner implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

        private byte[] last_tsuid = null;

        private String tsuid_string = "";

        /**
       * Fetches the next set of rows from the scanner and adds this class as
       * a callback
       * @return A meaningless deferred to wait on until all data rows have
       * been processed.
       */
        public Object scan() {
            return scanner.nextRows().addCallback(this).addErrback(err_back);
        }

        @Override
        public Object call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                result.callback(null);
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                try {
                    final byte[] tsuid = UniqueId.getTSUIDFromKey(row.get(0).key(), TSDB.metrics_width(), Const.TIMESTAMP_BYTES);
                    // so we save time
                    if (last_tsuid != null && Arrays.equals(last_tsuid, tsuid)) {
                        continue;
                    }
                    last_tsuid = tsuid;
                    // see if we've already processed this tsuid and if so, continue
                    if (processed_tsuids.contains(Arrays.hashCode(tsuid))) {
                        continue;
                    }
                    tsuid_string = UniqueId.uidToString(tsuid);
                    /**
             * An error callback used to catch issues with a particular timeseries
             * or UIDMeta such as a missing UID name. We want to continue
             * processing when this happens so we'll just log the error and
             * the user can issue a command later to clean up orphaned meta
             * entries.
             */
                    final class RowErrBack implements Callback<Object, Exception> {

                        @Override
                        public Object call(Exception e) throws Exception {
                            Throwable ex = e;
                            while (ex.getClass().equals(DeferredGroupException.class)) {
                                if (ex.getCause() == null) {
                                    LOG.warn("Unable to get to the root cause of the DGE");
                                    break;
                                }
                                ex = ex.getCause();
                            }
                            if (ex.getClass().equals(IllegalStateException.class)) {
                                LOG.error("Invalid data when processing TSUID [" + tsuid_string + "]: " + ex.getMessage());
                            } else if (ex.getClass().equals(IllegalArgumentException.class)) {
                                LOG.error("Invalid data when processing TSUID [" + tsuid_string + "]: " + ex.getMessage());
                            } else if (ex.getClass().equals(NoSuchUniqueId.class)) {
                                LOG.warn("Timeseries [" + tsuid_string + "] includes a non-existant UID: " + ex.getMessage());
                            } else {
                                LOG.error("Unknown exception processing row: " + row, ex);
                            }
                            return null;
                        }
                    }
                    // add tsuid to the processed list
                    processed_tsuids.add(Arrays.hashCode(tsuid));
                    // we may have a new TSUID or UIDs, so fetch the timestamp of the 
                    // row for use as the "created" time. Depending on speed we could 
                    // parse datapoints, but for now the hourly row time is enough
                    final long timestamp = Bytes.getUnsignedInt(row.get(0).key(), Const.SALT_WIDTH() + TSDB.metrics_width());
                    LOG.debug("[" + thread_id + "] Processing TSUID: " + tsuid_string + "  row timestamp: " + timestamp);
                    // now process the UID metric meta data
                    final byte[] metric_uid_bytes = Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width());
                    final String metric_uid = UniqueId.uidToString(metric_uid_bytes);
                    Long last_get = metric_uids.get(metric_uid);
                    if (last_get == null || last_get == 0 || timestamp < last_get) {
                        // fetch and update. Returns default object if the meta doesn't
                        // exist, so we can just call sync on this to create a missing
                        // entry
                        final UidCB cb = new UidCB(UniqueIdType.METRIC, metric_uid_bytes, timestamp);
                        final Deferred<Boolean> process_uid = UIDMeta.getUIDMeta(tsdb, UniqueIdType.METRIC, metric_uid_bytes).addCallbackDeferring(cb).addErrback(new RowErrBack());
                        storage_calls.add(process_uid);
                        metric_uids.put(metric_uid, timestamp);
                    }
                    // loop through the tags and process their meta
                    final List<byte[]> tags = UniqueId.getTagsFromTSUID(tsuid_string);
                    int idx = 0;
                    for (byte[] tag : tags) {
                        final UniqueIdType type = (idx % 2 == 0) ? UniqueIdType.TAGK : UniqueIdType.TAGV;
                        idx++;
                        final String uid = UniqueId.uidToString(tag);
                        // check the maps to see if we need to bother updating
                        if (type == UniqueIdType.TAGK) {
                            last_get = tagk_uids.get(uid);
                        } else {
                            last_get = tagv_uids.get(uid);
                        }
                        if (last_get != null && last_get != 0 && last_get <= timestamp) {
                            continue;
                        }
                        // fetch and update. Returns default object if the meta doesn't
                        // exist, so we can just call sync on this to create a missing
                        // entry
                        final UidCB cb = new UidCB(type, tag, timestamp);
                        final Deferred<Boolean> process_uid = UIDMeta.getUIDMeta(tsdb, type, tag).addCallbackDeferring(cb).addErrback(new RowErrBack());
                        storage_calls.add(process_uid);
                        if (type == UniqueIdType.TAGK) {
                            tagk_uids.put(uid, timestamp);
                        } else {
                            tagv_uids.put(uid, timestamp);
                        }
                    }
                    // handle the timeseries meta last so we don't record it if one
                    // or more of the UIDs had an issue
                    final Deferred<Boolean> process_tsmeta = TSMeta.getTSMeta(tsdb, tsuid_string).addCallbackDeferring(new TSMetaCB(tsuid, timestamp)).addErrback(new RowErrBack());
                    storage_calls.add(process_tsmeta);
                } catch (RuntimeException e) {
                    LOG.error("Processing row " + row + " failed with exception: " + e.getMessage());
                    LOG.debug("Row: " + row + " stack trace: ", e);
                }
            }
            /**
         * A buffering callback used to avoid StackOverflowError exceptions
         * where the list of deferred calls can exceed the limit. Instead we'll
         * process the Scanner's limit in rows, wait for all of the storage
         * calls to complete, then continue on to the next set.
         */
            final class ContinueCB implements Callback<Object, ArrayList<Boolean>> {

                @Override
                public Object call(ArrayList<Boolean> puts) throws Exception {
                    storage_calls.clear();
                    return scan();
                }
            }
            /**
         * Catch exceptions in one of the grouped calls and continue scanning.
         * Without this the user may not see the exception and the thread will
         * just die silently.
         */
            final class ContinueEB implements Callback<Object, Exception> {

                @Override
                public Object call(Exception e) throws Exception {
                    Throwable ex = e;
                    while (ex.getClass().equals(DeferredGroupException.class)) {
                        if (ex.getCause() == null) {
                            LOG.warn("Unable to get to the root cause of the DGE");
                            break;
                        }
                        ex = ex.getCause();
                    }
                    LOG.error("[" + thread_id + "] Upstream Exception: ", ex);
                    return scan();
                }
            }
            // call ourself again but wait for the current set of storage calls to
            // complete so we don't OOM
            Deferred.group(storage_calls).addCallback(new ContinueCB()).addErrback(new ContinueEB());
            return null;
        }
    }
    final MetaScanner scanner = new MetaScanner();
    try {
        scanner.scan();
        result.joinUninterruptibly();
        LOG.info("[" + thread_id + "] Complete");
    } catch (Exception e) {
        LOG.error("[" + thread_id + "] Scanner Exception", e);
        throw new RuntimeException("[" + thread_id + "] Scanner exception", e);
    }
}

Example 2 with UIDMeta

use of net.opentsdb.meta.UIDMeta in project opentsdb by OpenTSDB.

the class TreeBuilder method parseTagvRule.

/**
   * Processes the custom tag value paired with a custom tag name. Routes to the 
   * {@link #processParsedValue} method after processing if successful.
   * If the custom tag group is null or empty for the tagv, or the tagv couldn't
   * be found, we just return.
   * @throws IllegalStateException if the tags UIDMeta array has not been set
   */
private void parseTagvRule() {
    if (meta.getTags() == null || meta.getTags().isEmpty()) {
        throw new IllegalStateException("Timeseries meta data was missing tags");
    }
    // first, find the tagv UIDMeta we're matching against
    UIDMeta tagv = null;
    for (UIDMeta tag : meta.getTags()) {
        if (tag.getType() == UniqueIdType.TAGV && tag.getName().equals(rule.getField())) {
            tagv = tag;
            break;
        }
    }
    if (tagv == null) {
        testMessage("No match on tagv [" + rule.getField() + "] for rule: " + rule);
        return;
    }
    // now scan the custom tags for a matching tag name and it's value
    testMessage("Matched tagv [" + rule.getField() + "] for rule: " + rule);
    final Map<String, String> custom = tagv.getCustom();
    if (custom != null && custom.containsKey(rule.getCustomField())) {
        if (custom.get(rule.getCustomField()) == null) {
            throw new IllegalStateException("Value for custom tagv field [" + rule.getCustomField() + "] was null");
        }
        processParsedValue(custom.get(rule.getCustomField()));
        testMessage("Matched custom tag [" + rule.getCustomField() + "] for rule: " + rule);
    } else {
        testMessage("No match on custom tag [" + rule.getCustomField() + "] for rule: " + rule);
        return;
    }
}

Example 3 with UIDMeta

use of net.opentsdb.meta.UIDMeta in project opentsdb by OpenTSDB.

the class UniqueIdRpc method parseUIDMetaQS.

/**
   * Used with verb overrides to parse out values from a query string
   * @param query The query to parse
   * @return An UIDMeta object with configured values
   * @throws BadRequestException if a required value was missing or could not
   * be parsed
   */
private UIDMeta parseUIDMetaQS(final HttpQuery query) {
    final String uid = query.getRequiredQueryStringParam("uid");
    final String type = query.getRequiredQueryStringParam("type");
    final UIDMeta meta = new UIDMeta(UniqueId.stringToUniqueIdType(type), uid);
    final String display_name = query.getQueryStringParam("display_name");
    if (display_name != null) {
        meta.setDisplayName(display_name);
    }
    final String description = query.getQueryStringParam("description");
    if (description != null) {
        meta.setDescription(description);
    }
    final String notes = query.getQueryStringParam("notes");
    if (notes != null) {
        meta.setNotes(notes);
    }
    return meta;
}

Example 4 with UIDMeta

use of net.opentsdb.meta.UIDMeta in project opentsdb by OpenTSDB.

the class TestTreeBuilder method before.

@Before
public void before() throws Exception {
    final Config config = new Config(false);
    tsdb = new TSDB(client, config);
    storage = new MockBase(tsdb, client, true, true, true, true);
    final List<byte[]> families = new ArrayList<byte[]>();
    families.add(Tree.TREE_FAMILY());
    storage.addTable(TREE_TABLE, families);
    treebuilder = new TreeBuilder(storage.getTSDB(), tree);
    PowerMockito.spy(Tree.class);
    PowerMockito.doReturn(Deferred.fromResult(tree)).when(Tree.class, "fetchTree", (TSDB) any(), anyInt());
    // set private fields via reflection so the UTs can change things at will
    Field tag_metric = TSMeta.class.getDeclaredField("metric");
    tag_metric.setAccessible(true);
    tag_metric.set(meta, metric);
    tag_metric.setAccessible(false);
    ArrayList<UIDMeta> tags = new ArrayList<UIDMeta>(4);
    tags.add(tagk1);
    tags.add(tagv1);
    tags.add(tagk2);
    tags.add(tagv2);
    Field tags_field = TSMeta.class.getDeclaredField("tags");
    tags_field.setAccessible(true);
    tags_field.set(meta, tags);
    tags_field.setAccessible(false);
    // store root
    final TreeMap<Integer, String> root_path = new TreeMap<Integer, String>();
    final Branch root = new Branch(tree.getTreeId());
    root.setDisplayName("ROOT");
    root_path.put(0, "ROOT");
    root.prependParentPath(root_path);
    storage.addColumn(TREE_TABLE, root.compileBranchId(), Tree.TREE_FAMILY(), BRANCH, (byte[]) toStorageJson.invoke(root));
}

Example 5 with UIDMeta

use of net.opentsdb.meta.UIDMeta in project opentsdb by OpenTSDB.

the class TestUniqueIdRpc method setupAssign.

/**
   * Sets up common mocks for UID assignment tests
   * @throws Exception if something goes pear shaped
   */
private void setupAssign() throws Exception {
    when(tsdb.assignUid("metric", "sys.cpu.0")).thenReturn(new byte[] { 0, 0, 1 });
    when(tsdb.assignUid("metric", "sys.cpu.1")).thenThrow(new IllegalArgumentException("Name already exists with UID: 000002"));
    when(tsdb.assignUid("metric", "sys.cpu.2")).thenReturn(new byte[] { 0, 0, 3 });
    when(tsdb.assignUid("tagk", "host")).thenReturn(new byte[] { 0, 0, 1 });
    when(tsdb.assignUid("tagk", "datacenter")).thenThrow(new IllegalArgumentException("Name already exists with UID: 000002"));
    when(tsdb.assignUid("tagk", "fqdn")).thenReturn(new byte[] { 0, 0, 3 });
    when(tsdb.assignUid("tagv", "localhost")).thenReturn(new byte[] { 0, 0, 1 });
    when(tsdb.assignUid("tagv", "myserver")).thenThrow(new IllegalArgumentException("Name already exists with UID: 000002"));
    when(tsdb.assignUid("tagv", "foo")).thenReturn(new byte[] { 0, 0, 3 });
    // setup UIDMeta objects for testing
    UIDMeta metric = new UIDMeta(UniqueIdType.METRIC, new byte[] { 0, 0, 1 }, "sys.cpu.0");
    metric.setDisplayName("System CPU");
    UIDMeta tagk = new UIDMeta(UniqueIdType.TAGK, new byte[] { 0, 0, 1 }, "host");
    tagk.setDisplayName("Server Name");
    UIDMeta tagv = new UIDMeta(UniqueIdType.TAGV, new byte[] { 0, 0, 1 }, "web01");
    tagv.setDisplayName("Web Server 1");
}