Examples with Cell net.opentsdb.core.Internal.Cell used on opensource projects

Example 1 with Cell

use of net.opentsdb.core.Internal.Cell in project opentsdb by OpenTSDB.

the class TestAppendDataPoints method repairDuplicates.

@Test
public void repairDuplicates() throws Exception {
    setDataPointStorage();
    Whitebox.setInternalState(config, "repair_appends", true);
    final KeyValue kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2, DPQ_S, DPV2));
    final AppendDataPoints adp = new AppendDataPoints();
    final Collection<Cell> cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(2, cells.size());
    final Iterator<Cell> iterator = cells.iterator();
    Cell cell = iterator.next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    cell = iterator.next();
    assertArrayEquals(DPV2, cell.value);
    assertEquals(1356998402000L, cell.timestamp(1356998400));
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier());
    assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value());
    verify(client, times(1)).put(any(PutRequest.class));
    adp.repairedDeferred().join();
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2), storage.getColumn(ROW_KEY, AppendDataPoints.APPEND_COLUMN_QUALIFIER));
}

Example 2 with Cell

use of net.opentsdb.core.Internal.Cell in project opentsdb by OpenTSDB.

the class TestAppendDataPoints method repairOutOfOrder.

@Test
public void repairOutOfOrder() throws Exception {
    setDataPointStorage();
    Whitebox.setInternalState(config, "repair_appends", true);
    final KeyValue kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_S, DPV2, DPQ_MS, DPV));
    final AppendDataPoints adp = new AppendDataPoints();
    final Collection<Cell> cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(2, cells.size());
    final Iterator<Cell> iterator = cells.iterator();
    Cell cell = iterator.next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    cell = iterator.next();
    assertArrayEquals(DPV2, cell.value);
    assertEquals(1356998402000L, cell.timestamp(1356998400));
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier());
    assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value());
    verify(client, times(1)).put(any(PutRequest.class));
    adp.repairedDeferred().join();
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2), storage.getColumn(ROW_KEY, AppendDataPoints.APPEND_COLUMN_QUALIFIER));
}

Example 3 with Cell

use of net.opentsdb.core.Internal.Cell in project opentsdb by OpenTSDB.

the class AppendDataPoints method parseKeyValue.

/**
   * Parses a column from storage, orders and drops newer duplicate data points.
   * The parsing will return both a Cell collection for debugging and add
   * the cells to concatenated qualifier and value arrays in the compacted data
   * point format so that the results can be merged with other non-append 
   * columns or rows.
   * <p>
   * WARNING: If the "tsd.core.repair_appends" config is set to true then this
   * method will issue puts against the database, overwriting the column with
   * sorted and de-duplicated data. It will only do this for rows that are at
   * least an hour old so as to avoid pounding current rows.
   * <p> 
   * TODO (CL) - allow for newer or older data points depending on a config.
   * @param tsdb The TSDB to which we belong
   * @param kv The key value t parse
   * @throws IllegalArgumentException if the given KV is not an append column
   * or we were unable to parse the value.
   */
public final Collection<Cell> parseKeyValue(final TSDB tsdb, final KeyValue kv) {
    if (kv.qualifier().length != 3 || kv.qualifier()[0] != APPEND_COLUMN_PREFIX) {
        // we'll support appends at different offsets.
        throw new IllegalArgumentException("Can not parse cell, it is not " + " an appended cell. It has a different qualifier " + Bytes.pretty(kv.qualifier()) + ", row key " + Bytes.pretty(kv.key()));
    }
    final boolean repair = tsdb.getConfig().repair_appends();
    final long base_time;
    try {
        base_time = Internal.baseTime(tsdb, kv.key());
    } catch (ArrayIndexOutOfBoundsException oob) {
        throw new IllegalDataException("Corrupted value: invalid row key: " + kv, oob);
    }
    int val_idx = 0;
    int val_length = 0;
    int qual_length = 0;
    // Time delta, extracted from the qualifier.
    int last_delta = -1;
    final Map<Integer, Internal.Cell> deltas = new TreeMap<Integer, Cell>();
    boolean has_duplicates = false;
    boolean out_of_order = false;
    boolean needs_repair = false;
    try {
        while (val_idx < kv.value().length) {
            byte[] q = Internal.extractQualifier(kv.value(), val_idx);
            System.arraycopy(kv.value(), val_idx, q, 0, q.length);
            val_idx = val_idx + q.length;
            int vlen = Internal.getValueLengthFromQualifier(q, 0);
            byte[] v = new byte[vlen];
            System.arraycopy(kv.value(), val_idx, v, 0, vlen);
            val_idx += vlen;
            int delta = Internal.getOffsetFromQualifier(q);
            final Cell duplicate = deltas.get(delta);
            if (duplicate != null) {
                // This is a duplicate cell, skip it
                has_duplicates = true;
                qual_length -= duplicate.qualifier.length;
                val_length -= duplicate.value.length;
            }
            qual_length += q.length;
            val_length += vlen;
            final Cell cell = new Cell(q, v);
            deltas.put(delta, cell);
            if (!out_of_order) {
                // order data
                if (delta <= last_delta) {
                    out_of_order = true;
                }
                last_delta = delta;
            }
        }
    } catch (ArrayIndexOutOfBoundsException oob) {
        throw new IllegalDataException("Corrupted value: couldn't break down" + " into individual values (consumed " + val_idx + " bytes, but was" + " expecting to consume " + (kv.value().length) + "): " + kv + ", cells so far: " + deltas.values(), oob);
    }
    if (has_duplicates || out_of_order) {
        if ((DateTime.currentTimeMillis() / 1000) - base_time > REPAIR_THRESHOLD) {
            needs_repair = true;
        }
    }
    // Check we consumed all the bytes of the value.
    if (val_idx != kv.value().length) {
        throw new IllegalDataException("Corrupted value: couldn't break down" + " into individual values (consumed " + val_idx + " bytes, but was" + " expecting to consume " + (kv.value().length) + "): " + kv + ", cells so far: " + deltas.values());
    }
    val_idx = 0;
    int qual_idx = 0;
    byte[] healed_cell = null;
    int healed_index = 0;
    this.value = new byte[val_length];
    this.qualifier = new byte[qual_length];
    if (repair && needs_repair) {
        healed_cell = new byte[val_length + qual_length];
    }
    for (final Cell cell : deltas.values()) {
        System.arraycopy(cell.qualifier, 0, this.qualifier, qual_idx, cell.qualifier.length);
        qual_idx += cell.qualifier.length;
        System.arraycopy(cell.value, 0, this.value, val_idx, cell.value.length);
        val_idx += cell.value.length;
        if (repair && needs_repair) {
            System.arraycopy(cell.qualifier, 0, healed_cell, healed_index, cell.qualifier.length);
            healed_index += cell.qualifier.length;
            System.arraycopy(cell.value, 0, healed_cell, healed_index, cell.value.length);
            healed_index += cell.value.length;
        }
    }
    if (repair && needs_repair) {
        LOG.debug("Repairing appended data column " + kv);
        final PutRequest put = new PutRequest(tsdb.table, kv.key(), TSDB.FAMILY(), kv.qualifier(), healed_cell);
        repaired_deferred = tsdb.getClient().put(put);
    }
    return deltas.values();
}

Example 4 with Cell

use of net.opentsdb.core.Internal.Cell in project opentsdb by OpenTSDB.

the class DumpSeries method formatKeyValue.

private static void formatKeyValue(final StringBuilder buf, final TSDB tsdb, final boolean importformat, final KeyValue kv, final long base_time, final String metric) {
    final String tags;
    if (importformat) {
        final StringBuilder tagsbuf = new StringBuilder();
        for (final Map.Entry<String, String> tag : Internal.getTags(tsdb, kv.key()).entrySet()) {
            tagsbuf.append(' ').append(tag.getKey()).append('=').append(tag.getValue());
        }
        tags = tagsbuf.toString();
    } else {
        tags = null;
    }
    final byte[] qualifier = kv.qualifier();
    final byte[] value = kv.value();
    final int q_len = qualifier.length;
    if (!AppendDataPoints.isAppendDataPoints(qualifier) && q_len % 2 != 0) {
        if (!importformat) {
            // custom data object, not a data point
            if (kv.qualifier()[0] == Annotation.PREFIX()) {
                appendAnnotation(buf, kv, base_time);
            } else {
                buf.append(Arrays.toString(value)).append("\t[Not a data point]");
            }
        }
    } else if (q_len == 2 || q_len == 4 && Internal.inMilliseconds(qualifier)) {
        // regular data point
        final Cell cell = Internal.parseSingleValue(kv);
        if (cell == null) {
            throw new IllegalDataException("Unable to parse row: " + kv);
        }
        if (!importformat) {
            appendRawCell(buf, cell, base_time);
        } else {
            buf.append(metric).append(' ');
            appendImportCell(buf, cell, base_time, tags);
        }
    } else {
        final Collection<Cell> cells;
        if (q_len == 3) {
            // append data points
            final AppendDataPoints adps = new AppendDataPoints();
            cells = adps.parseKeyValue(tsdb, kv);
        } else {
            // compacted column
            cells = Internal.extractDataPoints(kv);
        }
        if (!importformat) {
            buf.append(Arrays.toString(kv.qualifier())).append('\t').append(Arrays.toString(kv.value())).append(" = ").append(cells.size()).append(" values:");
        }
        int i = 0;
        for (Cell cell : cells) {
            if (!importformat) {
                buf.append("\n    ");
                appendRawCell(buf, cell, base_time);
            } else {
                buf.append(metric).append(' ');
                appendImportCell(buf, cell, base_time, tags);
                if (i < cells.size() - 1) {
                    buf.append("\n");
                }
            }
            i++;
        }
    }
}

Example 5 with Cell

use of net.opentsdb.core.Internal.Cell in project opentsdb by OpenTSDB.

the class TestAppendDataPoints method parseKeyValue.

@Test
public void parseKeyValue() throws Exception {
    KeyValue kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_S, DPV));
    AppendDataPoints adp = new AppendDataPoints();
    Collection<Cell> cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(1, cells.size());
    Cell cell = cells.iterator().next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998402000L, cell.timestamp(1356998400));
    assertArrayEquals(DPQ_S, adp.qualifier());
    assertArrayEquals(DPV, adp.value());
    verify(client, never()).put(any(PutRequest.class));
    kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_MS, DPV));
    adp = new AppendDataPoints();
    cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(1, cells.size());
    cell = cells.iterator().next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    assertArrayEquals(DPQ_MS, adp.qualifier());
    assertArrayEquals(DPV, adp.value());
    verify(client, never()).put(any(PutRequest.class));
    // some odd offset
    kv = new KeyValue(ROW_KEY, CF, new byte[] { AppendDataPoints.APPEND_COLUMN_PREFIX, 42, 42 }, MockBase.concatByteArrays(DPQ_MS, DPV));
    adp = new AppendDataPoints();
    cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(1, cells.size());
    cell = cells.iterator().next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    assertArrayEquals(DPQ_MS, adp.qualifier());
    assertArrayEquals(DPV, adp.value());
    verify(client, never()).put(any(PutRequest.class));
    kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2));
    adp = new AppendDataPoints();
    cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(2, cells.size());
    Iterator<Cell> iterator = cells.iterator();
    cell = iterator.next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    cell = iterator.next();
    assertArrayEquals(DPV2, cell.value);
    assertEquals(1356998402000L, cell.timestamp(1356998400));
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier());
    assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value());
    verify(client, never()).put(any(PutRequest.class));
    // out of order
    kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_S, DPV2, DPQ_MS, DPV));
    adp = new AppendDataPoints();
    cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(2, cells.size());
    iterator = cells.iterator();
    cell = iterator.next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    cell = iterator.next();
    assertArrayEquals(DPV2, cell.value);
    assertEquals(1356998402000L, cell.timestamp(1356998400));
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier());
    assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value());
    verify(client, never()).put(any(PutRequest.class));
    // duplicates
    kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_MS, DPV, DPQ_S, DPV2, DPQ_S, DPV2));
    adp = new AppendDataPoints();
    cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(2, cells.size());
    iterator = cells.iterator();
    cell = iterator.next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    cell = iterator.next();
    assertArrayEquals(DPV2, cell.value);
    assertEquals(1356998402000L, cell.timestamp(1356998400));
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier());
    assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value());
    verify(client, never()).put(any(PutRequest.class));
    // duplicates AND out of order, just a mess
    kv = new KeyValue(ROW_KEY, CF, AppendDataPoints.APPEND_COLUMN_QUALIFIER, MockBase.concatByteArrays(DPQ_S, DPV2, DPQ_MS, DPV, DPQ_MS, DPV, DPQ_S, DPV2, DPQ_MS, DPV));
    adp = new AppendDataPoints();
    cells = adp.parseKeyValue(tsdb, kv);
    assertEquals(2, cells.size());
    iterator = cells.iterator();
    cell = iterator.next();
    assertArrayEquals(DPV, cell.value);
    assertEquals(1356998400128L, cell.timestamp(1356998400));
    cell = iterator.next();
    assertArrayEquals(DPV2, cell.value);
    assertEquals(1356998402000L, cell.timestamp(1356998400));
    assertArrayEquals(MockBase.concatByteArrays(DPQ_MS, DPQ_S), adp.qualifier());
    assertArrayEquals(MockBase.concatByteArrays(DPV, DPV2), adp.value());
    verify(client, never()).put(any(PutRequest.class));
}