Examples with InMemoryMetricsTable io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable used on opensource projects

Example 6 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class EntityTableTest method testRecycleAfterMaxId.

@Test
public void testRecycleAfterMaxId() throws Exception {
    InMemoryTableService.create("testRecycleId");
    MetricsTable table = new InMemoryMetricsTable("testRecycleId");
    EntityTable entityTable = new EntityTable(table, 101);
    // only have 100 entries as maxId.
    for (long i = 1; i <= 500; i++) {
        entityTable.getId("app", "app" + i);
    }
    // we call getName for the 100 entries, it will be the latest entries 401-500
    for (long i = 1; i <= 100; i++) {
        Assert.assertEquals("app" + String.valueOf(400 + i), entityTable.getName(i, "app"));
    }
}

Example 7 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class EntityTableTest method testGetName.

@Test
public void testGetName() throws Exception {
    InMemoryTableService.create("testGetName");
    MetricsTable table = new InMemoryMetricsTable("testGetName");
    EntityTable entityTable = new EntityTable(table);
    // Create some entities.
    for (int i = 1; i <= 10; i++) {
        Assert.assertEquals((long) i, entityTable.getId("app", "app" + i));
    }
    // Reverse lookup
    for (int i = 1; i <= 10; i++) {
        Assert.assertEquals("app" + i, entityTable.getName(i, "app"));
    }
}

Example 8 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class FactTableTest method testMaxResolution.

@Test
public void testMaxResolution() throws Exception {
    // we use Integer.MAX_VALUE as resolution to compute all-time total values
    InMemoryTableService.create("TotalsEntityTable");
    InMemoryTableService.create("TotalsDataTable");
    int resolution = Integer.MAX_VALUE;
    // should not matter when resolution is max
    int rollTimebaseInterval = 3600;
    FactTable table = new FactTable(new InMemoryMetricsTable("TotalsDataTable"), new EntityTable(new InMemoryMetricsTable("TotalsEntityTable")), resolution, rollTimebaseInterval);
    // ts is expected in seconds
    long ts = System.currentTimeMillis() / 1000;
    int count = 1000;
    for (int i = 0; i < count; i++) {
        for (int k = 0; k < 10; k++) {
            // shift one day
            writeInc(table, "metric" + k, ts + i * 60 * 60 * 24, i * k, "dim" + k, "value" + k);
        }
    }
    for (int k = 0; k < 10; k++) {
        // 0, 0 should match timestamp of all data points
        FactScan scan = new FactScan(0, 0, "metric" + k, dimValues("dim" + k, "value" + k));
        Table<String, List<DimensionValue>, List<TimeValue>> expected = HashBasedTable.create();
        expected.put("metric" + k, dimValues("dim" + k, "value" + k), ImmutableList.of(new TimeValue(0, k * count * (count - 1) / 2)));
        assertScan(table, expected, scan);
    }
}

Example 9 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class FactTableTest method testCache.

@Test
public void testCache() throws Exception {
    String tableName = "testCacheTable";
    String entityTableName = "testCacheEntityTable";
    InMemoryTableService.create(tableName);
    InMemoryTableService.create(entityTableName);
    int resolution = 5;
    InMemoryMetricsTable metricsTable = new InMemoryMetricsTable(tableName);
    FactTable table = new FactTable(metricsTable, new EntityTable(new InMemoryMetricsTable(entityTableName)), resolution, 2);
    // set the metrics collector for the table
    FactTableMetricsCollector metricsCollector = new FactTableMetricsCollector(resolution);
    table.setMetricsCollector(metricsCollector);
    // Initially the cache should be empty
    Assert.assertEquals(0, table.getFactCounterCache().size());
    // add some value with current ts
    long timestampNow = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) / resolution * resolution;
    List<DimensionValue> dims = dimValues("dim1", "dim2");
    List<Fact> metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    // Since no previous add to the metric store, these increment should still be considered as COUNTER, and the
    // cache should be updated.
    Assert.assertEquals(10, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(0, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow, value);
    }
    // Add metrics older than the current timestamp, these increment should still be considered as COUNTER, and the
    // cache should NOT be updated.
    metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow - 5, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    Assert.assertEquals(10, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(0, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow, value);
    }
    // Now insert metrics newer than the current timestamp, the increment should be considered as GAUGE, and the cache
    // should be updated
    metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow + 5, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    Assert.assertEquals(0, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(10, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow + 5, value);
    }
}

Example 10 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class FactCodecTest method test.

@Test
public void test() {
    InMemoryTableService.create("FactCodecTest");
    MetricsTable table = new InMemoryMetricsTable("FactCodecTest");
    int resolution = 10;
    int rollTimebaseInterval = 2;
    FactCodec codec = new FactCodec(new EntityTable(table), resolution, rollTimebaseInterval);
    // testing encoding with multiple dimensions
    List<DimensionValue> dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value3"));
    // note: we use seconds everywhere and rely on this
    long ts = 1422312915;
    byte[] rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    byte[] column = codec.createColumn(ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("myMetric", codec.getMeasureName(rowKey));
    // testing encoding without one dimension
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue"));
    rowKey = codec.createRowKey(dimensionValues, "mySingleTagMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("mySingleTagMetric", codec.getMeasureName(rowKey));
    // testing encoding without empty dimensions
    rowKey = codec.createRowKey(new ArrayList<DimensionValue>(), "myNoTagsMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(new ArrayList<DimensionValue>(), codec.getDimensionValues(rowKey));
    Assert.assertEquals("myNoTagsMetric", codec.getMeasureName(rowKey));
    // testing null metric
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue"));
    rowKey = codec.createRowKey(dimensionValues, "mySingleTagMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("mySingleTagMetric", codec.getMeasureName(rowKey));
    // testing null value
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", null), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myNullTagMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("myNullTagMetric", codec.getMeasureName(rowKey));
    // testing fuzzy mask for fuzzy stuff in row key
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), // any value is accepted
    new DimensionValue("dimension2", null), new DimensionValue("dimension3", "value3"));
    byte[] mask = codec.createFuzzyRowMask(dimensionValues, "myMetric");
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    FuzzyRowFilter filter = new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(rowKey, mask)));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "annnnnnnnnny"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value12"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value13"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    // fuzzy in value should match the "null" value
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", null), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric2", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    rowKey = codec.createRowKey(dimensionValues, null, ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    rowKey = codec.createRowKey(new ArrayList<DimensionValue>(), "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    // testing fuzzy mask for fuzzy metric
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue"));
    rowKey = codec.createRowKey(dimensionValues, null, ts);
    mask = codec.createFuzzyRowMask(dimensionValues, null);
    filter = new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(rowKey, mask)));
    rowKey = codec.createRowKey(dimensionValues, "annyyy", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    rowKey = codec.createRowKey(dimensionValues, "zzzzzzzzzzzz", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue2"));
    rowKey = codec.createRowKey(dimensionValues, "metric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
// todo: test prefix of multi dimension valued row key is not same one dimension valued row key
// todo: test that rollTimebaseInterval applies well
}