Examples with InMemoryMetricsTable io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable used on opensource projects

Example 11 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class EntityTableTest method testRecycleAfterMaxId.

@Test
public void testRecycleAfterMaxId() throws Exception {
    InMemoryTableService.create("testRecycleId");
    MetricsTable table = new InMemoryMetricsTable("testRecycleId");
    EntityTable entityTable = new EntityTable(table, 101);
    // only have 100 entries as maxId.
    for (long i = 1; i <= 500; i++) {
        entityTable.getId("app", "app" + i);
    }
    // we call getName for the 100 entries, it will be the latest entries 401-500
    for (long i = 1; i <= 100; i++) {
        Assert.assertEquals("app" + String.valueOf(400 + i), entityTable.getName(i, "app"));
    }
}

Example 12 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class FactCodecTest method test.

@Test
public void test() {
    InMemoryTableService.create("FactCodecTest");
    MetricsTable table = new InMemoryMetricsTable("FactCodecTest");
    int resolution = 10;
    int rollTimebaseInterval = 2;
    FactCodec codec = new FactCodec(new EntityTable(table), resolution, rollTimebaseInterval);
    // testing encoding with multiple dimensions
    List<DimensionValue> dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value3"));
    // note: we use seconds everywhere and rely on this
    long ts = 1422312915;
    byte[] rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    byte[] column = codec.createColumn(ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("myMetric", codec.getMeasureName(rowKey));
    // testing encoding without one dimension
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue"));
    rowKey = codec.createRowKey(dimensionValues, "mySingleTagMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("mySingleTagMetric", codec.getMeasureName(rowKey));
    // testing encoding without empty dimensions
    rowKey = codec.createRowKey(new ArrayList<DimensionValue>(), "myNoTagsMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(new ArrayList<DimensionValue>(), codec.getDimensionValues(rowKey));
    Assert.assertEquals("myNoTagsMetric", codec.getMeasureName(rowKey));
    // testing null metric
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue"));
    rowKey = codec.createRowKey(dimensionValues, "mySingleTagMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("mySingleTagMetric", codec.getMeasureName(rowKey));
    // testing null value
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", null), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myNullTagMetric", ts);
    Assert.assertEquals((ts / resolution) * resolution, codec.getTimestamp(rowKey, column));
    Assert.assertEquals(dimensionValues, codec.getDimensionValues(rowKey));
    Assert.assertEquals("myNullTagMetric", codec.getMeasureName(rowKey));
    // testing fuzzy mask for fuzzy stuff in row key
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), // any value is accepted
    new DimensionValue("dimension2", null), new DimensionValue("dimension3", "value3"));
    byte[] mask = codec.createFuzzyRowMask(dimensionValues, "myMetric");
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    FuzzyRowFilter filter = new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(rowKey, mask)));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "annnnnnnnnny"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value12"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value13"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    // fuzzy in value should match the "null" value
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", null), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("dimension1", "value1"), new DimensionValue("dimension2", "value2"), new DimensionValue("dimension3", "value3"));
    rowKey = codec.createRowKey(dimensionValues, "myMetric2", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    rowKey = codec.createRowKey(dimensionValues, null, ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    rowKey = codec.createRowKey(new ArrayList<DimensionValue>(), "myMetric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
    // testing fuzzy mask for fuzzy metric
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue"));
    rowKey = codec.createRowKey(dimensionValues, null, ts);
    mask = codec.createFuzzyRowMask(dimensionValues, null);
    filter = new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(rowKey, mask)));
    rowKey = codec.createRowKey(dimensionValues, "annyyy", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    rowKey = codec.createRowKey(dimensionValues, "zzzzzzzzzzzz", ts);
    Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey));
    dimensionValues = ImmutableList.of(new DimensionValue("myTag", "myValue2"));
    rowKey = codec.createRowKey(dimensionValues, "metric", ts);
    Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
// todo: test prefix of multi dimension valued row key is not same one dimension valued row key
// todo: test that rollTimebaseInterval applies well
}

Example 13 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class FactTableTest method testQuery.

@Test
public void testQuery() throws Exception {
    InMemoryTableService.create("QueryEntityTable");
    InMemoryTableService.create("QueryDataTable");
    int resolution = 10;
    int rollTimebaseInterval = 2;
    FactTable table = new FactTable(new InMemoryMetricsTable("QueryDataTable"), new EntityTable(new InMemoryMetricsTable("QueryEntityTable")), resolution, rollTimebaseInterval);
    // aligned to start of resolution bucket
    // "/1000" because time is expected to be in seconds
    long ts = ((System.currentTimeMillis() / 1000) / resolution) * resolution;
    for (int i = 0; i < 3; i++) {
        for (int k = 1; k < 3; k++) {
            // note: "+i" to ts here and below doesn't affect results, just to confirm
            // that data points are rounded to the resolution
            writeInc(table, "metric" + k, ts + i * resolution + i, k + i, "dim1", "value1", "dim2", "value2");
            writeInc(table, "metric" + k, ts + i * resolution + i, 2 * k + i, "dim1", "value2", "dim2", "value2");
            writeInc(table, "metric" + k, ts + i * resolution + i, 3 * k + i, "dim1", "value2", "dim2", "value1");
            writeInc(table, "metric" + k, ts + i * resolution + i, 4 * k + i, "dim1", "value1", "dim2", "value3");
            // null value in dim matches only fuzzy ("any")
            writeInc(table, "metric" + k, ts + i * resolution + i, 5 * k + i, "dim1", null, "dim2", "value3");
        }
    }
    Table<String, List<DimensionValue>, List<TimeValue>> expected;
    FactScan scan;
    for (int i = 1; i < 3; i++) {
        // all time points
        scan = new FactScan(ts - resolution, ts + 3 * resolution, "metric" + i, dimValues("dim1", "value1", "dim2", "value2"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1, i + 2));
        assertScan(table, expected, scan);
        // time points since second interval
        scan = new FactScan(ts + resolution, ts + 3 * resolution, "metric" + i, dimValues("dim1", "value1", "dim2", "value2"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        assertScan(table, expected, scan);
        // time points before third interval
        scan = new FactScan(ts - resolution, ts + resolution, "metric" + i, dimValues("dim1", "value1", "dim2", "value2"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1));
        assertScan(table, expected, scan);
        // time points for fuzzy dim2 since second interval
        scan = new FactScan(ts + resolution, ts + 3 * resolution, // null stands for any
        "metric" + i, dimValues("dim1", "value1", "dim2", null));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts + resolution, resolution, 4 * i + 1, 4 * i + 2));
        assertScan(table, expected, scan);
        // time points for fuzzy dim1 before third interval
        scan = new FactScan(ts - resolution, ts + resolution, // null stands for any
        "metric" + i, dimValues("dim1", null, "dim2", "value3"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts, resolution, 4 * i, 4 * i + 1));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts, resolution, 5 * i, 5 * i + 1));
        assertScan(table, expected, scan);
        // time points for both fuzzy dims before third interval
        scan = new FactScan(ts - resolution, ts + resolution, // null stands for any
        "metric" + i, dimValues("dim1", null, "dim2", null));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value1"), timeValues(ts, resolution, 3 * i, 3 * i + 1));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value2"), timeValues(ts, resolution, 2 * i, 2 * i + 1));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts, resolution, 4 * i, 4 * i + 1));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts, resolution, 5 * i, 5 * i + 1));
        assertScan(table, expected, scan);
        // time points for both fuzzy dims since third interval
        scan = new FactScan(ts + resolution, ts + 3 * resolution, // null stands for any
        "metric" + i, dimValues("dim1", null, "dim2", null));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value1"), timeValues(ts + resolution, resolution, 3 * i + 1, 3 * i + 2));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value2"), timeValues(ts + resolution, resolution, 2 * i + 1, 2 * i + 2));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts + resolution, resolution, 4 * i + 1, 4 * i + 2));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts + resolution, resolution, 5 * i + 1, 5 * i + 2));
        assertScan(table, expected, scan);
    }
    // all time points
    scan = new FactScan(ts - resolution, ts + 3 * resolution, dimValues("dim1", "value1", "dim2", "value2"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1, i + 2));
    }
    assertScan(table, expected, scan);
    // time points since second interval
    scan = new FactScan(ts + resolution, ts + 3 * resolution, dimValues("dim1", "value1", "dim2", "value2"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
    }
    assertScan(table, expected, scan);
    // time points before third interval
    scan = new FactScan(ts - resolution, ts + resolution, dimValues("dim1", "value1", "dim2", "value2"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1));
    }
    assertScan(table, expected, scan);
    // time points for fuzzy dim2 since second interval
    scan = new FactScan(ts + resolution, ts + 3 * resolution, dimValues("dim1", "value1", "dim2", null));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts + resolution, resolution, 4 * i + 1, 4 * i + 2));
    }
    assertScan(table, expected, scan);
    // time points for fuzzy dim1 before third interval (very important case - caught some bugs)
    scan = new FactScan(ts - resolution, ts + resolution, dimValues("dim1", null, "dim2", "value3"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts, resolution, 4 * i, 4 * i + 1));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts, resolution, 5 * i, 5 * i + 1));
    }
    assertScan(table, expected, scan);
}

Example 14 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class FactTableTest method testSearch.

@Test
public void testSearch() throws Exception {
    InMemoryTableService.create("SearchEntityTable");
    InMemoryTableService.create("SearchDataTable");
    int resolution = Integer.MAX_VALUE;
    int rollTimebaseInterval = 2;
    FactTable table = new FactTable(new InMemoryMetricsTable("SearchDataTable"), new EntityTable(new InMemoryMetricsTable("SearchEntityTable")), resolution, rollTimebaseInterval);
    // aligned to start of resolution bucket
    // "/1000" because time is expected to be in seconds
    long ts = ((System.currentTimeMillis() / 1000) / resolution) * resolution;
    List<String> aggregationList = ImmutableList.of("dim1", "dim2", "dim3", "dim4");
    for (int i = 0; i < 2; i++) {
        writeInc(table, "metric-a" + i, ts + i, i, "dim1", "value1", "dim2", "value2", "dim3", "value3", "dim4", "value4");
        writeInc(table, "metric-b" + i, ts + i, i, "dim1", "value2", "dim2", "value2", "dim3", "x3", "dim4", "x4");
        writeInc(table, "metric-c" + i, ts + i, i, "dim1", "value2", "dim2", "value2", "dim3", null, "dim4", "y4");
        writeInc(table, "metric-d" + i, ts + i, i, "dim1", "value1", "dim2", "value3", "dim3", "y3", "dim4", null);
    }
    Map<String, String> slice = Maps.newHashMap();
    slice.put("dim1", "value2");
    slice.put("dim2", "value2");
    slice.put("dim3", null);
    // verify search dims
    testTagSearch(table, aggregationList, ImmutableMap.of("dim2", "value2"), ImmutableSet.of(new DimensionValue("dim1", "value1"), new DimensionValue("dim1", "value2")));
    testTagSearch(table, aggregationList, ImmutableMap.of("dim1", "value1"), ImmutableSet.of(new DimensionValue("dim2", "value2"), new DimensionValue("dim2", "value3")));
    testTagSearch(table, aggregationList, ImmutableMap.<String, String>of(), ImmutableSet.of(new DimensionValue("dim1", "value1"), new DimensionValue("dim1", "value2")));
    testTagSearch(table, aggregationList, ImmutableMap.of("dim1", "value2", "dim2", "value2"), ImmutableSet.of(new DimensionValue("dim3", "x3"), new DimensionValue("dim4", "y4")));
    testTagSearch(table, aggregationList, slice, ImmutableSet.of(new DimensionValue("dim4", "x4"), new DimensionValue("dim4", "y4")));
    testTagSearch(table, aggregationList, ImmutableMap.of("dim1", "value2", "dim2", "value3", "dim3", "y3"), ImmutableSet.<DimensionValue>of());
    // verify search metrics
    testMetricNamesSearch(table, aggregationList, ImmutableMap.of("dim1", "value1", "dim2", "value2", "dim3", "value3"), ImmutableSet.<String>of("metric-a0", "metric-a1"));
    testMetricNamesSearch(table, aggregationList, ImmutableMap.of("dim2", "value2"), ImmutableSet.<String>of("metric-a0", "metric-a1", "metric-b0", "metric-b1", "metric-c0", "metric-c1"));
    testMetricNamesSearch(table, aggregationList, slice, ImmutableSet.of("metric-b0", "metric-b1", "metric-c0", "metric-c1"));
}

Example 15 with InMemoryMetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable in project cdap by caskdata.

the class EntityTableTest method testGetId.

@Test
public void testGetId() throws Exception {
    InMemoryTableService.create("testGetId");
    MetricsTable table = new InMemoryMetricsTable("testGetId");
    EntityTable entityTable = new EntityTable(table);
    // Make sure it is created sequentially
    for (int i = 1; i <= 10; i++) {
        Assert.assertEquals((long) i, entityTable.getId("app", "app" + i));
    }
    // It should get the same value (from cache)
    for (int i = 1; i <= 10; i++) {
        Assert.assertEquals((long) i, entityTable.getId("app", "app" + i));
    }
    // Construct another entityTable, it should load from storage.
    entityTable = new EntityTable(table);
    for (int i = 1; i <= 10; i++) {
        Assert.assertEquals((long) i, entityTable.getId("app", "app" + i));
    }
    // ID for different type should have ID starts from 1 again.
    for (int i = 1; i <= 10; i++) {
        Assert.assertEquals((long) i, entityTable.getId("flow", "flow" + i));
    }
}