Examples with MetricsTable co.cask.cdap.data2.dataset2.lib.table.MetricsTable used on opensource projects

Example 21 with MetricsTable

use of co.cask.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class CubeModule method register.

@Override
public void register(DatasetDefinitionRegistry registry) {
    DatasetDefinition<Table, ? extends DatasetAdmin> tableDef = registry.get("table");
    DatasetDefinition<MetricsTable, ? extends DatasetAdmin> metricsTableDef = registry.get(MetricsTable.class.getName());
    registry.add(new CubeDatasetDefinition(FULL_NAME, tableDef, metricsTableDef));
    registry.add(new CubeDatasetDefinition(SHORT_NAME, tableDef, metricsTableDef));
}

Example 22 with MetricsTable

use of co.cask.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method testDataMigration.

@Test
public void testDataMigration() throws Exception {
    MetricsTable v2Table = getTable("V2Table");
    MetricsTable v3Table = getTable("V3Table");
    v2Table.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR).put(A, mapOf(A, Bytes.toLong(A), B, Bytes.toLong(B))).put(B, mapOf(A, Bytes.toLong(A), B, Bytes.toLong(B))).put(X, mapOf(A, Bytes.toLong(A), B, Bytes.toLong(B))).build());
    // add just column A value for key X in table v3, so this is an increment, while column B is a gauge.
    v3Table.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR).put(X, mapOf(A, Bytes.toLong(A))).build());
    MetricsTableMigration metricsTableMigration = new MetricsTableMigration("V2Table", v2Table, "V3Table", v3Table);
    Assert.assertTrue(isMetricsDataAvailable(v2Table));
    metricsTableMigration.transferData(1);
    Assert.assertEquals(Bytes.toLong(A), Bytes.toLong(v3Table.get(A, A)));
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(v3Table.get(A, B)));
    Assert.assertEquals(Bytes.toLong(A), Bytes.toLong(v3Table.get(B, A)));
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(v3Table.get(B, B)));
    // this is an increment
    Assert.assertEquals(Bytes.toLong(A) * 2, Bytes.toLong(v3Table.get(X, A)));
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(v3Table.get(X, B)));
    Assert.assertFalse(isMetricsDataAvailable(v2Table));
}

Example 23 with MetricsTable

use of co.cask.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method testCombinedTablePut.

@Test
public void testCombinedTablePut() throws Exception {
    MetricsTable v2Table = getTable("v2Table");
    MetricsTable v3Table = getTable("v3Table");
    MetricsTable combinedMetricsTable = new CombinedHBaseMetricsTable(v2Table, v3Table, 1, cConf, dsFramework);
    // Already existing data on v2
    v2Table.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR).put(A, mapOf(A, Bytes.toLong(A), B, Bytes.toLong(B))).put(B, mapOf(A, Bytes.toLong(A), B, Bytes.toLong(B))).build());
    Assert.assertEquals(Bytes.toLong(A), Bytes.toLong(v2Table.get(A, A)));
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(v2Table.get(A, B)));
    Assert.assertEquals(Bytes.toLong(A), Bytes.toLong(v2Table.get(B, A)));
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(v2Table.get(B, B)));
    // Add some gauge metrics to v3 tables
    combinedMetricsTable.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR).put(B, mapOf(B, Bytes.toLong(B), C, Bytes.toLong(C))).put(C, mapOf(P, Bytes.toLong(P), X, Bytes.toLong(X))).build());
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(combinedMetricsTable.get(B, B)));
    Assert.assertEquals(Bytes.toLong(C), Bytes.toLong(combinedMetricsTable.get(B, C)));
    Assert.assertEquals(Bytes.toLong(X), Bytes.toLong(combinedMetricsTable.get(C, X)));
    Assert.assertEquals(Bytes.toLong(P), Bytes.toLong(combinedMetricsTable.get(C, P)));
    Scanner combinedScanner = combinedMetricsTable.scan(null, null, null);
    Row firstRow = combinedScanner.next();
    Assert.assertEquals(1L, Bytes.toLong(firstRow.getRow()));
    Iterator<Map.Entry<byte[], byte[]>> colIterator = firstRow.getColumns().entrySet().iterator();
    Map.Entry<byte[], byte[]> column = colIterator.next();
    Assert.assertEquals(1L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(1L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(2L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(2L, Bytes.toLong(column.getValue()));
    Row secondRow = combinedScanner.next();
    Assert.assertEquals(2L, Bytes.toLong(secondRow.getRow()));
    colIterator = secondRow.getColumns().entrySet().iterator();
    column = colIterator.next();
    Assert.assertEquals(1L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(1L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(2L, Bytes.toLong(column.getKey()));
    // this should be latest value whichh is 2
    Assert.assertEquals(2L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(3L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(3L, Bytes.toLong(column.getValue()));
    Row thirdRow = combinedScanner.next();
    Assert.assertEquals(3L, Bytes.toLong(thirdRow.getRow()));
    colIterator = thirdRow.getColumns().entrySet().iterator();
    column = colIterator.next();
    Assert.assertEquals(4L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(4L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(7L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(7L, Bytes.toLong(column.getValue()));
}

Example 24 with MetricsTable

use of co.cask.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method testConcurrentIncrement.

@Override
@Test
public void testConcurrentIncrement() throws Exception {
    String testConcurrentIncrement = "testConcurrentIncrement";
    final MetricsTable table = getTable(testConcurrentIncrement);
    final int rounds = 500;
    Map<byte[], Long> inc1 = ImmutableMap.of(X, 1L, Y, 2L);
    Map<byte[], Long> inc2 = ImmutableMap.of(Y, 1L, Z, 2L);
    // HTable used by HBaseMetricsTable is not thread safe, so each thread must use a separate instance
    // HBaseMetricsTable does not support mixed increment and incrementAndGet so the
    // updates and assertions here are different from MetricsTableTest.testConcurrentIncrement()
    Collection<? extends Thread> threads = ImmutableList.of(new IncThread(getTable(testConcurrentIncrement), A, inc1, rounds), new IncThread(getTable(testConcurrentIncrement), A, inc2, rounds), new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 5, rounds), new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 2, rounds));
    for (Thread t : threads) {
        t.start();
    }
    for (Thread t : threads) {
        t.join();
        if (t instanceof Closeable) {
            ((Closeable) t).close();
        }
    }
    assertEquals(rounds, Bytes.toLong(table.get(A, X)));
    assertEquals(3 * rounds, Bytes.toLong(table.get(A, Y)));
    assertEquals(2 * rounds, Bytes.toLong(table.get(A, Z)));
    assertEquals(7 * rounds, Bytes.toLong(table.get(A, R)));
}

Example 25 with MetricsTable

use of co.cask.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method getTable.

@Override
protected MetricsTable getTable(String name) throws Exception {
    // add v3 so that all the tests are performed for v3 table
    DatasetId metricsDatasetInstanceId = getDatasetId(name);
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(true).build();
    return DatasetsUtil.getOrCreateDataset(dsFramework, metricsDatasetInstanceId, MetricsTable.class.getName(), props, null);
}