Search in sources :

Example 31 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method testCombinedTablePut.

@Test
public void testCombinedTablePut() throws Exception {
    MetricsTable v2Table = getTable("v2Table");
    MetricsTable v3Table = getTable("v3Table");
    MetricsTable combinedMetricsTable = new CombinedHBaseMetricsTable(v2Table, v3Table, 1, cConf, dsFramework);
    // Already existing data on v2
    v2Table.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR).put(A, mapOf(A, Bytes.toLong(A), B, Bytes.toLong(B))).put(B, mapOf(A, Bytes.toLong(A), B, Bytes.toLong(B))).build());
    Assert.assertEquals(Bytes.toLong(A), Bytes.toLong(v2Table.get(A, A)));
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(v2Table.get(A, B)));
    Assert.assertEquals(Bytes.toLong(A), Bytes.toLong(v2Table.get(B, A)));
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(v2Table.get(B, B)));
    // Add some gauge metrics to v3 tables
    combinedMetricsTable.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR).put(B, mapOf(B, Bytes.toLong(B), C, Bytes.toLong(C))).put(C, mapOf(P, Bytes.toLong(P), X, Bytes.toLong(X))).build());
    Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(combinedMetricsTable.get(B, B)));
    Assert.assertEquals(Bytes.toLong(C), Bytes.toLong(combinedMetricsTable.get(B, C)));
    Assert.assertEquals(Bytes.toLong(X), Bytes.toLong(combinedMetricsTable.get(C, X)));
    Assert.assertEquals(Bytes.toLong(P), Bytes.toLong(combinedMetricsTable.get(C, P)));
    Scanner combinedScanner = combinedMetricsTable.scan(null, null, null);
    Row firstRow = combinedScanner.next();
    Assert.assertEquals(1L, Bytes.toLong(firstRow.getRow()));
    Iterator<Map.Entry<byte[], byte[]>> colIterator = firstRow.getColumns().entrySet().iterator();
    Map.Entry<byte[], byte[]> column = colIterator.next();
    Assert.assertEquals(1L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(1L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(2L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(2L, Bytes.toLong(column.getValue()));
    Row secondRow = combinedScanner.next();
    Assert.assertEquals(2L, Bytes.toLong(secondRow.getRow()));
    colIterator = secondRow.getColumns().entrySet().iterator();
    column = colIterator.next();
    Assert.assertEquals(1L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(1L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(2L, Bytes.toLong(column.getKey()));
    // this should be latest value whichh is 2
    Assert.assertEquals(2L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(3L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(3L, Bytes.toLong(column.getValue()));
    Row thirdRow = combinedScanner.next();
    Assert.assertEquals(3L, Bytes.toLong(thirdRow.getRow()));
    colIterator = thirdRow.getColumns().entrySet().iterator();
    column = colIterator.next();
    Assert.assertEquals(4L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(4L, Bytes.toLong(column.getValue()));
    column = colIterator.next();
    Assert.assertEquals(7L, Bytes.toLong(column.getKey()));
    Assert.assertEquals(7L, Bytes.toLong(column.getValue()));
}
Also used : Scanner(co.cask.cdap.api.dataset.table.Scanner) MetricsTable(co.cask.cdap.data2.dataset2.lib.table.MetricsTable) SortedMap(java.util.SortedMap) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) Row(co.cask.cdap.api.dataset.table.Row) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) SortedMap(java.util.SortedMap) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) MetricsTableTest(co.cask.cdap.data2.dataset2.lib.table.MetricsTableTest) Test(org.junit.Test)

Example 32 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method testConcurrentIncrement.

@Override
@Test
public void testConcurrentIncrement() throws Exception {
    String testConcurrentIncrement = "testConcurrentIncrement";
    final MetricsTable table = getTable(testConcurrentIncrement);
    final int rounds = 500;
    Map<byte[], Long> inc1 = ImmutableMap.of(X, 1L, Y, 2L);
    Map<byte[], Long> inc2 = ImmutableMap.of(Y, 1L, Z, 2L);
    // HTable used by HBaseMetricsTable is not thread safe, so each thread must use a separate instance
    // HBaseMetricsTable does not support mixed increment and incrementAndGet so the
    // updates and assertions here are different from MetricsTableTest.testConcurrentIncrement()
    Collection<? extends Thread> threads = ImmutableList.of(new IncThread(getTable(testConcurrentIncrement), A, inc1, rounds), new IncThread(getTable(testConcurrentIncrement), A, inc2, rounds), new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 5, rounds), new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 2, rounds));
    for (Thread t : threads) {
        t.start();
    }
    for (Thread t : threads) {
        t.join();
        if (t instanceof Closeable) {
            ((Closeable) t).close();
        }
    }
    assertEquals(rounds, Bytes.toLong(table.get(A, X)));
    assertEquals(3 * rounds, Bytes.toLong(table.get(A, Y)));
    assertEquals(2 * rounds, Bytes.toLong(table.get(A, Z)));
    assertEquals(7 * rounds, Bytes.toLong(table.get(A, R)));
}
Also used : MetricsTable(co.cask.cdap.data2.dataset2.lib.table.MetricsTable) Closeable(java.io.Closeable) MetricsTableTest(co.cask.cdap.data2.dataset2.lib.table.MetricsTableTest) Test(org.junit.Test)

Example 33 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method getTable.

@Override
protected MetricsTable getTable(String name) throws Exception {
    // add v3 so that all the tests are performed for v3 table
    DatasetId metricsDatasetInstanceId = getDatasetId(name);
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(true).build();
    return DatasetsUtil.getOrCreateDataset(dsFramework, metricsDatasetInstanceId, MetricsTable.class.getName(), props, null);
}
Also used : MetricsTable(co.cask.cdap.data2.dataset2.lib.table.MetricsTable) DatasetProperties(co.cask.cdap.api.dataset.DatasetProperties) DatasetId(co.cask.cdap.proto.id.DatasetId)

Example 34 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method getTable.

@Override
protected MetricsTable getTable(String name) throws Exception {
    // add v3 so that all the tests are performed for v3 table
    DatasetId metricsDatasetInstanceId = getDatasetId(name);
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(true).build();
    return DatasetsUtil.getOrCreateDataset(dsFramework, metricsDatasetInstanceId, MetricsTable.class.getName(), props, null);
}
Also used : MetricsTable(io.cdap.cdap.data2.dataset2.lib.table.MetricsTable) DatasetProperties(io.cdap.cdap.api.dataset.DatasetProperties) DatasetId(io.cdap.cdap.proto.id.DatasetId)

Example 35 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class HBaseMetricsTableTest method testConcurrentIncrement.

@Override
@Test
public void testConcurrentIncrement() throws Exception {
    String testConcurrentIncrement = "testConcurrentIncrement";
    final MetricsTable table = getTable(testConcurrentIncrement);
    final int rounds = 500;
    Map<byte[], Long> inc1 = ImmutableMap.of(X, 1L, Y, 2L);
    Map<byte[], Long> inc2 = ImmutableMap.of(Y, 1L, Z, 2L);
    // HTable used by HBaseMetricsTable is not thread safe, so each thread must use a separate instance
    // HBaseMetricsTable does not support mixed increment and incrementAndGet so the
    // updates and assertions here are different from MetricsTableTest.testConcurrentIncrement()
    Collection<? extends Thread> threads = ImmutableList.of(new IncThread(getTable(testConcurrentIncrement), A, inc1, rounds), new IncThread(getTable(testConcurrentIncrement), A, inc2, rounds), new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 5, rounds), new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 2, rounds));
    for (Thread t : threads) {
        t.start();
    }
    for (Thread t : threads) {
        t.join();
        if (t instanceof Closeable) {
            ((Closeable) t).close();
        }
    }
    assertEquals(rounds, Bytes.toLong(table.get(A, X)));
    assertEquals(3 * rounds, Bytes.toLong(table.get(A, Y)));
    assertEquals(2 * rounds, Bytes.toLong(table.get(A, Z)));
    assertEquals(7 * rounds, Bytes.toLong(table.get(A, R)));
}
Also used : MetricsTable(io.cdap.cdap.data2.dataset2.lib.table.MetricsTable) Closeable(java.io.Closeable) MetricsTableTest(io.cdap.cdap.data2.dataset2.lib.table.MetricsTableTest) Test(org.junit.Test)

Aggregations

MetricsTable (co.cask.cdap.data2.dataset2.lib.table.MetricsTable)19 Test (org.junit.Test)16 MetricsTable (io.cdap.cdap.data2.dataset2.lib.table.MetricsTable)15 DatasetId (co.cask.cdap.proto.id.DatasetId)6 AbstractModule (com.google.inject.AbstractModule)6 InMemoryMetricsTable (io.cdap.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable)6 InMemoryMetricsTable (co.cask.cdap.data2.dataset2.lib.table.inmemory.InMemoryMetricsTable)5 MetricsTableTest (co.cask.cdap.data2.dataset2.lib.table.MetricsTableTest)4 CombinedHBaseMetricsTable (co.cask.cdap.data2.dataset2.lib.table.hbase.CombinedHBaseMetricsTable)4 DatasetManagementException (co.cask.cdap.api.dataset.DatasetManagementException)3 DatasetModule (co.cask.cdap.api.dataset.module.DatasetModule)3 Row (co.cask.cdap.api.dataset.table.Row)3 Scanner (co.cask.cdap.api.dataset.table.Scanner)3 Table (co.cask.cdap.api.dataset.table.Table)3 Constants (co.cask.cdap.common.conf.Constants)3 ExternalDatasetModule (co.cask.cdap.data2.dataset2.lib.external.ExternalDatasetModule)3 MetadataDatasetModule (co.cask.cdap.data2.metadata.dataset.MetadataDatasetModule)3 LineageDatasetModule (co.cask.cdap.data2.metadata.lineage.LineageDatasetModule)3 UsageDatasetModule (co.cask.cdap.data2.registry.UsageDatasetModule)3 HBaseQueueDatasetModule (co.cask.cdap.data2.transaction.queue.hbase.HBaseQueueDatasetModule)3