Examples with MetaStore org.smartdata.metastore.MetaStore used on opensource projects

Example 26 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testAccessCountTableManager.

@Test
public void testAccessCountTableManager() throws InterruptedException {
    MetaStore adapter = mock(MetaStore.class);
    // Used by AccessCountTableAggregator
    AccessCountTableManager manager = new AccessCountTableManager(adapter);
    Long firstDayEnd = 24 * 60 * 60 * 1000L;
    AccessCountTable accessCountTable = new AccessCountTable(firstDayEnd - 5 * 1000, firstDayEnd);
    manager.addTable(accessCountTable);
    Thread.sleep(5000);
    Map<TimeGranularity, AccessCountTableDeque> map = manager.getTableDeques();
    AccessCountTableDeque second = map.get(TimeGranularity.SECOND);
    Assert.assertTrue(second.size() == 1);
    Assert.assertEquals(second.peek(), accessCountTable);
    AccessCountTableDeque minute = map.get(TimeGranularity.MINUTE);
    AccessCountTable minuteTable = new AccessCountTable(firstDayEnd - 60 * 1000, firstDayEnd);
    Assert.assertTrue(minute.size() == 1);
    Assert.assertEquals(minute.peek(), minuteTable);
    AccessCountTableDeque hour = map.get(TimeGranularity.HOUR);
    AccessCountTable hourTable = new AccessCountTable(firstDayEnd - 60 * 60 * 1000, firstDayEnd);
    Assert.assertTrue(hour.size() == 1);
    Assert.assertEquals(hour.peek(), hourTable);
    AccessCountTableDeque day = map.get(TimeGranularity.DAY);
    AccessCountTable dayTable = new AccessCountTable(firstDayEnd - 24 * 60 * 60 * 1000, firstDayEnd);
    Assert.assertTrue(day.size() == 1);
    Assert.assertEquals(day.peek(), dayTable);
}

Example 27 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testGetTablesCornerCase2.

@Test
public void testGetTablesCornerCase2() throws MetaStoreException {
    MetaStore adapter = mock(MetaStore.class);
    TableEvictor tableEvictor = new CountEvictor(adapter, 20);
    Map<TimeGranularity, AccessCountTableDeque> map = new HashMap<>();
    AccessCountTableDeque minute = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstMinute = new AccessCountTable(0L, Constants.ONE_MINUTE_IN_MILLIS);
    minute.addAndNotifyListener(firstMinute);
    map.put(TimeGranularity.MINUTE, minute);
    AccessCountTableDeque secondDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstFiveSeconds = new AccessCountTable(55 * Constants.ONE_SECOND_IN_MILLIS, 60 * Constants.ONE_SECOND_IN_MILLIS);
    AccessCountTable secondFiveSeconds = new AccessCountTable(60 * Constants.ONE_SECOND_IN_MILLIS, 65 * Constants.ONE_SECOND_IN_MILLIS);
    AccessCountTable thirdFiveSeconds = new AccessCountTable(110 * Constants.ONE_SECOND_IN_MILLIS, 115 * Constants.ONE_SECOND_IN_MILLIS);
    secondDeque.addAndNotifyListener(firstFiveSeconds);
    secondDeque.addAndNotifyListener(secondFiveSeconds);
    secondDeque.addAndNotifyListener(thirdFiveSeconds);
    map.put(TimeGranularity.SECOND, secondDeque);
    List<AccessCountTable> result = AccessCountTableManager.getTables(map, adapter, Constants.ONE_MINUTE_IN_MILLIS);
    Assert.assertTrue(result.size() == 3);
    Assert.assertTrue(result.get(0).equals(firstFiveSeconds));
    Assert.assertFalse(result.get(0).isEphemeral());
    Assert.assertTrue(result.get(1).equals(secondFiveSeconds));
    Assert.assertTrue(result.get(2).equals(thirdFiveSeconds));
}

Example 28 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestTableAggregator method testGetTopN.

@Test
public void testGetTopN() throws Exception {
    createTables(databaseTester.getConnection());
    IDataSet dataSet = new XmlDataSet(getClass().getClassLoader().getResourceAsStream("accessCountTable.xml"));
    databaseTester.setDataSet(dataSet);
    databaseTester.onSetup();
    MetaStore metaStore = new MetaStore(druidPool);
    prepareFiles(metaStore);
    AccessCountTable table1 = new AccessCountTable("table1", 0L, 0L, false);
    AccessCountTable table2 = new AccessCountTable("table2", 0L, 0L, false);
    AccessCountTable table3 = new AccessCountTable("table3", 0L, 0L, false);
    List<FileAccessInfo> accessInfos = metaStore.getHotFiles(Arrays.asList(table1, table2, table3), 1);
    Assert.assertTrue(accessInfos.size() == 1);
    FileAccessInfo expected1 = new FileAccessInfo(103L, "/file3", 7);
    Assert.assertTrue(accessInfos.get(0).equals(expected1));
    List<FileAccessInfo> accessInfos2 = metaStore.getHotFiles(Arrays.asList(table1, table2, table3), 2);
    List<FileAccessInfo> expected2 = Arrays.asList(expected1, new FileAccessInfo(102L, "/file2", 6));
    Assert.assertTrue(accessInfos2.size() == expected2.size());
    Assert.assertTrue(accessInfos2.containsAll(expected2));
}

Example 29 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testAccessFileNotInNamespace.

@Test
public void testAccessFileNotInNamespace() throws Exception {
    AccessCountTableManager manager = initTestEnvironment();
    List<FileAccessEvent> accessEvents = new ArrayList<>();
    accessEvents.add(new FileAccessEvent("file1", 0));
    accessEvents.add(new FileAccessEvent("file2", 1));
    accessEvents.add(new FileAccessEvent("file2", 2));
    accessEvents.add(new FileAccessEvent("file3", 2));
    accessEvents.add(new FileAccessEvent("file3", 3));
    accessEvents.add(new FileAccessEvent("file3", 4));
    accessEvents.add(new FileAccessEvent("file4", 5));
    accessEvents.add(new FileAccessEvent("file3", 5000));
    accessEvents.add(new FileAccessEvent("file3", 10000));
    manager.onAccessEventsArrived(accessEvents);
    assertTableEquals(new AccessCountTable(0L, 5000L).getTableName(), "expect1");
    assertTableEquals(new AccessCountTable(5000L, 10000L).getTableName(), "expect2");
    accessEvents.clear();
    accessEvents.add(new FileAccessEvent("file4", 10001));
    accessEvents.add(new FileAccessEvent("file4", 10002));
    accessEvents.add(new FileAccessEvent("file3", 15000));
    accessEvents.add(new FileAccessEvent("file4", 15001));
    accessEvents.add(new FileAccessEvent("file3", 20000));
    manager.onAccessEventsArrived(accessEvents);
    assertTableEquals(new AccessCountTable(10000L, 15000L).getTableName(), "expect2");
    assertTableEquals(new AccessCountTable(15000L, 20000L).getTableName(), "expect2");
    insertNewFile(new MetaStore(druidPool), "file4", 4L);
    accessEvents.clear();
    accessEvents.add(new FileAccessEvent("file4", 25000));
    manager.onAccessEventsArrived(accessEvents);
    assertTableEquals(new AccessCountTable(20000L, 25000L).getTableName(), "expect3");
}