Examples with TimeGranularity org.smartdata.metastore.utils.TimeGranularity used on opensource projects

Example 1 with TimeGranularity

use of org.smartdata.metastore.utils.TimeGranularity in project SSM by Intel-bigdata.

the class AccessCountTableManager method getTablesDuring.

// Todo: multi-thread issue
private static List<AccessCountTable> getTablesDuring(final Map<TimeGranularity, AccessCountTableDeque> tableDeques, MetaStore metaStore, final long length, final long endTime, final TimeGranularity timeGranularity) throws MetaStoreException {
    long startTime = endTime - length;
    AccessCountTableDeque tables = tableDeques.get(timeGranularity);
    List<AccessCountTable> results = new ArrayList<>();
    for (Iterator<AccessCountTable> iterator = tables.iterator(); iterator.hasNext(); ) {
        // Here we assume that the tables are all sorted by time.
        AccessCountTable table = iterator.next();
        if (table.getEndTime() > startTime) {
            if (table.getStartTime() >= startTime) {
                results.add(table);
                startTime = table.getEndTime();
            } else if (table.getStartTime() < startTime) {
                // table that already exists, so this situation should be avoided.
                if (!tableExists(tableDeques, startTime, table.getEndTime())) {
                    AccessCountTable splitTable = new AccessCountTable(startTime, table.getEndTime(), true);
                    metaStore.createProportionTable(splitTable, table);
                    results.add(splitTable);
                    startTime = table.getEndTime();
                }
            }
        }
    }
    if (startTime != endTime && !timeGranularity.equals(TimeGranularity.SECOND)) {
        TimeGranularity fineGrained = TimeUtils.getFineGarinedGranularity(timeGranularity);
        results.addAll(getTablesDuring(tableDeques, metaStore, endTime - startTime, endTime, fineGrained));
    }
    return results;
}

Example 2 with TimeGranularity

use of org.smartdata.metastore.utils.TimeGranularity in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testGetTablesCornerCase.

@Test
public void testGetTablesCornerCase() throws MetaStoreException {
    MetaStore adapter = mock(MetaStore.class);
    TableEvictor tableEvictor = new CountEvictor(adapter, 20);
    Map<TimeGranularity, AccessCountTableDeque> map = new HashMap<>();
    AccessCountTableDeque minute = new AccessCountTableDeque(tableEvictor);
    map.put(TimeGranularity.MINUTE, minute);
    AccessCountTableDeque secondDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstFiveSeconds = new AccessCountTable(0L, 5 * Constants.ONE_SECOND_IN_MILLIS);
    AccessCountTable secondFiveSeconds = new AccessCountTable(5 * Constants.ONE_SECOND_IN_MILLIS, 10 * Constants.ONE_SECOND_IN_MILLIS);
    secondDeque.addAndNotifyListener(firstFiveSeconds);
    secondDeque.addAndNotifyListener(secondFiveSeconds);
    map.put(TimeGranularity.SECOND, secondDeque);
    List<AccessCountTable> result = AccessCountTableManager.getTables(map, adapter, 2 * Constants.ONE_MINUTE_IN_MILLIS);
    Assert.assertTrue(result.size() == 2);
    Assert.assertTrue(result.get(0).equals(firstFiveSeconds));
    Assert.assertTrue(result.get(1).equals(secondFiveSeconds));
}

Example 3 with TimeGranularity

use of org.smartdata.metastore.utils.TimeGranularity in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testGetTables.

@Test
public void testGetTables() throws MetaStoreException {
    MetaStore adapter = mock(MetaStore.class);
    TableEvictor tableEvictor = new CountEvictor(adapter, 20);
    Map<TimeGranularity, AccessCountTableDeque> map = new HashMap<>();
    AccessCountTableDeque dayDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstDay = new AccessCountTable(0L, Constants.ONE_DAY_IN_MILLIS);
    dayDeque.addAndNotifyListener(firstDay);
    map.put(TimeGranularity.DAY, dayDeque);
    AccessCountTableDeque hourDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstHour = new AccessCountTable(23 * Constants.ONE_HOUR_IN_MILLIS, 24 * Constants.ONE_HOUR_IN_MILLIS);
    AccessCountTable secondHour = new AccessCountTable(24 * Constants.ONE_HOUR_IN_MILLIS, 25 * Constants.ONE_HOUR_IN_MILLIS);
    hourDeque.addAndNotifyListener(firstHour);
    hourDeque.addAndNotifyListener(secondHour);
    map.put(TimeGranularity.HOUR, hourDeque);
    AccessCountTableDeque minuteDeque = new AccessCountTableDeque(tableEvictor);
    Integer numMins = 25 * 60;
    AccessCountTable firstMin = new AccessCountTable((numMins - 1) * Constants.ONE_MINUTE_IN_MILLIS, numMins * Constants.ONE_MINUTE_IN_MILLIS);
    AccessCountTable secondMin = new AccessCountTable(numMins * Constants.ONE_MINUTE_IN_MILLIS, (numMins + 1) * Constants.ONE_MINUTE_IN_MILLIS);
    minuteDeque.addAndNotifyListener(firstMin);
    minuteDeque.addAndNotifyListener(secondMin);
    map.put(TimeGranularity.MINUTE, minuteDeque);
    AccessCountTableDeque secondDeque = new AccessCountTableDeque(tableEvictor);
    Integer numSeconds = (25 * 60 + 1) * 60;
    AccessCountTable firstFiveSeconds = new AccessCountTable((numSeconds - 5) * Constants.ONE_SECOND_IN_MILLIS, numSeconds * Constants.ONE_SECOND_IN_MILLIS);
    AccessCountTable secondFiveSeconds = new AccessCountTable(numSeconds * Constants.ONE_SECOND_IN_MILLIS, (numSeconds + 5) * Constants.ONE_SECOND_IN_MILLIS);
    secondDeque.addAndNotifyListener(firstFiveSeconds);
    secondDeque.addAndNotifyListener(secondFiveSeconds);
    map.put(TimeGranularity.SECOND, secondDeque);
    List<AccessCountTable> firstResult = AccessCountTableManager.getTables(map, adapter, (numSeconds + 5) * Constants.ONE_SECOND_IN_MILLIS);
    Assert.assertTrue(firstResult.size() == 4);
    Assert.assertEquals(firstResult.get(0), firstDay);
    Assert.assertEquals(firstResult.get(1), secondHour);
    Assert.assertEquals(firstResult.get(2), secondMin);
    Assert.assertEquals(firstResult.get(3), secondFiveSeconds);
    List<AccessCountTable> secondResult = AccessCountTableManager.getTables(map, adapter, numSeconds * Constants.ONE_SECOND_IN_MILLIS);
    Assert.assertTrue(secondResult.size() == 4);
    AccessCountTable expectDay = new AccessCountTable(5 * Constants.ONE_SECOND_IN_MILLIS, Constants.ONE_DAY_IN_MILLIS);
    Assert.assertEquals(expectDay, secondResult.get(0));
    List<AccessCountTable> thirdResult = AccessCountTableManager.getTables(map, adapter, secondFiveSeconds.getEndTime() - 23 * Constants.ONE_HOUR_IN_MILLIS);
    Assert.assertTrue(thirdResult.size() == 4);
    Assert.assertEquals(thirdResult.get(0), firstHour);
    List<AccessCountTable> fourthResult = AccessCountTableManager.getTables(map, adapter, secondFiveSeconds.getEndTime() - 24 * Constants.ONE_HOUR_IN_MILLIS);
    Assert.assertTrue(fourthResult.size() == 3);
    Assert.assertEquals(fourthResult.get(0), secondHour);
}

Example 4 with TimeGranularity

use of org.smartdata.metastore.utils.TimeGranularity in project SSM by Intel-bigdata.

the class AccessCountTableManager method tableExists.

private static boolean tableExists(final Map<TimeGranularity, AccessCountTableDeque> tableDeques, long start, long end) {
    TimeGranularity granularity = TimeUtils.getGranularity(end - start);
    AccessCountTable fakeTable = new AccessCountTable(start, end);
    return tableDeques.containsKey(granularity) && tableDeques.get(granularity).contains(fakeTable);
}

Example 5 with TimeGranularity

use of org.smartdata.metastore.utils.TimeGranularity in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testAccessCountTableManager.

@Test
public void testAccessCountTableManager() throws InterruptedException {
    MetaStore adapter = mock(MetaStore.class);
    // Used by AccessCountTableAggregator
    AccessCountTableManager manager = new AccessCountTableManager(adapter);
    Long firstDayEnd = 24 * 60 * 60 * 1000L;
    AccessCountTable accessCountTable = new AccessCountTable(firstDayEnd - 5 * 1000, firstDayEnd);
    manager.addTable(accessCountTable);
    Thread.sleep(5000);
    Map<TimeGranularity, AccessCountTableDeque> map = manager.getTableDeques();
    AccessCountTableDeque second = map.get(TimeGranularity.SECOND);
    Assert.assertTrue(second.size() == 1);
    Assert.assertEquals(second.peek(), accessCountTable);
    AccessCountTableDeque minute = map.get(TimeGranularity.MINUTE);
    AccessCountTable minuteTable = new AccessCountTable(firstDayEnd - 60 * 1000, firstDayEnd);
    Assert.assertTrue(minute.size() == 1);
    Assert.assertEquals(minute.peek(), minuteTable);
    AccessCountTableDeque hour = map.get(TimeGranularity.HOUR);
    AccessCountTable hourTable = new AccessCountTable(firstDayEnd - 60 * 60 * 1000, firstDayEnd);
    Assert.assertTrue(hour.size() == 1);
    Assert.assertEquals(hour.peek(), hourTable);
    AccessCountTableDeque day = map.get(TimeGranularity.DAY);
    AccessCountTable dayTable = new AccessCountTable(firstDayEnd - 24 * 60 * 60 * 1000, firstDayEnd);
    Assert.assertTrue(day.size() == 1);
    Assert.assertEquals(day.peek(), dayTable);
}