Examples with CachedFileStatus org.smartdata.model.CachedFileStatus used on opensource projects

Example 1 with CachedFileStatus

use of org.smartdata.model.CachedFileStatus in project SSM by Intel-bigdata.

the class TestCacheFileDao method testGetCachedFileStatus.

@Test
public void testGetCachedFileStatus() throws Exception {
    cacheFileDao.insert(6L, "testPath", 1490918400000L, 234567L, 456);
    CachedFileStatus cachedFileStatus = new CachedFileStatus(6L, "testPath", 1490918400000L, 234567L, 456);
    cacheFileDao.insert(19L, "testPath", 1490918400000L, 234567L, 456);
    cacheFileDao.insert(23L, "testPath", 1490918400000L, 234567L, 456);
    CachedFileStatus dbcachedFileStatus = cacheFileDao.getById(6);
    Assert.assertTrue(dbcachedFileStatus.equals(cachedFileStatus));
    List<CachedFileStatus> cachedFileList = cacheFileDao.getAll();
    List<Long> fids = cacheFileDao.getFids();
    Assert.assertTrue(fids.size() == 3);
    Assert.assertTrue(cachedFileList.get(0).getFid() == 6);
    Assert.assertTrue(cachedFileList.get(1).getFid() == 19);
    Assert.assertTrue(cachedFileList.get(2).getFid() == 23);
}

Example 2 with CachedFileStatus

use of org.smartdata.model.CachedFileStatus in project SSM by Intel-bigdata.

the class TestMetaStore method testInsertDeleteCachedFiles.

@Test
public void testInsertDeleteCachedFiles() throws Exception {
    metaStore.insertCachedFiles(80L, "testPath", 123456L, 234567L, 456);
    Assert.assertTrue(metaStore.getCachedFileStatus(80L).getFromTime() == 123456L);
    // Update record with 80l id
    Assert.assertTrue(metaStore.updateCachedFiles(80L, 234568L, 460));
    Assert.assertTrue(metaStore.getCachedFileStatus().get(0).getLastAccessTime() == 234568L);
    List<CachedFileStatus> list = new LinkedList<>();
    list.add(new CachedFileStatus(321L, "testPath", 113334L, 222222L, 222));
    metaStore.insertCachedFiles(list);
    Assert.assertTrue(metaStore.getCachedFileStatus(321L).getNumAccessed() == 222);
    Assert.assertTrue(metaStore.getCachedFileStatus().size() == 2);
    // Delete one record
    metaStore.deleteCachedFile(321L);
    Assert.assertTrue(metaStore.getCachedFileStatus().size() == 1);
    // Clear all records
    metaStore.deleteAllCachedFile();
    Assert.assertTrue(metaStore.getCachedFileStatus().size() == 0);
    metaStore.insertCachedFiles(80L, "testPath", 123456L, 234567L, 456);
}

Example 3 with CachedFileStatus

use of org.smartdata.model.CachedFileStatus in project SSM by Intel-bigdata.

the class TestCachedListFetcher method testFetcher.

@Test
public void testFetcher() throws Exception {
    String pathPrefix = "/fileTest/cache/";
    String[] fids = { "5", "7", "9", "10" };
    Path dir = new Path(pathPrefix);
    dfs.mkdirs(dir);
    dfs.setStoragePolicy(dir, "HOT");
    List<FileInfo> fileInfos = new ArrayList<>();
    for (int i = 0; i < fids.length; i++) {
        CacheFileAction cacheAction = new CacheFileAction();
        String path = pathPrefix + fids[i];
        FSDataOutputStream out = dfs.create(new Path(path));
        out.writeChars("testUncache");
        out.close();
        // Create cache pool
        CacheScheduler.createCachePool(dfsClient);
        fileInfos.add(createFileStatus(pathPrefix + fids[i]));
        cacheAction.setContext(smartContext);
        cacheAction.setDfsClient(dfsClient);
        Map<String, String> args = new HashMap();
        args.put(CacheFileAction.FILE_PATH, path);
        cacheAction.init(args);
        cacheAction.run();
    // System.out.println(cacheAction.isCached(path));
    }
    metaStore.insertFiles(fileInfos.toArray(new FileInfo[fileInfos.size()]));
    List<FileInfo> ret = metaStore.getFile();
    Assert.assertTrue(ret.size() == fids.length);
    cachedListFetcher.start();
    Thread.sleep(1000);
    List<CachedFileStatus> cachedFileStatuses = cachedListFetcher.getCachedList();
    Assert.assertTrue(cachedFileStatuses.size() == fids.length);
    int unCachedSize = 2;
    for (int i = 0; i < unCachedSize; i++) {
        UncacheFileAction uncacheFileAction = new UncacheFileAction();
        String path = pathPrefix + fids[i];
        fileInfos.add(createFileStatus("fileTest/cache/" + fids[i]));
        uncacheFileAction.setContext(smartContext);
        uncacheFileAction.setDfsClient(dfsClient);
        Map<String, String> args = new HashMap();
        args.put(UncacheFileAction.FILE_PATH, path);
        uncacheFileAction.init(args);
        uncacheFileAction.run();
    }
    // System.out.println(uncacheFileAction .isCached(path));
    Thread.sleep(2000);
    cachedFileStatuses = cachedListFetcher.getCachedList();
    Assert.assertTrue(cachedFileStatuses.size() == fids.length - unCachedSize);
}

Example 4 with CachedFileStatus

use of org.smartdata.model.CachedFileStatus in project SSM by Intel-bigdata.

the class CacheFileDao method insert.

public void insert(long fid, String path, long fromTime, long lastAccessTime, int numAccessed) {
    SimpleJdbcInsert simpleJdbcInsert = new SimpleJdbcInsert(dataSource);
    simpleJdbcInsert.setTableName("cached_file");
    simpleJdbcInsert.execute(toMap(new CachedFileStatus(fid, path, fromTime, lastAccessTime, numAccessed)));
}

Example 5 with CachedFileStatus

use of org.smartdata.model.CachedFileStatus in project SSM by Intel-bigdata.

the class CacheFileDao method update.

public void update(Map<String, Long> pathToIds, List<FileAccessEvent> events) {
    Map<Long, CachedFileStatus> idToStatus = new HashMap<>();
    List<CachedFileStatus> cachedFileStatuses = getAll();
    for (CachedFileStatus status : cachedFileStatuses) {
        idToStatus.put(status.getFid(), status);
    }
    Collection<Long> cachedIds = idToStatus.keySet();
    Collection<Long> needToUpdate = CollectionUtils.intersection(cachedIds, pathToIds.values());
    if (!needToUpdate.isEmpty()) {
        Map<Long, Integer> idToCount = new HashMap<>();
        Map<Long, Long> idToLastTime = new HashMap<>();
        for (FileAccessEvent event : events) {
            Long fid = pathToIds.get(event.getPath());
            if (needToUpdate.contains(fid)) {
                if (!idToCount.containsKey(fid)) {
                    idToCount.put(fid, 0);
                }
                idToCount.put(fid, idToCount.get(fid) + 1);
                if (!idToLastTime.containsKey(fid)) {
                    idToLastTime.put(fid, event.getTimestamp());
                }
                idToLastTime.put(fid, Math.max(event.getTimestamp(), idToLastTime.get(fid)));
            }
        }
        for (Long fid : needToUpdate) {
            Integer newAccessCount = idToStatus.get(fid).getNumAccessed() + idToCount.get(fid);
            this.update(fid, idToLastTime.get(fid), newAccessCount);
        }
    }
}