Examples with FileInfo org.smartdata.model.FileInfo used on opensource projects

Example 41 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestRuleManager method testMultiThreadChangeState.

@Test
public void testMultiThreadChangeState() throws Exception {
    String rule = "file: every 1s \n | length > 10 | cache";
    long now = System.currentTimeMillis();
    long length = 100;
    long fid = 10000;
    FileInfo[] files = { new FileInfo("/tmp/testfile", fid, length, false, (short) 3, 1024, now, now, (short) 1, null, null, (byte) 3, (byte) 0) };
    metaStore.insertFiles(files);
    long rid = ruleManager.submitRule(rule, RuleState.ACTIVE);
    long start = System.currentTimeMillis();
    int nThreads = 2;
    Thread[] threads = new Thread[nThreads];
    for (int i = 0; i < nThreads; i++) {
        threads[i] = new Thread(new StateChangeWorker(rid));
    }
    for (Thread t : threads) {
        t.start();
    }
    for (Thread t : threads) {
        t.join();
    }
    long end = System.currentTimeMillis();
    System.out.println("Time used = " + (end - start) + " ms");
    // This is needed due to async threads
    Thread.sleep(1000);
    RuleInfo res = ruleManager.getRuleInfo(rid);
    System.out.println(res);
    Thread.sleep(5000);
    RuleInfo after = ruleManager.getRuleInfo(rid);
    System.out.println(after);
    if (res.getState() == RuleState.ACTIVE) {
        Assert.assertTrue(after.getNumCmdsGen() - res.getNumCmdsGen() <= 6);
    } else {
        Assert.assertTrue(after.getNumCmdsGen() == res.getNumCmdsGen());
    }
}

Example 42 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestMetaStore method testGetFiles.

@Test
public void testGetFiles() throws Exception {
    String pathString = "/tmp/des";
    long length = 123L;
    boolean isDir = false;
    int blockReplication = 1;
    long blockSize = 128 * 1024L;
    long modTime = 123123123L;
    long accessTime = 123123120L;
    String owner = "root";
    String group = "admin";
    long fileId = 56L;
    byte storagePolicy = 0;
    byte erasureCodingPolicy = 0;
    FileInfo fileInfo = new FileInfo(pathString, fileId, length, isDir, (short) blockReplication, blockSize, modTime, accessTime, (short) 1, owner, group, storagePolicy, erasureCodingPolicy);
    metaStore.insertFile(fileInfo);
    FileInfo dbFileInfo = metaStore.getFile(56);
    Assert.assertTrue(dbFileInfo.equals(fileInfo));
    dbFileInfo = metaStore.getFile("/tmp/des");
    Assert.assertTrue(dbFileInfo.equals(fileInfo));
}

Example 43 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method prepareFiles.

private void prepareFiles(MetaStore metaStore) throws MetaStoreException {
    List<FileInfo> statusInternals = new ArrayList<>();
    for (int id = 1; id < 4; id++) {
        statusInternals.add(new FileInfo("file" + id, id, 123L, false, (short) 1, 128 * 1024L, 123123123L, 123123120L, (short) 1, "root", "admin", (byte) 0, (byte) 0));
    }
    metaStore.insertFiles(statusInternals.toArray(new FileInfo[0]));
}

Example 44 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestFileInfoDao method testInsetGetDeleteFiles.

@Test
public void testInsetGetDeleteFiles() throws Exception {
    String path = "/testFile";
    long length = 123L;
    boolean isDir = false;
    short blockReplication = 1;
    long blockSize = 128 * 1024L;
    long modTime = 123123123L;
    long accessTime = 123123120L;
    short permission = 1;
    String owner = "root";
    String group = "admin";
    long fileId = 312321L;
    byte storagePolicy = 0;
    byte erasureCodingPolicy = 0;
    FileInfo fileInfo = new FileInfo(path, fileId, length, isDir, blockReplication, blockSize, modTime, accessTime, permission, owner, group, storagePolicy, erasureCodingPolicy);
    fileInfoDao.insert(fileInfo);
    FileInfo file1 = fileInfoDao.getByPath("/testFile");
    Assert.assertTrue(fileInfo.equals(file1));
    FileInfo file2 = fileInfoDao.getById(fileId);
    Assert.assertTrue(fileInfo.equals(file2));
    FileInfo fileInfo1 = new FileInfo(path, fileId + 1, length, isDir, blockReplication, blockSize, modTime, accessTime, permission, owner, group, storagePolicy, erasureCodingPolicy);
    fileInfoDao.insert(fileInfo1);
    List<FileInfo> fileInfos = fileInfoDao.getFilesByPrefix("/testaaFile");
    Assert.assertTrue(fileInfos.size() == 0);
    fileInfos = fileInfoDao.getFilesByPrefix("/testFile");
    Assert.assertTrue(fileInfos.size() == 2);
    fileInfoDao.deleteById(fileId);
    fileInfos = fileInfoDao.getAll();
    Assert.assertTrue(fileInfos.size() == 1);
    fileInfoDao.deleteAll();
    fileInfos = fileInfoDao.getAll();
    Assert.assertTrue(fileInfos.size() == 0);
}

Example 45 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class SmallFilePlugin method getValidSmallFiles.

/**
 * Get valid small files according to container file.
 */
private List<String> getValidSmallFiles(FileInfo containerFileInfo, List<String> smallFileList) throws MetaStoreException {
    // Get container file len
    long containerFileLen = containerFileInfo.getLength();
    // Sort small file list for getting most eligible small files
    List<String> ret = new ArrayList<>();
    List<FileInfo> smallFileInfos = metaStore.getFilesByPaths(smallFileList);
    Collections.sort(smallFileInfos, new Comparator<FileInfo>() {

        @Override
        public int compare(FileInfo a, FileInfo b) {
            return Long.compare(a.getLength(), b.getLength());
        }
    });
    // Get small files can be compacted to container file
    for (FileInfo fileInfo : smallFileInfos) {
        long fileLen = fileInfo.getLength();
        if (fileLen > 0) {
            containerFileLen += fileLen;
            if (containerFileLen < containerFileSizeThreshold * 1.2) {
                ret.add(fileInfo.getPath());
            }
            if (containerFileLen >= containerFileSizeThreshold) {
                break;
            }
        }
    }
    if (!ret.isEmpty()) {
        return ret;
    } else {
        return null;
    }
}