Examples with S3FileState org.smartdata.model.S3FileState used on opensource projects

Example 1 with S3FileState

use of org.smartdata.model.S3FileState in project SSM by Intel-bigdata.

the class TestCopy2S3Scheduler method testOnS3.

@Test(timeout = 45000)
public void testOnS3() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    SmartAdmin admin = new SmartAdmin(smartContext.getConf());
    DistributedFileSystem dfs = cluster.getFileSystem();
    final String srcPath = "/src/";
    dfs.mkdirs(new Path(srcPath));
    List<String> sps = new ArrayList<>();
    // Write to src
    for (int i = 0; i < 3; i++) {
        // Create test files
        // Not 0 because this file may be not be truncated yet
        sps.add(srcPath + i);
        DFSTestUtil.createFile(dfs, new Path(srcPath + i), 10, (short) 1, 1);
    }
    do {
        Thread.sleep(1000);
        if (metaStore.getFilesByPaths(sps).size() == sps.size()) {
            break;
        }
    } while (true);
    for (String p : sps) {
        metaStore.insertUpdateFileState(new S3FileState(p));
    }
    long ruleId = admin.submitRule("file: path matches \"/src/*\"| copy2s3 -dest s3a://xxxctest/dest/", RuleState.ACTIVE);
    Thread.sleep(2500);
    List<ActionInfo> actions = metaStore.getActions(ruleId, 0);
    Assert.assertEquals(0, actions.size());
}

Example 2 with S3FileState

use of org.smartdata.model.S3FileState in project SSM by Intel-bigdata.

the class Copy2S3Scheduler method onActionFinished.

@Override
public void onActionFinished(CmdletInfo cmdletInfo, ActionInfo actionInfo, int actionIndex) {
    String path = actionInfo.getArgs().get(HdfsAction.FILE_PATH);
    if (actionInfo.isFinished() && actionInfo.isSuccessful()) {
        // Insert fileState
        try {
            metaStore.insertUpdateFileState(new S3FileState(path));
        } catch (MetaStoreException e) {
            LOG.error("Failed to insert file state.", e);
        }
    }
    // unlock filelock
    if (ifLocked(path)) {
        unLockTheFile(path);
        LOG.debug("unlocked copy2s3 file {}", path);
    }
}

Example 3 with S3FileState

use of org.smartdata.model.S3FileState in project SSM by Intel-bigdata.

the class MetaStore method getFileState.

/**
 * Get FileState of the given path.
 *
 * @param path
 * @return
 * @throws MetaStoreException
 */
public FileState getFileState(String path) throws MetaStoreException {
    FileState fileState;
    try {
        fileState = fileStateDao.getByPath(path);
        // Fetch info from corresponding table to regenerate a specific file state
        switch(fileState.getFileType()) {
            case NORMAL:
                fileState = new NormalFileState(path);
                break;
            case COMPACT:
                fileState = smallFileDao.getFileStateByPath(path);
                break;
            case COMPRESSION:
                CompressionFileState compressionFileState = getCompressionInfo(path);
                if (compressionFileState != null) {
                    compressionFileState.setFileStage(fileState.getFileStage());
                    fileState = compressionFileState;
                }
                break;
            case S3:
                fileState = new S3FileState(path);
                break;
            default:
        }
    } catch (EmptyResultDataAccessException e1) {
        fileState = new NormalFileState(path);
    } catch (Exception e2) {
        throw new MetaStoreException(e2);
    }
    return fileState;
}