Examples with ActionInfo org.smartdata.model.ActionInfo used on opensource projects

Example 6 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class TestCmdletManager method testWithoutCluster.

@Test
public void testWithoutCluster() throws MetaStoreException, IOException, InterruptedException {
    long cmdletId = 10;
    long actionId = 101;
    MetaStore metaStore = mock(MetaStore.class);
    Assert.assertNotNull(metaStore);
    when(metaStore.getMaxCmdletId()).thenReturn(cmdletId);
    when(metaStore.getMaxActionId()).thenReturn(actionId);
    CmdletDispatcher dispatcher = mock(CmdletDispatcher.class);
    Assert.assertNotNull(dispatcher);
    when(dispatcher.canDispatchMore()).thenReturn(true);
    ServerContext serverContext = new ServerContext(new SmartConf(), metaStore);
    serverContext.setServiceMode(ServiceMode.HDFS);
    CmdletManager cmdletManager = new CmdletManager(serverContext);
    cmdletManager.init();
    cmdletManager.setDispatcher(dispatcher);
    cmdletManager.start();
    cmdletManager.submitCmdlet("echo");
    Thread.sleep(500);
    verify(metaStore, times(1)).insertCmdlets(any(CmdletInfo[].class));
    verify(metaStore, times(1)).insertActions(any(ActionInfo[].class));
    Thread.sleep(500);
    long startTime = System.currentTimeMillis();
    ActionStatus actionStatus = new ActionStatus(cmdletId, true, actionId, startTime, null);
    StatusReport statusReport = new StatusReport(Arrays.asList(actionStatus));
    cmdletManager.updateStatus(statusReport);
    ActionInfo actionInfo = cmdletManager.getActionInfo(actionId);
    CmdletInfo cmdletInfo = cmdletManager.getCmdletInfo(cmdletId);
    Assert.assertNotNull(actionInfo);
    cmdletManager.updateStatus(new CmdletStatusUpdate(cmdletId, System.currentTimeMillis(), CmdletState.EXECUTING));
    CmdletInfo info = cmdletManager.getCmdletInfo(cmdletId);
    Assert.assertNotNull(info);
    Assert.assertEquals(info.getParameters(), "echo");
    Assert.assertEquals(info.getAids().size(), 1);
    Assert.assertTrue(info.getAids().get(0) == actionId);
    Assert.assertEquals(info.getState(), CmdletState.EXECUTING);
    long finishTime = System.currentTimeMillis();
    actionStatus = new ActionStatus(cmdletId, true, actionId, null, startTime, finishTime, null, true);
    statusReport = new StatusReport(Arrays.asList(actionStatus));
    cmdletManager.updateStatus(statusReport);
    Assert.assertTrue(actionInfo.isFinished());
    Assert.assertTrue(actionInfo.isSuccessful());
    Assert.assertEquals(actionInfo.getCreateTime(), startTime);
    Assert.assertEquals(actionInfo.getFinishTime(), finishTime);
    Assert.assertEquals(cmdletInfo.getState(), CmdletState.DONE);
    cmdletManager.updateStatus(new CmdletStatusUpdate(cmdletId, System.currentTimeMillis(), CmdletState.DONE));
    Assert.assertEquals(info.getState(), CmdletState.DONE);
    Thread.sleep(500);
    verify(metaStore, times(2)).insertCmdlets(any(CmdletInfo[].class));
    verify(metaStore, times(2)).insertActions(any(ActionInfo[].class));
    cmdletManager.stop();
}

Example 7 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class TestCopy2S3Scheduler method testDir.

@Test(timeout = 45000)
public void testDir() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    SmartAdmin admin = new SmartAdmin(smartContext.getConf());
    DistributedFileSystem dfs = cluster.getFileSystem();
    final String srcPath = "/src/";
    dfs.mkdirs(new Path(srcPath));
    // Write to src
    for (int i = 0; i < 3; i++) {
        // Create test files
        DFSTestUtil.createFile(dfs, new Path(srcPath + i), 1024, (short) 1, 1);
    }
    long ruleId = admin.submitRule("file: path matches \"/src/*\"| copy2s3 -dest s3a://xxxctest/dest/", RuleState.ACTIVE);
    List<ActionInfo> actions;
    do {
        actions = metaStore.getActions(ruleId, 0);
        Thread.sleep(1000);
    } while (actions.size() < 3);
}

Example 8 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class TestCopy2S3Scheduler method testOnS3.

@Test(timeout = 45000)
public void testOnS3() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    SmartAdmin admin = new SmartAdmin(smartContext.getConf());
    DistributedFileSystem dfs = cluster.getFileSystem();
    final String srcPath = "/src/";
    dfs.mkdirs(new Path(srcPath));
    List<String> sps = new ArrayList<>();
    // Write to src
    for (int i = 0; i < 3; i++) {
        // Create test files
        // Not 0 because this file may be not be truncated yet
        sps.add(srcPath + i);
        DFSTestUtil.createFile(dfs, new Path(srcPath + i), 10, (short) 1, 1);
    }
    do {
        Thread.sleep(1000);
        if (metaStore.getFilesByPaths(sps).size() == sps.size()) {
            break;
        }
    } while (true);
    for (String p : sps) {
        metaStore.insertUpdateFileState(new S3FileState(p));
    }
    long ruleId = admin.submitRule("file: path matches \"/src/*\"| copy2s3 -dest s3a://xxxctest/dest/", RuleState.ACTIVE);
    Thread.sleep(2500);
    List<ActionInfo> actions = metaStore.getActions(ruleId, 0);
    Assert.assertEquals(0, actions.size());
}

Example 9 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class CmdletManager method batchSyncCmdAction.

private void batchSyncCmdAction() throws Exception {
    if (cacheCmd.size() == 0 && tobeDeletedCmd.size() == 0) {
        return;
    }
    List<CmdletInfo> cmdletInfos = new ArrayList<>();
    List<ActionInfo> actionInfos = new ArrayList<>();
    List<CmdletInfo> cmdletFinished = new ArrayList<>();
    LOG.debug("Number of cached cmds {}", cacheCmd.size());
    int todelSize;
    synchronized (cacheCmd) {
        synchronized (tobeDeletedCmd) {
            todelSize = tobeDeletedCmd.size();
            for (Long cid : tobeDeletedCmd) {
                cacheCmd.remove(cid);
            }
        }
        for (Long cid : cacheCmd.keySet()) {
            CmdletInfo cmdletInfo = cacheCmd.remove(cid);
            if (cmdletInfo.getState() != CmdletState.DISABLED) {
                cmdletInfos.add(cmdletInfo);
                for (Long aid : cmdletInfo.getAids()) {
                    ActionInfo actionInfo = idToActions.get(aid);
                    if (actionInfo != null) {
                        actionInfos.add(actionInfo);
                    }
                }
            }
            if (CmdletState.isTerminalState(cmdletInfo.getState())) {
                cmdletFinished.add(cmdletInfo);
            }
            if (cmdletInfos.size() >= cacheCmdTh) {
                break;
            }
        }
        for (CmdletInfo cmdletInfo : cmdletFinished) {
            idToCmdlets.remove(cmdletInfo.getCid());
            try {
                tracker.untrack(cmdletInfo.getCid());
            } catch (Exception e) {
                LOG.warn("Failed to untrack task!", e);
            }
            for (Long aid : cmdletInfo.getAids()) {
                idToActions.remove(aid);
            }
        }
    }
    if (cmdletInfos.size() > 0) {
        LOG.debug("Number of cmds {} to submit", cmdletInfos.size());
        try {
            metaStore.insertActions(actionInfos.toArray(new ActionInfo[actionInfos.size()]));
            metaStore.insertCmdlets(cmdletInfos.toArray(new CmdletInfo[cmdletInfos.size()]));
        } catch (MetaStoreException e) {
            LOG.error("CmdletIds -> [ {} ], submit to DB error", cmdletInfos, e);
        }
    }
    if (todelSize > 0) {
        List<Long> del = new LinkedList<>();
        synchronized (tobeDeletedCmd) {
            del.addAll(tobeDeletedCmd.subList(0, todelSize > cacheCmdTh ? cacheCmdTh : todelSize));
            tobeDeletedCmd.removeAll(del);
        }
        if (del.size() > 0) {
            LOG.debug("Number of cmds {} to delete", del.size());
            try {
                metaStore.batchDeleteCmdlet(del);
                metaStore.batchDeleteCmdletActions(del);
            } catch (MetaStoreException e) {
                LOG.error("CmdletIds -> [ {} ], delete from DB error", del, e);
            }
        }
    }
}

Example 10 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class CmdletManager method createActionInfos.

protected List<ActionInfo> createActionInfos(CmdletDescriptor cmdletDescriptor, long cid) throws IOException {
    List<ActionInfo> actionInfos = new ArrayList<>();
    for (int index = 0; index < cmdletDescriptor.getActionSize(); index++) {
        Map<String, String> args = cmdletDescriptor.getActionArgs(index);
        ActionInfo actionInfo = new ActionInfo(maxActionId.getAndIncrement(), cid, cmdletDescriptor.getActionName(index), args, "", "", false, 0, false, 0, 0);
        actionInfos.add(actionInfo);
    }
    return actionInfos;
}