Examples with CmdletDescriptor org.smartdata.model.CmdletDescriptor used on opensource projects

Example 1 with CmdletDescriptor

use of org.smartdata.model.CmdletDescriptor in project SSM by Intel-bigdata.

the class TestCmdletManager method testReloadCmdletsInDB.

@Test(timeout = 40000)
public void testReloadCmdletsInDB() throws Exception {
    waitTillSSMExitSafeMode();
    CmdletManager cmdletManager = ssm.getCmdletManager();
    // Stop cmdletmanager
    // cmdletManager.stop();
    cmdletManager.setTimeout(1000);
    MetaStore metaStore = ssm.getMetaStore();
    String cmd = "write -file /test -length 1024; read -file /test";
    CmdletDescriptor cmdletDescriptor = generateCmdletDescriptor(cmd);
    long submitTime = System.currentTimeMillis();
    CmdletInfo cmdletInfo0 = new CmdletInfo(0, cmdletDescriptor.getRuleId(), CmdletState.DISPATCHED, cmdletDescriptor.getCmdletString(), submitTime, submitTime);
    CmdletInfo cmdletInfo1 = new CmdletInfo(1, cmdletDescriptor.getRuleId(), CmdletState.PENDING, cmdletDescriptor.getCmdletString(), submitTime, submitTime);
    List<ActionInfo> actionInfos0 = cmdletManager.createActionInfos(cmdletDescriptor, cmdletInfo0.getCid());
    flushToDB(metaStore, actionInfos0, cmdletInfo0);
    List<ActionInfo> actionInfos1 = cmdletManager.createActionInfos(cmdletDescriptor, cmdletInfo1.getCid());
    flushToDB(metaStore, actionInfos1, cmdletInfo1);
    // init cmdletmanager
    cmdletManager.init();
    // cmdletManager.start();
    CmdletInfo cmdlet0 = cmdletManager.getCmdletInfo(cmdletInfo0.getCid());
    CmdletInfo cmdlet1 = cmdletManager.getCmdletInfo(cmdletInfo1.getCid());
    while (cmdlet0.getState() != CmdletState.FAILED && cmdlet1.getState() != CmdletState.DONE) {
        Thread.sleep(100);
    }
}

Example 2 with CmdletDescriptor

use of org.smartdata.model.CmdletDescriptor in project SSM by Intel-bigdata.

the class TestCmdletManager method testGetListDeleteCmdlet.

@Test
public void testGetListDeleteCmdlet() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    String cmd = "allssd -file /testMoveFile/file1 ; cache -file /testCacheFile ; " + "write -file /test -length 1024";
    CmdletDescriptor cmdletDescriptor = generateCmdletDescriptor(cmd);
    CmdletInfo cmdletInfo = new CmdletInfo(0, cmdletDescriptor.getRuleId(), CmdletState.PENDING, cmdletDescriptor.getCmdletString(), 123178333L, 232444994L);
    CmdletInfo[] cmdlets = { cmdletInfo };
    metaStore.insertCmdlets(cmdlets);
    CmdletManager cmdletManager = ssm.getCmdletManager();
    Assert.assertTrue(cmdletManager.listCmdletsInfo(1, null).size() == 1);
    Assert.assertTrue(cmdletManager.getCmdletInfo(0) != null);
    cmdletManager.deleteCmdlet(0);
    Assert.assertTrue(cmdletManager.listCmdletsInfo(1, null).size() == 0);
}

Example 3 with CmdletDescriptor

use of org.smartdata.model.CmdletDescriptor in project SSM by Intel-bigdata.

the class ErasureCodingPlugin method onNewRuleExecutor.

@Override
public void onNewRuleExecutor(RuleInfo ruleInfo, TranslateResult tResult) {
    long ruleId = ruleInfo.getId();
    CmdletDescriptor des = tResult.getCmdDescriptor();
    for (int i = 0; i < des.getActionSize(); i++) {
        if (des.getActionName(i).equals("ec")) {
            String policy = des.getActionArgs(i).get("-policy");
            if (policy == null) {
                continue;
            }
            if (!ecPolicies.containsKey(ruleId)) {
                ecPolicies.put(ruleId, new ArrayList<String>());
            }
            ecPolicies.get(ruleId).add(policy);
        }
    }
}

Example 4 with CmdletDescriptor

use of org.smartdata.model.CmdletDescriptor in project SSM by Intel-bigdata.

the class RuleExecutor method submitCmdlets.

private int submitCmdlets(RuleInfo ruleInfo, List<String> files) {
    long ruleId = ruleInfo.getId();
    if (files == null || files.size() == 0 || ruleManager.getCmdletManager() == null) {
        return 0;
    }
    int nSubmitted = 0;
    List<RuleExecutorPlugin> plugins = RuleExecutorPluginManager.getPlugins();
    String template = tr.getCmdDescriptor().toCmdletString();
    for (String file : files) {
        if (!exited) {
            try {
                CmdletDescriptor cmd = new CmdletDescriptor(template, ruleId);
                cmd.setCmdletParameter(CmdletDescriptor.HDFS_FILE_PATH, file);
                for (RuleExecutorPlugin plugin : plugins) {
                    cmd = plugin.preSubmitCmdletDescriptor(ruleInfo, tr, cmd);
                }
                long cid = ruleManager.getCmdletManager().submitCmdlet(cmd);
                // Not really submitted if cid is -1.
                if (cid != -1) {
                    nSubmitted++;
                }
            } catch (QueueFullException e) {
                break;
            } catch (IOException e) {
                // it's common here, ignore this and continue submit
                LOG.debug("Failed to submit cmdlet for file: " + file, e);
            } catch (ParseException e) {
                LOG.error("Failed to submit cmdlet for file: " + file, e);
            }
        } else {
            break;
        }
    }
    return nSubmitted;
}

Example 5 with CmdletDescriptor

use of org.smartdata.model.CmdletDescriptor in project SSM by Intel-bigdata.

the class CmdletManager method reloadCmdletsInDB.

private void reloadCmdletsInDB() throws IOException {
    LOG.info("reloading the dispatched and pending cmdlets in DB.");
    List<CmdletInfo> cmdletInfos;
    try {
        cmdletInfos = metaStore.getCmdlets(CmdletState.DISPATCHED);
        if (cmdletInfos != null && cmdletInfos.size() != 0) {
            for (CmdletInfo cmdletInfo : cmdletInfos) {
                // track reload cmdlets
                CmdletDescriptor cmdletDescriptor = CmdletDescriptor.fromCmdletString(cmdletInfo.getParameters());
                cmdletDescriptor.setRuleId(cmdletInfo.getRid());
                tracker.track(cmdletInfo.getCid(), cmdletDescriptor);
                List<ActionInfo> actionInfos = getActions(cmdletInfo.getAids());
                for (ActionInfo actionInfo : actionInfos) {
                    actionInfo.setCreateTime(cmdletInfo.getGenerateTime());
                    actionInfo.setFinishTime(System.currentTimeMillis());
                    // Recover scheduler status according to dispatched action.
                    recoverSchedulerStatus(actionInfo);
                }
                syncCmdAction(cmdletInfo, actionInfos);
            }
        }
        cmdletInfos = metaStore.getCmdlets(CmdletState.PENDING);
        if (cmdletInfos != null && cmdletInfos.size() != 0) {
            for (CmdletInfo cmdletInfo : cmdletInfos) {
                CmdletDescriptor cmdletDescriptor = CmdletDescriptor.fromCmdletString(cmdletInfo.getParameters());
                cmdletDescriptor.setRuleId(cmdletInfo.getRid());
                // Pending task also needs to be tracked.
                tracker.track(cmdletInfo.getCid(), cmdletDescriptor);
                LOG.debug(String.format("Reload pending cmdlet: {}", cmdletInfo));
                List<ActionInfo> actionInfos = getActions(cmdletInfo.getAids());
                syncCmdAction(cmdletInfo, actionInfos);
            }
        }
    } catch (MetaStoreException e) {
        LOG.error("DB connection error occurs when ssm is reloading cmdlets!");
        return;
    } catch (ParseException pe) {
        LOG.error("Failed to parse cmdlet string for tracking task", pe);
    }
}