Examples with MetaStore org.smartdata.metastore.MetaStore used on opensource projects

Example 6 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestCmdletManager method testReloadCmdletsInDB.

@Test(timeout = 40000)
public void testReloadCmdletsInDB() throws Exception {
    waitTillSSMExitSafeMode();
    CmdletManager cmdletManager = ssm.getCmdletManager();
    // Stop cmdletmanager
    // cmdletManager.stop();
    cmdletManager.setTimeout(1000);
    MetaStore metaStore = ssm.getMetaStore();
    String cmd = "write -file /test -length 1024; read -file /test";
    CmdletDescriptor cmdletDescriptor = generateCmdletDescriptor(cmd);
    long submitTime = System.currentTimeMillis();
    CmdletInfo cmdletInfo0 = new CmdletInfo(0, cmdletDescriptor.getRuleId(), CmdletState.DISPATCHED, cmdletDescriptor.getCmdletString(), submitTime, submitTime);
    CmdletInfo cmdletInfo1 = new CmdletInfo(1, cmdletDescriptor.getRuleId(), CmdletState.PENDING, cmdletDescriptor.getCmdletString(), submitTime, submitTime);
    List<ActionInfo> actionInfos0 = cmdletManager.createActionInfos(cmdletDescriptor, cmdletInfo0.getCid());
    flushToDB(metaStore, actionInfos0, cmdletInfo0);
    List<ActionInfo> actionInfos1 = cmdletManager.createActionInfos(cmdletDescriptor, cmdletInfo1.getCid());
    flushToDB(metaStore, actionInfos1, cmdletInfo1);
    // init cmdletmanager
    cmdletManager.init();
    // cmdletManager.start();
    CmdletInfo cmdlet0 = cmdletManager.getCmdletInfo(cmdletInfo0.getCid());
    CmdletInfo cmdlet1 = cmdletManager.getCmdletInfo(cmdletInfo1.getCid());
    while (cmdlet0.getState() != CmdletState.FAILED && cmdlet1.getState() != CmdletState.DONE) {
        Thread.sleep(100);
    }
}

Example 7 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestCmdletManager method testGetListDeleteCmdlet.

@Test
public void testGetListDeleteCmdlet() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    String cmd = "allssd -file /testMoveFile/file1 ; cache -file /testCacheFile ; " + "write -file /test -length 1024";
    CmdletDescriptor cmdletDescriptor = generateCmdletDescriptor(cmd);
    CmdletInfo cmdletInfo = new CmdletInfo(0, cmdletDescriptor.getRuleId(), CmdletState.PENDING, cmdletDescriptor.getCmdletString(), 123178333L, 232444994L);
    CmdletInfo[] cmdlets = { cmdletInfo };
    metaStore.insertCmdlets(cmdlets);
    CmdletManager cmdletManager = ssm.getCmdletManager();
    Assert.assertTrue(cmdletManager.listCmdletsInfo(1, null).size() == 1);
    Assert.assertTrue(cmdletManager.getCmdletInfo(0) != null);
    cmdletManager.deleteCmdlet(0);
    Assert.assertTrue(cmdletManager.listCmdletsInfo(1, null).size() == 0);
}

Example 8 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestCmdletManager method testWithoutCluster.

@Test
public void testWithoutCluster() throws MetaStoreException, IOException, InterruptedException {
    long cmdletId = 10;
    long actionId = 101;
    MetaStore metaStore = mock(MetaStore.class);
    Assert.assertNotNull(metaStore);
    when(metaStore.getMaxCmdletId()).thenReturn(cmdletId);
    when(metaStore.getMaxActionId()).thenReturn(actionId);
    CmdletDispatcher dispatcher = mock(CmdletDispatcher.class);
    Assert.assertNotNull(dispatcher);
    when(dispatcher.canDispatchMore()).thenReturn(true);
    ServerContext serverContext = new ServerContext(new SmartConf(), metaStore);
    serverContext.setServiceMode(ServiceMode.HDFS);
    CmdletManager cmdletManager = new CmdletManager(serverContext);
    cmdletManager.init();
    cmdletManager.setDispatcher(dispatcher);
    cmdletManager.start();
    cmdletManager.submitCmdlet("echo");
    Thread.sleep(500);
    verify(metaStore, times(1)).insertCmdlets(any(CmdletInfo[].class));
    verify(metaStore, times(1)).insertActions(any(ActionInfo[].class));
    Thread.sleep(500);
    long startTime = System.currentTimeMillis();
    ActionStatus actionStatus = new ActionStatus(cmdletId, true, actionId, startTime, null);
    StatusReport statusReport = new StatusReport(Arrays.asList(actionStatus));
    cmdletManager.updateStatus(statusReport);
    ActionInfo actionInfo = cmdletManager.getActionInfo(actionId);
    CmdletInfo cmdletInfo = cmdletManager.getCmdletInfo(cmdletId);
    Assert.assertNotNull(actionInfo);
    cmdletManager.updateStatus(new CmdletStatusUpdate(cmdletId, System.currentTimeMillis(), CmdletState.EXECUTING));
    CmdletInfo info = cmdletManager.getCmdletInfo(cmdletId);
    Assert.assertNotNull(info);
    Assert.assertEquals(info.getParameters(), "echo");
    Assert.assertEquals(info.getAids().size(), 1);
    Assert.assertTrue(info.getAids().get(0) == actionId);
    Assert.assertEquals(info.getState(), CmdletState.EXECUTING);
    long finishTime = System.currentTimeMillis();
    actionStatus = new ActionStatus(cmdletId, true, actionId, null, startTime, finishTime, null, true);
    statusReport = new StatusReport(Arrays.asList(actionStatus));
    cmdletManager.updateStatus(statusReport);
    Assert.assertTrue(actionInfo.isFinished());
    Assert.assertTrue(actionInfo.isSuccessful());
    Assert.assertEquals(actionInfo.getCreateTime(), startTime);
    Assert.assertEquals(actionInfo.getFinishTime(), finishTime);
    Assert.assertEquals(cmdletInfo.getState(), CmdletState.DONE);
    cmdletManager.updateStatus(new CmdletStatusUpdate(cmdletId, System.currentTimeMillis(), CmdletState.DONE));
    Assert.assertEquals(info.getState(), CmdletState.DONE);
    Thread.sleep(500);
    verify(metaStore, times(2)).insertCmdlets(any(CmdletInfo[].class));
    verify(metaStore, times(2)).insertActions(any(ActionInfo[].class));
    cmdletManager.stop();
}

Example 9 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestTableAggregator method testAggregate.

@Test
public void testAggregate() throws Exception {
    createTables(databaseTester.getConnection());
    IDataSet dataSet = new XmlDataSet(getClass().getClassLoader().getResourceAsStream("accessCountTable.xml"));
    databaseTester.setDataSet(dataSet);
    databaseTester.onSetup();
    MetaStore metaStore = new MetaStore(druidPool);
    prepareFiles(metaStore);
    AccessCountTable result = new AccessCountTable("actual", 0L, 0L, false);
    AccessCountTable table1 = new AccessCountTable("table1", 0L, 0L, false);
    AccessCountTable table2 = new AccessCountTable("table2", 0L, 0L, false);
    AccessCountTable table3 = new AccessCountTable("table3", 0L, 0L, false);
    metaStore.aggregateTables(result, Lists.newArrayList(table1, table2, table3));
    ITable actual = databaseTester.getConnection().createTable(result.getTableName());
    ITable expect = databaseTester.getDataSet().getTable("expect");
    Assertion.assertEquals(expect, actual);
}

Example 10 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestRuleManager method init.

@Before
public void init() throws Exception {
    initDao();
    smartConf = new SmartConf();
    metaStore = new MetaStore(druidPool);
    ServerContext serverContext = new ServerContext(smartConf, metaStore);
    serverContext.setServiceMode(ServiceMode.HDFS);
    ruleManager = new RuleManager(serverContext, null, null);
    ruleManager.init();
    ruleManager.start();
}