Examples with MetaStore org.smartdata.metastore.MetaStore used on opensource projects

Example 11 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class SmartServer method initWith.

public void initWith() throws Exception {
    LOG.info("Start Init Smart Server");
    HadoopUtil.setSmartConfByHadoop(conf);
    MetaStore metaStore = MetaStoreUtils.getDBAdapter(conf);
    context = new ServerContext(conf, metaStore);
    initServiceMode(conf);
    engine = new SmartEngine(context);
    rpcServer = new SmartRpcServer(this, conf);
    zeppelinServer = new SmartZeppelinServer(conf, engine);
    LOG.info("Finish Init Smart Server");
}

Example 12 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestCopy2S3Scheduler method testDir.

@Test(timeout = 45000)
public void testDir() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    SmartAdmin admin = new SmartAdmin(smartContext.getConf());
    DistributedFileSystem dfs = cluster.getFileSystem();
    final String srcPath = "/src/";
    dfs.mkdirs(new Path(srcPath));
    // Write to src
    for (int i = 0; i < 3; i++) {
        // Create test files
        DFSTestUtil.createFile(dfs, new Path(srcPath + i), 1024, (short) 1, 1);
    }
    long ruleId = admin.submitRule("file: path matches \"/src/*\"| copy2s3 -dest s3a://xxxctest/dest/", RuleState.ACTIVE);
    List<ActionInfo> actions;
    do {
        actions = metaStore.getActions(ruleId, 0);
        Thread.sleep(1000);
    } while (actions.size() < 3);
}

Example 13 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestCopy2S3Scheduler method testOnS3.

@Test(timeout = 45000)
public void testOnS3() throws Exception {
    waitTillSSMExitSafeMode();
    MetaStore metaStore = ssm.getMetaStore();
    SmartAdmin admin = new SmartAdmin(smartContext.getConf());
    DistributedFileSystem dfs = cluster.getFileSystem();
    final String srcPath = "/src/";
    dfs.mkdirs(new Path(srcPath));
    List<String> sps = new ArrayList<>();
    // Write to src
    for (int i = 0; i < 3; i++) {
        // Create test files
        // Not 0 because this file may be not be truncated yet
        sps.add(srcPath + i);
        DFSTestUtil.createFile(dfs, new Path(srcPath + i), 10, (short) 1, 1);
    }
    do {
        Thread.sleep(1000);
        if (metaStore.getFilesByPaths(sps).size() == sps.size()) {
            break;
        }
    } while (true);
    for (String p : sps) {
        metaStore.insertUpdateFileState(new S3FileState(p));
    }
    long ruleId = admin.submitRule("file: path matches \"/src/*\"| copy2s3 -dest s3a://xxxctest/dest/", RuleState.ACTIVE);
    Thread.sleep(2500);
    List<ActionInfo> actions = metaStore.getActions(ruleId, 0);
    Assert.assertEquals(0, actions.size());
}

Example 14 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestAccessEventAggregator method testAccessEventAggregator.

@Test
public void testAccessEventAggregator() throws MetaStoreException {
    MetaStore adapter = mock(MetaStore.class);
    AccessCountTableManager manager = mock(AccessCountTableManager.class);
    AccessEventAggregator aggregator = new AccessEventAggregator(adapter, manager);
    aggregator.addAccessEvents(Lists.newArrayList(new FileAccessEvent("", 3000)));
    verify(adapter, never()).execute(anyString());
    aggregator.addAccessEvents(Lists.newArrayList(new FileAccessEvent("", 6000)));
    verify(adapter, times(1)).execute(anyString());
    verify(manager, times(1)).addTable(any(AccessCountTable.class));
    aggregator.addAccessEvents(Lists.newArrayList(new FileAccessEvent("abc", 8000), new FileAccessEvent("def", 14000), new FileAccessEvent("", 18000)));
    verify(adapter, times(3)).execute(anyString());
    verify(manager, times(3)).addTable(any(AccessCountTable.class));
}

Example 15 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestAlluxioEntryFetcher method setUp.

@Before
public void setUp() throws Exception {
    localAlluxioCluster = new LocalAlluxioCluster(2);
    localAlluxioCluster.initConfiguration();
    Configuration.set(PropertyKey.WEB_RESOURCES, PathUtils.concatPath(System.getProperty("user.dir"), "src/test/webapp"));
    localAlluxioCluster.start();
    fs = localAlluxioCluster.getClient();
    initDao();
    metaStore = new MetaStore(druidPool);
}