Examples with MetaStore org.smartdata.metastore.MetaStore used on opensource projects

Example 16 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestRuleExecutor method initActionDao.

@Before
public void initActionDao() throws Exception {
    initDao();
    metaStoreHelper = new MetaStoreHelper(druidPool.getDataSource());
    adapter = new MetaStore(druidPool);
}

Example 17 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class MetaStoreUtils method getDBAdapter.

public static MetaStore getDBAdapter(SmartConf conf) throws MetaStoreException {
    URL pathUrl = ClassLoader.getSystemResource("");
    String path = pathUrl.getPath();
    characterTakeUpBytes = conf.getInt(SmartConfKeys.SMART_METASTORE_CHARACTER_TAKEUP_BYTES_KEY, SmartConfKeys.SMART_METASTORE_CHARACTER_TAKEUP_BYTES_DEFAULT);
    String fileName = "druid.xml";
    String expectedCpPath = path + fileName;
    LOG.info("Expected DB connection pool configuration path = " + expectedCpPath);
    File cpConfigFile = new File(expectedCpPath);
    if (cpConfigFile.exists()) {
        LOG.info("Using pool configure file: " + expectedCpPath);
        Properties p = new Properties();
        try {
            p.loadFromXML(new FileInputStream(cpConfigFile));
            String url = conf.get(SmartConfKeys.SMART_METASTORE_DB_URL_KEY);
            if (url != null) {
                p.setProperty("url", url);
            }
            String purl = p.getProperty("url");
            if (purl == null || purl.length() == 0) {
                // For testing
                purl = getDefaultSqliteDB();
                p.setProperty("url", purl);
                LOG.warn("Database URL not specified, using " + purl);
            }
            if (purl.startsWith(MetaStoreUtils.MYSQL_URL_PREFIX)) {
                String dbName = getMysqlDBName(purl);
                for (String name : DB_NAME_NOT_ALLOWED) {
                    if (dbName.equals(name)) {
                        throw new MetaStoreException(String.format("The database %s in mysql is for DB system use, " + "please appoint other database in druid.xml.", name));
                    }
                }
            }
            try {
                String pw = conf.getPasswordFromHadoop(SmartConfKeys.SMART_METASTORE_PASSWORD);
                if (pw != null && pw != "") {
                    p.setProperty("password", pw);
                }
            } catch (IOException e) {
                LOG.info("Can not get metastore password from hadoop provision credentials," + " use the one configured in druid.xml .");
            }
            for (String key : p.stringPropertyNames()) {
                if (key.equals("password")) {
                    LOG.info("\t" + key + " = **********");
                } else {
                    LOG.info("\t" + key + " = " + p.getProperty(key));
                }
            }
            return new MetaStore(new DruidPool(p));
        } catch (Exception e) {
            if (e instanceof InvalidPropertiesFormatException) {
                throw new MetaStoreException("Malformat druid.xml, please check the file.", e);
            } else {
                throw new MetaStoreException(e);
            }
        }
    } else {
        LOG.info("DB connection pool config file " + expectedCpPath + " NOT found.");
    }
    // Get Default configure from druid-template.xml
    fileName = "druid-template.xml";
    expectedCpPath = path + fileName;
    LOG.info("Expected DB connection pool configuration path = " + expectedCpPath);
    cpConfigFile = new File(expectedCpPath);
    LOG.info("Using pool configure file: " + expectedCpPath);
    Properties p = new Properties();
    try {
        p.loadFromXML(new FileInputStream(cpConfigFile));
    } catch (Exception e) {
        throw new MetaStoreException(e);
    }
    String url = conf.get(SmartConfKeys.SMART_METASTORE_DB_URL_KEY);
    if (url != null) {
        p.setProperty("url", url);
    }
    for (String key : p.stringPropertyNames()) {
        LOG.info("\t" + key + " = " + p.getProperty(key));
    }
    return new MetaStore(new DruidPool(p));
}

Example 18 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testGetTablesCornerCase.

@Test
public void testGetTablesCornerCase() throws MetaStoreException {
    MetaStore adapter = mock(MetaStore.class);
    TableEvictor tableEvictor = new CountEvictor(adapter, 20);
    Map<TimeGranularity, AccessCountTableDeque> map = new HashMap<>();
    AccessCountTableDeque minute = new AccessCountTableDeque(tableEvictor);
    map.put(TimeGranularity.MINUTE, minute);
    AccessCountTableDeque secondDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstFiveSeconds = new AccessCountTable(0L, 5 * Constants.ONE_SECOND_IN_MILLIS);
    AccessCountTable secondFiveSeconds = new AccessCountTable(5 * Constants.ONE_SECOND_IN_MILLIS, 10 * Constants.ONE_SECOND_IN_MILLIS);
    secondDeque.addAndNotifyListener(firstFiveSeconds);
    secondDeque.addAndNotifyListener(secondFiveSeconds);
    map.put(TimeGranularity.SECOND, secondDeque);
    List<AccessCountTable> result = AccessCountTableManager.getTables(map, adapter, 2 * Constants.ONE_MINUTE_IN_MILLIS);
    Assert.assertTrue(result.size() == 2);
    Assert.assertTrue(result.get(0).equals(firstFiveSeconds));
    Assert.assertTrue(result.get(1).equals(secondFiveSeconds));
}

Example 19 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testGetTables.

@Test
public void testGetTables() throws MetaStoreException {
    MetaStore adapter = mock(MetaStore.class);
    TableEvictor tableEvictor = new CountEvictor(adapter, 20);
    Map<TimeGranularity, AccessCountTableDeque> map = new HashMap<>();
    AccessCountTableDeque dayDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstDay = new AccessCountTable(0L, Constants.ONE_DAY_IN_MILLIS);
    dayDeque.addAndNotifyListener(firstDay);
    map.put(TimeGranularity.DAY, dayDeque);
    AccessCountTableDeque hourDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstHour = new AccessCountTable(23 * Constants.ONE_HOUR_IN_MILLIS, 24 * Constants.ONE_HOUR_IN_MILLIS);
    AccessCountTable secondHour = new AccessCountTable(24 * Constants.ONE_HOUR_IN_MILLIS, 25 * Constants.ONE_HOUR_IN_MILLIS);
    hourDeque.addAndNotifyListener(firstHour);
    hourDeque.addAndNotifyListener(secondHour);
    map.put(TimeGranularity.HOUR, hourDeque);
    AccessCountTableDeque minuteDeque = new AccessCountTableDeque(tableEvictor);
    Integer numMins = 25 * 60;
    AccessCountTable firstMin = new AccessCountTable((numMins - 1) * Constants.ONE_MINUTE_IN_MILLIS, numMins * Constants.ONE_MINUTE_IN_MILLIS);
    AccessCountTable secondMin = new AccessCountTable(numMins * Constants.ONE_MINUTE_IN_MILLIS, (numMins + 1) * Constants.ONE_MINUTE_IN_MILLIS);
    minuteDeque.addAndNotifyListener(firstMin);
    minuteDeque.addAndNotifyListener(secondMin);
    map.put(TimeGranularity.MINUTE, minuteDeque);
    AccessCountTableDeque secondDeque = new AccessCountTableDeque(tableEvictor);
    Integer numSeconds = (25 * 60 + 1) * 60;
    AccessCountTable firstFiveSeconds = new AccessCountTable((numSeconds - 5) * Constants.ONE_SECOND_IN_MILLIS, numSeconds * Constants.ONE_SECOND_IN_MILLIS);
    AccessCountTable secondFiveSeconds = new AccessCountTable(numSeconds * Constants.ONE_SECOND_IN_MILLIS, (numSeconds + 5) * Constants.ONE_SECOND_IN_MILLIS);
    secondDeque.addAndNotifyListener(firstFiveSeconds);
    secondDeque.addAndNotifyListener(secondFiveSeconds);
    map.put(TimeGranularity.SECOND, secondDeque);
    List<AccessCountTable> firstResult = AccessCountTableManager.getTables(map, adapter, (numSeconds + 5) * Constants.ONE_SECOND_IN_MILLIS);
    Assert.assertTrue(firstResult.size() == 4);
    Assert.assertEquals(firstResult.get(0), firstDay);
    Assert.assertEquals(firstResult.get(1), secondHour);
    Assert.assertEquals(firstResult.get(2), secondMin);
    Assert.assertEquals(firstResult.get(3), secondFiveSeconds);
    List<AccessCountTable> secondResult = AccessCountTableManager.getTables(map, adapter, numSeconds * Constants.ONE_SECOND_IN_MILLIS);
    Assert.assertTrue(secondResult.size() == 4);
    AccessCountTable expectDay = new AccessCountTable(5 * Constants.ONE_SECOND_IN_MILLIS, Constants.ONE_DAY_IN_MILLIS);
    Assert.assertEquals(expectDay, secondResult.get(0));
    List<AccessCountTable> thirdResult = AccessCountTableManager.getTables(map, adapter, secondFiveSeconds.getEndTime() - 23 * Constants.ONE_HOUR_IN_MILLIS);
    Assert.assertTrue(thirdResult.size() == 4);
    Assert.assertEquals(thirdResult.get(0), firstHour);
    List<AccessCountTable> fourthResult = AccessCountTableManager.getTables(map, adapter, secondFiveSeconds.getEndTime() - 24 * Constants.ONE_HOUR_IN_MILLIS);
    Assert.assertTrue(fourthResult.size() == 3);
    Assert.assertEquals(fourthResult.get(0), secondHour);
}

Example 20 with MetaStore

use of org.smartdata.metastore.MetaStore in project SSM by Intel-bigdata.

the class TestInotifyEventApplier method init.

@Before
public void init() throws Exception {
    initDao();
    metaStore = new MetaStore(druidPool);
}